perf(xlsx): import pandas only for spreadsheets (1.7x faster startup, 41 MB less memory)

packages/markitdown-ocr/tests/test_xlsx_inheritance.py

@@ -8,6 +8,7 @@
 import zipfile
 
 import openpyxl
+import pandas
 from openpyxl.drawing.image import Image as SheetImage
 from openpyxl.drawing.spreadsheet_drawing import (
     AbsoluteAnchor,
@@ -269,7 +270,7 @@ def test_inherited_repairs_and_native_table_fixes_reach_ocr(
     service = _service()
     repair = Mock(wraps=_xlsx_converter._repair_sheetview_show_zeroes)
     monkeypatch.setattr(_xlsx_converter, "_repair_sheetview_show_zeroes", repair)
-    native_read = _xlsx_converter.pd.read_excel
+    native_read = pandas.read_excel
     calls = []
 
     def fixed_read(*args: Any, **kwargs: Any):
@@ -278,7 +279,7 @@ def fixed_read(*args: Any, **kwargs: Any):
         sheets["Cells"].iloc[0, 0] = "Future shared fix"
         return sheets
 
-    monkeypatch.setattr(_xlsx_converter.pd, "read_excel", fixed_read)
+    monkeypatch.setattr(pandas, "read_excel", fixed_read)
     result = _convert(XlsxConverterWithOCR(service), stream.getvalue())
     assert "Future shared fix" in result and "<native>" not in result
     assert result.count("[Image OCR]") == 2

packages/markitdown/src/markitdown/converters/_xlsx_converter.py

@@ -1,29 +1,38 @@
+import importlib
 import io
 import re
-import sys
 import zipfile
 from contextlib import contextmanager
+from types import ModuleType
 from typing import BinaryIO, Any, Iterator, Optional
 from ._html_converter import HtmlConverter
 from .._base_converter import DocumentConverter, DocumentConverterResult
 from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE
 from .._stream_info import StreamInfo
 
-# Try loading optional (but in this case, required) dependencies
-# Save reporting of any exceptions for later
-_xlsx_dependency_exc_info = None
-try:
-    import pandas as pd
-    import openpyxl  # noqa: F401
-except ImportError:
-    _xlsx_dependency_exc_info = sys.exc_info()
-
-_xls_dependency_exc_info = None
-try:
-    import pandas as pd  # noqa: F811
-    import xlrd  # noqa: F401
-except ImportError:
-    _xls_dependency_exc_info = sys.exc_info()
+
+def _import_pandas(
+    engine: str, *, converter: str, extension: str, feature: str
+) -> ModuleType:
+    """Import pandas and the given Excel engine for a spreadsheet conversion.
+
+    pandas is the slowest dependency to import, and only spreadsheets need it, so
+    it is imported on the first spreadsheet conversion instead of with this module.
+    """
+    try:
+        import pandas
+
+        importlib.import_module(engine)
+    except ImportError as exc:
+        raise MissingDependencyException(
+            MISSING_DEPENDENCY_MESSAGE.format(
+                converter=converter,
+                extension=extension,
+                feature=feature,
+            )
+        ) from exc
+    return pandas
+
 
 ACCEPTED_XLSX_MIME_TYPE_PREFIXES = [
     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
@@ -46,6 +55,7 @@
 
 @contextmanager
 def _read_xlsx_sheets(
+    pd: ModuleType,
     file_stream: BinaryIO,
 ) -> Iterator[tuple[dict[str, Any], BinaryIO]]:
     start_pos = file_stream.tell()
@@ -126,22 +136,15 @@ def convert(
         stream_info: StreamInfo,
         **kwargs: Any,  # Options to pass to the converter
     ) -> DocumentConverterResult:
-        # Check the dependencies
-        if _xlsx_dependency_exc_info is not None:
-            raise MissingDependencyException(
-                MISSING_DEPENDENCY_MESSAGE.format(
-                    converter=type(self).__name__,
-                    extension=".xlsx",
-                    feature="xlsx",
-                )
-            ) from _xlsx_dependency_exc_info[
-                1
-            ].with_traceback(  # type: ignore[union-attr]
-                _xlsx_dependency_exc_info[2]
-            )
+        pd = _import_pandas(
+            "openpyxl",
+            converter=type(self).__name__,
+            extension=".xlsx",
+            feature="xlsx",
+        )
 
         md_content = ""
-        with _read_xlsx_sheets(file_stream) as (sheets, workbook_stream):
+        with _read_xlsx_sheets(pd, file_stream) as (sheets, workbook_stream):
             images = None
             if type(self)._image_to_html is not XlsxConverter._image_to_html:
                 from ..converter_utils._xlsx_images import _XlsxImages
@@ -223,19 +226,12 @@ def convert(
         stream_info: StreamInfo,
         **kwargs: Any,  # Options to pass to the converter
     ) -> DocumentConverterResult:
-        # Load the dependencies
-        if _xls_dependency_exc_info is not None:
-            raise MissingDependencyException(
-                MISSING_DEPENDENCY_MESSAGE.format(
-                    converter=type(self).__name__,
-                    extension=".xls",
-                    feature="xls",
-                )
-            ) from _xls_dependency_exc_info[
-                1
-            ].with_traceback(  # type: ignore[union-attr]
-                _xls_dependency_exc_info[2]
-            )
+        pd = _import_pandas(
+            "xlrd",
+            converter=type(self).__name__,
+            extension=".xls",
+            feature="xls",
+        )
 
         sheets = pd.read_excel(file_stream, sheet_name=None, engine="xlrd")
         md_content = ""

packages/markitdown/tests/test_xlsx.py

@@ -2,13 +2,16 @@
 
 import inspect
 import io
+import subprocess
+import sys
 import zipfile
 from pathlib import Path
 from typing import Any, BinaryIO, Callable, Optional
 from unittest.mock import Mock
 from xml.etree import ElementTree as ET
 
 import openpyxl
+import pandas
 import pytest
 from PIL import Image
 from bs4 import BeautifulSoup
@@ -241,8 +244,8 @@ def test_default_converter_does_not_parse_images_or_change_native_output(
     monkeypatch.setattr(
         _xlsx_images, "_XlsxImages", Mock(side_effect=AssertionError("extra parsing"))
     )
-    reader = Mock(wraps=_xlsx_converter.pd.read_excel)
-    monkeypatch.setattr(_xlsx_converter.pd, "read_excel", reader)
+    reader = Mock(wraps=pandas.read_excel)
+    monkeypatch.setattr(pandas, "read_excel", reader)
 
     class NativeSubclass(XlsxConverter):
         pass
@@ -333,7 +336,7 @@ def test_native_table_failure_is_not_silently_skipped(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     error = TypeError("not the repairable error")
-    monkeypatch.setattr(_xlsx_converter.pd, "read_excel", Mock(side_effect=error))
+    monkeypatch.setattr(pandas, "read_excel", Mock(side_effect=error))
     render = Mock(return_value="<p>recognized</p>")
     with pytest.raises(TypeError) as caught:
         _convert(_ImageConverter(render), _workbook())
@@ -453,10 +456,7 @@ def test_missing_dependencies_and_legacy_xls_stay_separate(
     data = (Path(__file__).parent / "test_files" / "test.xls").read_bytes()
     legacy = XlsConverter()
     expected = legacy.convert(io.BytesIO(data), StreamInfo(extension=".xls")).markdown
-    dependency = ImportError("openpyxl unavailable")
-    monkeypatch.setattr(
-        _xlsx_converter, "_xlsx_dependency_exc_info", (ImportError, dependency, None)
-    )
+    monkeypatch.setitem(sys.modules, "openpyxl", None)
     with pytest.raises(MissingDependencyException, match=r"\[xlsx\]"):
         XlsxConverter().convert(io.BytesIO(), _INFO)
     assert not hasattr(legacy, "_image_to_html")
@@ -467,6 +467,17 @@ def test_missing_dependencies_and_legacy_xls_stay_separate(
     )
 
 
+def test_pandas_is_imported_only_for_spreadsheets() -> None:
+    code = (
+        "import sys, markitdown; markitdown.MarkItDown(); "
+        "print(sorted({'pandas', 'openpyxl', 'xlrd'} & set(sys.modules)))"
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", code], capture_output=True, text=True, check=True
+    )
+    assert result.stdout.strip() == "[]"
+
+
 # Conversion regressions