fix: convert CSV fields above the parser limit

packages/markitdown/src/markitdown/converters/_csv_converter.py

@@ -1,6 +1,7 @@
 import csv
 import io
 import re
+import threading
 from typing import BinaryIO, Any
 from charset_normalizer import from_bytes
 from .._base_converter import DocumentConverter, DocumentConverterResult
@@ -18,6 +19,9 @@
 # The lookbehind avoids retrying from each position inside a backslash run.
 _PIPE_ESCAPE_RE = re.compile(r"(?<!\\)(\\*)\|")
 
+# Serialize converter-owned changes to the process-wide CSV field limit.
+_CSV_FIELD_LIMIT_LOCK = threading.Lock()
+
 
 def _escape_table_cell(value: str) -> str:
     r"""Escape a CSV value so it is safe inside a Markdown table cell.
@@ -57,6 +61,11 @@ def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
 class CsvConverter(DocumentConverter):
     """
     Converts CSV files to Markdown tables.
+
+    Oversized fields are parsed with a temporarily raised process-wide CSV
+    field limit. The lock coordinates CsvConverter instances only; unrelated
+    CSV readers can observe the temporary limit, and changes made by other
+    code to that limit are not synchronized with this converter.
     """
 
     def __init__(self):
@@ -100,8 +109,18 @@ def convert(
         content = content.lstrip("\ufeff")
 
         # Parse CSV content
-        reader = csv.reader(io.StringIO(content, newline=""))
-        rows = list(reader)
+        with _CSV_FIELD_LIMIT_LOCK:
+            try:
+                rows = list(csv.reader(io.StringIO(content, newline="")))
+            except csv.Error:
+                original_limit = csv.field_size_limit()
+                if len(content) <= original_limit:
+                    raise
+                try:
+                    csv.field_size_limit(len(content))
+                    rows = list(csv.reader(io.StringIO(content, newline="")))
+                finally:
+                    csv.field_size_limit(original_limit)
         _trim_outer_blank_rows(rows)
 
         if not rows:

packages/markitdown/tests/test_csv.py

@@ -1,6 +1,9 @@
 """CSV conversion, escaping, blank rows, and line endings."""
 
+import csv
 import io
+import threading
+from concurrent.futures import ThreadPoolExecutor
 
 import pytest
 
@@ -15,6 +18,46 @@ def converter() -> MarkItDown:
     return MarkItDown(enable_plugins=False)
 
 
+@pytest.mark.parametrize(
+    "value",
+    ["x" * 131072, "x" * 131073, "é" * 131073],
+    ids=["at-limit", "above-limit", "unicode-above-limit"],
+)
+def test_csv_long_fields_preserve_table_and_parser_limit(
+    converter: MarkItDown, value: str
+) -> None:
+    original_limit = csv.field_size_limit()
+
+    result = converter.convert_stream(
+        io.BytesIO(f"content,other\n{value},ok\n".encode("utf-8")),
+        stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+    )
+
+    assert result.markdown == f"| content | other |\n| --- | --- |\n| {value} | ok |"
+    assert csv.field_size_limit() == original_limit
+
+
+def test_csv_concurrent_long_fields_preserve_table_and_parser_limit() -> None:
+    original_limit = csv.field_size_limit()
+    start = threading.Barrier(2)
+
+    def convert(value: str) -> str:
+        converter = MarkItDown(enable_plugins=False)
+        start.wait(timeout=10)
+        return converter.convert_stream(
+            io.BytesIO(f"content,other\n{value},ok\n".encode("utf-8")),
+            stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+        ).markdown
+
+    values = ["x" * 131073, "é" * 131074]
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        results = list(pool.map(convert, values))
+
+    for value, result in zip(values, results):
+        assert result == f"| content | other |\n| --- | --- |\n| {value} | ok |"
+    assert csv.field_size_limit() == original_limit
+
+
 @pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"])
 @pytest.mark.parametrize("final_newline", [False, True])
 def test_csv_record_separators_produce_a_table(