diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 331b0712a3..d7630d0268 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -1,6 +1,7 @@ import csv import io import re +import threading from typing import BinaryIO, Any from charset_normalizer import from_bytes from .._base_converter import DocumentConverter, DocumentConverterResult @@ -18,6 +19,9 @@ # The lookbehind avoids retrying from each position inside a backslash run. _PIPE_ESCAPE_RE = re.compile(r"(? str: r"""Escape a CSV value so it is safe inside a Markdown table cell. @@ -57,6 +61,11 @@ def _trim_outer_blank_rows(rows: list[list[str]]) -> None: class CsvConverter(DocumentConverter): """ Converts CSV files to Markdown tables. + + Oversized fields are parsed with a temporarily raised process-wide CSV + field limit. The lock coordinates CsvConverter instances only; unrelated + CSV readers can observe the temporary limit, and changes made by other + code to that limit are not synchronized with this converter. """ def __init__(self): @@ -100,8 +109,18 @@ def convert( content = content.lstrip("\ufeff") # Parse CSV content - reader = csv.reader(io.StringIO(content, newline="")) - rows = list(reader) + with _CSV_FIELD_LIMIT_LOCK: + try: + rows = list(csv.reader(io.StringIO(content, newline=""))) + except csv.Error: + original_limit = csv.field_size_limit() + if len(content) <= original_limit: + raise + try: + csv.field_size_limit(len(content)) + rows = list(csv.reader(io.StringIO(content, newline=""))) + finally: + csv.field_size_limit(original_limit) _trim_outer_blank_rows(rows) if not rows: diff --git a/packages/markitdown/tests/test_csv.py b/packages/markitdown/tests/test_csv.py index d21742bfbc..af88086d9a 100644 --- a/packages/markitdown/tests/test_csv.py +++ b/packages/markitdown/tests/test_csv.py @@ -1,6 +1,9 @@ """CSV conversion, escaping, blank rows, and line endings.""" +import csv import io +import threading +from concurrent.futures import ThreadPoolExecutor import pytest @@ -15,6 +18,46 @@ def converter() -> MarkItDown: return MarkItDown(enable_plugins=False) +@pytest.mark.parametrize( + "value", + ["x" * 131072, "x" * 131073, "é" * 131073], + ids=["at-limit", "above-limit", "unicode-above-limit"], +) +def test_csv_long_fields_preserve_table_and_parser_limit( + converter: MarkItDown, value: str +) -> None: + original_limit = csv.field_size_limit() + + result = converter.convert_stream( + io.BytesIO(f"content,other\n{value},ok\n".encode("utf-8")), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + + assert result.markdown == f"| content | other |\n| --- | --- |\n| {value} | ok |" + assert csv.field_size_limit() == original_limit + + +def test_csv_concurrent_long_fields_preserve_table_and_parser_limit() -> None: + original_limit = csv.field_size_limit() + start = threading.Barrier(2) + + def convert(value: str) -> str: + converter = MarkItDown(enable_plugins=False) + start.wait(timeout=10) + return converter.convert_stream( + io.BytesIO(f"content,other\n{value},ok\n".encode("utf-8")), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ).markdown + + values = ["x" * 131073, "é" * 131074] + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(convert, values)) + + for value, result in zip(values, results): + assert result == f"| content | other |\n| --- | --- |\n| {value} | ok |" + assert csv.field_size_limit() == original_limit + + @pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) @pytest.mark.parametrize("final_newline", [False, True]) def test_csv_record_separators_produce_a_table(