From 2dfbfd9ace43eabfc12fe59d0b17ae63b4a1e70c Mon Sep 17 00:00:00 2001 From: KennySimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:58:22 +0800 Subject: [PATCH 1/2] fix: convert CSV fields above the parser limit --- .../markitdown/converters/_csv_converter.py | 18 +++++++++++++++-- packages/markitdown/tests/test_csv.py | 20 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 331b0712a3..1b4df6743d 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -1,6 +1,7 @@ import csv import io import re +import threading from typing import BinaryIO, Any from charset_normalizer import from_bytes from .._base_converter import DocumentConverter, DocumentConverterResult @@ -18,6 +19,9 @@ # The lookbehind avoids retrying from each position inside a backslash run. _PIPE_ESCAPE_RE = re.compile(r"(? str: r"""Escape a CSV value so it is safe inside a Markdown table cell. @@ -100,8 +104,18 @@ def convert( content = content.lstrip("\ufeff") # Parse CSV content - reader = csv.reader(io.StringIO(content, newline="")) - rows = list(reader) + with _CSV_FIELD_LIMIT_LOCK: + try: + rows = list(csv.reader(io.StringIO(content, newline=""))) + except csv.Error: + original_limit = csv.field_size_limit() + if len(content) <= original_limit: + raise + try: + csv.field_size_limit(len(content)) + rows = list(csv.reader(io.StringIO(content, newline=""))) + finally: + csv.field_size_limit(original_limit) _trim_outer_blank_rows(rows) if not rows: diff --git a/packages/markitdown/tests/test_csv.py b/packages/markitdown/tests/test_csv.py index d21742bfbc..cb1eac2ea1 100644 --- a/packages/markitdown/tests/test_csv.py +++ b/packages/markitdown/tests/test_csv.py @@ -1,5 +1,6 @@ """CSV conversion, escaping, blank rows, and line endings.""" +import csv import io import pytest @@ -15,6 +16,25 @@ def converter() -> MarkItDown: return MarkItDown(enable_plugins=False) +@pytest.mark.parametrize( + "value", + ["x" * 131072, "x" * 131073, "é" * 131073], + ids=["at-limit", "above-limit", "unicode-above-limit"], +) +def test_csv_long_fields_preserve_table_and_parser_limit( + converter: MarkItDown, value: str +) -> None: + original_limit = csv.field_size_limit() + + result = converter.convert_stream( + io.BytesIO(f"content,other\n{value},ok\n".encode("utf-8")), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + + assert result.markdown == f"| content | other |\n| --- | --- |\n| {value} | ok |" + assert csv.field_size_limit() == original_limit + + @pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) @pytest.mark.parametrize("final_newline", [False, True]) def test_csv_record_separators_produce_a_table( From 97f2ac40d0a120db066edde7b402ed63b110db31 Mon Sep 17 00:00:00 2001 From: KennySimpson <70093673+KennyMcSimpson@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:47:42 +0800 Subject: [PATCH 2/2] test: cover concurrent CSV conversions and document the field limit --- .../markitdown/converters/_csv_converter.py | 5 ++++ packages/markitdown/tests/test_csv.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 1b4df6743d..d7630d0268 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -61,6 +61,11 @@ def _trim_outer_blank_rows(rows: list[list[str]]) -> None: class CsvConverter(DocumentConverter): """ Converts CSV files to Markdown tables. + + Oversized fields are parsed with a temporarily raised process-wide CSV + field limit. The lock coordinates CsvConverter instances only; unrelated + CSV readers can observe the temporary limit, and changes made by other + code to that limit are not synchronized with this converter. """ def __init__(self): diff --git a/packages/markitdown/tests/test_csv.py b/packages/markitdown/tests/test_csv.py index cb1eac2ea1..af88086d9a 100644 --- a/packages/markitdown/tests/test_csv.py +++ b/packages/markitdown/tests/test_csv.py @@ -2,6 +2,8 @@ import csv import io +import threading +from concurrent.futures import ThreadPoolExecutor import pytest @@ -35,6 +37,27 @@ def test_csv_long_fields_preserve_table_and_parser_limit( assert csv.field_size_limit() == original_limit +def test_csv_concurrent_long_fields_preserve_table_and_parser_limit() -> None: + original_limit = csv.field_size_limit() + start = threading.Barrier(2) + + def convert(value: str) -> str: + converter = MarkItDown(enable_plugins=False) + start.wait(timeout=10) + return converter.convert_stream( + io.BytesIO(f"content,other\n{value},ok\n".encode("utf-8")), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ).markdown + + values = ["x" * 131073, "é" * 131074] + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(convert, values)) + + for value, result in zip(values, results): + assert result == f"| content | other |\n| --- | --- |\n| {value} | ok |" + assert csv.field_size_limit() == original_limit + + @pytest.mark.parametrize("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"]) @pytest.mark.parametrize("final_newline", [False, True]) def test_csv_record_separators_produce_a_table(