diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.py index ffbcbd990c..4181fe1688 100644 --- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py +++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py @@ -9,6 +9,7 @@ # Pattern for MasterFormat-style partial numbering (e.g., ".1", ".2", ".10") PARTIAL_NUMBERING_PATTERN = re.compile(r"^\.\d+$") +PARTIAL_NUMBERING_PREFIX_PATTERN = re.compile(r"^\.\d+(\s+|$)") def _merge_partial_numbering_lines(text: str) -> str: @@ -24,7 +25,10 @@ def _merge_partial_numbering_lines(text: str) -> str: .1 The intent of this Request for Proposal... - This function merges them back together. + This function merges them back together. When consecutive partial numberings + appear (e.g., an unnumbered section stub or consecutive sub-items like + '.1\n.2\nContractor shall...'), standalone numbers are preserved on their own + lines rather than being collapsed into an invalid compound token like '.1 .2'. """ lines = text.split("\n") result_lines: list[str] = [] @@ -41,13 +45,15 @@ def _merge_partial_numbering_lines(text: str) -> str: while j < len(lines) and not lines[j].strip(): j += 1 - if j < len(lines): + # Only merge if the next line is actual content, not another partial numbering + if j < len(lines) and not PARTIAL_NUMBERING_PREFIX_PATTERN.match(lines[j].strip()): # Merge the partial numbering with the next line next_line = lines[j].strip() result_lines.append(f"{stripped} {next_line}") i = j + 1 # Skip past the merged line else: - # No next line to merge with, keep as is + # Next line is also a partial numbering, or no next line to merge with; + # preserve the standalone partial numbering as-is. result_lines.append(line) i += 1 else: diff --git a/packages/markitdown/tests/test_pdf.py b/packages/markitdown/tests/test_pdf.py index c4f8c64a9c..b780f4c766 100644 --- a/packages/markitdown/tests/test_pdf.py +++ b/packages/markitdown/tests/test_pdf.py @@ -1195,6 +1195,158 @@ def test_multiple_partial_numberings_all_merged(self): isolated_count == 0 ), f"Found {isolated_count} isolated partial numberings that weren't merged" + def test_merge_partial_numbering_consecutive_standalone(self): + """Test that two consecutive partial numberings are not merged together into a single token.""" + text = ( + ".1\n" + ".2\n" + "Contractor shall furnish all materials.\n" + ".3\n" + "Work shall comply with local codes." + ) + expected = ( + ".1\n" + ".2 Contractor shall furnish all materials.\n" + ".3 Work shall comply with local codes." + ) + assert _merge_partial_numbering_lines(text) == expected + + def test_merge_partial_numbering_multiple_consecutive_and_trailing(self): + """Test three or more consecutive partial numberings and trailing standalone numbers.""" + three_consecutive = ( + ".1\n" + ".2\n" + ".3\n" + "Final requirements and specification details." + ) + expected_three = ( + ".1\n" + ".2\n" + ".3 Final requirements and specification details." + ) + assert _merge_partial_numbering_lines(three_consecutive) == expected_three + + trailing_numbers = ".1\n.2\n.3" + expected_trailing = ".1\n.2\n.3" + assert _merge_partial_numbering_lines(trailing_numbers) == expected_trailing + + @pytest.mark.parametrize( + "separator", + ["\n", "\n\n", "\n \n\t\n"], + ids=["single-blank-line", "multiple-blank-lines", "whitespace-lines"], + ) + def test_merge_partial_numbering_consecutive_with_blank_lines(self, separator): + """Test consecutive partial numbers separated by blank lines preserve their structure.""" + text = ( + f".1{separator}" + ".2\n" + "Contractor shall furnish all materials." + ) + expected = ( + f".1{separator}" + ".2 Contractor shall furnish all materials." + ) + assert _merge_partial_numbering_lines(text) == expected + + def test_merge_partial_numbering_preceding_inline_numbered_line(self): + """Test standalone partial number does not merge into a line already prefixed with a partial number.""" + text = ( + ".1\n" + ".2 Contractor shall furnish all materials.\n" + ".3 Work shall comply with local codes." + ) + expected = ( + ".1\n" + ".2 Contractor shall furnish all materials.\n" + ".3 Work shall comply with local codes." + ) + assert _merge_partial_numbering_lines(text) == expected + + @pytest.mark.parametrize( + "first_num,second_num", + [ + (".1", ".2"), + (".9", ".10"), + (".10", ".11"), + (".99", ".100"), + (".1", ".100"), + ], + ) + def test_merge_partial_numbering_varying_digit_lengths(self, first_num, second_num): + """Test consecutive partial numbering protection across varying digit lengths.""" + text = ( + f"{first_num}\n" + f"{second_num}\n" + "Technical specification content." + ) + expected = ( + f"{first_num}\n" + f"{second_num} Technical specification content." + ) + assert _merge_partial_numbering_lines(text) == expected + + def test_merge_partial_numbering_preserves_multiple_blank_lines_between_sections(self): + """Test multiple blank lines between sections with consecutive numbers are preserved.""" + text = ( + "Section 01 00 00\n\n\n" + ".1\n\n" + "Preliminary note.\n\n\n" + ".2\n\n" + ".3\n\n" + "Execution requirements.\n\n\n" + ".4\n\n" + "Quality assurance." + ) + expected = ( + "Section 01 00 00\n\n\n" + ".1 Preliminary note.\n\n\n" + ".2\n\n" + ".3 Execution requirements.\n\n\n" + ".4 Quality assurance." + ) + assert _merge_partial_numbering_lines(text) == expected + + def test_end_to_end_pdf_conversion_with_consecutive_partial_numbering(self): + """Test full MarkItDown PDF conversion pipeline preserves consecutive partial numbering.""" + pdf_path = os.path.join(TEST_FILES_DIR, "test.pdf") + simulated_pdfminer_text = ( + ".1\n" + ".2\n" + "Contractor shall furnish all materials.\n" + ".3\n" + "Work shall comply with local codes." + ) + with patch("pdfminer.high_level.extract_text", return_value=simulated_pdfminer_text): + result = MarkItDown().convert(pdf_path) + + assert ".1 .2" not in result.text_content + assert ".1\n.2 Contractor shall furnish all materials." in result.text_content + assert ".3 Work shall comply with local codes." in result.text_content + + @pytest.mark.parametrize( + "non_partial_pattern", + [ + ".5 mg", + "1.2", + ".a", + "..1", + "section .1", + "... ellipsis", + ], + ids=["unit-suffix", "standard-decimal", "letter-suffix", "double-dot", "text-prefix", "ellipsis"], + ) + def test_merge_partial_numbering_boundary_conditions(self, non_partial_pattern): + """Test boundary conditions ensure non-partial numbering lines are unaffected.""" + text = ( + f"{non_partial_pattern}\n" + "Following line of text." + ) + expected = ( + f"{non_partial_pattern}\n" + "Following line of text." + ) + assert _merge_partial_numbering_lines(text) == expected + # Page cleanup and extraction fallback