fix: prefer cp1252 when the charset detector ties

packages/markitdown/src/markitdown/_charset_utils.py

@@ -0,0 +1,65 @@
+from typing import Optional
+
+from charset_normalizer import CharsetMatch, from_bytes
+
+# S and Z with caron share these bytes in cp1250 and cp1252. They are common in
+# Central European text and rare in Western European text.
+_CARON_BYTES = frozenset(b"\x8a\x8e\x9a\x9e")
+# Letters in cp1250 that are symbols in cp1252: Ł Ą Ż ł ą Ľ ľ ż read as
+# £ ¥ ¯ ³ ¹ ¼ ¾ ¿. Followed by a letter, they point to cp1250, except ¿, which
+# opens Spanish questions ("¿Qué") and counts only between two letters.
+_CP1250_LETTER_BYTES = frozenset(b"\xa3\xa5\xaf\xb3\xb9\xbc\xbe\xbf")
+_INVERTED_QUESTION_MARK = 0xBF
+# ő in cp1250, õ in cp1252. Portuguese, the Western language that uses õ,
+# has it only before "e" ("razões", "põe").
+_O_DOUBLE_ACUTE = 0xF5
+
+
+def _is_ascii_letter(byte: int) -> bool:
+    return 0x41 <= byte <= 0x5A or 0x61 <= byte <= 0x7A
+
+
+def _looks_like_cp1250(data: bytes) -> bool:
+    if not _CARON_BYTES.isdisjoint(data):
+        return True
+    for i, byte in enumerate(data):
+        following = data[i + 1] if i + 1 < len(data) else None
+        if (
+            byte in _CP1250_LETTER_BYTES
+            and following is not None
+            and _is_ascii_letter(following)
+            and (
+                byte != _INVERTED_QUESTION_MARK
+                or (i > 0 and _is_ascii_letter(data[i - 1]))
+            )
+        ):
+            return True
+        if byte == _O_DOUBLE_ACUTE and following not in (ord("e"), ord("E")):
+            return True
+    return False
+
+
+def best_charset_match(data: bytes) -> Optional[CharsetMatch]:
+    """Return the detector's best match, breaking ties between cp1252 and cp1250.
+
+    Western European text in cp1252 often scores exactly like cp1250 (or a
+    non-Latin code page), and the detector then picks by name, so "crème"
+    comes out as "crčme". On a tie in both chaos and coherence, take cp1250 if
+    the bytes point to it, and cp1252 otherwise. Text with no such bytes, such
+    as Czech "Česká" or Croatian without š or ž, stays ambiguous and goes to
+    cp1252.
+    """
+    matches = from_bytes(data)
+    best = matches.best()
+    if best is None:
+        return best
+
+    preferred = "cp1250" if _looks_like_cp1250(data) else "cp1252"
+    for match in matches:
+        if (
+            match.encoding == preferred
+            and match.chaos == best.chaos
+            and match.coherence == best.coherence
+        ):
+            return match
+    return best

packages/markitdown/src/markitdown/_markitdown.py

@@ -15,9 +15,9 @@
 from warnings import warn
 import requests
 import magika
-import charset_normalizer
 import codecs
 
+from ._charset_utils import best_charset_match
 from ._stream_info import StreamInfo
 from ._uri_utils import parse_data_uri, file_uri_to_path
 
@@ -782,7 +782,7 @@ def _get_stream_info_guesses(
                     # Complete a split UTF-8 character at the sample boundary.
                     file_stream.seek(cur_pos)
                     stream_page = _read_charset_sample(file_stream)
-                    charset_result = charset_normalizer.from_bytes(stream_page).best()
+                    charset_result = best_charset_match(stream_page)
 
                     if charset_result is not None:
                         charset = self._normalize_charset(charset_result.encoding)

packages/markitdown/src/markitdown/converters/_csv_converter.py

@@ -2,7 +2,7 @@
 import io
 import re
 from typing import BinaryIO, Any
-from charset_normalizer import from_bytes
+from .._charset_utils import best_charset_match
 from .._base_converter import DocumentConverter, DocumentConverterResult
 from .._stream_info import StreamInfo
 
@@ -88,7 +88,7 @@ def convert(
             content = file_stream.read().decode(stream_info.charset)
         else:
             data = file_stream.read()
-            detected = from_bytes(data).best()
+            detected = best_charset_match(data)
             content = (
                 str(detected)
                 if detected is not None

packages/markitdown/src/markitdown/converters/_plain_text_converter.py

@@ -1,5 +1,5 @@
 from typing import BinaryIO, Any
-from charset_normalizer import from_bytes
+from .._charset_utils import best_charset_match
 from .._base_converter import DocumentConverter, DocumentConverterResult
 from .._stream_info import StreamInfo
 
@@ -56,7 +56,7 @@ def convert(
             text_content = file_stream.read().decode(stream_info.charset)
         else:
             data = file_stream.read()
-            detected = from_bytes(data).best()
+            detected = best_charset_match(data)
             text_content = (
                 str(detected)
                 if detected is not None

packages/markitdown/tests/test_module_misc.py

@@ -805,6 +805,101 @@ def test_explicit_charset_still_takes_precedence(markitdown: MarkItDown) -> None
     assert result.markdown == data.decode("cp1252")
 
 
+# Single-byte code page ties
+#
+# Western European text in cp1252 often scores exactly like cp1250 or a non-Latin
+# code page, and the detector then picks by name: "crème" came out as "crčme"
+# and "Café" as "Cafﻠ". Central European text in cp1250 must still decode as
+# cp1250.
+
+_CP1252_TEXTS = [
+    "Café crème",
+    "Le café crème coûte 3 € à la boulangerie près de l'église.",
+    "Señor Muñoz, año próximo",
+    "A informação está disponível na página de ações.",
+    "Blåbærgrød og rødgrød med fløde på ø-hoppet i Århus.",
+]
+
+_CP1250_TEXTS = [
+    "Zażółć gęślą jaźń. Łódź jest dużym miastem w Polsce.",
+    "Příliš žluťoučký kůň úpěl ďábelské ódy.",
+    "Árvíztűrő tükörfúrógép és őszintén szólva.",
+    "Čađava šuma, žuta ćuprija i đak u Zagrebu.",
+]
+
+
+@pytest.mark.parametrize(
+    "text", _CP1252_TEXTS, ids=["fr-short", "fr", "es", "pt", "da"]
+)
+def test_western_european_cp1252_text_is_decoded(
+    markitdown: MarkItDown, text: str
+) -> None:
+    result = markitdown.convert_stream(
+        io.BytesIO(text.encode("cp1252")), file_extension=".txt"
+    )
+    assert result.markdown == text
+
+
+@pytest.mark.parametrize("text", _CP1250_TEXTS, ids=["pl", "cs", "hu", "hr"])
+def test_central_european_cp1250_text_is_still_decoded(
+    markitdown: MarkItDown, text: str
+) -> None:
+    result = markitdown.convert_stream(
+        io.BytesIO(text.encode("cp1250")), file_extension=".txt"
+    )
+    assert result.markdown == text
+
+
+# Short Central European samples that tie with cp1252 in the detector. Their
+# bytes still point to cp1250: ą and ł read as ¹ and ³ inside words, Ľ as ¼
+# before a letter, and ő as õ, which Portuguese only has before "e".
+_CP1250_TIED_TEXTS = [
+    "Pociąg był pełny.",
+    "Ľudia sa rodia slobodní",
+    "egyenlő jogokkal",
+]
+
+# Western European samples with the same bytes in positions cp1250 does not use.
+_CP1252_LOOKALIKE_TEXTS = [
+    "As razões e as lições",
+    "¿Qué tal?",
+    "Smith¹ wrote it",
+    "Nº 5, rue de l'Église",
+]
+
+
+@pytest.mark.parametrize("text", _CP1250_TIED_TEXTS, ids=["pl", "sk", "hu"])
+def test_central_european_cp1250_tie_is_decoded(
+    markitdown: MarkItDown, text: str
+) -> None:
+    result = markitdown.convert_stream(
+        io.BytesIO(text.encode("cp1250")), file_extension=".txt"
+    )
+    assert result.markdown == text
+
+
+@pytest.mark.parametrize(
+    "text", _CP1252_LOOKALIKE_TEXTS, ids=["pt", "es", "footnote", "numero"]
+)
+def test_western_european_cp1252_lookalike_is_decoded(
+    markitdown: MarkItDown, text: str
+) -> None:
+    result = markitdown.convert_stream(
+        io.BytesIO(text.encode("cp1252")), file_extension=".txt"
+    )
+    assert result.markdown == text
+
+
+def test_western_european_cp1252_csv_is_decoded(markitdown: MarkItDown) -> None:
+    body = "Nom,Ville\r\nCafé Crème,Besançon\r\n"
+    result = markitdown.convert_stream(
+        io.BytesIO(body.encode("cp1252")), file_extension=".csv"
+    )
+    assert result.markdown == (
+        "| Nom | Ville |\n| --- | --- |\n| Café Crème | Besançon |"
+    )
+
+
 # Undetectable charsets
 
 # Tests for bytes no charset decodes.