fix(magazine): suspect-OCR check uses page language and page vocabulary

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

CHANGELOG.md

@@ -15,6 +15,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
 - **Clean up boxes now also fits text boxes to their text.** A side that cuts through letters grows (at most a letter or two, never into a neighbouring box) and empty margins are trimmed. Boxes you drew or resized by hand are never touched.
 
 ### Fixed
+- **The quality check no longer flags correct text as "suspect OCR".** The word check guessed the language per fragment (`iPhones statt Token` read as English) and flagged valid German; it now uses the page's language, the same lenient German rules as the spellchecker, and treats words that recur on the page (names, brands) as vocabulary. Page 61 of the measured issue went from 11 suspect regions to 3 (one a genuinely truncated word), and the issue from 758 to 610 before the language fix's full effect.
 - Suspended compounds (`Programmier- und Hausaufgaben`) no longer lose their hyphen when regions are merged or layouts are grouped; already-ingested `Programmierund` shows in the spelling list with a one-click repair.
 
 ### Changed

core/magazine.py

@@ -40,7 +40,8 @@ import requests
 from core.retry import request_with_retries
 from core.pronunciations import (
     _SUSPENDED_COMPOUND_WORDS, detect_article_language, language_for_block,
-    normalize_ocr_text, prepare_text_for_tts, remove_linebreak_hyphens, suspect_ocr_tokens,
+    normalize_ocr_text, prepare_text_for_tts, recurring_tokens, remove_linebreak_hyphens,
+    suspect_ocr_tokens,
 )
 from dataclasses import dataclass, field, asdict
 from pathlib import Path
@@ -2823,11 +2824,17 @@ def verify_ocr_recognition(doc_dir: Path, layout: dict[str, Any], page_no: int,
     blocks = [dict(b) for b in layout.get("blocks", [])]
     checked = corrected = 0
     eligible = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE}
+    # A fragment is too short to say which language it is in (``iPhones statt Token`` reads as
+    # English); a page of body text is not.
+    page_language = detect_article_language(b.get("text") or "" for b in blocks
+                                            if b["page"] == page_no and b["type"] in eligible)
+    page_vocabulary = recurring_tokens(b.get("text") or "" for b in blocks
+                                       if b["page"] == page_no and b["type"] in eligible)
     for index, block in enumerate(blocks):
         if block["page"] != page_no or block["type"] not in eligible:
             continue
         old = " ".join((block.get("text") or "").split())
-        old_suspects = suspect_ocr_tokens(old)
+        old_suspects = suspect_ocr_tokens(old, page_language, page_vocabulary)
         if not old_suspects:
             continue
         checked += 1
@@ -2841,7 +2848,7 @@ def verify_ocr_recognition(doc_dir: Path, layout: dict[str, Any], page_no: int,
             continue
         similarity = difflib.SequenceMatcher(None, old.casefold(), candidate.casefold(),
                                              autojunk=False).ratio()
-        new_suspects = suspect_ocr_tokens(candidate)
+        new_suspects = suspect_ocr_tokens(candidate, page_language, page_vocabulary)
         length_ratio = len(candidate) / max(1, len(old))
         if (similarity >= 0.55 and 0.6 <= length_ratio <= 1.6
                 and len(new_suspects) < len(old_suspects)):
@@ -3041,8 +3048,11 @@ def audit_page_coverage(doc_dir: Path, layout: dict[str, Any], page_no: int) ->
     png = doc_dir / "pages" / f"{page_no:04d}.png"
     blocks = [b for b in layout.get("blocks", []) if b["page"] == page_no]
     eligible = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE}
+    page_language = detect_article_language(b.get("text") or "" for b in blocks if b["type"] in eligible)
+    page_vocabulary = recurring_tokens(b.get("text") or "" for b in blocks if b["type"] in eligible)
     suspects = [b["id"] for b in blocks
-                if b["type"] in eligible and suspect_ocr_tokens(" ".join((b.get("text") or "").split()))]
+                if b["type"] in eligible
+                and suspect_ocr_tokens(" ".join((b.get("text") or "").split()), page_language, page_vocabulary)]
     text_blocks = [b for b in blocks if b["type"] not in VISUAL_TYPES]
     overlaps = [[a["id"], b["id"]] for i, a in enumerate(text_blocks) for b in text_blocks[i + 1:]
                 if max(_bbox_intersection_ratio(a["bbox"], b["bbox"]),

core/pronunciations.py

@@ -109,7 +109,7 @@ def _english_spellchecker():
         return None
 
 
-def suspect_ocr_tokens(text: str, language: str = "auto") -> list[str]:
+def suspect_ocr_tokens(text: str, language: str = "auto", ignore=()) -> list[str]:
     """Return dictionary-unknown tokens worth checking against the page image.
 
     This deliberately does not provide replacements. Magazine text contains
@@ -117,23 +117,23 @@ def suspect_ocr_tokens(text: str, language: str = "auto") -> list[str]:
     dictionary output is only a trigger for a second OCR reading of the real
     crop, never authority to rewrite printed text by itself.
     """
-    lang = detect_text_language(text) if language == "auto" else language
+    from core.spelling import find_spelling_issues
-    checker = _german_spellchecker() if lang == "de" else _english_spellchecker()
+    skip = {str(w).casefold() for w in ignore}
-    if checker is None:
+    return list(dict.fromkeys(i["token"] for i in find_spelling_issues(text or "", language)
-        return []
+                              if i["token"].casefold() not in skip))
-    tokens = _SPELL_WORD_RE.findall(text or "")
+
-    unknown = []
+
-    for index, token in enumerate(tokens):
+def recurring_tokens(texts, minimum: int = 2) -> set[str]:
-        # Acronyms are handled by the pronunciation map; title-cased words in
+    """Words that appear at least `minimum` times across `texts` (lower-cased).
-        # the middle of a sentence are much more likely to be names/products.
+
-        # German capitalises every noun, so title case cannot distinguish a
+    A misreading differs each time it happens; a word printed again and again is a name, a
-        # name from an OCR error there. False positives are harmless because
+    brand or a term of the article's subject, however unfamiliar it is to a dictionary.
-        # this list only triggers a guarded image re-read.
+    """
-        if token.isupper() or (lang != "de" and index and token[:1].isupper()) or len(token) < 4:
+    counts: dict[str, int] = {}
-            continue
+    for text in texts:
-        if token.casefold() not in checker:
+        for token in _SPELL_WORD_RE.findall(text or ""):
-            unknown.append(token)
+            counts[token.casefold()] = counts.get(token.casefold(), 0) + 1
-    return list(dict.fromkeys(unknown))
+    return {token for token, n in counts.items() if n >= minimum}
 
 
 def _match_case(source: str, replacement: str) -> str:

core/spelling.py

@@ -11,9 +11,8 @@ from __future__ import annotations
 import re
 from typing import Iterable
 
+from core import pronunciations as _pron
 from core.pronunciations import (
-    _english_spellchecker,
-    _german_spellchecker,
     _match_case,
     _umlaut_variants,
     detect_article_language,
@@ -32,7 +31,7 @@ _RECURRING = 3
 
 
 def _checker(language: str):
-    return _german_spellchecker() if language == "de" else _english_spellchecker()
+    return _pron._german_spellchecker() if language == "de" else _pron._english_spellchecker()
 
 
 _INFLECTIONS = ("e", "en", "er", "es", "em", "n", "s", "t", "te", "ten", "tes", "st", "ste", "sten", "end", "ende")

tests/test_magazine.py

@@ -1622,7 +1622,7 @@ class OcrQualityPassTests(unittest.TestCase):
             'blocks': [block('copy', 1, 40, 50, 200, 100, old).to_json()],
         }
         with mock.patch('core.magazine.suspect_ocr_tokens',
-                        side_effect=lambda text: ['Softwarentwicklng'] if text == old else []), \
+                        side_effect=lambda text, *a, **k: ['Softwarentwicklng'] if text == old else []), \
              mock.patch('core.magazine.extract_region_text', return_value=fixed):
             updated, checked, corrected = verify_ocr_recognition(
                 pathlib.Path('/tmp/doc'), layout, 1)

tests/test_magazine_routes.py

@@ -1604,8 +1604,8 @@ class SpellingRouteTests(unittest.TestCase):
             def candidates(self, w): return {x for x in words if x != w and abs(len(x) - len(w)) <= 1 and sum(a != b for a, b in zip(x, w)) <= 1}
             word_usage_frequency = None
         from unittest.mock import patch
-        self._patches = [patch('core.spelling._german_spellchecker', return_value=Checker()),
+        self._patches = [patch('core.pronunciations._german_spellchecker', return_value=Checker()),
-                         patch('core.spelling._english_spellchecker', return_value=Checker())]
+                         patch('core.pronunciations._english_spellchecker', return_value=Checker())]
         for p in self._patches:
             p.start()
 

tests/test_pronunciations.py

@@ -48,7 +48,7 @@ class PronunciationTests(unittest.TestCase):
     def test_spellcheck_only_flags_suspects_and_never_rewrites_text(self):
         class Checker:
             def __contains__(self, word):
-                return word != 'softwarentwicklng'
+                return word in {'die', 'hilft', 'openai'}
 
         from unittest import mock
         with mock.patch('core.pronunciations._german_spellchecker', return_value=Checker()):

tests/test_spelling.py

@@ -24,8 +24,8 @@ class Checker:
 
 def patched(fn):
     def wrapper(*args, **kwargs):
-        with mock.patch("core.spelling._german_spellchecker", return_value=Checker(DE)), \
+        with mock.patch("core.pronunciations._german_spellchecker", return_value=Checker(DE)), \
-             mock.patch("core.spelling._english_spellchecker", return_value=Checker(EN)):
+             mock.patch("core.pronunciations._english_spellchecker", return_value=Checker(EN)):
             return fn(*args, **kwargs)
     return wrapper