From 559c1650107d5894bf617a97941e79a5cd626846 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Mon, 28 Sep 2026 08:23:09 +0200 Subject: [PATCH] fix(magazine): suspect-OCR check uses page language and page vocabulary Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + core/magazine.py | 18 ++++++++++++++---- core/pronunciations.py | 36 +++++++++++++++++------------------ core/spelling.py | 5 ++--- tests/test_magazine.py | 2 +- tests/test_magazine_routes.py | 4 ++-- tests/test_pronunciations.py | 2 +- tests/test_spelling.py | 4 ++-- 8 files changed, 41 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45ea2ac..04853c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ยท versioned wi - **Clean up boxes now also fits text boxes to their text.** A side that cuts through letters grows (at most a letter or two, never into a neighbouring box) and empty margins are trimmed. Boxes you drew or resized by hand are never touched. ### Fixed +- **The quality check no longer flags correct text as "suspect OCR".** The word check guessed the language per fragment (`iPhones statt Token` read as English) and flagged valid German; it now uses the page's language, the same lenient German rules as the spellchecker, and treats words that recur on the page (names, brands) as vocabulary. Page 61 of the measured issue went from 11 suspect regions to 3 (one a genuinely truncated word), and the issue from 758 to 610 before the language fix's full effect. - Suspended compounds (`Programmier- und Hausaufgaben`) no longer lose their hyphen when regions are merged or layouts are grouped; already-ingested `Programmierund` shows in the spelling list with a one-click repair. ### Changed diff --git a/core/magazine.py b/core/magazine.py index c950f98..d1d439b 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -40,7 +40,8 @@ import requests from core.retry import request_with_retries from core.pronunciations import ( _SUSPENDED_COMPOUND_WORDS, detect_article_language, language_for_block, - normalize_ocr_text, prepare_text_for_tts, remove_linebreak_hyphens, suspect_ocr_tokens, + normalize_ocr_text, prepare_text_for_tts, recurring_tokens, remove_linebreak_hyphens, + suspect_ocr_tokens, ) from dataclasses import dataclass, field, asdict from pathlib import Path @@ -2823,11 +2824,17 @@ def verify_ocr_recognition(doc_dir: Path, layout: dict[str, Any], page_no: int, blocks = [dict(b) for b in layout.get("blocks", [])] checked = corrected = 0 eligible = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE} + # A fragment is too short to say which language it is in (``iPhones statt Token`` reads as + # English); a page of body text is not. + page_language = detect_article_language(b.get("text") or "" for b in blocks + if b["page"] == page_no and b["type"] in eligible) + page_vocabulary = recurring_tokens(b.get("text") or "" for b in blocks + if b["page"] == page_no and b["type"] in eligible) for index, block in enumerate(blocks): if block["page"] != page_no or block["type"] not in eligible: continue old = " ".join((block.get("text") or "").split()) - old_suspects = suspect_ocr_tokens(old) + old_suspects = suspect_ocr_tokens(old, page_language, page_vocabulary) if not old_suspects: continue checked += 1 @@ -2841,7 +2848,7 @@ def verify_ocr_recognition(doc_dir: Path, layout: dict[str, Any], page_no: int, continue similarity = difflib.SequenceMatcher(None, old.casefold(), candidate.casefold(), autojunk=False).ratio() - new_suspects = suspect_ocr_tokens(candidate) + new_suspects = suspect_ocr_tokens(candidate, page_language, page_vocabulary) length_ratio = len(candidate) / max(1, len(old)) if (similarity >= 0.55 and 0.6 <= length_ratio <= 1.6 and len(new_suspects) < len(old_suspects)): @@ -3041,8 +3048,11 @@ def audit_page_coverage(doc_dir: Path, layout: dict[str, Any], page_no: int) -> png = doc_dir / "pages" / f"{page_no:04d}.png" blocks = [b for b in layout.get("blocks", []) if b["page"] == page_no] eligible = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE} + page_language = detect_article_language(b.get("text") or "" for b in blocks if b["type"] in eligible) + page_vocabulary = recurring_tokens(b.get("text") or "" for b in blocks if b["type"] in eligible) suspects = [b["id"] for b in blocks - if b["type"] in eligible and suspect_ocr_tokens(" ".join((b.get("text") or "").split()))] + if b["type"] in eligible + and suspect_ocr_tokens(" ".join((b.get("text") or "").split()), page_language, page_vocabulary)] text_blocks = [b for b in blocks if b["type"] not in VISUAL_TYPES] overlaps = [[a["id"], b["id"]] for i, a in enumerate(text_blocks) for b in text_blocks[i + 1:] if max(_bbox_intersection_ratio(a["bbox"], b["bbox"]), diff --git a/core/pronunciations.py b/core/pronunciations.py index 91bd62a..284b445 100644 --- a/core/pronunciations.py +++ b/core/pronunciations.py @@ -109,7 +109,7 @@ def _english_spellchecker(): return None -def suspect_ocr_tokens(text: str, language: str = "auto") -> list[str]: +def suspect_ocr_tokens(text: str, language: str = "auto", ignore=()) -> list[str]: """Return dictionary-unknown tokens worth checking against the page image. This deliberately does not provide replacements. Magazine text contains @@ -117,23 +117,23 @@ def suspect_ocr_tokens(text: str, language: str = "auto") -> list[str]: dictionary output is only a trigger for a second OCR reading of the real crop, never authority to rewrite printed text by itself. """ - lang = detect_text_language(text) if language == "auto" else language - checker = _german_spellchecker() if lang == "de" else _english_spellchecker() - if checker is None: - return [] - tokens = _SPELL_WORD_RE.findall(text or "") - unknown = [] - for index, token in enumerate(tokens): - # Acronyms are handled by the pronunciation map; title-cased words in - # the middle of a sentence are much more likely to be names/products. - # German capitalises every noun, so title case cannot distinguish a - # name from an OCR error there. False positives are harmless because - # this list only triggers a guarded image re-read. - if token.isupper() or (lang != "de" and index and token[:1].isupper()) or len(token) < 4: - continue - if token.casefold() not in checker: - unknown.append(token) - return list(dict.fromkeys(unknown)) + from core.spelling import find_spelling_issues + skip = {str(w).casefold() for w in ignore} + return list(dict.fromkeys(i["token"] for i in find_spelling_issues(text or "", language) + if i["token"].casefold() not in skip)) + + +def recurring_tokens(texts, minimum: int = 2) -> set[str]: + """Words that appear at least `minimum` times across `texts` (lower-cased). + + A misreading differs each time it happens; a word printed again and again is a name, a + brand or a term of the article's subject, however unfamiliar it is to a dictionary. + """ + counts: dict[str, int] = {} + for text in texts: + for token in _SPELL_WORD_RE.findall(text or ""): + counts[token.casefold()] = counts.get(token.casefold(), 0) + 1 + return {token for token, n in counts.items() if n >= minimum} def _match_case(source: str, replacement: str) -> str: diff --git a/core/spelling.py b/core/spelling.py index 0dcb082..077ad0a 100644 --- a/core/spelling.py +++ b/core/spelling.py @@ -11,9 +11,8 @@ from __future__ import annotations import re from typing import Iterable +from core import pronunciations as _pron from core.pronunciations import ( - _english_spellchecker, - _german_spellchecker, _match_case, _umlaut_variants, detect_article_language, @@ -32,7 +31,7 @@ _RECURRING = 3 def _checker(language: str): - return _german_spellchecker() if language == "de" else _english_spellchecker() + return _pron._german_spellchecker() if language == "de" else _pron._english_spellchecker() _INFLECTIONS = ("e", "en", "er", "es", "em", "n", "s", "t", "te", "ten", "tes", "st", "ste", "sten", "end", "ende") diff --git a/tests/test_magazine.py b/tests/test_magazine.py index d110e01..19ce58b 100644 --- a/tests/test_magazine.py +++ b/tests/test_magazine.py @@ -1622,7 +1622,7 @@ class OcrQualityPassTests(unittest.TestCase): 'blocks': [block('copy', 1, 40, 50, 200, 100, old).to_json()], } with mock.patch('core.magazine.suspect_ocr_tokens', - side_effect=lambda text: ['Softwarentwicklng'] if text == old else []), \ + side_effect=lambda text, *a, **k: ['Softwarentwicklng'] if text == old else []), \ mock.patch('core.magazine.extract_region_text', return_value=fixed): updated, checked, corrected = verify_ocr_recognition( pathlib.Path('/tmp/doc'), layout, 1) diff --git a/tests/test_magazine_routes.py b/tests/test_magazine_routes.py index 52eeda0..160612b 100644 --- a/tests/test_magazine_routes.py +++ b/tests/test_magazine_routes.py @@ -1604,8 +1604,8 @@ class SpellingRouteTests(unittest.TestCase): def candidates(self, w): return {x for x in words if x != w and abs(len(x) - len(w)) <= 1 and sum(a != b for a, b in zip(x, w)) <= 1} word_usage_frequency = None from unittest.mock import patch - self._patches = [patch('core.spelling._german_spellchecker', return_value=Checker()), - patch('core.spelling._english_spellchecker', return_value=Checker())] + self._patches = [patch('core.pronunciations._german_spellchecker', return_value=Checker()), + patch('core.pronunciations._english_spellchecker', return_value=Checker())] for p in self._patches: p.start() diff --git a/tests/test_pronunciations.py b/tests/test_pronunciations.py index ce46435..3c26494 100644 --- a/tests/test_pronunciations.py +++ b/tests/test_pronunciations.py @@ -48,7 +48,7 @@ class PronunciationTests(unittest.TestCase): def test_spellcheck_only_flags_suspects_and_never_rewrites_text(self): class Checker: def __contains__(self, word): - return word != 'softwarentwicklng' + return word in {'die', 'hilft', 'openai'} from unittest import mock with mock.patch('core.pronunciations._german_spellchecker', return_value=Checker()): diff --git a/tests/test_spelling.py b/tests/test_spelling.py index 7c56ab6..c52cc43 100644 --- a/tests/test_spelling.py +++ b/tests/test_spelling.py @@ -24,8 +24,8 @@ class Checker: def patched(fn): def wrapper(*args, **kwargs): - with mock.patch("core.spelling._german_spellchecker", return_value=Checker(DE)), \ - mock.patch("core.spelling._english_spellchecker", return_value=Checker(EN)): + with mock.patch("core.pronunciations._german_spellchecker", return_value=Checker(DE)), \ + mock.patch("core.pronunciations._english_spellchecker", return_value=Checker(EN)): return fn(*args, **kwargs) return wrapper