From 130fd273c61e2780f8cfef28957e22fcf8cc2b9b Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Wed, 30 Sep 2026 12:31:44 +0200 Subject: [PATCH] fix(magazine): read a turned page through a ladder (turned, original with carried boxes, tesseract) Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 3 + Manual.md | 7 ++- architecture/magazine-ingestion.md | 9 ++- core/magazine.py | 80 +++++++++++++++++++++++-- routes/magazine.py | 14 ++++- tests/test_magazine_rotation.py | 95 +++++++++++++++++++++++++++++- 6 files changed, 196 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a628c4..17a020f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi --- ## [Unreleased] +### Fixed +- **Turning a page no longer fails when the OCR engine cannot read the page in its new orientation.** On a real page (c't p. 156) the vision engine read the sideways image well (word-F1 0.878) but returned an empty answer for the turned page, so the turn was rolled back with "The OCR engine returned nothing usable". A turn now reads through a ladder: the turned page with the vision engines, then the page as scanned with its boxes carried into the turned page, then tesseract; the first plausible reading wins (on that page: the second, 13 regions, 168 words, F1 0.878), a picture page with no text can still be turned, and only if nothing can read it is the page put back. + ### Added - **Turn a page that was scanned sideways or upside down.** *Turn left*, *Turn right* and *Turn 180°* in the review toolbar turn the page image, carry its boxes along so reviewed articles still match, and read the page again (`POST /api/magazine/{id}/pages/{page}/rotate`). The turn is remembered in the magazine's metadata and applied whenever pages are rendered, so re-importing does not undo it. If the re-read fails, the page goes back exactly as it was. Not covered by Undo (it changes the image): the opposite button turns it back. Measured on a real page scanned sideways: word-F1 0.016 as scanned, 0.876 after turning and re-reading. Page images now revalidate (`Cache-Control: no-cache`) so a turned page shows at once. diff --git a/Manual.md b/Manual.md index 2cbd8b2..5c8c3b7 100644 --- a/Manual.md +++ b/Manual.md @@ -411,7 +411,12 @@ about 0.88). The turn is remembered: re-running detection or re-importing does n articles that used the page go back to "not approved" where the new boxes do not line up with the old ones. **Undo does not cover a turn** (it changes the page image); if you turned the wrong way, press the opposite button. -**Errors.** Without a vision OCR engine nothing is changed and you are told why. If the engine is down or returns nothing, the page image is +**How the page is read after a turn.** An OCR engine can read a page well in one orientation and return nothing in another (on a real +page it read the sideways image at word-F1 0.88 and gave an empty answer for the same page turned upright). So after a turn the app tries, in +order: the turned page with your vision engines; then the page as it was scanned, with the boxes carried over into the turned page; then tesseract +on the turned page. The first plausible reading wins, and the finished message says which was used. A picture page with no text can be turned too. + +**Errors.** Without a vision OCR engine nothing is changed and you are told why. If neither the engines nor tesseract can read the page, the page image is put back exactly as it was and nothing is saved. A page that has a real text layer in the PDF is read from its image after a turn, because the PDF's own text positions belong to the unturned page. diff --git a/architecture/magazine-ingestion.md b/architecture/magazine-ingestion.md index dddcf7e..ff9aedd 100644 --- a/architecture/magazine-ingestion.md +++ b/architecture/magazine-ingestion.md @@ -488,7 +488,14 @@ boxes were still in the sideways geometry. If the read fails, raises or returns layout, articles nor metadata are saved. On success the total turn per page is stored in `meta.json` `page_rotations` (four quarter turns delete the entry); `extract_pages(page_rotations=...)` applies it with `page.set_rotation` whenever a page is rendered, so re-ingesting does not flip the page back, and on a turned page the PDF's own text and embedded-image boxes (unturned coordinates) are ignored in favour of OCR of the render. -A turn takes no undo snapshot: Undo restores layout and articles but not the image, which would put old-orientation boxes on the turned page. The page +The re-read of a turned page goes through `read_turned_page_regions`, a ladder, because a vision engine can read one orientation and return +nothing in another: measured on a real page (c't 156, Baidu Unlimited-OCR) the sideways image read at word-F1 0.878 while the upright image +returned an empty reply in every mode and size tried (base/tiny/gundam, 1.5x upscale; padding to a square hit the repetition-loop guard). +Rungs: the turned page with the vision chain; the original page bytes read with the vision chain and every region box carried across with +`rotate_bbox` ("vision-original", chosen on that page: 13 regions, 168 words, F1 0.878); tesseract on the turned page. A reading wins if it is at +least 40 characters and `ocr_quality_score` >= 0.55; otherwise the longest non-empty one is used; "empty" (tesseract ran, saw nothing) lets a picture +page be turned; nothing at all raises and rolls the image back. `reocr_page` takes the regions via its new `regions` argument. The job result carries +`read_with`. A turn takes no undo snapshot: Undo restores layout and articles but not the image, which would put old-orientation boxes on the turned page. The page image endpoint now answers `Cache-Control: no-cache` so a turned page shows at once. Measured on a real page scanned sideways (Baidu Unlimited-OCR): word-F1 0.016 read as scanned, 0.876 after a turn and re-read. Limitation: an automatic "this page looks sideways" hint would need tesseract's orientation data (`osd.traineddata`), which the image does not ship. diff --git a/core/magazine.py b/core/magazine.py index cb12940..6e3b16c 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -2902,11 +2902,78 @@ def layout_with_rotated_page(layout: dict[str, Any], page_no: int, degrees: int) return out +def _regions_text(regions: Sequence[dict[str, Any]] | None) -> str: + return " ".join(str(r.get("text") or "") for r in regions or []).strip() + + +def _usable_reading(regions: Sequence[dict[str, Any]] | None) -> bool: + text = _regions_text(regions) + return len(text) >= 40 and ocr_quality_score(text) >= 0.55 + + +def read_turned_page_regions(png: Path, original_png: bytes, degrees: int, + engines: Sequence[tuple[str | None, str | None]], + lang: str = "deu+eng", timeout: float = 150.0 + ) -> tuple[list[dict[str, Any]] | None, str]: + """Read a page that has just been turned, trying the ways that can work in order; returns (regions, how). + + A vision engine is not equally good at every orientation: on a real page scanned sideways the engine read + the sideways image well (word-F1 0.88) and returned *nothing* for the same page turned upright, at every + size it was tried. So when a turn is asked for: + 1. the turned page with the vision engines ("vision"); + 2. the page as it was, with the boxes carried across into the turned page ("vision-original") -- + text read in the orientation the engine copes with, placed where it now is; + 3. tesseract on the turned page ("tesseract"). + The first reading that is long and plausible enough wins. If none is, the longest non-empty one is used + rather than giving up. `how` is "empty" when tesseract ran and found no text at all (a picture page): an + empty reading is then genuine, not a failure. (None, "failed") means nothing could read the page, so the + caller must put the page back. + """ + import shutil + import tempfile + from PIL import Image + d = normalise_rotation(degrees) + candidates: list[tuple[list[dict[str, Any]], str]] = [] + + turned = vision_ocr_page_chain(png, list(engines), timeout=timeout) + if _usable_reading(turned): + return turned, "vision" + if turned: + candidates.append((turned, "vision")) + + with tempfile.TemporaryDirectory(prefix="turn-") as tmp: + original = Path(tmp) / "original.png" + original.write_bytes(original_png) + with Image.open(original) as im: + width_px, height_px = im.size + regions = vision_ocr_page_chain(original, list(engines), timeout=timeout) + if regions: + carried = [{**r, "bbox": rotate_bbox(r["bbox"], width_px, height_px, d)} for r in regions if r.get("bbox")] + if _usable_reading(carried): + return carried, "vision-original" + if carried: + candidates.append((carried, "vision-original")) + + paragraphs = ocr_page(png, lang) + local = [{"label": "text", "bbox": list(p["bbox"]), "text": p["text"]} for p in paragraphs if p.get("text")] + if _usable_reading(local): + return local, "tesseract" + if local: + candidates.append((local, "tesseract")) + + if candidates: + return max(candidates, key=lambda c: len(_regions_text(c[0]))) + if shutil.which("tesseract"): + return [], "empty" # tesseract ran and saw no text: a picture page, not a failed read + return None, "failed" + + def reocr_page(doc_dir: Path, layout: dict[str, Any], page_no: int, vision_ocr_url: str, vision_ocr_model: str, vision_ocr_fallback_url: str | None = None, vision_ocr_fallback_model: str | None = None, - timeout: float = 150.0) -> dict[str, Any] | None: + timeout: float = 150.0, + regions: list[dict[str, Any]] | None = None) -> dict[str, Any] | None: """Force one page through the vision OCR chain again with the engine the user picked, replacing only that page's blocks in an already-ingested layout. @@ -2933,11 +3000,12 @@ def reocr_page(doc_dir: Path, layout: dict[str, Any], page_no: int, if not page_meta: return None - engines = [(vision_ocr_url, vision_ocr_model), - (vision_ocr_fallback_url, vision_ocr_fallback_model)] - regions = vision_ocr_page_chain(png, engines, timeout=timeout) - if not regions: - return None + if regions is None: # the caller did not read the page itself (a turned page is read by read_turned_page_regions) + engines = [(vision_ocr_url, vision_ocr_model), + (vision_ocr_fallback_url, vision_ocr_fallback_model)] + regions = vision_ocr_page_chain(png, engines, timeout=timeout) + if not regions: + return None repair_drop_cap_omissions(png, regions) scale_x = page_meta["width"] / max(1.0, _png_width(png, fitz)) diff --git a/routes/magazine.py b/routes/magazine.py index a56778f..2c73df7 100644 --- a/routes/magazine.py +++ b/routes/magazine.py @@ -989,8 +989,17 @@ def _reocr_page_sync(doc_id: str, job_id: str, page: int, # The old boxes move with the page, so what was approved still lines up when the new ones are matched. layout = mag.layout_with_rotated_page(layout, page, rotate_degrees) background_job_update(job_id, stage="reocr", detail=f"Re-reading page {page}") + regions, how = None, "" + if rotate_degrees: + # A vision engine can read a page in one orientation and return nothing in another, so a turned + # page is read by the ladder in mag.read_turned_page_regions, not by the single-engine call. + regions, how = mag.read_turned_page_regions( + png, original_png, rotate_degrees, + [(vision_ocr_url, vision_ocr_model), (vision_ocr_fallback_url, vision_ocr_fallback_model)]) + if regions is None: + raise RuntimeError("Neither the OCR engines nor tesseract could read the page") updated = mag.reocr_page(d, layout, page, vision_ocr_url, vision_ocr_model, - vision_ocr_fallback_url, vision_ocr_fallback_model) + vision_ocr_fallback_url, vision_ocr_fallback_model, regions=regions) if updated is None: raise RuntimeError("The OCR engine returned nothing usable for this page") background_job_update(job_id, stage="advertisements", detail="Recognising paragraphs, headings and advertisements") @@ -1034,7 +1043,8 @@ def _reocr_page_sync(doc_id: str, job_id: str, page: int, _seal_undo(d, token) background_job_update(job_id, stage="done", done=True, detail=f"Page {page} " + ("turned and re-read" if rotate_degrees else "re-read"), - result={"page": page, "undo": None if rotate_degrees else token, "rotation": rotate_degrees}) + result={"page": page, "undo": None if rotate_degrees else token, + "rotation": rotate_degrees, "read_with": how}) except Exception as exc: # noqa: BLE001 - surfaced to UI if original_png is not None: try: diff --git a/tests/test_magazine_rotation.py b/tests/test_magazine_rotation.py index 22c1d0e..2cd1a95 100644 --- a/tests/test_magazine_rotation.py +++ b/tests/test_magazine_rotation.py @@ -102,6 +102,73 @@ class LayoutTests(unittest.TestCase): mag.layout_with_rotated_page(self.layout(), 9, 90) +class LadderTests(unittest.TestCase): + """read_turned_page_regions: which reading wins, and how boxes from the original orientation are carried.""" + GOOD = "Dies ist ein vollstaendiger deutscher Satz mit genug Woertern fuer eine brauchbare Lesung der Seite." + + def setUp(self): + d = Path(tempfile.mkdtemp(prefix="ttvc-ladder-")) + self.turned = d / "turned.png" + Image.new("RGB", (200, 100), "white").save(self.turned) # the page after a 90 degree turn (was 100 wide, 200 tall) + buf = __import__("io").BytesIO() + Image.new("RGB", (100, 200), "white").save(buf, "PNG") + self.original = buf.getvalue() + self.calls = [] + + def run_ladder(self, vision, tesseract=None, which="/usr/bin/tesseract"): + def chain(png, engines, timeout=0): + self.calls.append(Path(png).name) + return vision[len(self.calls) - 1] if len(self.calls) <= len(vision) else None + with mock.patch.object(mag, "vision_ocr_page_chain", side_effect=chain), \ + mock.patch.object(mag, "ocr_page", return_value=tesseract or []) as ocr, \ + mock.patch("shutil.which", return_value=which): + out = mag.read_turned_page_regions(self.turned, self.original, 90, [("http://x", "m")]) + self.ocr = ocr + return out + + def region(self, text, bbox=(10, 20, 30, 40)): + return {"label": "text", "bbox": list(bbox), "text": text} + + def test_a_good_reading_of_the_turned_page_wins_at_once(self): + regions, how = self.run_ladder([[self.region(self.GOOD)]]) + self.assertEqual(how, "vision") + self.assertEqual(self.calls, ["turned.png"]) # nothing else was tried + self.ocr.assert_not_called() + + def test_when_the_engine_returns_nothing_the_original_orientation_is_read_and_its_boxes_carried_across(self): + regions, how = self.run_ladder([None, [self.region(self.GOOD, (10, 20, 30, 40))]]) + self.assertEqual(how, "vision-original") + self.assertEqual(regions[0]["bbox"], [140, 10, 40, 30]) # rotate_bbox of the box on the 100x200 original, 90 degrees + self.assertEqual(regions[0]["text"], self.GOOD) + self.assertEqual(self.calls, ["turned.png", "original.png"]) + + def test_tesseract_is_the_last_rung(self): + para = {"bbox": [5, 5, 50, 20], "text": self.GOOD, "confidence": 90, "line_height": 8} + regions, how = self.run_ladder([None, None], tesseract=[para]) + self.assertEqual(how, "tesseract") + self.assertEqual(regions, [{"label": "text", "bbox": [5, 5, 50, 20], "text": self.GOOD}]) + + def test_a_short_or_implausible_reading_does_not_win_but_the_best_of_them_is_used_if_nothing_else_does(self): + regions, how = self.run_ladder([[self.region("Seite 7")], [self.region("zzz qqq xxx")]], tesseract=[]) + self.assertIn(how, ("vision", "vision-original")) + self.assertTrue(regions) + self.assertEqual(regions[0]["text"], "zzz qqq xxx" if how == "vision-original" else "Seite 7") + + def test_a_picture_page_is_an_empty_reading_not_a_failure_when_tesseract_ran(self): + self.assertEqual(self.run_ladder([None, None], tesseract=[]), ([], "empty")) + + def test_without_tesseract_and_without_any_reading_it_is_a_failure(self): + self.assertEqual(self.run_ladder([None, None], tesseract=[], which=None), (None, "failed")) + + def test_the_original_bytes_are_used_not_the_turned_file(self): + self.run_ladder([None, [self.region(self.GOOD)]]) + self.assertEqual(self.calls[1], "original.png") + + def test_bad_degrees_are_refused(self): + with self.assertRaises(ValueError): + mag.read_turned_page_regions(self.turned, self.original, 45, []) + + class JobTests(unittest.TestCase): """_reocr_page_sync with a turn: success persists it, failure puts the page back.""" @@ -136,9 +203,12 @@ class JobTests(unittest.TestCase): p.start() self.addCleanup(p.stop) - def run_job(self, degrees, reread): - with mock.patch.object(self.routes.mag, "reocr_page", side_effect=reread) as m: + def run_job(self, degrees, reread, ladder=None): + ladder = ladder or (lambda *a, **k: ([{"label": "text", "bbox": [1, 1, 5, 5], "text": "x" * 50}], "vision")) + with mock.patch.object(self.routes.mag, "reocr_page", side_effect=reread) as m, \ + mock.patch.object(self.routes.mag, "read_turned_page_regions", side_effect=ladder) as lad: self.routes._reocr_page_sync("a" * 32, "job", 3, "http://ocr", "m", None, None, degrees) + self.ladder = lad return m def test_success_turns_the_image_remembers_it_and_re_measures_the_page(self): @@ -180,8 +250,29 @@ class JobTests(unittest.TestCase): self.assertEqual(self.png.read_bytes(), self.original) self.assertIn("engine down", self.jobs[-1]["error"]) + def test_a_turn_reads_through_the_ladder_and_hands_its_regions_to_the_re_read(self): + regions = [{"label": "text", "bbox": [1, 2, 3, 4], "text": "gelesen"}] + m = self.run_job(90, lambda d, layout, page, *a, **k: layout, ladder=lambda *a, **k: (regions, "vision-original")) + self.assertEqual(m.call_args.kwargs["regions"], regions) + self.assertEqual(self.jobs[-1]["result"]["read_with"], "vision-original") + # the ladder gets the turned page, the bytes of the page as it was, and the turn + args = self.ladder.call_args.args + self.assertEqual((args[0], args[1], args[2]), (self.png, self.original, 90)) + + def test_when_nothing_can_read_the_page_it_is_put_back(self): + self.run_job(90, lambda d, layout, page, *a, **k: layout, ladder=lambda *a, **k: (None, "failed")) + self.assertEqual(self.png.read_bytes(), self.original) + self.assertIn("Neither the OCR engines nor tesseract", self.jobs[-1]["error"]) + self.assertNotIn("page_rotations", json.loads((self.d / "meta.json").read_text())) + + def test_a_picture_page_with_no_text_can_still_be_turned(self): + self.run_job(90, lambda d, layout, page, *a, **k: layout, ladder=lambda *a, **k: ([], "empty")) + self.assertEqual(self.jobs[-1]["stage"], "done") + self.assertEqual(json.loads((self.d / "meta.json").read_text())["page_rotations"], {"3": 90}) + def test_without_a_turn_the_job_behaves_as_before(self): self.run_job(0, lambda d, layout, page, *a, **k: layout) + self.ladder.assert_not_called() self.routes._snapshot_for_undo.assert_called_once() self.assertEqual(self.png.read_bytes(), self.original) self.assertNotIn("page_rotations", json.loads((self.d / "meta.json").read_text()))