diff --git a/CHANGELOG.md b/CHANGELOG.md index 47e6b16..fd19dae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi - **"Article ends" is only claimed when it is true.** An article counts as ended when it is signed off (`(rah) et`, `(hag)` — OCR noise after the code is tolerated) or when the next region, on the same page or at the top of the next, is a headline, deck or byline. Otherwise the last box shows a clickable *no end marker — link page N ›* chip that adds the next page's text in one click. +- **Photos and graphics that no box covers get a figure region automatically** (a dense unboxed blob of real size; thin, line-structured ink is text and is left alone). New figures join the article with the most text on their page. +- **Text that is not narratable page furniture stays out of linked articles:** running heads that recur on 3+ pages, lists of page numbers and reference/source lists are no longer pulled in when continuation pages are linked. An article made only of text you already approved becomes approved. +- The per-page fix and the whole-issue run keep `quality_report.json` in step, so the thumbnail badges and the list of pages needing a decision match the layout as it is now. Running heads and adverts are no longer audited for clipped edges or overlaps. + ### Fixed - **Reading order is now applied automatically after import, *OCR again* and every per-page fix**, not only by the manual *Fix reading order* button. Until now the order after import was whatever the vision/LLM step returned, so a column's short last box could be read after the whole other column; on the measured issue 101 of 204 pages had a saved order that differs from the column-aware order. Pages whose order you fixed by hand stay untouched, except after *OCR again*, when every region is new and a hand order can no longer apply. - **The review list shows each region's whole text**, not the first 220 characters, so every misspelling gets its red wavy underline (like a word processor). The engine's `[Non-Text]` marker and hairline boxes (under 3 pt tall) are now treated as OCR noise by *Clean up boxes* and after a page re-OCR, so they no longer show up as near-invisible regions. Fitted boxes keep a little air on grown sides. diff --git a/core/magazine.py b/core/magazine.py index 1701021..c42b8ce 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -3144,6 +3144,40 @@ def detect_inset_pullquotes(doc_dir: Path, layout: dict[str, Any], pages: Iterab return layout, added +_PICTURE_DENSITY = 0.35 # inked share of an unboxed blob above which it is a picture, not text +_PICTURE_MIN_PT = 30.0 + + +def add_uncovered_pictures(doc_dir: Path, layout: dict[str, Any], page_no: int) -> tuple[dict[str, Any], list[str]]: + """Give a figure region to each photo or graphic that no box covers. + + The page audit lists ink outside every box. A dense blob of a real size is a picture the + layout model missed (text is thin and line-structured, ink share well under a third), so it + becomes a figure -- cropped for playback, and no longer reported as missing text. Returns the + layout and the new image ids. Idempotent: a covered picture is not found again. + """ + audit = audit_page_coverage(doc_dir, layout, page_no) + if not audit.get("uncovered"): + return layout, [] + meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None) + ink = page_ink_mask(doc_dir / "pages" / f"{page_no:04d}.png") + if meta is None or ink is None: + return layout, [] + height, width = ink.shape + sx, sy = width / max(1.0, float(meta["width"])), height / max(1.0, float(meta["height"])) + added: list[str] = [] + for x, y, w, h in audit["uncovered"]: + if w < _PICTURE_MIN_PT or h < _PICTURE_MIN_PT: + continue + region = ink[int(y * sy):int((y + h) * sy), int(x * sx):int((x + w) * sx)] + if region.size == 0 or float(region.mean()) < _PICTURE_DENSITY: + continue + before = {i["id"] for i in layout.get("images", [])} + layout = add_manual_region(doc_dir, layout, page_no, [x, y, w, h], "image") + added += [i["id"] for i in layout.get("images", []) if i["id"] not in before] + return layout, added + + def split_around_pullquotes(doc_dir: Path, layout: dict[str, Any], pages: Iterable[int] | None = None) -> tuple[dict[str, Any], list[dict[str, str]]]: """Cut a paragraph's box where its text wraps around an inset pull quote. @@ -3373,7 +3407,8 @@ def audit_page_coverage(doc_dir: Path, layout: dict[str, Any], page_no: int) -> suspects = [b["id"] for b in blocks if b["type"] in eligible and suspect_ocr_tokens(" ".join((b.get("text") or "").split()), page_language, page_vocabulary)] - text_blocks = [b for b in blocks if b["type"] not in VISUAL_TYPES] + # Running heads and adverts are never narrated: a box edge or an overlap there is not a problem. + text_blocks = [b for b in blocks if b["type"] not in VISUAL_TYPES and b["type"] not in (BLOCK_FURNITURE, BLOCK_AD)] overlaps = [[a["id"], b["id"]] for i, a in enumerate(text_blocks) for b in text_blocks[i + 1:] if max(_bbox_intersection_ratio(a["bbox"], b["bbox"]), _bbox_intersection_ratio(b["bbox"], a["bbox"])) >= _AUDIT_OVERLAP_RATIO] @@ -4679,6 +4714,26 @@ def _is_placeholder_article(article: dict[str, Any]) -> bool: return bool(_PLACEHOLDER_TITLE.fullmatch((article.get("title") or "").strip())) +_REFERENCE_HEAD_RE = re.compile(r"^\s*(literatur|quellen|studien, weitere quellen|weiterführende|links?\b|anmerkungen?\b)", re.I) + + +def looks_like_page_junk(text: str, repeats: int = 1) -> bool: + """Text that is on a page but is not something to narrate: a running head that recurs on many + pages, a list of page numbers, or a reference/source list. Never used to delete anything the + reviewer approved -- only to keep such text out of an article automatically.""" + t = " ".join((text or "").split()) + if not t: + return True + letters = sum(c.isalpha() for c in t) + if letters < 0.5 * len(t.replace(" ", "")) and len(t) < 200: + return True # "66 90,98 50,56 82, 114 ..." + if repeats >= 3 and len(t) < 80: + return True # the same short line on 3+ pages: a running head + if _REFERENCE_HEAD_RE.match(t) and len(t) < 400: + return True + return False + + def auto_link_pages(layout: dict[str, Any], articles: list[dict[str, Any]], max_gap_pages: int = 2) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Attach text that no real article owns to the article running from the page before. @@ -4700,8 +4755,16 @@ def auto_link_pages(layout: dict[str, Any], articles: list[dict[str, Any]], approved_ids = {b for a in articles if a.get("status") == "approved" for b in a["blocks"]} blocks = {b["id"]: b for b in layout.get("blocks", [])} by_page: dict[int, list[dict[str, Any]]] = {} + seen: dict[str, int] = {} for b in layout.get("blocks", []): if b["type"] in NARRATABLE and (b.get("text") or "").strip(): + key = re.sub(r"\d+", "#", " ".join(b["text"].split()).casefold()) + seen[key] = seen.get(key, 0) + 1 + for b in layout.get("blocks", []): + if b["type"] in NARRATABLE and (b.get("text") or "").strip(): + key = re.sub(r"\d+", "#", " ".join(b["text"].split()).casefold()) + if b["id"] not in owner and looks_like_page_junk(b["text"], seen[key]): + continue # not narratable page furniture: leave it out by_page.setdefault(b["page"], []).append(b) report = {"linked": 0, "created": 0, "unlinked": [], "dropped": 0} taken = {int(m.group(1)) for a in articles if (m := re.fullmatch(r"article-(\d+)", a["id"]))} @@ -4783,6 +4846,12 @@ def auto_link_pages(layout: dict[str, Any], articles: list[dict[str, Any]], report["dropped"] += 1 continue kept.append(a) + # An article whose every text region the reviewer had already approved (in whichever article + # it sat before) is as reviewed as it was: do not make it be approved a second time. + for a in kept: + narrated = [b for b in a["blocks"] if blocks.get(b, {}).get("type") in NARRATABLE] + if a.get("status") == "detected" and narrated and all(b in approved_ids for b in narrated): + a["status"] = "approved" return kept, report diff --git a/routes/magazine.py b/routes/magazine.py index f5be95d..25c22d4 100644 --- a/routes/magazine.py +++ b/routes/magazine.py @@ -1066,6 +1066,34 @@ def _vision_args(settings: dict[str, Any]) -> tuple[str | None, str | None, str pick("vision_ocr_fallback_url"), pick("vision_ocr_fallback_model")) +def _refresh_quality_rows(d: Path, layout: dict[str, Any], pages: list[int]) -> None: + """Re-measure `pages` and update their rows in quality_report.json, so the thumbnail badges and + the list of pages needing a decision match the layout as it is now (a per-page fix otherwise + left the last whole-issue run's numbers standing).""" + path = d / _QUALITY_REPORT + try: + report = json.loads(path.read_text("utf-8")) + except (OSError, ValueError): + return + rows = {r["page"]: r for r in report.get("pages", [])} + for page in pages: + if mag.page_is_advertisement(layout, page): + continue + fresh = mag.audit_page_coverage(d, layout, page) + row = rows.get(page, {"page": page}) + row.update({k: fresh[k] for k in ("uncovered", "uncovered_ratio", "suspect_blocks", "overlaps", "clipped", "measured")}) + spell = row.get("spelling") or [] + row["needs_review"] = bool(row["uncovered"] or row["overlaps"] or row["clipped"] or row.get("error") + or row.get("duplicates")) + row["words_to_check"] = bool(row["suspect_blocks"] or spell) + rows[page] = row + report["pages"] = [rows[p] for p in sorted(rows)] + summary = report.setdefault("summary", {}) + summary["pages_needing_review"] = [r["page"] for r in report["pages"] if r.get("needs_review")] + summary["pages_with_words"] = [r["page"] for r in report["pages"] if r.get("words_to_check") and not r.get("needs_review")] + path.write_text(json.dumps(report, ensure_ascii=False), encoding="utf-8") + + def _fix_page_geometry(d: Path, layout: dict[str, Any], articles: list[dict[str, Any]], page: int, settings: dict[str, Any]) -> tuple[dict[str, Any], dict[str, int], bool]: """The deterministic box fixes for one page, in the order that lets each rely on the last. @@ -1087,8 +1115,17 @@ def _fix_page_geometry(d: Path, layout: dict[str, Any], articles: list[dict[str, layout = mag.reorder_page(layout, page) if url and model and any(m.get("lines_added") for m in snapped): layout, _ = mag.reread_grown_regions(d, layout, snapped, url, model, fb_url, fb_model) + layout, pictures = mag.add_uncovered_pictures(d, layout, page) changed = _add_split_paragraphs_to_articles(articles, cleanup) + if pictures and articles: + # A new figure belongs to the article with the most text on its page. + on_page = {b["id"] for b in layout.get("blocks", []) if b["page"] == page} + owner = max(articles, key=lambda a: len(on_page & set(a.get("blocks") or [])), default=None) + if owner is not None and on_page & set(owner.get("blocks") or []): + owner["images"] = list(dict.fromkeys([*(owner.get("images") or []), *pictures])) + changed = True counts = {"noise_retyped": noise, "pullquotes_found": len(quotes), "paragraphs_cut": len(wrap), + "pictures_added": len(pictures), "boxes_fitted": len(snapped), "bylines_split": len(cleanup.get("byline_split", []))} return layout, counts, changed @@ -1259,6 +1296,7 @@ def _autofix_page_sync(doc_id: str, job_id: str, page: int, mode: str, settings: mag.save_layout(d, layout) if moved: mag.save_articles(d, articles) + _refresh_quality_rows(d, layout, [page]) _seal_undo(d, token) done["undo"] = token fixed = sum(v for k, v in done.items() if k != "undo" and isinstance(v, int)) diff --git a/tests/test_magazine.py b/tests/test_magazine.py index 524ab58..f23397a 100644 --- a/tests/test_magazine.py +++ b/tests/test_magazine.py @@ -3002,6 +3002,15 @@ class AutoLinkPagesTests(unittest.TestCase): self.assertEqual(out[0]["status"], "approved") # the stand-ins were approved: still reviewed self.assertEqual((report["linked"], report["dropped"]), (2, 2)) + def test_a_detected_article_made_only_of_already_approved_text_becomes_approved(self): + blocks = [self.blk("t", 1, "title", "Der Butler", 0), self.blk("a", 1, "body", "Der Text beginnt und", 1), + self.blk("b", 2, "body", "geht weiter.", 0)] + arts = [self.art("article-001", "Der Butler", [1], ["t", "a"], "detected"), + self.art("article-002", "Page 1", [1], ["t", "a"], "approved"), + self.art("article-003", "Page 2", [2], ["b"], "approved")] + out = self.link(blocks, arts)[0] + self.assertEqual([(a["id"], a["status"]) for a in out], [("article-001", "approved")]) + def test_text_nobody_approved_sends_the_article_back_to_review(self): blocks = [self.blk("t", 1, "title", "Der Butler", 0), self.blk("a", 1, "body", "Der Text beginnt und", 1), self.blk("b", 2, "body", "geht weiter.", 0)] @@ -3078,3 +3087,62 @@ class ArticleEndedTests(unittest.TestCase): def test_a_headline_two_pages_on_does_not_end_it(self): self.assertFalse(self.ended([("a", 1, "body", "Ohne Kürzel am Ende"), ("h", 3, "title", "Viel später")], ["a"])) + + +class PageJunkTests(unittest.TestCase): + def test_page_lists_running_heads_and_reference_lists_are_junk(self): + from core.magazine import looks_like_page_junk + self.assertTrue(looks_like_page_junk("66 90,98 50,56 82, 114, 120 148, 168 172, 194")) + self.assertTrue(looks_like_page_junk("46 Arbeitswelt. Wirtschaft", repeats=5)) + self.assertTrue(looks_like_page_junk("Literatur [1] Trinkwalder, Wunderbare Einhörner")) + self.assertTrue(looks_like_page_junk("Studien, weitere Quellen und Tools:")) + self.assertFalse(looks_like_page_junk("Der Konzern braucht eine, die iPhone, iPad und Mac unentbehrlich macht.")) + self.assertFalse(looks_like_page_junk("Ein kurzer Satz.", repeats=1)) + + def test_auto_link_leaves_such_text_out_of_the_article(self): + from core.magazine import auto_link_pages + blk = lambda i, pg, ty, t, n: {"id": i, "page": pg, "type": ty, "text": t, "reading_order": n, "bbox": [0, 0, 1, 1]} + layout = {"pages": [], "images": [], "blocks": [ + blk("t", 1, "title", "Der Butler", 0), blk("a", 1, "body", "Der Text beginnt und", 1), + blk("b", 2, "body", "geht auf Seite zwei weiter.", 0), blk("j", 2, "body", "66 90,98 50,56 82, 114", 1)]} + art = lambda i, ti, pg, bl, st: {"id": i, "title": ti, "author": "", "pages": pg, "blocks": bl, "images": [], "status": st, + "confidence": 0.5, "section": "", "continued_to": [], "audio_status": "none"} + out, _ = auto_link_pages(layout, [art("article-001", "Der Butler", [1], ["t", "a"], "approved"), art("article-002", "Page 2", [2], ["b"], "approved")]) + self.assertEqual(out[0]["blocks"], ["t", "a", "b"]) + + +class UncoveredPictureTests(unittest.TestCase): + def setUp(self): + from pathlib import Path + from PIL import Image, ImageDraw + self.dir = Path(tempfile.mkdtemp(prefix="ttvc-pic-")) + (self.dir / "pages").mkdir() + im = Image.new("RGB", (600, 800), (248, 242, 244)) + draw = ImageDraw.Draw(im) + draw.rectangle([100, 200, 300, 350], fill=(60, 90, 140)) # a photo: dense ink, 200x150 + for y in range(500, 560, 12): # text lines: thin, sparse + draw.rectangle([100, y, 300, y + 3], fill=(93, 91, 92)) + im.save(self.dir / "pages" / "0001.png") + self.layout = {"pages": [{"page": 1, "width": 600, "height": 800}], "images": [], "blocks": []} + + def tearDown(self): + shutil.rmtree(self.dir, ignore_errors=True) + + def test_a_dense_unboxed_blob_becomes_a_figure_and_text_lines_do_not(self): + from core.magazine import add_uncovered_pictures + out, added = add_uncovered_pictures(self.dir, self.layout, 1) + self.assertEqual(len(added), 1) + x, y, w, h = out["images"][0]["bbox"] + self.assertTrue(90 <= x <= 105 and 190 <= y <= 205 and w >= 195 and h >= 145, out["images"][0]["bbox"]) + self.assertTrue((self.dir / "images" / out["images"][0]["filename"]).is_file()) + + def test_running_it_again_finds_nothing_new(self): + from core.magazine import add_uncovered_pictures + once, _ = add_uncovered_pictures(self.dir, self.layout, 1) + self.assertEqual(add_uncovered_pictures(self.dir, once, 1)[1], []) + + def test_running_heads_and_adverts_are_not_audited_for_clipped_edges(self): + from core.magazine import audit_page_coverage + layout = dict(self.layout, blocks=[{"id": "f", "page": 1, "type": "furniture", "text": "45", "bbox": [100, 500, 100, 10]}, + {"id": "a", "page": 1, "type": "ad", "text": "Kaufen", "bbox": [100, 520, 100, 10]}]) + self.assertEqual(audit_page_coverage(self.dir, layout, 1)["clipped"], []) diff --git a/tests/test_magazine_routes.py b/tests/test_magazine_routes.py index d06f0c3..d80fa2b 100644 --- a/tests/test_magazine_routes.py +++ b/tests/test_magazine_routes.py @@ -1462,6 +1462,7 @@ class QualityRunJobTests(unittest.TestCase): patch('routes.magazine.mag.recover_missing_text', return_value=(layout, 1)), \ patch('routes.magazine.mag.verify_ocr_recognition', return_value=(layout, 3, 2)) as verify, \ patch('routes.magazine.mag.audit_page_coverage', side_effect=audit_row), \ + patch('routes.magazine.mag.add_uncovered_pictures', side_effect=lambda d, layout, page: (layout, [])), \ patch('routes.magazine.mag.save_layout'), \ patch('routes.magazine._LIB', lib), \ patch('routes.magazine.background_job_update') as update: