From 55f147d7a7bb63a362311827a13af53e7f173aba Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Sat, 12 Sep 2026 10:17:00 +0200 Subject: [PATCH] fix(magazine): derive figure boundaries from the page render A vision OCR engine grounds a figure far less precisely than it transcribes one. Over all 138 figure regions of a 204-page issue the transcribed contents were right while the boxes were routinely offset, undersized, or sitting on the neighbouring text column -- page 23's card grid was reported at x 0.24-0.54 where the graphic runs 0.41-0.95. snap_visual_bboxes() reduces the page render to an ink-coverage grid, seeds a connected-component search with the engine's box, and re-joins tiles a gutter apart so a card grid comes back as one figure. Text blocks are excluded from the mask, which is what stops a figure swallowing the column beside it. A candidate must be solidly inked or framed on nearly all of its rows and columns (admitting a white-interior screenshot), may not shrink below 30% of the seed, and reverts when two figures land on the same graphic; anything else keeps the box it had. Runs before the figure crop in both OCR paths, so the images shown during playback are framed correctly too. Replaces refine_picture_ bounds(), which only recognised coloured panels and had no text fence. Also adds "Fix figure boundaries" in review and POST /api/magazine/{id}/figures/refine behind it, so an issue ingested before this pass can be corrected without re-running OCR. Verified live against c't KI Wissen 2026: 78 of 138 boxes corrected, the rest declined. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 17 ++ Manual.md | 11 +- PROJECT_MAP.md | 5 + VERSION | 2 +- architecture/magazine-ingestion.md | 55 +++++++ core/magazine.py | 256 ++++++++++++++++++++++++----- routes/magazine.py | 47 ++++++ static/js/magazine.js | 45 ++++- tests/test_magazine.py | 100 +++++++++++ tests/test_magazine_routes.py | 58 +++++++ 10 files changed, 550 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ae2fc..aa61624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi --- +## [1.27.0] — 2026-09-12 + +### Fixed +- **Figure boundaries are now derived from the scanned page, not from the OCR engine's guess.** + - A vision OCR engine locates a photo far less precisely than it reads type: over all 138 figure regions of a 204-page issue, figure boxes were routinely offset, undersized, or sitting on the neighbouring text column while the transcribed text of the same regions was correct. + - New `snap_visual_bboxes()` pass reduces the page render to an ink-coverage grid, seeds a connected-component search with the engine's box, and re-joins tiles a gutter apart — so a grid of cards or a chart beside its key comes back as one figure. Text blocks are excluded from the mask, which stops a figure swallowing the column beside it. + - Conservative by design: a candidate must be solidly inked *or* framed on nearly all of its rows and columns (which is what admits a white-interior UI screenshot), may not shrink below 30% of the engine's box, and reverts when two figures are pulled onto the same graphic. Anything it cannot place confidently keeps the box it had. + - Runs before the figure crop in both OCR paths, so the images shown during playback are framed correctly too. Corrected 78 of 138 figure boxes on the reference issue. + - Replaces `refine_picture_bounds()`, which only recognised coloured panels and had no text fence. + +### Added +- **"Fix figure boundaries" in magazine review**, and `POST /api/magazine/{id}/figures/refine` behind it. + - Re-snaps every figure box in an issue as a restart-safe background job. No OCR engine is called, so an issue ingested before this release is corrected without paying for its transcription again. + - Text, article grouping and existing approvals are untouched — safe to run halfway through a review. Reports how many boundaries moved. + +--- + ## [1.26.14] — 2026-09-10 ### Added diff --git a/Manual.md b/Manual.md index ef20ae4..a242ddc 100644 --- a/Manual.md +++ b/Manual.md @@ -47,7 +47,14 @@ turned into a podcast discussion between two or three voices you pick. mistaken for unclaimed text. If OCR quality is poor, **"OCR again"** re-reads just the current page through the configured vision engine (or **"OCR again (whole magazine)"** from the contents screen for every - page, which also resets any articles already reviewed). If the layout + page, which also resets any articles already reviewed). If a figure's + box is drawn in the wrong place or at the wrong size — a common miss, + because an OCR engine locates a photo far less precisely than it reads + type — **"Fix figure boundaries"** re-derives every figure box in the + issue from the scanned pages themselves. It calls no OCR engine, changes + no text and touches no approval, so it is safe to run at any point, + including on an issue you are halfway through reviewing; boxes it cannot + place confidently are left exactly as they were. If the layout engine missed a region entirely, **"Draw it"** lets you click-drag a box directly on the page image and add it as text (typed or OCRed) or as a new figure. Confirm the boundaries, correct the title, then **Approve**. @@ -99,6 +106,8 @@ unreviewed boundaries may include a neighbouring article or an advert. | "No LLM model configured" | `llm_model` is empty. | Set it in Settings. | | "LLM did not return usable JSON" | The model ignored the format. | Retry, or pick a stronger model. | | "Article has too little text" | Under 200 characters — usually a stub. | Check the boundaries in review. | +| A figure box sits over the wrong part of the page | The OCR engine grounded the figure loosely. | Press **"Fix figure boundaries"**; if the graphic is too pale to detect, draw the region by hand. | +| "Fix figure boundaries" reports nothing corrected | Every box already matched its page, or none could be placed with confidence. | Nothing is wrong; correct any remaining box by drawing it. | ### Privacy diff --git a/PROJECT_MAP.md b/PROJECT_MAP.md index f7956b1..69171a2 100644 --- a/PROJECT_MAP.md +++ b/PROJECT_MAP.md @@ -78,6 +78,11 @@ which is the writable volume. real measured audio duration, not true forced alignment. - No formal cross-engine OCR accuracy benchmark exists (two engines wired: ocr-baidu-unlimited-ocr primary, KDL Frontier Parser nano fallback). +- Figure boundaries are re-derived from the page render (`snap_visual_bboxes`) + because vision OCR grounds figures loosely; it corrected 78 of 138 figure + boxes on `c't KI Wissen 2026` and deliberately declines the rest. A pale + chart or line drawing on white is below its ink threshold and keeps the + engine's box — draw those by hand in review. - No `npm run qa` aggregate command yet; Python tests run via `unittest`, browser tests via `playwright test`. - The app has no authentication for the browser UI and no per-user data diff --git a/VERSION b/VERSION index f38d4cf..e43da41 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.26.14 +1.27.0 \ No newline at end of file diff --git a/architecture/magazine-ingestion.md b/architecture/magazine-ingestion.md index e6ce877..98ed956 100644 --- a/architecture/magazine-ingestion.md +++ b/architecture/magazine-ingestion.md @@ -34,6 +34,9 @@ PDF the vision call failed/timed out/was unreachable → build_vision_blocks() vision regions -> Blocks; a direct label bypasses classify_blocks entirely (see below) + → snap_visual_bboxes() figure/chart/table/code boxes re-derived from the page + render, because grounding is the weakest part of a + vision engine's answer (see below) → classify_blocks() title / deck / byline / body / caption / pullquote / sidebar / furniture / ad / figure -- for native/tesseract blocks, and any vision region left ambiguous ("text") @@ -357,6 +360,58 @@ fallback and not the default. A formal cross-engine accuracy benchmark remains TODO P2. +### Figure bounds come from the render, not the engine + +A vision OCR engine grounds a figure far less precisely than it transcribes +one. Measured over all 138 figure regions of `c't KI Wissen 2026` (204 pages, +Baidu Unlimited-OCR): text boxes matched the page render closely enough to +use as-is, while figure boxes were routinely offset, undersized, or landed on +the neighbouring text column — page 23's card grid was reported at +`x 0.24–0.54` of the page where the graphic actually runs `0.41–0.95`, and +page 20's full-width illustration was cut off at a third of its height. The +transcribed *contents* of those same regions were right, which is the useful +signal: the model reliably says **which** graphic it means, so the render is +asked **where** that graphic ends. + +`snap_visual_bboxes()` (core/magazine.py) does that, on a coarse grid of the +rendered page — `page_ink_grid()` reduces it to the inked fraction of each +10 px cell, which separates graphics from type cleanly: body copy covers +roughly a quarter of the cells it runs through, a photo or filled panel +covers nearly all of them. The engine's box seeds a connected-component +search over the "solid" cells (scipy `ndimage.label`); components a gutter +apart that line up are re-joined, so a grid of cards or a chart beside its +key comes back as one figure rather than four. + +Three fences keep it from making things worse, because a box confidently +snapped onto the wrong thing is worse than a loose one the user can see: + +- **Text blocks are excluded from the mask up front.** They are the one + trustworthy boundary on the page, and without them a figure happily + swallows the column beside it. +- **Acceptance is explicit.** The result must be either solidly inked + (`_SNAP_MIN_FILL`) or framed on nearly all of its rows *and* columns — the + second test is what admits a UI screenshot, which is white inside and inked + only at its edge — and it may not shrink below `_SNAP_MIN_SEED_AREA` of the + engine's box. Anything else keeps the engine's box untouched. +- **Two figures pulled onto one graphic revert.** Either the page is one + picture the engine read as two or the reverse; neither reading can be + trusted over the other. + +Result on that issue: 78 of 138 boxes corrected, none made worse on visual +inspection of the sample, the rest left alone (pale charts on white, mostly, +which the ink test correctly declines). The pass is idempotent — an +already-correct box seeds the same graphic again — which is what allows it to +run outside ingestion: `refresh_visual_regions()` re-snaps on every +re-detection, and `POST /api/magazine/{id}/figures/refine` re-snaps a whole +issue as a background job, so a document ingested before this pass existed is +corrected without paying for its transcription a second time. It replaced +`refine_picture_bounds()`, which only recognised *coloured* panels and had no +text fence. + +The snap runs before `crop_visual_regions()` in both OCR paths: the figure +files shown during playback are cropped from these boxes, so a box corrected +late would leave a mis-framed image on disk. + The Active OCR Engine panel (Settings → Engines → OCR & Document Parsing) mirrors the Active Language Model panel exactly — same endpoint/model selects, same `/api/conversation/llm-models` fetch (it just proxies to diff --git a/core/magazine.py b/core/magazine.py index 3b78933..22a2e06 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -1143,6 +1143,201 @@ def build_vision_blocks(regions: list[dict[str, Any]], page_no: int, return out +# ── Figure bounds from the page render ─────────────────────────────────────── +# +# A vision OCR engine grounds a figure far less precisely than it transcribes +# one: on c't KI Wissen 2026 the label and the read-out contents were right +# while the four numbers landed on the neighbouring text column at two thirds +# of the real size (page 23), or covered only the top third of a full-width +# illustration (page 20). The rendered page itself carries the answer -- a +# photo, a chart or a UI screenshot is a densely inked rectangle fenced off +# from the type area by white gutters -- so the model's box is used only to +# say WHICH graphic is meant and the render says where that graphic ends. +# +# Deliberately conservative: every rule below can decline, and declining +# leaves the model's box untouched. A wrong box the user can see and redraw +# is a nuisance; a box confidently snapped onto the wrong thing is worse. + +_SNAP_CELL_PX = 10 # grid resolution -- ~2 mm at the 110 dpi page render +_SNAP_SOLID = 0.55 # inked fraction marking a cell as graphic, not type +_SNAP_GAP_CELLS = 3 # white gutter a tiled graphic is allowed to span +_SNAP_MIN_FILL = 0.45 # an accepted box is this solidly inked ... +_SNAP_MIN_FRAME = 0.9 # ... or framed on this share of its rows AND columns +_SNAP_MIN_SEED_AREA = 0.30 # ... and never shrinks below this much of the seed +_SNAP_MIN_OVERLAP = 0.02 # a graphic must touch the seed by at least this much + + +def page_ink_grid(png_path: Path, cell: int = _SNAP_CELL_PX): + """Coarse map of the page render: inked fraction of each `cell`-px square. + + "Inked" means visibly darker than the paper or visibly coloured, so a + grey scan background does not count as ink. Body type covers roughly a + quarter of the cells it runs through; a photo or a filled panel covers + nearly all of them, which is the separation snap_visual_bboxes() relies + on. Returns None when the render cannot be read. + """ + try: + import numpy as np + from PIL import Image + with Image.open(png_path) as im: + rgb = np.asarray(im.convert("RGB"), dtype=np.float32) + except Exception: + return None + lightness = rgb.max(axis=2) + saturation = lightness - rgb.min(axis=2) + paper = float(np.percentile(lightness, 90)) or 255.0 + ink = (lightness < paper * 0.88) | (saturation > 40) + rows, cols = (n // cell for n in ink.shape) + if rows < 4 or cols < 4: + return None + return ink[:rows * cell, :cols * cell].reshape( + rows, cell, cols, cell).mean(axis=(1, 3)) + + +def _snap_seed(coverage, seed: Sequence[int], blocked) -> list[int] | None: + """Grow the graphic the seed cell-box points at, or None to keep the seed. + + Cells already claimed by a text block are excluded up front: text boxes + come back from the same engine far more accurately than figure boxes + (verified against the page render for every block of the issue above), + so they are the one trustworthy fence against a figure swallowing the + column beside it. + """ + from scipy import ndimage + + solid = (coverage >= _SNAP_SOLID) & ~blocked + labels, count = ndimage.label(solid) + if not count: + return None + slices = ndimage.find_objects(labels) + boxes = [[s[1].start, s[0].start, s[1].stop - 1, s[0].stop - 1] + for s in slices] + sizes = ndimage.sum_labels(solid, labels, range(1, count + 1)) + + sx0, sy0, sx1, sy1 = seed + seed_area = max(1, (sx1 - sx0 + 1) * (sy1 - sy0 + 1)) + overlaps = [_cell_overlap(b, seed) for b in boxes] + best = max(range(count), key=overlaps.__getitem__) + if overlaps[best] < _SNAP_MIN_OVERLAP * seed_area: + return None + + # A figure split by white gutters (a grid of cards, a chart beside its + # key) arrives as several components; re-join the ones that line up with + # what we have so far, then re-test -- joining one may bring the next + # within reach. + box = list(boxes[best]) + joined = {best} + growing = True + while growing: + growing = False + for i, other in enumerate(boxes): + if i in joined or sizes[i] < 4 or not _cells_adjacent(box, other): + continue + box = [min(box[0], other[0]), min(box[1], other[1]), + max(box[2], other[2]), max(box[3], other[3])] + joined.add(i) + growing = True + + area = (box[2] - box[0] + 1) * (box[3] - box[1] + 1) + if area < _SNAP_MIN_SEED_AREA * seed_area: + return None + inside = coverage[box[1]:box[3] + 1, box[0]:box[2] + 1] + if inside.mean() >= _SNAP_MIN_FILL: + return box + # A screenshot or a boxed insert is mostly white inside but inked right + # around its edge, so a solid frame counts as a graphic too. + edge = solid[box[1]:box[3] + 1, box[0]:box[2] + 1] + if (edge.any(axis=1).mean() >= _SNAP_MIN_FRAME + and edge.any(axis=0).mean() >= _SNAP_MIN_FRAME): + return box + return None + + +def _cell_overlap(a: Sequence[int], b: Sequence[int]) -> int: + """Area, in grid cells, shared by two inclusive [x0, y0, x1, y1] boxes.""" + return (max(0, min(a[2], b[2]) - max(a[0], b[0]) + 1) + * max(0, min(a[3], b[3]) - max(a[1], b[1]) + 1)) + + +def _cells_adjacent(box: Sequence[int], other: Sequence[int]) -> bool: + """True when `other` is a gutter away from `box` and lines up with it.""" + gap = _SNAP_GAP_CELLS + if not (other[0] <= box[2] + gap and other[2] >= box[0] - gap + and other[1] <= box[3] + gap and other[3] >= box[1] - gap): + return False + # Side by side or stacked, not merely diagonal: a tile of the same figure + # shares most of a row or most of a column with it. + shared_rows = max(0, min(other[3], box[3]) - max(other[1], box[1]) + 1) + shared_cols = max(0, min(other[2], box[2]) - max(other[0], box[0]) + 1) + return (shared_rows >= 0.5 * min(other[3] - other[1] + 1, box[3] - box[1] + 1) + or shared_cols >= 0.5 * min(other[2] - other[0] + 1, box[2] - box[0] + 1)) + + +def snap_visual_bboxes(png_path: Path, blocks: Sequence[Block], + page_meta: dict[str, Any]) -> int: + """Correct every visual block's bbox against the page render, in place. + + Returns how many boxes moved. Silent no-op when the render is missing or + the imaging stack is unavailable -- the layout stays exactly as the OCR + engine reported it, which is what happened before this pass existed. + """ + visuals = [b for b in blocks if b.type in VISUAL_TYPES] + if not visuals: + return 0 + try: + import numpy as np + import scipy.ndimage # noqa: F401 -- required by _snap_seed + except ImportError: + return 0 + coverage = page_ink_grid(png_path) + if coverage is None: + return 0 + rows, cols = coverage.shape + page_w = float(page_meta.get("width") or 0) or 1.0 + page_h = float(page_meta.get("height") or 0) or 1.0 + cells_x, cells_y = cols / page_w, rows / page_h + + def to_cells(bbox: Sequence[float]) -> list[int]: + x, y, w, h = bbox + return [min(cols - 1, max(0, int(x * cells_x))), + min(rows - 1, max(0, int(y * cells_y))), + min(cols - 1, max(0, int((x + w) * cells_x))), + min(rows - 1, max(0, int((y + h) * cells_y)))] + + blocked = np.zeros(coverage.shape, dtype=bool) + for b in blocks: + if b.type in VISUAL_TYPES or not b.text.strip(): + continue + x0, y0, x1, y1 = to_cells(b.bbox) + blocked[y0:y1 + 1, x0:x1 + 1] = True + + snapped = [_snap_seed(coverage, to_cells(b.bbox), blocked) for b in visuals] + # Two figures pulled onto the same graphic means the page was read as + # one picture where the engine saw two (or the reverse); neither reading + # can be trusted over the other, so both keep their original box. + for i, a in enumerate(snapped): + for j in range(i + 1, len(snapped)): + c = snapped[j] + if not a or not c: + continue + smaller = min((a[2] - a[0] + 1) * (a[3] - a[1] + 1), + (c[2] - c[0] + 1) * (c[3] - c[1] + 1)) + if _cell_overlap(a, c) > 0.5 * smaller: + snapped[i] = snapped[j] = None + a = None + + moved = 0 + for block, box in zip(visuals, snapped): + if box is None: + continue + bbox = [box[0] / cells_x, box[1] / cells_y, + (box[2] + 1 - box[0]) / cells_x, (box[3] + 1 - box[1]) / cells_y] + if any(abs(a - b) > 1.0 for a, b in zip(bbox, block.bbox)): + moved += 1 + block.bbox = bbox + return moved + + def ocr_page(png_path: Path, lang: str = "deu+eng", min_confidence: float = 40.0) -> list[dict[str, Any]]: """OCR one rendered page, returning paragraph boxes from tesseract's TSV. @@ -1355,8 +1550,13 @@ def extract_pages(pdf_path: Path, out_dir: Path, render_dpi: int = 110, # conflicting reading on top of it. vision_page_count += 1 blocks = vision_blocks + page_meta = {"width": rect.width, "height": rect.height} + # Before cropping: the crop is taken from the block's bbox, + # so a box the engine placed loosely would be saved as a + # mis-framed figure file as well as a mis-drawn review box. + snap_visual_bboxes(png, blocks, page_meta) detected_images = crop_visual_regions(out_dir, page_no, - {"width": rect.width, "height": rect.height}, blocks) + page_meta, blocks) if detected_images: page_images = detected_images elif needs_ocr: @@ -1604,6 +1804,7 @@ def reocr_page(doc_dir: Path, layout: dict[str, Any], page_no: int, scale_x = page_meta["width"] / max(1.0, _png_width(png, fitz)) new_blocks = build_vision_blocks(regions, page_no, scale_x, id_prefix="r") + snap_visual_bboxes(png, new_blocks, page_meta) all_blocks = [Block(**b) for b in layout.get("blocks", [])] kept = [b for b in all_blocks if b.page != page_no] @@ -1734,54 +1935,23 @@ def _crop_page_png(doc_dir: Path, page_no: int, page_meta: dict[str, Any], im.crop(box_px).save(dest) -def refine_picture_bounds(png: Path, page_meta: dict, bbox: Sequence[float]) -> list[float]: - """Snap an approximate picture box to a large, solid coloured panel. - - Text and small decorations do not qualify. Grayscale or fragmented - artwork keeps the model's box instead of guessing a new boundary. - """ - import numpy as np - from PIL import Image - from scipy import ndimage - with Image.open(png) as image: - rgb = np.asarray(image.convert("RGB"), dtype=np.int16) - height, width = rgb.shape[:2] - mask = (rgb.max(axis=2) - rgb.min(axis=2) > 28) & (rgb.min(axis=2) < 225) - mask = ndimage.binary_closing(mask, iterations=3) - labels, _ = ndimage.label(mask) - candidates = [] - x, y, w, h = bbox - for i, slices in enumerate(ndimage.find_objects(labels), 1): - if slices is None: - continue - sy, sx = slices - area = (sy.stop - sy.start) * (sx.stop - sx.start) - if area < width * height * .025 or area > width * height * .85: - continue - if np.count_nonzero(labels[slices] == i) / area < .7: - continue - box = [sx.start / width * page_meta["width"], sy.start / height * page_meta["height"], - (sx.stop - sx.start) / width * page_meta["width"], (sy.stop - sy.start) / height * page_meta["height"]] - overlap = max(0, min(x+w, box[0]+box[2])-max(x, box[0])) * max(0, min(y+h, box[1]+box[3])-max(y, box[1])) - score = overlap / max(1, min(w*h, box[2]*box[3])) - if score > .6: - candidates.append((score, box)) - return max(candidates, key=lambda c: c[0])[1] if candidates else list(bbox) - - def refresh_visual_regions(doc_dir: Path, layout: dict, page: int | None = None) -> None: - """Regenerate semantic crops and avoid treating a scanned page as a figure.""" + """Re-snap figure bounds, regenerate the crops taken from them, and avoid + treating a whole scanned page as one figure.""" for meta in layout.get("pages", []): page_no = meta["page"] if page is not None and page_no != page: continue blocks = [Block(**b) for b in layout.get("blocks", []) if b["page"] == page_no] - for block in blocks: - if block.type == BLOCK_FIGURE and block.source == "llm_layout": - block.bbox = refine_picture_bounds(doc_dir / "pages" / f"{page_no:04d}.png", meta, block.bbox) - for saved in layout["blocks"]: - if saved["id"] == block.id: - saved["bbox"] = block.bbox + # Re-snapping here, not only at OCR time, is what lets an issue + # ingested before this pass existed pick up corrected figure boxes + # without paying for a second OCR run of the whole document. It is + # idempotent: an already-correct box seeds the same graphic again. + if snap_visual_bboxes(doc_dir / "pages" / f"{page_no:04d}.png", blocks, meta): + snapped = {b.id: b.bbox for b in blocks if b.type in VISUAL_TYPES} + for saved in layout["blocks"]: + if saved["id"] in snapped: + saved["bbox"] = snapped[saved["id"]] crops = crop_visual_regions(doc_dir, page_no, meta, blocks) kept = [] for img in layout.get("images", []): diff --git a/routes/magazine.py b/routes/magazine.py index e7f9f56..fe978ea 100644 --- a/routes/magazine.py +++ b/routes/magazine.py @@ -646,6 +646,53 @@ async def magazine_reocr_page(doc_id: str, page: int, request: Request): return {"job_id": job_id} +def _refine_figures_sync(doc_id: str, job_id: str) -> None: + """Re-snap every figure box in a document against its page renders. + + Restart-safe like the other layout jobs: a 200-page issue means 200 page + renders read and re-cropped, far too long for a request/response cycle. + Cheap compared with re-ingesting, though -- no OCR engine is called, so + an issue read before the snapping pass existed can be corrected without + paying for the transcription again, and without disturbing the text, + the article grouping, or any approval the user has already given. + """ + d = _LIB / doc_id + try: + layout = mag.load_layout(d) + if not layout: + raise FileNotFoundError("No layout yet -- run ingestion first") + pages = layout.get("pages", []) + before = {b["id"]: list(b["bbox"]) for b in layout.get("blocks", [])} + for i, meta in enumerate(pages, 1): + background_job_update(job_id, stage="figures", + detail=f"Checking figure boundaries on page {i} of {len(pages)}") + mag.refresh_visual_regions(d, layout, meta["page"]) + moved = sum(1 for b in layout.get("blocks", []) + if before.get(b["id"]) and b["bbox"] != before[b["id"]]) + mag.save_layout(d, layout) + background_job_update( + job_id, stage="done", done=True, result={"moved": moved}, + detail=f"{moved} figure boundar{'y' if moved == 1 else 'ies'} corrected" + if moved else "Every figure boundary already matched its page") + except Exception as exc: # noqa: BLE001 - surfaced to UI + background_job_update(job_id, stage="failed", done=True, error=str(exc)[:1000]) + + +@router.post("/api/magazine/{doc_id}/figures/refine") +async def magazine_refine_figures(doc_id: str): + """Re-check every figure boundary in this issue against the page renders. + + For an issue whose figure boxes came back loose from the OCR engine: the + boxes are re-derived from the rendered pages and the figure crops taken + from them are rebuilt. Text, articles and approvals are untouched. + """ + _require_magazine(doc_id) + job_id = uuid.uuid4().hex + background_job_create(job_id, "magazine-refine-figures", fingerprint=doc_id) + asyncio.create_task(asyncio.to_thread(_refine_figures_sync, doc_id, job_id)) + return {"job_id": job_id} + + def _add_region_sync(doc_id: str, job_id: str, page: int, bbox: list[float], kind: str, text: str, vision_ocr_url: str | None, vision_ocr_model: str | None, vision_ocr_fallback_url: str | None, vision_ocr_fallback_model: str | None) -> None: diff --git a/static/js/magazine.js b/static/js/magazine.js index da137f3..ee865d6 100644 --- a/static/js/magazine.js +++ b/static/js/magazine.js @@ -740,9 +740,12 @@ function renderReview() { ' Advertisement' + '' + '' + + '' + '' + - '' + + '' + ''; html += '
'; @@ -1173,6 +1176,45 @@ async function magReocrPage() { renderReview(); } +// Re-derives every figure box in the issue from the page renders. Separate +// from "OCR again" because no OCR engine is involved: it only corrects where +// the figures are, never what the page says, so it is safe to run on an issue +// that is half reviewed already. +async function magRefineFigures() { + const btn = $('mag-refine-figures-btn'); + const statusEl = $('mag-reocr-status'); + if (btn) btn.disabled = true; + if (statusEl) statusEl.textContent = 'Checking figure boundaries…'; + let jobId; + try { + const r = await api('/api/magazine/' + state.doc.id + '/figures/refine', { method: 'POST' }); + jobId = r.job_id; + } catch (e) { + if (btn) btn.disabled = false; + if (statusEl) statusEl.textContent = ''; + return say('Could not start the check: ' + e.message, 'err'); + } + let failed = false; + for (;;) { + let job; + try { job = await api('/api/jobs/' + jobId); } catch (_) { break; } + if (job.done) { + if (job.error) { failed = true; say('Figure check failed: ' + job.error, 'err'); } + else say(job.detail || 'Figure boundaries checked', 'ok'); + break; + } + if (statusEl && job.detail) statusEl.textContent = job.detail; + await new Promise((r) => setTimeout(r, 2000)); + } + if (btn) btn.disabled = false; + if (statusEl) statusEl.textContent = ''; + if (failed) return; + state.layout = null; + await ensureLayout(); + _magReviewChecked = null; + renderReview(); +} + async function magApprove() { const a = state.article; if (!a) return; @@ -1805,6 +1847,7 @@ root.magReviewFocusBlock = magReviewFocusBlock; root.magReviewHighlightBox = magReviewHighlightBox; root.magReviewToggleBox = magReviewToggleBox; root.magReocrPage = magReocrPage; +root.magRefineFigures = magRefineFigures; root.magReviewToggleImage = magReviewToggleImage; root.magReviewToggleDraw = magReviewToggleDraw; root.magReviewCancelDrawPanel = magReviewCancelDrawPanel; diff --git a/tests/test_magazine.py b/tests/test_magazine.py index 6489f88..ee875eb 100644 --- a/tests/test_magazine.py +++ b/tests/test_magazine.py @@ -19,6 +19,9 @@ os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-magazine-test- from core.magazine import ( ocr_page, cluster_running_furniture, + snap_visual_bboxes, + page_ink_grid, + BLOCK_FIGURE, looks_like_real_title, Article, BLOCK_AD, @@ -1435,5 +1438,102 @@ class ManualRegionTests(unittest.TestCase): self.assertEqual(len(names), 2) +class SnapVisualBboxesTests(unittest.TestCase): + """A vision OCR engine grounds a figure loosely; the render is the truth. + + The synthetic page mirrors the real failure seen on c't KI Wissen 2026: + a solid graphic on the right, a text column on the left, and a reported + figure box that straddles both at a fraction of the real size. + """ + + PAGE_W, PAGE_H = 582.0, 678.0 # PDF points + PX_W, PX_H = 890, 1037 # rendered pixels + FIGURE_PX = (400, 100, 850, 400) # x0, y0, x1, y1 + TEXT_PX = (60, 100, 350, 500) + + def setUp(self): + from PIL import Image, ImageDraw + self.dir = pathlib.Path(tempfile.mkdtemp()) + (self.dir / "pages").mkdir() + page = Image.new("RGB", (self.PX_W, self.PX_H), (255, 255, 255)) + draw = ImageDraw.Draw(page) + draw.rectangle(self.FIGURE_PX, fill=(40, 90, 200)) + x0, y0, x1, y1 = self.TEXT_PX + for y in range(y0, y1, 12): # type covers ~a third of its cells + draw.rectangle((x0, y, x1, y + 3), fill=(20, 20, 20)) + self.png = self.dir / "pages" / "0001.png" + page.save(self.png) + self.meta = {"page": 1, "width": self.PAGE_W, "height": self.PAGE_H} + + def tearDown(self): + shutil.rmtree(self.dir, ignore_errors=True) + + def _pt(self, px_box): + """Pixel [x0, y0, x1, y1] as a PDF-point [x, y, w, h] bbox.""" + sx, sy = self.PAGE_W / self.PX_W, self.PAGE_H / self.PX_H + x0, y0, x1, y1 = px_box + return [x0 * sx, y0 * sy, (x1 - x0) * sx, (y1 - y0) * sy] + + def _text_block(self): + return Block(id="t1", page=1, bbox=self._pt(self.TEXT_PX), + type=BLOCK_BODY, text="body copy " * 40) + + def _figure_block(self, px_box): + return Block(id="f1", page=1, bbox=self._pt(px_box), type=BLOCK_FIGURE, + text="figure", source="vision_label") + + def test_grid_separates_graphic_from_type(self): + grid = page_ink_grid(self.png) + self.assertIsNotNone(grid) + rows, cols = grid.shape + fig = grid[int(200 / self.PX_H * rows), int(600 / self.PX_W * cols)] + txt = grid[int(200 / self.PX_H * rows), int(200 / self.PX_W * cols)] + self.assertGreater(fig, 0.9) + self.assertLess(txt, 0.5) + + def test_offset_box_snaps_onto_the_graphic(self): + block = self._figure_block((200, 60, 600, 250)) + moved = snap_visual_bboxes(self.png, [block, self._text_block()], self.meta) + self.assertEqual(moved, 1) + want = self._pt(self.FIGURE_PX) + for got, expected in zip(block.bbox, want): + self.assertAlmostEqual(got, expected, delta=12.0) + + def test_snap_does_not_swallow_the_text_column(self): + block = self._figure_block((200, 60, 600, 250)) + snap_visual_bboxes(self.png, [block, self._text_block()], self.meta) + self.assertGreater(block.bbox[0], self._pt(self.TEXT_PX)[0] + 20) + + def test_snapping_is_idempotent(self): + block = self._figure_block((200, 60, 600, 250)) + blocks = [block, self._text_block()] + snap_visual_bboxes(self.png, blocks, self.meta) + settled = list(block.bbox) + self.assertEqual(snap_visual_bboxes(self.png, blocks, self.meta), 0) + self.assertEqual(block.bbox, settled) + + def test_box_over_type_only_is_left_alone(self): + block = self._figure_block((60, 100, 350, 500)) + before = list(block.bbox) + self.assertEqual( + snap_visual_bboxes(self.png, [block, self._text_block()], self.meta), 0) + self.assertEqual(block.bbox, before) + + def test_two_figures_on_one_graphic_keep_their_boxes(self): + left = self._figure_block((410, 110, 600, 380)) + right = Block(id="f2", page=1, bbox=self._pt((610, 110, 840, 380)), + type=BLOCK_FIGURE, text="figure", source="vision_label") + before = [list(left.bbox), list(right.bbox)] + self.assertEqual(snap_visual_bboxes(self.png, [left, right], self.meta), 0) + self.assertEqual([left.bbox, right.bbox], before) + + def test_missing_render_is_a_no_op(self): + block = self._figure_block((200, 60, 600, 250)) + before = list(block.bbox) + self.assertEqual( + snap_visual_bboxes(self.dir / "pages" / "0099.png", [block], self.meta), 0) + self.assertEqual(block.bbox, before) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_magazine_routes.py b/tests/test_magazine_routes.py index ffa5309..7c7bab0 100644 --- a/tests/test_magazine_routes.py +++ b/tests/test_magazine_routes.py @@ -160,3 +160,61 @@ class AdvertisementReviewTests(unittest.TestCase): self.assertEqual(len(layout['blocks']), 1) self.assertEqual(layout['blocks'][0]['type'], 'ad') self.assertEqual(layout['blocks'][0]['bbox'], [0, 0, 600, 800]) + + +class RefineFiguresJobTests(unittest.TestCase): + """Document-wide figure re-snap: reports what moved, keeps text untouched.""" + + def _layout(self): + return { + "pages": [{"page": 1, "width": 600, "height": 800}, + {"page": 2, "width": 600, "height": 800}], + "blocks": [ + {"id": "fig", "page": 1, "type": "figure", "bbox": [10, 10, 50, 50], "text": ""}, + {"id": "copy", "page": 2, "type": "body", "bbox": [0, 0, 100, 100], "text": "Body"}, + ], + } + + def test_moved_boxes_are_counted_and_saved(self): + from unittest.mock import patch + from routes.magazine import _refine_figures_sync + layout = self._layout() + + def refresh(_dir, doc, page=None): + if page == 1: + doc["blocks"][0]["bbox"] = [20, 20, 90, 90] + + with patch('routes.magazine.mag.load_layout', return_value=layout), \ + patch('routes.magazine.mag.refresh_visual_regions', side_effect=refresh), \ + patch('routes.magazine.mag.save_layout') as save, \ + patch('routes.magazine.background_job_update') as update: + _refine_figures_sync('a' * 32, 'job') + save.assert_called_once() + done = [c.kwargs for c in update.call_args_list if c.kwargs.get('done')] + self.assertEqual(done[-1]['stage'], 'done') + self.assertEqual(done[-1]['result'], {'moved': 1}) + self.assertEqual(layout['blocks'][1]['bbox'], [0, 0, 100, 100]) + + def test_nothing_to_correct_still_finishes_cleanly(self): + from unittest.mock import patch + from routes.magazine import _refine_figures_sync + with patch('routes.magazine.mag.load_layout', return_value=self._layout()), \ + patch('routes.magazine.mag.refresh_visual_regions'), \ + patch('routes.magazine.mag.save_layout'), \ + patch('routes.magazine.background_job_update') as update: + _refine_figures_sync('a' * 32, 'job') + done = [c.kwargs for c in update.call_args_list if c.kwargs.get('done')] + self.assertEqual(done[-1]['result'], {'moved': 0}) + self.assertNotIn('error', done[-1]) + + def test_a_document_with_no_layout_reports_failure(self): + from unittest.mock import patch + from routes.magazine import _refine_figures_sync + with patch('routes.magazine.mag.load_layout', return_value=None), \ + patch('routes.magazine.mag.save_layout') as save, \ + patch('routes.magazine.background_job_update') as update: + _refine_figures_sync('a' * 32, 'job') + save.assert_not_called() + done = [c.kwargs for c in update.call_args_list if c.kwargs.get('done')] + self.assertEqual(done[-1]['stage'], 'failed') + self.assertIn('No layout', done[-1]['error'])