"""Layout-aware magazine ingestion.

Magazines are not books: a single page routinely carries several unrelated
articles, advertisements, sidebars and pull quotes, and an article often
continues twenty pages later.  Treating a page as the unit of narration
therefore produces audio that jumps mid-sentence into an advert.  The unit
here is the *article*, reconstructed from classified layout blocks.

Pipeline (see architecture/magazine-ingestion.md):

    PDF
     -> extract_pages()        native text blocks with bbox/font via PyMuPDF
     -> ocr_page()             tesseract TSV fallback for scanned pages
     -> classify_blocks()      title / deck / byline / body / caption / ...
     -> assign_reading_order() column detection, then top-to-bottom per column
     -> detect_contents()      Stage A: locate and parse the table of contents
     -> group_articles()       Stage C: group blocks, link continuations
     -> (human review)         Stage D happens in the UI; nothing is narrated
                               until an article is approved.

PyMuPDF is imported lazily inside the I/O functions so that the pure geometry
and classification helpers -- which hold all the fiddly logic worth testing --
can be exercised without the native dependency or a sample PDF.
"""
from __future__ import annotations

import base64
import difflib
import json
import logging
import math
import re
import statistics
import subprocess
import tempfile
import unicodedata

import requests

from core.retry import request_with_retries
from core.pronunciations import (
    _SUSPENDED_COMPOUND_WORDS, detect_article_language, language_for_block,
    normalize_ocr_text, prepare_text_for_tts, recurring_tokens, remove_linebreak_hyphens,
    suspect_ocr_tokens,
)
from dataclasses import dataclass, field, asdict
from pathlib import Path
from typing import Any, Iterable, Sequence
from urllib.parse import urlsplit

# ── Block taxonomy ───────────────────────────────────────────────────────────
# Ordered roughly by how much they matter to narration.  Only NARRATABLE types
# are ever sent to TTS; the rest are kept for display and provenance.
BLOCK_TITLE = "title"
BLOCK_HEADING = "heading"
BLOCK_DECK = "deck"            # standfirst / subtitle under the headline
BLOCK_BYLINE = "byline"
BLOCK_BODY = "body"
BLOCK_QUOTE = "quote"          # quoted speech that belongs in narration
BLOCK_CAPTION = "caption"
BLOCK_PULLQUOTE = "pullquote"  # verbatim repeat of body text, blown up
BLOCK_SIDEBAR = "sidebar"
BLOCK_FURNITURE = "furniture"  # running heads, folios, page numbers
BLOCK_AD = "ad"
BLOCK_FIGURE = "figure"
BLOCK_CHART = "chart"
BLOCK_TABLE = "table"
BLOCK_CODE = "code"
VISUAL_TYPES = (BLOCK_FIGURE, BLOCK_CHART, BLOCK_TABLE, BLOCK_CODE)

NARRATABLE = (BLOCK_TITLE, BLOCK_HEADING, BLOCK_DECK, BLOCK_BYLINE, BLOCK_BODY, BLOCK_QUOTE)

# Text that marks an article jumping to (or arriving from) another page.
_CONT_TO = re.compile(
    r"\((?:continued|cont\.?)\s+on\s+page\s+(\d{1,4})\)"
    r"|continued\s+on\s+page\s+(\d{1,4})"
    r"|fortsetzung\s+auf\s+seite\s+(\d{1,4})"
    r"|weiter\s+auf\s+seite\s+(\d{1,4})",
    re.I,
)
_CONT_FROM = re.compile(
    r"continued\s+from\s+page\s+(\d{1,4})"
    r"|fortsetzung\s+von\s+seite\s+(\d{1,4})",
    re.I,
)

_BYLINE = re.compile(
    r"^\s*(?:by|von|text|words|story|interview)\s*[:  ]\s*(.{2,80})$",
    re.I,
)

# Headings that mark a contents page, and the section headers inside one.
_TOC_MARKERS = (
    "contents", "table of contents", "in this issue", "inside this issue",
    "features", "departments", "columns", "regulars",
    "inhalt", "inhaltsverzeichnis", "in dieser ausgabe", "rubriken", "themen",
)
# "The future of batteries ....... 18"  /  "The future of batteries   18"
_TOC_ENTRY = re.compile(r"^(.{3,120}?)[\s.…·•\-]{2,}(\d{1,4})\s*$")

_ADVERT_MARKERS = ("advertisement", "anzeige", "sponsored", "promotion")


@dataclass
class Block:
    """One laid-out region of a page, in PDF user-space coordinates.

    bbox is [x, y, width, height] with the origin at the page's top-left, which
    is what the reflowed view needs to draw a highlight over the rendered page.
    """
    id: str
    page: int
    bbox: list[float]
    type: str
    text: str
    reading_order: int = 0
    confidence: float = 1.0
    font_size: float = 0.0
    font_name: str = ""
    bold: bool = False
    column: int = 0
    region_label: str = ""
    continues_from_previous_page: bool = False
    source: str = "native"          # "native" | "ocr" | "vision_ocr" | "vision_label"
    # The author line's text before an author line that ran into the first sentence was split
    # (clean_up_regions). Part of the model so every path that rebuilds blocks keeps it -- and does
    # not crash on it (drawing a region did, "unexpected keyword argument 'original_text'").
    original_text: str = ""
    # The text as recognised, kept when the reader first edits it by hand (set_region_text), so the
    # review can show what changed and put it back. Empty = never edited.
    ocr_text: str = ""
    # box_manual: the reviewer placed or resized this box, so snap_text_boxes never moves it.
    # box_snapped: snap_text_boxes fitted it to its text. Both are model fields for the same
    # reason as original_text: every path that rebuilds blocks with Block(**b) must accept them.
    box_manual: bool = False
    box_snapped: bool = False

    def to_json(self) -> dict[str, Any]:
        data = asdict(self)
        for optional in ("original_text", "ocr_text", "box_manual", "box_snapped"):
            if not data[optional]:
                del data[optional]
        return data


@dataclass
class Article:
    id: str
    title: str
    author: str = ""
    pages: list[int] = field(default_factory=list)
    blocks: list[str] = field(default_factory=list)
    images: list[str] = field(default_factory=list)
    status: str = "detected"        # detected | approved | rejected
    confidence: float = 0.0
    section: str = ""
    continued_to: list[int] = field(default_factory=list)
    continued_from: list[int] = field(default_factory=list)   # 'continued from page N' seen on this article's pages
    audio_status: str = "none"      # none | narration | podcast | both

    def to_json(self) -> dict[str, Any]:
        return asdict(self)


# ── Geometry helpers (pure) ──────────────────────────────────────────────────

def bbox_overlaps(a: Sequence[float], b: Sequence[float], tol: float = 0.0) -> bool:
    """True when two [x, y, w, h] boxes intersect, allowing a tolerance."""
    ax, ay, aw, ah = a
    bx, by, bw, bh = b
    return not (
        ax + aw + tol < bx or bx + bw + tol < ax
        or ay + ah + tol < by or by + bh + tol < ay
    )


def detect_columns(blocks: Sequence[Block], page_width: float,
                   min_gap_ratio: float = 0.015) -> list[tuple[float, float]]:
    """Infer the column grid from where blocks are *absent*.

    Magazines vary the column count per page, so a fixed grid is wrong.  The
    reliable signal is vertical whitespace running the height of the page: we
    project every block onto the x axis, then split wherever the uncovered gap
    is wider than a few percent of the page.  Returns (left, right) spans.

    1.5%, not the rounder 3.5% this started at: measured on a real scanned
    c't magazine page, the actual gutter between two genuine columns was
    only ~2.1% of the page width -- narrower than the old threshold, which
    made detect_columns see one merged column and, combined with a wide
    caption bridging the same gap (see assign_reading_order), produced
    every block on the page landing in column=0.
    """
    spans = sorted(
        ((b.bbox[0], b.bbox[0] + b.bbox[2]) for b in blocks if b.bbox[2] > 1),
        key=lambda s: s[0],
    )
    if not spans:
        return [(0.0, page_width)]

    merged: list[list[float]] = [list(spans[0])]
    min_gap = page_width * min_gap_ratio
    for left, right in spans[1:]:
        if left - merged[-1][1] > min_gap:
            merged.append([left, right])
        else:
            merged[-1][1] = max(merged[-1][1], right)
    return [(m[0], m[1]) for m in merged]


# Blocks that are not running text: they never define a column or split the page.
_NOT_TEXT_FLOW = (BLOCK_FURNITURE, BLOCK_AD, BLOCK_FIGURE, BLOCK_CHART, BLOCK_TABLE, BLOCK_CODE)

_QUOTE_PAIRS = (("»", "«"), ("„", "“"), ("“", "”"), ("«", "»"), ('"', '"'), ("‚", "‘"))


def is_pullquote_like(block: Block) -> bool:
    """A short display quote that repeats the article: read after the page's running text.

    Types are not trusted on their own: on a real scanned page the layout model typed the actual
    pull-quote ("»Wenn man hundertmal die KI nutzt…«") as body text, and typed a left-column body
    paragraph as a pull-quote. A pull-quote is short and wrapped in quotation marks; a long block
    is running text whatever it was labelled.
    """
    text = " ".join((block.text or "").split())
    if not text or len(text) > 260:
        return False
    if block.type == BLOCK_PULLQUOTE:
        return True
    return any(text.startswith(open_) and text.rstrip(".!? ").endswith(close) for open_, close in _QUOTE_PAIRS)


def column_anchors(blocks: Sequence[Block], page_width: float) -> tuple[list[float], float]:
    """Where the text columns begin, and the typical column width.

    Found from the LEFT EDGES of column-width blocks, which cluster tightly, rather than from
    whitespace between columns. The whitespace signal failed on a real scanned c't page: the OCR
    boxes were loose enough that the gutter shrank to 4-6 points (under any sane threshold), and a
    pull-quote laid across the gutter bridged the columns outright -- every block came back as
    column 0 and the two columns were read interleaved. Left edges were unaffected: 25-29 and
    236-240. Columns must be most of a column width apart (the heavier of two close candidates
    wins), so a narrow stray box starting mid-column does not invent a column of its own. The
    weight floor is deliberately low: on page 193 the left column is three paragraphs under a
    screenshot, 19% of the right column's text, and a 25% floor dropped it and interleaved the page.
    Measured over a whole 204-page issue, 5-15% all score the same.
    """
    flow = [b for b in blocks if b.bbox[2] > 1 and b.type not in _NOT_TEXT_FLOW]
    if not flow:
        return [0.0], page_width
    # The column width comes from running text only. Measured over every block, a page-wide photo
    # and the headline pushed it to 317 points against real columns of ~210 (page 88), which made
    # a plain right-column paragraph look as if it spanned two columns.
    body = [b for b in flow if b.type == BLOCK_BODY] or flow
    widths = sorted(b.bbox[2] for b in body)
    col_w = widths[len(widths) // 2]
    candidates = sorted((b for b in flow if 0.55 * col_w <= b.bbox[2] <= 1.4 * col_w), key=lambda b: b.bbox[0])
    tol = page_width * 0.04
    clusters: list[list[Block]] = []
    for b in candidates:
        if clusters and b.bbox[0] - clusters[-1][-1].bbox[0] <= tol:
            clusters[-1].append(b)
        else:
            clusters.append([b])
    if not clusters:
        return [min(b.bbox[0] for b in flow)], col_w
    weighted = [(min(b.bbox[0] for b in c), sum(b.bbox[3] for b in c)) for c in clusters]
    heaviest = max(w for _, w in weighted)
    anchors: list[tuple[float, float]] = []
    for x, weight in weighted:
        if weight < 0.10 * heaviest:
            continue
        if anchors and x - anchors[-1][0] < 0.6 * col_w:
            if weight > anchors[-1][1]:
                anchors[-1] = (x, weight)
            continue
        anchors.append((x, weight))
    return [x for x, _ in anchors] or [weighted[0][0]], col_w


def assign_reading_order(blocks: list[Block], page_width: float) -> list[Block]:
    """Order blocks the way a person reads a magazine page.

    1. Text spanning two or more columns (headline, standfirst) is read in turn from the top,
       and splits the page into bands: everything under it comes after it.
    2. Within a band, each column top to bottom, columns left to right. Sorting purely by y
       interleaves parallel columns and splices one column's sentence into the next.
    3. Short pull-quotes after the page's running text (they repeat the article, so reading
       one mid-column interrupts a sentence), then furniture (running heads, folios).

    Captions never split the page into bands: a caption under a photo in the middle of a page
    would otherwise tear both columns in half.
    """
    anchors, col_w = column_anchors(blocks, page_width)
    # Columns never overlap: each ends where the next begins.
    spans = [(a, min(a + col_w, anchors[i + 1]) if i + 1 < len(anchors) else a + col_w)
             for i, a in enumerate(anchors)]

    def overlap(b: Block, span: tuple[float, float]) -> float:
        return max(0.0, min(b.bbox[0] + b.bbox[2], span[1]) - max(b.bbox[0], span[0]))

    def column_of(b: Block) -> int:
        best = max(range(len(spans)), key=lambda i: (overlap(b, spans[i]), -i))
        if overlap(b, spans[best]) > 0:
            return best
        # Outside every column (a margin note): the nearest column by left edge.
        return min(range(len(anchors)), key=lambda i: abs(b.bbox[0] - anchors[i]))

    def crosses_columns(b: Block) -> bool:
        if len(spans) < 2:
            return b.bbox[2] > page_width * 0.6
        return sum(1 for s in spans if overlap(b, s) >= 0.3 * col_w) >= 2

    def own_strip(b: Block) -> bool:
        """A real headline sits in a horizontal strip of its own. A wide box lying over the column
        boxes (a duplicate region covering the same text, seen on pages 78 and 154) does not."""
        top, bottom = b.bbox[1], b.bbox[1] + b.bbox[3]
        for other in blocks:
            if other is b or other.type in _NOT_TEXT_FLOW or crosses_columns(other):
                continue
            shared = min(bottom, other.bbox[1] + other.bbox[3]) - max(top, other.bbox[1])
            if shared > 0.3 * max(1.0, min(b.bbox[3], other.bbox[3])):
                return False
        return True

    def splits_page(b: Block) -> bool:
        """Text across two or more columns, in a strip of its own: read before what lies under it.
        Captions never split the page -- one in the middle would tear both columns."""
        if b.type in _NOT_TEXT_FLOW or b.type in (BLOCK_CAPTION, BLOCK_PULLQUOTE) or is_pullquote_like(b):
            return False
        return crosses_columns(b) and own_strip(b)

    def photo_splits_page(b: Block) -> bool:
        """A photo with no running text beside it separates the text above from the text below.

        Page 176: a caption beside a screenshot at the top, the two columns under it. Without this
        the caption was read between the columns and split "…zumal sie auch | helfen könnte…". A
        photo inset beside column text (page 164) does not split anything. Paragraph gaps are never
        used for this: in neighbouring columns they often line up, and treating them as breaks
        interleaves the columns again.
        """
        if b.type not in _NOT_TEXT_FLOW or b.type in (BLOCK_FURNITURE, BLOCK_AD):
            return False
        top, bottom = b.bbox[1], b.bbox[1] + b.bbox[3]
        for other in blocks:
            if other.type in _NOT_TEXT_FLOW or is_pullquote_like(other):
                continue
            if min(bottom, other.bbox[1] + other.bbox[3]) - max(top, other.bbox[1]) > 2.0:
                return False
        return True

    for b in blocks:
        b.column = column_of(b)

    breakers = {id(b) for b in blocks if splits_page(b) or photo_splits_page(b)}
    band_tops = sorted(b.bbox[1] for b in blocks if id(b) in breakers)
    spanning = lambda b: id(b) in breakers  # noqa: E731

    def key(b: Block) -> tuple:
        if b.type == BLOCK_FURNITURE:
            return (3, 0, 0, round(b.bbox[1], 1), b.bbox[0])
        if is_pullquote_like(b):
            return (2, 0, b.column, round(b.bbox[1], 1), b.bbox[0])
        # The band is the number of breakers starting at or above this block: a breaker opens its
        # own band, so it follows the columns above it and precedes those below.
        band = sum(1 for top in band_tops if top <= b.bbox[1] + 0.5)
        if spanning(b):
            return (1, band, 0, round(b.bbox[1], 1), b.bbox[0])
        return (1, band, 1 + b.column, round(b.bbox[1], 1), b.bbox[0])

    ordered = sorted(blocks, key=key)
    for i, b in enumerate(ordered):
        b.reading_order = i
    return ordered


# ── Classification (pure) ────────────────────────────────────────────────────

def body_font_size(blocks: Iterable[Block]) -> float:
    """The document's body size: the char-weighted median of block sizes.

    Weighting by length stops a page of huge display type from redefining what
    "normal" means -- body copy always dominates by character count.
    """
    sizes: list[float] = []
    for b in blocks:
        if b.font_size > 0 and b.text.strip():
            sizes.extend([b.font_size] * max(1, len(b.text) // 20))
    return statistics.median(sizes) if sizes else 10.0


def _normalise(text: str) -> str:
    text = unicodedata.normalize("NFKC", text or "")
    return re.sub(r"\s+", " ", text).strip()


def looks_like_byline(text: str) -> tuple[bool, str]:
    """Detect 'By Jane Smith' / 'Von Jane Smith', returning the name."""
    m = _BYLINE.match(_normalise(text))
    if not m:
        return False, ""
    name = m.group(1).strip(" .,;:")
    # A byline is a name, not a sentence: reject anything long or punctuated.
    if len(name) > 60 or name.count(" ") > 5 or name.endswith((".", "!", "?")):
        return False, ""
    return True, name


def find_continuation(text: str) -> tuple[int | None, int | None]:
    """Return (continues_on_page, continued_from_page) mentioned in the text."""
    to_m = _CONT_TO.search(text or "")
    from_m = _CONT_FROM.search(text or "")
    to_page = next((int(g) for g in to_m.groups() if g), None) if to_m else None
    from_page = next((int(g) for g in from_m.groups() if g), None) if from_m else None
    return to_page, from_page


def strip_continuation_markers(text: str) -> str:
    """Remove '(continued on page N)' style cues before text is narrated.

    These are a print layout convention -- meaningless, and slightly absurd,
    spoken aloud ("open paren continued on page eight close paren"). The
    layout store keeps the original text; only the narration copy is cleaned.
    """
    cleaned = _CONT_TO.sub("", text or "")
    cleaned = _CONT_FROM.sub("", cleaned)
    return re.sub(r"\s+", " ", cleaned).strip()


def classify_blocks(blocks: list[Block], page_height: float, page_width: float,
                    repeated: set[str] | None = None,
                    image_boxes: Sequence[Sequence[float]] = (),
                    base_size: float | None = None) -> list[Block]:
    """Assign a semantic type to every block on one page.

    Font size is judged relative to the document body size rather than in
    absolute points, so the same rules hold for a pocket magazine and a
    broadsheet.

    base_size must be measured across the whole document, not this page: a page
    carrying a full-bleed photograph and one caption has no body copy of its
    own, so a per-page baseline would make that caption its own reference and
    classify it as body text.  It falls back to the page when not supplied.
    """
    repeated = repeated or set()
    base = base_size if base_size and base_size > 0 else body_font_size(blocks)

    for b in blocks:
        # A vision OCR engine that explicitly labeled this region (footer,
        # title, page_number, caption -- see build_vision_blocks) is a more
        # direct signal than any font-size heuristic can reconstruct after
        # the fact; second-guessing it here would throw that away.
        if b.source == "vision_label":
            continue

        text = _normalise(b.text)
        if not text:
            b.type = BLOCK_FURNITURE
            b.confidence = 0.5
            continue

        ratio = (b.font_size / base) if base else 1.0
        near_top = b.bbox[1] < page_height * 0.07
        near_bottom = b.bbox[1] + b.bbox[3] > page_height * 0.93
        short = len(text) <= 90
        lower = text.lower()

        # Running heads, folios and page numbers repeat at the same edge on
        # many pages -- that repetition is what identifies them, not position
        # alone, since a headline can also sit at the top.
        if (near_top or near_bottom) and (text in repeated or text.isdigit()):
            b.type, b.confidence = BLOCK_FURNITURE, 0.95
            continue

        if any(m in lower for m in _ADVERT_MARKERS) and short:
            b.type, b.confidence = BLOCK_AD, 0.8
            continue

        is_byline, _ = looks_like_byline(text)
        if is_byline:
            b.type, b.confidence = BLOCK_BYLINE, 0.9
            continue

        # A caption sits directly beneath or beside a figure and is set smaller
        # than body copy.  Both conditions are needed: small text elsewhere is
        # usually a footnote, and body-sized text under an image is often the
        # article continuing past it.
        if ratio < 0.92 and any(
            bbox_overlaps(b.bbox, box, tol=max(page_height, page_width) * 0.02)
            for box in image_boxes
        ):
            b.type, b.confidence = BLOCK_CAPTION, 0.75
            continue

        if ratio >= 1.6 and short:
            b.type, b.confidence = BLOCK_TITLE, 0.9 if ratio >= 2.0 else 0.7
            continue

        if 1.18 <= ratio < 1.6 and len(text) <= 300:
            # Display-sized text mid-column is a pull quote; at the head of a
            # page it is the deck introducing the article.
            b.type = BLOCK_DECK if b.bbox[1] < page_height * 0.45 else BLOCK_PULLQUOTE
            b.confidence = 0.6
            continue

        b.type, b.confidence = BLOCK_BODY, 0.85

    return blocks


def mark_advert_pages(blocks: list[Block]) -> list[Block]:
    """Spread an explicit advertisement label across its whole page.

    Publishers print "ADVERTISEMENT" once to disclaim the page, not each block
    on it.  Without this the ad's own display type reads as a headline and the
    page gets claimed by whichever feature precedes it.
    """
    # A small ad label beside substantial prose does not establish a full-page ad.
    if any(b.type == BLOCK_BODY and len(b.text) > 160 for b in blocks):
        return blocks
    if not any(b.type == BLOCK_AD and b.source not in ("vision_label", "llm_ad", "llm_layout") for b in blocks):
        return blocks
    for b in blocks:
        if b.type != BLOCK_FURNITURE:
            b.type, b.confidence = BLOCK_AD, 0.8
    return blocks


def mark_pullquotes(blocks: list[Block]) -> list[Block]:
    """Demote display text that merely repeats nearby body copy.

    Narrating a pull quote reads the same sentence twice, a few seconds apart,
    which sounds like a skipping record.
    """
    body_text = _normalise(" ".join(
        b.text for b in blocks if b.type == BLOCK_BODY
    )).lower()
    if not body_text:
        return blocks
    for b in blocks:
        if b.type not in (BLOCK_DECK, BLOCK_PULLQUOTE):
            continue
        probe = _normalise(b.text).lower().strip(' "“”«»')
        if len(probe) > 25 and probe in body_text:
            b.type, b.confidence = BLOCK_PULLQUOTE, 0.9
    return blocks


def mark_sidebars(blocks: list[Block], page_width: float,
                  narrow_ratio: float = 0.7) -> list[Block]:
    """Reclassify body text sitting in an anomalously narrow column as a
    sidebar, so it is excluded from an article's narration by default rather
    than requiring a manual exclusion in review every time.

    Geometry-only, like the rest of this module's classification: a sidebar
    or pull-out box is reliably narrower than the page's regular column grid
    (a boxed insert set off from the main flow), which needs no image
    analysis to detect. This is a heuristic, not a certainty -- a genuinely
    narrow-column page design (or a caption sitting in its own slim column)
    can trigger a false positive, which is exactly why review still shows
    every sidebar block and lets it be included with one click rather than
    silently discarding it.

    Full-width elements (a headline spanning the whole page) are excluded
    from the column analysis here, the same way assign_reading_order treats
    them specially: left in, one would bridge two genuinely separate columns
    into a single merged span (its bbox overlaps both), hiding a real narrow
    sidebar column entirely. This computes its own column partition rather
    than trusting `.column` from assign_reading_order, since that partition
    was built from the FULL block list and its indices would not line up
    with this narrower one.
    """
    candidates = [b for b in blocks if b.bbox[2] <= page_width * 0.6]
    columns = detect_columns(candidates, page_width)
    if len(columns) < 2:
        return blocks
    widths = [right - left for left, right in columns]
    widest = max(widths)
    narrow_spans = [columns[i] for i, w in enumerate(widths) if w < widest * narrow_ratio]
    if not narrow_spans:
        return blocks
    for b in blocks:
        if b.type != BLOCK_BODY or b.bbox[2] > page_width * 0.6:
            continue
        centre = b.bbox[0] + b.bbox[2] / 2
        if any(left <= centre <= right for left, right in narrow_spans):
            b.type, b.confidence = BLOCK_SIDEBAR, 0.6
    return blocks


def find_repeated_lines(pages_text: Sequence[Sequence[str]],
                        threshold: float = 0.25) -> set[str]:
    """Lines appearing near the same edge on many pages: page furniture."""
    if not pages_text:
        return set()
    counts: dict[str, int] = {}
    for lines in pages_text:
        for line in {_normalise(t) for t in lines if _normalise(t)}:
            counts[line] = counts.get(line, 0) + 1
    minimum = max(3, int(len(pages_text) * threshold))
    # Long lines that recur are more likely a repeated standfirst than a
    # running head, so cap the length.
    return {t for t, n in counts.items() if n >= minimum and len(t) <= 80}


def looks_like_real_title(text: str) -> bool:
    """Guards the headline-fallback article start against OCR noise.

    A stray misread character or chart-axis fragment ("I", "@)X", ". |") can
    classify as BLOCK_TITLE by size alone; without this it would spawn its
    own article. Confirmed live on a scanned magazine: single-character OCR
    artifacts at running-footer size otherwise start empty articles.
    """
    letters = sum(1 for c in (text or "") if c.isalpha())
    return letters >= 4


def _digit_stripped(text: str) -> str:
    return re.sub(r"\d+", "", text or "")


def cluster_running_furniture(blocks: Sequence[Block], page_heights: dict[int, float],
                              similarity: float = 0.6, min_count: int = 3) -> None:
    """Reclassify near-edge blocks that recur across pages as furniture, even
    when OCR reads the running header/footer differently each time.

    find_repeated_lines() needs an EXACT match, which OCR defeats: a magazine
    tag like "c't Wissen 2026" came back as "cE KI-WISSEN", "ce Wissen 2136",
    "cA Wissen 2036" across different pages of the same scan -- confirmed
    live, and the dominant cause of a 28-page scan producing 26 near-1-per-
    page "articles" instead of grouping around real headlines. Only the page
    number reliably varies page to page, so digit-stripped fuzzy similarity
    catches what exact matching misses. Mutates blocks in place.
    """
    candidates: list[tuple[Block, str]] = []
    for b in blocks:
        if b.type in (BLOCK_FURNITURE, BLOCK_AD):
            continue
        text = _normalise(b.text)
        if not text or len(text) > 80:
            continue
        height = page_heights.get(b.page, 842.0)
        near_top = b.bbox[1] < height * 0.07
        near_bottom = b.bbox[1] + b.bbox[3] > height * 0.93
        if not (near_top or near_bottom):
            continue
        key = _digit_stripped(text).lower().strip()
        if len(key) < 3:
            continue
        candidates.append((b, key))

    assigned = [False] * len(candidates)
    for i, (_, key_i) in enumerate(candidates):
        if assigned[i]:
            continue
        cluster = [i]
        for j in range(i + 1, len(candidates)):
            if assigned[j]:
                continue
            if difflib.SequenceMatcher(None, key_i, candidates[j][1]).ratio() >= similarity:
                cluster.append(j)
        if len(cluster) >= min_count:
            for idx in cluster:
                candidates[idx][0].type = BLOCK_FURNITURE
                candidates[idx][0].confidence = 0.75
                assigned[idx] = True


# ── Stage A: the table of contents ───────────────────────────────────────────

def score_contents_page(lines: Sequence[str]) -> float:
    """How strongly one page looks like a table of contents (0..1)."""
    if not lines:
        return 0.0
    joined = " ".join(lines).lower()
    marker_hits = sum(1 for m in _TOC_MARKERS if m in joined)
    entry_hits = sum(1 for line in lines if _TOC_ENTRY.match(_normalise(line)))
    # Either signal alone is weak: a feature can be titled "Features", and a
    # dotted leader appears in ordinary tables too.
    score = min(1.0, marker_hits * 0.25) * 0.4 + min(1.0, entry_hits / 6) * 0.6
    return round(score, 3)


def parse_contents(lines: Sequence[str]) -> list[dict[str, Any]]:
    """Pull (title, page, section) entries out of a contents page."""
    entries: list[dict[str, Any]] = []
    section = ""
    for raw in lines:
        line = _normalise(raw)
        if not line:
            continue
        low = line.lower().strip(": ")
        if low in _TOC_MARKERS and low not in ("contents", "inhalt"):
            section = line.strip(": ")
            continue
        m = _TOC_ENTRY.match(line)
        if not m:
            continue
        title, page = m.group(1).strip(" .…·•-"), int(m.group(2))
        if len(title) < 3 or page <= 0 or page > 2000:
            continue
        entries.append({"title": title, "page": page, "section": section})
    return entries


# ── Stage C: grouping blocks into articles ───────────────────────────────────

def _title_key(text: str) -> str:
    return re.sub(r"[^a-z0-9]+", "", _normalise(text).lower())


def group_articles(blocks: Sequence[Block],
                   contents: Sequence[dict[str, Any]],
                   page_count: int,
                   skip_pages: set[int] | None = None) -> list[Article]:
    """Group ordered blocks into articles.

    The contents listing, when present, is the strongest available signal: it
    states both the title and the starting page, so it anchors boundaries that
    layout alone would guess at.  Pages not claimed by any listing still get
    articles from their own headlines, because a contents page never lists
    everything (and some issues have no contents page at all).
    """
    by_page: dict[int, list[Block]] = {}
    for b in blocks:
        by_page.setdefault(b.page, []).append(b)
    for page_blocks in by_page.values():
        page_blocks.sort(key=lambda b: b.reading_order)

    # Where does each article start?  A contents entry pins a start page; a
    # title block on a page with no entry starts one there.
    starts: dict[int, dict[str, Any]] = {}
    for entry in contents:
        page = entry["page"]
        if 1 <= page <= page_count:
            starts.setdefault(page, {
                "title": entry["title"],
                "section": entry.get("section", ""),
                "confidence": 0.9,
            })

    skip_pages = skip_pages or set()
    for page in sorted(by_page):
        if page in starts or page in skip_pages:
            continue
        # A page carrying nothing but advertising and page furniture has no
        # article on it -- the cover and the ad pages otherwise each become a
        # spurious entry the reviewer has to reject by hand on every issue.
        if not any(b.type in NARRATABLE and b.type != BLOCK_TITLE
                   for b in by_page[page]):
            continue
        page_blocks = by_page[page]
        titles = [b for b in page_blocks if b.type == BLOCK_TITLE
                 and looks_like_real_title(b.text)]
        # A large subheading, chart title or pull-out on a continuation page
        # is frequently classified as a title.  It is not a new article when
        # body copy already precedes it in reading order.  Genuine page-level
        # article leads either come first, or have a nearby byline/deck.
        if titles:
            title = titles[0]
            before = [b for b in page_blocks
                      if b.reading_order < title.reading_order and b.type == BLOCK_BODY]
            following = [b for b in page_blocks
                         if title.reading_order < b.reading_order <= title.reading_order + 3]
            has_lead_context = any(b.type in (BLOCK_BYLINE, BLOCK_DECK) for b in following)
            if before and not has_lead_context:
                titles = []
        if titles:
            starts[page] = {
                "title": _normalise(titles[0].text),
                "section": "",
                "confidence": 0.55,
            }

    if not starts:
        return []

    ordered_starts = sorted(starts)
    articles: list[Article] = []
    for i, start_page in enumerate(ordered_starts):
        end_page = (ordered_starts[i + 1] - 1) if i + 1 < len(ordered_starts) else page_count
        info = starts[start_page]
        art = Article(
            id=f"article-{i + 1:03d}",
            title=info["title"],
            section=info.get("section", ""),
            confidence=info["confidence"],
        )
        for page in range(start_page, end_page + 1):
            page_blocks = by_page.get(page, [])
            if not page_blocks:
                continue
            # Only claim a page the article actually occupies.  A full-page
            # advert sitting between two features starts no article of its
            # own, so it would otherwise be swallowed by the preceding one and
            # highlighted as part of it in the PDF view.
            #
            # A sidebar/pull-out box (mark_sidebars) is excluded the same way
            # advert text is: automatically kept out of narration by default,
            # rather than requiring a manual exclusion in review every time.
            # It still shows in review (blocksForPages lists every block on
            # the article's pages) and can be ticked back in with one click.
            contributed = False
            for b in page_blocks:
                if b.type in (BLOCK_FURNITURE, BLOCK_AD, BLOCK_SIDEBAR, BLOCK_PULLQUOTE, *VISUAL_TYPES):
                    continue
                contributed = True
                art.blocks.append(b.id)
                if b.type == BLOCK_BYLINE and not art.author:
                    _, name = looks_like_byline(b.text)
                    art.author = name
                to_page, _ = find_continuation(b.text)
                if to_page and to_page not in art.continued_to:
                    art.continued_to.append(to_page)
            # "Continued from page N" is usually set as small print the classifier files as furniture or a
            # sidebar, so look at every short line on the page, not only at what is narrated. A long block that
            # merely mentions the phrase is prose, not a jump marker.
            for b in page_blocks:
                if len(b.text or "") > _MAX_JUMP_MARKER_CHARS:
                    continue
                _, from_page = find_continuation(b.text)
                if from_page and from_page != page and from_page not in art.continued_from:
                    art.continued_from.append(from_page)
            if contributed:
                art.pages.append(page)

        # An article the pipeline could not fill with body copy is almost
        # certainly a false start (an advert headline, a section divider).
        has_body = any(
            b.type == BLOCK_BODY for b in blocks if b.id in set(art.blocks)
        )
        if not has_body:
            art.confidence = round(art.confidence * 0.4, 3)
        articles.append(art)

    return _link_continuations(articles, page_count)


_MAX_JUMP_MARKER_CHARS = 160   # a "continued from" line is a few words; longer text is prose that quotes the phrase


def _absorb(head: Article, tail: Article) -> None:
    head.pages.extend(p for p in tail.pages if p not in head.pages)
    head.blocks.extend(tail.blocks)
    head.images.extend(tail.images)
    tail.status = "merged"


def _link_continuations(articles: list[Article], page_count: int) -> list[Article]:
    """Attach jump pages to their originating article, in both directions.

    Forward: the head says "continued on page 88" -- the article that starts on page 88 is its tail.
    Backward: the tail says "continued from page 27" -- the article that occupies page 27 is its head.
    Without this, the tail of a feature parked at the back of the issue is orphaned and narrated as a
    separate stub that begins mid-sentence. A forward link wins when both exist (it is processed first,
    and a merged article is never absorbed twice). An article a contents entry anchors (confidence >= 0.9)
    keeps its own identity either way.
    """
    by_start = {a.pages[0]: a for a in articles if a.pages}
    for art in articles:
        for target in art.continued_to:
            if not (1 <= target <= page_count):
                continue
            tail = by_start.get(target)
            if tail is None or tail is art or tail.status == "merged":
                continue
            if tail.confidence >= 0.9:
                continue
            _absorb(art, tail)
    for tail in articles:
        if tail.status == "merged" or tail.confidence >= 0.9 or not tail.pages:
            continue
        for source in tail.continued_from:
            if not (1 <= source <= page_count) or source >= tail.pages[0]:
                continue        # a tail can only follow a page before it
            heads = [a for a in articles if a is not tail and a.status != "merged" and source in a.pages]
            if len(heads) != 1:
                continue        # no article there, or ambiguous (a page shared by several): leave it to the reviewer
            _absorb(heads[0], tail)
            break
    return [a for a in articles if a.status != "merged"]


def estimate_listen_seconds(blocks: Sequence[Block], article: Article,
                            words_per_minute: float = 155.0) -> int:
    """Rough narration length, for the 'estimated listening time' column."""
    ids = set(article.blocks)
    words = sum(
        len(b.text.split()) for b in blocks
        if b.id in ids and b.type in NARRATABLE
    )
    return int(round(words / words_per_minute * 60)) if words else 0


# ── PDF and OCR I/O ──────────────────────────────────────────────────────────

def _require_fitz():
    """Import PyMuPDF under its current name, falling back to the old alias.

    1.24 renamed the module to ``pymupdf``; ``fitz`` still resolves but emits a
    deprecation warning and is slated for removal.
    """
    try:
        import pymupdf
        return pymupdf
    except ImportError:
        pass
    try:
        import fitz
        return fitz
    except ImportError as exc:  # pragma: no cover - environment guard
        raise RuntimeError(
            "PyMuPDF is required for magazine ingestion; install pymupdf (pip install pymupdf) "
            "or rebuild your Docker container."
        ) from exc


# Labels a vision OCR engine may return for a region, mapped onto the same
# taxonomy tesseract/native blocks use. Anything else ("text", or a label the
# engine invents) is treated as ambiguous body copy and still goes through
# classify_blocks() for finer distinctions (byline, deck, ...).
_VISION_LABEL_TYPES = {
    # Headlines / Titles / Headers
    "title": BLOCK_TITLE,
    "headline": BLOCK_TITLE,
    "article_headline": BLOCK_TITLE,
    "main_title": BLOCK_TITLE,
    "heading": BLOCK_HEADING,
    "header": BLOCK_HEADING,
    "section_heading": BLOCK_HEADING,
    "subheading": BLOCK_HEADING,
    "subtitle": BLOCK_HEADING,
    "deck": BLOCK_HEADING,

    # Paragraphs / Text / Body
    "paragraph": BLOCK_BODY,
    "body": BLOCK_BODY,
    "content": BLOCK_BODY,
    "list": BLOCK_BODY,
    "list_item": BLOCK_BODY,
    "author": BLOCK_BYLINE,
    "byline": BLOCK_BYLINE,

    # Sidenotes / Sidebars / Quotes / Footnotes
    "sidenote": BLOCK_SIDEBAR,
    "sidebar": BLOCK_SIDEBAR,
    "aside_text": BLOCK_SIDEBAR,
    "aside": BLOCK_SIDEBAR,
    "footnote": BLOCK_SIDEBAR,
    "footnotes": BLOCK_SIDEBAR,
    "note": BLOCK_SIDEBAR,
    "commentary": BLOCK_SIDEBAR,
    "quote": BLOCK_QUOTE,
    "blockquote": BLOCK_QUOTE,
    "pullquote": BLOCK_PULLQUOTE,

    # Page numbers / Furniture / Folios / Sources
    "page_number": BLOCK_FURNITURE,
    "pagenumber": BLOCK_FURNITURE,
    "page": BLOCK_FURNITURE,
    "header_text": BLOCK_FURNITURE,
    "footer": BLOCK_FURNITURE,
    "running_header": BLOCK_FURNITURE,
    "running_footer": BLOCK_FURNITURE,
    "folio": BLOCK_FURNITURE,
    "head": BLOCK_FURNITURE,
    "source": BLOCK_FURNITURE,
    "sources": BLOCK_FURNITURE,
    "citation": BLOCK_FURNITURE,
    "citations": BLOCK_FURNITURE,

    # Images / Figures / Photos
    "image": BLOCK_FIGURE,
    "figure": BLOCK_FIGURE,
    "picture": BLOCK_FIGURE,
    "photo": BLOCK_FIGURE,
    "illustration": BLOCK_FIGURE,
    "non_text": BLOCK_FIGURE,
    "nontext": BLOCK_FIGURE,
    "drawing": BLOCK_FIGURE,

    # Graphics / Charts / Diagrams / Maps / Treemaps
    "chart": BLOCK_CHART,
    "graphic": BLOCK_CHART,
    "infographic": BLOCK_CHART,
    "statistics": BLOCK_CHART,
    "diagram": BLOCK_CHART,
    "map": BLOCK_CHART,
    "treemap": BLOCK_CHART,
    "plot": BLOCK_CHART,
    "flowchart": BLOCK_CHART,

    # Tables
    "table": BLOCK_TABLE,
    "grid": BLOCK_TABLE,
    "matrix": BLOCK_TABLE,

    # Code / Formulas / Math
    "code": BLOCK_CODE,
    "code_block": BLOCK_CODE,
    "formula": BLOCK_CODE,
    "equation": BLOCK_CODE,
    "math": BLOCK_CODE,

    # Captions
    "caption": BLOCK_CAPTION,
    "image_caption": BLOCK_CAPTION,
    "figure_caption": BLOCK_CAPTION,
    "table_caption": BLOCK_CAPTION,

    # Ads
    "advertisement": BLOCK_AD,
    "advert": BLOCK_AD,
    "ad": BLOCK_AD,
}

# One label + its four-number bbox, anchoring a region; content runs until
# the next such marker. Tolerates DeepSeek-OCR / Unlimited-OCR / PaddleOCR shapes
# like "text [241, 90, 588, 249]", "<|ref|>title<|/ref|><|det|>[[x0, y0, x1, y1]]<|/det|>", etc.
_VISION_REGION = re.compile(
    r"(?m)"
    r"(?:<\|ref\|>\s*(?P<ref>[^<]+?)\s*<\|/ref\|>\s*)?"
    r"(?:<\|det\|>\s*)?"
    r"(?P<label>[a-zA-Z0-9_\- ]+?)\s*"
    r"\[\[?\s*(?P<x0>-?\d+(?:\.\d+)?)\s*,\s*(?P<y0>-?\d+(?:\.\d+)?)\s*,\s*(?P<x1>-?\d+(?:\.\d+)?)\s*,\s*(?P<y1>-?\d+(?:\.\d+)?)\s*\]\]?"
    r"(?:\s*<\|/det\|>)?"
    r"\s*:?"
)

_UNLIMITED_OCR_REGION = re.compile(
    r"(?m)"
    r"(?:<\|ref\|>\s*(?P<ref>[^<]+?)\s*<\|/ref\|>\s*)?"
    r"<\|det\|>\s*"
    r"(?P<label>[a-zA-Z0-9_\- ]+?)\s*"
    r"(?:\[\[?\s*(?P<x0>-?\d+(?:\.\d+)?)\s*,\s*(?P<y0>-?\d+(?:\.\d+)?)\s*,\s*(?P<x1>-?\d+(?:\.\d+)?)\s*,\s*(?P<y1>-?\d+(?:\.\d+)?)\s*\]\]?)?"
    r"\s*<\|/det\|>"
)


def ocr_quality_score(text: str) -> float:
    """Rough plausibility that OCR/extracted text is real prose, 0..1."""
    text = (text or "").strip()
    if not text:
        return 0.0
    words = text.split()
    if not words:
        return 0.0

    def _stripped(w: str) -> str:
        return w.strip(".,;:!?()[]{}\"'|-–—")

    garbage = sum(1 for w in words if not _stripped(w))
    letters = sum(1 for c in text if c.isalpha())
    total = len(text.replace(" ", "")) or 1
    score = (letters / total) * (1.0 - garbage / len(words))
    return round(max(0.0, min(1.0, score)), 3)


def estimate_font_size_from_area(width: float, height: float, text_length: int,
                                 char_width_ratio: float = 0.5,
                                 line_height_ratio: float = 1.2) -> float:
    """Estimate a representative font size for a paragraph-level box."""
    if width <= 0 or height <= 0 or text_length <= 0:
        return 0.0
    denom = text_length * char_width_ratio * line_height_ratio
    return (width * height / denom) ** 0.5 if denom > 0 else 0.0


# A vision engine asked to transcribe a photo or screenshot it cannot read
# sometimes answers with a description of its own failure instead of leaving
# the region empty -- confirmed live on c't KI Wissen 2026 pages 82, 87 and
# 124, where a terminal screenshot and two photos each came back tiled into
# dozens of tiny regions, all carrying one of these two sentences verbatim.
# Downstream this reads as a wall of near-duplicate narration text laid over
# an image, which _whole_page_region's repetition check does not catch
# because each *region* carries the sentence once -- the loop is across
# regions, not within one string. Matching the wording is a second signal
# on top of exact repetition (see _drop_repeated_region_text below); either
# alone risks a false positive, together they do not.
_VISION_REFUSAL_PHRASES = (
    "too blurry to recognize", "too blurry to read", "cannot recognize the text",
    "cannot read the text", "unable to recognize the text", "unable to read the text",
    "ground truth image", "the image is too", "not clear enough to",
)


def _looks_like_vision_refusal(text: str) -> bool:
    lowered = (text or "").strip().lower()
    return len(lowered) >= 15 and any(phrase in lowered for phrase in _VISION_REFUSAL_PHRASES)


def _drop_repeated_region_text(regions: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """Blank out text that is repeated verbatim across several regions and
    reads like the engine describing its own failure rather than the page.

    Legitimate page text essentially never repeats a whole sentence across
    three or more distinct boxes; a hallucinated refusal, tiled once per
    fragment of an unreadable image, does exactly that. Requiring both the
    repetition and the refusal wording (see _looks_like_vision_refusal)
    keeps a genuinely repeated short phrase, such as a running header,
    from being caught by this.
    """
    counts: dict[str, int] = {}
    for region in regions:
        key = (region.get("text") or "").strip().lower()
        if key:
            counts[key] = counts.get(key, 0) + 1
    noisy = {text for text, count in counts.items() if count >= 3 and _looks_like_vision_refusal(text)}
    if not noisy:
        return regions
    logging.getLogger(__name__).warning(
        "Vision OCR repeated a refusal sentence across %d regions; discarding its text",
        sum(counts[t] for t in noisy),
    )
    out = []
    for region in regions:
        if (region.get("text") or "").strip().lower() in noisy:
            region = dict(region)
            region["text"] = ""
        out.append(region)
    return out


def parse_vision_ocr_regions(raw: str) -> list[dict[str, Any]]:
    """Parse a vision OCR engine's labeled-region transcription."""
    matches = list(_VISION_REGION.finditer(raw or ""))
    candidates = []
    for i, m in enumerate(matches):
        gd = m.groupdict()
        raw_label = (gd.get("ref") or gd.get("label") or "text").strip().strip("[]")
        label = raw_label.lower().replace("-", "_").replace(" ", "_")
        try:
            x0, y0, x1, y1 = (float(gd[k]) for k in ("x0", "y0", "x1", "y1"))
        except (ValueError, TypeError):
            continue
        if x1 <= x0 or y1 <= y0:
            continue
        start = m.end()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(raw or "")
        text = (raw or "")[start:end].strip()
        candidates.append({"label": label, "bbox": [x0, y0, x1 - x0, y1 - y0], "text": text})
    regions = []
    for region in _drop_repeated_region_text(candidates):
        if not region["text"] and _VISION_LABEL_TYPES.get(region["label"]) not in VISUAL_TYPES:
            continue
        regions.append(region)
    return regions


def parse_unlimited_ocr_regions(raw: str) -> list[dict[str, Any]]:
    """Parse Unlimited-OCR's native ``<|det|>`` document-layout markers."""
    matches = list(_UNLIMITED_OCR_REGION.finditer(raw or ""))
    candidates = []
    for i, match in enumerate(matches):
        gd = match.groupdict()
        raw_label = (gd.get("ref") or gd.get("label") or "").strip().strip("[]")
        label = raw_label.lower().replace("-", "_").replace(" ", "_")
        try:
            x0, y0, x1, y1 = (float(gd[k]) for k in ("x0", "y0", "x1", "y1"))
        except (ValueError, TypeError):
            continue
        if x1 <= x0 or y1 <= y0:
            continue
        start = match.end()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(raw or "")
        text = (raw or "")[start:end].strip()
        if label in {"non_text", "nontext", "photo", "drawing", "picture"}:
            label = "image"
        elif label in {"map", "graphic", "infographic", "diagram", "treemap", "statistics"}:
            label = "chart"
        candidates.append({"label": label, "bbox": [x0, y0, x1 - x0, y1 - y0], "text": text,
                           "native_normalized_bbox": True})
    regions = []
    for region in _drop_repeated_region_text(candidates):
        if not region["text"] and _VISION_LABEL_TYPES.get(region["label"]) not in VISUAL_TYPES:
            continue
        regions.append(region)
    return regions


def _whole_page_region(content: str, png_path: Path) -> list[dict[str, Any]] | None:
    """Fallback for an engine that answers with plain text and no coordinate
    markers at all (confirmed live: KDL Frontier Parser nano -- fast, but no
    "label [x0,y0,x1,y1]" structure the way ocr-baidu-unlimited-ocr gives).

    Treated as one ambiguous "text" region spanning the whole page: real
    text with no layout information beats no text at all, but it does mean
    column/reading-order structure is lost for that page -- classify_blocks
    still runs on it (source="vision_ocr", not "vision_label"), same as any
    other ambiguous region.
    """
    text = (content or "").strip()
    # A vision model can occasionally echo a formatting instruction instead
    # of reading the page.  The observed failure is a long sequence of literal
    # ``\\n `equation` `` tokens (not real line breaks), often prefixed with
    # "Do not use special symbols".  It scores highly under the generic
    # letter-ratio heuristic because "equation" is a normal word, so reject
    # this distinct response shape before turning it into a page-sized text
    # block.  Returning None lets vision_ocr_page_chain try the configured
    # fallback engine, then preserves the existing OCR if both fail.
    escaped_newlines = len(re.findall(r"\\n", text))
    equation_tokens = len(re.findall(r"`?\s*\\?equation\b", text, re.IGNORECASE))
    prompt_echo = "do not use special symbols" in text.lower()
    # Another observed loop starts with genuine page text and then repeats a
    # fabricated LaTex fragment (``\\therefore m = \\frac{3}{11}``) until the
    # response limit.  It has no instruction echo or `equation` token, so it
    # needs its own narrow signature.  Requiring both uncommon controls many
    # times avoids rejecting ordinary prose that merely mentions one formula.
    therefore_tokens = len(re.findall(r"\\therefore\b", text, re.IGNORECASE))
    fraction_tokens = len(re.findall(r"\\frac\s*\{", text, re.IGNORECASE))
    latex_loop = therefore_tokens >= 4 and fraction_tokens >= 4
    if (escaped_newlines >= 4 and equation_tokens >= 4) or (
        prompt_echo and escaped_newlines >= 2 and equation_tokens >= 2
    ) or latex_loop:
        logging.getLogger(__name__).warning("Vision OCR returned a repeated formatting artifact; discarding response")
        return None
    # The two signatures above are narrow matches for specific observed
    # failures; this is the general case -- any short phrase repeated many
    # times back-to-back (see vision_ocr_page's docstring on the same check).
    if _looks_like_repetition_loop(text):
        logging.getLogger(__name__).warning("Vision OCR returned a repeated phrase loop; discarding response")
        return None
    if len(text) < 20 or ocr_quality_score(text) < _VISION_QUALITY_THRESHOLD:
        return None
    try:
        from PIL import Image
        with Image.open(png_path) as im:
            width, height = im.size
    except Exception:
        return None
    return [{"label": "text", "bbox": [0.0, 0.0, float(width), float(height)], "text": text}]


# A short phrase repeating many times back-to-back, unlike the two narrow
# formatting-echo signatures above, is not specific to one prompt or one
# engine -- it is the generic decoding-loop failure any local vision model
# can fall into on a busy chart or graphic. Confirmed live: a Qwen VL engine
# transcribing a population chart produced "das Produkt, das Produkt, das
# Produkt, ..." several hundred times over, filling the whole response
# budget. Genuine text essentially never repeats an exact phrase this many
# times consecutively, so this is a safe, general trigger rather than
# another one-off signature -- but a leader-dot table of contents line
# ("......................") legitimately repeats a short unit too, which is
# why the repeated unit must contain a real word, not just punctuation.
_REPETITION_RUN = re.compile(r"(.{2,80}?)\1{7,}", re.DOTALL)
_REPETITION_WORD = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ]{2,}")


def _looks_like_repetition_loop(text: str) -> bool:
    """True when `text` degenerates into the same short phrase repeated
    immediately back-to-back many times -- see vision_ocr_page."""
    return any(_REPETITION_WORD.search(m.group(1))
              for m in _REPETITION_RUN.finditer(text or ""))


# A recogniser stuck in a loop can also emit one endless token with no spaces at all ("2.1.1.1.1.1 ..." thousands of
# characters long, seen on a real shopping-advert page). _looks_like_repetition_loop deliberately ignores a repeated unit
# without a real word, so leader dots and rules survive -- but a short unit that contains a letter or digit, repeated
# this many times back to back, is never real text: a phone number, a price, a section number or a URL do not come
# close to 30 identical repeats.
_GLUED_RUN = re.compile(r"(\S{1,6}?)\1{29,}")
_GLUED_UNIT_CONTENT = re.compile(r"[A-Za-z0-9À-ÖØ-öø-ÿ]")


def glued_repetition_share(text: str) -> float:
    """The share of `text` taken up by glued runs of a short letter/digit unit repeated 30+ times (0.0 - 1.0)."""
    text = text or ""
    if not text:
        return 0.0
    covered = sum(m.end() - m.start() for m in _GLUED_RUN.finditer(text) if _GLUED_UNIT_CONTENT.search(m.group(1)))
    return covered / len(text)


def vision_ocr_page(png_path: Path, base_url: str, model: str,
                    timeout: float = 150.0) -> list[dict[str, Any]] | None:
    """OCR one rendered page with a connected vision-capable OCR engine.

    Returns None on any failure (unreachable engine, timeout, unusable
    output) so the caller can fall back to the next engine or tesseract --
    this must never be the reason ingestion fails outright. Confirmed live
    against ocr-baidu-unlimited-ocr: markedly more accurate German text than
    tesseract on the same scanned page, and it labels running headers/
    footers and page numbers directly, which is the dominant source of the
    over-fragmentation cluster_running_furniture() otherwise has to recover
    after the fact from noisy OCR variants of the same repeating line.
    """
    try:
        image_b64 = base64.b64encode(png_path.read_bytes()).decode("ascii")
    except OSError:
        return None

    # Settings also accept a bare server address; OpenAI-compatible servers
    # expose their API under /v1. Preserve explicit proxy/API prefixes.
    base_url = base_url.strip().rstrip("/")
    if not urlsplit(base_url).path:
        base_url += "/v1"

    unlimited_ocr = "unlimited-ocr" in model.lower()
    prompt = "<|grounding|>Convert the document to markdown." if unlimited_ocr else (
        "Recognise the page layout and transcribe visible text verbatim. "
        "Return one region per headline, title, paragraph, sidenote, sidebar, "
        "image, figure, chart, graphic, map, diagram, table, code box, "
        "caption, header, footer, or page number. Use the format "
        "label [x0, y0, x1, y1]transcription with pixel coordinates in the "
        "supplied image, one region starting on each new line. Labels: "
        "title, heading, paragraph, sidenote, sidebar, image, figure, chart, "
        "graphic, map, table, code, caption, header, footer, page_number. "
        "Enclose a whole chart, diagram, map, graphic, or table in one box, "
        "not separate axis labels or cells. Preserve table rows and columns "
        "as Markdown and code indentation and line breaks. Images, figures, and "
        "charts may have empty transcription; do not invent descriptions. "
        "Do not merge neighbouring articles or include commentary."
    )
    payload = {
        "model": model,
        "messages": [{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {
                "url": f"data:image/png;base64,{image_b64}",
            }},
        ]}],
        "max_tokens": 4000,
        "temperature": 0.0,
    }
    # Unlimited-OCR's official vLLM recipe requires its per-request
    # NGramPerReqLogitsProcessor settings. The container already exposes the
    # processor, but without these fields it does nothing and the model can
    # run into a repeated chart/formula loop. Keep the vendor-specific fields
    # off generic OpenAI-compatible vision engines.
    if unlimited_ocr:
        payload.update({
            "skip_special_tokens": False,
            "ngram_size": 35,
            "window_size": 128,
            "images_config": {"image_mode": "gundam"},
        })
    try:
        resp = request_with_retries(
            "POST", f"{base_url.rstrip('/')}/chat/completions",
            json=payload, timeout=timeout, attempts=2,
        )
        resp.raise_for_status()
        content = resp.json()["choices"][0]["message"]["content"]
    except requests.exceptions.RequestException as exc:
        status = getattr(getattr(exc, "response", None), "status_code", None)
        logging.getLogger(__name__).warning("Vision OCR request failed (%s%s)",
                                           type(exc).__name__, f", HTTP {status}" if status else "")
        return None
    except (ValueError, KeyError, IndexError, TypeError):
        logging.getLogger(__name__).warning("Vision OCR returned an invalid completion response")
        return None
    if not isinstance(content, str):
        return None
    if _looks_like_repetition_loop(content):
        # Worse than no response at all: a repeated phrase baked into a
        # block's text would be narrated verbatim. Treating it as a failed
        # call gives the existing fallback chain (vision_ocr_page_chain) a
        # real shot instead.
        logging.getLogger(__name__).warning("Vision OCR returned a repeated phrase loop; discarding response")
        return None

    regions = parse_unlimited_ocr_regions(content) if unlimited_ocr else parse_vision_ocr_regions(content)
    if regions and unlimited_ocr:
        # Native markers use the model input's 0–999-ish coordinate space;
        # blocks downstream expect coordinates of the rendered PNG.
        try:
            from PIL import Image
            with Image.open(png_path) as im:
                width, height = im.size
            for region in regions:
                if region.pop("native_normalized_bbox", False):
                    x, y, w, h = region["bbox"]
                    region["bbox"] = [x / 1000 * width, y / 1000 * height,
                                      w / 1000 * width, h / 1000 * height]
        except Exception:
            return None
    return regions or _whole_page_region(content, png_path)


def vision_ocr_page_chain(png_path: Path,
                          engines: Sequence[tuple[str, str]],
                          timeout: float = 150.0) -> list[dict[str, Any]] | None:
    """Try each configured vision OCR engine in order, first success wins.

    A primary engine (labeled regions, best quality) with a faster or
    differently-shaped secondary as a real fallback -- not just tesseract --
    when the primary is unreachable or times out. Engines are (url, model)
    pairs; empty/blank entries are skipped.
    """
    for url, model in engines:
        if not (url and model):
            continue
        regions = vision_ocr_page(png_path, url, model, timeout=timeout)
        if regions:
            return regions
    return None


_LOWERCASE_REGION_START = re.compile(r"^[a-zäöüß]")
_DETACHED_DROP_CAP = re.compile(r"^([A-ZÄÖÜ])\s+([a-zäöüß]{3,})\b")
_SPLIT_KI_DROP_CAP = re.compile(r"^K\s+[Il1](?=\s|[.,;:!?])")
_SHORT_DETACHED_DROP_CAPS = (
    (re.compile(r"^A\s+m\b"), "Am"),
    (re.compile(r"^B\s+i\s+s\b"), "Bis"),
)


def _join_detached_drop_cap(text: str) -> str:
    """Join OCR fragments caused by an oversized paragraph initial.

    ``KI`` needs its own rule: OCR commonly returns the ordinary-sized I as
    ``I``, ``l`` or ``1`` after a separately detected drop-cap K. Treat that
    narrow, word-bounded shape as the acronym, while leaving ordinary phrases
    such as ``K ist`` alone. The short-word rules cover confirmed document
    cases that the deliberately conservative 3-letter remainder rule cannot.
    """
    joined = _SPLIT_KI_DROP_CAP.sub("KI", text, count=1)
    for pattern, replacement in _SHORT_DETACHED_DROP_CAPS:
        joined = pattern.sub(replacement, joined, count=1)
    return _DETACHED_DROP_CAP.sub(r"\1\2", joined, count=1)


def prefer_drop_cap_repair(original: str, candidate: str) -> str:
    """Use a local OCR reading when it demonstrably restores a missing first line.

    Layout OCR models commonly treat an oversized initial as a separate visual
    object, then begin the paragraph transcription later. The remainder may be
    a lower-case word fragment (``tert, dass`` in ``eingetrichtert, dass``) or
    a valid capitalised word after an entire omitted line (``Herrn, verwaltet``).
    A second reading is accepted only when most saved text still appears in it;
    capitalised remainders additionally require a substantial multi-word prefix.
    """
    old = " ".join((original or "").split())
    # Tesseract often emits the ordinary first-line letters *before* the
    # tall initial because their tops are higher: ``ahrelang …\nJ tert …``.
    # Put that isolated capital back at the front before comparing readings.
    lines = [" ".join(line.split()) for line in (candidate or "").splitlines() if line.strip()]
    if len(lines) >= 2 and _LOWERCASE_REGION_START.match(lines[0]):
        cap_line = re.match(r"^([A-ZÄÖÜ])\s+(.+)$", lines[1])
        if cap_line:
            lines = [cap_line.group(1) + lines[0], cap_line.group(2), *lines[2:]]
    new = " ".join(lines).lstrip("|¦[]{}() ")
    new = _join_detached_drop_cap(new)
    old_starts_lower = bool(_LOWERCASE_REGION_START.match(old))
    if not old or not new or not new[:1].isupper():
        return old
    if len(new) < max(40, int(len(old) * 0.85)) or ocr_quality_score(new) < 0.55:
        return old
    matcher = difflib.SequenceMatcher(None, old.lower(), new.lower(), autojunk=False)
    matches = matcher.get_matching_blocks()
    coverage = sum(m.size for m in matches) / max(1, len(old))
    first_substantial = next((m for m in matches if m.size >= 8 and m.a <= 20), None)
    if coverage < 0.65 or first_substantial is None or not (1 <= first_substantial.b <= 250):
        return old
    # Keep the trusted existing body verbatim and take only the missing
    # prefix from the local reading. Besides avoiding a wholesale downgrade,
    # this also prevents a crop touching the next paragraph by a few pixels
    # from appending that paragraph a second time.
    prefix = new[:first_substantial.b].rstrip()
    # A valid capitalised remainder can hide a whole omitted first line
    # ("Herrn, verwaltet ..." after a large initial E). For that shape,
    # accept only a substantial multi-word prefix. The older lowercase case
    # may legitimately restore only the tail of one split word.
    if not old_starts_lower and (len(prefix) < 8 or len(prefix.split()) < 2):
        return old
    if prefix.endswith("-"):
        prefix = prefix[:-1]       # printed line-break hyphen: eingetrich- + tert
        return prefix + old
    # Join only when the matched remainder begins inside a word in the
    # candidate. If it begins after whitespace, the missing material was a
    # complete word/line (``Skripte | schreiben``), not a split word
    # (``eingetrich|tert``).
    joins_word = (old_starts_lower and first_substantial.b > 0
                  and new[first_substantial.b - 1:first_substantial.b].isalnum()
                  and old[:1].isalnum())
    return prefix + ("" if joins_word or old[:1] in ",.;:!?" else " ") + old


def repair_drop_cap_omissions(png_path: Path, regions: list[dict[str, Any]],
                              lang: str = "deu+eng") -> int:
    """Re-read only suspicious paragraph crops with Tesseract.

    This is intentionally a narrow second opinion, not a full second OCR pass:
    normal regions cost nothing, and a replacement is made only through
    :func:`prefer_drop_cap_repair`'s overlap guard.
    """
    repaired = 0
    for region in regions:
        if region.get("label") not in {"text", "paragraph", "body"}:
            continue
        text = " ".join((region.get("text") or "").split())
        # The primary OCR can see both pieces but preserve the visual gap
        # after an oversized initial ("S prechende").  Join it here, before
        # the lower-case omission check below; previously this rule only ran
        # on the secondary Tesseract reading and missed that common result.
        joined = _join_detached_drop_cap(text)
        if joined != text:
            region["text"] = joined
            repaired += 1

    def follows_article_header(index: int, region: dict[str, Any]) -> bool:
        """Whether this body box begins just below a title/deck/byline."""
        x, y, w, _ = (float(v) for v in region.get("bbox", [0, 0, 0, 0]))
        for previous in regions[:index]:
            if previous.get("label") not in {
                    "title", "headline", "author", "byline", "deck", "standfirst"}:
                continue
            px, py, pw, ph = (float(v) for v in previous.get("bbox", [0, 0, 0, 0]))
            horizontal = max(0.0, min(x + w, px + pw) - max(x, px))
            gap = y - (py + ph)
            if horizontal >= 0.3 * max(1.0, min(w, pw)) and -10 <= gap <= 250:
                return True
        return False

    suspicious = [r for i, r in enumerate(regions)
                  if r.get("label") in {"text", "paragraph", "body"}
                  and len((r.get("text") or "").strip()) >= 80
                  and (_LOWERCASE_REGION_START.match((r.get("text") or "").strip())
                       or follows_article_header(i, r))]
    if not suspicious:
        return repaired
    try:
        from PIL import Image
        image = Image.open(png_path)
    except Exception:
        return repaired
    try:
        for region in suspicious:
            x, y, w, h = (float(v) for v in region["bbox"])
            pad = 6
            crop = image.crop((max(0, int(x) - pad), max(0, int(y) - pad),
                               min(image.width, int(math.ceil(x + w)) + pad),
                               min(image.height, int(math.ceil(y + h)) + pad)))
            with tempfile.NamedTemporaryFile(prefix="drop-cap-", suffix=".png", delete=False) as tmp:
                crop_path = Path(tmp.name)
            try:
                crop.save(crop_path)
                proc = subprocess.run(
                    ["tesseract", str(crop_path), "stdout", "-l", lang, "--psm", "6"],
                    capture_output=True, text=True, timeout=60, check=True,
                )
                replacement = prefer_drop_cap_repair(region["text"], proc.stdout)
                if replacement != " ".join(region["text"].split()):
                    region["text"] = replacement
                    repaired += 1
            except (subprocess.SubprocessError, FileNotFoundError, OSError):
                continue
            finally:
                crop_path.unlink(missing_ok=True)
    finally:
        image.close()
    return repaired


# One line per visual type -- a chart needs "what trend", a table needs
# "what's listed", not the generic "what's shown" that suits a photo. Every
# variant still asks for one or two plain sentences: this is read aloud
# between the surrounding paragraphs, not a caption someone reads at their
# own pace, so it has to stay short regardless of how much detail the image
# could support.
_DESCRIBE_PROMPTS = {
    BLOCK_CHART: (
        "This image is a chart or graph from a magazine article. In one or "
        "two plain sentences, for a listener who cannot see it, describe "
        "what kind of chart it is, what it measures, and the general trend "
        "or most notable values. Do not invent a number you cannot actually "
        "read in the image. No markdown, no preamble -- the description "
        "text only."
    ),
    BLOCK_TABLE: (
        "This image is a table from a magazine article. In one or two "
        "plain sentences, for a listener who cannot see it, summarise what "
        "it lists -- do not read out individual cells. No markdown, no "
        "preamble -- the description text only."
    ),
    BLOCK_CODE: (
        "This image is a code listing or code box from a magazine article. "
        "In one plain sentence, for a listener who cannot see it, describe "
        "what it appears to show (the language and general purpose) -- do "
        "not read the code line by line. No markdown, no preamble -- the "
        "description text only."
    ),
    BLOCK_FIGURE: (
        "This image is a photo or illustration from a magazine article. In "
        "one or two plain sentences, for a listener who cannot see it, "
        "describe what it shows, including any text visible in the image "
        "itself. No markdown, no preamble -- the description text only."
    ),
}


def describe_visual_region(image_path: Path, region_type: str, base_url: str,
                           model: str, timeout: float = 90.0) -> str | None:
    """Describe one already-cropped figure/chart/table/code image in plain
    language, for optional inclusion in an article's spoken narration.

    A deliberately different task from vision_ocr_page(): that prompt
    explicitly forbids inventing a description ("Images, figures, and
    charts may have empty transcription; do not invent descriptions") --
    correct for transcription, wrong here, where a description is exactly
    what is wanted. Reuses the same engine (already vision-capable and
    already configured) with its own prompt instead, and the same
    reliability guards -- a repeated-phrase decoding loop is exactly as
    possible describing a busy chart as transcribing one.

    Returns None on any failure (unreachable engine, timeout, empty or
    degenerate response) so the caller can try a fallback engine or leave
    the figure without a description -- never partway through one.
    """
    try:
        image_b64 = base64.b64encode(image_path.read_bytes()).decode("ascii")
    except OSError:
        return None

    base_url = base_url.strip().rstrip("/")
    if not urlsplit(base_url).path:
        base_url += "/v1"

    prompt = _DESCRIBE_PROMPTS.get(region_type, _DESCRIBE_PROMPTS[BLOCK_FIGURE])
    unlimited_ocr = "unlimited-ocr" in model.lower()
    payload = {
        "model": model,
        "messages": [{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {
                "url": f"data:image/png;base64,{image_b64}",
            }},
        ]}],
        "max_tokens": 300,
        "temperature": 0.2,
    }
    if unlimited_ocr:
        # Same anti-repetition settings as vision_ocr_page -- same model,
        # same decoding-loop risk, regardless of the prompt asked of it.
        payload.update({
            "skip_special_tokens": False,
            "ngram_size": 35,
            "window_size": 128,
            "images_config": {"image_mode": "gundam"},
        })
    try:
        resp = request_with_retries(
            "POST", f"{base_url.rstrip('/')}/chat/completions",
            json=payload, timeout=timeout, attempts=2,
        )
        resp.raise_for_status()
        content = resp.json()["choices"][0]["message"]["content"]
    except requests.exceptions.RequestException as exc:
        status = getattr(getattr(exc, "response", None), "status_code", None)
        logging.getLogger(__name__).warning("Figure description request failed (%s%s)",
                                           type(exc).__name__, f", HTTP {status}" if status else "")
        return None
    except (ValueError, KeyError, IndexError, TypeError):
        logging.getLogger(__name__).warning("Figure description returned an invalid completion response")
        return None
    if not isinstance(content, str):
        return None
    text = content.strip().strip("`").strip()
    if not text or len(text) < 8:
        return None
    if _looks_like_repetition_loop(text):
        logging.getLogger(__name__).warning("Figure description returned a repeated phrase loop; discarding response")
        return None
    # A long response is either the model ignoring "one or two sentences" or
    # a loop this pass's own repetition check didn't happen to catch --
    # either way, narration wants a short aside, not a paragraph. Cut at a
    # sentence boundary rather than mid-word.
    limit = 600
    if len(text) > limit:
        cut = text.rfind(". ", 0, limit)
        text = text[:cut + 1] if cut > limit // 2 else text[:limit].rstrip() + "…"
    return text


def describe_visual_region_chain(image_path: Path, region_type: str,
                                 engines: Sequence[tuple[str, str]],
                                 timeout: float = 90.0) -> str | None:
    """Try each configured vision engine in order, first usable description
    wins -- same shape as vision_ocr_page_chain."""
    for url, model in engines:
        if not (url and model):
            continue
        text = describe_visual_region(image_path, region_type, url, model, timeout=timeout)
        if text:
            return text
    return None


def build_vision_blocks(regions: list[dict[str, Any]], page_no: int,
                        scale_x: float, id_prefix: str) -> list[Block]:
    """Turn one page's vision OCR regions into Blocks.

    A label the engine gives directly (footer/page_number/header/title/
    caption) is trusted outright -- source="vision_label" makes
    classify_blocks() skip it rather than second-guess it from font size,
    which vision OCR does not reliably provide (see estimate_font_size_from_
    area's docstring: it is a paragraph-area estimate, not a measurement).
    An unrecognised or "text" label is left ambiguous (BLOCK_BODY,
    source="vision_ocr") and still goes through the normal classification
    pass for finer distinctions such as byline or deck.
    """
    out = []
    # Vision OCR occasionally labels a whole text column (or an advert
    # containing text) as an image as well as returning the actual text
    # regions.  Keeping both produces the large, overlapping boxes users see
    # after re-OCR.  Text detections are the more useful source of truth in
    # that case; retain a visual region only when it has meaningful area not
    # covered by a text region.
    scaled = [
        (region, tuple(v * scale_x for v in region["bbox"]))
        for region in regions
    ]
    text_boxes = [
        box for region, box in scaled
        if _VISION_LABEL_TYPES.get(region.get("label")) not in VISUAL_TYPES
        and region.get("text", "").strip()
    ]

    def covered_by_text(box: tuple[float, float, float, float]) -> bool:
        x, y, w, h = box
        area = max(0.0, w) * max(0.0, h)
        if area <= 0:
            return True
        covered = 0.0
        for tx, ty, tw, th in text_boxes:
            covered += max(0.0, min(x + w, tx + tw) - max(x, tx)) * \
                max(0.0, min(y + h, ty + th) - max(y, ty))
        return covered / area >= 0.35

    # Some vision responses emit a broad ``image`` box for a complete column
    # and a tighter figure box for the actual photograph in that column.  The
    # broad box is not useful as an image crop and can hide the text beneath
    # it in the review UI.  Prefer the contained, tighter visual detection.
    visual_boxes = [
        (idx, box) for idx, (region, box) in enumerate(scaled)
        if _VISION_LABEL_TYPES.get(region.get("label")) in VISUAL_TYPES
    ]

    def contains(outer: tuple[float, float, float, float],
                 inner: tuple[float, float, float, float]) -> bool:
        ox, oy, ow, oh = outer
        ix, iy, iw, ih = inner
        return (ix >= ox and iy >= oy and ix + iw <= ox + ow
                and iy + ih <= oy + oh)

    shadowed_visuals = {
        outer_idx for outer_idx, outer in visual_boxes
        if any(outer_idx != inner_idx and contains(outer, inner)
               and inner[2] * inner[3] <= outer[2] * outer[3] * 0.65
               for inner_idx, inner in visual_boxes)
    }
    visual_regions = [box for idx, box in visual_boxes
                      if idx not in shadowed_visuals and not covered_by_text(box)]

    # OCR often transcribes labels printed inside a screenshot as ordinary
    # paragraphs.  Keep the screenshot as one image region; those inner
    # detections must not become separately narrated article text.
    contained_text = {
        idx for idx, (region, box) in enumerate(scaled)
        if _VISION_LABEL_TYPES.get(region.get("label")) not in VISUAL_TYPES
        and any(contains(visual, box) for visual in visual_regions)
    }

    for i, region in enumerate(regions):
        x, y, w, h = (v * scale_x for v in region["bbox"])
        label = region["label"]
        text = region["text"]
        mapped = _VISION_LABEL_TYPES.get(label)
        if i in shadowed_visuals:
            continue
        if i in contained_text and mapped not in (BLOCK_CAPTION, BLOCK_FURNITURE):
            continue
        if mapped in VISUAL_TYPES and h > w * 1.25:
            # A common page-136 response extends the photo box through its
            # caption.  When OCR also returned that caption, stop the image
            # at the first strongly overlapping text box in its lower half.
            for tx, ty, tw, th in text_boxes:
                overlap = max(0.0, min(x + w, tx + tw) - max(x, tx))
                if (overlap >= 0.55 * min(w, tw)
                        and ty > y + h * 0.35 and ty < y + h * 0.9):
                    h = max(1.0, ty - y)
                    break
        if mapped in VISUAL_TYPES and covered_by_text((x, y, w, h)):
            continue
        if mapped is not None:
            block = Block(
                id=f"{id_prefix}{page_no:04d}-v{i:03d}",
                page=page_no, bbox=[x, y, w, h], type=mapped, text=text,
                confidence=0.85, source="vision_label",
            )
        else:
            block = Block(
                id=f"{id_prefix}{page_no:04d}-v{i:03d}",
                page=page_no, bbox=[x, y, w, h], type=BLOCK_BODY, text=text,
                confidence=0.8,
                font_size=estimate_font_size_from_area(w, h, len(text)),
                source="vision_ocr",
            )
        block.region_label = label
        out.append(block)
    return out


# ── Figure bounds from the page render ───────────────────────────────────────
#
# A vision OCR engine grounds a figure far less precisely than it transcribes
# one: on c't KI Wissen 2026 the label and the read-out contents were right
# while the four numbers landed on the neighbouring text column at two thirds
# of the real size (page 23), or covered only the top third of a full-width
# illustration (page 20).  The rendered page itself carries the answer -- a
# photo, a chart or a UI screenshot is a densely inked rectangle fenced off
# from the type area by white gutters -- so the model's box is used only to
# say WHICH graphic is meant and the render says where that graphic ends.
#
# Two passes over the render, not one. A strict "solid ink" pass finds and
# repositions onto a densely inked anchor -- a photo, a filled UI panel, a
# card in a tiled grid -- exactly the way a photo behaves. But a bar chart's
# thin bars, hairline gridlines and axis labels rarely reach that density
# (confirmed on page 18 of the same issue: most bar rows measured 0.25-0.5
# coverage against the 0.55 solid threshold, so the strict pass anchored
# only on the one fully-solid element -- the vertical zero-line -- and
# returned a box cropped to a third of the real chart). A second, looser
# pass grows that anchor -- or, lacking one, the seed itself -- through any
# real mark at all, which is what recovers the rest of a sparse chart.
#
# Both passes are fenced by the same `blocked` cells: real text, and (new)
# every *other* figure's own seed area, so the looser pass cannot bridge a
# real gutter between two adjacent figures the way a raw blank-run
# allowance could -- confirmed against two head-and-shoulders portraits on
# page 31 sitting only ~12pt (1.8 grid cells) apart, well inside what the
# blank-run tolerance alone would cross.
#
# Deliberately conservative: every rule below can decline, and declining
# leaves the model's box untouched.  A wrong box the user can see and redraw
# is a nuisance; a box confidently snapped onto the wrong thing is worse.

_SNAP_CELL_PX = 10        # grid resolution -- ~2 mm at the 110 dpi page render
_SNAP_SOLID = 0.55        # inked fraction marking a cell as graphic, not type
_SNAP_GAP_CELLS = 3       # white gutter a tiled graphic is allowed to span
_SNAP_MIN_FILL = 0.45     # an accepted box is this solidly inked ...
_SNAP_MIN_FRAME = 0.9     # ... or framed on this share of its rows AND columns
_SNAP_MIN_SEED_AREA = 0.30   # ... and never shrinks below this much of the seed
_SNAP_MIN_OVERLAP = 0.005  # a graphic must touch the seed by at least this much --
                           # low on purpose: a seed can be offset far enough
                           # that it barely grazes the real graphic's edge
                           # (see page 70 of c't KI Wissen 2026, where the
                           # 0.02 first drafted here missed a 3-of-220-cell
                           # graze by a hair and needed a second snap pass
                           # before it could reposition at all)
_SNAP_ANY_INK = 0.15      # the loose pass's "there is a real mark here" floor --
                          # above the ~0.02-0.11 scan-noise floor measured in
                          # genuinely blank chart gaps, below the ~0.25+ every
                          # real bar/line/label measured at
_SNAP_MAX_BLANK_RUN = 2   # cells of true blank the loose pass may cross --
                          # covers a chart's own internal row gaps (measured
                          # at 1, occasionally 2 cells) without relying on
                          # this alone to keep two adjacent figures apart
_SNAP_BLOCKED_WALL = 0.7  # a row/column blocked across this much of a
                          # candidate box is real text the box must not
                          # cross, even though merging separate ink
                          # components into one rectangle (see _snap_seed)
                          # can otherwise enclose it geometrically
_SNAP_JOINT_ROUNDS = 6    # rounds of re-seeding every visual on a page from
                          # each other's last result (see snap_visual_bboxes)
                          # -- enough to settle the common case and to reach
                          # a deterministic stopping point for the rare pair
                          # that does not truly converge


def page_ink_mask(png_path: Path):
    """Full-resolution boolean map of the page render: True where a pixel is visibly darker than
    the paper or visibly coloured. None when the render cannot be read."""
    try:
        import numpy as np
        from PIL import Image
        with Image.open(png_path) as im:
            rgb = np.asarray(im.convert("RGB"), dtype=np.float32)
    except Exception:
        return None
    lightness = rgb.max(axis=2)
    saturation = lightness - rgb.min(axis=2)
    paper = float(np.percentile(lightness, 90)) or 255.0
    return (lightness < paper * 0.88) | (saturation > 40)


def page_ink_grid(png_path: Path, cell: int = _SNAP_CELL_PX):
    """Coarse map of the page render: inked fraction of each `cell`-px square.

    "Inked" means visibly darker than the paper or visibly coloured, so a
    grey scan background does not count as ink.  Body type covers roughly a
    quarter of the cells it runs through; a photo or a filled panel covers
    nearly all of them, which is the separation snap_visual_bboxes() relies
    on.  Returns None when the render cannot be read.
    """
    ink = page_ink_mask(png_path)
    if ink is None:
        return None
    rows, cols = (n // cell for n in ink.shape)
    if rows < 4 or cols < 4:
        return None
    return ink[:rows * cell, :cols * cell].reshape(
        rows, cell, cols, cell).mean(axis=(1, 3))


def page_ink_grid_points(png_path: Path, page_meta: dict, cell: int = _SNAP_CELL_PX):
    """page_ink_grid(), plus each cell's size in the page's own point units (as bboxes are stored
    in), for a caller that has no reason to know the render's pixel size -- see
    region_looks_like_a_picture, which screens a proposed visual region against this."""
    grid = page_ink_grid(png_path, cell)
    if grid is None:
        return None
    try:
        from PIL import Image
        with Image.open(png_path) as im:
            w_px, h_px = im.size
    except Exception:
        return None
    if not w_px or not h_px or not page_meta.get("width") or not page_meta.get("height"):
        return None
    return grid, page_meta["width"] * cell / w_px, page_meta["height"] * cell / h_px


_SNAP_STABLE_IOU = 0.7    # a result must reproduce at least this much of
                          # itself (intersection over union) when fed back
                          # in as its own seed, or it is discarded -- see
                          # _snap_seed's stability check


def _snap_seed(coverage, seed: Sequence[int], blocked) -> list[int] | None:
    """Grow the graphic the seed cell-box points at, or None to keep the seed.

    A result is only trusted once it has proven it will not move again:
    since `snap_visual_bboxes` is meant to be safe to run repeatedly (an
    already-corrected box seeding the same graphic again), a candidate this
    function itself would revise on a second look is exactly the case that
    must not be published. Two adjacent dark photos sharing a caption strip
    (page 158 of c't KI Wissen 2026) is the confirmed real case: which one
    "owns" the ambiguous shared area flips depending on subtle seed-size
    effects, so neither box ever settles. Declining -- keeping the model's
    original box -- is safer than publishing a box the very next run would
    replace with a substantially different one.
    """
    box = _snap_seed_once(coverage, seed, blocked)
    if box is None:
        return None
    settled = _snap_seed_once(coverage, box, blocked)
    if settled is None:
        return None
    intersection = _cell_overlap(box, settled)
    union = ((box[2] - box[0] + 1) * (box[3] - box[1] + 1)
             + (settled[2] - settled[0] + 1) * (settled[3] - settled[1] + 1)
             - intersection)
    if union == 0 or intersection / union < _SNAP_STABLE_IOU:
        return None
    return box


def _snap_seed_once(coverage, seed: Sequence[int], blocked) -> list[int] | None:
    """One application of the snap rules -- see _snap_seed for the public,
    stability-checked entry point that wraps this.

    Cells already claimed by a text block -- or by another figure's own
    seed area, see snap_visual_bboxes -- are excluded up front: text boxes
    come back from the same engine far more accurately than figure boxes
    (verified against the page render for every block of the issue above),
    so they are the one trustworthy fence against a figure swallowing the
    column, or the neighbouring figure, beside it.
    """
    import numpy as np
    from scipy import ndimage

    solid = (coverage >= _SNAP_SOLID) & ~blocked
    seed_area = max(1, (seed[2] - seed[0] + 1) * (seed[3] - seed[1] + 1))
    seed_mask = np.zeros(solid.shape, dtype=bool)
    sx0, sy0, sx1, sy1 = seed
    seed_mask[sy0:sy1 + 1, sx0:sx1 + 1] = True

    # A figure split by white gutters -- a grid of cards, or (as much as a
    # dashed zero-line and a dense axis-label strip can be) a chart -- is
    # several disconnected solid components. Bridging small gaps BEFORE
    # labelling, rather than iteratively re-joining whichever component the
    # seed happens to touch first, is what makes the result independent of
    # the seed's own size: re-running this function on an already-corrected
    # (and now larger) box must pick the same components it did the first
    # time, not a different one that merely overlaps the new seed more by
    # raw area. Confirmed unstable on page 18's chart before this was
    # connectivity- rather than seed-driven -- the dense axis-label strip
    # briefly out-competed the thin zero-line for "best overlap" once the
    # seed grew to cover both, shrinking an already-correct box back down.
    connected = solid
    if _SNAP_GAP_CELLS > 0:
        structure = ndimage.generate_binary_structure(2, 1)
        for _ in range(_SNAP_GAP_CELLS):
            connected = ndimage.binary_dilation(connected, structure=structure) & ~blocked

    anchor = None
    labels, count = ndimage.label(connected)
    if count:
        overlaps = ndimage.sum_labels(seed_mask, labels, range(1, count + 1))
        best = max(range(count), key=lambda i: overlaps[i])
        if overlaps[best] >= _SNAP_MIN_OVERLAP * seed_area:
            # The bounding box of this component's ORIGINAL solid cells,
            # not its dilated footprint -- dilation only decides which
            # solid patches count as one figure, it must not itself widen
            # the box.
            ys, xs = np.where(solid & (labels == best + 1))
            if xs.size:
                anchor = [int(xs.min()), int(ys.min()), int(xs.max()), int(ys.max())]
                # A bounding box is a rectangle, not the component's actual
                # shape, so it can still enclose a blocked text wall lying
                # between two merged patches even though neither patch's
                # own cells ever touched it.
                anchor = _trim_blocked_edges(anchor, blocked)

    # The solid pass alone stops here for a photo or a UI panel -- dense
    # enough that anchor already is the whole figure. A chart's sparse bars
    # and axis rules need the second, looser pass below to be included at
    # all; without a solid anchor to grow from, grow straight from the seed
    # instead of giving up outright, since the seed usually still overlaps
    # at least part of the real graphic even when none of it is solid.
    grow_from = anchor if anchor is not None else seed
    any_ink = (coverage >= _SNAP_ANY_INK) & ~blocked
    gx0, gy0, gx1, gy1 = grow_from
    if not any_ink[gy0:gy1 + 1, gx0:gx1 + 1].any():
        # Nothing at all under the anchor/seed -- a blank margin, or a pale
        # chart/white-on-white screenshot below even the loose floor (see
        # the known limitation in Manual.md). Declining is safer than
        # guessing.
        if anchor is None:
            return None
        box = anchor
    else:
        box = _grow_to_bounds(coverage, blocked, grow_from)

    area = (box[2] - box[0] + 1) * (box[3] - box[1] + 1)
    if area < _SNAP_MIN_SEED_AREA * seed_area:
        return None

    if box == grow_from:
        # Growth found nothing beyond the anchor itself -- fall back to the
        # original density test, which only ever runs here because a real
        # solid anchor was required to reach this point.
        if anchor is None:
            return None
        inside = coverage[box[1]:box[3] + 1, box[0]:box[2] + 1]
        if inside.mean() >= _SNAP_MIN_FILL:
            return box
        # A screenshot or a boxed insert is mostly white inside but inked
        # right around its edge, so a solid frame counts as a graphic too.
        edge = solid[box[1]:box[3] + 1, box[0]:box[2] + 1]
        if (edge.any(axis=1).mean() >= _SNAP_MIN_FRAME
                and edge.any(axis=0).mean() >= _SNAP_MIN_FRAME):
            return box
        return None
    # Growth extended the anchor/seed through real, connected marks fenced
    # by text and other figures -- trust it without re-testing density,
    # which a chart's mostly-white interior fails by construction.
    return box


def _grow_to_bounds(coverage, blocked, box: Sequence[int]) -> list[int]:
    """Extend a box outward through content too sparse for the solid test.

    Each of the four sides advances one cell at a time through any real
    mark at all (`_SNAP_ANY_INK`), which is what a bar chart's thin bars,
    hairline axis and small print register as even though none of them
    individually reach the solid-ink threshold above. A side stops the
    instant it reaches a blocked cell -- real text, or another figure's own
    seed area -- or after `_SNAP_MAX_BLANK_RUN` consecutive cells with no
    mark at all, whichever comes first, so growth cannot bridge a real
    gutter between two adjacent figures or wander off into open margin.
    """
    rows, cols = coverage.shape
    x0, y0, x1, y1 = box
    any_ink = (coverage >= _SNAP_ANY_INK) & ~blocked

    def scan(pos: int, step: int, limit: int, lo: int, hi: int, horizontal: bool) -> int:
        blank_run = 0
        while True:
            nxt = pos + step
            if nxt < 0 or nxt >= limit:
                break
            line_blocked = blocked[nxt, lo:hi + 1] if horizontal else blocked[lo:hi + 1, nxt]
            if line_blocked.any():
                break
            line_ink = any_ink[nxt, lo:hi + 1] if horizontal else any_ink[lo:hi + 1, nxt]
            if line_ink.any():
                blank_run = 0
            else:
                blank_run += 1
                if blank_run > _SNAP_MAX_BLANK_RUN:
                    pos -= step * (blank_run - 1)   # roll back the trailing blank run
                    break
            pos = nxt
        return pos

    new_y0 = scan(y0, -1, rows, x0, x1, True)
    new_y1 = scan(y1, 1, rows, x0, x1, True)
    new_x0 = scan(x0, -1, cols, y0, y1, False)
    new_x1 = scan(x1, 1, cols, y0, y1, False)
    return [min(x0, new_x0), min(y0, new_y0), max(x1, new_x1), max(y1, new_y1)]


def _trim_blocked_edges(box: Sequence[int], blocked) -> list[int]:
    """Pull each edge of `box` back to the nearest real text wall inside it.

    Merging separate ink components into one bounding rectangle (see
    _snap_seed) can enclose a text block that happens to sit between two
    pieces of the same graphic, even though that block's own cells were
    excluded from the ink search itself -- the merge only produces a
    rectangle, not the components' actual shape, so the rectangle can still
    geometrically span over blocked cells lying between them. Confirmed on
    page 22 of c't KI Wissen 2026: a chart's candidate box reached two text
    lines above the chart itself, because a small mark on the chart's own
    top edge and the graphic's body were merged across a gap that also
    happened to contain the tail of an unrelated paragraph.

    A row or column blocked across most of the box's width/height is
    unambiguously real text, not a stray corner overlap, and is trusted
    over the rectangle's own extent -- searched outward from the box's own
    centre so a wall near one edge trims that edge without disturbing the
    other three.
    """
    x0, y0, x1, y1 = box
    width, height = x1 - x0 + 1, y1 - y0 + 1
    sub = blocked[y0:y1 + 1, x0:x1 + 1]
    row_frac = sub.mean(axis=1)
    col_frac = sub.mean(axis=0)

    new_y0, new_y1 = y0, y1
    for r in range(height // 2, -1, -1):
        if row_frac[r] >= _SNAP_BLOCKED_WALL:
            new_y0 = y0 + r + 1
            break
    for r in range(height // 2, height):
        if row_frac[r] >= _SNAP_BLOCKED_WALL:
            new_y1 = y0 + r - 1
            break
    new_x0, new_x1 = x0, x1
    for c in range(width // 2, -1, -1):
        if col_frac[c] >= _SNAP_BLOCKED_WALL:
            new_x0 = x0 + c + 1
            break
    for c in range(width // 2, width):
        if col_frac[c] >= _SNAP_BLOCKED_WALL:
            new_x1 = x0 + c - 1
            break
    if new_y0 > new_y1 or new_x0 > new_x1:
        return list(box)   # trimming would invert the box -- leave it be
    return [new_x0, new_y0, new_x1, new_y1]


def _cell_overlap(a: Sequence[int], b: Sequence[int]) -> int:
    """Area, in grid cells, shared by two inclusive [x0, y0, x1, y1] boxes."""
    return (max(0, min(a[2], b[2]) - max(a[0], b[0]) + 1)
            * max(0, min(a[3], b[3]) - max(a[1], b[1]) + 1))


def _refine_dense_visual_edges(png_path: Path, box: Sequence[int],
                               page_w: float, page_h: float) -> list[float] | None:
    """Resolve the coarse grid's sub-cell ambiguity for dense photographs.

    The grid safely finds the right graphic, but a ten-pixel cell can leave
    paper on whichever edges the true boundary crosses mid-cell. That creates
    the visibly mixed crop (margin on left/bottom, none on top/right). Dense
    figures have a strong per-pixel transition, so refine all four edges from
    the render. Sparse charts deliberately stay on the safer coarse path.
    """
    try:
        import numpy as np
        from PIL import Image
        with Image.open(png_path) as im:
            rgb = np.asarray(im.convert("RGB"), dtype=np.float32)
    except Exception:
        return None
    lightness = rgb.max(axis=2)
    saturation = lightness - rgb.min(axis=2)
    paper = float(np.percentile(lightness, 90)) or 255.0
    ink = (lightness < paper * 0.88) | (saturation > 40)
    height, width = ink.shape
    x0 = max(0, int(box[0] * _SNAP_CELL_PX))
    y0 = max(0, int(box[1] * _SNAP_CELL_PX))
    x1 = min(width, int((box[2] + 1) * _SNAP_CELL_PX))
    y1 = min(height, int((box[3] + 1) * _SNAP_CELL_PX))
    if x1 <= x0 or y1 <= y0 or float(ink[y0:y1, x0:x1].mean()) < _SNAP_MIN_FILL:
        return None

    radius = _SNAP_CELL_PX * 2

    def best_run(flags, original_start: int, original_end: int, offset: int):
        runs = []
        start = None
        for index, on in enumerate(list(flags) + [False]):
            if on and start is None:
                start = index
            elif not on and start is not None:
                absolute = (offset + start, offset + index)
                overlap = max(0, min(absolute[1], original_end) -
                              max(absolute[0], original_start))
                runs.append((overlap, absolute[1] - absolute[0], absolute))
                start = None
        return max(runs, default=(0, 0, None))[2]

    wx0, wx1 = max(0, x0 - radius), min(width, x1 + radius)
    col_run = best_run(ink[y0:y1, wx0:wx1].mean(axis=0) >= 0.35,
                       x0, x1, wx0)
    if not col_run:
        return None
    rx0, rx1 = col_run
    wy0, wy1 = max(0, y0 - radius), min(height, y1 + radius)
    row_run = best_run(ink[wy0:wy1, rx0:rx1].mean(axis=1) >= 0.35,
                       y0, y1, wy0)
    if not row_run:
        return None
    ry0, ry1 = row_run
    return [rx0 * page_w / width, ry0 * page_h / height,
            (rx1 - rx0) * page_w / width, (ry1 - ry0) * page_h / height]


def snap_visual_bboxes(png_path: Path, blocks: Sequence[Block],
                       page_meta: dict[str, Any]) -> int:
    """Correct every visual block's bbox against the page render, in place.

    Returns how many boxes moved.  Silent no-op when the render is missing or
    the imaging stack is unavailable -- the layout stays exactly as the OCR
    engine reported it, which is what happened before this pass existed.
    """
    visuals = [b for b in blocks if b.type in VISUAL_TYPES]
    if not visuals:
        return 0
    try:
        import numpy as np
        import scipy.ndimage  # noqa: F401  -- required by _snap_seed
    except ImportError:
        return 0
    coverage = page_ink_grid(png_path)
    if coverage is None:
        return 0
    rows, cols = coverage.shape
    page_w = float(page_meta.get("width") or 0) or 1.0
    page_h = float(page_meta.get("height") or 0) or 1.0
    cells_x, cells_y = cols / page_w, rows / page_h

    def to_cells(bbox: Sequence[float]) -> list[int]:
        x, y, w, h = bbox
        return [min(cols - 1, max(0, int(x * cells_x))),
                min(rows - 1, max(0, int(y * cells_y))),
                min(cols - 1, max(0, int((x + w) * cells_x))),
                min(rows - 1, max(0, int((y + h) * cells_y)))]

    text_blocked = np.zeros(coverage.shape, dtype=bool)
    for b in blocks:
        # Advertisement geometry comes from the same vision model whose loose
        # figure boxes this pass is repairing.  Treating that often-wrong ad
        # rectangle as immutable text blocked the real graphic below it and
        # left both the orange figure crops and red advert outline displaced.
        if b.type in (*VISUAL_TYPES, BLOCK_AD) or not b.text.strip():
            continue
        x0, y0, x1, y1 = to_cells(b.bbox)
        text_blocked[y0:y1 + 1, x0:x1 + 1] = True

    visual_seeds = [to_cells(b.bbox) for b in visuals]

    def blocked_for(i: int, current: list[list[int]]):
        # A figure's loose growth pass (see _grow_to_bounds) must not bridge
        # into a neighbouring figure's own territory just because the gutter
        # between them is narrow and mostly blank -- two head-and-shoulders
        # portraits on the same page can sit under 2 grid cells apart.
        # Blocking every *other* figure's own current area, not just text,
        # fences that gutter regardless of how little ink is in it.
        blocked = text_blocked.copy()
        for j, other in enumerate(current):
            if j == i:
                continue
            ox0, oy0, ox1, oy1 = other
            blocked[oy0:oy1 + 1, ox0:ox1 + 1] = True
        return blocked

    def colliding_pairs(boxes: list[list[int]]) -> set[int]:
        # Two figures pulled onto the same graphic means the page was read
        # as one picture where the engine saw two (or the reverse); neither
        # reading can be trusted over the other, so both keep their
        # original box.
        hit: set[int] = set()
        for i, a in enumerate(boxes):
            for j in range(i + 1, len(boxes)):
                c = boxes[j]
                smaller = min((a[2] - a[0] + 1) * (a[3] - a[1] + 1),
                              (c[2] - c[0] + 1) * (c[3] - c[1] + 1))
                if _cell_overlap(a, c) > 0.5 * smaller:
                    hit.add(i)
                    hit.add(j)
        return hit

    # Ambiguity is decided once, up front, fenced against the engine's own
    # ORIGINAL seeds -- not against whatever the iterative refinement below
    # settles on. Iterating first and only then checking for collisions
    # would let two seeds legitimately ambiguous about the same graphic
    # mutually fence each other into a tidy, confident-looking split of it
    # instead of ever being recognised as the same disputed claim.
    first_round = [_snap_seed(coverage, seed, blocked_for(i, visual_seeds))
                   for i, seed in enumerate(visual_seeds)]
    ambiguous = colliding_pairs([box or seed for box, seed in zip(first_round, visual_seeds)])

    # Fencing figure i against figure j's box, and vice versa, means each
    # one's result depends on where the OTHER currently sits -- a mutual
    # dependency a single pass cannot resolve consistently. Repeating the
    # whole page's worth of visuals, each round seeded by the previous
    # round's output, converges that to a joint fixed point in the common
    # case, and -- for the rare pair that never truly settles (two dark
    # photos sharing a caption strip, confirmed on page 158 of c't KI
    # Wissen 2026) -- reliably lands on the SAME deterministic answer every
    # time this function is called, because a later call starts from
    # exactly the state this loop stops at rather than re-diverging. An
    # already-ambiguous visual is frozen at its original seed throughout,
    # both as a participant (it will be discarded anyway) and as a fence
    # for its neighbours (its disputed reading must not move anyone else).
    current = list(visual_seeds)
    for _ in range(_SNAP_JOINT_ROUNDS):
        nxt = [visual_seeds[i] if i in ambiguous else
               (_snap_seed(coverage, seed, blocked_for(i, current)) or seed)
               for i, seed in enumerate(current)]
        if nxt == current:
            break
        current = nxt

    reverted = ambiguous | colliding_pairs(current)
    final = [None if i in reverted else current[i] for i in range(len(current))]

    moved = 0
    for index, (block, box) in enumerate(zip(visuals, final)):
        if box is None:
            continue
        refined = _refine_dense_visual_edges(png_path, box, page_w, page_h)
        if box == visual_seeds[index] and refined is None:
            continue
        bbox = (refined or [box[0] / cells_x, box[1] / cells_y,
                            (box[2] + 1 - box[0]) / cells_x,
                            (box[3] + 1 - box[1]) / cells_y])
        if any(abs(a - b) > 1.0 for a, b in zip(bbox, block.bbox)):
            moved += 1
        block.bbox = bbox
    return moved


def ocr_page(png_path: Path, lang: str = "deu+eng",
             min_confidence: float = 40.0) -> list[dict[str, Any]]:
    """OCR one rendered page, returning paragraph boxes from tesseract's TSV.

    Grouped by (block, paragraph), not by line: PyMuPDF's native path already
    aggregates a paragraph's spans into one Block with one representative font
    size, and OCR must match that shape or the two paths behave inconsistently
    downstream. Confirmed live on a scanned magazine -- one Block per LINE let
    a single anomalous line (a chart axis number, a headline's wrapped second
    line, scan noise) independently trigger BLOCK_TITLE classification, and a
    28-page scan produced ~11 "title" blocks per page instead of grouping
    around real headlines. font_size uses the MEDIAN of the paragraph's own
    line heights rather than the paragraph's total bbox height (which would
    scale with line count, not type size) -- a genuine multi-line headline
    still measures large because each of its few lines is uniformly large;
    one noisy line inside an otherwise normal paragraph no longer dominates.
    TSV rather than plain text because the coordinates are the whole point:
    they are what lets an OCR'd page take part in the same column and article
    grouping as a native-text page.
    """
    try:
        proc = subprocess.run(
            ["tesseract", str(png_path), "stdout", "-l", lang, "tsv"],
            capture_output=True, text=True, timeout=180, check=True,
        )
    except (subprocess.SubprocessError, FileNotFoundError, OSError):
        return []

    paragraphs: dict[tuple[int, int], dict[str, Any]] = {}
    for row in proc.stdout.splitlines()[1:]:
        parts = row.split("\t")
        if len(parts) < 12:
            continue
        try:
            block_num, par_num, line_num = int(parts[2]), int(parts[3]), int(parts[4])
            left, top, width, height = (int(parts[6]), int(parts[7]),
                                        int(parts[8]), int(parts[9]))
            conf = float(parts[10])
        except ValueError:
            continue
        word = parts[11].strip()
        if not word or conf < min_confidence:
            continue
        para_key = (block_num, par_num)
        para = paragraphs.setdefault(para_key, {"lines": {}})
        line = para["lines"].setdefault(line_num, {
            "text": [], "x0": left, "y0": top,
            "x1": left + width, "y1": top + height, "conf": [],
        })
        line["text"].append(word)
        line["x0"] = min(line["x0"], left)
        line["y0"] = min(line["y0"], top)
        line["x1"] = max(line["x1"], left + width)
        line["y1"] = max(line["y1"], top + height)
        line["conf"].append(conf)

    out = []
    for _, para in sorted(paragraphs.items()):
        lines = [l for _, l in sorted(para["lines"].items()) if l["text"]]
        if not lines:
            continue
        x0 = min(l["x0"] for l in lines)
        y0 = min(l["y0"] for l in lines)
        x1 = max(l["x1"] for l in lines)
        y1 = max(l["y1"] for l in lines)
        heights = [float(l["y1"] - l["y0"]) for l in lines]
        confs = [c for l in lines for c in l["conf"]]
        out.append({
            "text": " ".join(" ".join(l["text"]) for l in lines),
            "bbox": [float(x0), float(y0), float(x1 - x0), float(y1 - y0)],
            "line_height": statistics.median(heights),
            "confidence": round(sum(confs) / len(confs) / 100.0, 3),
        })
    return out


# ── Image import ─────────────────────────────────────────────────────────────

IMAGE_IMPORT_MAX_FILES = 300
IMAGE_IMPORT_MAX_BYTES = 25 * 1024 * 1024        # per image
IMAGE_IMPORT_MAX_PIXELS = 60_000_000             # refuses decompression bombs before decoding
_ASSUMED_SCAN_DPI = 150                          # when an image carries no (or an absurd) resolution


class ImageImportError(ValueError):
    """An uploaded page image was rejected; the message is safe to show the user."""


def _sniff_image(data: bytes) -> str | None:
    """'png' or 'jpeg' judged by the file's own header, never by its name or declared type."""
    if data[:8] == b"\x89PNG\r\n\x1a\n":
        return "png"
    if data[:3] == b"\xff\xd8\xff":
        return "jpeg"
    return None


def images_to_pdf(images: Sequence[tuple[str, bytes]], dest: Path) -> int:
    """Combine page images (PNG/JPEG, in the given order) into one PDF at `dest`; returns the page count.

    Each image becomes one page sized from its own resolution (so the app's fixed-dpi render of the
    page comes out at the scan's real proportions), with EXIF rotation applied. The result has no text
    layer, so ingestion OCRs every page exactly as for a scanned PDF. Nothing is written on failure.
    """
    if not images:
        raise ImageImportError("Choose at least one image")
    if len(images) > IMAGE_IMPORT_MAX_FILES:
        raise ImageImportError(f"At most {IMAGE_IMPORT_MAX_FILES} images per magazine")
    import io
    import pymupdf
    from PIL import Image, ImageOps

    doc = pymupdf.open()
    try:
        for name, data in images:
            label = Path(name or "image").name[:80]
            if len(data) > IMAGE_IMPORT_MAX_BYTES:
                raise ImageImportError(f"{label} is larger than {IMAGE_IMPORT_MAX_BYTES // (1024 * 1024)} MB")
            kind = _sniff_image(data)
            if kind is None:
                raise ImageImportError(f"{label} is not a PNG or JPEG image")
            try:
                with Image.open(io.BytesIO(data)) as im:
                    width, height = im.size
                    if width * height > IMAGE_IMPORT_MAX_PIXELS:
                        raise ImageImportError(f"{label} has too many pixels")
                    dpi = im.info.get("dpi", (0, 0))[0] or 0
                    if im.getexif().get(0x0112, 1) not in (0, 1):      # EXIF says the pixels are rotated/flipped
                        upright = ImageOps.exif_transpose(im)
                        buf = io.BytesIO()
                        upright.convert("RGB" if kind == "jpeg" else upright.mode).save(
                            buf, format="JPEG" if kind == "jpeg" else "PNG", **({"quality": 95} if kind == "jpeg" else {}))
                        data = buf.getvalue()
                        width, height = upright.size
            except ImageImportError:
                raise
            except Exception as e:  # truncated / corrupt file
                raise ImageImportError(f"{label} could not be read as an image") from e
            # 72 (and anything below 100) is the placeholder most tools write, not a real scan resolution.
            dpi = float(dpi) if 100 <= float(dpi or 0) <= 1200 else _ASSUMED_SCAN_DPI
            w_pt, h_pt = width * 72.0 / dpi, height * 72.0 / dpi
            page = doc.new_page(width=w_pt, height=h_pt)
            page.insert_image(pymupdf.Rect(0, 0, w_pt, h_pt), stream=data)
        tmp = dest.with_suffix(".part")
        doc.save(str(tmp))
        tmp.replace(dest)
        return len(images)
    finally:
        doc.close()


_VISION_QUALITY_THRESHOLD = 0.5  # below this, existing native/OCR text is escalated


def extract_pages(pdf_path: Path, out_dir: Path, render_dpi: int = 110,
                  ocr_lang: str = "deu+eng",
                  vision_ocr_url: str | None = None,
                  vision_ocr_model: str | None = None,
                  vision_ocr_fallback_url: str | None = None,
                  vision_ocr_fallback_model: str | None = None,
                  vision_ocr_timeout: float = 150.0,
                  force_vision_ocr: bool = False,
                  progress: Any = None,
                  page_rotations: dict[int, int] | None = None) -> dict[str, Any]:
    """Run the full extraction for one PDF, writing page renders and images.

    `page_rotations` ({page: clockwise degrees}) are turns the reader applied to pages scanned sideways; a
    rendered page is turned the same way whenever it is (re)rendered, so a re-import does not undo them.

    When vision_ocr_url/model are given, a page whose text needs OCR (or
    whose already-extracted text scores as mostly garbage, native or
    tesseract alike -- see ocr_quality_score) is escalated to that engine
    first, with a second vision engine (vision_ocr_fallback_*) tried next if
    given, and tesseract only as the last resort if every vision call fails
    or every engine is unreachable. Confirmed live against
    ocr-baidu-unlimited-ocr: markedly more accurate transcription of a
    scanned German magazine than tesseract, and it labels running headers/
    footers/page numbers directly (see build_vision_blocks), which is the
    dominant source of the over-fragmentation cluster_running_furniture()
    otherwise has to recover after the fact from noisy OCR variants of the
    same repeating line. KDL Frontier Parser nano, tested as the fallback
    engine, answers faster but returns plain text with no layout coordinates
    at all -- handled by _whole_page_region as one ambiguous full-page block.

    Returns the layout document: every block on every page, classified and in
    reading order, plus the raw contents-page candidates.
    """
    fitz = _require_fitz()
    pages_dir = out_dir / "pages"
    images_dir = out_dir / "images"
    pages_dir.mkdir(parents=True, exist_ok=True)
    images_dir.mkdir(parents=True, exist_ok=True)
    vision_engines = [(vision_ocr_url, vision_ocr_model),
                      (vision_ocr_fallback_url, vision_ocr_fallback_model)]
    vision_enabled = bool(vision_ocr_url and vision_ocr_model)
    vision_page_count = 0

    doc = fitz.open(str(pdf_path))
    try:
        raw_pages: list[dict[str, Any]] = []

        for pno in range(doc.page_count):
            page = doc.load_page(pno)
            page_no = pno + 1
            extra_turn = int((page_rotations or {}).get(page_no, 0)) % 360
            if extra_turn:
                page.set_rotation((page.rotation + extra_turn) % 360)
            rect = page.rect

            png = pages_dir / f"{page_no:04d}.png"
            if not png.exists():
                pix = page.get_pixmap(dpi=render_dpi)
                pix.save(str(png))

            image_boxes: list[list[float]] = []
            page_images: list[dict[str, Any]] = []
            # The PDF's own text and image boxes are in the page's unturned coordinates; on a page the reader
            # turned, the render (and OCR of it) is the truth.
            for img_index, info in enumerate([] if extra_turn else page.get_images(full=True)):
                rects: list[list[float]] = []
                try:
                    for r in page.get_image_rects(info[0]):
                        rects.append([r.x0, r.y0, r.width, r.height])
                except Exception:
                    continue
                image_boxes.extend(rects)
                name = f"p{page_no:04d}-{img_index:02d}"
                if _extract_image(fitz, doc, info[0], images_dir, name) and rects:
                    # A few embedded images repeat (a logo stamped on every
                    # page); each placement still gets its own bbox so a
                    # caption or figure position can be matched precisely.
                    for ri, box in enumerate(rects):
                        page_images.append({
                            "id": name if ri == 0 else f"{name}-{ri}",
                            "page": page_no,
                            "bbox": box,
                            "filename": f"{name}.png",
                        })

            blocks: list[Block] = []
            data = {} if extra_turn else page.get_text("dict")
            for bi, blk in enumerate(data.get("blocks", [])):
                if blk.get("type") != 0:      # 0 = text, 1 = image
                    continue
                texts, sizes, fonts, bolds = [], [], [], []
                for line in blk.get("lines", []):
                    for span in line.get("spans", []):
                        if not span.get("text", "").strip():
                            continue
                        texts.append(span["text"])
                        sizes.append(float(span.get("size", 0)))
                        fonts.append(span.get("font", ""))
                        bolds.append(bool(span.get("flags", 0) & 2 ** 4))
                text = " ".join(texts).strip()
                if not text:
                    continue
                x0, y0, x1, y1 = blk["bbox"]
                blocks.append(Block(
                    id=f"b{page_no:04d}-{bi:03d}",
                    page=page_no,
                    bbox=[x0, y0, x1 - x0, y1 - y0],
                    type=BLOCK_BODY,
                    text=text,
                    font_size=statistics.median(sizes) if sizes else 0.0,
                    font_name=max(set(fonts), key=fonts.count) if fonts else "",
                    bold=sum(bolds) > len(bolds) / 2 if bolds else False,
                ))

            # A page whose native text is negligible is a scan (or is drawn as
            # outlines); OCR is the only way to reach its words. A page WITH
            # native/embedded text is not necessarily good text -- a
            # pre-OCR'd scan (e.g. from scanner-bundled OCR software) can
            # carry a poor text layer that this same escalation should reach.
            native_chars = sum(len(b.text) for b in blocks)
            needs_ocr = native_chars < 120
            native_quality = (
                ocr_quality_score(" ".join(b.text for b in blocks))
                if not needs_ocr else 0.0
            )
            needs_escalation = force_vision_ocr or needs_ocr or native_quality < _VISION_QUALITY_THRESHOLD

            vision_blocks: list[Block] | None = None
            if vision_enabled and needs_escalation:
                regions = vision_ocr_page_chain(png, vision_engines,
                                                timeout=vision_ocr_timeout)
                if regions:
                    repair_drop_cap_omissions(png, regions, ocr_lang)
                    scale_x = rect.width / max(1.0, _png_width(png, fitz))
                    vision_blocks = build_vision_blocks(
                        regions, page_no, scale_x, id_prefix="b")

            if vision_blocks is not None:
                # Replaces whatever was already there -- including a poor
                # native/pre-OCR'd text layer, rather than layering a second,
                # conflicting reading on top of it.
                vision_page_count += 1
                blocks = vision_blocks
                page_meta = {"width": rect.width, "height": rect.height}
                # Before cropping: the crop is taken from the block's bbox,
                # so a box the engine placed loosely would be saved as a
                # mis-framed figure file as well as a mis-drawn review box.
                snap_visual_bboxes(png, blocks, page_meta)
                detected_images = crop_visual_regions(out_dir, page_no,
                    page_meta, blocks)
                if detected_images:
                    page_images = detected_images
            elif needs_ocr:
                scale_x = rect.width / max(1.0, _png_width(png, fitz))
                for li, para in enumerate(ocr_page(png, ocr_lang)):
                    bx, by, bw, bh = para["bbox"]
                    blocks.append(Block(
                        id=f"b{page_no:04d}-o{li:03d}",
                        page=page_no,
                        bbox=[bx * scale_x, by * scale_x, bw * scale_x, bh * scale_x],
                        type=BLOCK_BODY,
                        text=para["text"],
                        confidence=para["confidence"],
                        # The paragraph's own line height, not its full bbox
                        # height -- that would scale with line count instead
                        # of type size (see ocr_page's docstring).
                        font_size=para["line_height"] * scale_x,
                        source="ocr",
                    ))
            # else: native text present and scored well enough -- unchanged.

            raw_pages.append({
                "page": page_no,
                "width": rect.width,
                "height": rect.height,
                "rotation": extra_turn,
                "blocks": blocks,
                "image_boxes": image_boxes,
                "images": page_images,
            })
            if progress:
                # Raw block text, pre-classification -- good enough for a
                # live "here is what was just read" preview during ingestion
                # (mirrors ocr-baidu-unlimited-ocr's own demo: source page on
                # one side, recognised text on the other, filled in as each
                # page finishes) without waiting for the document-wide passes
                # below.
                try:
                    progress(page_no, doc.page_count, blocks, page_images, page_size=(rect.width, rect.height))
                except TypeError:
                    try:
                        progress(page_no, doc.page_count, blocks, page_images)
                    except TypeError:
                        progress(page_no, doc.page_count, blocks)

        # Furniture can only be identified across the whole document, so
        # classification waits until every page has been read.
        repeated = find_repeated_lines(
            [[b.text for b in p["blocks"]] for p in raw_pages]
        )
        all_blocks: list[Block] = []
        contents_candidates: list[dict[str, Any]] = []

        # One baseline for the whole issue (see classify_blocks).
        doc_base = body_font_size(
            [b for p in raw_pages for b in p["blocks"]]
        )

        for p in raw_pages:
            classify_blocks(p["blocks"], p["height"], p["width"],
                            repeated=repeated, image_boxes=p["image_boxes"],
                            base_size=doc_base)
            mark_advert_pages(p["blocks"])
            mark_pullquotes(p["blocks"])
            assign_reading_order(p["blocks"], p["width"])
            mark_sidebars(p["blocks"], p["width"])
            all_blocks.extend(p["blocks"])

        # Cross-page, so it runs once over every block rather than per page
        # (see cluster_running_furniture's docstring for why exact-match
        # find_repeated_lines above is not enough once OCR is involved).
        page_heights = {p["page"]: p["height"] for p in raw_pages}
        cluster_running_furniture(all_blocks, page_heights)

        for p in raw_pages:
            lines = [b.text for b in sorted(p["blocks"], key=lambda b: b.reading_order)]
            score = score_contents_page(lines)
            if score >= 0.45:
                contents_candidates.append({
                    "page": p["page"],
                    "score": score,
                    "entries": parse_contents(lines),
                })

        all_images = [img for p in raw_pages for img in p["images"]]
        for img in all_images:
            img["reading_order"] = image_reading_position(img, all_blocks)

        return {
            "page_count": doc.page_count,
            "pages": [{"page": p["page"], "width": p["width"], "height": p["height"],
                       **({"rotation": p["rotation"]} if p.get("rotation") else {})}
                      for p in raw_pages],
            "blocks": [b.to_json() for b in all_blocks],
            "images": all_images,
            "contents_candidates": contents_candidates,
            "vision_ocr_pages": vision_page_count,
        }
    finally:
        doc.close()


def _png_width(png: Path, fitz) -> float:
    try:
        return float(fitz.Pixmap(str(png)).width)
    except Exception:
        return 1.0


def _extract_image(fitz, doc, xref: int, images_dir: Path, name: str) -> bool:
    """Save one embedded image, skipping the tiny rules and spacers.

    Returns whether a file now exists at that name, so the caller can decide
    whether to record it as a real figure -- without this, a page's rules and
    spacers were silently skipped on disk but still implicitly "existed" as
    far as anything downstream knew, since nothing checked.
    """
    try:
        pix = fitz.Pixmap(doc, xref)
        if pix.width < 64 or pix.height < 64:
            return False
        if pix.n - pix.alpha >= 4:              # CMYK needs converting first
            pix = fitz.Pixmap(fitz.csRGB, pix)
        target = images_dir / f"{name}.png"
        if not target.exists():
            pix.save(str(target))
        return True
    except Exception:
        return False


# ── Images: linking a figure to the article it illustrates ──────────────────

def assign_images_to_articles(images: Sequence[dict[str, Any]],
                              blocks: Sequence[Block],
                              articles: Sequence[Article]) -> None:
    """Attach each extracted figure to the article whose column it falls in.

    A figure belongs to whichever article occupies its page; when two
    articles share a page (a feature ending, a new one starting lower down),
    the figure goes to whichever article has the nearer block by vertical
    position, so a photo is not shown under the wrong headline. Mutates
    articles in place.
    """
    by_page: dict[int, list[Article]] = {}
    for art in articles:
        for page in art.pages:
            by_page.setdefault(page, []).append(art)

    blocks_by_page: dict[int, list[Block]] = {}
    for b in blocks:
        blocks_by_page.setdefault(b.page, []).append(b)

    for img in images:
        if img.get("type") == BLOCK_AD:
            continue
        candidates = by_page.get(img["page"], [])
        if not candidates:
            continue
        if len(candidates) == 1:
            candidates[0].images.append(img["id"])
            continue
        # Several articles share this page: pick by nearest block vertically,
        # since that is what a reader would associate the figure with.
        img_y = img["bbox"][1]
        best, best_dist = candidates[0], float("inf")
        for art in candidates:
            own_blocks = [b for b in blocks_by_page.get(img["page"], [])
                         if b.id in set(art.blocks)]
            if not own_blocks:
                continue
            dist = min(abs(b.bbox[1] - img_y) for b in own_blocks)
            if dist < best_dist:
                best, best_dist = art, dist
        best.images.append(img["id"])


def image_reading_position(image: dict[str, Any], blocks: Sequence[Block]) -> int:
    """Approximate reading_order for an image, for ordering it among sentences.

    Uses the nearest block on the same page whose top sits at or above the
    image (so a photo is shown once the surrounding text has been reached,
    not before), falling back to the closest block on the page if the image
    sits above all of them.
    """
    same_page = [b for b in blocks if b.page == image["page"]]
    if not same_page:
        return 0
    above = [b for b in same_page if b.bbox[1] <= image["bbox"][1]]
    pool = above or same_page
    nearest = min(pool, key=lambda b: abs(b.bbox[1] - image["bbox"][1]))
    return nearest.reading_order


# ── Persistence ──────────────────────────────────────────────────────────────

def build_articles(layout: dict[str, Any]) -> list[dict[str, Any]]:
    """Turn a stored layout document into detected articles with estimates."""
    blocks = [Block(**b) for b in layout.get("blocks", [])]
    candidates = layout.get("contents_candidates") or []
    best = max(candidates, key=lambda c: c["score"], default=None)
    entries = best["entries"] if best else []

    # The contents page lists articles; it is not one.
    skip = {c["page"] for c in candidates if c.get("score", 0) >= 0.45}
    articles = group_articles(blocks, entries, layout.get("page_count", 0),
                              skip_pages=skip)
    assign_images_to_articles(layout.get("images") or [], blocks, articles)
    out = []
    for art in articles:
        rec = art.to_json()
        rec["estimated_seconds"] = estimate_listen_seconds(blocks, art)
        out.append(rec)
    return out


# ── Turning a page that was scanned sideways or upside down ───────────────────

ROTATIONS = (90, 180, 270)      # clockwise degrees a reader can ask for


def normalise_rotation(degrees: Any) -> int:
    """90, 180 or 270 (clockwise). -90 means 270; anything else is refused."""
    try:
        d = int(degrees) % 360
    except (TypeError, ValueError):
        raise ValueError("degrees must be 90, 180 or 270") from None
    if d not in ROTATIONS:
        raise ValueError("degrees must be 90, 180 or 270")
    return d


def rotate_bbox(bbox: Sequence[float], width: float, height: float, degrees: int) -> list[float]:
    """Where a [x, y, w, h] box on a page of width x height lands after the page is turned clockwise."""
    x, y, w, h = (float(v) for v in bbox)
    d = normalise_rotation(degrees)
    if d == 90:
        return [height - (y + h), x, h, w]
    if d == 180:
        return [width - (x + w), height - (y + h), w, h]
    return [y, width - (x + w), h, w]          # 270


def rotate_page_image(png: Path, degrees: int) -> None:
    """Turn a rendered page clockwise in place (written to a temporary file first)."""
    from PIL import Image
    d = normalise_rotation(degrees)
    op = {90: Image.Transpose.ROTATE_270, 180: Image.Transpose.ROTATE_180, 270: Image.Transpose.ROTATE_90}[d]
    tmp = png.with_suffix(".rot.tmp")
    try:
        with Image.open(png) as im:
            im.transpose(op).save(tmp, format="PNG")
        tmp.replace(png)
    finally:
        tmp.unlink(missing_ok=True)


def layout_with_rotated_page(layout: dict[str, Any], page_no: int, degrees: int) -> dict[str, Any]:
    """The layout with one page turned: its size swapped for a quarter turn, `rotation` accumulated, and the
    boxes already on it carried to where they are now, so whatever was approved still lines up with them."""
    d = normalise_rotation(degrees)
    meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if meta is None:
        raise ValueError("Page not found")
    w, h = float(meta["width"]), float(meta["height"])
    out = dict(layout)
    out["pages"] = [
        ({**p, "width": h, "height": w, "rotation": (int(p.get("rotation") or 0) + d) % 360} if d != 180
         else {**p, "rotation": (int(p.get("rotation") or 0) + d) % 360}) if p["page"] == page_no else p
        for p in layout.get("pages", [])]
    out["blocks"] = [{**b, "bbox": rotate_bbox(b["bbox"], w, h, d)} if b.get("page") == page_no else b
                     for b in layout.get("blocks", [])]
    out["images"] = [{**i, "bbox": rotate_bbox(i["bbox"], w, h, d)} if i.get("page") == page_no and i.get("bbox") else i
                     for i in layout.get("images", [])]
    return out


def _regions_text(regions: Sequence[dict[str, Any]] | None) -> str:
    return " ".join(str(r.get("text") or "") for r in regions or []).strip()


def _usable_reading(regions: Sequence[dict[str, Any]] | None) -> bool:
    text = _regions_text(regions)
    return len(text) >= 40 and ocr_quality_score(text) >= 0.55


def read_turned_page_regions(png: Path, original_png: bytes, degrees: int,
                             engines: Sequence[tuple[str | None, str | None]],
                             lang: str = "deu+eng", timeout: float = 150.0
                             ) -> tuple[list[dict[str, Any]] | None, str]:
    """Read a page that has just been turned, trying the ways that can work in order; returns (regions, how).

    A vision engine is not equally good at every orientation: on a real page scanned sideways the engine read
    the sideways image well (word-F1 0.88) and returned *nothing* for the same page turned upright, at every
    size it was tried. So when a turn is asked for:
      1. the turned page with the vision engines ("vision");
      2. the page as it was, with the boxes carried across into the turned page ("vision-original") --
         text read in the orientation the engine copes with, placed where it now is;
      3. tesseract on the turned page ("tesseract").
    The first reading that is long and plausible enough wins. If none is, the longest non-empty one is used
    rather than giving up. `how` is "empty" when tesseract ran and found no text at all (a picture page): an
    empty reading is then genuine, not a failure. (None, "failed") means nothing could read the page, so the
    caller must put the page back.
    """
    import shutil
    import tempfile
    from PIL import Image
    d = normalise_rotation(degrees)
    candidates: list[tuple[list[dict[str, Any]], str]] = []

    turned = vision_ocr_page_chain(png, list(engines), timeout=timeout)
    if _usable_reading(turned):
        return turned, "vision"
    if turned:
        candidates.append((turned, "vision"))

    with tempfile.TemporaryDirectory(prefix="turn-") as tmp:
        original = Path(tmp) / "original.png"
        original.write_bytes(original_png)
        with Image.open(original) as im:
            width_px, height_px = im.size
        regions = vision_ocr_page_chain(original, list(engines), timeout=timeout)
        if regions:
            carried = [{**r, "bbox": rotate_bbox(r["bbox"], width_px, height_px, d)} for r in regions if r.get("bbox")]
            if _usable_reading(carried):
                return carried, "vision-original"
            if carried:
                candidates.append((carried, "vision-original"))

    paragraphs = ocr_page(png, lang)
    local = [{"label": "text", "bbox": list(p["bbox"]), "text": p["text"]} for p in paragraphs if p.get("text")]
    if _usable_reading(local):
        return local, "tesseract"
    if local:
        candidates.append((local, "tesseract"))

    if candidates:
        return max(candidates, key=lambda c: len(_regions_text(c[0])))
    if shutil.which("tesseract"):
        return [], "empty"         # tesseract ran and saw no text: a picture page, not a failed read
    return None, "failed"


def reocr_page(doc_dir: Path, layout: dict[str, Any], page_no: int,
               vision_ocr_url: str, vision_ocr_model: str,
               vision_ocr_fallback_url: str | None = None,
               vision_ocr_fallback_model: str | None = None,
               timeout: float = 150.0,
               regions: list[dict[str, Any]] | None = None) -> dict[str, Any] | None:
    """Force one page through the vision OCR chain again with the engine the
    user picked, replacing only that page's blocks in an already-ingested
    layout.

    Unlike extract_pages()'s automatic escalation, this always calls the
    engine regardless of the page's existing text quality -- a user pressing
    "OCR again" has already judged the result unusable and is not asking for
    another quality gate. Deliberately does NOT call build_articles(): an
    article the user already reviewed and approved must not be silently
    re-grouped and lose that approval just because one of its pages was
    re-read. The caller decides whether to re-run detection separately.

    Returns the updated layout, or None if the page has no rendered PNG or
    the engine returned nothing usable (network failure, empty response) --
    the caller keeps the existing blocks in that case, never discards good
    data for a failed retry.
    """
    fitz = _require_fitz()
    png = doc_dir / "pages" / f"{page_no:04d}.png"
    if not png.is_file():
        return None

    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta:
        return None

    if regions is None:        # the caller did not read the page itself (a turned page is read by read_turned_page_regions)
        engines = [(vision_ocr_url, vision_ocr_model),
                   (vision_ocr_fallback_url, vision_ocr_fallback_model)]
        regions = vision_ocr_page_chain(png, engines, timeout=timeout)
        if not regions:
            return None
    repair_drop_cap_omissions(png, regions)

    scale_x = page_meta["width"] / max(1.0, _png_width(png, fitz))
    new_blocks = build_vision_blocks(regions, page_no, scale_x, id_prefix="r")
    snap_visual_bboxes(png, new_blocks, page_meta)

    all_blocks = [Block(**b) for b in layout.get("blocks", [])]
    kept = [b for b in all_blocks if b.page != page_no]

    # The baseline must come from the rest of the document, not this page
    # alone -- see classify_blocks' docstring on why a single page (which may
    # have no body copy of its own) cannot be its own reference.
    doc_base = body_font_size(kept) or body_font_size(all_blocks)
    classify_blocks(new_blocks, page_meta["height"], page_meta["width"], base_size=doc_base)
    mark_advert_pages(new_blocks)
    mark_pullquotes(new_blocks)
    assign_reading_order(new_blocks, page_meta["width"])
    mark_sidebars(new_blocks, page_meta["width"])

    merged = kept + new_blocks
    page_heights = {p["page"]: p["height"] for p in layout.get("pages", [])}
    cluster_running_furniture(merged, page_heights)

    layout = dict(layout)
    layout["blocks"] = [b.to_json() for b in merged]
    detected_images = crop_visual_regions(doc_dir, page_no, page_meta, new_blocks)
    if detected_images:
        layout["images"] = [i for i in layout.get("images", []) if i["page"] != page_no or "-manual-" in i["id"]] + detected_images
    layout["vision_ocr_pages"] = int(layout.get("vision_ocr_pages") or 0) + 1
    # The page's boxes are all new, so an order set by hand for the old ones no longer applies.
    layout["pages"] = [{k: v for k, v in p.items() if k != "order_locked"} if p["page"] == page_no else p
                       for p in layout.get("pages", [])]
    return layout


def _bbox_intersection_ratio(a: Sequence[float], b: Sequence[float]) -> float:
    overlap = (max(0.0, min(a[0] + a[2], b[0] + b[2]) - max(a[0], b[0]))
               * max(0.0, min(a[1] + a[3], b[1] + b[3]) - max(a[1], b[1])))
    area = max(1.0, float(a[2]) * float(a[3]))
    return overlap / area


def merge_missing_text_candidates(layout: dict[str, Any], page_no: int,
                                  candidates: Sequence[Block]) -> tuple[dict[str, Any], int]:
    """Add independently OCR'd text regions that the saved layout omitted.

    Candidates overlapping existing text or a known picture are rejected.
    New regions deliberately remain outside article selections so a reviewer
    sees them unchecked and decides whether they belong to the article.
    """
    existing = [Block(**b) for b in layout.get("blocks", []) if b["page"] == page_no]
    visuals = [b.bbox for b in existing if b.type in VISUAL_TYPES]
    visuals += [i["bbox"] for i in layout.get("images", []) if i["page"] == page_no]
    text_blocks = [b for b in existing if b.type not in VISUAL_TYPES]
    ids = {b["id"] for b in layout.get("blocks", [])}
    added: list[Block] = []
    for candidate in candidates:
        text = " ".join((candidate.text or "").split())
        if len(text) < 12 or ocr_quality_score(text) < 0.55:
            continue
        if any(_bbox_intersection_ratio(candidate.bbox, box) >= 0.6 for box in visuals):
            continue
        duplicate = False
        for saved in text_blocks + added:
            overlap = _bbox_intersection_ratio(candidate.bbox, saved.bbox)
            similarity = difflib.SequenceMatcher(
                None, text.casefold(), " ".join(saved.text.split()).casefold(), autojunk=False
            ).ratio()
            if overlap >= 0.25 or similarity >= 0.72:
                duplicate = True
                break
        if duplicate:
            continue
        number = 0
        while f"q{page_no:04d}-m{number:03d}" in ids:
            number += 1
        candidate.id = f"q{page_no:04d}-m{number:03d}"
        candidate.text = text
        candidate.source = "quality_missing_text"
        ids.add(candidate.id)
        added.append(candidate)
    if not added:
        return layout, 0
    updated = dict(layout)
    all_blocks = [Block(**b) for b in layout.get("blocks", [])] + added
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if page_meta and not page_meta.get("order_locked"):
        assign_reading_order([b for b in all_blocks if b.page == page_no], page_meta["width"])
    updated["blocks"] = [b.to_json() for b in all_blocks]
    return updated, len(added)


def recover_missing_text(doc_dir: Path, layout: dict[str, Any], page_no: int,
                         lang: str = "deu+eng") -> tuple[dict[str, Any], int]:
    """Use local OCR as an independent completeness check without replacing boxes."""
    fitz = _require_fitz()
    png = doc_dir / "pages" / f"{page_no:04d}.png"
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not png.is_file() or not page_meta:
        return layout, 0
    scale = page_meta["width"] / max(1.0, _png_width(png, fitz))
    candidates = [Block(
        id="", page=page_no, bbox=[float(v) * scale for v in region["bbox"]],
        type=BLOCK_BODY, text=region.get("text", ""),
        confidence=float(region.get("confidence") or 0.5),
        font_size=float(region.get("line_height") or 0.0) * scale,
        source="quality_missing_text",
    ) for region in ocr_page(png, lang)]
    base = body_font_size([Block(**b) for b in layout.get("blocks", [])])
    classify_blocks(candidates, page_meta["height"], page_meta["width"], base_size=base)
    return merge_missing_text_candidates(layout, page_no, candidates)


def verify_ocr_recognition(doc_dir: Path, layout: dict[str, Any], page_no: int,
                           vision_ocr_url: str | None = None,
                           vision_ocr_model: str | None = None,
                           vision_ocr_fallback_url: str | None = None,
                           vision_ocr_fallback_model: str | None = None,
                           timeout: float = 150.0) -> tuple[dict[str, Any], int, int]:
    """Re-read dictionary-suspicious regions and accept only a better image reading."""
    updated = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    checked = corrected = 0
    eligible = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE}
    # A fragment is too short to say which language it is in (``iPhones statt Token`` reads as
    # English); a page of body text is not.
    page_language = detect_article_language(b.get("text") or "" for b in blocks
                                            if b["page"] == page_no and b["type"] in eligible)
    page_vocabulary = recurring_tokens(b.get("text") or "" for b in blocks
                                       if b["page"] == page_no and b["type"] in eligible)
    for index, block in enumerate(blocks):
        if block["page"] != page_no or block["type"] not in eligible:
            continue
        old = " ".join((block.get("text") or "").split())
        old_suspects = suspect_ocr_tokens(old, page_language, page_vocabulary)
        if not old_suspects:
            continue
        checked += 1
        candidate = extract_region_text(
            doc_dir, updated, page_no, block["bbox"],
            vision_ocr_url, vision_ocr_model,
            vision_ocr_fallback_url, vision_ocr_fallback_model, timeout,
        )
        candidate = _join_detached_drop_cap(" ".join((candidate or "").split()))
        if not candidate or candidate == old:
            continue
        similarity = difflib.SequenceMatcher(None, old.casefold(), candidate.casefold(),
                                             autojunk=False).ratio()
        new_suspects = suspect_ocr_tokens(candidate, page_language, page_vocabulary)
        length_ratio = len(candidate) / max(1, len(old))
        if (similarity >= 0.55 and 0.6 <= length_ratio <= 1.6
                and len(new_suspects) < len(old_suspects)):
            blocks[index]["text"] = candidate
            corrected += 1
    updated["blocks"] = blocks
    return updated, checked, corrected


_AUDIT_MIN_CELL_INK = 0.08       # a text-bearing cell is inked well below a photo's; this still
                                  # ignores paper grain and scanner noise
_AUDIT_MIN_CELLS = 10            # ~one short word line at the 10 px grid; smaller specks are dust
_AUDIT_OVERLAP_RATIO = 0.3       # two text boxes sharing this much of the smaller one's area
_AUDIT_CLIP_INK = 0.10           # ink density in the cell column just outside a box's side edge:
                                  # text running past the box means its first/last letters are cut
_AUDIT_EDGE_FRACTION = 0.03      # scan borders / spine shadows live here and are never article text


_AUDIT_CLIP_MAX_INK = 0.7        # above this the strip is a picture or a filled panel, not text
_AUDIT_GRAPHIC_MARGIN = 10.0    # points: unboxed ink this close to a picture/chart/ad belongs to it
_AUDIT_CLIP_STRIP_PX = 14      # how far outside a box edge to look for ink that belongs to it (~1-2 letters)


_RULE_MAX_INK_ROWS = 6          # pixel rows with ink: a rule or dotted leader is a few, a text line is 8+


def _is_rule_only(pixels, blob: Sequence[float], page_meta: dict[str, Any]) -> bool:
    """True when the ink inside `blob` is only a thin rule or a dotted leader (the dots between a
    table-of-contents entry and its page number), which has no letters and nothing to narrate."""
    height, width = pixels.shape
    sx, sy = width / max(1.0, float(page_meta["width"])), height / max(1.0, float(page_meta["height"]))
    x, y, w, h = blob
    region = pixels[max(0, int(y * sy)):int((y + h) * sy) + 1, max(0, int(x * sx)):int((x + w) * sx) + 1]
    if region.size == 0:
        return False
    return int((region.sum(axis=1) >= 3).sum()) <= _RULE_MAX_INK_ROWS


_LINE_MIN_LONG_PX, _LINE_MAX_THIN_PX = 45, 8          # a thin, long connected shape is a rule
_FRAME_MIN_LONG_PX, _FRAME_MAX_FILL = 90, 0.2         # a big, mostly hollow shape is a frame or a scribble
_GLYPH_MIN_PX, _GLYPH_MAX_PX, _GLYPH_MIN_INK = 4, 80, 6
_MIN_GLYPHS, _MIN_GLYPH_SHARE = 6, 0.25
_PICTURE_MIN_SIDE_PX, _PICTURE_MIN_FILL = 20, 0.5


def _is_lines_or_frames_only(pixels, blob: Sequence[float], page_meta: dict[str, Any]) -> bool:
    """True when the ink inside `blob` is rules, frames, chart axes or a signature rather than text.

    Text is many small separate marks (letters) sitting in rows. Rules, table and chart lines,
    screenshot outlines and a signature are a few large connected shapes: long and thin, or big and
    mostly hollow. The blob is dropped when fewer than six letter-sized marks are left, or they hold
    under a quarter of the ink. Framed *text* is kept: its letters are many small marks.
    """
    try:
        import numpy as np
        from scipy import ndimage
    except Exception:
        return False
    height, width = pixels.shape
    sx, sy = width / max(1.0, float(page_meta["width"])), height / max(1.0, float(page_meta["height"]))
    x, y, w, h = blob
    region = pixels[max(0, int(y * sy)):int((y + h) * sy) + 1, max(0, int(x * sx)):int((x + w) * sx) + 1]
    if region.size == 0:
        return False
    labels, count = ndimage.label(region, structure=np.ones((3, 3), dtype=int))
    if count == 0:
        return True
    sizes = np.bincount(labels.ravel())[1:]
    glyphs = glyph_ink = total = 0
    for index, sl in enumerate(ndimage.find_objects(labels), 1):
        ink = int(sizes[index - 1])
        if ink < _GLYPH_MIN_INK:
            continue
        box_h, box_w = sl[0].stop - sl[0].start, sl[1].stop - sl[1].start
        long_side, thin_side = max(box_h, box_w), min(box_h, box_w)
        total += ink
        if thin_side >= _PICTURE_MIN_SIDE_PX and ink / float(box_h * box_w) >= _PICTURE_MIN_FILL:
            return False                 # a solid picture: the figure logic decides, not this filter
        is_line = (long_side >= _LINE_MIN_LONG_PX and thin_side <= _LINE_MAX_THIN_PX) or (
            long_side >= _FRAME_MIN_LONG_PX and ink / float(box_h * box_w) < _FRAME_MAX_FILL)
        if not is_line and _GLYPH_MIN_PX <= long_side <= _GLYPH_MAX_PX:
            glyphs += 1
            glyph_ink += ink
    return total == 0 or glyphs < _MIN_GLYPHS or glyph_ink / float(total) < _MIN_GLYPH_SHARE


def _clipped_edges(pixels, page_meta: dict[str, Any], text_blocks: list[dict[str, Any]],
                   boxes: list[Sequence[float]]) -> list[dict[str, Any]]:
    """Text boxes whose edge cuts through text: ink in a thin strip just outside a side, ignoring
    whatever another box already covers (a neighbouring column is not clipping)."""
    import numpy as np
    height, width = pixels.shape
    sx, sy = width / max(1.0, float(page_meta["width"])), height / max(1.0, float(page_meta["height"]))
    covered = np.zeros_like(pixels)
    for x, y, w, h in boxes:
        covered[max(0, int(y * sy)):int((y + h) * sy) + 1, max(0, int(x * sx)):int((x + w) * sx) + 1] = True
    found = []
    for block in text_blocks:
        x, y, w, h = block["bbox"]
        left, right = int(x * sx), int((x + w) * sx) + 1
        top, bottom = int(y * sy), int((y + h) * sy) + 1
        strips = {"left": (top, bottom, left - _AUDIT_CLIP_STRIP_PX, left),
                  "right": (top, bottom, right, right + _AUDIT_CLIP_STRIP_PX),
                  "top": (top - _AUDIT_CLIP_STRIP_PX, top, left, right),
                  "bottom": (bottom, bottom + _AUDIT_CLIP_STRIP_PX, left, right)}
        for side, (r0, r1, c0, c1) in strips.items():
            r0, c0, r1, c1 = max(0, r0), max(0, c0), min(height, r1), min(width, c1)
            if r1 <= r0 or c1 <= c0:
                continue
            free = ~covered[r0:r1, c0:c1]
            if free.sum() < 40:
                continue
            ink_share = (pixels[r0:r1, c0:c1] & free).sum() / free.sum()
            # Text is ink between the lines; a strip that is (nearly) solid ink is a photo or a
            # filled panel behind the headline, not letters cut off by the box.
            if not (_AUDIT_CLIP_INK <= ink_share <= _AUDIT_CLIP_MAX_INK):
                continue
            if not _strip_continues_the_text(pixels, side, (r0, r1, c0, c1), free, (top, bottom, left, right)):
                continue
            found.append({"id": block["id"], "side": side})
    return found


def _strip_continues_the_text(pixels, side: str, strip: tuple[int, int, int, int], free,
                              box: tuple[int, int, int, int]) -> bool:
    """Whether the ink in a strip beside a box looks like the box's own text running on.

    Letters cut off by a box edge sit on the same text lines as the box's own text, so beside a
    left/right edge the strip's inked pixel rows fall on rows where the box itself has ink; and the
    ink is intermittent (letters and gaps), not a line. A screenshot frame, a chart border or the
    edge of a photo is a solid vertical line or does not follow the box's lines. Above/below, a
    thin horizontal rule is not a cut line. A box with no ink of its own cannot be judged and is
    kept as reported.
    """
    r0, r1, c0, c1 = strip
    top, bottom, left, right = box
    ink = pixels[r0:r1, c0:c1] & free
    if side in ("left", "right"):
        inner = pixels[max(0, top):bottom, max(0, left):right]
        strip_rows = ink.any(axis=1)
        if inner.size == 0 or not inner.any() or not strip_rows.any():
            return True
        inner_rows = inner.any(axis=1)
        rows = min(len(strip_rows), len(inner_rows))
        strip_rows, inner_rows = strip_rows[:rows], inner_rows[:rows]
        on_lines = float((strip_rows & inner_rows).sum()) / float(strip_rows.sum())
        solid = float(strip_rows.sum()) / max(1, rows)
        return on_lines >= 0.75 and solid <= 0.9
    rows_inked = int(ink.any(axis=1).sum())
    coverage = float(ink.any(axis=0).sum()) / max(1, ink.shape[1])
    return not (rows_inked <= 3 and coverage >= 0.7)


_OVERLAP_RANK = {BLOCK_TITLE: 0, BLOCK_HEADING: 0, BLOCK_DECK: 1, BLOCK_BYLINE: 1, BLOCK_BODY: 2,
                 BLOCK_QUOTE: 2, BLOCK_PULLQUOTE: 3, BLOCK_CAPTION: 3, BLOCK_SIDEBAR: 3}
_OVERLAP_MIN_KEEP = 0.4      # never trim a box down to less than this share of its own area


def resolve_box_overlaps(layout: dict[str, Any], pages: Iterable[int] | None = None) -> tuple[dict[str, Any], list[str]]:
    """Trim boxes of lower rank where they run over a higher-ranked one.

    A caption or sidebar drawn over a headline, or a headline box over body text, covers the same
    printed area twice. The box of lower rank (headline > deck/byline > body > caption/sidebar/
    pull quote) is cut back to the largest part that no longer overlaps the higher one -- a
    rectangle above, below, left or right of it -- unless that would leave under 40% of it, in
    which case both stay for a human (the audit still reports the pair). Boxes of equal rank, hand
    placed boxes and anything not text are left alone; only geometry changes. Returns the layout
    and the ids of the trimmed boxes.
    """
    wanted = set(pages) if pages is not None else None
    layout = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    trimmed: list[str] = []
    by_page: dict[int, list[dict[str, Any]]] = {}
    for b in blocks:
        if b["type"] in _OVERLAP_RANK and (wanted is None or b["page"] in wanted):
            by_page.setdefault(b["page"], []).append(b)
    for page_blocks in by_page.values():
        for i, a in enumerate(page_blocks):
            for b in page_blocks[i + 1:]:
                ra, rb = _OVERLAP_RANK[a["type"]], _OVERLAP_RANK[b["type"]]
                if ra == rb:
                    continue
                low, high = (a, b) if ra > rb else (b, a)
                if low.get("box_manual") or low.get("source") == "manual":
                    continue
                if (low["type"] == BLOCK_PULLQUOTE and low.get("source") == "inset_pullquote"
                        and _text_found_in(low.get("text") or "", high.get("text") or "") >= 0.8
                        and _bbox_intersection_ratio(low["bbox"], high["bbox"]) >= 0.5):
                    # the detector found a quote the layout engine had already read as text
                    low.update(type=BLOCK_FURNITURE, region_label="duplicate", confidence=0.9)
                    trimmed.append(low["id"])
                    continue
                if max(_bbox_intersection_ratio(low["bbox"], high["bbox"]),
                       _bbox_intersection_ratio(high["bbox"], low["bbox"])) < _AUDIT_OVERLAP_RATIO:
                    continue
                x, y, w, h = low["bbox"]
                hx, hy, hw, hh = high["bbox"]
                ix0, iy0 = max(x, hx), max(y, hy)
                ix1, iy1 = min(x + w, hx + hw), min(y + h, hy + hh)
                if ix1 <= ix0 or iy1 <= iy0:
                    continue
                options = [[x, y, w, iy0 - y], [x, iy1, w, y + h - iy1],
                           [x, y, ix0 - x, h], [ix1, y, x + w - ix1, h]]
                best = max(options, key=lambda o: max(0.0, o[2]) * max(0.0, o[3]))
                if best[2] < 6 or best[3] < 6 or best[2] * best[3] < _OVERLAP_MIN_KEEP * w * h:
                    continue
                low["bbox"] = [round(v, 1) for v in best]
                low["box_snapped"] = True
                if low["id"] not in trimmed:
                    trimmed.append(low["id"])
    layout["blocks"] = blocks
    return layout, trimmed


_MIN_TEXT_BOX_HEIGHT = 3.0    # points: a text box thinner than this holds no line of text
_SNAP_TEXT_STEP_PX = 2        # grow/shrink granularity
_SNAP_TEXT_MAX_GROW_PX = 16   # never grow a side sideways further than about a letter or two
_SNAP_TEXT_MAX_GROW_TITLE_PX = 70   # headlines, decks: large type, the last word often sticks out
_SNAP_TEXT_MAX_GROW_DOWN_PX = 90   # a paragraph cut short at the top/bottom can be a few lines
_SNAP_TEXT_LINE_GAP_PX = 14   # blank pixels between two text lines that still count as one block
_SNAP_TEXT_PAD_PX = 2         # air left between the text and its snapped edge
_SNAP_TEXT_MIN_PX = 8         # a snapped box keeps at least this size
_SNAP_TEXT_MIN_MOVE_PX = 3    # smaller corrections are not worth changing a box for
_SNAP_TEXT_LINE_INK = 0.004   # a pixel line with less ink than this share is empty margin


def snap_text_boxes(doc_dir: Path, layout: dict[str, Any],
                    pages: Iterable[int] | None = None) -> tuple[dict[str, Any], list[dict[str, Any]]]:
    """Fit each text box to the ink of its own text, using the page render.

    Two corrections, both bounded: a side that cuts through letters (ink continues just outside
    it, where no other box is) grows until the text ends, by at most a letter or two; empty
    margin inside a side is trimmed back to the first line with ink, leaving a little air. A
    box the reviewer drew or resized by hand is never touched, and neither is one that no page
    render exists for. Geometry only -- the text is not changed. Returns the layout and one
    report row per box that moved.
    """
    try:
        import numpy as np
    except Exception:
        return layout, []
    wanted = set(pages) if pages is not None else None
    snappable = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE}
    layout = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    moved: list[dict[str, Any]] = []
    for meta in layout.get("pages", []):
        page_no = meta["page"]
        if wanted is not None and page_no not in wanted:
            continue
        png = doc_dir / "pages" / f"{page_no:04d}.png"
        if not png.is_file():
            continue
        pixels = page_ink_mask(png)
        if pixels is None:
            continue
        height, width = pixels.shape
        sx, sy = width / max(1.0, float(meta["width"])), height / max(1.0, float(meta["height"]))
        on_page = [b for b in blocks if b["page"] == page_no]
        images = [i for i in layout.get("images", []) if i["page"] == page_no]

        def to_px(bbox: Sequence[float]) -> list[int]:
            x, y, w, h = bbox
            return [max(0, int(round(x * sx))), max(0, int(round(y * sy))),
                    min(width, int(round((x + w) * sx))), min(height, int(round((y + h) * sy)))]

        for block in on_page:
            if (block["type"] not in snappable or block.get("source") == "manual"
                    or block.get("box_manual")):
                continue
            left, top, right, bottom = to_px(block["bbox"])
            if right - left < _SNAP_TEXT_MIN_PX or bottom - top < _SNAP_TEXT_MIN_PX:
                continue
            others = np.zeros_like(pixels)
            for other in on_page:
                if other is not block:
                    ol, ot, orr, ob = to_px(other["bbox"])
                    others[ot:ob, ol:orr] = True
            for image in images:
                il, it, ir, ib = to_px(image["bbox"])
                others[it:ib, il:ir] = True
            original = (left, top, right, bottom)

            def strip_has_text(r0: int, r1: int, c0: int, c1: int) -> bool:
                r0, c0, r1, c1 = max(0, r0), max(0, c0), min(height, r1), min(width, c1)
                if r1 <= r0 or c1 <= c0:
                    return False
                free = ~others[r0:r1, c0:c1]
                return bool(free.sum() >= 12 and (pixels[r0:r1, c0:c1] & free).sum() / free.sum() >= _AUDIT_CLIP_INK)

            def blocked(r0: int, r1: int, c0: int, c1: int) -> bool:
                """The strip a side would grow into already belongs to another box."""
                r0, c0, r1, c1 = max(0, r0), max(0, c0), min(height, r1), min(width, c1)
                return bool(r1 > r0 and c1 > c0 and others[r0:r1, c0:c1].any())

            step = _SNAP_TEXT_STEP_PX
            gap = _SNAP_TEXT_LINE_GAP_PX
            side = (_SNAP_TEXT_MAX_GROW_TITLE_PX if block["type"] in (BLOCK_TITLE, BLOCK_HEADING, BLOCK_DECK)
                    else _SNAP_TEXT_MAX_GROW_PX)          # display type runs past a box by more
            budget = {"left": side, "right": side,
                      "top": _SNAP_TEXT_MAX_GROW_DOWN_PX, "bottom": _SNAP_TEXT_MAX_GROW_DOWN_PX}
            while True:            # each side has its own allowance; stop when none can grow
                grew = False
                if budget["left"] > 0 and left - step >= 0 and not blocked(top, bottom, left - step, left) and strip_has_text(top, bottom, left - step, left):
                    left -= step; budget["left"] -= step; grew = True
                if budget["right"] > 0 and right + step <= width and not blocked(top, bottom, right, right + step) and strip_has_text(top, bottom, right, right + step):
                    right += step; budget["right"] -= step; grew = True
                if budget["top"] > 0 and top - step >= 0 and not blocked(top - step, top, left, right) and strip_has_text(top - gap, top, left, right):
                    top -= step; budget["top"] -= step; grew = True
                if budget["bottom"] > 0 and bottom + step <= height and not blocked(bottom, bottom + step, left, right) and strip_has_text(bottom, bottom + gap, left, right):
                    bottom += step; budget["bottom"] -= step; grew = True
                if not grew:
                    break
            if (left, top, right, bottom) != original:
                # a little air past the last inked line, so descenders are not touching the edge
                pad = _SNAP_TEXT_PAD_PX
                if left - pad >= 0 and not blocked(top, bottom, left - pad, left):
                    left -= pad
                if right + pad <= width and not blocked(top, bottom, right, right + pad):
                    right += pad
                if top - pad >= 0 and not blocked(top - pad, top, left, right):
                    top -= pad
                if bottom + pad <= height and not blocked(bottom, bottom + pad, left, right):
                    bottom += pad
                original_grown = True
            else:
                original_grown = False
            if not original_grown:
                # Not clipped: trim the empty margins instead. A line is empty margin when it
                # holds (almost) no ink of its own.
                def line_empty(r0: int, r1: int, c0: int, c1: int) -> bool:
                    area = max(1, (r1 - r0) * (c1 - c0))
                    return (pixels[r0:r1, c0:c1].sum() / area) <= _SNAP_TEXT_LINE_INK

                while right - left - step >= _SNAP_TEXT_MIN_PX and line_empty(top, bottom, left, left + step):
                    left += step
                while right - left - step >= _SNAP_TEXT_MIN_PX and line_empty(top, bottom, right - step, right):
                    right -= step
                while bottom - top - step >= _SNAP_TEXT_MIN_PX and line_empty(top, top + step, left, right):
                    top += step
                while bottom - top - step >= _SNAP_TEXT_MIN_PX and line_empty(bottom - step, bottom, left, right):
                    bottom -= step
                if (left, top, right, bottom) != original:
                    left, top = max(0, left - _SNAP_TEXT_PAD_PX), max(0, top - _SNAP_TEXT_PAD_PX)
                    right, bottom = min(width, right + _SNAP_TEXT_PAD_PX), min(height, bottom + _SNAP_TEXT_PAD_PX)
            grown_down = bottom - original[3]
            moves = [abs(a - b) for a, b in zip((left, top, right, bottom), original)]
            if max(moves) < _SNAP_TEXT_MIN_MOVE_PX:
                continue
            block["bbox"] = [round(left / sx, 1), round(top / sy, 1),
                             round((right - left) / sx, 1), round((bottom - top) / sy, 1)]
            block["box_snapped"] = True
            moved.append({"page": page_no, "id": block["id"],
                          "grew": (left, top) < original[:2] or (right, bottom) > original[2:],
                          # lines were added under/over the box: its text was probably cut short
                          "lines_added": grown_down > 8 or original[1] - top > 8})
    layout["blocks"] = blocks
    return layout, moved


_PULLQUOTE_MIN_W, _PULLQUOTE_MIN_H = 55.0, 28.0      # points
_PULLQUOTE_MAX_W, _PULLQUOTE_MAX_H = 230.0, 100.0
_PULLQUOTE_MAX_INK = 0.25       # share of the box that is coloured ink: text, not a filled chart
_PULLQUOTE_PAPER = 0.6          # share of the box that is plain paper: a quote sits on the page, a photo/ad does not
_PULLQUOTE_COVERED = 0.3        # a blob this covered by a headline/figure box is that, not a quote


def detect_inset_pullquotes(doc_dir: Path, layout: dict[str, Any], pages: Iterable[int] | None = None,
                            vision_ocr_url: str | None = None, vision_ocr_model: str | None = None,
                            vision_ocr_fallback_url: str | None = None,
                            vision_ocr_fallback_model: str | None = None) -> tuple[dict[str, Any], list[str]]:
    """Find a coloured display quote inset into the text and give it a pull-quote region of its own.

    Detection layouts wrap body text around such an inset, so no box covers it and the body
    paragraph's rectangle runs over it. The quote is printed in a display colour (teal on the
    measured issue) unlike the grey/black body text: a dense-enough blob of that colour, of
    quote size, that no headline, figure or existing pull quote already covers. The new region is
    typed pull quote (never read aloud); its text is read from the crop when a vision engine is
    available. Returns the layout and the new region ids.
    """
    try:
        import numpy as np
        from PIL import Image
        from scipy import ndimage
    except Exception:
        return layout, []
    wanted = set(pages) if pages is not None else None
    layout = dict(layout)
    blocks = list(layout.get("blocks", []))
    added: list[str] = []
    counter = 0
    for meta in layout.get("pages", []):
        page_no = meta["page"]
        if wanted is not None and page_no not in wanted:
            continue
        png = doc_dir / "pages" / f"{page_no:04d}.png"
        if not png.is_file() or page_is_advertisement({**layout, "blocks": blocks}, page_no):
            continue
        try:
            with Image.open(png) as im:
                rgb = np.asarray(im.convert("RGB"), dtype=np.int16)
        except Exception:
            continue
        height, width, _ = rgb.shape
        sx, sy = width / max(1.0, float(meta["width"])), height / max(1.0, float(meta["height"]))
        r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2]
        coloured = (g - r > 12) & (b - r > 12) & (rgb.max(axis=2) < 190)
        if coloured.sum() < 400:
            continue
        labels, _ = ndimage.label(ndimage.binary_dilation(coloured, iterations=10))
        cover = [bb["bbox"] for bb in blocks if bb["page"] == page_no and bb["type"] in (
            BLOCK_TITLE, BLOCK_HEADING, BLOCK_DECK, BLOCK_BYLINE, BLOCK_PULLQUOTE, BLOCK_SIDEBAR,
            BLOCK_CAPTION, *VISUAL_TYPES)]
        cover += [im_["bbox"] for im_ in layout.get("images", []) if im_["page"] == page_no]
        found = 0
        for index, sl in enumerate(ndimage.find_objects(labels), 1):
            ys, xs = sl
            w_pt, h_pt = (xs.stop - xs.start) / sx, (ys.stop - ys.start) / sy
            ink = int((coloured[sl] & (labels[sl] == index)).sum())
            if ink < 400 or not (_PULLQUOTE_MIN_W <= w_pt <= _PULLQUOTE_MAX_W
                                 and _PULLQUOTE_MIN_H <= h_pt <= _PULLQUOTE_MAX_H):
                continue
            if ink / max(1, (xs.stop - xs.start) * (ys.stop - ys.start)) > _PULLQUOTE_MAX_INK:
                continue
            region = rgb[sl]
            paper = (region.max(axis=2) - region.min(axis=2) < 28) & (region.max(axis=2) > 205)
            if paper.mean() < _PULLQUOTE_PAPER:
                continue
            # A quote is set in one muted display colour; an illustration or infographic on a pale
            # ground also has strongly saturated areas (red/orange/violet), a quote has none.
            if ((region.max(axis=2) - region.min(axis=2)) > 70).mean() > 0.01:
                continue
            box = [xs.start / sx, ys.start / sy, w_pt, h_pt]
            if any(_area_inside(box, c) >= _PULLQUOTE_COVERED for c in cover):
                continue
            found += 1
            if found > 2:
                break
            counter += 1
            text = ""
            if vision_ocr_url and vision_ocr_model:
                try:
                    text = " ".join((extract_region_text(doc_dir, layout, page_no, box, vision_ocr_url,
                                                         vision_ocr_model, vision_ocr_fallback_url,
                                                         vision_ocr_fallback_model) or "").split())
                except Exception:
                    text = ""
            new_id = f"pq{page_no:04d}-{counter}"
            blocks.append(Block(id=new_id, page=page_no, bbox=[round(v, 1) for v in box],
                                type=BLOCK_PULLQUOTE, text=text or "(no text recognised: inset quote)",
                                confidence=0.6, source="inset_pullquote", region_label="text").to_json())
            added.append(new_id)
    layout["blocks"] = blocks
    return layout, added


_PICTURE_DENSITY = 0.35     # inked share of an unboxed blob above which it is a picture, not text
_PICTURE_MIN_PT = 30.0


def add_uncovered_pictures(doc_dir: Path, layout: dict[str, Any], page_no: int) -> tuple[dict[str, Any], list[str]]:
    """Give a figure region to each photo or graphic that no box covers.

    The page audit lists ink outside every box. A dense blob of a real size is a picture the
    layout model missed (text is thin and line-structured, ink share well under a third), so it
    becomes a figure -- cropped for playback, and no longer reported as missing text. Returns the
    layout and the new image ids. Idempotent: a covered picture is not found again.
    """
    audit = audit_page_coverage(doc_dir, layout, page_no)
    if not audit.get("uncovered"):
        return layout, []
    meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    ink = page_ink_mask(doc_dir / "pages" / f"{page_no:04d}.png")
    if meta is None or ink is None:
        return layout, []
    height, width = ink.shape
    sx, sy = width / max(1.0, float(meta["width"])), height / max(1.0, float(meta["height"]))
    added: list[str] = []
    for x, y, w, h in audit["uncovered"]:
        if w < _PICTURE_MIN_PT or h < _PICTURE_MIN_PT:
            continue
        region = ink[int(y * sy):int((y + h) * sy), int(x * sx):int((x + w) * sx)]
        if region.size == 0 or float(region.mean()) < _PICTURE_DENSITY:
            continue
        before = {i["id"] for i in layout.get("images", [])}
        layout = add_manual_region(doc_dir, layout, page_no, [x, y, w, h], "image")
        added += [i["id"] for i in layout.get("images", []) if i["id"] not in before]
    return layout, added


def split_around_pullquotes(doc_dir: Path, layout: dict[str, Any],
                            pages: Iterable[int] | None = None) -> tuple[dict[str, Any], list[dict[str, str]]]:
    """Cut a paragraph's box where its text wraps around an inset pull quote.

    One rectangle cannot follow text that runs full width above a quote and in a narrower
    column beside it, so the box would sit over the quote. The paragraph is measured from the
    page render: its text lines are found, lines that end left of the quote (or sit beside it)
    are 'narrow', the rest 'full'. The box and the text are cut between the two runs, the text
    in proportion to the ink on each side and only at a space. A paragraph whose every line is
    narrow just gets its box narrowed. Returns the layout and one {id, new_id} row per new
    piece, in reading order, for the caller to add to the articles.
    """
    try:
        import numpy as np
    except Exception:
        return layout, []
    wanted = set(pages) if pages is not None else None
    layout = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    rows: list[dict[str, str]] = []
    for meta in layout.get("pages", []):
        page_no = meta["page"]
        if wanted is not None and page_no not in wanted:
            continue
        quotes = [b for b in blocks if b["page"] == page_no and b["type"] == BLOCK_PULLQUOTE]
        png = doc_dir / "pages" / f"{page_no:04d}.png"
        if not quotes or not png.is_file():
            continue
        ink = page_ink_mask(png)
        if ink is None:
            continue
        height, width = ink.shape
        sx, sy = width / max(1.0, float(meta["width"])), height / max(1.0, float(meta["height"]))
        for quote in quotes:
            qx, qy, qw, qh = quote["bbox"]
            q_left, q_top, q_bottom = int(qx * sx), int(qy * sy), int((qy + qh) * sy)
            q_right = int((qx + qw) * sx)
            for block in [b for b in blocks if b["page"] == page_no and b["type"] == BLOCK_BODY
                          and not b.get("box_manual") and (b.get("text") or "").strip()]:
                if _area_inside(quote["bbox"], block["bbox"]) < 0.15:
                    continue
                bx, by, bw, bh = block["bbox"]
                left, top = max(0, int(bx * sx)), max(0, int(by * sy))
                right, bottom = min(width, int((bx + bw) * sx)), min(height, int((by + bh) * sy))
                if q_left <= left + 20 or right - left < 40:
                    continue
                area = ink[top:bottom, left:right].copy()
                area[max(0, q_top - top):max(0, q_bottom - top), max(0, q_left - left):max(0, q_right - left)] = False
                profile = area.sum(axis=1)
                # Relative to this paragraph's own ink density: set-tight lines and scan
                # bleed-through leave no row truly empty, only clearly lighter.
                threshold = max(2.0, 0.3 * float(np.percentile(profile, 90)))
                bands, start, gap = [], None, 0
                for y, count in enumerate(profile):
                    if count > threshold:
                        if start is None:
                            start = y
                        gap = 0
                    elif start is not None:
                        gap += 1
                        if gap > 1:
                            bands.append((start, y - gap))
                            start, gap = None, 0
                if start is not None:
                    bands.append((start, len(profile) - 1 - gap))
                bands = [(a, b) for a, b in bands if b - a >= 4]
                if len(bands) < 3:
                    continue
                # A line is narrow when it ends before the quote's left edge and sits at or just above
                # the quote (text starts wrapping a line before the quote's own box does).
                line_height = float(np.median([b - a + 1 for a, b in bands]))
                labels = []
                for a, b in bands:
                    cols = np.nonzero(area[a:b + 1].any(axis=0))[0]
                    extent = int(cols.max()) + left if len(cols) else left
                    beside = top + b >= q_top - 2.2 * line_height and top + a < q_bottom + 2
                    after = top + a >= q_bottom - 2
                    if extent <= q_left + 6 and (beside or after):
                        labels.append("narrow")
                    elif after and labels and labels[-1] == "narrow" and extent <= q_left + 6 + 0.5 * (right - q_left):
                        labels.append("narrow")
                    else:
                        labels.append("full")
                if "narrow" not in labels:
                    continue
                # Characters per line follow the line's printed width (its left-to-right extent),
                # which is steadier than its ink, so the text is cut where the lines are.
                weights = []
                for a, b in bands:
                    cols = np.nonzero(area[a:b + 1].any(axis=0))[0]
                    weights.append(max(1, int(cols.max() - cols.min() + 1)) if len(cols) else 1)
                runs: list[list[int]] = []                 # [label index run start, end)
                for i, label in enumerate(labels):
                    if runs and labels[runs[-1][0]] == label:
                        runs[-1][1] = i + 1
                    else:
                        runs.append([i, i + 1])
                text = " ".join(block["text"].split())
                total = float(sum(weights))
                cuts, acc = [], 0.0
                for run in runs[:-1]:
                    acc += sum(weights[run[0]:run[1]])
                    at = int(round(acc / total * len(text)))
                    space = min((i for i in range(len(text)) if text[i] == " "),
                                key=lambda i: abs(i - at), default=None)
                    if space is None:
                        cuts = []
                        break
                    cuts.append(space)
                if runs and (len(cuts) != len(runs) - 1 or cuts != sorted(set(cuts)) or (cuts and (cuts[0] <= 0 or cuts[-1] >= len(text) - 1))):
                    continue
                pieces = []
                previous = 0
                for k, run in enumerate(runs):
                    end = cuts[k] if k < len(cuts) else len(text)
                    y0 = top + bands[run[0]][0] - 2
                    y1 = top + bands[run[1] - 1][1] + 3
                    narrow = labels[run[0]] == "narrow"
                    x1 = min(right, q_left - 3) if narrow else right
                    pieces.append({"text": text[previous:end].strip(),
                                   "bbox": [round(left / sx, 1), round(max(0, y0) / sy, 1),
                                            round((x1 - left) / sx, 1), round((min(height, y1) - max(0, y0)) / sy, 1)]})
                    previous = end
                if any(not piece["text"] for piece in pieces):
                    continue
                taken = {b["id"] for b in blocks}
                serial = 0

                def new_id() -> str:
                    nonlocal serial
                    while f"t{page_no:04d}-{serial:03d}" in taken:
                        serial += 1
                    taken.add(f"t{page_no:04d}-{serial:03d}")
                    return f"t{page_no:04d}-{serial:03d}"

                built = [dict(block, text=pieces[0]["text"], bbox=pieces[0]["bbox"], box_snapped=False)]
                for piece in pieces[1:]:
                    extra = {**block, "id": new_id(), "text": piece["text"], "bbox": piece["bbox"],
                             "reading_order": block.get("reading_order", 0) + 0.5}
                    extra.pop("original_text", None)
                    built.append(extra)
                for a_, b_ in zip(built, built[1:]):
                    rows.append({"id": a_["id"], "new_id": b_["id"]})
                blocks = [x for b in blocks for x in (built if b["id"] == block["id"] else [b])]
    if rows:
        touched = {b["page"] for b in blocks if any(b["id"] == r["new_id"] for r in rows)}
        for page_no in touched:
            page = sorted((b for b in blocks if b["page"] == page_no), key=lambda b: b.get("reading_order", 0))
            position = {b["id"]: i for i, b in enumerate(page)}
            blocks = [dict(b, reading_order=position[b["id"]]) if b["page"] == page_no else b for b in blocks]
    layout["blocks"] = blocks
    return layout, rows


_SEAM_WINDOW_PX = 16          # how far around a shared boundary to look for the gap between two lines
_SEAM_MIN_SIDE_PX = 10        # each box keeps at least this much height after a seam moves
_SEAM_STRIP_MIN_PX = 6        # a box that gained less than this needs no text read for the new strip


def mend_column_seams(doc_dir: Path, layout: dict[str, Any],
                      pages: Iterable[int] | None = None) -> tuple[dict[str, Any], list[dict[str, Any]]]:
    """Make vertically stacked boxes of one column meet at the white gap between two text lines.

    Detection often cuts a column into slices that overlap or leave a line in the seam: the
    boundary runs through a line of text, so that line is in neither box (both report a clipped
    edge, and neither can grow because the other is in the way). For each pair of neighbouring
    boxes whose shared boundary overlaps or cuts through ink, the boundary moves to the nearest
    gap of at least two blank pixel rows -- down (to the box below's top) when the upper paragraph
    is still running, up otherwise, and to the middle of an overlap. Text and x-extent are not
    changed here. Returns the layout and one row per box that gained lines:
    {id, strip: [x, y, w, h], where: "end" | "start"}, the newly covered strip whose text the
    caller reads (read_seam_strips) and attaches after / before the box's text.
    """
    try:
        import numpy as np
    except Exception:
        return layout, []
    wanted = set(pages) if pages is not None else None
    snappable = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR}
    layout = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    strips: list[dict[str, Any]] = []
    for meta in layout.get("pages", []):
        page_no = meta["page"]
        if wanted is not None and page_no not in wanted:
            continue
        png = doc_dir / "pages" / f"{page_no:04d}.png"
        ink = page_ink_mask(png) if png.is_file() else None
        if ink is None:
            continue
        height, width = ink.shape
        sx, sy = width / max(1.0, float(meta["width"])), height / max(1.0, float(meta["height"]))
        on_page = [b for b in blocks if b["page"] == page_no and b["type"] in snappable
                   and not b.get("box_manual") and b.get("source") != "manual" and (b.get("text") or "").strip()]
        for upper in sorted(on_page, key=lambda b: b["bbox"][1]):
            ax, ay, aw, ah = upper["bbox"]
            below = [b for b in on_page if b is not upper and b["bbox"][1] > ay + 0.3 * ah
                     and min(ax + aw, b["bbox"][0] + b["bbox"][2]) - max(ax, b["bbox"][0])
                     >= 0.6 * min(aw, b["bbox"][2])]
            if not below:
                continue
            lower = min(below, key=lambda b: b["bbox"][1])
            bx, by, bw, bh = lower["bbox"]
            a_bot, b_top = int(round((ay + ah) * sy)), int(round(by * sy))
            left = max(0, int(max(ax, bx) * sx))
            right = min(width, int(min(ax + aw, bx + bw) * sx))
            if right - left < 20:
                continue
            lo = max(0, min(a_bot, b_top) - _SEAM_WINDOW_PX)
            hi = min(height, max(a_bot, b_top) + _SEAM_WINDOW_PX)
            if hi - lo < 6:
                continue
            profile = ink[lo:hi, left:right].sum(axis=1).astype(float)
            peak = float(profile.max())
            if peak <= 0:
                continue

            def cuts_text(row: int) -> bool:
                r = row - lo
                return bool(profile[max(0, r - 1):r + 2].max() > 0.25 * peak)

            overlap = a_bot - b_top
            if not (overlap > 2 or cuts_text(a_bot) or cuts_text(b_top)):
                continue
            blank = profile <= 0.12 * peak
            runs, i = [], 0
            while i < len(blank):
                if blank[i]:
                    j = i
                    while j < len(blank) and blank[j]:
                        j += 1
                    if j - i >= 2:
                        runs.append((i, j))
                    i = j
                else:
                    i += 1
            if not runs:
                continue
            running = not _SENTENCE_END_RE.search((upper.get("text") or "").rstrip())
            centre = lambda r: (r[0] + r[1]) / 2
            if overlap > 2:                                    # both boxes claim it: split the difference
                run = min(runs, key=lambda r: abs(centre(r) - ((a_bot + b_top) / 2 - lo)))
            elif running:
                # The upper paragraph is still going: a line sitting in the seam is its next line, so
                # the seam goes after that line (the first gap below where the lower box starts).
                below_start = [r for r in runs if centre(r) >= max(a_bot, b_top) - lo - 1]
                run = min(below_start, key=centre) if below_start else min(runs, key=lambda r: abs(centre(r) - (b_top - lo)))
            else:
                # It ended: a line in the seam opens the paragraph below, so the seam goes before it.
                above_end = [r for r in runs if centre(r) <= min(a_bot, b_top) - lo + 1]
                run = max(above_end, key=centre) if above_end else min(runs, key=lambda r: abs(centre(r) - (a_bot - lo)))
            seam = lo + (run[0] + run[1]) // 2
            if seam - int(ay * sy) < _SEAM_MIN_SIDE_PX or int((by + bh) * sy) - seam < _SEAM_MIN_SIDE_PX:
                continue
            new_bottom, new_top = seam / sy, (seam + 1) / sy
            old_bottom, old_top = ay + ah, by
            # A line in the seam runs the width of the column, so a box that gains one takes the
            # union width of the two stacked boxes (a short last-lines slice is narrower than that).
            ux0, ux1 = min(ax, bx), max(ax + aw, bx + bw)
            grew_down = (new_bottom - old_bottom) * sy > _SEAM_STRIP_MIN_PX
            grew_up = (old_top - new_top) * sy > _SEAM_STRIP_MIN_PX
            upper["bbox"] = ([ux0, ay, ux1 - ux0, round(new_bottom - ay, 1)] if grew_down
                             else [ax, ay, aw, round(new_bottom - ay, 1)])
            lower["bbox"] = ([ux0, round(new_top, 1), ux1 - ux0, round(by + bh - new_top, 1)] if grew_up
                             else [bx, round(new_top, 1), bw, round(by + bh - new_top, 1)])
            upper["box_snapped"] = lower["box_snapped"] = True
            if grew_down:
                strips.append({"id": upper["id"], "where": "end",
                               "strip": [ux0, round(old_bottom, 1), ux1 - ux0, round(new_bottom - old_bottom, 1)]})
            if grew_up:
                # The line the old top cut through belongs to the strip too: read down to its end.
                line_end = next((lo + r[0] for r in runs if lo + r[0] >= int(old_top * sy)), int(old_top * sy))
                strips.append({"id": lower["id"], "where": "start",
                               "strip": [ux0, round(new_top, 1), ux1 - ux0, round(max(old_top, line_end / sy) - new_top, 1)]})
    layout["blocks"] = blocks
    return layout, strips


def read_seam_strips(doc_dir: Path, layout: dict[str, Any], strips: Sequence[dict[str, Any]],
                     vision_ocr_url: str | None = None, vision_ocr_model: str | None = None,
                     vision_ocr_fallback_url: str | None = None,
                     vision_ocr_fallback_model: str | None = None) -> tuple[dict[str, Any], int]:
    """Read the lines a mended seam newly covers and attach them to the box that now owns them.

    Only the strip is read, so nothing has to agree with an earlier reading of the whole box; the
    words go after the text of the box above or before the text of the box below. Reading uses the
    vision engine when it answers and local OCR otherwise. Returns the layout and how many strips
    were attached; an unreadable or implausible strip is skipped (the box keeps its text).
    """
    layout = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    by_id = {b["id"]: b for b in blocks}
    attached = 0
    for row in strips:
        block = by_id.get(row["id"])
        if block is None:
            continue
        try:
            text = " ".join((extract_region_text(doc_dir, layout, block["page"], row["strip"], vision_ocr_url,
                                                 vision_ocr_model, vision_ocr_fallback_url,
                                                 vision_ocr_fallback_model) or "").split())
        except Exception:
            continue
        if len(text) < 3 or ocr_quality_score(text) < 0.4:
            continue
        old = block["text"].strip()
        block["text"] = f"{old} {text}" if row["where"] == "end" else f"{text} {old}"
        attached += 1
    layout["blocks"] = blocks
    return layout, attached


def reread_grown_regions(doc_dir: Path, layout: dict[str, Any], moved: Sequence[dict[str, Any]],
                         vision_ocr_url: str | None, vision_ocr_model: str | None,
                         vision_ocr_fallback_url: str | None = None,
                         vision_ocr_fallback_model: str | None = None,
                         timeout: float = 150.0) -> tuple[dict[str, Any], int]:
    """Re-read boxes that snap_text_boxes extended by whole lines: their text was cut short too.

    The new reading replaces the old only when it is longer, still starts with the old text (so
    a re-read that wandered into another paragraph is refused) and reads like prose. Returns the
    layout and how many regions were extended.
    """
    layout = dict(layout)
    blocks = [dict(b) for b in layout.get("blocks", [])]
    by_id = {b["id"]: b for b in blocks}
    extended = 0
    for row in moved:
        block = by_id.get(row["id"])
        if not row.get("lines_added") or block is None:
            continue
        old = " ".join((block.get("text") or "").split())
        try:
            new = extract_region_text(doc_dir, layout, block["page"], block["bbox"], vision_ocr_url,
                                      vision_ocr_model, vision_ocr_fallback_url, vision_ocr_fallback_model, timeout)
        except Exception:
            continue
        new = " ".join((new or "").split())
        head = re.sub(r"\W+", "", old.casefold())[:24]
        if (new and len(new) > len(old) + 8 and head and head in re.sub(r"\W+", "", new.casefold())
                and ocr_quality_score(new) >= 0.55):
            block["text"] = new
            extended += 1
    layout["blocks"] = blocks
    return layout, extended


def duplicate_article_groups(articles: Sequence[dict[str, Any]]) -> list[list[str]]:
    """Groups of articles that share most of their text regions -- the same words would be
    narrated once per article. Two articles are grouped when the smaller one shares at least
    half its regions with the other."""
    sets = [(a["id"], set(a.get("blocks") or [])) for a in articles if a.get("blocks")]
    parent = {i: i for i, _ in sets}

    def find(i: str) -> str:
        while parent[i] != i:
            parent[i] = parent[parent[i]]
            i = parent[i]
        return i

    for n, (ia, sa) in enumerate(sets):
        for ib, sb in sets[n + 1:]:
            if len(sa & sb) >= 0.5 * min(len(sa), len(sb)):
                parent[find(ia)] = find(ib)
    groups: dict[str, list[str]] = {}
    for i, _ in sets:
        groups.setdefault(find(i), []).append(i)
    return [g for g in groups.values() if len(g) > 1]


def audit_page_coverage(doc_dir: Path, layout: dict[str, Any], page_no: int) -> dict[str, Any]:
    """Report inked areas of a page render that no region box covers.

    Pure measurement: it never edits the layout. `uncovered` holds page-point boxes of ink
    blobs outside every block/image box (any type, so furniture and pictures count as
    covered); `uncovered_ratio` is that ink's share of all ink on the page. `suspect_blocks`
    lists text regions whose words still look mis-read after any verification pass.
    """
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    png = doc_dir / "pages" / f"{page_no:04d}.png"
    blocks = [b for b in layout.get("blocks", []) if b["page"] == page_no]
    eligible = set(NARRATABLE) | {BLOCK_CAPTION, BLOCK_SIDEBAR, BLOCK_PULLQUOTE}
    page_language = detect_article_language(b.get("text") or "" for b in blocks if b["type"] in eligible)
    page_vocabulary = recurring_tokens(b.get("text") or "" for b in blocks if b["type"] in eligible)
    suspects = [b["id"] for b in blocks
                if b["type"] in eligible
                and suspect_ocr_tokens(" ".join((b.get("text") or "").split()), page_language, page_vocabulary)]
    # Running heads and adverts are never narrated: a box edge or an overlap there is not a problem.
    text_blocks = [b for b in blocks if b["type"] not in VISUAL_TYPES and b["type"] not in (BLOCK_FURNITURE, BLOCK_AD)]
    overlaps = [[a["id"], b["id"]] for i, a in enumerate(text_blocks) for b in text_blocks[i + 1:]
                if max(_bbox_intersection_ratio(a["bbox"], b["bbox"]),
                       _bbox_intersection_ratio(b["bbox"], a["bbox"])) >= _AUDIT_OVERLAP_RATIO]
    report: dict[str, Any] = {"page": page_no, "uncovered": [], "uncovered_ratio": 0.0,
                              "suspect_blocks": suspects, "overlaps": overlaps, "clipped": [],
                              "measured": False}
    if not page_meta or not png.is_file():
        return report
    try:
        import numpy as np
        from scipy import ndimage
    except Exception:
        return report
    grid_info = page_ink_grid_points(png, page_meta)
    if grid_info is None:
        return report
    grid, cell_w, cell_h = grid_info
    ink = grid >= _AUDIT_MIN_CELL_INK
    covered = np.zeros_like(ink)
    boxes = [b["bbox"] for b in blocks] + [i["bbox"] for i in layout.get("images", [])
                                            if i["page"] == page_no]
    for x, y, w, h in boxes:
        r0, r1 = max(0, int(y / cell_h) - 1), min(ink.shape[0], int(math.ceil((y + h) / cell_h)) + 1)
        c0, c1 = max(0, int(x / cell_w) - 1), min(ink.shape[1], int(math.ceil((x + w) / cell_w)) + 1)
        covered[r0:r1, c0:c1] = True
    pixels = page_ink_mask(png)
    if pixels is not None:
        report["clipped"] = _clipped_edges(pixels, page_meta, text_blocks, boxes)
    graphic_boxes = ([i["bbox"] for i in layout.get("images", []) if i["page"] == page_no]
                     + [b["bbox"] for b in blocks if b["type"] in VISUAL_TYPES or b["type"] == BLOCK_AD])
    er, ec = int(ink.shape[0] * _AUDIT_EDGE_FRACTION), int(ink.shape[1] * _AUDIT_EDGE_FRACTION)
    edge = np.zeros_like(ink)
    edge[:er, :] = edge[-er:, :] = True
    edge[:, :ec] = edge[:, -ec:] = True
    stray = ink & ~covered & ~edge
    report["measured"] = True
    total = int(ink.sum())
    if not stray.any() or not total:
        return report
    # Bridge the gaps between letters and lines so one paragraph is one blob.
    labels, count = ndimage.label(ndimage.binary_dilation(stray, iterations=2))
    kept_cells = 0
    for index in range(1, count + 1):
        rows, cols = np.nonzero((labels == index) & stray)
        if len(rows) < _AUDIT_MIN_CELLS:
            continue
        blob = [round(float(cols.min() * cell_w), 1), round(float(rows.min() * cell_h), 1),
                round(float((cols.max() + 1 - cols.min()) * cell_w), 1),
                round(float((rows.max() + 1 - rows.min()) * cell_h), 1)]
        # Labels of a chart, text set into a photo or an advert's own wording are part of that
        # graphic, not narratable text a box forgot: a blob in or beside one is not reported.
        if any(bbox_overlaps(blob, g, tol=_AUDIT_GRAPHIC_MARGIN) for g in graphic_boxes):
            continue
        if pixels is not None and (_is_rule_only(pixels, blob, page_meta) or _is_lines_or_frames_only(pixels, blob, page_meta)):
            continue                     # a printed rule or dotted leader, not letters
        kept_cells += len(rows)
        report["uncovered"].append(blob)
    report["uncovered_ratio"] = round(kept_cells / total, 4)
    return report


_INVENTED_VISUAL_MIN_INK = 0.4   # see the guard below apply_semantic_regions; a real photo, chart
                                  # or table measured 0.56-0.97 on confirmed real pages of the same
                                  # issue, a text area the model mistook for one 0.10-0.14 -- wide
                                  # margin either side of this floor


def region_looks_like_a_picture(bbox: Sequence[float], page_ink) -> bool:
    """Whether a NEW visual region (see apply_semantic_regions) is inked densely enough, over its
    own area, to plausibly be a photo, chart or table rather than a text area or column gap the
    model mistook for one. `page_ink` is (coverage_grid, cell_w, cell_h) in the same page-point
    units as `bbox`, as produced by page_ink_grid() -- or None to skip the check (no page render to
    check against, e.g. in a unit test), which always passes.
    """
    if page_ink is None:
        return True
    grid, cell_w, cell_h = page_ink
    if grid is None or not cell_w or not cell_h:
        return True
    x, y, w, h = bbox
    r0, r1 = max(0, int(y / cell_h)), min(grid.shape[0], int(math.ceil((y + h) / cell_h)))
    c0, c1 = max(0, int(x / cell_w)), min(grid.shape[1], int(math.ceil((x + w) / cell_w)))
    if r1 <= r0 or c1 <= c0:
        return True   # too small to sample a page-render cell -- err toward keeping it
    return float(grid[r0:r1, c0:c1].mean()) >= _INVENTED_VISUAL_MIN_INK


def apply_semantic_regions(blocks: list[dict], regions: list[dict], page_meta: dict | None = None,
                           page_ink=None) -> list[dict]:
    """Group existing OCR fragments without allowing invented IDs or text.

    A paragraph may continue from the bottom of one column at the top of the
    next.  The semantic model correctly recognises that as one logical
    paragraph, but collapsing both OCR fragments into one rectangular bbox
    paints the union over all of the unrelated text between them.  Keep one
    output region per physical column in that case; narration can still read
    the adjacent regions continuously.
    """
    known = {b["id"]: b for b in blocks}
    visual_boxes = [b["bbox"] for b in blocks if b.get("type") in VISUAL_TYPES]
    used = set()
    out = []
    allowed = set(NARRATABLE) | set(VISUAL_TYPES) | {BLOCK_AD, BLOCK_SIDEBAR, BLOCK_CAPTION, BLOCK_FURNITURE, BLOCK_PULLQUOTE}

    # Infer the grid from the ungrouped OCR boxes, while their real column
    # geometry is still available.  Doing this after forming a union is too
    # late: the union itself bridges the gutter and hides the two columns.
    semantic_spans: list[tuple[float, float]] = []
    semantic_col_w = 0.0
    if page_meta and page_meta.get("width"):
        geometry_blocks = [
            Block(id=b["id"], page=b["page"], bbox=list(b["bbox"]),
                  type=b["type"], text=b.get("text", ""))
            for b in blocks
        ]
        anchors, semantic_col_w = column_anchors(geometry_blocks, float(page_meta["width"]))
        semantic_spans = [
            (anchor, min(anchor + semantic_col_w, anchors[i + 1])
             if i + 1 < len(anchors) else anchor + semantic_col_w)
            for i, anchor in enumerate(anchors)
        ]

    def physical_groups(members: list[dict], kind: str) -> list[list[dict]]:
        """Partition a logical body paragraph at genuine column boundaries."""
        if kind != BLOCK_BODY or len(members) < 2 or len(semantic_spans) < 2:
            return [members]

        def overlap(member: dict, span: tuple[float, float]) -> float:
            x, _, width, _ = member["bbox"]
            return max(0.0, min(x + width, span[1]) - max(x, span[0]))

        assignments = []
        for member in members:
            column = max(range(len(semantic_spans)),
                         key=lambda i: (overlap(member, semantic_spans[i]), -i))
            # A wide standfirst or a tiny detached glyph is not evidence that
            # a paragraph crosses columns.  Each split member must occupy a
            # useful fraction of its assigned text column.
            if overlap(member, semantic_spans[column]) < 0.45 * min(
                    semantic_col_w, member["bbox"][2]):
                return [members]
            assignments.append(column)
        if len(set(assignments)) < 2:
            return [members]

        grouped: dict[int, list[dict]] = {}
        for member, column in zip(members, assignments):
            grouped.setdefault(column, []).append(member)
        return [sorted(grouped[column], key=lambda b: (b["bbox"][1], b["bbox"][0]))
                for column in sorted(grouped)]

    def append_group(members: list[dict], kind: str, label: str,
                     visual_box: list[float] | None, continues: bool) -> None:
        # The semantic model returns a set of block IDs, not a dependable
        # transcription order. Joining in its arbitrary ID order moved the
        # first line extracted from a run-on byline to the paragraph's end.
        members = sorted(members, key=lambda b: (
            b.get("reading_order", 10**9), b["bbox"][1], b["bbox"][0]))
        x = min(b["bbox"][0] for b in members)
        y = min(b["bbox"][1] for b in members)
        right = max(b["bbox"][0] + b["bbox"][2] for b in members)
        bottom = max(b["bbox"][1] + b["bbox"][3] for b in members)
        separator = "\n" if kind in (BLOCK_CODE, BLOCK_TABLE) else " "
        text = separator.join(b["text"] for b in members)
        if kind in NARRATABLE:
            # Closes a printed line break (eingetrich- tert) but keeps a suspended
            # compound (Programmier- und Hausaufgaben) as printed.
            text = remove_linebreak_hyphens(text)
        out.append(dict(members[0], bbox=visual_box or [x, y, right - x, bottom - y], text=text,
                        type=kind, region_label=label, source="llm_layout",
                        reading_order=len(out), continues_from_previous_page=continues))

    for region in regions:
        if not isinstance(region, dict):
            raise ValueError("Invalid semantic region")
        ids = region.get("block_ids")
        label = region.get("type")
        kind = _VISION_LABEL_TYPES.get(label, label)
        visual_box = None
        if "bbox" in region and kind in (*VISUAL_TYPES, BLOCK_AD):
            bbox = region["bbox"]
            if not page_meta or not isinstance(bbox, list) or len(bbox) != 4 or not all(
                    isinstance(v, (int, float)) and math.isfinite(v) and 0 <= v <= 1000 for v in bbox):
                raise ValueError("Invalid visual bounding box")
            x0, y0, x1, y1 = bbox
            if x1 <= x0 or y1 <= y0:
                raise ValueError("Empty visual region")
            visual_box = [x0 / 1000 * page_meta["width"], y0 / 1000 * page_meta["height"],
                          (x1 - x0) / 1000 * page_meta["width"], (y1 - y0) / 1000 * page_meta["height"]]
        if not ids and kind in VISUAL_TYPES and visual_box:
            # A "new" picture the model invents (block_ids: []) is meant for a graphic OCR never
            # produced a block for. Confirmed on a real issue: it sometimes instead draws that box
            # over a column gap and the text below it (a table-of-contents heading and its intro
            # paragraph in one case) -- which would otherwise bury real paragraphs under an empty
            # "Figure" placeholder. A real photo/chart/table is a densely inked area of the page
            # render; that case is not, so page_ink screens it out before it is ever created.
            if not region_looks_like_a_picture(visual_box, page_ink):
                continue
            out.append(Block(id=f"p{page_meta['page']:04d}-visual-{len(out)}", page=page_meta['page'],
                             bbox=visual_box, type=kind, text="", reading_order=len(out),
                             source="llm_layout", region_label=label).to_json())
            continue
        if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
            raise ValueError("Invalid region IDs")
        if kind not in allowed or len(set(ids)) != len(ids) or any(i not in known or i in used for i in ids):
            raise ValueError("Unknown or repeated semantic region")
        members = [known[i] for i in ids]
        if len({b["page"] for b in members}) != 1:
            raise ValueError("A region cannot span pages")
        # A semantic model can call ordinary lower-page body copy a caption.
        # Preserve that label only when there is actually a nearby figure.
        # Explicit captions from the primary layout engine remain untouched.
        if (kind == BLOCK_CAPTION
                and all(member.get("type") == BLOCK_BODY for member in members)):
            x = min(member["bbox"][0] for member in members)
            y = min(member["bbox"][1] for member in members)
            right = max(member["bbox"][0] + member["bbox"][2] for member in members)
            bottom = max(member["bbox"][1] + member["bbox"][3] for member in members)
            member_box = [x, y, right - x, bottom - y]
            tolerance = max(float(page_meta.get("width", 0)) if page_meta else 0,
                            float(page_meta.get("height", 0)) if page_meta else 0) * 0.02
            if not any(bbox_overlaps(member_box, box, tol=tolerance)
                       for box in visual_boxes):
                kind = BLOCK_BODY
        if any(b["type"] == BLOCK_AD for b in members):
            if not all(b["type"] == BLOCK_AD for b in members):
                raise ValueError("Cannot merge advertising and editorial text")
            kind = BLOCK_AD
        used.update(ids)
        groups = physical_groups(members, kind)
        for i, group in enumerate(groups):
            append_group(group, kind, label, visual_box,
                         i == 0 and region.get("continues_from_previous_page") is True)
    # Incomplete model output must not silently remove document content.
    return out + [b for b in blocks if b["id"] not in used]


def remap_reviewed_page(articles: list[dict], old_layout: dict, layout: dict, page: int) -> None:
    """Reconnect reviewed articles when OCR replaces the page's region IDs."""
    old = {b["id"]: b for b in old_layout.get("blocks", []) if b["page"] == page}
    owners = {a["id"]: [old[i] for i in a.get("blocks", []) if i in old] for a in articles}
    affected = [a for a in articles if page in a.get("pages", []) or owners[a["id"]]]
    for a in affected:
        a["blocks"] = [i for i in a.get("blocks", []) if i not in old]
        known_images = {i["id"] for i in layout.get("images", [])}
        a["images"] = [i for i in a.get("images", []) if i in known_images]
        if a.get("status") == "approved":
            a["status"] = "detected"
    for block in layout.get("blocks", []):
        if block["page"] != page or block["type"] not in NARRATABLE or not affected:
            continue
        x, y, w, h = block["bbox"]
        def overlap(a):
            return sum(max(0, min(x + w, b["bbox"][0] + b["bbox"][2]) - max(x, b["bbox"][0])) *
                       max(0, min(y + h, b["bbox"][1] + b["bbox"][3]) - max(y, b["bbox"][1]))
                       for b in owners[a["id"]])
        best = max(affected, key=overlap)
        if overlap(best) >= w * h * 0.35:
            best["blocks"].append(block["id"])
    # Newly recognised visual regions follow the closest article on the page.
    objects = [Article(**{k: v for k, v in a.items() if k in Article.__dataclass_fields__}) for a in affected]
    new_images = [i for i in layout.get("images", []) if i["page"] == page and i.get("source_block")]
    for a in objects:
        a.images = [i for i in a.images if i not in {im["id"] for im in new_images}]
    assign_images_to_articles(new_images, [Block(**b) for b in layout.get("blocks", [])], objects)
    for a, obj in zip(affected, objects):
        a["images"] = obj.images


def _crop_page_png(doc_dir: Path, page_no: int, page_meta: dict[str, Any],
                   bbox: Sequence[float], dest: Path) -> None:
    """Crop one region (PDF-point bbox) out of a page's rendered PNG."""
    from PIL import Image
    png = doc_dir / "pages" / f"{page_no:04d}.png"
    with Image.open(png) as im:
        sx, sy = im.width / page_meta["width"], im.height / page_meta["height"]
        box_px = (
            max(0, int(bbox[0] * sx)), max(0, int(bbox[1] * sy)),
            min(im.width, int((bbox[0] + bbox[2]) * sx)),
            min(im.height, int((bbox[1] + bbox[3]) * sy)),
        )
        im.crop(box_px).save(dest)


def refresh_visual_regions(doc_dir: Path, layout: dict, page: int | None = None) -> None:
    """Re-snap figure bounds, regenerate the crops taken from them, and avoid
    treating a whole scanned page as one figure."""
    for meta in layout.get("pages", []):
        page_no = meta["page"]
        if page is not None and page_no != page:
            continue
        blocks = [Block(**b) for b in layout.get("blocks", []) if b["page"] == page_no]
        visuals = [b for b in blocks if b.type in VISUAL_TYPES]

        def intersection_area(a: Sequence[float], b: Sequence[float]) -> float:
            return (max(0.0, min(a[0] + a[2], b[0] + b[2]) - max(a[0], b[0]))
                    * max(0.0, min(a[1] + a[3], b[1] + b[3]) - max(a[1], b[1])))

        # Remember which visual seeds made up each advert before moving them.
        # Once they land on the rendered artwork, their union is better advert
        # geometry than the model's displaced red rectangle.  The coverage
        # floor avoids shrinking a mixed text advert around one incidental icon.
        advert_visuals: dict[str, list[str]] = {}
        for advert in (b for b in blocks if b.type == BLOCK_AD):
            related = [v for v in visuals
                       if intersection_area(v.bbox, advert.bbox) >= 0.5 * v.bbox[2] * v.bbox[3]]
            covered = sum(intersection_area(v.bbox, advert.bbox) for v in related)
            if related and covered >= 0.25 * advert.bbox[2] * advert.bbox[3]:
                advert_visuals[advert.id] = [v.id for v in related]
        # Re-snapping here, not only at OCR time, is what lets an issue
        # ingested before this pass existed pick up corrected figure boxes
        # without paying for a second OCR run of the whole document. It is
        # idempotent: an already-correct box seeds the same graphic again.
        if snap_visual_bboxes(doc_dir / "pages" / f"{page_no:04d}.png", blocks, meta):
            snapped = {b.id: b.bbox for b in blocks if b.type in VISUAL_TYPES}
            for advert in (b for b in blocks if b.id in advert_visuals):
                parts = [b for b in visuals if b.id in advert_visuals[advert.id]]
                if not parts:
                    continue
                x0 = min(b.bbox[0] for b in parts)
                y0 = min(b.bbox[1] for b in parts)
                x1 = max(b.bbox[0] + b.bbox[2] for b in parts)
                y1 = max(b.bbox[1] + b.bbox[3] for b in parts)
                advert.bbox = [x0, y0, x1 - x0, y1 - y0]
                snapped[advert.id] = advert.bbox
            for saved in layout["blocks"]:
                if saved["id"] in snapped:
                    saved["bbox"] = snapped[saved["id"]]
        crops = crop_visual_regions(doc_dir, page_no, meta, blocks)
        kept = []
        for img in layout.get("images", []):
            if img["page"] != page_no or "-manual-" in img["id"]:
                kept.append(img)
                continue
            if img.get("source_block"):
                continue
            x, y, w, h = img["bbox"]
            if w * h >= meta["width"] * meta["height"] * .85 and sum(len(b.text) for b in blocks) > 120:
                continue
            if any(abs(x - c["bbox"][0]) < 5 and abs(y - c["bbox"][1]) < 5 and
                   abs(w - c["bbox"][2]) < 10 and abs(h - c["bbox"][3]) < 10 for c in crops):
                continue
            # Embedded graphics inside an identified advert are advertising too.
            if any(b.type == BLOCK_AD and x >= b.bbox[0] and y >= b.bbox[1] and
                   x + w <= b.bbox[0] + b.bbox[2] and y + h <= b.bbox[1] + b.bbox[3] for b in blocks):
                img = dict(img, type=BLOCK_AD)
            kept.append(img)
        for crop in crops:
            crop["reading_order"] = image_reading_position(crop, blocks)
        layout["images"] = kept + crops


def crop_visual_regions(doc_dir: Path, page_no: int, page_meta: dict,
                        blocks: Sequence[Block]) -> list[dict[str, Any]]:
    """Keep charts, tables and code boxes intact as page crops for playback."""
    images = []
    for b in blocks:
        if b.type not in VISUAL_TYPES:
            continue
        name = f"p{page_no:04d}-region-{b.id}"
        (doc_dir / "images").mkdir(parents=True, exist_ok=True)
        _crop_page_png(doc_dir, page_no, page_meta, b.bbox, doc_dir / "images" / f"{name}.png")
        images.append({"id": name, "page": page_no, "bbox": b.bbox,
                       "filename": f"{name}.png", "type": b.type,
                       "source_block": b.id, "text": b.text,
                       "reading_order": b.reading_order})
    return images


def add_manual_region(doc_dir: Path, layout: dict[str, Any], page_no: int,
                      bbox: Sequence[float], kind: str, text: str = "",
                      vision_ocr_url: str | None = None,
                      vision_ocr_model: str | None = None,
                      vision_ocr_fallback_url: str | None = None,
                      vision_ocr_fallback_model: str | None = None,
                      timeout: float = 150.0) -> dict[str, Any]:
    """Add a region the user drew by hand -- for text or a figure OCR/layout
    detection missed entirely, not for correcting one already found (use
    review's tick/untick or reocr_page for that).

    kind "image" crops the region into its own figure file and adds it to
    layout["images"]. kind "text" adds a new Block: if `text` is supplied it
    is used verbatim (the user typed it themselves), otherwise the crop is
    sent through the vision OCR chain if one is configured -- best-effort,
    the block is still added with a placeholder if that returns nothing,
    since the user drawing this region at all means they judged the text
    important enough to include even without a clean read.

    Deliberately does not re-run classify_blocks on the new region: the user
    is drawing it precisely because automatic detection already failed on
    it, so re-applying the same heuristics would not be trustworthy. Text is
    trusted as stated (BLOCK_BODY, narratable); explicit review labels map to
    their matching semantic block types and only then receive a fresh
    reading-order position on its page.
    """
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta:
        raise ValueError("Unknown page")
    if len(bbox) != 4 or bbox[2] <= 0 or bbox[3] <= 0:
        raise ValueError("bbox must be [x, y, width, height] with a positive size")
    layout = dict(layout)

    if kind == "image":
        images_dir = doc_dir / "images"
        images_dir.mkdir(parents=True, exist_ok=True)
        existing = list(images_dir.glob(f"p{page_no:04d}-manual-*.png"))
        name = f"p{page_no:04d}-manual-{len(existing):03d}"
        _crop_page_png(doc_dir, page_no, page_meta, bbox, images_dir / f"{name}.png")
        images = list(layout.get("images", []))
        # Placed after every auto-detected image on the page by reading
        # order -- a reasonable default for a figure the pipeline missed;
        # the user can still see and move past it like any other figure.
        page_images = [i for i in images if i["page"] == page_no]
        next_ro = max([i.get("reading_order", 0) for i in page_images], default=0) + 1
        if page_meta.get("order_locked"):
            # A hand-ordered page: the new figure goes last rather than into a guessed position.
            next_ro = 1 + max([i.get("reading_order", 0) for i in page_images]
                              + [b.get("reading_order", 0) for b in layout.get("blocks", []) if b["page"] == page_no],
                              default=-1)
        images.append({"id": name, "page": page_no, "bbox": list(bbox),
                       "filename": f"{name}.png", "reading_order": next_ro})
        layout["images"] = images
        layout, _ = fold_text_into_picture(layout, page_no, bbox)
        return layout

    manual_type, region_label = {
        "text": (BLOCK_BODY, None),
        "body": (BLOCK_BODY, None),
        "title": (BLOCK_TITLE, None),
        "heading": (BLOCK_HEADING, None),
        "deck": (BLOCK_DECK, None),
        "byline": (BLOCK_BYLINE, None),
        "quote": (BLOCK_QUOTE, None),
        "caption": (BLOCK_CAPTION, None),
        "pullquote": (BLOCK_PULLQUOTE, None),
        "sidebar": (BLOCK_SIDEBAR, None),
        "page_number": (BLOCK_FURNITURE, "page_number"),
        "source": (BLOCK_FURNITURE, "source"),
        "in_picture": (BLOCK_FURNITURE, "in_picture"),
        "ocr_noise": (BLOCK_FURNITURE, "ocr_noise"),
        "duplicate": (BLOCK_FURNITURE, "duplicate"),
        "table": (BLOCK_TABLE, None),
        "ad": (BLOCK_AD, None),
    }.get(kind, (None, None))
    if manual_type is None:
        raise ValueError("Unsupported manual region kind")

    final_text = text.strip()
    if not final_text:
        crop_path = doc_dir / "pages" / f".manual-crop-{page_no}.png"
        try:
            _crop_page_png(doc_dir, page_no, page_meta, bbox, crop_path)
            # Same two-step read as extract_region_text (used by "read again"
            # on an existing region): vision OCR first, but a region drawn
            # while that backend is unreachable -- confirmed live, both the
            # primary and fallback vision endpoints were refusing connections
            # -- must not always come back empty when the local Tesseract
            # pass could have read it instead.
            if vision_ocr_url and vision_ocr_model:
                try:
                    engines = [(vision_ocr_url, vision_ocr_model),
                              (vision_ocr_fallback_url, vision_ocr_fallback_model)]
                    regions = vision_ocr_page_chain(crop_path, engines, timeout=timeout)
                    if regions:
                        final_text = " ".join(r["text"] for r in regions if r.get("text")).strip()
                except Exception:                   # noqa: BLE001 - fall through to local OCR
                    logging.getLogger(__name__).warning(
                        "vision OCR unreachable for a manually drawn region; falling back to local OCR")
            if not final_text:
                final_text = " ".join(p.get("text", "") for p in ocr_page(crop_path) if p.get("text")).strip()
        finally:
            crop_path.unlink(missing_ok=True)
    if not final_text:
        final_text = "(no text recognised -- click to edit)"

    blocks = [Block(**b) for b in layout.get("blocks", [])]
    existing_ids = {b.id for b in blocks}
    i = 0
    while f"m{page_no:04d}-{i:03d}" in existing_ids:
        i += 1
    new_block = Block(id=f"m{page_no:04d}-{i:03d}", page=page_no, bbox=list(bbox),
                      type=manual_type, text=final_text, confidence=0.5, source="manual",
                      region_label=region_label)

    page_blocks = [b for b in blocks if b.page == page_no] + [new_block]
    if page_meta.get("order_locked"):
        # A hand-ordered page keeps its order; the new region is read last until the reader moves it.
        new_block.reading_order = 1 + max([b.reading_order for b in page_blocks[:-1]]
                                          + [i.get("reading_order", 0) for i in layout.get("images", [])
                                             if i["page"] == page_no], default=-1)
    else:
        assign_reading_order(page_blocks, page_meta["width"])
    others = [b for b in blocks if b.page != page_no]
    layout["blocks"] = [b.to_json() for b in others + page_blocks]
    return layout


def extract_region_text(doc_dir: Path, layout: dict[str, Any], page_no: int,
                        bbox: Sequence[float], vision_ocr_url: str | None = None,
                        vision_ocr_model: str | None = None,
                        vision_ocr_fallback_url: str | None = None,
                        vision_ocr_fallback_model: str | None = None,
                        timeout: float = 150.0) -> str:
    """OCR a user-selected rectangle without changing the saved layout.

    Used for metadata fields such as article title/author: unlike a manual
    region, the selection is a one-off extraction and must not become prose
    that is later narrated.
    """
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta or len(bbox) != 4 or bbox[2] <= 0 or bbox[3] <= 0:
        raise ValueError("Invalid page selection")
    with tempfile.NamedTemporaryFile(prefix=f".extract-{page_no}-", suffix=".png",
                                     dir=doc_dir / "pages", delete=False) as tmp:
        crop_path = Path(tmp.name)
    try:
        _crop_page_png(doc_dir, page_no, page_meta, bbox, crop_path)
        regions = None
        if vision_ocr_url and vision_ocr_model:
            try:
                regions = vision_ocr_page_chain(crop_path, [
                    (vision_ocr_url, vision_ocr_model),
                    (vision_ocr_fallback_url, vision_ocr_fallback_model),
                ], timeout=timeout)
            except Exception:                       # noqa: BLE001 - fall through to local OCR
                logging.getLogger(__name__).warning(
                    "vision OCR unreachable for a re-read region; falling back to local OCR")
        if regions:
            repair_drop_cap_omissions(crop_path, regions)
            return _join_detached_drop_cap(
                " ".join(r.get("text", "") for r in regions if r.get("text")).strip())
        return _join_detached_drop_cap(
            " ".join(p.get("text", "") for p in ocr_page(crop_path) if p.get("text")).strip())
    finally:
        crop_path.unlink(missing_ok=True)


def update_region(doc_dir: Path, layout: dict[str, Any], page_no: int, block_id: str,
                  bbox: Sequence[float] | None, reocr: bool = False,
                  vision_ocr_url: str | None = None,
                  vision_ocr_model: str | None = None,
                  vision_ocr_fallback_url: str | None = None,
                  vision_ocr_fallback_model: str | None = None,
                  timeout: float = 150.0) -> tuple[dict[str, Any], str | None]:
    """Move/resize a region the layout engine already found, after the user
    dragged its box to correct a misplaced or mis-sized detection.

    Not for a region detection missed entirely (add_manual_region covers
    that) -- block_id must already exist in this page's blocks or images.
    A text block can also be re-OCRed at its corrected geometry (`reocr`);
    an image region is always re-cropped from the page render when its bbox
    changes, since its saved file is exactly that crop. Returns the updated
    layout and, when reocr produced text, that text (None otherwise, so the
    caller can tell "no re-OCR requested" apart from "re-OCR found nothing").
    """
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta:
        raise ValueError("Unknown page")
    if bbox is not None and (len(bbox) != 4 or bbox[2] <= 0 or bbox[3] <= 0):
        raise ValueError("bbox must be [x, y, width, height] with a positive size")
    layout = dict(layout)

    blocks = list(layout.get("blocks", []))
    block_idx = next((i for i, b in enumerate(blocks) if b["id"] == block_id), None)
    if block_idx is not None:
        block = dict(blocks[block_idx])
        old_bbox = list(block.get("bbox") or [])
        old_text = str(block.get("text") or "")
        if bbox is not None:
            block["bbox"] = list(bbox)
            block["box_manual"] = True          # placed by hand: snap_text_boxes leaves it alone
        new_text = None
        if reocr:
            new_text = extract_region_text(
                doc_dir, layout, page_no, block["bbox"],
                vision_ocr_url, vision_ocr_model,
                vision_ocr_fallback_url, vision_ocr_fallback_model, timeout)
            if new_text:
                # A vision re-read can omit the first printed line when a
                # paragraph begins with a drop cap. If the corrected box
                # still includes its old top edge, use the existing text as
                # a guarded second reading to restore only that lost prefix.
                # Moving the top edge down intentionally must still be able
                # to remove text, hence the geometry check.
                same_top = (bbox is None or not old_bbox
                            or float(block["bbox"][1]) <= float(old_bbox[1])
                               + max(3.0, float(old_bbox[3]) * 0.15))
                if same_top:
                    new_text = prefer_drop_cap_repair(new_text, old_text)
                block["text"] = new_text
                block.pop("ocr_text", None)   # freshly read: the new reading is the original
        blocks[block_idx] = block
        layout["blocks"] = blocks
        return layout, new_text

    images = list(layout.get("images", []))
    image_idx = next((i for i, im in enumerate(images) if im["id"] == block_id), None)
    if image_idx is None:
        raise ValueError("Region not found on this page")
    image = dict(images[image_idx])
    if bbox is not None:
        image["bbox"] = list(bbox)
        filename = image.get("filename") or f"{block_id}.png"
        (doc_dir / "images").mkdir(parents=True, exist_ok=True)
        _crop_page_png(doc_dir, page_no, page_meta, bbox, doc_dir / "images" / filename)
        image["filename"] = filename
    images[image_idx] = image
    layout["images"] = images
    return layout, None


# ── Box clean-up ────────────────────────────────────────────────────────────
# Each rule was sized on a real 204-page scanned issue (2 038 boxes) before it was written, and each
# one only changes what a box IS, never deletes it: the types below are not read aloud, show in the
# review list, and can be changed back with the type picker. A split byline keeps its original text.

_VISUAL_TYPES = (BLOCK_FIGURE, BLOCK_CHART, BLOCK_TABLE, BLOCK_CODE)
_BYLINE_RUNON = re.compile(r"^(Von|By|Text)\s+(.*)$", re.IGNORECASE)


def _area_inside(a: Sequence[float], b: Sequence[float]) -> float:
    """Share of box a's area that lies inside box b."""
    w = max(0.0, min(a[0] + a[2], b[0] + b[2]) - max(a[0], b[0]))
    h = max(0.0, min(a[1] + a[3], b[1] + b[3]) - max(a[1], b[1]))
    return (w * h) / max(1e-6, a[2] * a[3])


def _text_found_in(small: str, big: str) -> float:
    """How much of `small` appears, in order, inside `big` (0..1), tolerant of OCR spacing."""
    import difflib
    s = " ".join(small.split()).lower()
    g = " ".join(big.split()).lower()
    if not s:
        return 0.0
    matcher = difflib.SequenceMatcher(None, s, g, autojunk=False)
    return sum(m.size for m in matcher.get_matching_blocks()) / len(s)


def split_runon_byline(text: str) -> tuple[str, str] | None:
    """Split "Von Andreas Weck Als Microsoft-CTO…" into ("Von Andreas Weck", "Als Microsoft-CTO…").

    The OCR box of an author line often runs into the first line of the article. A name is two
    words (a middle initial like "M." makes three), extended by "und/and/&" plus another name for
    co-authors; whatever follows is the start of the article text. Returns None when there is
    nothing to split -- including a plain two-author byline ("Von Sabrina Patsch und Morton
    Swimmer") and a sentence that merely starts with "Von" ("Von Familienmitgliedern …").
    """
    m = _BYLINE_RUNON.match(" ".join((text or "").split()))
    if not m:
        return None
    lead, words = m.group(1), m.group(2).split()

    def take_name(i: int) -> int:
        """Index after one name starting at words[i], or -1 when words[i:] does not start with one."""
        if i + 1 >= len(words) or not all(w[:1].isupper() for w in words[i:i + 2]):
            return -1
        j = i + 2
        if j < len(words) and re.fullmatch(r"[A-ZÄÖÜ]\.", words[i + 1]) and words[j][:1].isupper():
            j += 1                                            # "Anna M. Müller"
        return j

    end = take_name(0)
    if end < 0:
        return None
    while end + 1 < len(words) and words[end].lower() in ("und", "and", "&"):
        nxt = take_name(end + 1)
        if nxt < 0:
            break
        end = nxt
    rest = " ".join(words[end:])
    if not rest:
        return None
    return f"{lead} {' '.join(words[:end])}", rest


def clean_up_regions(layout: dict[str, Any], pages: Iterable[int] | None = None) -> tuple[dict[str, Any], dict[str, list]]:
    """Fix the box problems measured on real scanned issues. Returns the layout and a report.

    * Text inside a picture (a label on a screenshot, a word on a slide in a photo), at most 80
      characters and at least 90% inside the picture, becomes "in_picture". Longer text is left
      alone: on the measured issue it included real running text caught inside a loose picture
      frame ("nutzen, ohne dass du …" continuing the previous sentence).
    * A box of two characters or fewer ("1", "V", "J 3", "#") becomes "ocr_noise". Longer short
      strings are left alone: they included a phone number, a price and a citation.
    * A box lying inside a bigger text box whose text already contains it (typically a subheading
      the body box also read) becomes "duplicate", so it is not read twice. Overlapping boxes with
      different text are left alone -- they were the large majority, and hiding one loses text.
    * An author line that runs into the article's first sentence is split: the name stays, the
      sentence becomes a paragraph box of its own, directly under the name in the same column, and
      is added to the articles that hold the author line. It is not glued onto "the next paragraph":
      measured on five real cases, the next paragraph in reading order was the right one only once
      (it was in the other column, or a sentence further on). If a paragraph on the page already
      starts with that sentence, only the name is kept.
    Returns the layout and a report; report["byline_split"] entries carry the new paragraph's id.
    """
    wanted = set(pages) if pages is not None else None
    report: dict[str, list] = {"in_picture": [], "ocr_noise": [], "duplicate": [], "byline_split": []}
    blocks = [dict(b) for b in layout.get("blocks", [])]
    by_page: dict[int, list[dict]] = {}
    for b in blocks:
        if wanted is None or b["page"] in wanted:
            by_page.setdefault(b["page"], []).append(b)
    pictures: dict[int, list] = {}
    for im in layout.get("images", []):
        pictures.setdefault(im["page"], []).append(im["bbox"])
    for b in blocks:
        if b["type"] in _VISUAL_TYPES:
            pictures.setdefault(b["page"], []).append(b["bbox"])

    def mark(b: dict, label: str) -> None:
        b["type"], b["region_label"], b["confidence"] = BLOCK_FURNITURE, label, 0.9
        report[label].append({"page": b["page"], "id": b["id"], "text": " ".join((b.get("text") or "").split())[:80]})

    owned = {im.get("source_block") for im in layout.get("images", []) if im.get("source_block")}
    for page_no, page_blocks in by_page.items():
        # What deleting a picture used to leave behind: an empty figure block whose image entry is gone. It is nothing
        # (no text, no picture to play), so it is re-typed as noise -- still listed, reversible, not narrated.
        for b in page_blocks:
            if b["type"] in _VISUAL_TYPES and not (b.get("text") or "").strip() and b["id"] not in owned:
                mark(b, "ocr_noise")
        text_boxes = [b for b in page_blocks if b["type"] not in _VISUAL_TYPES and b["type"] != BLOCK_FURNITURE]
        for b in text_boxes:
            text = " ".join((b.get("text") or "").split())
            if text.startswith("(no text recognised"):
                continue
            if (len(re.sub(r"[\W_]", "", text)) <= 2 or text.lower().startswith("[non-text")
                    or b["bbox"][3] < _MIN_TEXT_BOX_HEIGHT
                    or (len(text) >= 60 and glued_repetition_share(text) >= 0.5)):
                # includes the engine's own "[Non-Text]", hairline boxes, and a region that is mostly one endless
                # repeated token (a recogniser stuck in a loop)
                mark(b, "ocr_noise")
            elif len(text) <= 80 and any(_area_inside(b["bbox"], p) >= 0.9 for p in pictures.get(page_no, [])):
                mark(b, "in_picture")
        for small in text_boxes:
            if small["type"] == BLOCK_FURNITURE or len((small.get("text") or "").strip()) < 4:
                continue
            for big in text_boxes:
                if big is small or big["type"] == BLOCK_FURNITURE:
                    continue
                if (big["bbox"][2] * big["bbox"][3] > small["bbox"][2] * small["bbox"][3]
                        and _area_inside(small["bbox"], big["bbox"]) >= 0.8
                        and _text_found_in(small.get("text") or "", big.get("text") or "") >= 0.9):
                    mark(small, "duplicate")
                    break
        taken = {x["id"] for x in blocks}
        for b in [x for x in page_blocks if x["type"] == BLOCK_BYLINE and not x.get("original_text")]:
            split = split_runon_byline(b.get("text") or "")
            if not split:
                continue
            name, rest = split
            b["original_text"], b["text"] = b["text"], name
            already = any(_text_found_in(rest[:60], (o.get("text") or "")[:len(rest) + 40]) >= 0.9
                          for o in page_blocks if o is not b and o["type"] == BLOCK_BODY)
            new_id = None
            if not already:
                n = 0
                while f"s{page_no:04d}-{n:03d}" in taken:
                    n += 1
                new_id = f"s{page_no:04d}-{n:03d}"
                taken.add(new_id)
                x, y, w, h = b["bbox"]
                # The sentence sits on the lower part of the author line's box; placing it there keeps
                # it directly after the name, in the same column, in reading order.
                blocks.append({**{k: v for k, v in b.items() if k not in ("original_text",)},
                               "id": new_id, "type": BLOCK_BODY, "text": rest, "region_label": "",
                               "bbox": [x, y + h * 0.5, w, max(1.0, h * 0.5)], "confidence": 0.7,
                               "reading_order": b.get("reading_order", 0)})
                page_blocks.append(blocks[-1])
            report["byline_split"].append({"page": page_no, "id": b["id"], "text": name,
                                           "moved": rest[:60], "new_id": new_id})

    layout = dict(layout)
    layout["blocks"] = blocks
    for page_no in {x["page"] for x in report["byline_split"] if x["new_id"]}:
        layout = reorder_page(layout, page_no)
    return layout, report


def fold_text_into_picture(layout: dict[str, Any], page_no: int, bbox: Sequence[float]) -> tuple[dict[str, Any], list[str]]:
    """Short text lying inside a picture becomes part of it (not read aloud). Returns the ids changed.

    Used when a region becomes a picture -- drawn as one, or changed into one. A chart's legend and
    axis labels had to be deleted by hand one by one after the chart was marked as an image (a real
    page carried 13 of them). Same limits as clean_up_regions: at most 80 characters, at least 90%
    inside, so real running text caught under a too-large frame is left alone.
    """
    changed: list[str] = []
    blocks = []
    for b in layout.get("blocks", []):
        text = " ".join((b.get("text") or "").split())
        if (b["page"] == page_no and b["type"] not in _VISUAL_TYPES and b["type"] != BLOCK_FURNITURE
                and text and len(text) <= 80 and _area_inside(b["bbox"], bbox) >= 0.9):
            b = dict(b, type=BLOCK_FURNITURE, region_label="in_picture", confidence=0.9)
            changed.append(b["id"])
        blocks.append(b)
    layout = dict(layout)
    layout["blocks"] = blocks
    return layout, changed


# What a region can be changed into from the review screen: text kinds map to a block type (and,
# for furniture, which kind of furniture); "image" turns the region into a figure crop.
REGION_TYPES: dict[str, tuple[str, str]] = {
    "body": (BLOCK_BODY, ""),
    "title": (BLOCK_TITLE, ""),
    "heading": (BLOCK_HEADING, ""),
    "deck": (BLOCK_DECK, ""),
    "byline": (BLOCK_BYLINE, ""),
    "quote": (BLOCK_QUOTE, ""),
    "caption": (BLOCK_CAPTION, ""),
    "pullquote": (BLOCK_PULLQUOTE, ""),
    "sidebar": (BLOCK_SIDEBAR, ""),
    "page_number": (BLOCK_FURNITURE, "page_number"),
    "source": (BLOCK_FURNITURE, "source"),
    "ad": (BLOCK_AD, ""),
    "in_picture": (BLOCK_FURNITURE, "in_picture"),
    "ocr_noise": (BLOCK_FURNITURE, "ocr_noise"),
    "duplicate": (BLOCK_FURNITURE, "duplicate"),
    "table": (BLOCK_TABLE, ""),
    "image": ("", ""),
}


def change_region_type(doc_dir: Path, layout: dict[str, Any], page_no: int, region_id: str,
                       new_type: str,
                       vision_ocr_url: str | None = None,
                       vision_ocr_model: str | None = None,
                       vision_ocr_fallback_url: str | None = None,
                       vision_ocr_fallback_model: str | None = None,
                       timeout: float = 150.0) -> tuple[dict[str, Any], str]:
    """Change what a detected region is. Returns the new layout and the region's id afterwards.

    Between text kinds only the type changes; the text and the box stay. Text to image crops the
    box into a figure and drops the text block; image to text reads the box with OCR (a placeholder
    when nothing is recognised, as for a hand-drawn region) and drops the figure. The id changes
    when a region crosses between text and image, so the caller can move article membership over.
    The page's reading order is recomputed, since a type decides where a region is read.
    """
    if new_type not in REGION_TYPES:
        raise ValueError(f"Unknown region type: {new_type}")
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta:
        raise ValueError("Unknown page")
    block = next((b for b in layout.get("blocks", []) if b["id"] == region_id and b["page"] == page_no), None)
    image = next((im for im in layout.get("images", []) if im["id"] == region_id and im["page"] == page_no), None)
    if block is None and image is None:
        raise ValueError("Region not found on this page")

    if new_type == "image":
        if image is not None:
            return layout, region_id
        before = {im["id"] for im in layout.get("images", [])}
        layout = add_manual_region(doc_dir, layout, page_no, block["bbox"], "image")
        new_id = next(im["id"] for im in layout["images"] if im["id"] not in before)
        layout, _ = fold_text_into_picture(delete_region(layout, region_id), page_no, block["bbox"])
        return reorder_page(layout, page_no), new_id

    block_type, region_label = REGION_TYPES[new_type]
    if block is not None:
        layout = dict(layout)
        layout["blocks"] = [dict(b, type=block_type, region_label=region_label, confidence=1.0)
                            if b["id"] == region_id else b for b in layout.get("blocks", [])]
        return reorder_page(layout, page_no), region_id

    # Image to text: read the box, then give the new block the chosen type.
    before = {b["id"] for b in layout.get("blocks", [])}
    layout = add_manual_region(doc_dir, delete_region(layout, region_id), page_no, image["bbox"], "text", "",
                               vision_ocr_url, vision_ocr_model,
                               vision_ocr_fallback_url, vision_ocr_fallback_model, timeout)
    new_id = next(b["id"] for b in layout["blocks"] if b["id"] not in before)
    layout["blocks"] = [dict(b, type=block_type, region_label=region_label) if b["id"] == new_id else b
                        for b in layout["blocks"]]
    return reorder_page(layout, page_no), new_id


def _page_text_blocks_in_order(layout: dict[str, Any], page_no: int) -> list[dict]:
    return sorted((b for b in layout.get("blocks", []) if b["page"] == page_no and b["type"] not in _VISUAL_TYPES),
                  key=lambda b: b.get("reading_order", 0))


def set_region_text(layout: dict[str, Any], page_no: int, region_id: str, text: str) -> dict[str, Any]:
    """Replace a text region's OCR text with what the reader typed (trusted verbatim)."""
    text = str(text or "").strip()
    if not text:
        raise ValueError("Text must not be empty -- delete the region instead")
    if len(text) > 20000:
        raise ValueError("Text is too long")
    if not any(b["id"] == region_id and b["page"] == page_no for b in layout.get("blocks", [])):
        raise ValueError("Region not found on this page")
    layout = dict(layout)

    def edited(b: dict[str, Any]) -> dict[str, Any]:
        before = str(b.get("text") or "")
        first_read = b.get("ocr_text") or before
        out = dict(b, text=text, confidence=1.0)
        if text == first_read:
            out.pop("ocr_text", None)          # typed back to the original: no longer an edit
        else:
            out["ocr_text"] = first_read
        return out

    layout["blocks"] = [edited(b) if b["id"] == region_id else b for b in layout.get("blocks", [])]
    return layout


def merge_with_next(layout: dict[str, Any], page_no: int, region_id: str) -> tuple[dict[str, Any], str]:
    """Join a text region with the one read after it (a paragraph the OCR cut in two).

    The first region keeps its id, type and place; the text is joined (a word split by a hyphen at
    the line end is closed up again); the box grows to cover both. Returns the layout and the id of
    the region that was folded in, so the caller can drop it from articles.
    """
    ordered = _page_text_blocks_in_order(layout, page_no)
    ids = [b["id"] for b in ordered]
    if region_id not in ids:
        raise ValueError("Region not found on this page")
    i = ids.index(region_id)
    if i + 1 >= len(ordered):
        raise ValueError("This is the last region on the page")
    first, second = ordered[i], ordered[i + 1]
    a, b = " ".join(first["text"].split()), " ".join(second["text"].split())
    joined = remove_linebreak_hyphens(f"{a} {b}") if re.search(r"[a-zäöüß]-$", a) and b[:1].islower() else f"{a} {b}"
    x1, y1 = min(first["bbox"][0], second["bbox"][0]), min(first["bbox"][1], second["bbox"][1])
    x2 = max(first["bbox"][0] + first["bbox"][2], second["bbox"][0] + second["bbox"][2])
    y2 = max(first["bbox"][1] + first["bbox"][3], second["bbox"][1] + second["bbox"][3])
    layout = dict(layout)
    layout["blocks"] = [dict(x, text=joined, bbox=[x1, y1, x2 - x1, y2 - y1]) if x["id"] == first["id"] else x
                        for x in layout.get("blocks", []) if x["id"] != second["id"]]
    return layout, second["id"]


def split_region(layout: dict[str, Any], page_no: int, region_id: str, at: int) -> tuple[dict[str, Any], str]:
    """Split a text region at a character position into two (two paragraphs the OCR fused).

    The box is cut horizontally in proportion to the text on each side, the second part gets a new
    id and is read directly after the first. Returns the layout and the new region's id.
    """
    block = next((b for b in layout.get("blocks", []) if b["id"] == region_id and b["page"] == page_no), None)
    if block is None:
        raise ValueError("Region not found on this page")
    text = block["text"]
    head, tail = text[:at].strip(), text[at:].strip()
    if not head or not tail:
        raise ValueError("Split inside the text, not at its start or end")
    taken = {b["id"] for b in layout.get("blocks", [])}
    n = 0
    while f"t{page_no:04d}-{n:03d}" in taken:
        n += 1
    new_id = f"t{page_no:04d}-{n:03d}"
    x, y, w, h = block["bbox"]
    share = len(head) / max(1, len(head) + len(tail))
    first = dict(block, text=head, bbox=[x, y, w, max(1.0, h * share)])
    second = {**block, "id": new_id, "text": tail, "bbox": [x, y + h * share, w, max(1.0, h * (1 - share))],
              "reading_order": block.get("reading_order", 0) + 0.5}
    second.pop("original_text", None)
    layout = dict(layout)
    blocks = []
    for b in layout.get("blocks", []):
        blocks.extend([first, second] if b["id"] == region_id else [b])
    # Renumber the page so the new part sits directly after the first, whatever else is locked.
    page = sorted((b for b in blocks if b["page"] == page_no), key=lambda b: b.get("reading_order", 0))
    position = {b["id"]: i for i, b in enumerate(page)}
    layout["blocks"] = [dict(b, reading_order=position[b["id"]]) if b["page"] == page_no else b for b in blocks]
    return layout, new_id


def set_page_order(layout: dict[str, Any], page_no: int, order: Sequence[str]) -> dict[str, Any]:
    """Set a page's reading order by hand and lock it, so nothing automatic reshuffles it again.

    `order` lists region ids (text blocks and figures) in the order they are read. Regions left
    out keep their current relative order after the listed ones, so ordering only the first few
    boxes is enough. Text and figures share one sequence, which is also where a figure is shown
    during playback.
    """
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta:
        raise ValueError("Unknown page")
    blocks = [b for b in layout.get("blocks", []) if b["page"] == page_no]
    images = [i for i in layout.get("images", []) if i["page"] == page_no]
    current = sorted(blocks + images, key=lambda r: (r.get("reading_order", 0), 0 if r in blocks else 1))
    known = {r["id"] for r in current}
    unknown = [rid for rid in order if rid not in known]
    if unknown:
        raise ValueError(f"Not on page {page_no}: {', '.join(map(str, unknown[:5]))}")
    listed: list[str] = []
    for rid in order:
        if rid not in listed:
            listed.append(rid)
    sequence = listed + [r["id"] for r in current if r["id"] not in listed]
    position = {rid: i for i, rid in enumerate(sequence)}
    layout = dict(layout)
    layout["blocks"] = [dict(b, reading_order=position[b["id"]]) if b["page"] == page_no else b
                        for b in layout.get("blocks", [])]
    layout["images"] = [dict(i, reading_order=position[i["id"]]) if i["page"] == page_no else i
                        for i in layout.get("images", [])]
    layout["pages"] = [dict(p, order_locked=True) if p["page"] == page_no else p for p in layout.get("pages", [])]
    return layout


def clear_page_order(layout: dict[str, Any], page_no: int) -> dict[str, Any]:
    """Undo a hand-made order: unlock the page and let the automatic reading order take over."""
    if not any(p["page"] == page_no for p in layout.get("pages", [])):
        raise ValueError("Unknown page")
    layout = dict(layout)
    layout["pages"] = [{k: v for k, v in p.items() if k != "order_locked"} if p["page"] == page_no else p
                       for p in layout.get("pages", [])]
    return reorder_page(layout, page_no)


def recompute_reading_order(layout: dict[str, Any]) -> tuple[dict[str, Any], list[int]]:
    """Re-derive the reading order of every page in an issue, without any OCR.

    For an issue ingested before the column detection improved. Only the order changes: text,
    boxes, types, articles and approvals stay as they are, and a page whose order the reader set by
    hand is left alone. Returns the layout and the pages whose order actually changed.
    """
    changed: list[int] = []
    for page_meta in layout.get("pages", []):
        page_no = page_meta["page"]
        before = [b["id"] for b in sorted((b for b in layout.get("blocks", []) if b["page"] == page_no),
                                          key=lambda b: b.get("reading_order", 0))]
        layout = reorder_page(layout, page_no)
        after = [b["id"] for b in sorted((b for b in layout.get("blocks", []) if b["page"] == page_no),
                                         key=lambda b: b.get("reading_order", 0))]
        if before != after:
            changed.append(page_no)
    return layout, changed


def reorder_page(layout: dict[str, Any], page_no: int) -> dict[str, Any]:
    """Recompute one page's reading order -- unless the reader fixed that order by hand."""
    page_meta = next((p for p in layout.get("pages", []) if p["page"] == page_no), None)
    if not page_meta or page_meta.get("order_locked") or not page_meta.get("width"):
        return layout
    # Only the position is written back, so fields the Block model does not carry survive.
    fields = Block.__dataclass_fields__
    page_dicts = [b for b in layout.get("blocks", []) if b["page"] == page_no]
    models = [Block(**{k: v for k, v in b.items() if k in fields}) for b in page_dicts]
    assign_reading_order(models, page_meta["width"])
    placed = {m.id: (m.reading_order, m.column) for m in models}
    layout = dict(layout)
    layout["blocks"] = [dict(b, reading_order=placed[b["id"]][0], column=placed[b["id"]][1])
                        if b["page"] == page_no else b for b in layout.get("blocks", [])]
    return layout


def region_ids_with(layout: dict[str, Any], region_id: str) -> set[str]:
    """The ids that go when `region_id` is deleted: the region itself and, for a picture, its other half.

    A picture region is stored twice: a visual block (figure/chart/table/code, empty or carrying OCR text) and an
    image entry whose `source_block` names it. The review list shows either one as a row, so deleting "the image"
    used to leave the block behind as a box with an empty paragraph in the same place (and the other way round).
    """
    ids = {region_id}
    for im in layout.get("images", []):
        if im["id"] == region_id and im.get("source_block"):
            twin = next((b for b in layout.get("blocks", []) if b["id"] == im["source_block"]), None)
            if twin is not None and twin.get("type") in VISUAL_TYPES:     # never take a text block that was retyped since
                ids.add(twin["id"])
    ids |= {im["id"] for im in layout.get("images", []) if im.get("source_block") == region_id}
    return ids


def delete_region(layout: dict[str, Any], block_id: str) -> dict[str, Any]:
    """Remove a region the layout engine got wrong entirely (not just misplaced) -- both halves of a picture region
    (see region_ids_with). An article referencing a removed id in its own blocks/images list is left as-is here:
    article_text() and the review screen treat an id no longer in the layout as absent, and the routes drop the
    ids from the articles themselves."""
    gone = region_ids_with(layout, block_id)
    layout = dict(layout)
    layout["blocks"] = [b for b in layout.get("blocks", []) if b["id"] not in gone]
    layout["images"] = [im for im in layout.get("images", []) if im["id"] not in gone]
    return layout


def save_layout(doc_dir: Path, layout: dict[str, Any]) -> None:
    (doc_dir / "layout.json").write_text(
        json.dumps(layout, ensure_ascii=False), "utf-8")


def load_layout(doc_dir: Path) -> dict[str, Any]:
    path = doc_dir / "layout.json"
    if not path.is_file():
        return {}
    try:
        return json.loads(path.read_text("utf-8"))
    except (ValueError, OSError):
        return {}


def save_articles(doc_dir: Path, articles: list[dict[str, Any]]) -> None:
    (doc_dir / "articles.json").write_text(
        json.dumps({"articles": articles}, ensure_ascii=False), "utf-8")


def load_articles(doc_dir: Path) -> list[dict[str, Any]]:
    path = doc_dir / "articles.json"
    if not path.is_file():
        return []
    try:
        return json.loads(path.read_text("utf-8")).get("articles", [])
    except (ValueError, OSError):
        return []


BLOCK_FIGURE_DESCRIPTION = "figure_description"


def page_is_advertisement(layout: dict[str, Any], page_no: int) -> bool:
    """A page is an advertisement when it was flagged as one, or detection found advertisement
    regions on it and nothing narratable -- a full-page ad needs no manual approval."""
    meta = next((p for p in layout.get("pages", []) if p.get("page") == page_no), None)
    if meta is None:
        return False
    if meta.get("advertisement"):
        return True
    on_page = [b for b in layout.get("blocks", []) if b.get("page") == page_no]
    return (any(b.get("type") == BLOCK_AD for b in on_page)
            and not any(b.get("type") in NARRATABLE for b in on_page))


def article_asides(layout: dict[str, Any], article: dict[str, Any],
                   all_articles: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:
    """Sidebars lying on this article's pages, to show beside the text but never narrate.

    A sidebar is kept out of narration (mark_sidebars), which also keeps it out of the article's own block
    list -- so the reading view never showed it. Only pages this article alone occupies are used: on a page
    shared by several articles there is no telling which one a sidebar belongs to, and showing it in the
    wrong one is worse than not showing it. Advertisement pages carry none. Each entry says which of the
    article's narrated regions it follows (`after`, or None = before the first on its page).
    """
    mine = {int(p) for p in article.get("pages") or []}
    shared = {int(p) for a in all_articles if a.get("id") != article.get("id") for p in a.get("pages") or []}
    pages = {p for p in mine - shared if not page_is_advertisement(layout, p)}
    narrated = set(article.get("blocks") or [])
    blocks = layout.get("blocks", [])
    out = []
    for b in blocks:
        if b.get("type") != BLOCK_SIDEBAR or b.get("page") not in pages or b.get("id") in narrated:
            continue
        text = strip_continuation_markers(str(b.get("text") or "")).strip()
        if not text:
            continue
        before = [x for x in blocks if x.get("page") == b["page"] and x.get("id") in narrated
                  and (x.get("reading_order") or 0) < (b.get("reading_order") or 0)]
        after = max(before, key=lambda x: x.get("reading_order") or 0)["id"] if before else None
        out.append({"id": b["id"], "page": b["page"], "bbox": b.get("bbox"), "text": text[:4000],
                    "after": after, "_order": (b["page"], b.get("reading_order") or 0)})
    out.sort(key=lambda a: a["_order"])
    for a in out:
        del a["_order"]
    return out


def article_text(layout: dict[str, Any], article: dict[str, Any],
                 include_captions: bool = False,
                 include_descriptions: bool = False,
                 fuse: bool = False,
                 glossary: dict[str, str] | None = None) -> list[dict[str, Any]]:
    """The narratable text of one article, in reading order.

    Returned per block so the player can highlight the source region and jump
    to the originating PDF page.

    A figure/chart/table/code region has no text block of its own to
    narrate -- see describe_visual_region() -- so an already-described
    image belonging to this article is synthesised into a
    BLOCK_FIGURE_DESCRIPTION entry at that image's own reading position when
    `include_descriptions` is set. Optional and off by default: generating
    a description costs a vision-model call the user may not have wanted
    for every figure, and a description read aloud mid-article changes the
    listening experience enough that it should be an explicit choice, not a
    silent side effect of one being available.

    `fuse` joins a paragraph that continues across a column or page break
    into one block (see fuse_paragraph_blocks) so narration does not stop
    mid-sentence at the seam. `glossary` is the article's own acronym
    glossary (see core.acronyms); the language is decided once for the whole
    article, because a short fragment alone is a poor language signal.
    """
    # A page the reviewer approved as an advertisement is never narrated, whatever article claims it.
    ad_pages = {p["page"] for p in layout.get("pages", []) if page_is_advertisement(layout, p["page"])}
    wanted = set(article.get("blocks") or [])
    types = set(NARRATABLE) | ({BLOCK_CAPTION} if include_captions else set())
    chosen = [b for b in layout.get("blocks", [])
              if b["id"] in wanted and b["type"] in types and b["page"] not in ad_pages]
    language = detect_article_language(strip_continuation_markers(b["text"]) for b in chosen)
    blocks = []
    for b in chosen:
        raw = strip_continuation_markers(b["text"])
        lang = language_for_block(raw, language)
        blocks.append(dict(
            b,
            language=lang,
            text=normalize_ocr_text(raw, lang),
            # Keep faithful print text for reading/highlighting, but give live
            # TTS the same pronunciation expansion as exported article audio.
            spoken_text=prepare_text_for_tts(raw, lang, glossary),
        ))
    if include_descriptions:
        wanted_images = set(article.get("images") or [])
        for img in layout.get("images", []):
            description = (img.get("description") or "").strip()
            if img["id"] not in wanted_images or not description or img["page"] in ad_pages:
                continue
            blocks.append({
                "id": f"desc-{img['id']}", "page": img["page"], "bbox": img["bbox"],
                "type": BLOCK_FIGURE_DESCRIPTION, "text": description,
                "reading_order": img.get("reading_order", 0),
            })
    blocks.sort(key=lambda b: (b["page"], b["reading_order"]))
    return fuse_paragraph_blocks(blocks, language, glossary) if fuse else blocks


# Punctuation that closes a sentence, optionally followed by closing quotes/brackets.
_SENTENCE_END_RE = re.compile(r"[.!?…:;]['\"”’»«›‹)\]]*$")
_WORD_TAIL_RE = re.compile(r"([^\W\d_]+)-$")
_WORD_HEAD_RE = re.compile(r"[^\W\d_]+")
# A block ending on one of these cannot be a finished sentence, even when the
# next fragment starts with a capital (every German noun does).
_OPEN_ENDING_WORDS = frozenset({
    "der", "die", "das", "den", "dem", "des", "ein", "eine", "einen", "einem", "einer", "eines",
    "und", "oder", "aber", "sowie", "als", "wie", "wenn", "dass", "weil", "denn", "doch",
    "von", "vom", "zu", "zum", "zur", "mit", "bei", "für", "auf", "aus", "in", "im", "an", "am",
    "nach", "über", "unter", "durch", "gegen", "ohne", "um", "ist", "sind", "wird", "werden",
    "hat", "haben", "nicht", "auch", "sich", "so",
    "the", "a", "an", "and", "or", "but", "of", "to", "in", "on", "at", "for", "with", "by",
    "from", "as", "that", "which", "is", "are", "was", "were", "be", "not", "than",
})
_FUSABLE = (BLOCK_BODY,)
_PASS_THROUGH = (BLOCK_CAPTION, BLOCK_FIGURE_DESCRIPTION)


def paragraph_continues(previous: str, following: str) -> bool:
    """True when `following` reads on from `previous` (same paragraph, next fragment).

    A finished sentence never continues. Otherwise a lower-case start, a
    trailing hyphen/comma or an ending on a function word (article,
    preposition, conjunction) says the sentence is still open; a fragment that
    merely lacks final punctuation and starts a new capitalised sentence is
    left alone so two genuine paragraphs are never glued together.
    """
    prev, nxt = (previous or "").rstrip(), (following or "").lstrip()
    if not prev or not nxt or _INTERVIEW_LABEL_RE.match(nxt):
        return False
    if prev.endswith(("-", "\u00ad")):
        return True
    if _SENTENCE_END_RE.search(prev):
        return False
    if nxt[0].islower() or nxt[0] in ",;)":
        return True
    words = _WORD_HEAD_RE.findall(prev)
    return prev.endswith(",") or bool(words and words[-1].casefold() in _OPEN_ENDING_WORDS)


def _join_fragments(previous: str, following: str, language: str) -> tuple[str, int]:
    """Join two paragraph fragments, repairing a word split at the seam.

    Returns the joined text and the offset in it where the second fragment's
    own text starts (a word rebuilt across the seam stays with the first).
    """
    prev, nxt = previous.rstrip(), following.lstrip()
    if prev.endswith("\u00ad"):
        prev = prev[:-1] + "-"
    tail = _WORD_TAIL_RE.search(prev)
    head = _WORD_HEAD_RE.match(nxt)
    if not (tail and head) or head.group(0).casefold() in _SUSPENDED_COMPOUND_WORDS:
        return prev + " " + nxt, len(prev) + 1       # also "Hardware- und Software"
    first = head.group(0)
    if first[:1].islower():                           # printed line break: eingetrich- + tert
        joined = prev[:tail.start()] + normalize_ocr_text(tail.group(1) + first, language) + nxt[len(first):]
        return joined, len(joined) - (len(nxt) - len(first))
    return prev + nxt, len(prev)                      # "KI-" + "Modelle": the hyphen is real


def fuse_paragraph_blocks(blocks: list[dict[str, Any]], language: str = "auto",
                          glossary: dict[str, str] | None = None) -> list[dict[str, Any]]:
    """Join body regions that are one paragraph split by a column or page break.

    Input is an article's blocks in reading order. Each fused block keeps the
    first region's id/page/bbox and gains `parts` -- (id, page, bbox, start,
    end) offsets into the fused text -- so the player can still highlight and
    jump to the region a sentence came from. Captions and figure descriptions
    that sit between two fragments are emitted after the paragraph, never in
    the middle of a sentence.
    """
    out: list[dict[str, Any]] = []
    group: dict[str, Any] | None = None
    held: list[dict[str, Any]] = []

    def part(b: dict[str, Any], start: int, end: int) -> dict[str, Any]:
        return {"id": b["id"], "page": b["page"], "bbox": b.get("bbox"), "start": start, "end": end}

    def flush() -> None:
        nonlocal group, held
        if group is not None:
            if len(group["parts"]) > 1:
                group["spoken_text"] = prepare_text_for_tts(group["text"], group.get("language") or language, glossary)
            out.append(group)
        out.extend(held)
        group, held = None, []

    for b in blocks:
        text = (b.get("text") or "").strip()
        if b["type"] in _PASS_THROUGH and group is not None:
            held.append(b)
            continue
        if (b["type"] in _FUSABLE and group is not None and text
                and paragraph_continues(group["text"], text)):
            joined, start = _join_fragments(group["text"], text, group.get("language") or language)
            group["parts"][-1]["end"] = start
            group["parts"].append(part(b, start, len(joined)))
            group["text"] = joined
            group["fused_ids"].append(b["id"])
            continue
        flush()
        if b["type"] in _FUSABLE and text:
            group = dict(b, text=text, fused_ids=[b["id"]])
            group["parts"] = [part(b, 0, len(text))]
        else:
            out.append(b)
    flush()
    return out


def _flow_relation(a: dict[str, Any], b: dict[str, Any]) -> str:
    """How the eye travels from region a to region b: below / column / page."""
    if a["page"] != b["page"]:
        return "page"
    ax, aw = a["bbox"][0], a["bbox"][2]
    bx, bw = b["bbox"][0], b["bbox"][2]
    overlap = max(0.0, min(ax + aw, bx + bw) - max(ax, bx))
    same_column = overlap >= 0.5 * min(aw, bw)          # also a wide headline above its column
    return "below" if same_column and b["bbox"][1] >= a["bbox"][1] else "column"


def article_flow(layout: dict[str, Any], article: dict[str, Any]) -> dict[str, Any]:
    """Where an article's text carries on, region to region, in reading order.

    Every consecutive pair of narrated regions is a link. `relation` says how the eye travels:
    `below` (the next region under it, same column), `column` (top of another column on the same
    page) or `page` (another page). `sentence` is true when a sentence runs on across the link
    (the same fusion narration uses), false when the next region simply starts a new paragraph.
    Also returns the article's first and last regions.
    """
    plain = article_text(layout, article)
    fused = article_text(layout, article, fuse=True)
    running: set[tuple[str, str]] = set()
    for block in fused:
        parts = block.get("parts") or []
        running.update((a["id"], b["id"]) for a, b in zip(parts, parts[1:]))
    narrated = [b for b in plain if b.get("type") in NARRATABLE]
    links = [{"from": a["id"], "to": b["id"], "relation": _flow_relation(a, b),
              "from_page": a["page"], "to_page": b["page"], "sentence": (a["id"], b["id"]) in running}
             for a, b in zip(narrated, narrated[1:])]
    last = narrated[-1] if narrated else None
    ended = False
    if last is not None:
        # Two signs an article stops: it is signed off ("(rah) ct"), or the next region -- on this
        # page, else at the top of the next -- is a headline, deck or byline: something new starts.
        flowing = sorted((b for b in layout.get("blocks", [])
                          if b["type"] in NARRATABLE and (b.get("text") or "").strip()),
                         key=lambda b: (b["page"], b.get("reading_order", 0)))
        at = next((i for i, b in enumerate(flowing) if b["id"] == last["id"]), None)
        following = flowing[at + 1] if at is not None and at + 1 < len(flowing) else None
        ended = bool(_ARTICLE_END_RE.search((last.get("text") or "").strip())
                     or (following is not None and following["type"] in _LINK_BOUNDARY_TYPES
                         and following["page"] - last["page"] <= 1)
                     or following is None)
    return {"links": links, "first": narrated[0]["id"] if narrated else None,
            "last": last["id"] if last else None,
            "last_page": last["page"] if last else None,
            "ended": ended}


_ARTICLE_END_RE = re.compile(r"\(\s*[A-Za-zÄÖÜäöü]{2,4}\s*\)\s*\S{0,4}\s*$")   # "(hag)", "(atr) c't": a signed end
_PLACEHOLDER_TITLE = re.compile(r"Page \d+")


def _is_placeholder_article(article: dict[str, Any]) -> bool:
    return bool(_PLACEHOLDER_TITLE.fullmatch((article.get("title") or "").strip()))


_REFERENCE_HEAD_RE = re.compile(r"^\s*(literatur|quellen|studien, weitere quellen|weiterführende|links?\b|anmerkungen?\b)", re.I)


def looks_like_page_junk(text: str, repeats: int = 1) -> bool:
    """Text that is on a page but is not something to narrate: a running head that recurs on many
    pages, a list of page numbers, or a reference/source list. Never used to delete anything the
    reviewer approved -- only to keep such text out of an article automatically."""
    t = " ".join((text or "").split())
    if not t:
        return True
    letters = sum(c.isalpha() for c in t)
    if letters < 0.5 * len(t.replace(" ", "")) and len(t) < 200:
        return True                                    # "66 90,98 50,56 82, 114 ..."
    if repeats >= 3 and len(t) < 80:
        return True                                    # the same short line on 3+ pages: a running head
    if _REFERENCE_HEAD_RE.match(t) and len(t) < 400:
        return True
    return False


def auto_link_pages(layout: dict[str, Any], articles: list[dict[str, Any]],
                    max_gap_pages: int = 2) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    """Attach text that no real article owns to the article running from the page before.

    Detection often finds an article's headline page and leaves its continuation pages as
    one-page stand-ins ("Page 61"). Reading in page order, text no real article owns is:
    a NEW article when a headline followed by a byline or deck starts it; otherwise a
    CONTINUATION of the article running just before, unless that article already ended (a signed
    end such as ``(hag)``) or is more than `max_gap_pages` pages back -- then it stays unlinked
    for a human. Stand-in articles whose text all moved are dropped; an approved article that
    gains text goes back to *detected* so nothing unreviewed is ever narrated.
    Returns the articles and a report {linked, created, unlinked, dropped}.
    """
    articles = [dict(a, blocks=list(a.get("blocks") or []), pages=list(a.get("pages") or [])) for a in articles]
    real = [a for a in articles if not _is_placeholder_article(a)]
    owner = {b: a for a in real for b in a["blocks"]}
    # Regions the reviewer already approved (as part of any article, stand-ins included): moving
    # them into another article does not make them unreviewed.
    approved_ids = {b for a in articles if a.get("status") == "approved" for b in a["blocks"]}
    blocks = {b["id"]: b for b in layout.get("blocks", [])}
    by_page: dict[int, list[dict[str, Any]]] = {}
    seen: dict[str, int] = {}
    for b in layout.get("blocks", []):
        if b["type"] in NARRATABLE and (b.get("text") or "").strip():
            key = re.sub(r"\d+", "#", " ".join(b["text"].split()).casefold())
            seen[key] = seen.get(key, 0) + 1
    for b in layout.get("blocks", []):
        if b["type"] in NARRATABLE and (b.get("text") or "").strip():
            key = re.sub(r"\d+", "#", " ".join(b["text"].split()).casefold())
            if b["id"] not in owner and looks_like_page_junk(b["text"], seen[key]):
                continue                               # not narratable page furniture: leave it out
            by_page.setdefault(b["page"], []).append(b)
    report = {"linked": 0, "created": 0, "unlinked": [], "dropped": 0}
    taken = {int(m.group(1)) for a in articles if (m := re.fullmatch(r"article-(\d+)", a["id"]))}
    moved: set[str] = set()
    running: dict[str, Any] | None = None
    running_page = 0
    running_tail = ""

    def next_id() -> str:
        n = max(taken, default=0) + 1
        taken.add(n)
        return f"article-{n:03d}"

    for page in sorted(by_page):
        ordered = sorted(by_page[page], key=lambda b: b.get("reading_order", 0))
        i = 0
        while i < len(ordered):
            block = ordered[i]
            if block["id"] in owner:
                running, running_page, running_tail = owner[block["id"]], page, block["text"]
                i += 1
                continue
            j = i                                        # collect this run of unowned text
            while j < len(ordered) and ordered[j]["id"] not in owner:
                j += 1
            segment = ordered[i:j]
            i = j
            cut = [0]
            for k, b in enumerate(segment):              # a headline + byline/deck opens a new piece
                nxt = [x["type"] for x in segment[k + 1:k + 3]]
                if k and b["type"] in (BLOCK_TITLE, BLOCK_HEADING) and (BLOCK_BYLINE in nxt or BLOCK_DECK in nxt):
                    cut.append(k)
            for start, end in zip(cut, cut[1:] + [len(segment)]):
                part = segment[start:end]
                head = part[0]
                opens = head["type"] in (BLOCK_TITLE, BLOCK_HEADING) and any(
                    x["type"] in (BLOCK_BYLINE, BLOCK_DECK) for x in part[1:3])
                if opens:
                    byline = next((x for x in part[1:4] if x["type"] == BLOCK_BYLINE), None)
                    fresh = Article(id=next_id(), title=" ".join(head["text"].split())[:300],
                                    author=re.sub(r"^(von|by)\s+", "", (byline or {}).get("text", ""), flags=re.I).strip()[:120],
                                    pages=sorted({x["page"] for x in part}), blocks=[x["id"] for x in part],
                                    confidence=0.55).to_json()
                    if all(x["id"] in approved_ids for x in part):
                        fresh["status"] = "approved"
                    articles.append(fresh)
                    real.append(fresh)
                    owner.update({b: fresh for b in fresh["blocks"]})
                    moved.update(fresh["blocks"])
                    running, running_page, running_tail = fresh, page, part[-1]["text"]
                    report["created"] += 1
                elif (running is not None and page - running_page <= max_gap_pages
                      and not _ARTICLE_END_RE.search(running_tail.strip())):
                    running["blocks"].extend(x["id"] for x in part)
                    running["pages"] = sorted(set(running["pages"]) | {x["page"] for x in part})
                    if running.get("status") == "approved" and not all(x["id"] in approved_ids for x in part):
                        running["status"] = "detected"      # gained text nobody has approved yet
                    owner.update({x["id"]: running for x in part})
                    moved.update(x["id"] for x in part)
                    running_page, running_tail = page, part[-1]["text"]
                    report["linked"] += len(part)
                else:
                    report["unlinked"].append(part[0]["id"])
    absorbed_by_page: dict[int, dict[str, Any]] = {}
    for article in real:
        for b in article["blocks"]:
            if b in moved and b in blocks:
                absorbed_by_page[blocks[b]["page"]] = article
    kept = []
    for a in articles:
        if _is_placeholder_article(a):
            # text that moved, or that a real article owns anyway, no longer needs a stand-in
            a["blocks"] = [b for b in a["blocks"] if b not in moved and b not in owner]
            still_text = any(blocks.get(b, {}).get("type") in NARRATABLE for b in a["blocks"])
            if not still_text:
                target = next((absorbed_by_page[p] for p in a.get("pages", []) if p in absorbed_by_page), None)
                if target is not None and a.get("images"):       # its pictures follow the text
                    target["images"] = list(dict.fromkeys([*target.get("images", []), *a["images"]]))
                report["dropped"] += 1
                continue
        kept.append(a)
    # An article whose every text region the reviewer had already approved (in whichever article
    # it sat before) is as reviewed as it was: do not make it be approved a second time.
    for a in kept:
        narrated = [b for b in a["blocks"] if blocks.get(b, {}).get("type") in NARRATABLE]
        if a.get("status") == "detected" and narrated and all(b in approved_ids for b in narrated):
            a["status"] = "approved"
    return kept, report


_LINK_BOUNDARY_TYPES = (BLOCK_TITLE, BLOCK_HEADING, BLOCK_DECK, BLOCK_BYLINE)


def blocks_to_link(layout: dict[str, Any], article: dict[str, Any], page: int, where: str) -> list[str]:
    """Regions on `page` to add so the article continues there (`after`) or starts there (`before`).

    after: from the first body region that carries the article on (the sentence runs on, or
    simply the page's first body region) down to, but not including, the next headline, deck or
    byline -- that is a different piece. before: from the page's last headline (the article's
    opening) to the end, or all its body text when it has none.
    """
    own = set(article.get("blocks") or [])
    on_page = sorted((b for b in layout.get("blocks", [])
                      if b["page"] == page and b["type"] in NARRATABLE and (b.get("text") or "").strip()
                      and b["id"] not in own), key=lambda b: b.get("reading_order", 0))
    if where == "before":
        starts = [i for i, b in enumerate(on_page) if b["type"] in _LINK_BOUNDARY_TYPES]
        chosen = on_page[starts[-1]:] if starts else [b for b in on_page if b["type"] == BLOCK_BODY]
        return [b["id"] for b in chosen]
    narrated = [b for b in article_text(layout, article) if b.get("type") in NARRATABLE]
    tail = (narrated[-1]["text"] if narrated else "")
    first = next((i for i, b in enumerate(on_page) if b["type"] == BLOCK_BODY), None)
    if first is None:
        return []
    # The paragraph that carries the article on: one starting lower-case is the surest sign of a
    # sentence running over; failing that, one the fusion rules accept; failing that, the first.
    bodies = [(i, normalize_ocr_text(strip_continuation_markers(b["text"])))
              for i, b in enumerate(on_page) if b["type"] == BLOCK_BODY]
    carry = next((i for i, t in bodies if t[:1].islower()), None)
    if carry is None:
        carry = next((i for i, t in bodies if paragraph_continues(tail, t)), None)
    first = first if carry is None else carry
    chosen = []
    for b in on_page[first:]:
        if b["type"] in _LINK_BOUNDARY_TYPES:
            break
        if b["type"] in (BLOCK_BODY, BLOCK_QUOTE):
            chosen.append(b["id"])
    return chosen


# ── Standalone audio (Stage E, optional) ─────────────────────────────────────
# The in-browser reader synthesises sentence-by-sentence into memory and never
# writes a file -- fine for reading inside this app, useless for a car head
# unit or a phone's own podcast/music player. This renders one article down
# to a single tagged MP3 a person can actually take with them.

_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?…])\s+(?=[A-ZÀ-Ü»„\"'‘])")


def split_sentences_for_tts(text: str) -> list[str]:
    return [p.strip() for p in _SENTENCE_SPLIT_RE.split(text) if p.strip()]


_INTERVIEW_LABEL_RE = re.compile(
    r"^\s*([A-ZÄÖÜ][A-Za-zÄÖÜäöüß.'’\-]+(?:\s+[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.'’\-]+){0,3})\s*:\s+(.+)$",
    re.S,
)


def interview_tts_segments(blocks: Sequence[dict[str, Any]]) -> list[dict[str, str | None]]:
    """Return article text segments with speaker labels for a two-person interview.

    Detection is intentionally strict: exactly two recurring labels must begin
    paragraph blocks.  Colons in ordinary prose therefore do not suddenly
    switch voices.  Speaker labels themselves are omitted from speech because
    the voice change already conveys who is talking.
    """
    parsed: list[tuple[str | None, str]] = []
    counts: dict[str, int] = {}
    labels: dict[str, str] = {}
    for block in blocks:
        text = (block.get("text") or "").strip()
        if not text:
            continue
        match = _INTERVIEW_LABEL_RE.match(text)
        if not match:
            parsed.append((None, text))
            continue
        label, body = match.group(1).strip(), match.group(2).strip()
        key = label.casefold()
        labels.setdefault(key, label)
        counts[key] = counts.get(key, 0) + 1
        parsed.append((key, body))

    recurring = [key for key, count in counts.items() if count >= 2]
    if len(recurring) != 2:
        return [{"speaker": None, "text": text} for _, text in parsed]
    allowed = set(recurring)
    return [
        {"speaker": labels[speaker] if speaker in allowed else None, "text": text}
        for speaker, text in parsed
    ]


def synthesize_article_audio(doc_dir: Path, article: dict[str, Any], layout: dict[str, Any],
                             voice: str, settings: dict[str, Any], *,
                             interview_voice: str = "",
                             album: str = "", cover_bytes: bytes | None = None,
                             cover_mime: str = "image/png", progress_cb=None) -> Path:
    """Render one article to a single MP3, tagged with title/artist/album,
    its own text as embedded lyrics, and a cover image.

    progress_cb(done, total), if given, is called after each sentence.
    """
    from pydub import AudioSegment
    import io as _io
    from core.tts_helpers import _tts_request_audio

    glossary = article.get("glossary")
    # Fused: a paragraph split by a column or page break is one block, so no
    # sentence is cut in two at the seam.
    blocks = article_text(layout, article, fuse=True, glossary=glossary)
    full_text = " ".join(b["text"] for b in blocks if (b.get("text") or "").strip()).strip()
    if not full_text:
        raise ValueError("Article has no narratable text -- approve it with at least one block ticked")
    language = detect_article_language(b["text"] for b in blocks)
    segments = interview_tts_segments(blocks) if interview_voice else [
        {"speaker": None, "text": b["text"], "spoken": b.get("spoken_text")}
        for b in blocks if (b.get("text") or "").strip()
    ]
    speakers = []
    for segment in segments:
        speaker = segment.get("speaker")
        if speaker and speaker not in speakers:
            speakers.append(speaker)
    voice_for = {speaker: (voice if i == 0 else interview_voice)
                 for i, speaker in enumerate(speakers[:2])}
    utterances: list[tuple[str, str]] = []
    utterance_segment: list[int] = []        # which segment each utterance came from, for the timing record
    for n, segment in enumerate(segments):
        spoken = segment.get("spoken") or prepare_text_for_tts(str(segment["text"]), language, glossary)
        segment_voice = voice_for.get(segment.get("speaker"), voice)
        for sentence in split_sentences_for_tts(spoken):
            utterances.append((sentence, segment_voice))
            utterance_segment.append(n)
    total = len(utterances)

    combined = AudioSegment.empty()
    spans: list[tuple[int, int]] = []        # [start_ms, end_ms) of every utterance in the final MP3
    for i, (sentence, sentence_voice) in enumerate(utterances):
        audio_bytes, media_type = _tts_request_audio(sentence, sentence_voice, settings)
        fmt = "wav" if audio_bytes[:4] == b"RIFF" or "wav" in media_type.lower() else None
        piece = AudioSegment.from_file(_io.BytesIO(audio_bytes), format=fmt)
        spans.append((len(combined), len(combined) + len(piece)))
        combined += piece
        if progress_cb:
            progress_cb(i + 1, total)

    audio_dir = doc_dir / "audio"
    audio_dir.mkdir(parents=True, exist_ok=True)
    out_path = audio_dir / f"{article['id']}.mp3"
    combined.export(out_path, format="mp3", bitrate="128k")
    try:
        tag_article_mp3(out_path, article, full_text, album=album,
                        cover_bytes=cover_bytes, cover_mime=cover_mime)
    except ImportError:
        # Tags (title, artist, cover, lyrics) are a convenience; losing them must not throw away minutes of
        # synthesis. The audio and its timings are complete without them.
        logging.getLogger(__name__).warning("mutagen is not installed: %s was saved without ID3 tags", out_path.name)
    write_audio_timings(out_path, segments, utterance_segment, spans, interview=bool(interview_voice))
    return out_path


def mp3_duration_seconds(path: Path) -> float | None:
    """Length of an MP3 in seconds: mutagen when installed, else ffprobe, else None (never raises)."""
    try:
        from mutagen import File as MutagenFile
        parsed = MutagenFile(path)
        if parsed is not None and parsed.info is not None and parsed.info.length:
            return float(parsed.info.length)
    except Exception:
        pass
    try:
        out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of",
                              "default=nw=1:nk=1", str(path)], capture_output=True, text=True, timeout=20, check=True).stdout
        value = float(out.strip())
        return value if value > 0 else None
    except (subprocess.SubprocessError, OSError, ValueError):
        return None


def write_audio_timings(mp3_path: Path, segments: Sequence[dict[str, Any]], utterance_segment: Sequence[int],
                        spans: Sequence[tuple[int, int]], *, interview: bool) -> None:
    """Record where every spoken sentence sits in the MP3 (`<id>.timings.json` beside it).

    Measured, not estimated: the sentences were rendered one by one and joined, so each one's length is
    known exactly. Holds no text -- only, per narrated segment, its character count (to notice later that
    the text changed) and the [start, end] milliseconds of each of its sentences. Best-effort: a failed
    write only costs the EPUB its exact timings.
    """
    per_segment: list[list[list[int]]] = [[] for _ in segments]
    for n, (start, end) in zip(utterance_segment, spans):
        per_segment[n].append([int(start), int(end)])
    record = {
        "version": 1, "interview": bool(interview), "duration_ms": int(spans[-1][1]) if spans else 0,
        "segments": [{"chars": len(str(seg.get("text") or "")), "times": times}
                     for seg, times in zip(segments, per_segment)],
    }
    try:
        tmp = mp3_path.with_suffix(".timings.tmp")
        tmp.write_text(json.dumps(record), "utf-8")
        tmp.replace(mp3_path.with_suffix(".timings.json"))
    except OSError:
        logging.getLogger(__name__).warning("Could not write sentence timings for %s", mp3_path.name)


def read_audio_timings(mp3_path: Path) -> dict[str, Any] | None:
    try:
        data = json.loads(mp3_path.with_suffix(".timings.json").read_text("utf-8"))
    except (OSError, ValueError):
        return None
    return data if isinstance(data, dict) and data.get("version") == 1 and isinstance(data.get("segments"), list) else None


def tag_article_mp3(path: Path, article: dict[str, Any], lyrics: str, *,
                    album: str = "", cover_bytes: bytes | None = None,
                    cover_mime: str = "image/png") -> None:
    from mutagen.id3 import ID3, ID3NoHeaderError, TIT2, TPE1, TALB, USLT, APIC

    try:
        tags = ID3(path)
    except ID3NoHeaderError:
        tags = ID3()
    title = (article.get("title") or "Untitled").strip()
    author = (article.get("author") or "").strip()
    tags.setall("TIT2", [TIT2(encoding=3, text=title)])
    tags.setall("TPE1", [TPE1(encoding=3, text=author or album or "Unknown")])
    if album:
        tags.setall("TALB", [TALB(encoding=3, text=album)])
    # 20000 chars matches the review/edit text-field limit used elsewhere for
    # article text -- long enough for any real article, short enough that a
    # malformed layout cannot balloon the tag.
    tags.setall("USLT", [USLT(encoding=3, lang="deu", desc="", text=lyrics[:20000])])
    if cover_bytes:
        tags.setall("APIC", [APIC(encoding=3, mime=cover_mime, type=3, desc="Cover", data=cover_bytes)])
    tags.save(path, v2_version=3)