From 0f4b28ba143e9e3bec2c0a7c4a61214d76d870fc Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Wed, 30 Sep 2026 10:20:35 +0200 Subject: [PATCH] feat(magazine): link a tail to its head from 'continued from page N' Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + TODO.md | 6 ++-- architecture/magazine-ingestion.md | 5 +-- core/magazine.py | 51 ++++++++++++++++++++------ tests/test_magazine.py | 58 ++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41e56d1..bb22602 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi ## [Unreleased] ### Added +- **"Continued from page N" now joins a feature's tail to its head.** Until now only the forward "continued on page N" reference linked pages. A tail that says "Fortsetzung von Seite 10" / "Continued from page 10" (found even when set as small furniture or sidebar text, on lines up to 160 characters) is attached to the single article on that page. Skipped when a contents entry anchors the tail, when the page is shared by several articles, or when the marker points forward or at an empty page. Articles gain a `continued_from` list. - **Magazine import from PNG/JPEG page images.** Choose or drop several scans or phone photos; they are checked by content (not by name), ordered by filename with natural numbering, turned into one scan-style PDF (page size from the image's own resolution, EXIF rotation applied, 72 dpi treated as unknown) and ingested like any scanned issue. Limits: 300 images, 25 MB and 60 megapixels each; a bad image rejects the whole upload and leaves nothing behind; an existing source is never replaced. Verified on real pages: OCR through the image path scored like the original PDF (F1 0.85 / 0.78). - `architecture/ocr-porting-prompt-general.md`: the OCR porting brief rewritten for general documents (letters, papers, forms, invoices). Adds privacy, pre-processing of skewed/photographed scans, document-type handling, field-level accuracy and handwriting, and tags every claim as measured on magazines, a general principle, or untested for these document types. diff --git a/TODO.md b/TODO.md index 22f550f..66eeebd 100644 --- a/TODO.md +++ b/TODO.md @@ -705,9 +705,9 @@ casting. The original PDF remains the visual source of truth while approved arti ### P2 — Advanced magazine intelligence -- [~] Improve cross-page continuation detection and automatic sidebar/advertisement exclusion. - *(sidebar exclusion done via mark_sidebars, a narrow-column geometry heuristic; cross-page continuation still only - uses the forward "continued on page N" reference, not "continued from")* +- [x] Improve cross-page continuation detection and automatic sidebar/advertisement exclusion. + *(sidebar exclusion via mark_sidebars, a narrow-column geometry heuristic; cross-page continuation now follows both + "continued on page N" and "continued from page N", 2026-09-29, unit-tested, not yet seen on a real issue)* - [x] Add image descriptions, chart/table handling, and optional spoken figure summaries. *(per-figure "Describe" in review generates a type-aware description — chart/table/code/figure each get their own prompt — via the configured vision engine with fallback; editable before approval; spoken only if "Read diff --git a/architecture/magazine-ingestion.md b/architecture/magazine-ingestion.md index 9bbee00..98da01c 100644 --- a/architecture/magazine-ingestion.md +++ b/architecture/magazine-ingestion.md @@ -718,8 +718,9 @@ Magazines, Settings, or any future section. - Word-level highlighting is an *estimate* (proportional to character length within the sentence's real measured audio duration), not true forced alignment — see "Word-level highlighting is estimated, not measured" above. -- `continued from page N` is parsed but not used for linking — only the forward - `continued on page N` reference drives it. +- `continued from page N` now links the tail to the article on page N (forward references still win; a contents-anchored + tail, a page shared by several articles, or a marker pointing forward or at an empty page changes nothing). Verified by + unit tests only: the measured c't issue contains no jump markers at all. - Reviews-style pages with no headline are absorbed by the preceding article when the contents page does not list them. - KDL Frontier Parser nano (the OCR fallback engine) returns no layout diff --git a/core/magazine.py b/core/magazine.py index 8cc22f1..2536823 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -152,6 +152,7 @@ class Article: confidence: float = 0.0 section: str = "" continued_to: list[int] = field(default_factory=list) + continued_from: list[int] = field(default_factory=list) # 'continued from page N' seen on this article's pages audio_status: str = "none" # none | narration | podcast | both def to_json(self) -> dict[str, Any]: @@ -806,6 +807,15 @@ def group_articles(blocks: Sequence[Block], to_page, _ = find_continuation(b.text) if to_page and to_page not in art.continued_to: art.continued_to.append(to_page) + # "Continued from page N" is usually set as small print the classifier files as furniture or a + # sidebar, so look at every short line on the page, not only at what is narrated. A long block that + # merely mentions the phrase is prose, not a jump marker. + for b in page_blocks: + if len(b.text or "") > _MAX_JUMP_MARKER_CHARS: + continue + _, from_page = find_continuation(b.text) + if from_page and from_page != page and from_page not in art.continued_from: + art.continued_from.append(from_page) if contributed: art.pages.append(page) @@ -821,12 +831,25 @@ def group_articles(blocks: Sequence[Block], return _link_continuations(articles, page_count) -def _link_continuations(articles: list[Article], page_count: int) -> list[Article]: - """Attach jump pages ('continued on page 88') to their originating article. +_MAX_JUMP_MARKER_CHARS = 160 # a "continued from" line is a few words; longer text is prose that quotes the phrase - Without this, the tail of a feature parked at the back of the issue is - orphaned and gets narrated as a separate stub article that begins - mid-sentence. + +def _absorb(head: Article, tail: Article) -> None: + head.pages.extend(p for p in tail.pages if p not in head.pages) + head.blocks.extend(tail.blocks) + head.images.extend(tail.images) + tail.status = "merged" + + +def _link_continuations(articles: list[Article], page_count: int) -> list[Article]: + """Attach jump pages to their originating article, in both directions. + + Forward: the head says "continued on page 88" -- the article that starts on page 88 is its tail. + Backward: the tail says "continued from page 27" -- the article that occupies page 27 is its head. + Without this, the tail of a feature parked at the back of the issue is orphaned and narrated as a + separate stub that begins mid-sentence. A forward link wins when both exist (it is processed first, + and a merged article is never absorbed twice). An article a contents entry anchors (confidence >= 0.9) + keeps its own identity either way. """ by_start = {a.pages[0]: a for a in articles if a.pages} for art in articles: @@ -836,14 +859,20 @@ def _link_continuations(articles: list[Article], page_count: int) -> list[Articl tail = by_start.get(target) if tail is None or tail is art or tail.status == "merged": continue - # Only absorb a genuine continuation: a real article listed in the - # contents keeps its own identity. if tail.confidence >= 0.9: continue - art.pages.extend(p for p in tail.pages if p not in art.pages) - art.blocks.extend(tail.blocks) - art.images.extend(tail.images) - tail.status = "merged" + _absorb(art, tail) + for tail in articles: + if tail.status == "merged" or tail.confidence >= 0.9 or not tail.pages: + continue + for source in tail.continued_from: + if not (1 <= source <= page_count) or source >= tail.pages[0]: + continue # a tail can only follow a page before it + heads = [a for a in articles if a is not tail and a.status != "merged" and source in a.pages] + if len(heads) != 1: + continue # no article there, or ambiguous (a page shared by several): leave it to the reviewer + _absorb(heads[0], tail) + break return [a for a in articles if a.status != "merged"] diff --git a/tests/test_magazine.py b/tests/test_magazine.py index 85fb7a9..61f036d 100644 --- a/tests/test_magazine.py +++ b/tests/test_magazine.py @@ -465,6 +465,64 @@ class ContinuationTests(unittest.TestCase): self.assertEqual(find_continuation("The page turned slowly."), (None, None)) +class BackwardContinuationTests(unittest.TestCase): + """A tail that says "continued from page N" is attached to the article on page N.""" + + def issue(self, tail_marker=None, marker_type=BLOCK_FURNITURE, marker_page=20): + blocks = [] + + def add(b, order): + b.reading_order = order + blocks.append(b) + add(block("b0010-000", 10, 50, 60, 470, 50, "The Future of Solid State Batteries", size=26.0, btype=BLOCK_TITLE), 0) + add(block("b0010-001", 10, 50, 130, 200, 300, "Body copy of the feature. " * 30), 1) + add(block("b0020-000", 20, 50, 60, 470, 50, "Lessons From The Lab Bench", size=26.0, btype=BLOCK_TITLE), 0) + add(block("b0020-001", 20, 50, 130, 200, 300, "More body copy after the jump. " * 30), 1) + if tail_marker: + add(block("b0020-009", marker_page, 50, 760, 200, 12, tail_marker, size=7.0, btype=marker_type), 9) + return blocks + + def test_continued_from_joins_the_tail_to_its_head(self): + arts = group_articles(self.issue("Fortsetzung von Seite 10"), [], page_count=30) + self.assertEqual(len(arts), 1) + self.assertEqual(arts[0].pages, [10, 20]) + self.assertIn("b0020-001", arts[0].blocks) + + def test_english_marker_in_a_sidebar_block_also_works(self): + arts = group_articles(self.issue("Continued from page 10", marker_type=BLOCK_SIDEBAR), [], page_count=30) + self.assertEqual([a.pages for a in arts], [[10, 20]]) + + def test_without_a_marker_the_two_stay_separate(self): + arts = group_articles(self.issue(), [], page_count=30) + self.assertEqual(len(arts), 2) + + def test_a_marker_pointing_at_an_empty_page_changes_nothing(self): + arts = group_articles(self.issue("Fortsetzung von Seite 15"), [], page_count=30) + self.assertEqual(len(arts), 2) + + def test_a_marker_pointing_forward_is_ignored(self): + arts = group_articles(self.issue("Fortsetzung von Seite 25"), [], page_count=30) + self.assertEqual(len(arts), 2) + + def test_long_prose_quoting_the_phrase_is_not_a_marker(self): + prose = "As the editors noted, the story was continued from page 10 of the previous issue. " * 4 + arts = group_articles(self.issue(prose, marker_type=BLOCK_BODY), [], page_count=30) + self.assertEqual(len(arts), 2) + + def test_a_tail_anchored_by_the_contents_keeps_its_identity(self): + contents = [{"title": "Lessons From The Lab Bench", "page": 20, "section": ""}] + arts = group_articles(self.issue("Fortsetzung von Seite 10"), contents, page_count=30) + self.assertEqual(len(arts), 2) + + def test_forward_and_backward_references_together_join_once(self): + blocks = self.issue("Fortsetzung von Seite 10") + blocks.append(block("b0010-009", 10, 50, 760, 200, 12, "Fortsetzung auf Seite 20", size=7.0, btype=BLOCK_BODY)) + blocks[-1].reading_order = 9 + arts = group_articles(blocks, [], page_count=30) + self.assertEqual(len(arts), 1) + self.assertEqual(len(arts[0].blocks), len(set(arts[0].blocks))) + + class OcrParagraphGroupingTests(unittest.TestCase): """ocr_page() groups tesseract's TSV by paragraph, not by line -- matching how the native PyMuPDF path already aggregates a paragraph's spans into