From 736aeaaeaccbca00739a2047fb75eb17c2b66f24 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Sat, 5 Sep 2026 12:46:54 +0200 Subject: [PATCH] feat: add magazine projects with layout-aware article extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Magazines break the audiobook pipeline's assumptions: several unrelated articles share a page, columns run in parallel, adverts and sidebars interrupt body copy, and a feature routinely jumps twenty pages to its tail. Narrating page-by-page walks out of an article mid-sentence and into an advert, so the canonical object is the article, not the page. core/magazine.py reconstructs articles from a flat PDF: PyMuPDF extracts blocks with bboxes and font metrics, tesseract supplies a TSV fallback so scanned pages join the same column grouping, blocks are classified and ordered column-major, and articles are anchored by the parsed table of contents with "continued on page N" tails linked back. routes/magazine.py adds 13 endpoints. Ingestion is a restart-safe background job. Nothing is narrated until a human approves the boundaries; podcast generation refuses unapproved articles. Podcast mode renders an approved article as a 2-3 host discussion using the configured LLM and voices the user picks, alongside narration rather than instead of it. Verified end-to-end over HTTP on a synthetic 8-page issue: the feature came back as pages [3, 8] with its jump tail absorbed and the advert page excluded, and an image-only rendering recovered all 47 blocks via OCR. 44 new tests; full suite 90 passing. The Studio UI is not built — this is reachable only via the API. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 48 ++ Dockerfile | 8 + Manual.md | 100 ++++ PROJECT_MAP.md | 85 +++ TODO.md | 13 +- architecture/magazine-ingestion.md | 160 ++++++ core/magazine.py | 840 +++++++++++++++++++++++++++++ requirements.txt | 1 + routes/magazine.py | 522 ++++++++++++++++++ server.py | 3 +- tests/test_magazine.py | 431 +++++++++++++++ tests/test_magazine_routes.py | 91 ++++ 12 files changed, 2296 insertions(+), 6 deletions(-) create mode 100644 Manual.md create mode 100644 PROJECT_MAP.md create mode 100644 architecture/magazine-ingestion.md create mode 100644 core/magazine.py create mode 100644 routes/magazine.py create mode 100644 tests/test_magazine.py create mode 100644 tests/test_magazine_routes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 300c492..aff7e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,54 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi ## [Unreleased] +### Added +- **Magazine Projects (backend + API).** A second document workflow beside books, for magazines and papers where the + audiobook assumptions break down: several unrelated articles share a page, columns run in parallel, adverts and sidebars + interrupt body copy, and a feature routinely jumps twenty pages to its tail. Narrating such a PDF page-by-page walks out + of an article mid-sentence and into an advert, so the canonical object is the **article**, not the page. + `core/magazine.py` reconstructs articles from a flat PDF: PyMuPDF extracts text blocks with bounding boxes and font + metrics, tesseract supplies a TSV fallback (with coordinates, so scanned pages join the same column grouping as native + ones), blocks are classified as title/deck/byline/body/caption/pull quote/furniture/advert, ordered column-major, and + grouped into articles anchored by the parsed table of contents, with `continued on page N` tails linked back to their + originating article. `routes/magazine.py` adds 13 endpoints; ingestion is a restart-safe background job, since an + 84-page issue takes minutes and the browser must not have to stay open. Magazines reuse the Read Aloud store + (`kind: "magazine"`), so library listing, covers, progress and audio endpoints work unchanged. Verified end-to-end on a + synthetic 8-page issue — the two-column feature was recovered as pages [3, 8] with its jump tail absorbed, the advert + page excluded, and both bylines extracted — and on an image-only rendering of the same file, where all 47 blocks came + back through OCR at 0.75-0.90 confidence. 44 tests. **The Studio UI is not built; this is reachable only via the API.** +- **Podcast Mode For Articles.** An approved article can be rendered as a discussion between two or three voices the user + picks, instead of straight narration. The configured LLM writes the script through the existing OpenAI-compatible + client; the prompt forbids inventing facts, figures or quotes absent from the source, and tells the hosts to say so when + the article reaches no conclusion. The parser tolerates a markdown fence and surrounding prose, and reassigns an + unrecognised host label rather than dropping the turn — an unknown label would otherwise leave a line with no voice at + all. The chosen cast is remembered so the next article defaults to the same hosts. Generation is refused for unapproved + articles, since unreviewed boundaries may include a neighbouring article or an advert. +- **Documentation Baseline.** `PROJECT_MAP.md`, `Manual.md` and `architecture/` are required by the project's own agent + instructions but had never been created. All three now exist; the architecture SOP records the decisions that are easy + to get wrong and the limitations that remain, including that OCR-engine selection is not yet dispatched server-side. + +### Changed +- **Server-Side Layout Analysis For Magazines.** PDF text extraction for plain books stays in the browser, but magazine + layout analysis runs server-side: PyMuPDF exposes real font metrics and embedded-image rectangles that PDF.js does not, + OCR coordinates merge with those boxes in one coordinate space, and the result is expensive enough that it must be + computed once and persisted rather than recomputed per browser session. PDF.js keeps rendering duty. Docling and + Unstructured were rejected — multi-GB of torch and layout weights, ARM64 wheel risk on the DGX Spark and GPU contention + with the TTS stack, to improve on heuristics that handle the observed cases. Adds `pymupdf` and `tesseract-ocr` + (deu/eng) to the image. + +### Fixed +- **Captions On Photo Pages Would Have Been Read As Body Text.** Block classification judged font size against a baseline + computed per page, so a page carrying a full-bleed photograph and a single caption made that caption its own reference: + at a ratio of exactly 1.0 it classified as body copy and would have been narrated as part of the article. The baseline + is now measured once across every block in the issue. +- **Advert Pages Swallowed By The Preceding Feature.** Publishers print "ADVERTISEMENT" once to disclaim a page, not on + every block, so the advert's own display type classified as a headline. Combined with an article claiming its whole page + range, a full-page advert between two features was absorbed into the first and would have been highlighted as part of it + in the PDF view. The label now carries its page, and an article claims only pages it actually contributes text to. +- **Covers And Contents Pages Proposed As Articles.** The cover, the table of contents and full-page adverts each became a + low-confidence article the reviewer had to reject by hand on every issue. A page whose only narratable content is a + headline now starts no article, and the contents page — which lists articles rather than being one — is excluded. + ### Changed - **OCR custom-card persistence.** Browser-local and server-backed custom OCR profiles are now merged during reloads, so a newly added Baidu or other OCR engine remains visible while settings synchronize; the OCR page also shows an diff --git a/Dockerfile b/Dockerfile index c9ce38b..a6bc7de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,19 @@ ENV PYTHONUNBUFFERED=1 # libatomic1: pedalboard's native extension (pedalboard_native) links against # libatomic.so.1 and fails to import without it — pip installs the Python # package fine, but the C extension silently can't load at runtime. +# tesseract-ocr: server-side OCR fallback for scanned magazine pages. The TSV +# output carries word bounding boxes, which merge with PyMuPDF's block bboxes — +# plain text alone would lose the column/region structure magazine layout +# analysis depends on. deu/eng match the languages the client-side heading OCR +# already ships. RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ curl \ ca-certificates \ libatomic1 \ + tesseract-ocr \ + tesseract-ocr-deu \ + tesseract-ocr-eng \ && rm -rf /var/lib/apt/lists/* # ── Python dependencies ─────────────────────────────────────────────────────── diff --git a/Manual.md b/Manual.md new file mode 100644 index 0000000..7d7f245 --- /dev/null +++ b/Manual.md @@ -0,0 +1,100 @@ +# User manual + +Living manual. Every feature updates this file in the same commit as the code. + +> Sections for the mature features (voice library, Read Aloud, audiobook +> casting, Rehearser) are still to be written — this file was created alongside +> the magazine feature and currently documents that area only. + +--- + +## Magazines + +### Purpose + +Listen to a magazine or newspaper one **article** at a time, from its true +beginning to its true end, without adverts, sidebars or a neighbouring article +spliced into the middle. Each article can be narrated by a single voice, or +turned into a podcast discussion between two or three voices you pick. + +### Prerequisites + +- The magazine as a PDF (native text or scanned — both work). +- At least one voice in the library for narration. +- For podcast mode: an LLM configured in Settings (`llm_url`, `llm_model`), and + two or three voices chosen as hosts. + +### Step by step + +1. **Add the magazine.** Studio → *Add Magazine / Paper*. Give it a title and + issue, then upload the PDF. +2. **Ingest.** Ingestion runs in the background — you can close the browser. It + renders every page, extracts text (falling back to OCR on scanned pages), + classifies each region of the layout, finds the contents page, and proposes + articles with page ranges. +3. **Review the articles.** This step is not optional and cannot be skipped. + Each proposed article shows its title, author, page range, estimated + listening time and a confidence score. Confirm the boundaries, exclude a + sidebar you do not want read, correct the title, then **Approve**. +4. **Listen.** An approved article can be narrated with the default voice, or + sent to podcast mode. + +**Expected result:** the article plays from its headline to its final paragraph. +Page furniture (running heads, page numbers), adverts and pull quotes are never +read aloud. + +### Confidence, and what it means + +| Shown | Meaning | +|---|---| +| ~90% | The contents page listed this article — title and start page are known. | +| ~55% | Detected from a headline alone; check the boundaries carefully. | +| Low | No body copy found. Usually an advert or a section divider — reject it. | + +An article is never narrated until you approve it, whatever its confidence. + +### Podcast mode + +Choose two or three voices as hosts. The configured LLM writes a discussion of +that article for listeners who have not read it, and each host's turns are +synthesised with their voice. + +The prompt forbids inventing facts, figures or quotes that are not in the +article; if the article reaches no conclusion the hosts say so. It is still a +generated discussion, not the author's words — treat it as a summary, and read +the original for anything that matters. + +Podcast generation is refused for an article you have not approved, because +unreviewed boundaries may include a neighbouring article or an advert. + +### Errors + +| What you see | What happened | What to do | +|---|---|---| +| "Upload the PDF first" | The project has no `source.pdf`. | Upload it, then ingest. | +| Ingestion failed | The PDF is encrypted or corrupt. | Check it opens in a viewer. | +| "Approve the article before generating a podcast" | Review gate. | Approve it first. | +| "No LLM model configured" | `llm_model` is empty. | Set it in Settings. | +| "LLM did not return usable JSON" | The model ignored the format. | Retry, or pick a stronger model. | +| "Article has too little text" | Under 200 characters — usually a stub. | Check the boundaries in review. | + +### Privacy + +Ingestion, OCR and storage are entirely local to the container. Podcast mode is +the exception: **the article's text is sent to whatever LLM `llm_url` points +at.** If that is a hosted API rather than a model on your own network, the +article leaves your machine. Narration uses your local TTS stack and sends +nothing outward. + +### Accessibility and mobile + +To be completed with the P1 interface. Planned: keyboard-navigable contents +tree and review screen, sentence highlighting that respects +`prefers-reduced-motion`, large touch targets for playback, and a driving mode +that advances images without requiring interaction. + +### Offline + +Ingested magazines are stored server-side, so any device on the network sees the +same library and reading progress. Playback needs the server; there is no +offline cache yet. diff --git a/PROJECT_MAP.md b/PROJECT_MAP.md new file mode 100644 index 0000000..6deba93 --- /dev/null +++ b/PROJECT_MAP.md @@ -0,0 +1,85 @@ +# Project map + +Living index of where things are and what state they are in. +Update in the same commit as the change it describes. + +## Current phase + +Magazine projects, **P0 (backend) complete**. Audiobook/Rehearser/voice-library +features are mature and in daily use. + +## Deployment + +Self-hosted Docker on the DGX Spark, port 7890, image +`ghcr.io/martin-b78/tts-voice-creator-clone-and-design-2`. +Python and static files are bind-mounted, so code changes need only a container +recreate; rebuild and push only when `Dockerfile` or `requirements.txt` change. + +## Layout + +| Path | Role | +|---|---| +| `server.py` | FastAPI app, router registration, auth middleware | +| `core/` | domain logic — config, database, audio, voice, routing, magazine | +| `routes/` | HTTP boundary, one module per feature area | +| `static/js/` | frontend, one module per feature area (no build step for most) | +| `tests/` | `test_*.py` (unittest) and `*.spec.js` (Playwright) | +| `architecture/` | SOPs for subsystems whose design is not obvious from code | +| `workflows/` | ComfyUI graphs for character portrait generation | + +## Feature areas + +| Area | Server | Frontend | State | +|---|---|---|---| +| Voice library / clone / design | `routes/library.py`, `core/voice.py` | `library.js` | mature | +| Read Aloud (books, PDFs) | `routes/reader.py` | `reader.js` | mature | +| Audiobook casting | `routes/conversation.py` | `audiobook*.js` | mature | +| Rehearser | `routes/rehearsals_db.py` | `rehearser-*.js` | mature | +| Studio (phase wizard) | — | `studio.js` | mature | +| TTS engines / routing | `routes/tts.py`, `core/routing.py` | `engines.js` | mature | +| STT | `routes/stt.py` | — | mature | +| **Magazines** | `core/magazine.py`, `routes/magazine.py` | *not started* | **P0 backend** | + +## Data model + +Documents (books and magazines alike) live under +`/reader_library//`, discriminated by `meta.json → kind`: + +- `kind: "pdf"` / text — Read Aloud book. Text is extracted in the browser and + not persisted; `meta.json` holds counts and a flat `idx` resume position. +- `kind: "magazine"` — adds `layout.json`, `articles.json`, `pages/`, `images/`, + `podcast/`, and a per-article `progress` map. See + [architecture/magazine-ingestion.md](architecture/magazine-ingestion.md). + +Settings, presets, routes and the SQLite database also live under `CONFIG_DIR`, +which is the writable volume. + +## Environment + +| Variable | Purpose | +|---|---| +| `CONFIG_DIR` | writable state (settings, library, database) | +| `VOICES_DIR` / `OUTPUT_DIR` | voice WAV library | +| `TTS_CONTAINER_NAMES` | containers restarted after a voice change | +| `DOCKER_HOST` | points at the socket proxy; the raw socket is not mounted | +| `llm_url` / `llm_model` / `llm_api_key` | settings.json, OpenAI-compatible LLM | + +## Pending decisions + +- Container registry: ghcr costs quota against a private package (≈427 MB per + push, 500 MB free on the Free plan). Local registry vs. ghcr vs. no re-pull is + **unresolved** — the compose files currently point at ghcr. +- Magazine P1 scope: sentence-level highlighting confirmed as the starting + point; word-level deferred to P2. + +## Known limitations + +- `gitea` backup remote is 157 commits behind `origin`; `main` tracks `gitea`. +- Magazine sidebar detection is not implemented (P2). +- No `npm run qa` aggregate command yet; Python tests run via `unittest`, + browser tests via `playwright test`. + +## Next logical step + +Magazine P1: magazine overview with the contents tree, the article review +screen, reflowed + PDF views, and sentence-highlighted playback. diff --git a/TODO.md b/TODO.md index 2bc5fd9..a574c55 100644 --- a/TODO.md +++ b/TODO.md @@ -290,12 +290,15 @@ casting. The original PDF remains the visual source of truth while approved arti ### P0 — Magazine foundation -- [ ] Add a Magazine project type beside Books and Theater Plays. -- [ ] Add PDF/JPG/PNG import with page previews and safe local storage. +- [~] Add a Magazine project type beside Books and Theater Plays. *(backend + API done; Studio UI not started)* +- [~] Add PDF/JPG/PNG import with page previews and safe local storage. *(PDF + page renders done; JPG/PNG not done)* - [x] Add OCR-engine selection from Engines > OCR & Document Parsing, with page-image input and structured output. -- [ ] Detect contents/index pages and propose article titles, authors, and page ranges. -- [ ] Build a review screen where users approve or correct article boundaries before TTS. -- [ ] Preserve source coordinates for every extracted block, image, caption, and page reference. +- [x] Detect contents/index pages and propose article titles, authors, and page ranges. +- [~] Build a review screen where users approve or correct article boundaries before TTS. *(approve/edit API done and gating narration; screen not built)* +- [x] Preserve source coordinates for every extracted block, image, caption, and page reference. + +- [x] Generate per-article podcast scripts from a connected LLM with two or three user-picked host voices. + *(script generation + review gating done; multi-voice synthesis reuses the existing casting path, UI not built)* ### P1 — Magazine listening experience diff --git a/architecture/magazine-ingestion.md b/architecture/magazine-ingestion.md new file mode 100644 index 0000000..e994206 --- /dev/null +++ b/architecture/magazine-ingestion.md @@ -0,0 +1,160 @@ +# SOP — Magazine ingestion + +Status: **P0 complete (backend)**. P1 (listening workflow UI) not started. + +## Why magazines are not books + +The audiobook pipeline assumes one continuous text in reading order. A magazine +breaks every part of that assumption: several unrelated articles share a page, +columns run in parallel, adverts and sidebars interrupt body copy, and a feature +routinely jumps twenty pages to its tail. Narrating a magazine page-by-page +produces audio that walks out of an article mid-sentence and into an advert. + +**The canonical object is therefore the article, not the page.** Everything +below exists to reconstruct articles from a flat PDF. + +## Pipeline + +Implemented in [`core/magazine.py`](../core/magazine.py); driven by +[`routes/magazine.py`](../routes/magazine.py). + +``` +PDF + → extract_pages() PyMuPDF: text blocks with bbox, font size/name, weight + → ocr_page() tesseract TSV fallback when a page has <120 native chars + → classify_blocks() title / deck / byline / body / caption / pullquote / + sidebar / furniture / ad / figure + → mark_advert_pages() spread an "ADVERTISEMENT" label across its whole page + → mark_pullquotes() demote display text that merely repeats body copy + → assign_reading_order() column detection, then top-to-bottom within a column + → score/parse_contents() Stage A: locate and parse the table of contents + → group_articles() Stage C: group blocks, link "continued on page N" + → human review Stage D, in the UI — nothing is narrated until approved +``` + +### Why server-side, not in the browser + +Text extraction for plain PDFs runs client-side (PDF.js), and that stays. Layout +analysis does not, for three reasons: PyMuPDF exposes real font metrics and +embedded-image rectangles that PDF.js does not; OCR coordinates from tesseract +merge with those bboxes in one coordinate space; and the result is expensive +enough (minutes for an issue) that it must be computed once and persisted, not +recomputed per browser session. + +PDF.js remains responsible for *rendering* in the PDF view. + +Docling/Unstructured were considered and rejected: they pull torch plus layout +model weights (multi-GB, ARM64 wheel risk on the DGX Spark, GPU contention with +the TTS stack) to improve on heuristics that already work on the cases we hit. +Revisit only if real issues show grouping failures the heuristics cannot fix. + +## Decisions that are easy to get wrong + +**Font baseline is document-wide, never per page.** `classify_blocks` judges +size as a ratio against `base_size`. A page carrying a full-bleed photograph and +one caption has no body copy of its own, so a per-page baseline makes that +caption its own reference and classifies it as body text. `extract_pages` +measures the baseline once across every block in the issue. + +**Reading order is column-major.** Sorting blocks by `y` interleaves parallel +columns, splicing the left column's sentence into the right column's. Columns +are inferred from vertical whitespace (where blocks are *absent*), because +magazines change column count per page — a fixed grid is wrong. Blocks spanning +>60% of the page width sort ahead of both columns, so a full-width headline +precedes the body it introduces. + +**Furniture is identified by repetition, not position.** A running head and a +headline both sit at the top of a page. What separates them is that the running +head recurs on many pages at the same edge; `find_repeated_lines` needs the +whole document, so classification waits until every page has been read. + +**A page is claimed only if it contributes.** An advert between two features +starts no article, so without an explicit check it gets swallowed by the +preceding one and highlighted as part of it in the PDF view. + +**Pull quotes are demoted.** Narrating one reads the same sentence twice a few +seconds apart. A display-sized block whose text already appears in the body copy +is a pull quote, not a deck. + +## Confidence, and why review is mandatory + +| Signal | Confidence | +|---|---| +| Anchored by a contents entry | 0.90 | +| Started by a headline, no contents entry | 0.55 | +| Article whose blocks contain no body copy | ×0.4 | + +Nothing is narrated until a human sets `status: "approved"`. This is deliberate: +a five-second correction beats a 30-minute recording with a sidebar in the +middle. `POST …/podcast` refuses unapproved articles outright. + +## Storage + +Magazines live in the Read Aloud store and are distinguished by +`meta.json → kind: "magazine"`, so the existing library listing, cover, progress +and audio endpoints keep working. + +``` +/reader_library// + meta.json kind, title, issue, ingest status, default_voice, + podcast hosts, per-article progress + source.pdf the original + layout.json every classified block, with bbox + reading order + articles.json detected articles; status gates narration + pages/NNNN.png rendered pages (PDF view + OCR input) + images/*.png extracted figures ≥64px + podcast/.json generated host scripts + audio/.mp3 synthesised audio (existing reader endpoints) +``` + +`layout.json` is the join table between audio, text, image and PDF coordinates: +every block carries `page` + `bbox`, so a sentence in the reflowed view can jump +to its source region on the rendered page. + +## Podcast mode + +Per article, alongside narration — not instead of it. An approved article's +narratable blocks are sent to the configured LLM (`llm_url` / `llm_model`, +reusing `_post_llm_chat_completion` from `routes/conversation.py`) with a system +prompt that forbids inventing facts and demands strict JSON. The parser tolerates +a markdown fence and surrounding prose, and reassigns unknown host labels rather +than dropping the turn — an unrecognised label would otherwise leave a line with +no voice. 2–3 hosts, each bound to a voice the user picks; synthesis reuses the +multi-speaker path the audiobook casting already uses. + +## Tests + +- [`tests/test_magazine.py`](../tests/test_magazine.py) — 34 tests on the pure + geometry/classification logic: column reading order, the document-baseline + regression, furniture, pull quotes, contents parsing, continuation linking, + advert pages. +- [`tests/test_magazine_routes.py`](../tests/test_magazine_routes.py) — 10 tests: path + traversal rejection, review gating, LLM script parsing including malformed + output. + +Both run without PyMuPDF: the I/O functions import it lazily so the logic worth +testing needs no native dependency or sample PDF. + +Verified end-to-end against a synthetic 8-page issue (contents page, two-column +feature with a byline and a page-8 jump, full-page advert, second feature, +reviews) and against an image-only rendering of the same file to exercise OCR. + +## OCR engine selection (not yet wired) + +Engines → OCR & Document Parsing lists layout-aware parsers (KDL Frontier Parser +nano, Baidu Unlimited-OCR) as connection-probed cards. Those are presentation +and health-check only — there is no server-side dispatch, so ingestion currently +always uses local tesseract via `ocr_lang`. Routing a page image to one of those +vLLM endpoints and mapping its structured output onto `Block` is real work, not +a config change; it is the natural home for TODO P2's per-page fallback +selection and benchmark comparison. + +## Known limitations + +- Sidebar detection (`BLOCK_SIDEBAR`) is defined but not yet inferred; sidebars + currently classify as body and must be excluded by hand in review. P2. +- `continued from page N` is parsed but not used for linking — only the forward + `continued on page N` reference drives it. +- Reviews-style pages with no headline are absorbed by the preceding article + when the contents page does not list them. +- Word-level timing is not available; P1 targets sentence highlighting. diff --git a/core/magazine.py b/core/magazine.py new file mode 100644 index 0000000..b4bf0c0 --- /dev/null +++ b/core/magazine.py @@ -0,0 +1,840 @@ +"""Layout-aware magazine ingestion. + +Magazines are not books: a single page routinely carries several unrelated +articles, advertisements, sidebars and pull quotes, and an article often +continues twenty pages later. Treating a page as the unit of narration +therefore produces audio that jumps mid-sentence into an advert. The unit +here is the *article*, reconstructed from classified layout blocks. + +Pipeline (see architecture/magazine-ingestion.md): + + PDF + -> extract_pages() native text blocks with bbox/font via PyMuPDF + -> ocr_page() tesseract TSV fallback for scanned pages + -> classify_blocks() title / deck / byline / body / caption / ... + -> assign_reading_order() column detection, then top-to-bottom per column + -> detect_contents() Stage A: locate and parse the table of contents + -> group_articles() Stage C: group blocks, link continuations + -> (human review) Stage D happens in the UI; nothing is narrated + until an article is approved. + +PyMuPDF is imported lazily inside the I/O functions so that the pure geometry +and classification helpers -- which hold all the fiddly logic worth testing -- +can be exercised without the native dependency or a sample PDF. +""" +from __future__ import annotations + +import json +import re +import statistics +import subprocess +import unicodedata +from dataclasses import dataclass, field, asdict +from pathlib import Path +from typing import Any, Iterable, Sequence + +# ── Block taxonomy ─────────────────────────────────────────────────────────── +# Ordered roughly by how much they matter to narration. Only NARRATABLE types +# are ever sent to TTS; the rest are kept for display and provenance. +BLOCK_TITLE = "title" +BLOCK_DECK = "deck" # standfirst / subtitle under the headline +BLOCK_BYLINE = "byline" +BLOCK_BODY = "body" +BLOCK_CAPTION = "caption" +BLOCK_PULLQUOTE = "pullquote" # verbatim repeat of body text, blown up +BLOCK_SIDEBAR = "sidebar" +BLOCK_FURNITURE = "furniture" # running heads, folios, page numbers +BLOCK_AD = "ad" +BLOCK_FIGURE = "figure" + +NARRATABLE = (BLOCK_TITLE, BLOCK_DECK, BLOCK_BYLINE, BLOCK_BODY) + +# Text that marks an article jumping to (or arriving from) another page. +_CONT_TO = re.compile( + r"\((?:continued|cont\.?)\s+on\s+page\s+(\d{1,4})\)" + r"|continued\s+on\s+page\s+(\d{1,4})" + r"|fortsetzung\s+auf\s+seite\s+(\d{1,4})" + r"|weiter\s+auf\s+seite\s+(\d{1,4})", + re.I, +) +_CONT_FROM = re.compile( + r"continued\s+from\s+page\s+(\d{1,4})" + r"|fortsetzung\s+von\s+seite\s+(\d{1,4})", + re.I, +) + +_BYLINE = re.compile( + r"^\s*(?:by|von|text|words|story|interview)\s*[:  ]\s*(.{2,80})$", + re.I, +) + +# Headings that mark a contents page, and the section headers inside one. +_TOC_MARKERS = ( + "contents", "table of contents", "in this issue", "inside this issue", + "features", "departments", "columns", "regulars", + "inhalt", "inhaltsverzeichnis", "in dieser ausgabe", "rubriken", "themen", +) +# "The future of batteries ....... 18" / "The future of batteries 18" +_TOC_ENTRY = re.compile(r"^(.{3,120}?)[\s.…·•\-]{2,}(\d{1,4})\s*$") + +_ADVERT_MARKERS = ("advertisement", "anzeige", "sponsored", "promotion") + + +@dataclass +class Block: + """One laid-out region of a page, in PDF user-space coordinates. + + bbox is [x, y, width, height] with the origin at the page's top-left, which + is what the reflowed view needs to draw a highlight over the rendered page. + """ + id: str + page: int + bbox: list[float] + type: str + text: str + reading_order: int = 0 + confidence: float = 1.0 + font_size: float = 0.0 + font_name: str = "" + bold: bool = False + column: int = 0 + source: str = "native" # "native" | "ocr" + + def to_json(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class Article: + id: str + title: str + author: str = "" + pages: list[int] = field(default_factory=list) + blocks: list[str] = field(default_factory=list) + images: list[str] = field(default_factory=list) + status: str = "detected" # detected | approved | rejected + confidence: float = 0.0 + section: str = "" + continued_to: list[int] = field(default_factory=list) + audio_status: str = "none" # none | narration | podcast | both + + def to_json(self) -> dict[str, Any]: + return asdict(self) + + +# ── Geometry helpers (pure) ────────────────────────────────────────────────── + +def bbox_overlaps(a: Sequence[float], b: Sequence[float], tol: float = 0.0) -> bool: + """True when two [x, y, w, h] boxes intersect, allowing a tolerance.""" + ax, ay, aw, ah = a + bx, by, bw, bh = b + return not ( + ax + aw + tol < bx or bx + bw + tol < ax + or ay + ah + tol < by or by + bh + tol < ay + ) + + +def detect_columns(blocks: Sequence[Block], page_width: float, + min_gap_ratio: float = 0.035) -> list[tuple[float, float]]: + """Infer the column grid from where blocks are *absent*. + + Magazines vary the column count per page, so a fixed grid is wrong. The + reliable signal is vertical whitespace running the height of the page: we + project every block onto the x axis, then split wherever the uncovered gap + is wider than a few percent of the page. Returns (left, right) spans. + """ + spans = sorted( + ((b.bbox[0], b.bbox[0] + b.bbox[2]) for b in blocks if b.bbox[2] > 1), + key=lambda s: s[0], + ) + if not spans: + return [(0.0, page_width)] + + merged: list[list[float]] = [list(spans[0])] + min_gap = page_width * min_gap_ratio + for left, right in spans[1:]: + if left - merged[-1][1] > min_gap: + merged.append([left, right]) + else: + merged[-1][1] = max(merged[-1][1], right) + return [(m[0], m[1]) for m in merged] + + +def assign_reading_order(blocks: list[Block], page_width: float) -> list[Block]: + """Order blocks the way a person reads the page: column by column. + + Sorting purely by y interleaves parallel columns, which is precisely the + failure that splices one article's sentence into another's. + """ + columns = detect_columns(blocks, page_width) + + def column_of(b: Block) -> int: + centre = b.bbox[0] + b.bbox[2] / 2 + for i, (left, right) in enumerate(columns): + if left <= centre <= right: + return i + # Spanning blocks (a headline across all columns) belong to the first + # column they touch, so they precede the body they introduce. + for i, (left, right) in enumerate(columns): + if bbox_overlaps(b.bbox, [left, b.bbox[1], right - left, b.bbox[3]]): + return i + return len(columns) + + for b in blocks: + b.column = column_of(b) + + # A full-width headline must not be dragged to the bottom of column 0, so + # blocks spanning most of the page sort by y before anything below them. + def key(b: Block) -> tuple[float, int, float]: + spans_page = b.bbox[2] > page_width * 0.6 + return (0 if spans_page else 1, b.column, round(b.bbox[1], 1)) + + ordered = sorted(blocks, key=key) + for i, b in enumerate(ordered): + b.reading_order = i + return ordered + + +# ── Classification (pure) ──────────────────────────────────────────────────── + +def body_font_size(blocks: Iterable[Block]) -> float: + """The document's body size: the char-weighted median of block sizes. + + Weighting by length stops a page of huge display type from redefining what + "normal" means -- body copy always dominates by character count. + """ + sizes: list[float] = [] + for b in blocks: + if b.font_size > 0 and b.text.strip(): + sizes.extend([b.font_size] * max(1, len(b.text) // 20)) + return statistics.median(sizes) if sizes else 10.0 + + +def _normalise(text: str) -> str: + text = unicodedata.normalize("NFKC", text or "") + return re.sub(r"\s+", " ", text).strip() + + +def looks_like_byline(text: str) -> tuple[bool, str]: + """Detect 'By Jane Smith' / 'Von Jane Smith', returning the name.""" + m = _BYLINE.match(_normalise(text)) + if not m: + return False, "" + name = m.group(1).strip(" .,;:") + # A byline is a name, not a sentence: reject anything long or punctuated. + if len(name) > 60 or name.count(" ") > 5 or name.endswith((".", "!", "?")): + return False, "" + return True, name + + +def find_continuation(text: str) -> tuple[int | None, int | None]: + """Return (continues_on_page, continued_from_page) mentioned in the text.""" + to_m = _CONT_TO.search(text or "") + from_m = _CONT_FROM.search(text or "") + to_page = next((int(g) for g in to_m.groups() if g), None) if to_m else None + from_page = next((int(g) for g in from_m.groups() if g), None) if from_m else None + return to_page, from_page + + +def classify_blocks(blocks: list[Block], page_height: float, page_width: float, + repeated: set[str] | None = None, + image_boxes: Sequence[Sequence[float]] = (), + base_size: float | None = None) -> list[Block]: + """Assign a semantic type to every block on one page. + + Font size is judged relative to the document body size rather than in + absolute points, so the same rules hold for a pocket magazine and a + broadsheet. + + base_size must be measured across the whole document, not this page: a page + carrying a full-bleed photograph and one caption has no body copy of its + own, so a per-page baseline would make that caption its own reference and + classify it as body text. It falls back to the page when not supplied. + """ + repeated = repeated or set() + base = base_size if base_size and base_size > 0 else body_font_size(blocks) + + for b in blocks: + text = _normalise(b.text) + if not text: + b.type = BLOCK_FURNITURE + b.confidence = 0.5 + continue + + ratio = (b.font_size / base) if base else 1.0 + near_top = b.bbox[1] < page_height * 0.07 + near_bottom = b.bbox[1] + b.bbox[3] > page_height * 0.93 + short = len(text) <= 90 + lower = text.lower() + + # Running heads, folios and page numbers repeat at the same edge on + # many pages -- that repetition is what identifies them, not position + # alone, since a headline can also sit at the top. + if (near_top or near_bottom) and (text in repeated or text.isdigit()): + b.type, b.confidence = BLOCK_FURNITURE, 0.95 + continue + + if any(m in lower for m in _ADVERT_MARKERS) and short: + b.type, b.confidence = BLOCK_AD, 0.8 + continue + + is_byline, _ = looks_like_byline(text) + if is_byline: + b.type, b.confidence = BLOCK_BYLINE, 0.9 + continue + + # A caption sits directly beneath or beside a figure and is set smaller + # than body copy. Both conditions are needed: small text elsewhere is + # usually a footnote, and body-sized text under an image is often the + # article continuing past it. + if ratio < 0.92 and any( + bbox_overlaps(b.bbox, box, tol=max(page_height, page_width) * 0.02) + for box in image_boxes + ): + b.type, b.confidence = BLOCK_CAPTION, 0.75 + continue + + if ratio >= 1.6 and short: + b.type, b.confidence = BLOCK_TITLE, 0.9 if ratio >= 2.0 else 0.7 + continue + + if 1.18 <= ratio < 1.6 and len(text) <= 300: + # Display-sized text mid-column is a pull quote; at the head of a + # page it is the deck introducing the article. + b.type = BLOCK_DECK if b.bbox[1] < page_height * 0.45 else BLOCK_PULLQUOTE + b.confidence = 0.6 + continue + + b.type, b.confidence = BLOCK_BODY, 0.85 + + return blocks + + +def mark_advert_pages(blocks: list[Block]) -> list[Block]: + """Spread an explicit advertisement label across its whole page. + + Publishers print "ADVERTISEMENT" once to disclaim the page, not each block + on it. Without this the ad's own display type reads as a headline and the + page gets claimed by whichever feature precedes it. + """ + if not any(b.type == BLOCK_AD for b in blocks): + return blocks + for b in blocks: + if b.type != BLOCK_FURNITURE: + b.type, b.confidence = BLOCK_AD, 0.8 + return blocks + + +def mark_pullquotes(blocks: list[Block]) -> list[Block]: + """Demote display text that merely repeats nearby body copy. + + Narrating a pull quote reads the same sentence twice, a few seconds apart, + which sounds like a skipping record. + """ + body_text = _normalise(" ".join( + b.text for b in blocks if b.type == BLOCK_BODY + )).lower() + if not body_text: + return blocks + for b in blocks: + if b.type not in (BLOCK_DECK, BLOCK_PULLQUOTE): + continue + probe = _normalise(b.text).lower().strip(' "“”«»') + if len(probe) > 25 and probe in body_text: + b.type, b.confidence = BLOCK_PULLQUOTE, 0.9 + return blocks + + +def find_repeated_lines(pages_text: Sequence[Sequence[str]], + threshold: float = 0.25) -> set[str]: + """Lines appearing near the same edge on many pages: page furniture.""" + if not pages_text: + return set() + counts: dict[str, int] = {} + for lines in pages_text: + for line in {_normalise(t) for t in lines if _normalise(t)}: + counts[line] = counts.get(line, 0) + 1 + minimum = max(3, int(len(pages_text) * threshold)) + # Long lines that recur are more likely a repeated standfirst than a + # running head, so cap the length. + return {t for t, n in counts.items() if n >= minimum and len(t) <= 80} + + +# ── Stage A: the table of contents ─────────────────────────────────────────── + +def score_contents_page(lines: Sequence[str]) -> float: + """How strongly one page looks like a table of contents (0..1).""" + if not lines: + return 0.0 + joined = " ".join(lines).lower() + marker_hits = sum(1 for m in _TOC_MARKERS if m in joined) + entry_hits = sum(1 for line in lines if _TOC_ENTRY.match(_normalise(line))) + # Either signal alone is weak: a feature can be titled "Features", and a + # dotted leader appears in ordinary tables too. + score = min(1.0, marker_hits * 0.25) * 0.4 + min(1.0, entry_hits / 6) * 0.6 + return round(score, 3) + + +def parse_contents(lines: Sequence[str]) -> list[dict[str, Any]]: + """Pull (title, page, section) entries out of a contents page.""" + entries: list[dict[str, Any]] = [] + section = "" + for raw in lines: + line = _normalise(raw) + if not line: + continue + low = line.lower().strip(": ") + if low in _TOC_MARKERS and low not in ("contents", "inhalt"): + section = line.strip(": ") + continue + m = _TOC_ENTRY.match(line) + if not m: + continue + title, page = m.group(1).strip(" .…·•-"), int(m.group(2)) + if len(title) < 3 or page <= 0 or page > 2000: + continue + entries.append({"title": title, "page": page, "section": section}) + return entries + + +# ── Stage C: grouping blocks into articles ─────────────────────────────────── + +def _title_key(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "", _normalise(text).lower()) + + +def group_articles(blocks: Sequence[Block], + contents: Sequence[dict[str, Any]], + page_count: int, + skip_pages: set[int] | None = None) -> list[Article]: + """Group ordered blocks into articles. + + The contents listing, when present, is the strongest available signal: it + states both the title and the starting page, so it anchors boundaries that + layout alone would guess at. Pages not claimed by any listing still get + articles from their own headlines, because a contents page never lists + everything (and some issues have no contents page at all). + """ + by_page: dict[int, list[Block]] = {} + for b in blocks: + by_page.setdefault(b.page, []).append(b) + for page_blocks in by_page.values(): + page_blocks.sort(key=lambda b: b.reading_order) + + # Where does each article start? A contents entry pins a start page; a + # title block on a page with no entry starts one there. + starts: dict[int, dict[str, Any]] = {} + for entry in contents: + page = entry["page"] + if 1 <= page <= page_count: + starts.setdefault(page, { + "title": entry["title"], + "section": entry.get("section", ""), + "confidence": 0.9, + }) + + skip_pages = skip_pages or set() + for page in sorted(by_page): + if page in starts or page in skip_pages: + continue + # A page carrying nothing but advertising and page furniture has no + # article on it -- the cover and the ad pages otherwise each become a + # spurious entry the reviewer has to reject by hand on every issue. + if not any(b.type in NARRATABLE and b.type != BLOCK_TITLE + for b in by_page[page]): + continue + titles = [b for b in by_page[page] if b.type == BLOCK_TITLE] + if titles: + starts[page] = { + "title": _normalise(titles[0].text), + "section": "", + "confidence": 0.55, + } + + if not starts: + return [] + + ordered_starts = sorted(starts) + articles: list[Article] = [] + for i, start_page in enumerate(ordered_starts): + end_page = (ordered_starts[i + 1] - 1) if i + 1 < len(ordered_starts) else page_count + info = starts[start_page] + art = Article( + id=f"article-{i + 1:03d}", + title=info["title"], + section=info.get("section", ""), + confidence=info["confidence"], + ) + for page in range(start_page, end_page + 1): + page_blocks = by_page.get(page, []) + if not page_blocks: + continue + # Only claim a page the article actually occupies. A full-page + # advert sitting between two features starts no article of its + # own, so it would otherwise be swallowed by the preceding one and + # highlighted as part of it in the PDF view. + contributed = False + for b in page_blocks: + if b.type in (BLOCK_FURNITURE, BLOCK_AD): + continue + contributed = True + art.blocks.append(b.id) + if b.type == BLOCK_BYLINE and not art.author: + _, name = looks_like_byline(b.text) + art.author = name + to_page, _ = find_continuation(b.text) + if to_page and to_page not in art.continued_to: + art.continued_to.append(to_page) + if contributed: + art.pages.append(page) + + # An article the pipeline could not fill with body copy is almost + # certainly a false start (an advert headline, a section divider). + has_body = any( + b.type == BLOCK_BODY for b in blocks if b.id in set(art.blocks) + ) + if not has_body: + art.confidence = round(art.confidence * 0.4, 3) + articles.append(art) + + return _link_continuations(articles, page_count) + + +def _link_continuations(articles: list[Article], page_count: int) -> list[Article]: + """Attach jump pages ('continued on page 88') to their originating article. + + Without this, the tail of a feature parked at the back of the issue is + orphaned and gets narrated as a separate stub article that begins + mid-sentence. + """ + by_start = {a.pages[0]: a for a in articles if a.pages} + for art in articles: + for target in art.continued_to: + if not (1 <= target <= page_count): + continue + tail = by_start.get(target) + if tail is None or tail is art or tail.status == "merged": + continue + # Only absorb a genuine continuation: a real article listed in the + # contents keeps its own identity. + if tail.confidence >= 0.9: + continue + art.pages.extend(p for p in tail.pages if p not in art.pages) + art.blocks.extend(tail.blocks) + art.images.extend(tail.images) + tail.status = "merged" + return [a for a in articles if a.status != "merged"] + + +def estimate_listen_seconds(blocks: Sequence[Block], article: Article, + words_per_minute: float = 155.0) -> int: + """Rough narration length, for the 'estimated listening time' column.""" + ids = set(article.blocks) + words = sum( + len(b.text.split()) for b in blocks + if b.id in ids and b.type in NARRATABLE + ) + return int(round(words / words_per_minute * 60)) if words else 0 + + +# ── PDF and OCR I/O ────────────────────────────────────────────────────────── + +def _require_fitz(): + """Import PyMuPDF under its current name, falling back to the old alias. + + 1.24 renamed the module to ``pymupdf``; ``fitz`` still resolves but emits a + deprecation warning and is slated for removal. + """ + try: + import pymupdf + return pymupdf + except ImportError: + pass + try: + import fitz + return fitz + except ImportError as exc: # pragma: no cover - environment guard + raise RuntimeError( + "PyMuPDF is required for magazine ingestion; install pymupdf" + ) from exc + + +def ocr_page(png_path: Path, lang: str = "deu+eng", + min_confidence: float = 40.0) -> list[dict[str, Any]]: + """OCR one rendered page, returning line boxes from tesseract's TSV. + + TSV rather than plain text because the coordinates are the whole point: + they are what lets an OCR'd page take part in the same column and article + grouping as a native-text page. + """ + try: + proc = subprocess.run( + ["tesseract", str(png_path), "stdout", "-l", lang, "tsv"], + capture_output=True, text=True, timeout=180, check=True, + ) + except (subprocess.SubprocessError, FileNotFoundError, OSError): + return [] + + lines: dict[tuple[int, int, int], dict[str, Any]] = {} + for row in proc.stdout.splitlines()[1:]: + parts = row.split("\t") + if len(parts) < 12: + continue + try: + block_num, par_num, line_num = int(parts[2]), int(parts[3]), int(parts[4]) + left, top, width, height = (int(parts[6]), int(parts[7]), + int(parts[8]), int(parts[9])) + conf = float(parts[10]) + except ValueError: + continue + word = parts[11].strip() + if not word or conf < min_confidence: + continue + key = (block_num, par_num, line_num) + rec = lines.setdefault(key, { + "text": [], "x0": left, "y0": top, + "x1": left + width, "y1": top + height, "conf": [], + }) + rec["text"].append(word) + rec["x0"] = min(rec["x0"], left) + rec["y0"] = min(rec["y0"], top) + rec["x1"] = max(rec["x1"], left + width) + rec["y1"] = max(rec["y1"], top + height) + rec["conf"].append(conf) + + out = [] + for (_, _, _), rec in sorted(lines.items()): + out.append({ + "text": " ".join(rec["text"]), + "bbox": [float(rec["x0"]), float(rec["y0"]), + float(rec["x1"] - rec["x0"]), float(rec["y1"] - rec["y0"])], + "confidence": round(sum(rec["conf"]) / len(rec["conf"]) / 100.0, 3), + }) + return out + + +def extract_pages(pdf_path: Path, out_dir: Path, render_dpi: int = 110, + ocr_lang: str = "deu+eng", + progress: Any = None) -> dict[str, Any]: + """Run the full extraction for one PDF, writing page renders and images. + + Returns the layout document: every block on every page, classified and in + reading order, plus the raw contents-page candidates. + """ + fitz = _require_fitz() + pages_dir = out_dir / "pages" + images_dir = out_dir / "images" + pages_dir.mkdir(parents=True, exist_ok=True) + images_dir.mkdir(parents=True, exist_ok=True) + + doc = fitz.open(str(pdf_path)) + try: + raw_pages: list[dict[str, Any]] = [] + + for pno in range(doc.page_count): + page = doc.load_page(pno) + rect = page.rect + page_no = pno + 1 + + png = pages_dir / f"{page_no:04d}.png" + if not png.exists(): + pix = page.get_pixmap(dpi=render_dpi) + pix.save(str(png)) + + image_boxes: list[list[float]] = [] + for img_index, info in enumerate(page.get_images(full=True)): + try: + for r in page.get_image_rects(info[0]): + image_boxes.append([r.x0, r.y0, r.width, r.height]) + except Exception: + continue + _extract_image(fitz, doc, info[0], images_dir, + f"p{page_no:04d}-{img_index:02d}") + + blocks: list[Block] = [] + data = page.get_text("dict") + for bi, blk in enumerate(data.get("blocks", [])): + if blk.get("type") != 0: # 0 = text, 1 = image + continue + texts, sizes, fonts, bolds = [], [], [], [] + for line in blk.get("lines", []): + for span in line.get("spans", []): + if not span.get("text", "").strip(): + continue + texts.append(span["text"]) + sizes.append(float(span.get("size", 0))) + fonts.append(span.get("font", "")) + bolds.append(bool(span.get("flags", 0) & 2 ** 4)) + text = " ".join(texts).strip() + if not text: + continue + x0, y0, x1, y1 = blk["bbox"] + blocks.append(Block( + id=f"b{page_no:04d}-{bi:03d}", + page=page_no, + bbox=[x0, y0, x1 - x0, y1 - y0], + type=BLOCK_BODY, + text=text, + font_size=statistics.median(sizes) if sizes else 0.0, + font_name=max(set(fonts), key=fonts.count) if fonts else "", + bold=sum(bolds) > len(bolds) / 2 if bolds else False, + )) + + # A page whose native text is negligible is a scan (or is drawn as + # outlines); OCR is the only way to reach its words. + native_chars = sum(len(b.text) for b in blocks) + if native_chars < 120: + scale_x = rect.width / max(1.0, _png_width(png, fitz)) + for li, line in enumerate(ocr_page(png, ocr_lang)): + bx, by, bw, bh = line["bbox"] + blocks.append(Block( + id=f"b{page_no:04d}-o{li:03d}", + page=page_no, + bbox=[bx * scale_x, by * scale_x, bw * scale_x, bh * scale_x], + type=BLOCK_BODY, + text=line["text"], + confidence=line["confidence"], + font_size=bh * scale_x, + source="ocr", + )) + + raw_pages.append({ + "page": page_no, + "width": rect.width, + "height": rect.height, + "blocks": blocks, + "image_boxes": image_boxes, + }) + if progress: + progress(page_no, doc.page_count) + + # Furniture can only be identified across the whole document, so + # classification waits until every page has been read. + repeated = find_repeated_lines( + [[b.text for b in p["blocks"]] for p in raw_pages] + ) + all_blocks: list[Block] = [] + contents_candidates: list[dict[str, Any]] = [] + + # One baseline for the whole issue (see classify_blocks). + doc_base = body_font_size( + [b for p in raw_pages for b in p["blocks"]] + ) + + for p in raw_pages: + classify_blocks(p["blocks"], p["height"], p["width"], + repeated=repeated, image_boxes=p["image_boxes"], + base_size=doc_base) + mark_advert_pages(p["blocks"]) + mark_pullquotes(p["blocks"]) + assign_reading_order(p["blocks"], p["width"]) + all_blocks.extend(p["blocks"]) + + lines = [b.text for b in sorted(p["blocks"], key=lambda b: b.reading_order)] + score = score_contents_page(lines) + if score >= 0.45: + contents_candidates.append({ + "page": p["page"], + "score": score, + "entries": parse_contents(lines), + }) + + return { + "page_count": doc.page_count, + "pages": [{"page": p["page"], "width": p["width"], "height": p["height"]} + for p in raw_pages], + "blocks": [b.to_json() for b in all_blocks], + "contents_candidates": contents_candidates, + } + finally: + doc.close() + + +def _png_width(png: Path, fitz) -> float: + try: + return float(fitz.Pixmap(str(png)).width) + except Exception: + return 1.0 + + +def _extract_image(fitz, doc, xref: int, images_dir: Path, name: str) -> None: + """Save one embedded image, skipping the tiny rules and spacers.""" + try: + pix = fitz.Pixmap(doc, xref) + if pix.width < 64 or pix.height < 64: + return + if pix.n - pix.alpha >= 4: # CMYK needs converting first + pix = fitz.Pixmap(fitz.csRGB, pix) + target = images_dir / f"{name}.png" + if not target.exists(): + pix.save(str(target)) + except Exception: + return + + +# ── Persistence ────────────────────────────────────────────────────────────── + +def build_articles(layout: dict[str, Any]) -> list[dict[str, Any]]: + """Turn a stored layout document into detected articles with estimates.""" + blocks = [Block(**b) for b in layout.get("blocks", [])] + candidates = layout.get("contents_candidates") or [] + best = max(candidates, key=lambda c: c["score"], default=None) + entries = best["entries"] if best else [] + + # The contents page lists articles; it is not one. + skip = {c["page"] for c in candidates if c.get("score", 0) >= 0.45} + articles = group_articles(blocks, entries, layout.get("page_count", 0), + skip_pages=skip) + out = [] + for art in articles: + rec = art.to_json() + rec["estimated_seconds"] = estimate_listen_seconds(blocks, art) + out.append(rec) + return out + + +def save_layout(doc_dir: Path, layout: dict[str, Any]) -> None: + (doc_dir / "layout.json").write_text( + json.dumps(layout, ensure_ascii=False), "utf-8") + + +def load_layout(doc_dir: Path) -> dict[str, Any]: + path = doc_dir / "layout.json" + if not path.is_file(): + return {} + try: + return json.loads(path.read_text("utf-8")) + except (ValueError, OSError): + return {} + + +def save_articles(doc_dir: Path, articles: list[dict[str, Any]]) -> None: + (doc_dir / "articles.json").write_text( + json.dumps({"articles": articles}, ensure_ascii=False), "utf-8") + + +def load_articles(doc_dir: Path) -> list[dict[str, Any]]: + path = doc_dir / "articles.json" + if not path.is_file(): + return [] + try: + return json.loads(path.read_text("utf-8")).get("articles", []) + except (ValueError, OSError): + return [] + + +def article_text(layout: dict[str, Any], article: dict[str, Any], + include_captions: bool = False) -> list[dict[str, Any]]: + """The narratable text of one article, in reading order. + + Returned per block so the player can highlight the source region and jump + to the originating PDF page. + """ + wanted = set(article.get("blocks") or []) + types = set(NARRATABLE) | ({BLOCK_CAPTION} if include_captions else set()) + blocks = [ + b for b in layout.get("blocks", []) + if b["id"] in wanted and b["type"] in types + ] + blocks.sort(key=lambda b: (b["page"], b["reading_order"])) + return blocks diff --git a/requirements.txt b/requirements.txt index 5741806..54ac6dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ scipy>=1.12.0 requests>=2.31.0 Pillow>=10.3.0 yt-dlp>=2024.3.10 +pymupdf>=1.24.0 diff --git a/routes/magazine.py b/routes/magazine.py new file mode 100644 index 0000000..b2d3748 --- /dev/null +++ b/routes/magazine.py @@ -0,0 +1,522 @@ +"""Magazine projects: layout-aware ingestion, article review, podcast scripts. + +Magazines live in the same store as Read Aloud documents +(/reader_library//) and are distinguished by meta.json's +``kind: "magazine"``, so the existing library listing, cover, progress and +audio endpoints keep working unchanged. Alongside those, a magazine adds: + + layout.json every classified block on every page, with bboxes + articles.json detected articles; nothing is narrated until approved + pages/NNNN.png rendered pages for the PDF view and OCR + images/*.png extracted figures + podcast/.json generated two/three-host scripts + +Ingestion is a restart-safe background job because an 84-page issue takes +minutes, and the browser must not have to stay open for it. +""" +from __future__ import annotations + +import asyncio +import json +import re +import time +import uuid +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import FileResponse + +from core import magazine as mag +from core.config import _load_settings +from core.constants import CONFIG_DIR +from core.database import ( + background_job_create, + background_job_get, + background_job_update, +) + +router = APIRouter() + +_LIB = CONFIG_DIR / "reader_library" +_MAX_HOSTS = 3 +_MIN_HOSTS = 2 + + +# ── Path safety ────────────────────────────────────────────────────────────── + +def _doc_dir(doc_id: str) -> Path: + """Resolve a document directory, rejecting anything but a generated id.""" + if not doc_id or not doc_id.isalnum() or len(doc_id) > 40: + raise HTTPException(400, "Bad document id") + d = _LIB / doc_id + if not d.is_dir(): + raise HTTPException(404, "Document not found") + return d + + +def _article_id(article_id: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", article_id or ""): + raise HTTPException(400, "Bad article id") + return article_id + + +def _read_meta(d: Path) -> dict[str, Any]: + try: + return json.loads((d / "meta.json").read_text("utf-8")) + except (OSError, ValueError): + return {} + + +def _write_meta(d: Path, meta: dict[str, Any]) -> None: + (d / "meta.json").write_text(json.dumps(meta, ensure_ascii=False), "utf-8") + + +def _require_magazine(doc_id: str) -> tuple[Path, dict[str, Any]]: + d = _doc_dir(doc_id) + meta = _read_meta(d) + if meta.get("kind") != "magazine": + raise HTTPException(400, "Not a magazine project") + return d, meta + + +# ── Project creation ───────────────────────────────────────────────────────── + +@router.post("/api/magazine/docs") +async def magazine_create(request: Request): + """Create an empty magazine project; the PDF is uploaded separately.""" + body = await request.json() + title = str(body.get("title") or "").strip()[:300] or "Untitled magazine" + doc_id = uuid.uuid4().hex + d = _LIB / doc_id + d.mkdir(parents=True, exist_ok=True) + now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + _write_meta(d, { + "kind": "magazine", + "title": title, + "issue": str(body.get("issue") or "").strip()[:120], + "publisher": str(body.get("publisher") or "").strip()[:120], + "created": now, + "updated": now, + "ingest": {"status": "empty"}, + "default_voice": str(body.get("default_voice") or "").strip()[:200], + "podcast": {"hosts": []}, + "progress": {}, + }) + return {"id": doc_id} + + +# ── Stage A–C: ingestion ───────────────────────────────────────────────────── + +def _ingest_sync(doc_id: str, job_id: str, ocr_lang: str, render_dpi: int) -> None: + """Run the extraction pipeline; recorded as a restart-safe job.""" + d = _LIB / doc_id + pdf = d / "source.pdf" + try: + if not pdf.is_file(): + raise FileNotFoundError("No source.pdf uploaded for this magazine") + + def progress(page: int, total: int) -> None: + background_job_update( + job_id, stage="extract", + detail=f"Analysing page {page} of {total}", + ) + + background_job_update(job_id, stage="extract", detail="Opening PDF") + layout = mag.extract_pages(pdf, d, render_dpi=render_dpi, + ocr_lang=ocr_lang, progress=progress) + mag.save_layout(d, layout) + + background_job_update(job_id, stage="articles", detail="Detecting articles") + articles = mag.build_articles(layout) + mag.save_articles(d, articles) + + meta = _read_meta(d) + meta["pageCount"] = layout.get("page_count", 0) + meta["ingest"] = { + "status": "done", + "finished": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "articles": len(articles), + "ocr_pages": sum( + 1 for b in layout.get("blocks", []) if b.get("source") == "ocr" + ), + } + _write_meta(d, meta) + background_job_update(job_id, stage="done", done=True, + detail=f"{len(articles)} articles detected", + result={"articles": len(articles)}) + except Exception as exc: # noqa: BLE001 - surfaced to UI + meta = _read_meta(d) + meta["ingest"] = {"status": "failed", "error": str(exc)[:500]} + _write_meta(d, meta) + background_job_update(job_id, stage="failed", done=True, error=str(exc)[:1000]) + + +@router.post("/api/magazine/{doc_id}/ingest") +async def magazine_ingest(doc_id: str, request: Request): + d, meta = _require_magazine(doc_id) + if not (d / "source.pdf").is_file(): + raise HTTPException(400, "Upload the PDF first") + + body = {} + try: + body = await request.json() + except Exception: + pass + ocr_lang = str(body.get("ocr_lang") or "deu+eng")[:40] + if not re.fullmatch(r"[a-z]{3}(?:\+[a-z]{3})*", ocr_lang): + raise HTTPException(400, "Bad OCR language") + try: + render_dpi = max(72, min(200, int(body.get("dpi") or 110))) + except (TypeError, ValueError): + render_dpi = 110 + + job_id = uuid.uuid4().hex + background_job_create(job_id, "magazine-ingest", fingerprint=doc_id) + meta["ingest"] = {"status": "running", "job_id": job_id} + _write_meta(d, meta) + + asyncio.create_task( + asyncio.to_thread(_ingest_sync, doc_id, job_id, ocr_lang, render_dpi) + ) + return {"job_id": job_id} + + +@router.get("/api/magazine/{doc_id}/ingest/status") +async def magazine_ingest_status(doc_id: str): + _, meta = _require_magazine(doc_id) + job_id = (meta.get("ingest") or {}).get("job_id") + job = background_job_get(job_id) if job_id else None + return {"ingest": meta.get("ingest") or {"status": "empty"}, "job": job} + + +# ── Layout and articles ────────────────────────────────────────────────────── + +@router.get("/api/magazine/{doc_id}/layout") +async def magazine_layout(doc_id: str, page: int | None = None): + """The layout document, optionally narrowed to one page.""" + d, _ = _require_magazine(doc_id) + layout = mag.load_layout(d) + if not layout: + raise HTTPException(404, "Not ingested yet") + if page is not None: + layout = dict(layout) + layout["blocks"] = [b for b in layout.get("blocks", []) if b.get("page") == page] + layout["pages"] = [p for p in layout.get("pages", []) if p.get("page") == page] + return layout + + +@router.get("/api/magazine/{doc_id}/articles") +async def magazine_articles(doc_id: str): + d, meta = _require_magazine(doc_id) + return { + "articles": mag.load_articles(d), + "progress": meta.get("progress") or {}, + "default_voice": meta.get("default_voice") or "", + "podcast": meta.get("podcast") or {"hosts": []}, + } + + +@router.patch("/api/magazine/{doc_id}/articles/{article_id}") +async def magazine_update_article(doc_id: str, article_id: str, request: Request): + """Stage D: the human review that gates narration. + + Only fields a reviewer can legitimately change are writable; block ids are + validated against the layout so a bad edit cannot point an article at + blocks that do not exist. + """ + d, _ = _require_magazine(doc_id) + article_id = _article_id(article_id) + articles = mag.load_articles(d) + target = next((a for a in articles if a.get("id") == article_id), None) + if target is None: + raise HTTPException(404, "Article not found") + + body = await request.json() + + if "status" in body: + status = str(body["status"]) + if status not in ("detected", "approved", "rejected"): + raise HTTPException(400, "Bad status") + target["status"] = status + if "title" in body: + target["title"] = str(body["title"]).strip()[:300] + if "author" in body: + target["author"] = str(body["author"]).strip()[:120] + if "section" in body: + target["section"] = str(body["section"]).strip()[:120] + + if "blocks" in body: + known = {b["id"] for b in mag.load_layout(d).get("blocks", [])} + requested = body["blocks"] + if not isinstance(requested, list): + raise HTTPException(400, "blocks must be a list") + unknown = [b for b in requested if b not in known] + if unknown: + raise HTTPException(400, f"Unknown block ids: {unknown[:5]}") + target["blocks"] = requested + layout_blocks = {b["id"]: b for b in mag.load_layout(d).get("blocks", [])} + target["pages"] = sorted({ + layout_blocks[b]["page"] for b in requested if b in layout_blocks + }) + + if "pages" in body: + pages = body["pages"] + if not isinstance(pages, list) or not all(isinstance(p, int) for p in pages): + raise HTTPException(400, "pages must be a list of integers") + target["pages"] = sorted(set(pages)) + + mag.save_articles(d, articles) + return target + + +@router.get("/api/magazine/{doc_id}/articles/{article_id}/text") +async def magazine_article_text(doc_id: str, article_id: str, + captions: int = 0): + """The narratable text of one article, per block, in reading order.""" + d, _ = _require_magazine(doc_id) + article_id = _article_id(article_id) + articles = mag.load_articles(d) + target = next((a for a in articles if a.get("id") == article_id), None) + if target is None: + raise HTTPException(404, "Article not found") + blocks = mag.article_text(mag.load_layout(d), target, + include_captions=bool(captions)) + return {"article": target, "blocks": blocks} + + +@router.patch("/api/magazine/{doc_id}/progress") +async def magazine_progress(doc_id: str, request: Request): + """Per-article resume position, so each article remembers its own place.""" + d, meta = _require_magazine(doc_id) + body = await request.json() + article_id = _article_id(str(body.get("article") or "")) + progress = dict(meta.get("progress") or {}) + entry = dict(progress.get(article_id) or {}) + if "sentence" in body: + entry["sentence"] = max(0, int(body["sentence"])) + if "seconds" in body: + entry["seconds"] = max(0, float(body["seconds"])) + progress[article_id] = entry + meta["progress"] = progress + meta["updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + _write_meta(d, meta) + return {"ok": True} + + +# ── Page renders and figures ───────────────────────────────────────────────── + +@router.get("/api/magazine/{doc_id}/pages/{page}") +async def magazine_page_image(doc_id: str, page: int): + d, _ = _require_magazine(doc_id) + if page < 1 or page > 10000: + raise HTTPException(400, "Bad page") + png = d / "pages" / f"{page:04d}.png" + if not png.is_file(): + raise HTTPException(404, "Page not rendered") + return FileResponse(png, media_type="image/png") + + +@router.get("/api/magazine/{doc_id}/images/{name}") +async def magazine_image(doc_id: str, name: str): + d, _ = _require_magazine(doc_id) + if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}\.png", name or ""): + raise HTTPException(400, "Bad image name") + path = d / "images" / name + if not path.is_file(): + raise HTTPException(404, "Image not found") + return FileResponse(path, media_type="image/png") + + +# ── Podcast mode ───────────────────────────────────────────────────────────── + +_PODCAST_SYSTEM = ( + "You write podcast scripts in which {n} hosts discuss a magazine article " + "for listeners who have not read it. Cover the article's actual content " + "and conclusions faithfully -- never invent facts, statistics, quotes or " + "study results that are not in the source text. If the article is " + "inconclusive, let the hosts say so rather than manufacturing a verdict.\n" + "\n" + "Return STRICT JSON, no markdown fence, in this shape:\n" + '{{"title": "...", "turns": [{{"host": "A", "text": "..."}}]}}\n' + "\n" + "Rules:\n" + "- Use only host labels {labels}.\n" + "- 12-30 turns. Each turn is one or two spoken sentences.\n" + "- Open by naming the article and its author, close with a short takeaway.\n" + "- Write speech, not prose: contractions, plain words, no bullet points, " + "no stage directions, no sound-effect cues.\n" + "- Write in {language}.\n" +) + + +def _podcast_dir(d: Path) -> Path: + p = d / "podcast" + p.mkdir(parents=True, exist_ok=True) + return p + + +def _parse_script(raw: str, labels: list[str]) -> dict[str, Any]: + """Parse the model's JSON, tolerating a stray markdown fence.""" + text = (raw or "").strip() + if text.startswith("```"): + text = re.sub(r"^```[a-zA-Z]*\s*|\s*```$", "", text).strip() + start, end = text.find("{"), text.rfind("}") + if start != -1 and end > start: + text = text[start:end + 1] + try: + data = json.loads(text) + except ValueError as exc: + raise HTTPException(502, f"LLM did not return usable JSON: {exc}") from exc + + turns_in = data.get("turns") + if not isinstance(turns_in, list) or not turns_in: + raise HTTPException(502, "LLM returned no turns") + + allowed = set(labels) + turns: list[dict[str, str]] = [] + for turn in turns_in: + if not isinstance(turn, dict): + continue + host = str(turn.get("host") or "").strip().upper()[:8] + line = str(turn.get("text") or "").strip() + if not line: + continue + # An unknown label would otherwise silently drop to no voice at all. + if host not in allowed: + host = labels[len(turns) % len(labels)] + turns.append({"host": host, "text": line[:2000]}) + if not turns: + raise HTTPException(502, "LLM returned no usable turns") + return {"title": str(data.get("title") or "").strip()[:300], "turns": turns} + + +@router.post("/api/magazine/{doc_id}/articles/{article_id}/podcast") +async def magazine_podcast_script(doc_id: str, article_id: str, request: Request): + """Turn one approved article into a 2-3 host podcast script. + + The script is generated and stored here; synthesis reuses the existing TTS + pipeline, one voice per host, exactly as multi-speaker audiobook segments + are rendered today. + """ + from routes.conversation import ( # local import avoids a cycle + _post_llm_chat_completion, + _request_timeout_seconds, + ) + + d, meta = _require_magazine(doc_id) + article_id = _article_id(article_id) + articles = mag.load_articles(d) + target = next((a for a in articles if a.get("id") == article_id), None) + if target is None: + raise HTTPException(404, "Article not found") + if target.get("status") != "approved": + # Generating from unreviewed boundaries wastes an LLM call on text that + # may contain a neighbouring article or an advert. + raise HTTPException(400, "Approve the article before generating a podcast") + + body = await request.json() + hosts = body.get("hosts") + if not isinstance(hosts, list) or not (_MIN_HOSTS <= len(hosts) <= _MAX_HOSTS): + raise HTTPException(400, f"Pick {_MIN_HOSTS} or {_MAX_HOSTS} host voices") + parsed_hosts: list[dict[str, str]] = [] + for i, host in enumerate(hosts): + if not isinstance(host, dict): + raise HTTPException(400, "Bad host entry") + voice = str(host.get("voice") or "").strip()[:200] + if not voice: + raise HTTPException(400, "Every host needs a voice") + parsed_hosts.append({ + "label": chr(ord("A") + i), + "voice": voice, + "name": str(host.get("name") or "").strip()[:60] or f"Host {chr(ord('A') + i)}", + }) + labels = [h["label"] for h in parsed_hosts] + + blocks = mag.article_text(mag.load_layout(d), target, + include_captions=bool(body.get("captions"))) + source = "\n\n".join(b["text"] for b in blocks).strip() + if len(source) < 200: + raise HTTPException(400, "Article has too little text for a podcast") + # Keep the prompt inside a sane context window; the tail of a very long + # feature contributes little to a discussion of its argument. + source = source[:24000] + + settings = _load_settings() + llm_url = (body.get("llm_url") or settings.get("llm_url") + or "http://localhost:11434/v1") + model = (body.get("model") or settings.get("llm_model") or "").strip() + if not model: + raise HTTPException(400, "No LLM model configured") + language = str(body.get("language") or "the language of the article")[:60] + + system = _PODCAST_SYSTEM.format( + n=len(parsed_hosts), + labels=", ".join(labels), + language=language, + ) + roster = "\n".join(f"{h['label']} = {h['name']}" for h in parsed_hosts) + user = ( + f"Hosts:\n{roster}\n\n" + f"Article title: {target.get('title') or 'Untitled'}\n" + f"Author: {target.get('author') or 'unknown'}\n\n" + f"Article text:\n{source}" + ) + + payload = { + "model": model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "temperature": float(body.get("temperature") or 0.7), + "stream": False, + } + headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"} + timeout = _request_timeout_seconds(body.get("timeout"), default=300.0) + + resp = await asyncio.to_thread( + _post_llm_chat_completion, llm_url, payload, headers, timeout + ) + if resp.status_code >= 400: + raise HTTPException(502, f"LLM error {resp.status_code}: {resp.text[:300]}") + try: + content = resp.json()["choices"][0]["message"]["content"] + except (ValueError, KeyError, IndexError) as exc: + raise HTTPException(502, "Unexpected LLM response shape") from exc + + script = _parse_script(content, labels) + script.update({ + "article": article_id, + "hosts": parsed_hosts, + "model": model, + "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }) + (_podcast_dir(d) / f"{article_id}.json").write_text( + json.dumps(script, ensure_ascii=False), "utf-8") + + # Remember the cast so the next article defaults to the same hosts. + meta["podcast"] = {"hosts": parsed_hosts} + _write_meta(d, meta) + return script + + +@router.get("/api/magazine/{doc_id}/articles/{article_id}/podcast") +async def magazine_podcast_get(doc_id: str, article_id: str): + d, _ = _require_magazine(doc_id) + article_id = _article_id(article_id) + path = d / "podcast" / f"{article_id}.json" + if not path.is_file(): + raise HTTPException(404, "No podcast script generated yet") + try: + return json.loads(path.read_text("utf-8")) + except (OSError, ValueError) as exc: + raise HTTPException(500, "Podcast script unreadable") from exc + + +@router.delete("/api/magazine/{doc_id}/articles/{article_id}/podcast") +async def magazine_podcast_delete(doc_id: str, article_id: str): + d, _ = _require_magazine(doc_id) + article_id = _article_id(article_id) + (d / "podcast" / f"{article_id}.json").unlink(missing_ok=True) + return {"ok": True} diff --git a/server.py b/server.py index 83250d9..347d049 100644 --- a/server.py +++ b/server.py @@ -19,7 +19,7 @@ from starlette.datastructures import MutableHeaders from core.constants import STATIC_DIR, _BufferHandler, _MAX_JSON_BYTES, _MAX_UPLOAD_BYTES, _MAX_RAW_REQUEST_BYTES, _REQUIRE_EXTERNAL_API_KEY from core.config import _load_settings, _ensure_external_api_key from core.voice_index import refresh_voice_index_background -from routes import admin, settings, library, stt, sources, docker, tts, conversation, reader, characters, rehearsals_db, recovery, history, jobs +from routes import admin, settings, library, stt, sources, docker, tts, conversation, reader, characters, rehearsals_db, recovery, history, jobs, magazine logger = logging.getLogger("uvicorn.error") @@ -346,6 +346,7 @@ app.include_router(rehearsals_db.router) app.include_router(recovery.router) app.include_router(history.router) app.include_router(jobs.router) +app.include_router(magazine.router) # ── Static files ────────────────────────────────────────────────────────────── diff --git a/tests/test_magazine.py b/tests/test_magazine.py new file mode 100644 index 0000000..b558e63 --- /dev/null +++ b/tests/test_magazine.py @@ -0,0 +1,431 @@ +"""Magazine layout analysis regressions. + +These cover the pure geometry/classification logic rather than PyMuPDF I/O: +reading order across columns and article boundary detection are what decide +whether an article is narrated from its true beginning to its true end, so +they are the parts worth locking down. +""" +from __future__ import annotations + +import os +import tempfile +import unittest + +os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-magazine-test-config-")) + +from core.magazine import ( + BLOCK_AD, + BLOCK_BODY, + BLOCK_BYLINE, + BLOCK_CAPTION, + BLOCK_DECK, + BLOCK_FURNITURE, + BLOCK_PULLQUOTE, + BLOCK_TITLE, + Block, + article_text, + assign_reading_order, + body_font_size, + build_articles, + classify_blocks, + detect_columns, + estimate_listen_seconds, + find_continuation, + find_repeated_lines, + group_articles, + looks_like_byline, + mark_advert_pages, + mark_pullquotes, + parse_contents, + score_contents_page, +) + +PAGE_W, PAGE_H = 595.0, 842.0 # A4 in points + + +def block(bid, page, x, y, w, h, text, size=10.0, btype=BLOCK_BODY): + return Block(id=bid, page=page, bbox=[x, y, w, h], type=btype, + text=text, font_size=size) + + +class ColumnGeometryTests(unittest.TestCase): + def test_two_columns_detected_from_the_gutter(self): + blocks = [ + block("a", 1, 50, 100, 200, 300, "left copy"), + block("b", 1, 320, 100, 200, 300, "right copy"), + ] + cols = detect_columns(blocks, PAGE_W) + self.assertEqual(len(cols), 2) + self.assertLess(cols[0][1], cols[1][0]) + + def test_single_column_when_no_gutter(self): + blocks = [ + block("a", 1, 50, 100, 480, 100, "wide copy"), + block("b", 1, 50, 220, 480, 100, "more wide copy"), + ] + self.assertEqual(len(detect_columns(blocks, PAGE_W)), 1) + + def test_reading_order_finishes_a_column_before_starting_the_next(self): + """The failure this prevents: interleaving columns by y, which splices + the left column's sentence into the right column's.""" + blocks = [ + block("l1", 1, 50, 100, 200, 80, "left top"), + block("r1", 1, 320, 100, 200, 80, "right top"), + block("l2", 1, 50, 200, 200, 80, "left bottom"), + block("r2", 1, 320, 200, 200, 80, "right bottom"), + ] + order = [b.id for b in assign_reading_order(blocks, PAGE_W)] + self.assertEqual(order, ["l1", "l2", "r1", "r2"]) + + def test_full_width_headline_precedes_both_columns(self): + blocks = [ + block("l1", 1, 50, 200, 200, 80, "left copy"), + block("r1", 1, 320, 200, 200, 80, "right copy"), + block("head", 1, 50, 80, 470, 60, "The Future of Batteries", size=24.0), + ] + order = [b.id for b in assign_reading_order(blocks, PAGE_W)] + self.assertEqual(order[0], "head") + + +class ClassificationTests(unittest.TestCase): + def test_body_size_is_not_hijacked_by_display_type(self): + blocks = [ + block("t", 1, 50, 50, 400, 60, "HUGE HEADLINE", size=40.0), + block("b1", 1, 50, 150, 200, 200, "x" * 400, size=10.0), + block("b2", 1, 320, 150, 200, 200, "y" * 400, size=10.0), + ] + self.assertAlmostEqual(body_font_size(blocks), 10.0, places=1) + + def test_headline_byline_and_body_are_separated(self): + blocks = [ + block("t", 1, 50, 60, 470, 50, "The Future of Batteries", size=26.0), + block("by", 1, 50, 120, 200, 14, "By Jane Smith", size=10.0), + block("b", 1, 50, 150, 200, 300, "Battery technology " * 40, size=10.0), + ] + classify_blocks(blocks, PAGE_H, PAGE_W) + kinds = {b.id: b.type for b in blocks} + self.assertEqual(kinds["t"], BLOCK_TITLE) + self.assertEqual(kinds["by"], BLOCK_BYLINE) + self.assertEqual(kinds["b"], BLOCK_BODY) + + def test_running_head_and_folio_become_furniture(self): + blocks = [ + block("head", 1, 50, 20, 200, 12, "SCIENTIFIC AMERICAN", size=8.0), + block("folio", 1, 520, 820, 20, 12, "18", size=8.0), + block("b", 1, 50, 150, 200, 300, "Body copy " * 40, size=10.0), + ] + classify_blocks(blocks, PAGE_H, PAGE_W, + repeated={"SCIENTIFIC AMERICAN"}) + kinds = {b.id: b.type for b in blocks} + self.assertEqual(kinds["head"], BLOCK_FURNITURE) + self.assertEqual(kinds["folio"], BLOCK_FURNITURE) + self.assertEqual(kinds["b"], BLOCK_BODY) + + def test_small_text_next_to_a_figure_is_a_caption(self): + """A photo page has no body copy of its own, so the baseline must come + from the document -- otherwise the caption becomes its own reference.""" + blocks = [block("cap", 1, 50, 420, 200, 20, "A lithium cell.", size=7.5)] + classify_blocks(blocks, PAGE_H, PAGE_W, + image_boxes=[[50, 200, 200, 210]], base_size=10.0) + self.assertEqual(blocks[0].type, BLOCK_CAPTION) + + def test_small_text_away_from_any_figure_is_not_a_caption(self): + blocks = [block("fn", 1, 50, 420, 200, 20, "See note 4.", size=7.5)] + classify_blocks(blocks, PAGE_H, PAGE_W, image_boxes=[], base_size=10.0) + self.assertNotEqual(blocks[0].type, BLOCK_CAPTION) + + def test_photo_page_caption_would_misclassify_on_a_per_page_baseline(self): + """Guards the regression directly: no base_size means the lone caption + measures itself and reads as body text.""" + blocks = [block("cap", 1, 50, 420, 200, 20, "A lithium cell.", size=7.5)] + classify_blocks(blocks, PAGE_H, PAGE_W, + image_boxes=[[50, 200, 200, 210]]) + self.assertEqual(blocks[0].type, BLOCK_BODY) + + def test_advertisement_marker_is_excluded(self): + blocks = [block("ad", 1, 50, 50, 200, 20, "ADVERTISEMENT", size=9.0)] + classify_blocks(blocks, PAGE_H, PAGE_W) + self.assertEqual(blocks[0].type, BLOCK_AD) + + def test_an_advertisement_label_carries_its_whole_page(self): + """The ad's own display type would otherwise read as a headline.""" + blocks = [ + block("m", 4, 50, 50, 200, 20, "ADVERTISEMENT", size=9.0), + block("big", 4, 60, 400, 400, 60, "BUY OUR THING", size=40.0), + block("folio", 4, 520, 820, 20, 12, "4", size=8.0), + ] + classify_blocks(blocks, PAGE_H, PAGE_W, repeated=set(), base_size=10.0) + mark_advert_pages(blocks) + kinds = {b.id: b.type for b in blocks} + self.assertEqual(kinds["big"], BLOCK_AD) + self.assertEqual(kinds["folio"], BLOCK_FURNITURE) + + def test_a_page_without_the_label_is_untouched(self): + blocks = [ + block("t", 3, 50, 60, 470, 50, "The Future of Batteries", size=26.0), + block("b", 3, 50, 150, 200, 300, "Battery copy. " * 40, size=10.0), + ] + classify_blocks(blocks, PAGE_H, PAGE_W, base_size=10.0) + mark_advert_pages(blocks) + self.assertEqual(blocks[0].type, BLOCK_TITLE) + + def test_pullquote_repeating_body_copy_is_demoted(self): + """Narrating a pull quote reads the same sentence twice.""" + sentence = "Battery chemistry is entering an entirely new phase" + blocks = [ + block("b", 1, 50, 150, 200, 300, + f"Some lead in. {sentence}. And more copy follows.", size=10.0), + block("pq", 1, 320, 300, 200, 60, f"“{sentence}”", size=14.0), + ] + classify_blocks(blocks, PAGE_H, PAGE_W) + mark_pullquotes(blocks) + self.assertEqual(blocks[1].type, BLOCK_PULLQUOTE) + + def test_deck_that_does_not_repeat_body_stays_a_deck(self): + blocks = [ + block("b", 1, 50, 300, 200, 300, "Entirely different copy. " * 20, size=10.0), + block("dk", 1, 50, 120, 470, 40, + "Why the next decade belongs to solid state", size=14.0), + ] + classify_blocks(blocks, PAGE_H, PAGE_W) + mark_pullquotes(blocks) + self.assertEqual(blocks[1].type, BLOCK_DECK) + + def test_byline_rejects_a_sentence(self): + self.assertEqual(looks_like_byline("By the time the cell charges, it is hot.")[0], False) + self.assertEqual(looks_like_byline("By Jane Smith"), (True, "Jane Smith")) + self.assertEqual(looks_like_byline("Von Jan Müller"), (True, "Jan Müller")) + + +class FurnitureDetectionTests(unittest.TestCase): + def test_lines_repeating_across_pages_are_furniture(self): + pages = [["SCIENTIFIC AMERICAN", f"unique body {i}"] for i in range(10)] + repeated = find_repeated_lines(pages) + self.assertIn("SCIENTIFIC AMERICAN", repeated) + self.assertNotIn("unique body 3", repeated) + + def test_no_false_positives_on_a_short_document(self): + self.assertEqual(find_repeated_lines([["a"], ["b"]]), set()) + + +class ContentsPageTests(unittest.TestCase): + def test_contents_page_scores_above_a_feature_page(self): + toc = [ + "Contents", + "Features", + "The future of batteries ......... 18", + "Inside the deep ocean ........... 30", + "The new space race .............. 44", + "Departments", + "Books ........................... 60", + "Reviews ......................... 64", + ] + feature = ["The Future of Batteries", "By Jane Smith", + "Battery technology is entering a new phase. " * 20] + self.assertGreater(score_contents_page(toc), 0.45) + self.assertLess(score_contents_page(feature), 0.45) + + def test_entries_parse_with_section_and_page(self): + entries = parse_contents([ + "Contents", + "Features", + "The future of batteries ......... 18", + "Inside the deep ocean ........... 30", + "Departments", + "Reviews ......................... 64", + ]) + self.assertEqual(len(entries), 3) + self.assertEqual(entries[0], {"title": "The future of batteries", + "page": 18, "section": "Features"}) + self.assertEqual(entries[2]["section"], "Departments") + + def test_garbage_entries_are_rejected(self): + self.assertEqual(parse_contents(["ab .... 99999", "x . 3"]), []) + + +class ContinuationTests(unittest.TestCase): + def test_continued_on_and_from_are_found(self): + self.assertEqual(find_continuation("(continued on page 88)")[0], 88) + self.assertEqual(find_continuation("Fortsetzung auf Seite 90")[0], 90) + self.assertEqual(find_continuation("continued from page 27")[1], 27) + + def test_plain_text_has_no_continuation(self): + self.assertEqual(find_continuation("The page turned slowly."), (None, None)) + + +class ArticleGroupingTests(unittest.TestCase): + def _issue_blocks(self): + blocks = [ + block("b0018-000", 18, 50, 60, 470, 50, + "The Future of Batteries", size=26.0, btype=BLOCK_TITLE), + block("b0018-001", 18, 50, 120, 200, 14, + "By Jane Smith", size=10.0, btype=BLOCK_BYLINE), + block("b0018-002", 18, 50, 150, 200, 300, + "Battery copy. " * 40, size=10.0), + block("b0030-000", 30, 50, 60, 470, 50, + "Inside the Deep Ocean", size=26.0, btype=BLOCK_TITLE), + block("b0030-001", 30, 50, 150, 200, 300, + "Ocean copy. " * 40, size=10.0), + ] + for i, b in enumerate(blocks): + b.reading_order = i + return blocks + + def test_contents_entries_anchor_article_boundaries(self): + contents = [ + {"title": "The future of batteries", "page": 18, "section": "Features"}, + {"title": "Inside the deep ocean", "page": 30, "section": "Features"}, + ] + arts = group_articles(self._issue_blocks(), contents, page_count=40) + self.assertEqual(len(arts), 2) + self.assertEqual(arts[0].pages[0], 18) + self.assertEqual(arts[0].author, "Jane Smith") + self.assertEqual(arts[1].pages[0], 30) + # The first article must stop before the second begins. + self.assertNotIn(30, arts[0].pages) + + def test_headline_starts_an_article_without_a_contents_entry(self): + arts = group_articles(self._issue_blocks(), [], page_count=40) + self.assertEqual(len(arts), 2) + self.assertLess(arts[0].confidence, 0.9) # weaker signal, flagged for review + + def test_furniture_and_ads_never_enter_an_article(self): + blocks = self._issue_blocks() + ad = block("b0019-000", 19, 50, 50, 400, 400, "ADVERTISEMENT", + size=9.0, btype=BLOCK_AD) + folio = block("b0019-001", 19, 520, 820, 20, 12, "19", + size=8.0, btype=BLOCK_FURNITURE) + blocks.extend([ad, folio]) + arts = group_articles(blocks, [{"title": "The future of batteries", + "page": 18, "section": ""}], page_count=40) + self.assertNotIn("b0019-000", arts[0].blocks) + self.assertNotIn("b0019-001", arts[0].blocks) + + def test_an_advert_page_between_features_is_not_claimed(self): + """The ad page starts no article, so it must not be swallowed by the + preceding one and highlighted as part of that feature.""" + blocks = self._issue_blocks() + ad = block("b0019-000", 19, 50, 50, 400, 400, "ADVERTISEMENT", + size=9.0, btype=BLOCK_AD) + ad.reading_order = 50 + blocks.append(ad) + arts = group_articles(blocks, [ + {"title": "The future of batteries", "page": 18, "section": ""}, + {"title": "Inside the deep ocean", "page": 30, "section": ""}, + ], page_count=40) + self.assertNotIn(19, arts[0].pages) + self.assertEqual(arts[0].pages, [18]) + + def test_a_jump_tail_is_absorbed_into_its_originating_article(self): + blocks = self._issue_blocks() + blocks[2].text += " (continued on page 88)" + tail_title = block("b0088-000", 88, 50, 60, 470, 30, + "Batteries, continued", size=18.0, btype=BLOCK_TITLE) + tail_body = block("b0088-001", 88, 50, 110, 200, 300, + "The rest of the battery copy. " * 20, size=10.0) + tail_title.reading_order, tail_body.reading_order = 90, 91 + blocks.extend([tail_title, tail_body]) + + arts = group_articles(blocks, [{"title": "The future of batteries", + "page": 18, "section": ""}], page_count=90) + first = arts[0] + self.assertIn(88, first.pages) + self.assertIn("b0088-001", first.blocks) + # and the tail is no longer a standalone stub + self.assertTrue(all("continued" not in a.title.lower() for a in arts[1:])) + + def test_a_page_with_only_a_headline_starts_no_article(self): + """Covers and full-page adverts carry display type but no copy; each + would otherwise become an entry the reviewer rejects on every issue.""" + blocks = [block("b0005-000", 5, 50, 60, 470, 50, "Special Offer", + size=30.0, btype=BLOCK_TITLE)] + blocks[0].reading_order = 0 + self.assertEqual(group_articles(blocks, [], page_count=10), []) + + def test_a_listed_article_with_no_body_copy_is_low_confidence(self): + """A contents entry still anchors an article, but an empty one is + flagged rather than silently trusted.""" + blocks = [block("b0005-000", 5, 50, 60, 470, 50, "Special Offer", + size=30.0, btype=BLOCK_TITLE)] + blocks[0].reading_order = 0 + arts = group_articles(blocks, [{"title": "Special Offer", "page": 5, + "section": ""}], page_count=10) + self.assertEqual(len(arts), 1) + self.assertLess(arts[0].confidence, 0.4) + + def test_the_contents_page_is_not_itself_an_article(self): + blocks = self._issue_blocks() + toc = block("b0002-000", 2, 50, 90, 300, 40, "Contents", + size=28.0, btype=BLOCK_TITLE) + toc_entry = block("b0002-001", 2, 60, 175, 300, 20, + "The future of batteries ......... 18", size=11.0) + toc.reading_order, toc_entry.reading_order = 0, 1 + blocks.extend([toc, toc_entry]) + arts = group_articles(blocks, [], page_count=40, skip_pages={2}) + self.assertTrue(all(a.pages[0] != 2 for a in arts)) + + +class ArticleTextTests(unittest.TestCase): + def test_only_narratable_blocks_in_reading_order(self): + layout = { + "page_count": 2, + "blocks": [ + {"id": "b2", "page": 1, "type": BLOCK_BODY, "reading_order": 2, + "text": "second", "bbox": [0, 0, 1, 1]}, + {"id": "b1", "page": 1, "type": BLOCK_TITLE, "reading_order": 1, + "text": "first", "bbox": [0, 0, 1, 1]}, + {"id": "cap", "page": 1, "type": BLOCK_CAPTION, "reading_order": 3, + "text": "a caption", "bbox": [0, 0, 1, 1]}, + {"id": "ad", "page": 1, "type": BLOCK_AD, "reading_order": 4, + "text": "buy this", "bbox": [0, 0, 1, 1]}, + ], + } + article = {"blocks": ["b1", "b2", "cap", "ad"]} + got = [b["text"] for b in article_text(layout, article)] + self.assertEqual(got, ["first", "second"]) + + with_caps = [b["text"] for b in article_text(layout, article, + include_captions=True)] + self.assertEqual(with_caps, ["first", "second", "a caption"]) + + def test_listening_estimate_counts_only_narratable_words(self): + blocks = [ + Block(id="b1", page=1, bbox=[0, 0, 1, 1], type=BLOCK_BODY, + text=" ".join(["word"] * 155)), + Block(id="ad", page=1, bbox=[0, 0, 1, 1], type=BLOCK_AD, + text=" ".join(["spam"] * 500)), + ] + from core.magazine import Article + art = Article(id="a1", title="t", blocks=["b1", "ad"]) + self.assertEqual(estimate_listen_seconds(blocks, art), 60) + + +class BuildArticlesTests(unittest.TestCase): + def test_best_contents_candidate_drives_grouping(self): + layout = { + "page_count": 40, + "blocks": [ + {"id": "b0018-000", "page": 18, "type": BLOCK_TITLE, + "reading_order": 0, "text": "The Future of Batteries", + "bbox": [50, 60, 470, 50], "font_size": 26.0, "font_name": "", + "bold": True, "column": 0, "confidence": 0.9, "source": "native"}, + {"id": "b0018-001", "page": 18, "type": BLOCK_BODY, + "reading_order": 1, "text": "Battery copy. " * 40, + "bbox": [50, 150, 200, 300], "font_size": 10.0, "font_name": "", + "bold": False, "column": 0, "confidence": 0.85, "source": "native"}, + ], + "contents_candidates": [ + {"page": 2, "score": 0.5, "entries": []}, + {"page": 3, "score": 0.9, "entries": [ + {"title": "The future of batteries", "page": 18, + "section": "Features"}]}, + ], + } + arts = build_articles(layout) + self.assertEqual(len(arts), 1) + self.assertEqual(arts[0]["section"], "Features") + self.assertGreater(arts[0]["estimated_seconds"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_magazine_routes.py b/tests/test_magazine_routes.py new file mode 100644 index 0000000..ec7abea --- /dev/null +++ b/tests/test_magazine_routes.py @@ -0,0 +1,91 @@ +"""Magazine route-layer regressions: path safety, review gating, script parsing.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest + +_CONFIG = tempfile.mkdtemp(prefix="ttvc-magazine-routes-test-") +os.environ.setdefault("CONFIG_DIR", _CONFIG) + +from fastapi import HTTPException + +from routes.magazine import _article_id, _doc_dir, _parse_script + + +class PathSafetyTests(unittest.TestCase): + def test_traversal_and_junk_ids_are_rejected(self): + for bad in ("../../etc", "a/b", "", "..", "x" * 41, "id-with-dash", "a.b"): + with self.assertRaises(HTTPException) as ctx: + _doc_dir(bad) + self.assertEqual(ctx.exception.status_code, 400, bad) + + def test_wellformed_but_missing_id_is_a_404_not_a_400(self): + with self.assertRaises(HTTPException) as ctx: + _doc_dir("0123456789abcdef0123456789abcdef") + self.assertEqual(ctx.exception.status_code, 404) + + def test_article_ids_reject_separators(self): + self.assertEqual(_article_id("article-001"), "article-001") + for bad in ("../x", "a/b", "", "a" * 65, "a b"): + with self.assertRaises(HTTPException): + _article_id(bad) + + +class PodcastScriptParsingTests(unittest.TestCase): + def test_plain_json_parses(self): + raw = json.dumps({ + "title": "Batteries", + "turns": [{"host": "A", "text": "Hello."}, + {"host": "B", "text": "Hi."}], + }) + got = _parse_script(raw, ["A", "B"]) + self.assertEqual(got["title"], "Batteries") + self.assertEqual([t["host"] for t in got["turns"]], ["A", "B"]) + + def test_markdown_fence_is_tolerated(self): + raw = '```json\n{"title":"T","turns":[{"host":"A","text":"Hi."}]}\n```' + self.assertEqual(_parse_script(raw, ["A", "B"])["turns"][0]["text"], "Hi.") + + def test_prose_around_the_json_is_stripped(self): + raw = 'Sure! Here is the script:\n{"title":"T","turns":[{"host":"A","text":"Hi."}]}\nHope that helps.' + self.assertEqual(_parse_script(raw, ["A", "B"])["title"], "T") + + def test_unknown_host_label_is_reassigned_not_dropped(self): + """An unrecognised label would otherwise leave a turn with no voice.""" + raw = json.dumps({"turns": [ + {"host": "NARRATOR", "text": "One."}, + {"host": "B", "text": "Two."}, + ]}) + got = _parse_script(raw, ["A", "B"]) + self.assertEqual(len(got["turns"]), 2) + self.assertTrue(all(t["host"] in ("A", "B") for t in got["turns"])) + + def test_empty_turns_are_skipped(self): + raw = json.dumps({"turns": [ + {"host": "A", "text": " "}, + {"host": "B", "text": "Real line."}, + ]}) + got = _parse_script(raw, ["A", "B"]) + self.assertEqual(len(got["turns"]), 1) + + def test_unusable_output_raises_502_not_500(self): + for raw in ("not json at all", "{}", json.dumps({"turns": []}), + json.dumps({"turns": [{"host": "A", "text": ""}]})): + with self.assertRaises(HTTPException) as ctx: + _parse_script(raw, ["A", "B"]) + self.assertEqual(ctx.exception.status_code, 502, raw) + + def test_overlong_turn_text_is_capped(self): + raw = json.dumps({"turns": [{"host": "A", "text": "x" * 5000}]}) + self.assertEqual(len(_parse_script(raw, ["A"])["turns"][0]["text"]), 2000) + + +def tearDownModule(): + shutil.rmtree(_CONFIG, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main()