From 788f186b9ce5d1f0342c69ab2d8896204a5ba39b Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Sat, 3 Oct 2026 00:09:45 +0200 Subject: [PATCH] fix(magazine): use sentence timings of interview renders for player and EPUB Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 3 +++ Manual.md | 2 +- architecture/magazine-ingestion.md | 6 +++-- core/epub_export.py | 34 +++++++++++++++++++------- core/magazine.py | 11 +++++++++ routes/magazine.py | 32 ++++++++++++++++++------ tests/test_magazine_routes.py | 39 ++++++++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 713ac84..55e0656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ยท versioned wi - **Share an article or a magazine as a link that opens straight in driving mode.** A *Share* button in the magazine toolbar (whole magazine), one in the playlist header and one in driving mode (current article) build `/#s-magazine?doc=&article=&mode=drive`. They use the browser's share sheet where there is one (it needs https, so on a plain-http LAN address they copy the link instead, with a polite aria-live toast, or show a box with the link if even copying is refused). Opening such a link, also on a first visit and also pasted into an already-open tab, opens the magazine and then driving mode for that article with audio, pictures and text; when the browser blocks autoplay a large *Tap to play* button appears. The link carries ids only (no key or password) and does not change who may reach the app; the ids are validated against a strict pattern and never put into markup, and anything malformed is ignored. Tested with Playwright (desktop deep link, blocked autoplay, hostile parameters, share sheet at 375 px, copy fallback). - **Filter the page grid by status.** Each legend entry is now a toggle button with the number of pages in that status; choose one or several to show only pages that have any of them. It combines with *Show only pages that need me*, the summary says how many pages are shown, *Show all pages* clears everything, a status no page has is greyed out, and the buttons announce their on/off state and work from the keyboard. +### Fixed +- **Sentence timings now also work for interview renders.** Every recorded audio file carries `"interview": true` (set whenever a second voice is configured, even without speaker labels), and the timings endpoint and EPUB export discarded all such records, so the player and the EPUB fell back to estimated sentence positions for every current article. The record is now matched to the text again, for interview renders by the label-stripped length the renderer recorded, so `/timings` returns exact (or anchored) sentence times and speech onsets for these articles too. Records for text edited since the render are still ignored. + ### Changed - **Driving mode: one display menu instead of the three-icon switch; fewer controls in the dock.** The top bar now has *Share*, a *menu* button (shows the current view), *Index* and *Exit*. The menu opens a flyout with *Picture*, *Lyrics* and *Compact* plus *Show text*, *Show pictures* (moved from the dock) and *Minimize text and picture*; it works with arrow keys, Home/End, Enter and Escape, closes on an outside tap, keeps 44 px targets, stays inside the screen from 320 px wide and never covers the dock. The playback speed button moved to the end of the slider row, after the time. The dock is now just the slider row and the transport buttons. Picture zoom stays on the picture. - **Driving mode has a new layout that always keeps the controls on screen.** The overlay is now one full-height column: a top bar (share, a *Picture / Lyrics / Compact* switch, minimize, close), a content area that shrinks and scrolls inside itself, and a controls dock pinned to the bottom (position slider, time, previous / -15 / play / +15 / next, speed, text, images). On an iPhone SE (375x667) the controls used to fall below the screen, the share button covered the article title and the top of the title was cut off; now the dock is always fully visible from 320x568 up to tablets, in landscape the picture and the three lines sit side by side, and long titles wrap to two lines. *Picture* fills the area (double-tap, pinch, wheel, +/- keys or the zoom buttons to zoom, drag to pan, Escape or the reset button to reset); *Lyrics* shows the whole text, the spoken sentence large and centered, tap a sentence to jump there, and scrolling by hand pauses following until *Back to current*; *Compact* (default) is the picture plus previous / current / next line. The minimize button shrinks everything to just the current line. The choice is remembered. Transport buttons stay perfect circles. diff --git a/Manual.md b/Manual.md index 47c7df9..a0c877b 100644 --- a/Manual.md +++ b/Manual.md @@ -423,7 +423,7 @@ read-aloud overlays (for example Thorium Reader, Apple Books, or Readium-based a **Expected result.** One chapter per article, a table of contents, the MP3s inside the file, and each sentence linked to its place in the audio. The file passes the W3C EPUBCheck (0 errors, 0 warnings on a real two-article, twelve-minute test). -**How exact is the highlighting.** Audio rendered **after 2026-09-30** records where every sentence starts and ends, so the +**How exact is the highlighting.** Audio rendered **after 2026-09-30** (including interview-voice renders) records where every sentence starts and ends, so the highlight is exact. Older audio has no such record: its sentences are placed by estimate, proportionally to their length across the real duration, so the highlight can drift by a sentence or two. The book's description states which it is; to get exact timings for an older article, regenerate its MP3. diff --git a/architecture/magazine-ingestion.md b/architecture/magazine-ingestion.md index 6ed769d..5fb602d 100644 --- a/architecture/magazine-ingestion.md +++ b/architecture/magazine-ingestion.md @@ -288,13 +288,15 @@ works regardless. MP3 export renders one TTS call per spoken sentence and joins them, so every sentence's position is known exactly. `synthesize_article_audio` now records it (`write_audio_timings`) in `audio/.timings.json`: per narrated segment its character count and the `[start_ms, end_ms]` of each of its sentences, plus the total duration and whether the audio used -interview voices. No text is stored. Older MP3s have no record. +interview voices (the flag is set whenever a second voice is configured, labels or not). No text is stored. Older MP3s have no record. + +Interview renders speak each block without its `Speaker:` label and record that shorter length, so `mag.spoken_chars_for(blocks, timings)` repeats the stripping (`interview_tts_segments`) and passes the expected lengths to `plan_sentences(..., spoken_chars)`; units carry `seg`/`nth` so the timings endpoint can pick the matching per-utterance speech onset. `core/epub_export.py` (pure packaging, no layout or OCR code) builds the book. `plan_sentences` splits each narrated block with the same sentence rule the MP3 and the reader use and assigns times in three tiers, reported per article: **exact** (record present, segment count and every segment's character count match the current text, and the sentence counts agree), **anchored** (paragraph start/end measured, its sentences share that span by length), **estimated** (no usable record, changed -text, or interview audio: each sentence's share of the real duration by spoken length). The package: `mimetype` stored first, +text: each sentence's share of the real duration by spoken length). The package: `mimetype` stored first, `container.xml`, an OPF 3.0 with `media:duration` per overlay and the active class, a nav document, one XHTML + one SMIL + the stored MP3 per article. All text is XML-escaped. `GET /api/magazine/{id}/epub[?article=id]` includes approved articles that have audio, builds into a temp file (removed after sending) and returns `application/epub+zip`. Validated with W3C EPUBCheck diff --git a/core/epub_export.py b/core/epub_export.py index 966eb05..4b11ceb 100644 --- a/core/epub_export.py +++ b/core/epub_export.py @@ -44,10 +44,16 @@ def _weight(text: str) -> float: def plan_sentences(blocks: Sequence[dict[str, Any]], duration_ms: float, - timings: dict[str, Any] | None) -> tuple[list[dict[str, Any]], str]: + timings: dict[str, Any] | None, + spoken_chars: Sequence[int] | None = None) -> tuple[list[dict[str, Any]], str]: """Split narrated blocks into sentences with [start, end] milliseconds; returns (units, quality). - Each unit: {block: index, text, start, end}. Blocks with no text are skipped. + Each unit: {block, text, start, end, seg, nth}: `seg` is the record's segment index, `nth` the + position among the segment's recorded utterances when the counts agree (else nth is None). + `spoken_chars`, one per narrated block, is the character count the renderer recorded for it; interview + renders strip the "Speaker:" label from the spoken text, so the caller passes the stripped lengths. + Without it the block's own text length is expected (a plain render). + Blocks with no text are skipped. """ narrated = [(i, b) for i, b in enumerate(blocks) if (b.get("text") or "").strip()] units: list[dict[str, Any]] = [] @@ -57,18 +63,28 @@ def plan_sentences(blocks: Sequence[dict[str, Any]], duration_ms: float, if not units: return [], "estimated" - segs = (timings or {}).get("segments") if timings and not timings.get("interview") else None - usable = isinstance(segs, list) and len(segs) == len(narrated) and all( - isinstance(s, dict) and s.get("chars") == len(str(b["text"])) and isinstance(s.get("times"), list) - and s["times"] for (_, b), s in zip(narrated, segs)) + segs = (timings or {}).get("segments") if timings else None + if timings and timings.get("interview"): + # An interview render's per-segment lengths can only be checked against the label-stripped text. + want = list(spoken_chars) if spoken_chars is not None else None + else: + want = [len(str(b["text"])) for _, b in narrated] + usable = want is not None and len(want) == len(narrated) and isinstance(segs, list) \ + and len(segs) == len(narrated) and all( + isinstance(s, dict) and s.get("chars") == w and isinstance(s.get("times"), list) + and s["times"] and all(isinstance(t, list) and len(t) == 2 for t in s["times"]) + for w, s in zip(want, segs)) if usable: quality = "exact" - for (i, b), seg in zip(narrated, segs): + for k, ((i, b), seg) in enumerate(zip(narrated, segs)): mine = [u for u in units if u["block"] == i] times = seg["times"] + for u in mine: + u["seg"], u["nth"] = k, None if len(times) == len(mine): - for u, (s0, s1) in zip(mine, times): + for n, (u, (s0, s1)) in enumerate(zip(mine, times)): u["start"], u["end"] = float(s0), float(s1) + u["nth"] = n else: # counts disagree: keep the paragraph exact, split by length quality = "anchored" if quality == "exact" else quality a, z = float(times[0][0]), float(times[-1][1]) @@ -131,7 +147,7 @@ def build_epub(dest: Path, *, title: str, issue: str, language: str, for n, art in enumerate(articles, 1): key = f"a{n:03d}" - units, tier = plan_sentences(art["blocks"], float(art["duration_ms"]), art.get("timings")) + units, tier = plan_sentences(art["blocks"], float(art["duration_ms"]), art.get("timings"), art.get("spoken_chars")) if not units: continue quality[str(art.get("title") or key)] = tier diff --git a/core/magazine.py b/core/magazine.py index 1f46cf3..bb2fe3a 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -5674,6 +5674,17 @@ def interview_tts_segments(blocks: Sequence[dict[str, Any]]) -> list[dict[str, s ] +def spoken_chars_for(blocks: Sequence[dict[str, Any]], timings: dict[str, Any] | None) -> list[int] | None: + """Per narrated block, the character count an interview render recorded (label stripped); None otherwise. + + The interview renderer speaks each block without its "Speaker:" prefix and records that shorter length, so + a timing record can only be matched to the text by repeating the same stripping. + """ + if not timings or not timings.get("interview"): + return None + return [len(str(seg["text"])) for seg in interview_tts_segments(blocks)] + + def synthesize_article_audio(doc_dir: Path, article: dict[str, Any], layout: dict[str, Any], voice: str, settings: dict[str, Any], *, interview_voice: str = "", diff --git a/routes/magazine.py b/routes/magazine.py index 36d7166..76bcbe5 100644 --- a/routes/magazine.py +++ b/routes/magazine.py @@ -727,16 +727,33 @@ async def magazine_article_timings(doc_id: str, article_id: str): if not timings or not timings.get("duration_ms"): return none blocks = mag.article_text(mag.load_layout(d), target, fuse=True, glossary=target.get("glossary")) - units, quality = epub_export.plan_sentences(blocks, float(timings["duration_ms"]), timings) + units, quality = epub_export.plan_sentences(blocks, float(timings["duration_ms"]), timings, + mag.spoken_chars_for(blocks, timings)) if quality == "estimated": return none # Speech onset: the TTS pads every sentence with a lead-in of silence, so the words start later than the # span does. Recorded at render time; measured from the decoded MP3 for audio rendered before that. - flat = [o for seg in timings["segments"] for o in (seg.get("onsets") or [])] - if len(flat) != len(units): - flat = (await asyncio.to_thread(mag.measure_speech_onsets, mp3, timings)) if quality == "exact" else None - if not flat or len(flat) != len(units): - flat = [0] * len(units) + segs = timings["segments"] + if not all(len(seg.get("onsets") or []) == len(seg.get("times") or []) for seg in segs): + measured = await asyncio.to_thread(mag.measure_speech_onsets, mp3, timings) + at = 0 + per_seg: list[list[int]] = [] + for seg in segs: + n = len(seg.get("times") or []) + per_seg.append(list((measured or [0] * (at + n))[at:at + n])) + at += n + else: + per_seg = [list(seg["onsets"]) if seg.get("onsets") else [0] * len(seg["times"]) for seg in segs] + flat, seen = [], set() + for u in units: + # An exact unit is one recorded utterance; in an anchored block (counts disagree) only the first + # sentence is known to start where the block's first utterance does. + on = per_seg[u["seg"]] if "seg" in u else [] + k = u.get("nth") + if k is None and u["block"] not in seen and on: + k = 0 + seen.add(u["block"]) + flat.append(on[k] if k is not None and k < len(on) else 0) return {"available": True, "quality": quality, "sentences": [ {"text": u["text"], "block": u["block"], "block_id": blocks[u["block"]].get("id"), "start": round(u["start"] / 1000, 3), "end": round(u["end"] / 1000, 3), @@ -3140,7 +3157,8 @@ async def magazine_epub(doc_id: str, article: str = ""): continue # an unreadable MP3 cannot be given timings at all duration_ms = int(length * 1000) parts.append({"title": a.get("title") or "", "author": a.get("author") or "", "blocks": blocks, - "mp3": mp3, "duration_ms": duration_ms, "timings": timings}) + "mp3": mp3, "duration_ms": duration_ms, "timings": timings, + "spoken_chars": mag.spoken_chars_for(blocks, timings)}) if not parts: raise HTTPException(404, "No approved article has audio yet") language = mag.detect_article_language(b["text"] for p in parts for b in p["blocks"]) diff --git a/tests/test_magazine_routes.py b/tests/test_magazine_routes.py index 4cb1966..2df9ba1 100644 --- a/tests/test_magazine_routes.py +++ b/tests/test_magazine_routes.py @@ -2391,6 +2391,45 @@ class ArticleTimingsRouteTests(unittest.TestCase): self.write({"version": 1, "interview": True, "duration_ms": 6000, "segments": []}) self.assertFalse(self.client.get(self.url()).json()["available"]) + def test_interview_render_record_is_used_like_a_plain_one(self): + # Shaped like a real audio/.timings.json: "interview": true is set whenever a second voice was + # configured, even when no speaker labels exist; segments are one per narrated block. + self.write({"version": 1, "interview": True, "duration_ms": 6000, "segments": [ + {"chars": len(self.t1), "times": [[0, 1500], [1500, 3000]], "onsets": [200, 300]}, + {"chars": len(self.t2), "times": [[3000, 6000]], "onsets": [500]}]}) + body = self.client.get(self.url()).json() + self.assertTrue(body["available"]) + self.assertEqual(body["quality"], "exact") + self.assertEqual([(s["start"], s["end"], s["speech"]) for s in body["sentences"]], + [(0.0, 1.5, 0.2), (1.5, 3.0, 1.8), (3.0, 6.0, 3.5)]) + + def test_interview_record_with_speaker_labels_matches_the_stripped_length(self): + # The renderer speaks "Frage: ..." without the label and records the shorter length. + import routes.magazine as routes + lay = json.loads((self.d / "layout.json").read_text()) + texts = ["Frage: Wie geht es Ihnen? Gut.", "Antwort: Sehr gut danke.", "Frage: Und sonst? Nichts.", + "Antwort: Alles gut."] + lay["blocks"] = [{"id": f"i{n}", "page": 1, "bbox": [0, n * 10, 10, 10], "type": "body", "text": t, + "reading_order": n} for n, t in enumerate(texts)] + (self.d / "layout.json").write_text(json.dumps(lay)) + arts = json.loads((self.d / "articles.json").read_text()) + arts["articles"][0]["blocks"] = [b["id"] for b in lay["blocks"]] + (self.d / "articles.json").write_text(json.dumps(arts)) + bodies = ["Wie geht es Ihnen? Gut.", "Sehr gut danke.", "Und sonst? Nichts.", "Alles gut."] + times = [[[0, 2000], [2000, 3000]], [[3000, 5000]], [[5000, 6000], [6000, 7000]], [[7000, 8000]]] + self.write({"version": 1, "interview": True, "duration_ms": 8000, "segments": [ + {"chars": len(b), "times": t} for b, t in zip(bodies, times)]}) + body = self.client.get(self.url()).json() + self.assertTrue(body["available"]) + self.assertEqual(body["quality"], "exact") + self.assertEqual([(s["start"], s["end"]) for s in body["sentences"]], + [(0.0, 2.0), (2.0, 3.0), (3.0, 5.0), (5.0, 6.0), (6.0, 7.0), (7.0, 8.0)]) + + def test_interview_record_for_changed_text_is_not_used(self): + self.write({"version": 1, "interview": True, "duration_ms": 6000, "segments": [ + {"chars": 5, "times": [[0, 3000]]}, {"chars": len(self.t2), "times": [[3000, 6000]]}]}) + self.assertFalse(self.client.get(self.url()).json()["available"]) + def test_unknown_article_is_404_and_bad_id_is_400(self): self.assertEqual(self.client.get(self.url("nope")).status_code, 404) self.assertEqual(self.client.get(self.url("a b")).status_code, 400)