From f88fefcee4634488cc26e2e8053e84d80f7411dc Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Fri, 2 Oct 2026 14:57:39 +0200 Subject: [PATCH] feat(magazine): driving mode shows three sentence lines and syncs per frame Previous/current/next sentence in a fixed-height box with a fade, tap to jump. The highlight is a binary search over sentence starts on audio.currentTime every animation frame, with forced resync on seeking, seeked, ratechange, play, pause and visibility; the marked word uses the sentence's own span. Playwright covers seeks, 1.5x/2x, rapid seeks. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 4 + Manual.md | 1 + static/js/magazine.js | 175 ++++++++++++++++++++++++-------- static/sections/s-magazine.html | 2 +- static/style.css | 22 +++- tests/magazine.spec.js | 157 +++++++++++++++++++++++++++- 6 files changed, 311 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5c5d92..72e747c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,11 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi - **Share an article or a magazine as a link that opens straight in driving mode.** A *Share* button in the magazine toolbar (whole magazine), one in the playlist header and one in driving mode (current article) build `/#s-magazine?doc=&article=&mode=drive`. They use the browser's share sheet where there is one (it needs https, so on a plain-http LAN address they copy the link instead, with a polite aria-live toast, or show a box with the link if even copying is refused). Opening such a link, also on a first visit and also pasted into an already-open tab, opens the magazine and then driving mode for that article with audio, pictures and text; when the browser blocks autoplay a large *Tap to play* button appears. The link carries ids only (no key or password) and does not change who may reach the app; the ids are validated against a strict pattern and never put into markup, and anything malformed is ignored. Tested with Playwright (desktop deep link, blocked autoplay, hostile parameters, share sheet at 375 px, copy fallback). - **Filter the page grid by status.** Each legend entry is now a toggle button with the number of pages in that status; choose one or several to show only pages that have any of them. It combines with *Show only pages that need me*, the summary says how many pages are shown, *Show all pages* clears everything, a status no page has is greyed out, and the buttons announce their on/off state and work from the keyboard. +### Changed +- **Driving mode shows three lines of text instead of the whole article.** The previous sentence (small), the current one (large, with the spoken word marked) and the next one (small) sit in a fixed-height box, so the controls below never jump. Lines fade in smoothly (no animation when your device asks for reduced motion), the text is not a live region so a screen reader does not read out every sentence, it is readable at 375 px, and tapping the previous or next line jumps to that sentence. Pictures work as before. + ### Fixed +- **Highlight stays in step through seeking, speed changes and background tabs.** The highlighted sentence is now read from the audio's own position on every animation frame (and immediately on seeking, seeked, rate change, play, pause and when the tab becomes visible) using a binary search over the sentence start times, so there is no cached position to go stale; a jump into a silent gap keeps the sentence just spoken, and the marked word follows the sentence's own start and end, weighted by word length. New browser tests cover seeking back and forward, 1.5x and 2x, speed changes and rapid repeated seeks. - **Audio and text now line up when the voice actually starts.** Measured on real rendered articles (an 8-sentence and a 71-sentence MP3): the recorded sentence boundaries are exact (no drift over 10 minutes, no encoder-delay offset: the MP3 carries a LAME/Info header, so browsers trim the encoder delay), but the voice engine pads every sentence with 0.4 to 0.9 s of silence before the first word, so the highlight lit up about 0.7 s early. New renders now record where speech begins in each sentence (`onsets` in `.timings.json`), older audio is measured from the decoded MP3 on request (in memory, nothing is written), and `/timings` returns it as `speech` per sentence. - **The item pictures on the character concept sheet show only objects, never hands, feet or people.** The lower-right tiles are now asked for as "a high-quality concept art prop sheet featuring isolated character accessories" on a plain, neutral studio background, still life, strictly inanimate objects, with no people, hands, faces or body parts; the item name (German names included) and the book's world and style stay in the prompt. A body-part list was also added to the item tile's negative prompt, which like the rest only acts if the cfg is raised above 1.0. Existing sheets need to be regenerated. - **Driving mode and the playlist highlight the sentence that is actually being spoken.** The player guessed where each sentence sits in the MP3 from its text length, so the highlighted line and the picture drifted from the voice (worst after seeking). The renderer already records the exact time of every sentence (`audio/.timings.json`); a new endpoint `GET /api/magazine/{doc}/articles/{id}/timings` returns them (`{available, sentences: [{text, block, block_id, start, end}]}`, seconds, from the same sentence planner the EPUB uses) and the player uses them, falling back to the old estimate when an MP3 has no usable record (older audio, interview audio, text edited since). The transcript is still fetched with captions, but the MP3 is built without them, so a caption now gets no time slot (the estimate no longer spends audio time on captions either). The highlight is derived only from the audio's current time against the timeline, so the slider, skip buttons, tapping a sentence, lock-screen seek and resume all stay in sync. Covered by a backend test (`ArticleTimingsRouteTests`) and a Playwright test with seeking. diff --git a/Manual.md b/Manual.md index 4769fcf..98b5afa 100644 --- a/Manual.md +++ b/Manual.md @@ -289,6 +289,7 @@ turned into a podcast discussion between two or three voices you pick. large touch target and keyboard reachable; the speed also applies to the playlist player. **Hands-free:** while a track plays, the lock screen, notification shade, Bluetooth headset and car display show the article title, magazine and first-page artwork and offer play/pause, previous/next article and ±15 s (and seeking, where the device offers it). In driving mode the screen is kept awake while audio plays and allowed to sleep when it is paused or you leave. Both are optional browser features; where one is missing or refused, playback itself is unaffected. Opening driving mode resumes the first unfinished article at the position you left it. + **Three lines of text:** driving mode shows the previous sentence (small), the current one (large, the spoken word marked) and the next (small) in a box of fixed height, so nothing moves around it. Tap or press Enter on the previous or next line to jump to that sentence. The text is deliberately not announced by screen readers sentence by sentence; the play/pause button and the position slider stay the way to control it. With reduced motion switched on in your device, lines change without animation. On a phone (375 px wide) the sentences wrap and a very long current sentence scrolls inside its box. Turn the text off with the *Text* option to see only the picture. **Text and picture follow the voice:** for audio generated since sentence times were recorded, the highlighted sentence (playlist and driving mode) and the picture shown come from the exact time each sentence was spoken, so they stay right after the slider, the 15-second buttons, a tap on a sentence, the lock screen or a resume. Captions are shown but are not in the MP3, so they are never highlighted. The highlight lights up a sentence just before its first word is heard (each sentence starts with a short silence, which is not counted). Older audio, interview-style audio and audio whose text was edited afterwards has no usable record and is followed by an estimate from sentence length (less exact); regenerate the audio to get exact sync. **Share a link:** the *Share* button in the toolbar (the whole magazine), the share button in the playlist header and the one at the top left of driving mode (the current article) share a link of the form `/#s-magazine?doc=&article=
&mode=drive`. Where the browser has a share sheet (phones, and any browser on https) it opens; otherwise the link is copied and a confirmation is announced (on a plain-http address a box with the link is shown to copy by hand). Opening the link goes straight to that article in driving mode with audio, pictures and text; if the browser refuses to start audio without a tap, a large *Tap to play* button appears. A link for the whole magazine opens at its first unfinished article. **Privacy and access:** the link contains only ids, no password or key. It is just an address: whoever opens it needs to reach this app the same way as for any other page (this app has no user accounts; access is controlled by your network or reverse proxy), and an unknown or removed article simply shows a message. Only articles that already have audio can be shared. A link whose parameters are not in the expected form is ignored. diff --git a/static/js/magazine.js b/static/js/magazine.js index 0dd6342..328205c 100644 --- a/static/js/magazine.js +++ b/static/js/magazine.js @@ -5118,12 +5118,16 @@ function savePlaylistProgress(completed, immediate) { renderPlaylistTracks(); } -// One {start, end, skip} per displayed sentence, in seconds on the MP3's own clock. +// One {start, end, speech, eff, skip} per displayed sentence, in seconds on the MP3's own clock. // The text is fetched with captions (shown), but the MP3 is built without them, so a caption sentence has no // slot (skip, zero width, sitting where the narration reaches it). Narrated sentences use the exact times the // renderer recorded (state.playlistExact, one entry per narrated sentence, from /timings) when there are any, -// else a length-based estimate over `duration`. The highlight only ever reads audio.currentTime against this, -// so seeking by slider, skip buttons, click, lock screen or resume cannot desync it. +// else a length-based estimate over `duration`. `speech` is when the words are first audible (each clip has a +// silent lead-in); `eff` is when the sentence lights up: just before its speech, never before its own start and +// never earlier than the sentence before it. The highlight only ever reads audio.currentTime against this +// (a binary search over `eff`), so seeking by slider, skip buttons, click, lock screen or resume cannot desync it. +const MAG_HIGHLIGHT_LEAD = 0.15; // seconds a sentence lights up before its first word is heard + function playlistTimeline(duration) { const sentences = state.playlistSentences; const exact = state.playlistExact; @@ -5134,10 +5138,11 @@ function playlistTimeline(duration) { if (exact && exact.length === narrated.length && exact.length) { let n = 0, cursor = 0; result = sentences.map((s) => { - if (s.type === 'caption') return { start: cursor, end: cursor, skip: true }; + if (s.type === 'caption') return { start: cursor, end: cursor, speech: cursor, skip: true }; const t = exact[n++]; cursor = t.end; - return { start: t.start, end: t.end, skip: false }; + const speech = Number.isFinite(t.speech) ? Math.min(t.end, Math.max(t.start, t.speech)) : t.start; + return { start: t.start, end: t.end, speech, skip: false }; }); } else { const weights = sentences.map((s) => s.type === 'caption' ? 0 : Math.max(6, s.text.replace(/\s+/g, ' ').length)); @@ -5146,29 +5151,63 @@ function playlistTimeline(duration) { result = weights.map((weight) => { const start = cursor / total * duration; cursor += weight; - return { start, end: cursor / total * duration, skip: weight === 0 }; + return { start, end: cursor / total * duration, speech: start, skip: weight === 0 }; }); } + const slots = []; + const pos = {}; + let prevEff = 0; + result.forEach((t, i) => { + if (t.skip) { t.eff = prevEff; return; } + t.eff = slots.length ? Math.max(prevEff, Math.min(t.end, Math.max(t.start, t.speech - MAG_HIGHLIGHT_LEAD))) : 0; + prevEff = t.eff; + pos[i] = slots.length; + slots.push(i); + }); + result.slots = slots; // indices of the narrated sentences, in order + result.pos = pos; // sentence index -> position in slots state.playlistTimelineKey = key; state.playlistTimelineCache = result; return result; } -// The sentence the audio is in: the first narrated slot that has not ended yet; past the end, the last one. +// The sentence the audio is in: the last narrated one whose lead-in has begun (binary search). Before the first +// that is the first; in a silent gap it stays on the sentence just spoken; a caption never takes the highlight. function playlistIndexAt(timeline, time) { - let index = timeline.findIndex((t) => !t.skip && time < t.end); - if (index < 0) { - for (let i = timeline.length - 1; i >= 0; i--) if (!timeline[i].skip) { index = i; break; } + const slots = timeline.slots || []; + if (!slots.length) return 0; + let lo = 0, hi = slots.length - 1, found = 0; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (timeline[slots[mid]].eff <= time) { found = mid; lo = mid + 1; } else hi = mid - 1; } - return Math.max(0, index); + return slots[found]; +} + +// Fraction (0..1) of the sentence's own spoken part [speech, end] that has played. +function playlistLocal(span, time) { + if (!span) return 0; + return Math.max(0, Math.min(0.999, (time - span.speech) / Math.max(0.1, span.end - span.speech))); +} + +// Which word is being said, by the share of the sentence's characters spoken so far. +function playlistWordAt(words, local) { + if (!words || !words.length) return -1; + const lens = words.map((w) => String(w).length + 1); + const total = lens.reduce((a, b) => a + b, 0); + let acc = 0; + for (let i = 0; i < lens.length; i++) { acc += lens[i]; if (local * total < acc) return i; } + return words.length - 1; } function magPlaylistSeekSentence(index) { const audio = $('mag-playlist-audio'); if (!audio || !state.playlistSentences[index]) return; const timeline = playlistTimeline(playlistDuration(state.article)); - if (timeline[index]) audio.currentTime = timeline[index].start; + const span = timeline[index]; + if (span) audio.currentTime = span.skip ? span.start : Math.min(span.end, span.eff + 0.005); audio.play().catch(() => {}); + magPlaylistSync(); } function renderPlaylistCompanion() { @@ -5208,20 +5247,22 @@ function magPlaylistSync() { const audio = $('mag-playlist-audio'); const a = state.articles.find((x) => x.id === state.playlistCurrent); if (!audio || !a) return; + const time = audio.currentTime; const duration = playlistDuration(a); const timeline = playlistTimeline(duration); - const index = playlistIndexAt(timeline, audio.currentTime); + const index = playlistIndexAt(timeline, time); + const span = timeline[index]; + const local = playlistLocal(span, time); + const wordIdx = playlistWordAt(state.playlistSentences[index]?.words, local); const follow = $('mag-playlist-karaoke')?.checked !== false; document.querySelectorAll('#mag-playlist-transcript [data-playlist-sentence]').forEach((el) => { el.classList.toggle('is-current', follow && Number(el.dataset.playlistSentence) === index); el.querySelectorAll('.mag-w').forEach((w) => w.classList.remove('is-current')); }); const line = document.querySelector('#mag-playlist-transcript [data-playlist-sentence="' + index + '"]'); - if (follow && line && timeline[index]) { - const local = Math.max(0, Math.min(0.999, (audio.currentTime - timeline[index].start) / - Math.max(0.1, timeline[index].end - timeline[index].start))); + if (follow && line && span) { const words = line.querySelectorAll('.mag-w'); - if (words.length) words[Math.min(words.length - 1, Math.floor(local * words.length))].classList.add('is-current'); + if (words[wordIdx]) words[wordIdx].classList.add('is-current'); if (state.playlistHighlightIndex !== index) { if (!prefersReducedMotion()) line.scrollIntoView({ block: 'center', behavior: 'smooth' }); else line.scrollIntoView({ block: 'center' }); @@ -5229,11 +5270,36 @@ function magPlaylistSync() { } } renderPlaylistVisual(index); - const pct = duration ? Math.min(100, audio.currentTime / duration * 100) : 0; - const span = timeline[index]; - const local = span ? Math.max(0, Math.min(0.999, (audio.currentTime - span.start) / - Math.max(0.1, span.end - span.start))) : 0; - if (state.driving) magDrivingSync(audio, duration, pct, index, local); + const pct = duration ? Math.min(100, time / duration * 100) : 0; + state.syncIndex = index; state.syncWord = wordIdx; state.syncTime = time; + if (state.driving) magDrivingSync(audio, duration, pct, index, local, timeline); +} + +// Per-frame check while playing: the full sync (DOM work) only runs when the sentence or word changed, or +// 4 times a second for the clock and slider. +function magPlaylistFrame() { + const audio = $('mag-playlist-audio'); + const a = state.articles.find((x) => x.id === state.playlistCurrent); + if (!audio || !a) return; + const time = audio.currentTime; + const timeline = playlistTimeline(playlistDuration(a)); + const index = playlistIndexAt(timeline, time); + const wordIdx = playlistWordAt(state.playlistSentences[index]?.words, playlistLocal(timeline[index], time)); + if (index !== state.syncIndex || wordIdx !== state.syncWord || Math.abs(time - (state.syncTime || 0)) >= 0.25) magPlaylistSync(); +} + +// One loop reads audio.currentTime (media time: playbackRate and seeks never skew it) every animation frame +// while audio plays. Events below force a resync for everything the loop cannot see (paused seeks, hidden tabs). +function magSyncLoopStart() { + if (state.syncRaf) return; + const tick = () => { + state.syncRaf = 0; + const audio = $('mag-playlist-audio'); + if (!audio || audio.paused || audio.ended) return; + magPlaylistFrame(); + state.syncRaf = requestAnimationFrame(tick); + }; + state.syncRaf = requestAnimationFrame(tick); } async function loadPlaylistCompanion(a) { @@ -5241,6 +5307,7 @@ async function loadPlaylistCompanion(a) { state.playlistSentences = []; state.playlistImages = []; state.playlistExact = null; + state.playlistTimelineKey = null; renderPlaylistCompanion(); try { await ensureLayout(); @@ -5253,9 +5320,10 @@ async function loadPlaylistCompanion(a) { if (token !== state.playlistLoadToken || state.playlistCurrent !== a.id) return; state.playlistSentences = splitSentences(data.blocks || []); state.playlistExact = timings && timings.available && Array.isArray(timings.sentences) - ? timings.sentences.map((u) => ({ start: Number(u.start), end: Number(u.end) })) + ? timings.sentences.map((u) => ({ start: Number(u.start), end: Number(u.end), speech: u.speech == null ? NaN : Number(u.speech) })) .filter((u) => Number.isFinite(u.start) && Number.isFinite(u.end)) : null; + state.playlistTimelineKey = null; state.playlistImages = articleImages(a); renderPlaylistCompanion(); } catch (_) { @@ -5805,8 +5873,11 @@ function magDrivingApplyToggles() { if (visual && !prefs.images) visual.hidden = true; } -// Called from magPlaylistSync on every timeupdate while the overlay is open. -function magDrivingSync(audio, duration, pct, sentenceIndex, local) { +// Called from magPlaylistSync whenever the sentence, word or clock changes while the overlay is open. +// The text shows three sentences only: the one before, the current one (largest, with the spoken word marked) +// and the one after. All lines are built once per article; only visibility and classes move, inside a +// fixed-height box, so nothing around it jumps. +function magDrivingSync(audio, duration, pct, sentenceIndex, local, timeline) { const prefs = magDrivingPrefs(); const seek = $('mag-driving-seek'); if (seek && document.activeElement !== seek) seek.value = String(Math.round(pct * 10)); @@ -5817,31 +5888,46 @@ function magDrivingSync(audio, duration, pct, sentenceIndex, local) { if (lyrics) { lyrics.hidden = !prefs.text || !sentences.length; if (!lyrics.hidden) { - // Whole article as karaoke text: built once per article, then only the current-sentence and - // current-word classes move. Word timing is estimated from the MP3 position, like the playlist. const key = state.playlistCurrent + ':' + sentences.length; if (lyrics.dataset.key !== key) { lyrics.dataset.key = key; - lyrics.dataset.index = ''; + lyrics.dataset.window = ''; lyrics.innerHTML = sentences.map((sent, i) => - '

' + + '

').join(''); - } - if (lyrics.dataset.index !== String(sentenceIndex)) { - lyrics.dataset.index = String(sentenceIndex); - lyrics.querySelectorAll('.mag-driving-line.is-current').forEach((el) => el.classList.remove('is-current')); - const line = lyrics.querySelector('.mag-driving-line[data-i="' + sentenceIndex + '"]'); - if (line) { - line.classList.add('is-current'); - // Scroll the box itself (not the page) so the current sentence sits in the middle. - const target = line.offsetTop - (lyrics.clientHeight - line.offsetHeight) / 2; - lyrics.scrollTo({ top: Math.max(0, target), behavior: prefersReducedMotion() ? 'auto' : 'smooth' }); + if (!lyrics.dataset.wired) { + lyrics.dataset.wired = '1'; + const jump = (el) => { const line = el.closest('.mag-driving-line.is-prev, .mag-driving-line.is-next'); if (line) magPlaylistSeekSentence(Number(line.dataset.i)); }; + lyrics.addEventListener('click', (e) => jump(e.target)); + lyrics.addEventListener('keydown', (e) => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); jump(e.target); } }); } } - const line = lyrics.querySelector('.mag-driving-line.is-current'); + const slots = (timeline && timeline.slots) || []; + const p = timeline && timeline.pos ? timeline.pos[sentenceIndex] : undefined; + const prev = p > 0 ? slots[p - 1] : -1, next = p >= 0 && p < slots.length - 1 ? slots[p + 1] : -1; + const windowKey = prev + ',' + sentenceIndex + ',' + next; + if (lyrics.dataset.window !== windowKey) { + (lyrics.dataset.window || '').split(',').forEach((i) => { + const el = lyrics.querySelector('.mag-driving-line[data-i="' + i + '"]'); + if (el) { el.hidden = true; el.classList.remove('is-prev', 'is-current', 'is-next'); el.removeAttribute('tabindex'); el.removeAttribute('role'); el.removeAttribute('aria-label'); } + }); + lyrics.dataset.window = windowKey; + [[prev, 'is-prev', 'Jump to the previous sentence'], [sentenceIndex, 'is-current', ''], [next, 'is-next', 'Jump to the next sentence']].forEach(([i, cls, label]) => { + const el = i >= 0 ? lyrics.querySelector('.mag-driving-line[data-i="' + i + '"]') : null; + if (!el) return; + el.hidden = false; + el.classList.add(cls); + if (label) { el.setAttribute('tabindex', '0'); el.setAttribute('role', 'button'); el.setAttribute('aria-label', label + ': ' + (sentences[i].text || '')); } + }); + // A very long sentence can outgrow the box: keep its start in view (the box itself scrolls, not the page). + const cur = lyrics.querySelector('.mag-driving-line.is-current'); + if (cur && lyrics.scrollHeight > lyrics.clientHeight) lyrics.scrollTo({ top: cur.offsetTop - 8, behavior: prefersReducedMotion() ? 'auto' : 'smooth' }); + } + const cur = lyrics.querySelector('.mag-driving-line.is-current'); lyrics.querySelectorAll('.mag-dw.is-current').forEach((w) => w.classList.remove('is-current')); - const words = line ? line.querySelectorAll('.mag-dw') : []; - if (words.length) words[Math.min(words.length - 1, Math.floor((local || 0) * words.length))].classList.add('is-current'); + const words = cur ? cur.querySelectorAll('.mag-dw') : []; + const wi = playlistWordAt(sentences[sentenceIndex]?.words, local || 0); + if (words[wi]) words[wi].classList.add('is-current'); } } const visual = $('mag-driving-visual'); @@ -5949,6 +6035,9 @@ function wirePlaylistAudio() { audio.addEventListener('ended', _magPlaylistNext); audio.addEventListener('playing', () => magDrivingStartPrompt(false)); audio.addEventListener('timeupdate', () => { magPlaylistSync(); savePlaylistProgress(false, false); magMediaSessionPosition(); }); + // The loop covers smooth playback; these cover every way the position can change without it noticing. + ['seeking', 'seeked', 'ratechange', 'durationchange', 'pause', 'ended'].forEach((ev) => audio.addEventListener(ev, magPlaylistSync)); + ['play', 'playing', 'seeked'].forEach((ev) => audio.addEventListener(ev, () => { magPlaylistSync(); magSyncLoopStart(); })); audio.addEventListener('loadedmetadata', () => { const a = state.articles.find((x) => x.id === state.playlistCurrent); if (a && Number.isFinite(audio.duration)) a.audio_seconds = audio.duration; @@ -5965,7 +6054,7 @@ function wirePlaylistAudio() { }); audio.addEventListener('loadedmetadata', magMediaSessionSync); // The browser drops a screen wake lock when the tab is hidden; take it again on return. - document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'visible') magDrivingWake(); }); + document.addEventListener('visibilitychange', () => { magPlaylistSync(); if (document.visibilityState === 'visible') { magDrivingWake(); magSyncLoopStart(); } }); } // ── Boot ───────────────────────────────────────────────────────────────────── diff --git a/static/sections/s-magazine.html b/static/sections/s-magazine.html index f1365e4..c5be38f 100644 --- a/static/sections/s-magazine.html +++ b/static/sections/s-magazine.html @@ -341,7 +341,7 @@ - +
diff --git a/static/style.css b/static/style.css index 6477f69..174ffc2 100644 --- a/static/style.css +++ b/static/style.css @@ -5453,12 +5453,24 @@ input[list="lang-codes"] { .mag-driving-opt:focus-visible { outline: 3px solid var(--accent); outline-offset: 2px; } @media (max-width: 860px) { .mag-driving-btn.is-small { width: 52px; height: 52px; font-size: 22px; } .mag-driving-controls { gap: 12px; } } -/* Driving mode karaoke text: whole article, current sentence + word highlighted */ -.mag-driving-lyrics { max-height: 34vh; overflow-y: auto; text-align: left; padding: 8px 14px; border-radius: 12px; - background: color-mix(in srgb, var(--surface) 70%, transparent); scroll-behavior: auto; } -.mag-driving-lyrics .mag-driving-line { margin: 0 0 12px; font-size: clamp(18px, 2.6vw, 26px); opacity: .55; color: var(--text); font-weight: 400; } -.mag-driving-lyrics .mag-driving-line.is-current { opacity: 1; font-weight: 600; font-size: clamp(20px, 3vw, 30px); +/* Driving mode text: three sentences only (previous, current, next) in a fixed-height box, so nothing jumps */ +.mag-driving-lyrics { height: clamp(210px, 40vh, 360px); max-height: none; overflow-y: auto; overscroll-behavior: contain; + display: flex; flex-direction: column; justify-content: safe center; gap: 12px; text-align: left; padding: 8px 14px; + border-radius: 12px; background: color-mix(in srgb, var(--surface) 70%, transparent); scroll-behavior: auto; } +.mag-driving-lyrics .mag-driving-line { margin: 0; flex: none; font-size: clamp(15px, 4vw, 20px); line-height: 1.35; + opacity: .6; color: var(--text); font-weight: 400; cursor: pointer; + display: -webkit-box; -webkit-line-clamp: 2; line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } +.mag-driving-lyrics .mag-driving-line[hidden] { display: none; } +.mag-driving-lyrics .mag-driving-line:focus-visible { outline: 3px solid var(--accent); outline-offset: 2px; } +.mag-driving-lyrics .mag-driving-line.is-current { opacity: 1; font-weight: 700; font-size: clamp(21px, 6vw, 32px); cursor: default; + display: block; overflow: visible; -webkit-line-clamp: unset; line-clamp: unset; border-left: 4px solid var(--accent); padding-left: 10px; } +@media (prefers-reduced-motion: no-preference) { + .mag-driving-lyrics .mag-driving-line.is-current { animation: mag-line-in .3s ease-out; } + .mag-driving-lyrics .mag-driving-line.is-prev, .mag-driving-lyrics .mag-driving-line.is-next { animation: mag-line-fade .3s ease-out; } +} +@keyframes mag-line-in { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: none; } } +@keyframes mag-line-fade { from { opacity: 0; } to { opacity: .6; } } .mag-dw.is-current { background: var(--accent); color: var(--on-accent, #fff); border-radius: 4px; padding: 0 3px; } .mag-driving-lyrics { position: relative; } diff --git a/tests/magazine.spec.js b/tests/magazine.spec.js index fb9ec49..8c92cb3 100644 --- a/tests/magazine.spec.js +++ b/tests/magazine.spec.js @@ -1699,7 +1699,8 @@ test.describe('Hands-free listening', () => { await expect(page.locator('#mag-playlist-transcript [data-playlist-sentence]')).toHaveCount(5); await page.evaluate(() => window.magOpenDriving()); await expect(page.locator('#mag-driving')).toBeVisible(); - await expect(page.locator('#mag-driving-lyrics .mag-driving-line')).toHaveCount(5); + await expect(page.locator('#mag-driving-lyrics .mag-driving-line')).toHaveCount(5); // all built, three shown + await expect(page.locator('#mag-driving-lyrics .mag-driving-line:not([hidden])')).toHaveCount(3); await page.evaluate(() => document.getElementById('mag-playlist-audio').pause()); const at = async (t) => { await page.evaluate((x) => { document.getElementById('mag-playlist-audio').currentTime = x; }, t); @@ -2576,3 +2577,157 @@ test.describe('Filtering the page grid by status', () => { await expect(chip(page, 'flagged')).toHaveAttribute('aria-pressed', 'true'); }); }); + + +// ── Audio/text sync: the highlight is always the sentence containing audio.currentTime ────────────────────── +test.describe('Driving mode: three-line text and robust audio/text sync', () => { + // Four 3 s sentences. Each has 0.7 s of silent lead-in (speech), so a sentence lights up 0.15 s before its + // words: 0, 3.55, 6.55, 9.55 (the first always from 0). + const EFF = [0, 3.55, 6.55, 9.55]; + const expectedAt = (t) => EFF.reduce((n, e, i) => (e <= t ? i : n), 0); + + async function openDriving(page, { speed } = {}) { + await mockBackend(page, { articleStatus: 'approved', hasAudio: true }); + const blocks = [{ id: 'b1', page: 3, bbox: [0, 0, 1, 1], type: 'body', + text: 'Alpha one is here. Bravo two is next. Charlie three follows. Delta four ends it.', reading_order: 0 }]; + await page.route(`**/api/magazine/${DOC_ID}/articles/article-001/text*`, route => route.fulfill({ json: { article: ARTICLE_APPROVED, blocks } })); + await page.route(`**/api/magazine/${DOC_ID}/articles/article-001/timings`, route => route.fulfill({ json: { available: true, quality: 'exact', + sentences: [0, 3, 6, 9].map((s, i) => ({ text: 'S' + i, block: 0, start: s, end: s + 3, speech: s + 0.7 })) } })); + await page.route(`**/api/magazine/${DOC_ID}/articles/article-001/audio`, route => { + const wav = silentWav(12000), range = /bytes=(\d+)-(\d*)/.exec(route.request().headers().range || ''); + if (!range) return route.fulfill({ body: wav, contentType: 'audio/wav', headers: { 'Accept-Ranges': 'bytes' } }); + const from = Number(range[1]), to = range[2] ? Number(range[2]) : wav.length - 1; + return route.fulfill({ status: 206, body: wav.subarray(from, to + 1), contentType: 'audio/wav', + headers: { 'Accept-Ranges': 'bytes', 'Content-Range': `bytes ${from}-${to}/${wav.length}` } }); + }); + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#app-skeleton', { state: 'detached', timeout: 15_000 }); + await navTo(page, 's-magazine'); + await page.locator('#mag-library-list .mag-card').first().click(); + await page.getByRole('button', { name: 'Playlist' }).click(); + await page.locator('.mag-playlist-track-play').click(); + await expect(page.locator('#mag-playlist-transcript [data-playlist-sentence]')).toHaveCount(4); + await page.evaluate(() => window.magOpenDriving()); + await expect(page.locator('#mag-driving')).toBeVisible(); + await expect.poll(() => page.evaluate(() => document.getElementById('mag-playlist-audio').duration)).toBeGreaterThan(11); + if (speed) await page.evaluate((v) => { const a = document.getElementById('mag-playlist-audio'); a.defaultPlaybackRate = v; a.playbackRate = v; }, speed); + } + const state = (page) => page.evaluate(() => { + const a = document.getElementById('mag-playlist-audio'); + const cur = document.querySelector('#mag-driving-lyrics .mag-driving-line.is-current'); + return { t: a.currentTime, current: cur ? Number(cur.dataset.i) : -1, + shown: [...document.querySelectorAll('#mag-driving-lyrics .mag-driving-line:not([hidden])')].map((e) => Number(e.dataset.i)) }; + }); + + test('shows only previous, current and next sentence, and the window moves with the audio', async ({ page }) => { + await openDriving(page); + await page.evaluate(() => document.getElementById('mag-playlist-audio').pause()); + for (const [t, shown] of [[1, [0, 1]], [4, [0, 1, 2]], [7, [1, 2, 3]], [11, [2, 3]]]) { + await page.evaluate((x) => { document.getElementById('mag-playlist-audio').currentTime = x; }, t); + await expect.poll(async () => (await state(page)).shown).toEqual(shown); + const s = await state(page); + expect(s.current).toBe(expectedAt(s.t)); + } + // the box is a fixed height: it does not grow or shrink as the lines change + const h1 = await page.locator('#mag-driving-lyrics').evaluate((e) => e.getBoundingClientRect().height); + await page.evaluate(() => { document.getElementById('mag-playlist-audio').currentTime = 1; }); + await expect.poll(async () => (await state(page)).current).toBe(0); + expect(await page.locator('#mag-driving-lyrics').evaluate((e) => e.getBoundingClientRect().height)).toBe(h1); + // not a live region: sentences are never announced one by one + expect(await page.locator('#mag-driving-lyrics').getAttribute('aria-live')).toBe('off'); + }); + + test('tapping the previous or next line jumps to that sentence', async ({ page }) => { + await openDriving(page); + await page.evaluate(() => { const a = document.getElementById('mag-playlist-audio'); a.pause(); a.currentTime = 4; }); + await expect.poll(async () => (await state(page)).current).toBe(1); + await page.locator('#mag-driving-lyrics .mag-driving-line.is-next').click(); + await expect.poll(async () => (await state(page)).current).toBe(2); + expect((await state(page)).t).toBeGreaterThanOrEqual(6.5); + await page.locator('#mag-driving-lyrics .mag-driving-line.is-prev').click(); + await expect.poll(async () => (await state(page)).current).toBe(1); + }); + + test('reduced motion turns the line transition off', async ({ page }) => { + await page.emulateMedia({ reducedMotion: 'reduce' }); + await openDriving(page); + await page.evaluate(() => { const a = document.getElementById('mag-playlist-audio'); a.pause(); a.currentTime = 4; }); + await expect.poll(async () => (await state(page)).current).toBe(1); + expect(await page.locator('#mag-driving-lyrics .mag-driving-line.is-current').evaluate((e) => getComputedStyle(e).animationName)).toBe('none'); + }); + + test('375 px wide: three lines fit the box and the page does not scroll sideways', async ({ page }) => { + await page.setViewportSize({ width: 375, height: 700 }); + await openDriving(page); + await page.evaluate(() => { const a = document.getElementById('mag-playlist-audio'); a.pause(); a.currentTime = 4; }); + await expect.poll(async () => (await state(page)).shown).toEqual([0, 1, 2]); + const box = await page.locator('#mag-driving-lyrics').boundingBox(); + expect(box.width).toBeLessThanOrEqual(375); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(375); + }); + + test('seeking back and forward lands on the sentence containing the time, also in the silent lead-in', async ({ page }) => { + await openDriving(page); + await page.evaluate(() => document.getElementById('mag-playlist-audio').pause()); + // 3.2 s is inside sentence 1's silent lead-in (it lights up at 3.55): still the sentence just spoken + for (const t of [10.5, 2, 7.5, 3.2, 3.6, 0.1, 6.54, 6.56, 11.9, 0]) { + await page.evaluate((x) => { document.getElementById('mag-playlist-audio').currentTime = x; }, t); + await expect.poll(async () => (await state(page)).current, { message: 'seek to ' + t }).toBe(expectedAt(t)); + } + }); + + for (const speed of [1.5, 2]) { + test(`playing at ${speed}x: highlight equals the sentence containing currentTime, and media time runs at the rate`, async ({ page }) => { + await openDriving(page, { speed }); + await page.evaluate(() => { const a = document.getElementById('mag-playlist-audio'); a.currentTime = 1.2; a.play(); }); + const samples = []; + const wall0 = Date.now(); + let t0 = null; + while (Date.now() - wall0 < 4500) { + const s = await state(page); + if (t0 === null) t0 = { t: s.t, wall: Date.now() }; + const nearEdge = EFF.some((e) => Math.abs(s.t - e) < 0.25); + if (!nearEdge) samples.push(s); + if (s.t > 11.5) break; + await page.waitForTimeout(60); + } + const rate = (await state(page)).t - t0.t; + expect(rate / ((Date.now() - t0.wall) / 1000)).toBeGreaterThan(speed * 0.7); // currentTime is media time + expect(samples.length).toBeGreaterThan(5); + for (const s of samples) expect(s.current, 'at ' + s.t).toBe(expectedAt(s.t)); + }); + } + + test('changing the speed mid-sentence does not move the highlight', async ({ page }) => { + await openDriving(page); + await page.evaluate(() => { const a = document.getElementById('mag-playlist-audio'); a.currentTime = 4; a.play(); }); + await expect.poll(async () => (await state(page)).current).toBe(1); + for (const v of [2, 0.75, 1.5]) { + await page.evaluate((x) => { document.getElementById('mag-playlist-audio').playbackRate = x; }, v); + const s = await state(page); + if (!EFF.some((e) => Math.abs(s.t - e) < 0.2)) expect(s.current).toBe(expectedAt(s.t)); + } + }); + + test('rapid repeated seeks settle on the final position with no stale highlight', async ({ page }) => { + await openDriving(page, { speed: 2 }); + await page.evaluate(() => document.getElementById('mag-playlist-audio').play()); + const finals = []; + for (let round = 0; round < 4; round++) { + const last = await page.evaluate(() => { + const a = document.getElementById('mag-playlist-audio'); + let x = 0; + for (let i = 0; i < 30; i++) { x = Math.random() * 11.5; a.currentTime = x; } + return x; + }); + await page.waitForTimeout(250); + const s = await state(page); + finals.push(last); + expect(s.t).toBeGreaterThan(last - 0.1); + if (!EFF.some((e) => Math.abs(s.t - e) < 0.25)) expect(s.current, 'after seek to ' + last).toBe(expectedAt(s.t)); + } + // a seek while paused is shown without any playback frame running + await page.evaluate(() => { const a = document.getElementById('mag-playlist-audio'); a.pause(); a.currentTime = 7; }); + await expect.poll(async () => (await state(page)).current).toBe(2); + }); +});