From 4ab345d1c122bddfb259d77f32127078ffe58cd3 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Sat, 19 Sep 2026 21:08:01 +0200 Subject: [PATCH] fix(audiobook): warm the model before span cast/verify; keep tests off the real LLM A cold-booting model server would otherwise time out the first passage into tag guessing. Documents that the custom attribution prompt applies to the classic engine only. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 2 ++ Manual.md | 2 ++ architecture/audiobook-attribution.md | 4 ++++ static/js/audiobook-cast-spans.js | 16 ++++++++++++++++ tests/audiobook-spans.spec.js | 14 ++++++++++++++ 5 files changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0737e39..2e3dd87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,11 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi - **Verify / recast for span-mode casts** (`audiobookVerifySpans`): the app's Recast unknown, 2nd Quality Run and auto-repair entry points now run span-natively for a span cast instead of the classic passes that re-emit text. **Verify is advisory**: one extra differently-windowed pass (~10 min per novel) that never changes a speaker — agreeing lines are marked `agreed`, disagreeing lines keep the first answer, are marked `disputed` and record the other pass's answer as `altSpeaker`; the confidence score reflects both. **Recast unknown** re-asks only the Unknown lines with reasoning and fuller context (`attribute_only` on `/api/attribute-spans`) and leaves any line the model cannot place Unknown. Segments now carry a `spanId`. ### Fixed +- Span cast and verify now send the model **warm-up request** first (as the classic engine does), so a cold-booting model server no longer times out the first passage into tag guessing. - The cast view's **2.4 Verify all characters** now reaches the advisory second pass for span casts (it previously passed only `repairNarration`, which would have run the Unknown-only recast); its tooltip describes what it does for a span cast. ### Changed +- The custom attribution prompt (Settings) applies to the classic engine only; span mode uses fixed instructions (documented). - Auto-repair no longer runs after a span-mode cast: Unknown lines stay visible for review instead of being guessed at. - **Span-cast confidence is calibrated on agreement between the two passes** instead of the old heuristics: agreed 82 % (Likely), disputed 48 % (Doubtful), Unknown 5 % (Guess), not yet verified 76 % (Probably) — pooled over three novels (~54 000 spoken words, 2 485 lines with a second opinion). The old bands claimed 16 % for lines that were 62 % right and the tag/role-label signals did not discriminate in span casts. Classic casts keep the old model. Running Verify therefore shrinks the "below 80 %" review queue to the disputed and Unknown lines. diff --git a/Manual.md b/Manual.md index a2e54a1..e9da4a6 100644 --- a/Manual.md +++ b/Manual.md @@ -55,6 +55,8 @@ character merge in the cast view for those. - A book whose speech is not marked with quotation marks (»…«, „…“, “…”, «…») uses the previous whole-passage method automatically. To force the previous method in this browser: `localStorage['ab-span-mode'] = '0'`. +- The custom attribution prompt in Settings only applies to the older whole-passage method; span + mode uses its own fixed instructions (it only asks who speaks each line). - Book text goes to the LLM you configured, exactly as before; nothing else is sent anywhere. Nothing works offline without a reachable LLM. - Accuracy: measured against hand-corrected casts of three novels, roughly 90–92 % of diff --git a/architecture/audiobook-attribution.md b/architecture/audiobook-attribution.md index ca96f6b..ef91a8c 100644 --- a/architecture/audiobook-attribution.md +++ b/architecture/audiobook-attribution.md @@ -62,6 +62,10 @@ accuracy gain on the same windows. An answer that parses to nothing is retried o larger token budget; lines the model skipped are retried once on their own. Calls are serialised with every other attribution call. +A model warm-up request (`/api/attribute-dialogue`, "Wake up.") runs first in both the cast and the verify, as in the +classic engine, so a cold-booting model server does not time out the first passage. The custom attribution +prompt (Settings) is part of the classic engine only; the span prompt is fixed in `core/attribution_spans.py`. + Naming rule: the model answers with the label the text uses ("Der Fremde", "Baronin", "Oberst") when a person is only referred to by role in the scene, instead of guessing another known character's name. Role → name resolution then happens book-wide in the existing diff --git a/static/js/audiobook-cast-spans.js b/static/js/audiobook-cast-spans.js index 6606fd7..df16cab 100644 --- a/static/js/audiobook-cast-spans.js +++ b/static/js/audiobook-cast-spans.js @@ -23,6 +23,19 @@ function _abPageAt(marks, offset) { return page; } +// A cold model server can take minutes to load the model; without this the first real request would +// time out and its passage fall back to tag guessing. Best effort: only a user abort propagates. +async function _abWarmUp(view, llm, language, signal) { + view.processing('Waking up LLM model (this may take a few minutes if cold-booting)…'); + try { + await audiobookFetchWithTimeout('/api/attribute-dialogue', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, signal, + body: JSON.stringify({ text: 'Wake up.', known_characters: [], recent: '', language, llm_url: llm.url, model: llm.model, + timeout_seconds: audiobookTimeoutSeconds(AUDIOBOOK_WARMUP_TIMEOUT_MS) }), + }, AUDIOBOOK_WARMUP_TIMEOUT_MS + 5000); + } catch (err) { if (err.name === 'AbortError') throw err; } +} + function _abWindowRequest(win, next, roster, recent, language, llm) { const following = (next ? next.spans.slice(0, 2) : []).map(s => s.llmText.slice(0, 300)).join(' '); return { @@ -74,6 +87,7 @@ async function audiobookCastSpans(overrideUrl, overrideModel, resume) { _abStartDraftAutosave(() => { if (allSegments.length) _abSaveDraft(allSegments, roster, text, completed, windows.length); }); try { + await _abWarmUp(view, llm, language, ac.signal); for (let i = startIndex; i < windows.length; i++) { if (_audiobook.cancel) break; const win = windows[i]; @@ -245,6 +259,7 @@ async function audiobookVerifySpans(overrideUrl, overrideModel, options = {}) { const asWindow = (lo, hi) => spans.slice(lo, hi).map(sp => ({ id: idOf(sp), kind: sp.kind, text: sp.text.length > 1500 ? sp.text.slice(0, 900) + ' [...] ' + sp.text.slice(-600) : sp.text })); try { + await _abWarmUp(view, llm, language, ac.signal); if (unknownOnly) { const pre = audiobookResolveUnknowns(segs, [], _audiobook.roster || []); if (pre.length) toast(`${pre.length} Unknown line${pre.length !== 1 ? 's' : ''} resolved by grammar rules`, 'success'); @@ -330,4 +345,5 @@ root.audiobookCastSpans = audiobookCastSpans; root.TTVC = root.TTVC || {}; root.TTVC.audiobook = root.TTVC.audiobook || {}; root.TTVC.audiobook.castSpans = audiobookCastSpans; +root.TTVC.audiobook.verifySpans = audiobookVerifySpans; })(window); diff --git a/tests/audiobook-spans.spec.js b/tests/audiobook-spans.spec.js index 4a48dff..9e4967b 100644 --- a/tests/audiobook-spans.spec.js +++ b/tests/audiobook-spans.spec.js @@ -54,6 +54,8 @@ async function seedCast(page, firstPass, book = BOOK) { async function mockSpanEndpoint(page, plan) { const calls = []; + // the warm-up request must never reach the real model server + await page.route('**/api/attribute-dialogue', route => route.fulfill({ json: { segments: [] } })); await page.route('**/api/attribute-spans', route => { const body = route.request().postDataJSON(); calls.push({ reasoning: !!body.want_reasoning, only: body.attribute_only || null, ids: body.spans.filter(s => s.kind === 'dialogue').map(s => s.id) }); @@ -124,6 +126,7 @@ test('verify leaves the cast untouched when the endpoint fails, and declines a c await openReader(page); await seedCast(page, { 'Na los!': 'Linosch', 'Vielen Dank!': 'Alrik' }); const before = await page.evaluate(() => JSON.stringify(window._audiobook.segments.map(s => [s.spanId, s.speaker]))); + await page.route('**/api/attribute-dialogue', route => route.fulfill({ json: { segments: [] } })); await page.route('**/api/attribute-spans', route => route.fulfill({ status: 503, json: { detail: 'down' } })); await page.evaluate(() => window.audiobookVerifySpans(undefined, undefined, {})); const after = await page.evaluate(() => JSON.stringify(window._audiobook.segments.map(s => [s.spanId, s.speaker]))); @@ -161,3 +164,14 @@ test('the "Verify all characters" menu action runs the advisory second pass, not const src = await page.evaluate(() => fetch('/static/js/audiobook-cast.js').then(r => r.text())); expect(src).toContain('repairNarration: true, verify: true'); }); + +test('verify wakes the model before the first real request', async ({ page }) => { + await openReader(page); + await seedCast(page, { 'Na los!': 'Linosch' }); + const order = []; + await page.route('**/api/attribute-dialogue', route => { order.push('warmup:' + route.request().postDataJSON().text); return route.fulfill({ json: { segments: [] } }); }); + await page.route('**/api/attribute-spans', route => { order.push('spans'); return route.fulfill({ json: { speakers: {}, missing: [], characters: [] } }); }); + await page.evaluate(() => window.audiobookVerifySpans(undefined, undefined, {})); + expect(order[0]).toBe('warmup:Wake up.'); + expect(order.filter(x => x === 'spans').length).toBeGreaterThan(0); +});