diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d3d207..b565670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi ## [Unreleased] ### Added +- Conversation Playground: Metrics view (Conversation | Metrics tabs) with Pipecat-style TTFB line charts for STT, LLM first token and TTS, plus session Token Usage (prompt/completion/total, "n/a" when the backend reports none). The stream now requests `stream_options.include_usage` (dropped and retried on 400/422) and forwards `usage` in the final stats event; the TTS value is now the time to the first audio chunk instead of including the remaining LLM generation time. - Conversation Playground: a Thinking on/off switch next to the Language Model (default off, setting `conversation_thinking`). Off sends `chat_template_kwargs.enable_thinking=false` (plus `think:false` for Ollama) with each reply request and never touches the system prompt; the empty-reply error now says to turn Thinking off. ### Fixed diff --git a/Manual.md b/Manual.md index c9caa06..a72d673 100644 --- a/Manual.md +++ b/Manual.md @@ -877,3 +877,21 @@ offline cache yet. **Offline:** works with a local LLM; no network beyond your configured endpoint. **Privacy:** the setting is stored in the local app settings only; nothing extra is sent anywhere. + +## Conversation Playground: Metrics view + +**Purpose:** see how fast each stage responds over a session, like the Pipecat Playground "Metrics" tab: Token Usage (prompt / completion / total) and a TTFB line chart each for STT, LLM (first token) and TTS (time to the first audio chunk, not the end of playback). + +**Prerequisites:** at least one completed turn in Conversation. Token counts need a backend that reports `usage` in streaming replies (Ollama, llama.cpp, vLLM and llama-swap do). + +**Usage:** switch the Conversation | Metrics tabs at the top (click, tap, or Left/Right/Home/End arrows). Each completed turn adds one point (X = time of day HH:MM:SS, Y = ms). Use "Show as table" under a chart for the numbers. The Clear button resets charts and token totals. + +**Expected result:** three charts with one point per turn (last 50 kept) and summed token counts. Voice turns include STT; typed turns have no STT point. The Latency panel in the Conversation tab is unchanged. + +**Errors:** a backend that sends no usage shows "n/a" (never 0). Failed turns add no point. With no turns yet, "No turns yet." is shown. + +**Mobile / accessibility:** charts stack on phones; 44 px tabs (tablist, aria-selected); every chart has a text summary (e.g. "LLM first token, last 142 ms, 5 points") and a table alternative; values are drawn as line plus markers, not colour only; no animation. + +**Offline:** works with a local LLM; no network beyond your configured endpoint. + +**Privacy:** metrics live only in the browser tab's memory and vanish on reload; nothing leaves the browser except the LLM call itself (which now asks the endpoint to include token usage). diff --git a/PROJECT_MAP.md b/PROJECT_MAP.md index e456569..089f74f 100644 --- a/PROJECT_MAP.md +++ b/PROJECT_MAP.md @@ -58,6 +58,7 @@ Two self-hosted Docker instances, both port 7890: | Voice library / clone / design | `routes/library.py`, `core/voice.py` | `library.js` | mature | | Voice archetype gallery (40 own German-first design voices, cached previews) | `core/voice_archetypes.py`, `routes/archetypes.py` | `voice-archetypes.js`, `s-design.html` | **new** — see [architecture/voice-archetypes.md](architecture/voice-archetypes.md) | | Read Aloud (books, PDFs) | `routes/reader.py` | `reader.js` | mature | +| Conversation Metrics view (TTFB charts STT/LLM/TTS, token usage) | `routes/conversation.py` (`_usage_from_chunk`, `_post_llm_stream`) | `conversation.js`, `s-conversation.html` | **new 2026-10-06** — in-memory, last 50 turns | | Audiobook casting (span mode: deterministic quote spans, LLM names speakers only; verify/recast) | `routes/conversation.py`, `core/attribution_spans.py` | `audiobook*.js`, `audiobook-spans.js`, `audiobook-cast-spans.js` | **new 2026-09-19** — see [architecture/audiobook-attribution.md](architecture/audiobook-attribution.md) | | Characters (avatar/concept identity, appearance forms) | `routes/characters.py`, `routes/conversation.py` | `library-characters.js`, `character-sheets.js` | **reworked 2026-09-19** — see [architecture/character-visual-identity.md](architecture/character-visual-identity.md) | | Rehearser | `routes/rehearsals_db.py` | `rehearser-*.js` | mature | diff --git a/routes/conversation.py b/routes/conversation.py index c8efee9..4650c3e 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -388,6 +388,28 @@ def _thinking_off_params(settings: dict, llm_url: str = "") -> dict: return params +def _usage_from_chunk(obj: dict) -> dict | None: + """Token counts from an OpenAI-style stream chunk's ``usage``, or None when absent/non-numeric.""" + u = obj.get("usage") + if not isinstance(u, dict): + return None + out = {k: u[k] for k in ("prompt_tokens", "completion_tokens", "total_tokens") + if isinstance(u.get(k), int) and not isinstance(u.get(k), bool)} + return out or None + + +def _post_llm_stream(url: str, payload: dict, headers: dict): + """POST a streaming chat request; on 400/422 retry once without the optional keys + (thinking switch, stream_options) that older backends reject.""" + resp = requests.post(url, json=payload, headers=headers, stream=True, timeout=120) + optional = [k for k in ("chat_template_kwargs", "think", "stream_options") if k in payload] + if resp.status_code in (400, 422) and optional: + for k in optional: + payload.pop(k, None) + resp = requests.post(url, json=payload, headers=headers, stream=True, timeout=120) + return resp + + def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str: """Inline synchronous persona rewrite; raises RuntimeError on failure.""" settings = _load_settings() @@ -6238,21 +6260,16 @@ async def conversation_turn( if eff_llm_model: llm_payload["model"] = eff_llm_model llm_payload.update(_thinking_off_params(settings, eff_llm_url)) + llm_payload["stream_options"] = {"include_usage": True} # dropped on 400/422 by _post_llm_stream _loop = asyncio.get_event_loop() _token_q: asyncio.Queue[tuple[str, str] | None] = asyncio.Queue() + _llm_usage: dict = {} # filled by _llm_thread from the stream's usage chunk; {} = server sent none def _llm_thread() -> None: try: _hdrs = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"} - resp = requests.post(f"{eff_llm_url}/chat/completions", json=llm_payload, - headers=_hdrs, stream=True, timeout=120) - if resp.status_code in (400, 422) and "chat_template_kwargs" in llm_payload: - # backend rejects the thinking switch: ask again without it - llm_payload.pop("chat_template_kwargs", None) - llm_payload.pop("think", None) - resp = requests.post(f"{eff_llm_url}/chat/completions", json=llm_payload, - headers=_hdrs, stream=True, timeout=120) + resp = _post_llm_stream(f"{eff_llm_url}/chat/completions", llm_payload, _hdrs) resp.raise_for_status() for raw_line in resp.iter_lines(): if not raw_line: @@ -6265,6 +6282,9 @@ async def conversation_turn( break try: obj = json.loads(payload_str) + _u = _usage_from_chunk(obj) + if _u: + _llm_usage.update(_u) d_obj = ((obj.get("choices") or [{}])[0].get("delta") or {}) reasoning = _reasoning_text(d_obj) content = d_obj.get("content") or "" @@ -6375,6 +6395,13 @@ async def conversation_turn( tts_tasks: list[asyncio.Task] = [] tts_texts: list[str] = [] # sentence text corresponding to each task tts_first_start: float | None = None + tts_first_done: list[float] = [] # when the first chunk's audio was ready (not when we got to await it) + + def _add_tts(chunk: str) -> None: + task = _make_tts_task(chunk, tts_voice, settings, tts_be, tts_sem, tts_emotion) + if not tts_tasks: + task.add_done_callback(lambda _t: tts_first_done.append(time.monotonic())) + tts_tasks.append(task) try: while True: @@ -6405,7 +6432,7 @@ async def conversation_turn( if tts_first_start is None: tts_first_start = time.monotonic() tts_texts.append(chunk_text) - tts_tasks.append(_make_tts_task(chunk_text, tts_voice, settings, tts_be, tts_sem, tts_emotion)) + _add_tts(chunk_text) except Exception as exc: yield sse({"type": "error", "stage": "llm", "message": str(exc)}) return @@ -6424,7 +6451,7 @@ async def conversation_turn( if tts_first_start is None: tts_first_start = time.monotonic() tts_texts.append(sent_buf.strip()) - tts_tasks.append(_make_tts_task(sent_buf.strip(), tts_voice, settings, tts_be, tts_sem, tts_emotion)) + _add_tts(sent_buf.strip()) llm_total_ms = int((time.monotonic() - t_llm) * 1000) yield sse({"type": "llm_done", "text": llm_text, @@ -6448,14 +6475,16 @@ async def conversation_turn( yield sse({"type": "error", "stage": "tts", "message": str(exc)}) return if i == 0: - tts_ms = int((time.monotonic() - tts_start) * 1000) + # time to first audio chunk; the LLM may still have been streaming when it was ready + tts_ms = int(((tts_first_done[0] if tts_first_done else time.monotonic()) - tts_start) * 1000) sentence_text = tts_texts[i] if i < len(tts_texts) else "" yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), "mime": mime, "text": sentence_text}) total_ms = int((time.monotonic() - t0) * 1000) yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms, - "llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms}) + "llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms, + "usage": dict(_llm_usage) or None}) yield sse({"type": "done"}) return StreamingResponse(generate(), media_type="text/event-stream", diff --git a/static/js/conversation.js b/static/js/conversation.js index e8f1a72..471021c 100644 --- a/static/js/conversation.js +++ b/static/js/conversation.js @@ -572,6 +572,81 @@ $('s-import-voices-file')?.addEventListener('change', async function () { turnHistory.insertBefore(item, turnHistory.firstChild); } + + // ── Metrics view (TTFB charts + token usage; in-memory, last 50 turns) ──── + const METRIC_MAX = 50; + let metricTurns = []; // {ts, stt, llm, tts} (ms, null when not measured) + let metricUsage = null; // {p, c, t} summed over turns whose server reported usage + const tr = s => (window.t ? window.t(s) : s); + const pad2 = n => String(n).padStart(2, '0'); + const hms = ts => { const d = new Date(ts); return `${pad2(d.getHours())}:${pad2(d.getMinutes())}:${pad2(d.getSeconds())}`; }; + + function recordMetrics(st) { + if (!st) return; + metricTurns.push({ ts: Date.now(), stt: st.stt_ms ?? null, llm: st.llm_ttft_ms ?? null, tts: st.tts_ms ?? null }); + if (metricTurns.length > METRIC_MAX) metricTurns.shift(); + const u = st.usage; + if (u && typeof u === 'object') { + metricUsage = metricUsage || { p: 0, c: 0, t: 0 }; + metricUsage.p += u.prompt_tokens || 0; + metricUsage.c += u.completion_tokens || 0; + metricUsage.t += u.total_tokens ?? ((u.prompt_tokens || 0) + (u.completion_tokens || 0)); + } + renderMetrics(); + } + + function chartSvg(pts, label) { + const W = 300, H = 140, L = 40, R = 10, T = 8, B = 22; + const max = Math.max(...pts.map(p => p.v), 1); + const t0 = pts[0].ts, span = pts[pts.length - 1].ts - t0; + const x = ts => L + (span ? (ts - t0) / span : 0.5) * (W - L - R); + const y = v => T + (1 - v / max) * (H - T - B); + const line = pts.map(p => `${x(p.ts).toFixed(1)},${y(p.v).toFixed(1)}`).join(' '); + const dots = pts.map(p => ``).join(''); + return ` + + + ${max} + 0 + ${hms(t0)} + ${span ? `${hms(pts[pts.length - 1].ts)}` : ''} + ${dots}`; + } + + function renderMetrics() { + const wrap = $('cm-charts'); if (!wrap) return; + $('cm-empty').hidden = metricTurns.length > 0; + const tok = (id, v) => { const el = $(id); if (el) el.textContent = v == null ? tr('n/a') : String(v); }; + tok('cm-tok-prompt', metricUsage?.p); tok('cm-tok-completion', metricUsage?.c); tok('cm-tok-total', metricUsage?.t); + if (!metricTurns.length) { wrap.innerHTML = ''; return; } + wrap.innerHTML = [['stt', 'STT', 'STT'], ['llm', 'LLM', 'LLM first token'], ['tts', 'TTS', 'TTS first audio']].map(([k, title, what]) => { + const pts = metricTurns.filter(m => m[k] != null).map(m => ({ ts: m.ts, v: m[k] })); + const summary = pts.length ? `${tr(what)}, ${tr('last')} ${pts[pts.length - 1].v} ms, ${pts.length} ${tr('points')}` : `${tr(what)}: ${tr('no data')}`; + const rows = pts.map(p => `${hms(p.ts)}${p.v} ms`).join(''); + return `

${title}

+ ${pts.length ? chartSvg(pts, summary) : ''}

${summary}

+
${tr('Show as table')} + ${rows}
${summary}
${tr('Time')}TTFB
`; + }).join(''); + } + + (function initMetricTabs() { + const tabs = [$('conv-tab-chat'), $('conv-tab-metrics')].filter(Boolean); + const panels = [$('conv-main-view'), $('conv-metrics-view')]; + if (tabs.length < 2) return; + function select(i, focus) { + tabs.forEach((tb, j) => { tb.setAttribute('aria-selected', String(i === j)); tb.tabIndex = i === j ? 0 : -1; if (panels[j]) panels[j].hidden = i !== j; }); + if (focus) tabs[i].focus(); + } + tabs.forEach((tb, i) => { + tb.addEventListener('click', () => select(i)); + tb.addEventListener('keydown', e => { + const k = { ArrowRight: (i + 1) % 2, ArrowLeft: (i + 1) % 2, Home: 0, End: 1 }[e.key]; + if (k != null) { e.preventDefault(); select(k, true); } + }); + }); + })(); + // ── Recording ──────────────────────────────────────────────────────────── function startRecTimer() { recStart = Date.now(); @@ -1100,6 +1175,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () { conversationHistory.push({ role: 'user', content: userBubble.textContent }); conversationHistory.push({ role: 'assistant', content: assistantText }); addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true); + recordMetrics(lastStats); scheduleAutoMic(); // triggers after audio queue drains } else if (evt.type === 'error') { clearAudio(); @@ -1224,6 +1300,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () { conversationHistory.push({ role: 'user', content: text }); conversationHistory.push({ role: 'assistant', content: assistantText }); addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true); + recordMetrics(lastStats); scheduleAutoMic(); // triggers after audio queue drains } else if (evt.type === 'error') { clearAudio(); @@ -1305,6 +1382,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () { if (turnHistory) turnHistory.innerHTML = '
No turns yet.
'; ['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; }); ['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; }); + metricTurns = []; metricUsage = null; renderMetrics(); }); llmFetchBtn?.addEventListener('click', fetchLlmModels); diff --git a/static/js/i18n.js b/static/js/i18n.js index 07474e3..db42306 100644 --- a/static/js/i18n.js +++ b/static/js/i18n.js @@ -94,6 +94,12 @@ const I18N_DICT = { 'Thinking': 'Denken', 'on': 'an', 'off': 'aus', 'Off = faster answers. On = better reasoning but slower, and some models may return an empty reply.': 'Aus = schnellere Antworten. An = besseres Schlussfolgern, aber langsamer; manche Modelle liefern eine leere Antwort.', + // Conversation metrics view + 'Metrics': 'Metriken', 'Conversation views': 'Unterhaltungsansichten', + 'Token Usage': 'Token-Verbrauch', 'TTFB Metrics': 'TTFB-Metriken', + 'Prompt tokens': 'Prompt-Token', 'Completion tokens': 'Completion-Token', 'Total tokens': 'Token gesamt', + 'n/a': 'k. A.', 'LLM first token': 'LLM erstes Token', 'TTS first audio': 'TTS erstes Audio', + 'last': 'zuletzt', 'points': 'Punkte', 'no data': 'keine Daten', 'Show as table': 'Als Tabelle anzeigen', 'Time': 'Zeit', // Inline text tags 'Inline text tags': 'Inline-Texttags', 'Off by default. When on, tags shape the speech in Try It Out, Read Aloud, the speech API and audiobooks; streaming and magazine audio only remove them.': 'Standardmäßig aus. Wenn aktiv, steuern Tags die Sprache in „Ausprobieren“, „Vorlesen“, der Sprach-API und Hörbüchern; Streaming und Magazin-Audio entfernen sie nur.', diff --git a/static/sections/s-conversation.html b/static/sections/s-conversation.html index d3aba3b..b3f5bee 100644 --- a/static/sections/s-conversation.html +++ b/static/sections/s-conversation.html @@ -6,6 +6,11 @@ +
+ + +
+

Engines & prompt

@@ -53,7 +58,7 @@
-
+
@@ -148,3 +153,16 @@
+ + + diff --git a/static/style.css b/static/style.css index b251807..4c15937 100644 --- a/static/style.css +++ b/static/style.css @@ -6040,3 +6040,28 @@ html[data-mag-night] #toast.success { color: var(--green); } html[data-mag-night #conv-llm-fetch-btn { min-width:44px; } .conv-llm-hint { flex-basis:100%; margin:0; font-size:12px; } @media (max-width: 640px) { #conv-llm-endpoint, #conv-llm-url { flex-basis:100%; width:100% !important; } } + +/* Conversation: Conversation | Metrics tabs and the metrics view */ +.conv-tabs { display:flex; gap:4px; margin:0 0 12px; border-bottom:1px solid var(--border); } +.conv-tab { min-height:44px; min-width:44px; padding:0 16px; background:none; border:0; border-bottom:3px solid transparent; color:var(--subtext); font:inherit; font-weight:600; cursor:pointer; } +.conv-tab[aria-selected="true"] { color:var(--text); border-bottom-color:var(--accent); } +.conv-tab:focus-visible { outline:2px solid var(--accent); outline-offset:-2px; } +.conv-main[hidden], .conv-metrics[hidden] { display:none; } +.conv-metrics { margin-top:14px; } +.conv-metrics-h { margin:14px 0 8px; font-size:14px; } +.conv-tokens { display:grid; grid-template-columns:repeat(3, 1fr); gap:10px; } +.conv-token { background:var(--surface); border:1px solid var(--border); border-radius:var(--radius); padding:12px; display:flex; flex-direction:column; gap:4px; } +.conv-token-l { font-size:12px; color:var(--subtext); } +.conv-token strong { font-size:22px; font-variant-numeric:tabular-nums; } +.conv-charts { display:grid; grid-template-columns:repeat(3, 1fr); gap:10px; } +.conv-chart { background:var(--surface); border:1px solid var(--border); border-radius:var(--radius); padding:12px; min-width:0; } +.conv-chart h4 { margin:0 0 6px; font-size:13px; } +.conv-chart svg { width:100%; height:auto; display:block; } +.conv-chart .cm-axis { stroke:var(--border); } +.conv-chart .cm-line { stroke:var(--accent); fill:none; stroke-width:2; } +.conv-chart .cm-dot { fill:var(--surface); stroke:var(--accent); stroke-width:2; } +.conv-chart text { fill:var(--subtext); font-size:10px; } +.conv-chart details summary { cursor:pointer; min-height:44px; display:flex; align-items:center; font-size:12px; color:var(--subtext); } +.conv-chart table { width:100%; font-size:12px; border-collapse:collapse; font-variant-numeric:tabular-nums; } +.conv-chart th, .conv-chart td { text-align:left; padding:2px 4px; border-bottom:1px solid var(--border); } +@media (max-width: 767px) { .conv-tokens, .conv-charts { grid-template-columns:1fr; } } diff --git a/tests/conversation-metrics.spec.js b/tests/conversation-metrics.spec.js new file mode 100644 index 0000000..18e3a61 --- /dev/null +++ b/tests/conversation-metrics.spec.js @@ -0,0 +1,84 @@ +// @ts-check +// Conversation Playground Metrics view (all /api routes mocked; no live writes). +const { test, expect } = require('./_guard'); + +const sse = (/** @type {any[]} */ evts) => evts.map(e => `data: ${JSON.stringify(e)}\n\n`).join(''); + +async function boot(page, { usage = true, lang = '' } = {}) { + let n = 0; + await page.route('**/api/**', route => route.request().method() === 'GET' ? route.fulfill({ json: {} }) : route.fulfill({ json: { ok: true } })); + await page.route('**/api/conversation/turn', route => { + n++; + const stats = { type: 'stats', stt_ms: 100 + n, llm_ttft_ms: 200 + n * 10, llm_total_ms: 900, tts_ms: 300 + n, total_ms: 1500, + usage: usage ? { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } : null }; + return route.fulfill({ contentType: 'text/event-stream', body: sse([ + { type: 'transcript', text: 'hi', stt_ms: null }, { type: 'token', delta: 'Hello.' }, + { type: 'llm_done', text: 'Hello.', llm_ttft_ms: 1, llm_total_ms: 2 }, stats, { type: 'done' }]) }); + }); + await page.route('**/static/dist/main.min.js*', r => r.abort()); + if (lang) await page.addInitScript(l => { try { localStorage.setItem('app-lang', l); } catch (e) {} }, lang); + await page.goto('/', { waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => typeof window.navTo === 'function', null, { timeout: 15_000 }); + await page.waitForSelector('#app-skeleton', { state: 'detached', timeout: 15_000 }); + await page.evaluate(() => window.navTo('s-conversation')); +} + +async function turn(page, i) { + await page.locator('#conv-text-input').fill('hello ' + i); + await page.locator('#conv-text-input').press('Enter'); + await expect(page.locator('#conv-turn-history .conv-hist-item')).toHaveCount(i); + await expect(page.locator('#conv-send-btn')).toBeEnabled(); +} + +test('3 turns -> 3 charts with 3 points, token sums, latency panel intact', async ({ page }) => { + await boot(page); + for (let i = 1; i <= 3; i++) await turn(page, i); + // existing Latency panel unchanged + for (const id of ['cpv-stt', 'cpv-ttft', 'cpv-llm', 'cpv-tts', 'cpv-total']) await expect(page.locator('#' + id)).not.toHaveText('—'); + await expect(page.locator('#conv-stats-panel .conv-stats-head', { hasText: 'Turn history' })).toBeVisible(); + + await page.getByRole('tab', { name: 'Metrics' }).click(); + await expect(page.locator('#conv-metrics-view')).toBeVisible(); + await expect(page.locator('#conv-main-view')).toBeHidden(); + await expect(page.locator('.conv-chart')).toHaveCount(3); + for (const k of ['stt', 'llm', 'tts']) { + await expect(page.locator(`.conv-chart[data-metric="${k}"] circle`)).toHaveCount(3); + await expect(page.locator(`.conv-chart[data-metric="${k}"] tbody tr`)).toHaveCount(3); + } + await expect(page.locator('.conv-chart[data-metric="llm"] svg')).toHaveAttribute('aria-label', 'LLM first token, last 230 ms, 3 points'); + await expect(page.locator('.conv-chart[data-metric="llm"] svg text').first()).toBeAttached(); + await expect(page.locator('#cm-tok-prompt')).toHaveText('30'); + await expect(page.locator('#cm-tok-completion')).toHaveText('15'); + await expect(page.locator('#cm-tok-total')).toHaveText('45'); + expect(await page.locator('.conv-chart[data-metric="stt"] tbody td').first().textContent()).toMatch(/^\d{2}:\d{2}:\d{2}$/); + + // Clear resets + await page.locator('#conv-clear-btn').click(); + await expect(page.locator('.conv-chart')).toHaveCount(0); + await expect(page.locator('#cm-empty')).toBeVisible(); + await expect(page.locator('#cm-tok-total')).toHaveText('n/a'); +}); + +test('no usage from the server shows n/a, never 0', async ({ page }) => { + await boot(page, { usage: false }); + await turn(page, 1); + await page.getByRole('tab', { name: 'Metrics' }).click(); + for (const id of ['prompt', 'completion', 'total']) await expect(page.locator('#cm-tok-' + id)).toHaveText('n/a'); + await expect(page.locator('.conv-chart[data-metric="llm"] circle')).toHaveCount(1); +}); + +test('empty state, keyboard toggle, 44px targets, German', async ({ page }) => { + await boot(page, { lang: 'de' }); + const chat = page.locator('#conv-tab-chat'), met = page.locator('#conv-tab-metrics'); + await expect(chat).toHaveText('Unterhaltung'); + expect((await met.boundingBox())?.height).toBeGreaterThanOrEqual(44); + await chat.focus(); + await page.keyboard.press('ArrowRight'); + await expect(met).toHaveAttribute('aria-selected', 'true'); + await expect(met).toBeFocused(); + await expect(page.locator('#cm-empty')).toBeVisible(); + await expect(page.locator('#cm-tok-total')).toHaveText('k. A.'); + await page.keyboard.press('ArrowLeft'); + await expect(chat).toHaveAttribute('aria-selected', 'true'); + await expect(page.locator('#conv-main-view')).toBeVisible(); +}); diff --git a/tests/test_conversation_metrics.py b/tests/test_conversation_metrics.py new file mode 100644 index 0000000..29ec831 --- /dev/null +++ b/tests/test_conversation_metrics.py @@ -0,0 +1,111 @@ +"""Metrics: usage forwarding from the LLM stream and first-audio-chunk (TTS TTFB) timing.""" +from __future__ import annotations + +import json +import os +import tempfile +import time +import unittest +from unittest import mock + +os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-metrics-test-config-")) + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +import routes.conversation as conversation + + +class _Resp: + def __init__(self, status=200, lines=()): + self.status_code, self._lines = status, lines + + def raise_for_status(self): + pass + + def iter_lines(self): + return iter(self._lines) + + +def _chunk(content=None, usage=None): + o = {"choices": [{"delta": {"content": content}}] if content is not None else []} + if usage is not None: + o["usage"] = usage + return b"data: " + json.dumps(o).encode() + + +class UsageParsing(unittest.TestCase): + def test_numbers_only(self): + self.assertEqual(conversation._usage_from_chunk({"usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}}), + {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}) + + def test_absent_or_junk_is_none(self): + for o in ({}, {"usage": None}, {"usage": {}}, {"usage": {"prompt_tokens": "x", "total_tokens": True}}): + self.assertIsNone(conversation._usage_from_chunk(o)) + + +class RetryWithoutOptionalKeys(unittest.TestCase): + def test_400_drops_stream_options_and_thinking(self): + sent = [] + resps = [_Resp(400), _Resp(200)] + + def fake(u, json=None, **kw): + sent.append(dict(json)) + return resps.pop(0) + + payload = {"stream": True, "stream_options": {"include_usage": True}, "chat_template_kwargs": {}, "think": False} + with mock.patch.object(conversation.requests, "post", side_effect=fake): + r = conversation._post_llm_stream("http://h/v1/chat/completions", payload, {}) + self.assertEqual(r.status_code, 200) + self.assertIn("stream_options", sent[0]) + self.assertEqual(sent[1], {"stream": True}) + + def test_no_retry_when_nothing_optional(self): + with mock.patch.object(conversation.requests, "post", return_value=_Resp(400)) as p: + self.assertEqual(conversation._post_llm_stream("u", {"stream": True}, {}).status_code, 400) + self.assertEqual(p.call_count, 1) + + +class TurnStream(unittest.TestCase): + def _turn(self, lines_fn, tts_delay=0.0): + app = FastAPI() + app.include_router(conversation.router) + sent = [] + + def fake_post(u, json=None, **kw): + sent.append(json) + return _Resp(lines=lines_fn()) + + def fake_tts(*a, **k): + time.sleep(tts_delay) + return b"RIFF", "audio/wav" + + with mock.patch.object(conversation, "_load_settings", return_value={"llm_url": "http://h:17861/v1"}), \ + mock.patch.object(conversation.requests, "post", side_effect=fake_post), \ + mock.patch.object(conversation, "_preview_request_audio", side_effect=fake_tts): + r = TestClient(app).post("/api/conversation/turn", data={"text": "Tell me about the sea please"}) + evts = [json.loads(l[5:]) for l in r.text.splitlines() if l.startswith("data:")] + return sent, next(e for e in evts if e["type"] == "stats") + + def test_usage_forwarded_and_requested(self): + sent, stats = self._turn(lambda: [_chunk("The sea is wide and deep. "), _chunk(usage={"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}), b"data: [DONE]"]) + self.assertEqual(sent[0]["stream_options"], {"include_usage": True}) + self.assertEqual(stats["usage"], {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}) + + def test_no_usage_is_null_not_zero(self): + _, stats = self._turn(lambda: [_chunk("The sea is wide and deep. "), b"data: [DONE]"]) + self.assertIsNone(stats["usage"]) + + def test_tts_ms_is_first_chunk_not_llm_duration(self): + def lines(): + yield _chunk("The sea is wide and deep. ") + time.sleep(0.6) # LLM keeps streaming long after the first sentence's audio is ready + yield _chunk("It is also very blue today. ") + yield b"data: [DONE]" + _, stats = self._turn(lines) + self.assertGreaterEqual(stats["llm_total_ms"], 600) + self.assertLess(stats["tts_ms"], 400) + + +if __name__ == "__main__": + unittest.main()