feat(conversation): metrics view with TTFB charts and token usage
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
60eb108360
commit
e7a25d80ff
@ -7,6 +7,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
- Conversation Playground: Metrics view (Conversation | Metrics tabs) with Pipecat-style TTFB line charts for STT, LLM first token and TTS, plus session Token Usage (prompt/completion/total, "n/a" when the backend reports none). The stream now requests `stream_options.include_usage` (dropped and retried on 400/422) and forwards `usage` in the final stats event; the TTS value is now the time to the first audio chunk instead of including the remaining LLM generation time.
|
||||
- Conversation Playground: a Thinking on/off switch next to the Language Model (default off, setting `conversation_thinking`). Off sends `chat_template_kwargs.enable_thinking=false` (plus `think:false` for Ollama) with each reply request and never touches the system prompt; the empty-reply error now says to turn Thinking off.
|
||||
|
||||
### Fixed
|
||||
|
||||
18
Manual.md
18
Manual.md
@ -877,3 +877,21 @@ offline cache yet.
|
||||
**Offline:** works with a local LLM; no network beyond your configured endpoint.
|
||||
|
||||
**Privacy:** the setting is stored in the local app settings only; nothing extra is sent anywhere.
|
||||
|
||||
## Conversation Playground: Metrics view
|
||||
|
||||
**Purpose:** see how fast each stage responds over a session, like the Pipecat Playground "Metrics" tab: Token Usage (prompt / completion / total) and a TTFB line chart each for STT, LLM (first token) and TTS (time to the first audio chunk, not the end of playback).
|
||||
|
||||
**Prerequisites:** at least one completed turn in Conversation. Token counts need a backend that reports `usage` in streaming replies (Ollama, llama.cpp, vLLM and llama-swap do).
|
||||
|
||||
**Usage:** switch the Conversation | Metrics tabs at the top (click, tap, or Left/Right/Home/End arrows). Each completed turn adds one point (X = time of day HH:MM:SS, Y = ms). Use "Show as table" under a chart for the numbers. The Clear button resets charts and token totals.
|
||||
|
||||
**Expected result:** three charts with one point per turn (last 50 kept) and summed token counts. Voice turns include STT; typed turns have no STT point. The Latency panel in the Conversation tab is unchanged.
|
||||
|
||||
**Errors:** a backend that sends no usage shows "n/a" (never 0). Failed turns add no point. With no turns yet, "No turns yet." is shown.
|
||||
|
||||
**Mobile / accessibility:** charts stack on phones; 44 px tabs (tablist, aria-selected); every chart has a text summary (e.g. "LLM first token, last 142 ms, 5 points") and a table alternative; values are drawn as line plus markers, not colour only; no animation.
|
||||
|
||||
**Offline:** works with a local LLM; no network beyond your configured endpoint.
|
||||
|
||||
**Privacy:** metrics live only in the browser tab's memory and vanish on reload; nothing leaves the browser except the LLM call itself (which now asks the endpoint to include token usage).
|
||||
|
||||
@ -58,6 +58,7 @@ Two self-hosted Docker instances, both port 7890:
|
||||
| Voice library / clone / design | `routes/library.py`, `core/voice.py` | `library.js` | mature |
|
||||
| Voice archetype gallery (40 own German-first design voices, cached previews) | `core/voice_archetypes.py`, `routes/archetypes.py` | `voice-archetypes.js`, `s-design.html` | **new** — see [architecture/voice-archetypes.md](architecture/voice-archetypes.md) |
|
||||
| Read Aloud (books, PDFs) | `routes/reader.py` | `reader.js` | mature |
|
||||
| Conversation Metrics view (TTFB charts STT/LLM/TTS, token usage) | `routes/conversation.py` (`_usage_from_chunk`, `_post_llm_stream`) | `conversation.js`, `s-conversation.html` | **new 2026-10-06** — in-memory, last 50 turns |
|
||||
| Audiobook casting (span mode: deterministic quote spans, LLM names speakers only; verify/recast) | `routes/conversation.py`, `core/attribution_spans.py` | `audiobook*.js`, `audiobook-spans.js`, `audiobook-cast-spans.js` | **new 2026-09-19** — see [architecture/audiobook-attribution.md](architecture/audiobook-attribution.md) |
|
||||
| Characters (avatar/concept identity, appearance forms) | `routes/characters.py`, `routes/conversation.py` | `library-characters.js`, `character-sheets.js` | **reworked 2026-09-19** — see [architecture/character-visual-identity.md](architecture/character-visual-identity.md) |
|
||||
| Rehearser | `routes/rehearsals_db.py` | `rehearser-*.js` | mature |
|
||||
|
||||
@ -388,6 +388,28 @@ def _thinking_off_params(settings: dict, llm_url: str = "") -> dict:
|
||||
return params
|
||||
|
||||
|
||||
def _usage_from_chunk(obj: dict) -> dict | None:
|
||||
"""Token counts from an OpenAI-style stream chunk's ``usage``, or None when absent/non-numeric."""
|
||||
u = obj.get("usage")
|
||||
if not isinstance(u, dict):
|
||||
return None
|
||||
out = {k: u[k] for k in ("prompt_tokens", "completion_tokens", "total_tokens")
|
||||
if isinstance(u.get(k), int) and not isinstance(u.get(k), bool)}
|
||||
return out or None
|
||||
|
||||
|
||||
def _post_llm_stream(url: str, payload: dict, headers: dict):
|
||||
"""POST a streaming chat request; on 400/422 retry once without the optional keys
|
||||
(thinking switch, stream_options) that older backends reject."""
|
||||
resp = requests.post(url, json=payload, headers=headers, stream=True, timeout=120)
|
||||
optional = [k for k in ("chat_template_kwargs", "think", "stream_options") if k in payload]
|
||||
if resp.status_code in (400, 422) and optional:
|
||||
for k in optional:
|
||||
payload.pop(k, None)
|
||||
resp = requests.post(url, json=payload, headers=headers, stream=True, timeout=120)
|
||||
return resp
|
||||
|
||||
|
||||
def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
|
||||
"""Inline synchronous persona rewrite; raises RuntimeError on failure."""
|
||||
settings = _load_settings()
|
||||
@ -6238,21 +6260,16 @@ async def conversation_turn(
|
||||
if eff_llm_model:
|
||||
llm_payload["model"] = eff_llm_model
|
||||
llm_payload.update(_thinking_off_params(settings, eff_llm_url))
|
||||
llm_payload["stream_options"] = {"include_usage": True} # dropped on 400/422 by _post_llm_stream
|
||||
|
||||
_loop = asyncio.get_event_loop()
|
||||
_token_q: asyncio.Queue[tuple[str, str] | None] = asyncio.Queue()
|
||||
_llm_usage: dict = {} # filled by _llm_thread from the stream's usage chunk; {} = server sent none
|
||||
|
||||
def _llm_thread() -> None:
|
||||
try:
|
||||
_hdrs = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
|
||||
resp = requests.post(f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||||
headers=_hdrs, stream=True, timeout=120)
|
||||
if resp.status_code in (400, 422) and "chat_template_kwargs" in llm_payload:
|
||||
# backend rejects the thinking switch: ask again without it
|
||||
llm_payload.pop("chat_template_kwargs", None)
|
||||
llm_payload.pop("think", None)
|
||||
resp = requests.post(f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||||
headers=_hdrs, stream=True, timeout=120)
|
||||
resp = _post_llm_stream(f"{eff_llm_url}/chat/completions", llm_payload, _hdrs)
|
||||
resp.raise_for_status()
|
||||
for raw_line in resp.iter_lines():
|
||||
if not raw_line:
|
||||
@ -6265,6 +6282,9 @@ async def conversation_turn(
|
||||
break
|
||||
try:
|
||||
obj = json.loads(payload_str)
|
||||
_u = _usage_from_chunk(obj)
|
||||
if _u:
|
||||
_llm_usage.update(_u)
|
||||
d_obj = ((obj.get("choices") or [{}])[0].get("delta") or {})
|
||||
reasoning = _reasoning_text(d_obj)
|
||||
content = d_obj.get("content") or ""
|
||||
@ -6375,6 +6395,13 @@ async def conversation_turn(
|
||||
tts_tasks: list[asyncio.Task] = []
|
||||
tts_texts: list[str] = [] # sentence text corresponding to each task
|
||||
tts_first_start: float | None = None
|
||||
tts_first_done: list[float] = [] # when the first chunk's audio was ready (not when we got to await it)
|
||||
|
||||
def _add_tts(chunk: str) -> None:
|
||||
task = _make_tts_task(chunk, tts_voice, settings, tts_be, tts_sem, tts_emotion)
|
||||
if not tts_tasks:
|
||||
task.add_done_callback(lambda _t: tts_first_done.append(time.monotonic()))
|
||||
tts_tasks.append(task)
|
||||
|
||||
try:
|
||||
while True:
|
||||
@ -6405,7 +6432,7 @@ async def conversation_turn(
|
||||
if tts_first_start is None:
|
||||
tts_first_start = time.monotonic()
|
||||
tts_texts.append(chunk_text)
|
||||
tts_tasks.append(_make_tts_task(chunk_text, tts_voice, settings, tts_be, tts_sem, tts_emotion))
|
||||
_add_tts(chunk_text)
|
||||
except Exception as exc:
|
||||
yield sse({"type": "error", "stage": "llm", "message": str(exc)})
|
||||
return
|
||||
@ -6424,7 +6451,7 @@ async def conversation_turn(
|
||||
if tts_first_start is None:
|
||||
tts_first_start = time.monotonic()
|
||||
tts_texts.append(sent_buf.strip())
|
||||
tts_tasks.append(_make_tts_task(sent_buf.strip(), tts_voice, settings, tts_be, tts_sem, tts_emotion))
|
||||
_add_tts(sent_buf.strip())
|
||||
|
||||
llm_total_ms = int((time.monotonic() - t_llm) * 1000)
|
||||
yield sse({"type": "llm_done", "text": llm_text,
|
||||
@ -6448,14 +6475,16 @@ async def conversation_turn(
|
||||
yield sse({"type": "error", "stage": "tts", "message": str(exc)})
|
||||
return
|
||||
if i == 0:
|
||||
tts_ms = int((time.monotonic() - tts_start) * 1000)
|
||||
# time to first audio chunk; the LLM may still have been streaming when it was ready
|
||||
tts_ms = int(((tts_first_done[0] if tts_first_done else time.monotonic()) - tts_start) * 1000)
|
||||
sentence_text = tts_texts[i] if i < len(tts_texts) else ""
|
||||
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(),
|
||||
"mime": mime, "text": sentence_text})
|
||||
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
|
||||
"llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms})
|
||||
"llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms,
|
||||
"usage": dict(_llm_usage) or None})
|
||||
yield sse({"type": "done"})
|
||||
|
||||
return StreamingResponse(generate(), media_type="text/event-stream",
|
||||
|
||||
@ -572,6 +572,81 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
turnHistory.insertBefore(item, turnHistory.firstChild);
|
||||
}
|
||||
|
||||
|
||||
// ── Metrics view (TTFB charts + token usage; in-memory, last 50 turns) ────
|
||||
const METRIC_MAX = 50;
|
||||
let metricTurns = []; // {ts, stt, llm, tts} (ms, null when not measured)
|
||||
let metricUsage = null; // {p, c, t} summed over turns whose server reported usage
|
||||
const tr = s => (window.t ? window.t(s) : s);
|
||||
const pad2 = n => String(n).padStart(2, '0');
|
||||
const hms = ts => { const d = new Date(ts); return `${pad2(d.getHours())}:${pad2(d.getMinutes())}:${pad2(d.getSeconds())}`; };
|
||||
|
||||
function recordMetrics(st) {
|
||||
if (!st) return;
|
||||
metricTurns.push({ ts: Date.now(), stt: st.stt_ms ?? null, llm: st.llm_ttft_ms ?? null, tts: st.tts_ms ?? null });
|
||||
if (metricTurns.length > METRIC_MAX) metricTurns.shift();
|
||||
const u = st.usage;
|
||||
if (u && typeof u === 'object') {
|
||||
metricUsage = metricUsage || { p: 0, c: 0, t: 0 };
|
||||
metricUsage.p += u.prompt_tokens || 0;
|
||||
metricUsage.c += u.completion_tokens || 0;
|
||||
metricUsage.t += u.total_tokens ?? ((u.prompt_tokens || 0) + (u.completion_tokens || 0));
|
||||
}
|
||||
renderMetrics();
|
||||
}
|
||||
|
||||
function chartSvg(pts, label) {
|
||||
const W = 300, H = 140, L = 40, R = 10, T = 8, B = 22;
|
||||
const max = Math.max(...pts.map(p => p.v), 1);
|
||||
const t0 = pts[0].ts, span = pts[pts.length - 1].ts - t0;
|
||||
const x = ts => L + (span ? (ts - t0) / span : 0.5) * (W - L - R);
|
||||
const y = v => T + (1 - v / max) * (H - T - B);
|
||||
const line = pts.map(p => `${x(p.ts).toFixed(1)},${y(p.v).toFixed(1)}`).join(' ');
|
||||
const dots = pts.map(p => `<circle class="cm-dot" cx="${x(p.ts).toFixed(1)}" cy="${y(p.v).toFixed(1)}" r="3"></circle>`).join('');
|
||||
return `<svg viewBox="0 0 ${W} ${H}" role="img" aria-label="${label}">
|
||||
<line class="cm-axis" x1="${L}" y1="${T}" x2="${L}" y2="${H - B}"></line>
|
||||
<line class="cm-axis" x1="${L}" y1="${H - B}" x2="${W - R}" y2="${H - B}"></line>
|
||||
<text x="${L - 4}" y="${T + 8}" text-anchor="end">${max}</text>
|
||||
<text x="${L - 4}" y="${H - B}" text-anchor="end">0</text>
|
||||
<text x="${L}" y="${H - 6}">${hms(t0)}</text>
|
||||
${span ? `<text x="${W - R}" y="${H - 6}" text-anchor="end">${hms(pts[pts.length - 1].ts)}</text>` : ''}
|
||||
<polyline class="cm-line" points="${line}"></polyline>${dots}</svg>`;
|
||||
}
|
||||
|
||||
function renderMetrics() {
|
||||
const wrap = $('cm-charts'); if (!wrap) return;
|
||||
$('cm-empty').hidden = metricTurns.length > 0;
|
||||
const tok = (id, v) => { const el = $(id); if (el) el.textContent = v == null ? tr('n/a') : String(v); };
|
||||
tok('cm-tok-prompt', metricUsage?.p); tok('cm-tok-completion', metricUsage?.c); tok('cm-tok-total', metricUsage?.t);
|
||||
if (!metricTurns.length) { wrap.innerHTML = ''; return; }
|
||||
wrap.innerHTML = [['stt', 'STT', 'STT'], ['llm', 'LLM', 'LLM first token'], ['tts', 'TTS', 'TTS first audio']].map(([k, title, what]) => {
|
||||
const pts = metricTurns.filter(m => m[k] != null).map(m => ({ ts: m.ts, v: m[k] }));
|
||||
const summary = pts.length ? `${tr(what)}, ${tr('last')} ${pts[pts.length - 1].v} ms, ${pts.length} ${tr('points')}` : `${tr(what)}: ${tr('no data')}`;
|
||||
const rows = pts.map(p => `<tr><td>${hms(p.ts)}</td><td>${p.v} ms</td></tr>`).join('');
|
||||
return `<section class="conv-chart" data-metric="${k}"><h4>${title}</h4>
|
||||
${pts.length ? chartSvg(pts, summary) : ''}<p class="sr-only">${summary}</p>
|
||||
<details><summary>${tr('Show as table')}</summary><table><caption class="sr-only">${summary}</caption>
|
||||
<thead><tr><th scope="col">${tr('Time')}</th><th scope="col">TTFB</th></tr></thead><tbody>${rows}</tbody></table></details></section>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
(function initMetricTabs() {
|
||||
const tabs = [$('conv-tab-chat'), $('conv-tab-metrics')].filter(Boolean);
|
||||
const panels = [$('conv-main-view'), $('conv-metrics-view')];
|
||||
if (tabs.length < 2) return;
|
||||
function select(i, focus) {
|
||||
tabs.forEach((tb, j) => { tb.setAttribute('aria-selected', String(i === j)); tb.tabIndex = i === j ? 0 : -1; if (panels[j]) panels[j].hidden = i !== j; });
|
||||
if (focus) tabs[i].focus();
|
||||
}
|
||||
tabs.forEach((tb, i) => {
|
||||
tb.addEventListener('click', () => select(i));
|
||||
tb.addEventListener('keydown', e => {
|
||||
const k = { ArrowRight: (i + 1) % 2, ArrowLeft: (i + 1) % 2, Home: 0, End: 1 }[e.key];
|
||||
if (k != null) { e.preventDefault(); select(k, true); }
|
||||
});
|
||||
});
|
||||
})();
|
||||
|
||||
// ── Recording ────────────────────────────────────────────────────────────
|
||||
function startRecTimer() {
|
||||
recStart = Date.now();
|
||||
@ -1100,6 +1175,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
conversationHistory.push({ role: 'user', content: userBubble.textContent });
|
||||
conversationHistory.push({ role: 'assistant', content: assistantText });
|
||||
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
|
||||
recordMetrics(lastStats);
|
||||
scheduleAutoMic(); // triggers after audio queue drains
|
||||
} else if (evt.type === 'error') {
|
||||
clearAudio();
|
||||
@ -1224,6 +1300,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
conversationHistory.push({ role: 'user', content: text });
|
||||
conversationHistory.push({ role: 'assistant', content: assistantText });
|
||||
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
|
||||
recordMetrics(lastStats);
|
||||
scheduleAutoMic(); // triggers after audio queue drains
|
||||
} else if (evt.type === 'error') {
|
||||
clearAudio();
|
||||
@ -1305,6 +1382,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
if (turnHistory) turnHistory.innerHTML = '<div class="conv-history-empty">No turns yet.</div>';
|
||||
['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; });
|
||||
['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; });
|
||||
metricTurns = []; metricUsage = null; renderMetrics();
|
||||
});
|
||||
|
||||
llmFetchBtn?.addEventListener('click', fetchLlmModels);
|
||||
|
||||
@ -94,6 +94,12 @@ const I18N_DICT = {
|
||||
'Thinking': 'Denken',
|
||||
'on': 'an', 'off': 'aus',
|
||||
'Off = faster answers. On = better reasoning but slower, and some models may return an empty reply.': 'Aus = schnellere Antworten. An = besseres Schlussfolgern, aber langsamer; manche Modelle liefern eine leere Antwort.',
|
||||
// Conversation metrics view
|
||||
'Metrics': 'Metriken', 'Conversation views': 'Unterhaltungsansichten',
|
||||
'Token Usage': 'Token-Verbrauch', 'TTFB Metrics': 'TTFB-Metriken',
|
||||
'Prompt tokens': 'Prompt-Token', 'Completion tokens': 'Completion-Token', 'Total tokens': 'Token gesamt',
|
||||
'n/a': 'k. A.', 'LLM first token': 'LLM erstes Token', 'TTS first audio': 'TTS erstes Audio',
|
||||
'last': 'zuletzt', 'points': 'Punkte', 'no data': 'keine Daten', 'Show as table': 'Als Tabelle anzeigen', 'Time': 'Zeit',
|
||||
// Inline text tags
|
||||
'Inline text tags': 'Inline-Texttags',
|
||||
'Off by default. When on, tags shape the speech in Try It Out, Read Aloud, the speech API and audiobooks; streaming and magazine audio only remove them.': 'Standardmäßig aus. Wenn aktiv, steuern Tags die Sprache in „Ausprobieren“, „Vorlesen“, der Sprach-API und Hörbüchern; Streaming und Magazin-Audio entfernen sie nur.',
|
||||
|
||||
@ -6,6 +6,11 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="conv-tabs" role="tablist" aria-label="Conversation views">
|
||||
<button type="button" role="tab" id="conv-tab-chat" class="conv-tab" aria-selected="true" aria-controls="conv-main-view" tabindex="0">Conversation</button>
|
||||
<button type="button" role="tab" id="conv-tab-metrics" class="conv-tab" aria-selected="false" aria-controls="conv-metrics-view" tabindex="-1">Metrics</button>
|
||||
</div>
|
||||
|
||||
<!-- Config bar -->
|
||||
<div class="conv-config-bar card" style="padding:14px">
|
||||
<h2><span class="mdi mdi-tune-variant"></span> Engines & prompt</h2>
|
||||
@ -53,7 +58,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Main area: chat + stats -->
|
||||
<div class="conv-main">
|
||||
<div class="conv-main" id="conv-main-view" role="tabpanel" aria-labelledby="conv-tab-chat">
|
||||
|
||||
<!-- Chat window -->
|
||||
<div class="conv-chat-panel">
|
||||
@ -148,3 +153,16 @@
|
||||
<span class="mdi mdi-chart-timeline-variant"></span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Metrics view (Pipecat-style): token usage + TTFB line charts, in-memory for this session -->
|
||||
<div class="conv-metrics" id="conv-metrics-view" role="tabpanel" aria-labelledby="conv-tab-metrics" hidden>
|
||||
<h3 class="conv-metrics-h">Token Usage</h3>
|
||||
<div class="conv-tokens">
|
||||
<div class="conv-token"><span class="conv-token-l">Prompt tokens</span><strong id="cm-tok-prompt">n/a</strong></div>
|
||||
<div class="conv-token"><span class="conv-token-l">Completion tokens</span><strong id="cm-tok-completion">n/a</strong></div>
|
||||
<div class="conv-token"><span class="conv-token-l">Total tokens</span><strong id="cm-tok-total">n/a</strong></div>
|
||||
</div>
|
||||
<h3 class="conv-metrics-h">TTFB Metrics</h3>
|
||||
<p class="conv-history-empty" id="cm-empty">No turns yet.</p>
|
||||
<div class="conv-charts" id="cm-charts"></div>
|
||||
</div>
|
||||
|
||||
@ -6040,3 +6040,28 @@ html[data-mag-night] #toast.success { color: var(--green); } html[data-mag-night
|
||||
#conv-llm-fetch-btn { min-width:44px; }
|
||||
.conv-llm-hint { flex-basis:100%; margin:0; font-size:12px; }
|
||||
@media (max-width: 640px) { #conv-llm-endpoint, #conv-llm-url { flex-basis:100%; width:100% !important; } }
|
||||
|
||||
/* Conversation: Conversation | Metrics tabs and the metrics view */
|
||||
.conv-tabs { display:flex; gap:4px; margin:0 0 12px; border-bottom:1px solid var(--border); }
|
||||
.conv-tab { min-height:44px; min-width:44px; padding:0 16px; background:none; border:0; border-bottom:3px solid transparent; color:var(--subtext); font:inherit; font-weight:600; cursor:pointer; }
|
||||
.conv-tab[aria-selected="true"] { color:var(--text); border-bottom-color:var(--accent); }
|
||||
.conv-tab:focus-visible { outline:2px solid var(--accent); outline-offset:-2px; }
|
||||
.conv-main[hidden], .conv-metrics[hidden] { display:none; }
|
||||
.conv-metrics { margin-top:14px; }
|
||||
.conv-metrics-h { margin:14px 0 8px; font-size:14px; }
|
||||
.conv-tokens { display:grid; grid-template-columns:repeat(3, 1fr); gap:10px; }
|
||||
.conv-token { background:var(--surface); border:1px solid var(--border); border-radius:var(--radius); padding:12px; display:flex; flex-direction:column; gap:4px; }
|
||||
.conv-token-l { font-size:12px; color:var(--subtext); }
|
||||
.conv-token strong { font-size:22px; font-variant-numeric:tabular-nums; }
|
||||
.conv-charts { display:grid; grid-template-columns:repeat(3, 1fr); gap:10px; }
|
||||
.conv-chart { background:var(--surface); border:1px solid var(--border); border-radius:var(--radius); padding:12px; min-width:0; }
|
||||
.conv-chart h4 { margin:0 0 6px; font-size:13px; }
|
||||
.conv-chart svg { width:100%; height:auto; display:block; }
|
||||
.conv-chart .cm-axis { stroke:var(--border); }
|
||||
.conv-chart .cm-line { stroke:var(--accent); fill:none; stroke-width:2; }
|
||||
.conv-chart .cm-dot { fill:var(--surface); stroke:var(--accent); stroke-width:2; }
|
||||
.conv-chart text { fill:var(--subtext); font-size:10px; }
|
||||
.conv-chart details summary { cursor:pointer; min-height:44px; display:flex; align-items:center; font-size:12px; color:var(--subtext); }
|
||||
.conv-chart table { width:100%; font-size:12px; border-collapse:collapse; font-variant-numeric:tabular-nums; }
|
||||
.conv-chart th, .conv-chart td { text-align:left; padding:2px 4px; border-bottom:1px solid var(--border); }
|
||||
@media (max-width: 767px) { .conv-tokens, .conv-charts { grid-template-columns:1fr; } }
|
||||
|
||||
84
tests/conversation-metrics.spec.js
Normal file
84
tests/conversation-metrics.spec.js
Normal file
@ -0,0 +1,84 @@
|
||||
// @ts-check
|
||||
// Conversation Playground Metrics view (all /api routes mocked; no live writes).
|
||||
const { test, expect } = require('./_guard');
|
||||
|
||||
const sse = (/** @type {any[]} */ evts) => evts.map(e => `data: ${JSON.stringify(e)}\n\n`).join('');
|
||||
|
||||
async function boot(page, { usage = true, lang = '' } = {}) {
|
||||
let n = 0;
|
||||
await page.route('**/api/**', route => route.request().method() === 'GET' ? route.fulfill({ json: {} }) : route.fulfill({ json: { ok: true } }));
|
||||
await page.route('**/api/conversation/turn', route => {
|
||||
n++;
|
||||
const stats = { type: 'stats', stt_ms: 100 + n, llm_ttft_ms: 200 + n * 10, llm_total_ms: 900, tts_ms: 300 + n, total_ms: 1500,
|
||||
usage: usage ? { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } : null };
|
||||
return route.fulfill({ contentType: 'text/event-stream', body: sse([
|
||||
{ type: 'transcript', text: 'hi', stt_ms: null }, { type: 'token', delta: 'Hello.' },
|
||||
{ type: 'llm_done', text: 'Hello.', llm_ttft_ms: 1, llm_total_ms: 2 }, stats, { type: 'done' }]) });
|
||||
});
|
||||
await page.route('**/static/dist/main.min.js*', r => r.abort());
|
||||
if (lang) await page.addInitScript(l => { try { localStorage.setItem('app-lang', l); } catch (e) {} }, lang);
|
||||
await page.goto('/', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(() => typeof window.navTo === 'function', null, { timeout: 15_000 });
|
||||
await page.waitForSelector('#app-skeleton', { state: 'detached', timeout: 15_000 });
|
||||
await page.evaluate(() => window.navTo('s-conversation'));
|
||||
}
|
||||
|
||||
async function turn(page, i) {
|
||||
await page.locator('#conv-text-input').fill('hello ' + i);
|
||||
await page.locator('#conv-text-input').press('Enter');
|
||||
await expect(page.locator('#conv-turn-history .conv-hist-item')).toHaveCount(i);
|
||||
await expect(page.locator('#conv-send-btn')).toBeEnabled();
|
||||
}
|
||||
|
||||
test('3 turns -> 3 charts with 3 points, token sums, latency panel intact', async ({ page }) => {
|
||||
await boot(page);
|
||||
for (let i = 1; i <= 3; i++) await turn(page, i);
|
||||
// existing Latency panel unchanged
|
||||
for (const id of ['cpv-stt', 'cpv-ttft', 'cpv-llm', 'cpv-tts', 'cpv-total']) await expect(page.locator('#' + id)).not.toHaveText('—');
|
||||
await expect(page.locator('#conv-stats-panel .conv-stats-head', { hasText: 'Turn history' })).toBeVisible();
|
||||
|
||||
await page.getByRole('tab', { name: 'Metrics' }).click();
|
||||
await expect(page.locator('#conv-metrics-view')).toBeVisible();
|
||||
await expect(page.locator('#conv-main-view')).toBeHidden();
|
||||
await expect(page.locator('.conv-chart')).toHaveCount(3);
|
||||
for (const k of ['stt', 'llm', 'tts']) {
|
||||
await expect(page.locator(`.conv-chart[data-metric="${k}"] circle`)).toHaveCount(3);
|
||||
await expect(page.locator(`.conv-chart[data-metric="${k}"] tbody tr`)).toHaveCount(3);
|
||||
}
|
||||
await expect(page.locator('.conv-chart[data-metric="llm"] svg')).toHaveAttribute('aria-label', 'LLM first token, last 230 ms, 3 points');
|
||||
await expect(page.locator('.conv-chart[data-metric="llm"] svg text').first()).toBeAttached();
|
||||
await expect(page.locator('#cm-tok-prompt')).toHaveText('30');
|
||||
await expect(page.locator('#cm-tok-completion')).toHaveText('15');
|
||||
await expect(page.locator('#cm-tok-total')).toHaveText('45');
|
||||
expect(await page.locator('.conv-chart[data-metric="stt"] tbody td').first().textContent()).toMatch(/^\d{2}:\d{2}:\d{2}$/);
|
||||
|
||||
// Clear resets
|
||||
await page.locator('#conv-clear-btn').click();
|
||||
await expect(page.locator('.conv-chart')).toHaveCount(0);
|
||||
await expect(page.locator('#cm-empty')).toBeVisible();
|
||||
await expect(page.locator('#cm-tok-total')).toHaveText('n/a');
|
||||
});
|
||||
|
||||
test('no usage from the server shows n/a, never 0', async ({ page }) => {
|
||||
await boot(page, { usage: false });
|
||||
await turn(page, 1);
|
||||
await page.getByRole('tab', { name: 'Metrics' }).click();
|
||||
for (const id of ['prompt', 'completion', 'total']) await expect(page.locator('#cm-tok-' + id)).toHaveText('n/a');
|
||||
await expect(page.locator('.conv-chart[data-metric="llm"] circle')).toHaveCount(1);
|
||||
});
|
||||
|
||||
test('empty state, keyboard toggle, 44px targets, German', async ({ page }) => {
|
||||
await boot(page, { lang: 'de' });
|
||||
const chat = page.locator('#conv-tab-chat'), met = page.locator('#conv-tab-metrics');
|
||||
await expect(chat).toHaveText('Unterhaltung');
|
||||
expect((await met.boundingBox())?.height).toBeGreaterThanOrEqual(44);
|
||||
await chat.focus();
|
||||
await page.keyboard.press('ArrowRight');
|
||||
await expect(met).toHaveAttribute('aria-selected', 'true');
|
||||
await expect(met).toBeFocused();
|
||||
await expect(page.locator('#cm-empty')).toBeVisible();
|
||||
await expect(page.locator('#cm-tok-total')).toHaveText('k. A.');
|
||||
await page.keyboard.press('ArrowLeft');
|
||||
await expect(chat).toHaveAttribute('aria-selected', 'true');
|
||||
await expect(page.locator('#conv-main-view')).toBeVisible();
|
||||
});
|
||||
111
tests/test_conversation_metrics.py
Normal file
111
tests/test_conversation_metrics.py
Normal file
@ -0,0 +1,111 @@
|
||||
"""Metrics: usage forwarding from the LLM stream and first-audio-chunk (TTS TTFB) timing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-metrics-test-config-"))
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import routes.conversation as conversation
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status=200, lines=()):
|
||||
self.status_code, self._lines = status, lines
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def iter_lines(self):
|
||||
return iter(self._lines)
|
||||
|
||||
|
||||
def _chunk(content=None, usage=None):
|
||||
o = {"choices": [{"delta": {"content": content}}] if content is not None else []}
|
||||
if usage is not None:
|
||||
o["usage"] = usage
|
||||
return b"data: " + json.dumps(o).encode()
|
||||
|
||||
|
||||
class UsageParsing(unittest.TestCase):
|
||||
def test_numbers_only(self):
|
||||
self.assertEqual(conversation._usage_from_chunk({"usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}}),
|
||||
{"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12})
|
||||
|
||||
def test_absent_or_junk_is_none(self):
|
||||
for o in ({}, {"usage": None}, {"usage": {}}, {"usage": {"prompt_tokens": "x", "total_tokens": True}}):
|
||||
self.assertIsNone(conversation._usage_from_chunk(o))
|
||||
|
||||
|
||||
class RetryWithoutOptionalKeys(unittest.TestCase):
|
||||
def test_400_drops_stream_options_and_thinking(self):
|
||||
sent = []
|
||||
resps = [_Resp(400), _Resp(200)]
|
||||
|
||||
def fake(u, json=None, **kw):
|
||||
sent.append(dict(json))
|
||||
return resps.pop(0)
|
||||
|
||||
payload = {"stream": True, "stream_options": {"include_usage": True}, "chat_template_kwargs": {}, "think": False}
|
||||
with mock.patch.object(conversation.requests, "post", side_effect=fake):
|
||||
r = conversation._post_llm_stream("http://h/v1/chat/completions", payload, {})
|
||||
self.assertEqual(r.status_code, 200)
|
||||
self.assertIn("stream_options", sent[0])
|
||||
self.assertEqual(sent[1], {"stream": True})
|
||||
|
||||
def test_no_retry_when_nothing_optional(self):
|
||||
with mock.patch.object(conversation.requests, "post", return_value=_Resp(400)) as p:
|
||||
self.assertEqual(conversation._post_llm_stream("u", {"stream": True}, {}).status_code, 400)
|
||||
self.assertEqual(p.call_count, 1)
|
||||
|
||||
|
||||
class TurnStream(unittest.TestCase):
|
||||
def _turn(self, lines_fn, tts_delay=0.0):
|
||||
app = FastAPI()
|
||||
app.include_router(conversation.router)
|
||||
sent = []
|
||||
|
||||
def fake_post(u, json=None, **kw):
|
||||
sent.append(json)
|
||||
return _Resp(lines=lines_fn())
|
||||
|
||||
def fake_tts(*a, **k):
|
||||
time.sleep(tts_delay)
|
||||
return b"RIFF", "audio/wav"
|
||||
|
||||
with mock.patch.object(conversation, "_load_settings", return_value={"llm_url": "http://h:17861/v1"}), \
|
||||
mock.patch.object(conversation.requests, "post", side_effect=fake_post), \
|
||||
mock.patch.object(conversation, "_preview_request_audio", side_effect=fake_tts):
|
||||
r = TestClient(app).post("/api/conversation/turn", data={"text": "Tell me about the sea please"})
|
||||
evts = [json.loads(l[5:]) for l in r.text.splitlines() if l.startswith("data:")]
|
||||
return sent, next(e for e in evts if e["type"] == "stats")
|
||||
|
||||
def test_usage_forwarded_and_requested(self):
|
||||
sent, stats = self._turn(lambda: [_chunk("The sea is wide and deep. "), _chunk(usage={"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}), b"data: [DONE]"])
|
||||
self.assertEqual(sent[0]["stream_options"], {"include_usage": True})
|
||||
self.assertEqual(stats["usage"], {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13})
|
||||
|
||||
def test_no_usage_is_null_not_zero(self):
|
||||
_, stats = self._turn(lambda: [_chunk("The sea is wide and deep. "), b"data: [DONE]"])
|
||||
self.assertIsNone(stats["usage"])
|
||||
|
||||
def test_tts_ms_is_first_chunk_not_llm_duration(self):
|
||||
def lines():
|
||||
yield _chunk("The sea is wide and deep. ")
|
||||
time.sleep(0.6) # LLM keeps streaming long after the first sentence's audio is ready
|
||||
yield _chunk("It is also very blue today. ")
|
||||
yield b"data: [DONE]"
|
||||
_, stats = self._turn(lines)
|
||||
self.assertGreaterEqual(stats["llm_total_ms"], 600)
|
||||
self.assertLess(stats["tts_ms"], 400)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue
Block a user