Show spoken sentence in typing bubble; fix VAD noise sensitivity

Sentence text in typing bubble:
- Backend sends 'text' field with each audio SSE event (the sentence
  being synthesised)
- Frontend audio queue stores {url, text} pairs
- playNextAudio() writes the sentence text into the '...' typing bubble
  when LLM tokens haven't arrived yet (convCurrentSentenceBubble)
- convCurrentSentenceBubble cleared as soon as first LLM token arrives
  so normal streaming takes over seamlessly

VAD noise fixes:
- VAD_THRESHOLD: 0.01 → 0.02 (background noise no longer counts as speech)
- VAD_MIN_REC_MS: 400 → 800ms (8/10s wait before silence detection starts,
  gives user time to begin speaking without initial noise triggering send)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

routes/conversation.py

@@ -817,6 +817,7 @@ async def conversation_turn(
         ttft_done = False
         sent_buf = ""
         tts_tasks: list[asyncio.Task] = []
+        tts_texts: list[str] = []   # sentence text corresponding to each task
         tts_first_start: float | None = None
 
         try:
@@ -841,6 +842,7 @@ async def conversation_turn(
                     if chunk_text:
                         if tts_first_start is None:
                             tts_first_start = time.monotonic()
+                        tts_texts.append(chunk_text)
                         tts_tasks.append(asyncio.create_task(
                             asyncio.to_thread(_preview_request_audio, chunk_text, tts_voice, settings, "", tts_be)
                         ))
@@ -852,6 +854,7 @@ async def conversation_turn(
         if sent_buf.strip():
             if tts_first_start is None:
                 tts_first_start = time.monotonic()
+            tts_texts.append(sent_buf.strip())
             tts_tasks.append(asyncio.create_task(
                 asyncio.to_thread(_preview_request_audio, sent_buf.strip(), tts_voice, settings, "", tts_be)
             ))
@@ -879,7 +882,9 @@ async def conversation_turn(
                 return
             if i == 0:
                 tts_ms = int((time.monotonic() - tts_start) * 1000)
-            yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), "mime": mime})
+            sentence_text = tts_texts[i] if i < len(tts_texts) else ""
+            yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(),
+                       "mime": mime, "text": sentence_text})
 
         total_ms = int((time.monotonic() - t0) * 1000)
         yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,

static/js/conversation.js

@@ -177,8 +177,8 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
   let vadRafId = null;
   let liveInterimText = '';
   let speechRec = null;
-  const VAD_THRESHOLD = 0.01;
+  const VAD_THRESHOLD = 0.02;   // raised to ignore background noise
-  const VAD_MIN_REC_MS = 400;
+  const VAD_MIN_REC_MS = 800;   // wait 800ms before VAD starts checking (avoids click/noise at start)
   const VAD_SILENCE_MS = 1000;
   // Whisper hallucinations on silence/noise — discard these from the live preview
   const HALLUCINATION_RE = /^(reich|danke\s*(schön)?|vielen\s*dank|thank\s*you|thanks|you|copyright|abonnieren|untertitel|zарегистрируйтесь)[.!?,\s]*$/i;
@@ -191,9 +191,10 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
   const audioQueue = [];
   let audioQueuePlaying = false;
   let audioQueueDrainCb = null;
-  let vadHadSpeech = false;    // true once RMS crossed threshold during this recording
+  let vadHadSpeech = false;         // true once RMS crossed threshold during this recording
-  let vadLastVoiceMs = 0;      // last timestamp speech was detected (for preview gate)
+  let vadLastVoiceMs = 0;           // last timestamp speech was detected (for preview gate)
-  let cancelNextBlob = false;  // set by VAD when no speech was detected → skip STT
+  let cancelNextBlob = false;       // set by VAD when no speech was detected → skip STT
+  let convCurrentSentenceBubble = null; // assistant bubble to update with current TTS sentence
 
   // ── Populate STT backends ────────────────────────────────────────────────
   async function loadConvSttBackends() {
@@ -396,6 +397,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
     if (!audioQueue.length) {
       audioQueuePlaying = false;
       convCurrentAudio = null;
+      convCurrentSentenceBubble = null;
       if (audioQueueDrainCb) {
         const cb = audioQueueDrainCb;
         audioQueueDrainCb = null;
@@ -406,20 +408,27 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
       return;
     }
     audioQueuePlaying = true;
-    const url = audioQueue.shift();
+    const item = audioQueue.shift(); // {url, text}
-    const el = new Audio(url);
+    // Show the sentence text in the typing bubble if it still shows "..."
+    if (convCurrentSentenceBubble && item.text) {
+      if (convCurrentSentenceBubble.querySelector('.conv-typing')) {
+        convCurrentSentenceBubble.innerHTML = '';
+        convCurrentSentenceBubble.textContent = item.text;
+      }
+    }
+    const el = new Audio(item.url);
     convCurrentAudio = el;
-    el.addEventListener('ended', () => { URL.revokeObjectURL(url); playNextAudio(); });
+    el.addEventListener('ended', () => { URL.revokeObjectURL(item.url); playNextAudio(); });
-    el.play().catch(() => { URL.revokeObjectURL(url); playNextAudio(); });
+    el.play().catch(() => { URL.revokeObjectURL(item.url); playNextAudio(); });
     if (micStatus) micStatus.textContent = 'Speaking…';
   }
 
-  function enqueueAudio(b64, mime) {
+  function enqueueAudio(b64, mime, text) {
     const binStr = atob(b64);
     const arr = new Uint8Array(binStr.length);
     for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
     const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' }));
-    audioQueue.push(url);
+    audioQueue.push({ url, text: text || '' });
     if (!audioQueuePlaying) playNextAudio();
   }
 
@@ -648,6 +657,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
     // Show user bubble seeded with live interim transcript (if available)
     const userBubble = addBubble('user', liveInterimText || '…');
     const assistantBubble = addTypingBubble();
+    convCurrentSentenceBubble = assistantBubble; // sentence text will appear here while audio plays
     let assistantText = '';
     let lastStats = null;
 
@@ -683,18 +693,18 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
             userBubble.textContent = evt.text || '(empty)';
             if (micStatus) micStatus.textContent = 'Generating reply…';
           } else if (evt.type === 'token') {
-            if (assistantBubble.querySelector('.conv-typing')) {
+            if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
-              assistantBubble.innerHTML = '';
+            convCurrentSentenceBubble = null; // LLM tokens take over the bubble now
-            }
             assistantText += evt.delta;
             assistantBubble.textContent = assistantText;
             chatWindow.scrollTop = chatWindow.scrollHeight;
           } else if (evt.type === 'llm_done') {
             assistantText = evt.text || assistantText;
             assistantBubble.textContent = assistantText;
+            convCurrentSentenceBubble = null;
             if (micStatus) micStatus.textContent = 'Synthesising speech…';
           } else if (evt.type === 'audio') {
-            enqueueAudio(evt.b64, evt.mime || 'audio/wav');
+            enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
           } else if (evt.type === 'stats') {
             lastStats = evt;
             updateStats(evt);
@@ -755,6 +765,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
 
     const userBubble = addBubble('user', text);
     const assistantBubble = addTypingBubble();
+    convCurrentSentenceBubble = assistantBubble;
     let assistantText = '';
     let lastStats = null;
 
@@ -790,15 +801,17 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
             if (micStatus) micStatus.textContent = 'Generating reply…';
           } else if (evt.type === 'token') {
             if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
+            convCurrentSentenceBubble = null;
             assistantText += evt.delta;
             assistantBubble.textContent = assistantText;
             chatWindow.scrollTop = chatWindow.scrollHeight;
           } else if (evt.type === 'llm_done') {
             assistantText = evt.text || assistantText;
             assistantBubble.textContent = assistantText;
+            convCurrentSentenceBubble = null;
             if (micStatus) micStatus.textContent = 'Synthesising speech…';
           } else if (evt.type === 'audio') {
-            enqueueAudio(evt.b64, evt.mime || 'audio/wav');
+            enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
           } else if (evt.type === 'stats') {
             lastStats = evt;
             updateStats(evt);