Pipeline TTS with LLM streaming for lower latency

Replace serial LLM-wait-TTS with overlapped execution:
- LLM streams via background thread → asyncio.Queue (non-blocking event loop)
- _sentence_split() detects sentence boundaries in the token stream
- asyncio.create_task fires TTS for each sentence immediately — TTS for
  sentence 1 runs while LLM is still generating sentences 2, 3, …
- Audio chunks stream to frontend in order as each task completes
- Time-to-first-audio drops from (LLM total + TTS total) to
  roughly (LLM time-to-first-sentence + TTS latency for one sentence)

Frontend audio queue:
- enqueueAudio() / playNextAudio() chain multi-chunk responses seamlessly
- clearAudio() stops playback and cancels queue on new turn or mic click
- scheduleAutoMic() waits for queue to drain before restarting mic
- Error paths clear the queue to avoid stale audio playing after failure

Also fix missing contextlib import (silent bug when audio temp files
needed cleanup in the STT path).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

routes/conversation.py

@@ -3,8 +3,11 @@ from __future__ import annotations
 
 import asyncio
 import base64
+import contextlib
 import io
 import json
+import re
+import threading
 import time
 import uuid
 import wave
@@ -30,6 +33,20 @@ from routes.stt import _transcribe_audio, _clean_stt_backend
 
 router = APIRouter()
 
+# ── Sentence-boundary helpers for pipelined TTS ───────────────────────────────
+
+_SENT_RE = re.compile(r'(?<=[.!?])\s+')
+_MIN_SENTENCE = 30  # min chars in buffer before we split
+
+def _sentence_split(buf: str) -> int:
+    """Return the index after the first sentence boundary, or -1."""
+    if len(buf) < _MIN_SENTENCE:
+        return -1
+    for m in _SENT_RE.finditer(buf):
+        if m.end() >= _MIN_SENTENCE:
+            return m.end()
+    return -1
+
 # ── LLM helpers ───────────────────────────────────────────────────────────────
 
 def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
@@ -736,51 +753,99 @@ async def conversation_turn(
             yield sse({"type": "error", "stage": "stt", "message": "No speech detected."})
             return
 
-        # 2. LLM stream
+        # 2. LLM stream — runs in a thread; tokens arrive via asyncio.Queue
+        #    so the event loop is never blocked and TTS can start on sentence 1
+        #    while the LLM is still generating sentences 2, 3, …
         messages = [{"role": "system", "content": system_prompt}]
         messages.extend(hist[-20:])
         messages.append({"role": "user", "content": transcript})
         llm_payload: dict = {"messages": messages, "stream": True, "temperature": 0.7, "max_tokens": 512}
         if llm_model:
             llm_payload["model"] = llm_model
+
+        _loop = asyncio.get_event_loop()
+        _token_q: asyncio.Queue[str | None] = asyncio.Queue()
+
+        def _llm_thread() -> None:
             try:
-            t_llm = time.monotonic()
+                resp = requests.post(
-            llm_resp = await asyncio.to_thread(lambda: requests.post(
                     f"{eff_llm_url}/chat/completions", json=llm_payload,
                     headers={"Authorization": "Bearer no-key"}, stream=True, timeout=120,
-            ))
+                )
-            llm_resp.raise_for_status()
+                resp.raise_for_status()
-            ttft_done = False
+                for raw_line in resp.iter_lines():
-            for raw_line in llm_resp.iter_lines():
                     if not raw_line:
                         continue
                     line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else str(raw_line)
                     if not line.startswith("data:"):
                         continue
-                chunk = line[5:].strip()
+                    payload_str = line[5:].strip()
-                if chunk == "[DONE]":
+                    if payload_str == "[DONE]":
                         break
                     try:
-                    obj = json.loads(chunk)
+                        obj = json.loads(payload_str)
                         d_obj = ((obj.get("choices") or [{}])[0].get("delta") or {})
-                    # Primary: visible content. Fallback: reasoning_content (Qwen3 /think tokens)
                         delta = d_obj.get("content") or d_obj.get("reasoning_content") or ""
                         if not delta and isinstance(obj.get("message"), dict):
                             delta = obj["message"].get("content") or ""
                         if delta:
+                            _loop.call_soon_threadsafe(_token_q.put_nowait, delta)
+                    except Exception:
+                        continue
+            except Exception as exc:
+                _loop.call_soon_threadsafe(_token_q.put_nowait, f"\x00ERR:{exc}")
+            finally:
+                _loop.call_soon_threadsafe(_token_q.put_nowait, None)
+
+        threading.Thread(target=_llm_thread, daemon=True).start()
+
+        t_llm = time.monotonic()
+        llm_ttft_ms: int | None = None
+        ttft_done = False
+        sent_buf = ""
+        tts_tasks: list[asyncio.Task] = []
+        tts_first_start: float | None = None
+
+        try:
+            while True:
+                delta = await _token_q.get()
+                if delta is None:
+                    break
+                if delta.startswith("\x00ERR:"):
+                    yield sse({"type": "error", "stage": "llm", "message": delta[5:]})
+                    return
                 if not ttft_done:
                     llm_ttft_ms = int((time.monotonic() - t_llm) * 1000)
                     ttft_done = True
                 llm_text += delta
+                sent_buf += delta
                 yield sse({"type": "token", "delta": delta})
-                except Exception:
+                # Fire TTS on sentence boundary — runs concurrently with LLM
-                    continue
+                split = _sentence_split(sent_buf)
+                if split > 0:
+                    chunk_text = sent_buf[:split].strip()
+                    sent_buf = sent_buf[split:]
+                    if chunk_text:
+                        if tts_first_start is None:
+                            tts_first_start = time.monotonic()
+                        tts_tasks.append(asyncio.create_task(
+                            asyncio.to_thread(_preview_request_audio, chunk_text, tts_voice, settings, "", tts_be)
+                        ))
+        except Exception as exc:
+            yield sse({"type": "error", "stage": "llm", "message": str(exc)})
+            return
+
+        # Flush any remaining text as a final TTS task
+        if sent_buf.strip():
+            if tts_first_start is None:
+                tts_first_start = time.monotonic()
+            tts_tasks.append(asyncio.create_task(
+                asyncio.to_thread(_preview_request_audio, sent_buf.strip(), tts_voice, settings, "", tts_be)
+            ))
+
         llm_total_ms = int((time.monotonic() - t_llm) * 1000)
         yield sse({"type": "llm_done", "text": llm_text,
                    "llm_ttft_ms": llm_ttft_ms, "llm_total_ms": llm_total_ms})
-        except Exception as e:
-            yield sse({"type": "error", "stage": "llm", "message": str(e)})
-            return
 
         if not llm_text.strip():
             yield sse({"type": "error", "stage": "llm",
@@ -789,21 +854,23 @@ async def conversation_turn(
                                   "or pick a non-thinking model in the Language Model dropdown."})
             return
 
-        # 3. TTS
+        # 3. Stream audio chunks in order — each chunk's TTS ran concurrently
+        #    with LLM generation, so first audio arrives much sooner than
+        #    waiting for the full response.
+        tts_start = tts_first_start or time.monotonic()
+        for i, task in enumerate(tts_tasks):
             try:
-            t_tts = time.monotonic()
+                audio_bytes, mime = await task
-            audio_bytes, mime = await asyncio.to_thread(
+            except Exception as exc:
-                _preview_request_audio, llm_text, tts_voice, settings, "", tts_be
+                yield sse({"type": "error", "stage": "tts", "message": str(exc)})
-            )
+                return
-            tts_ms = int((time.monotonic() - t_tts) * 1000)
+            if i == 0:
-            total_ms = int((time.monotonic() - t0) * 1000)
+                tts_ms = int((time.monotonic() - tts_start) * 1000)
             yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), "mime": mime})
+
+        total_ms = int((time.monotonic() - t0) * 1000)
         yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
                    "llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms})
-        except Exception as e:
-            yield sse({"type": "error", "stage": "tts", "message": str(e)})
-            return
-
         yield sse({"type": "done"})
 
     return StreamingResponse(generate(), media_type="text/event-stream",

static/js/conversation.js

@@ -186,6 +186,9 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
   let previewTranscribing = false;
   let convCurrentAudio = null;
   let autoMicGeneration = 0;
+  const audioQueue = [];
+  let audioQueuePlaying = false;
+  let audioQueueDrainCb = null;
 
   // ── Populate STT backends ────────────────────────────────────────────────
   async function loadConvSttBackends() {
@@ -376,6 +379,45 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
     if (micTimer) micTimer.textContent = '';
   }
 
+  // ── Audio queue — plays multi-chunk TTS responses sequentially ──────────────
+  function clearAudio() {
+    if (convCurrentAudio) { try { convCurrentAudio.pause(); } catch(_){} convCurrentAudio = null; }
+    audioQueue.length = 0;
+    audioQueuePlaying = false;
+    audioQueueDrainCb = null;
+  }
+
+  function playNextAudio() {
+    if (!audioQueue.length) {
+      audioQueuePlaying = false;
+      convCurrentAudio = null;
+      if (audioQueueDrainCb) {
+        const cb = audioQueueDrainCb;
+        audioQueueDrainCb = null;
+        setTimeout(cb, 150);
+      } else if (micStatus && !isProcessing) {
+        micStatus.textContent = 'Ready';
+      }
+      return;
+    }
+    audioQueuePlaying = true;
+    const url = audioQueue.shift();
+    const el = new Audio(url);
+    convCurrentAudio = el;
+    el.addEventListener('ended', () => { URL.revokeObjectURL(url); playNextAudio(); });
+    el.play().catch(() => { URL.revokeObjectURL(url); playNextAudio(); });
+    if (micStatus) micStatus.textContent = 'Speaking…';
+  }
+
+  function enqueueAudio(b64, mime) {
+    const binStr = atob(b64);
+    const arr = new Uint8Array(binStr.length);
+    for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
+    const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' }));
+    audioQueue.push(url);
+    if (!audioQueuePlaying) playNextAudio();
+  }
+
   // ── Chunked Whisper preview (fallback for browsers without SpeechRecognition) ──
   async function transcribeForPreview() {
     if (previewTranscribing || !recChunks.length) return;
@@ -408,8 +450,11 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
       if (gen !== autoMicGeneration || isProcessing) return;
       startRecording().catch(() => {});
     }
-    if (convCurrentAudio && !convCurrentAudio.ended) {
+    if (audioQueuePlaying || audioQueue.length > 0) {
-      convCurrentAudio.addEventListener('ended', () => setTimeout(tryStart, 200), { once: true });
+      // Audio still queued — trigger after queue drains
+      audioQueueDrainCb = tryStart;
+    } else if (convCurrentAudio && !convCurrentAudio.ended) {
+      convCurrentAudio.addEventListener('ended', () => setTimeout(tryStart, 150), { once: true });
     } else {
       setTimeout(tryStart, 300);
     }
@@ -554,7 +599,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
   // ── Send turn via SSE ─────────────────────────────────────────────────────
   async function processBlob(blob) {
     autoMicGeneration++; // cancel any pending auto-mic from previous turn
-    convCurrentAudio = null;
+    clearAudio();
     turnCount++;
     const turnN = turnCount;
     const t0 = Date.now();
@@ -618,17 +663,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
             assistantBubble.textContent = assistantText;
             if (micStatus) micStatus.textContent = 'Synthesising speech…';
           } else if (evt.type === 'audio') {
-            const mime = evt.mime || 'audio/wav';
+            enqueueAudio(evt.b64, evt.mime || 'audio/wav');
-            const binStr = atob(evt.b64);
-            const arr = new Uint8Array(binStr.length);
-            for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
-            const audioBlob = new Blob([arr], { type: mime });
-            const url = URL.createObjectURL(audioBlob);
-            const audioEl = new Audio(url);
-            convCurrentAudio = audioEl;
-            audioEl.addEventListener('ended', () => { URL.revokeObjectURL(url); convCurrentAudio = null; });
-            audioEl.play().catch(() => {});
-            if (micStatus) micStatus.textContent = 'Speaking…';
           } else if (evt.type === 'stats') {
             lastStats = evt;
             updateStats(evt);
@@ -636,17 +671,15 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
             conversationHistory.push({ role: 'user', content: userBubble.textContent });
             conversationHistory.push({ role: 'assistant', content: assistantText });
             addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
-            scheduleAutoMic();
+            scheduleAutoMic(); // triggers after audio queue drains
-            if (!handsFreeToggle?.checked && micStatus) micStatus.textContent = 'Ready';
           } else if (evt.type === 'error') {
+            clearAudio();
             const wrap = assistantBubble.closest('.conv-bubble-wrap');
             if (wrap) wrap.remove();
             const stage = evt.stage?.toUpperCase() || 'ERR';
             let msg = evt.message || 'Unknown error';
-            // Surface the actual server error detail, not the raw HTTP noise
             const detailMatch = msg.match(/HTTP \d+:\s*(.+)/s);
             if (detailMatch) msg = detailMatch[1].trim();
-            // Truncate very long stack traces
             if (msg.length > 300) msg = msg.slice(0, 300) + '…';
             addErrorBubble(`[${stage}] ${msg}`);
             addHistoryItem(turnN, Date.now() - t0, false);
@@ -655,6 +688,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
         }
       }
     } catch(e) {
+      clearAudio();
       const wrap = assistantBubble.closest('.conv-bubble-wrap');
       if (wrap) wrap.remove();
       addErrorBubble(e.message);
@@ -676,7 +710,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
     text = text.trim();
     if (!text || isProcessing) return;
     autoMicGeneration++; // cancel any pending auto-mic
-    convCurrentAudio = null;
+    clearAudio();
     isProcessing = true;
     if (sendBtn) sendBtn.disabled = true;
     if (textInput) { textInput.disabled = true; textInput.value = ''; }
@@ -733,17 +767,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
             assistantBubble.textContent = assistantText;
             if (micStatus) micStatus.textContent = 'Synthesising speech…';
           } else if (evt.type === 'audio') {
-            const mime = evt.mime || 'audio/wav';
+            enqueueAudio(evt.b64, evt.mime || 'audio/wav');
-            const binStr = atob(evt.b64);
-            const arr = new Uint8Array(binStr.length);
-            for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
-            const audioBlob = new Blob([arr], { type: mime });
-            const url = URL.createObjectURL(audioBlob);
-            const audioEl = new Audio(url);
-            convCurrentAudio = audioEl;
-            audioEl.addEventListener('ended', () => { URL.revokeObjectURL(url); convCurrentAudio = null; });
-            audioEl.play().catch(() => {});
-            if (micStatus) micStatus.textContent = 'Speaking…';
           } else if (evt.type === 'stats') {
             lastStats = evt;
             updateStats(evt);
@@ -751,9 +775,9 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
             conversationHistory.push({ role: 'user', content: text });
             conversationHistory.push({ role: 'assistant', content: assistantText });
             addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
-            scheduleAutoMic();
+            scheduleAutoMic(); // triggers after audio queue drains
-            if (!handsFreeToggle?.checked && micStatus) micStatus.textContent = 'Ready';
           } else if (evt.type === 'error') {
+            clearAudio();
             const wrap = assistantBubble.closest('.conv-bubble-wrap');
             if (wrap) wrap.remove();
             addErrorBubble(`[${evt.stage?.toUpperCase() || 'ERR'}] ${evt.message || 'Unknown error'}`);
@@ -763,6 +787,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
         }
       }
     } catch(e) {
+      clearAudio();
       const wrap = assistantBubble.closest('.conv-bubble-wrap');
       if (wrap) wrap.remove();
       addErrorBubble(e.message);
@@ -784,6 +809,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
     if (mediaRecorder && mediaRecorder.state === 'recording') {
       stopRecording();
     } else {
+      clearAudio(); // stop agent if still speaking
       startRecording();
     }
   });