diff --git a/routes/conversation.py b/routes/conversation.py index 7cb9e59..a249628 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -817,6 +817,7 @@ async def conversation_turn( ttft_done = False sent_buf = "" tts_tasks: list[asyncio.Task] = [] + tts_texts: list[str] = [] # sentence text corresponding to each task tts_first_start: float | None = None try: @@ -841,6 +842,7 @@ async def conversation_turn( if chunk_text: if tts_first_start is None: tts_first_start = time.monotonic() + tts_texts.append(chunk_text) tts_tasks.append(asyncio.create_task( asyncio.to_thread(_preview_request_audio, chunk_text, tts_voice, settings, "", tts_be) )) @@ -852,6 +854,7 @@ async def conversation_turn( if sent_buf.strip(): if tts_first_start is None: tts_first_start = time.monotonic() + tts_texts.append(sent_buf.strip()) tts_tasks.append(asyncio.create_task( asyncio.to_thread(_preview_request_audio, sent_buf.strip(), tts_voice, settings, "", tts_be) )) @@ -879,7 +882,9 @@ async def conversation_turn( return if i == 0: tts_ms = int((time.monotonic() - tts_start) * 1000) - yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), "mime": mime}) + sentence_text = tts_texts[i] if i < len(tts_texts) else "" + yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), + "mime": mime, "text": sentence_text}) total_ms = int((time.monotonic() - t0) * 1000) yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms, diff --git a/static/js/conversation.js b/static/js/conversation.js index 97cdd11..fe15614 100644 --- a/static/js/conversation.js +++ b/static/js/conversation.js @@ -177,8 +177,8 @@ $('s-import-voices-file')?.addEventListener('change', async function () { let vadRafId = null; let liveInterimText = ''; let speechRec = null; - const VAD_THRESHOLD = 0.01; - const VAD_MIN_REC_MS = 400; + const VAD_THRESHOLD = 0.02; // raised to ignore background noise + const VAD_MIN_REC_MS = 800; // wait 800ms before VAD starts checking (avoids click/noise at start) const VAD_SILENCE_MS = 1000; // Whisper hallucinations on silence/noise — discard these from the live preview const HALLUCINATION_RE = /^(reich|danke\s*(schön)?|vielen\s*dank|thank\s*you|thanks|you|copyright|abonnieren|untertitel|zарегистрируйтесь)[.!?,\s]*$/i; @@ -191,9 +191,10 @@ $('s-import-voices-file')?.addEventListener('change', async function () { const audioQueue = []; let audioQueuePlaying = false; let audioQueueDrainCb = null; - let vadHadSpeech = false; // true once RMS crossed threshold during this recording - let vadLastVoiceMs = 0; // last timestamp speech was detected (for preview gate) - let cancelNextBlob = false; // set by VAD when no speech was detected → skip STT + let vadHadSpeech = false; // true once RMS crossed threshold during this recording + let vadLastVoiceMs = 0; // last timestamp speech was detected (for preview gate) + let cancelNextBlob = false; // set by VAD when no speech was detected → skip STT + let convCurrentSentenceBubble = null; // assistant bubble to update with current TTS sentence // ── Populate STT backends ──────────────────────────────────────────────── async function loadConvSttBackends() { @@ -396,6 +397,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () { if (!audioQueue.length) { audioQueuePlaying = false; convCurrentAudio = null; + convCurrentSentenceBubble = null; if (audioQueueDrainCb) { const cb = audioQueueDrainCb; audioQueueDrainCb = null; @@ -406,20 +408,27 @@ $('s-import-voices-file')?.addEventListener('change', async function () { return; } audioQueuePlaying = true; - const url = audioQueue.shift(); - const el = new Audio(url); + const item = audioQueue.shift(); // {url, text} + // Show the sentence text in the typing bubble if it still shows "..." + if (convCurrentSentenceBubble && item.text) { + if (convCurrentSentenceBubble.querySelector('.conv-typing')) { + convCurrentSentenceBubble.innerHTML = ''; + convCurrentSentenceBubble.textContent = item.text; + } + } + const el = new Audio(item.url); convCurrentAudio = el; - el.addEventListener('ended', () => { URL.revokeObjectURL(url); playNextAudio(); }); - el.play().catch(() => { URL.revokeObjectURL(url); playNextAudio(); }); + el.addEventListener('ended', () => { URL.revokeObjectURL(item.url); playNextAudio(); }); + el.play().catch(() => { URL.revokeObjectURL(item.url); playNextAudio(); }); if (micStatus) micStatus.textContent = 'Speaking…'; } - function enqueueAudio(b64, mime) { + function enqueueAudio(b64, mime, text) { const binStr = atob(b64); const arr = new Uint8Array(binStr.length); for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i); const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' })); - audioQueue.push(url); + audioQueue.push({ url, text: text || '' }); if (!audioQueuePlaying) playNextAudio(); } @@ -648,6 +657,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () { // Show user bubble seeded with live interim transcript (if available) const userBubble = addBubble('user', liveInterimText || '…'); const assistantBubble = addTypingBubble(); + convCurrentSentenceBubble = assistantBubble; // sentence text will appear here while audio plays let assistantText = ''; let lastStats = null; @@ -683,18 +693,18 @@ $('s-import-voices-file')?.addEventListener('change', async function () { userBubble.textContent = evt.text || '(empty)'; if (micStatus) micStatus.textContent = 'Generating reply…'; } else if (evt.type === 'token') { - if (assistantBubble.querySelector('.conv-typing')) { - assistantBubble.innerHTML = ''; - } + if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = ''; + convCurrentSentenceBubble = null; // LLM tokens take over the bubble now assistantText += evt.delta; assistantBubble.textContent = assistantText; chatWindow.scrollTop = chatWindow.scrollHeight; } else if (evt.type === 'llm_done') { assistantText = evt.text || assistantText; assistantBubble.textContent = assistantText; + convCurrentSentenceBubble = null; if (micStatus) micStatus.textContent = 'Synthesising speech…'; } else if (evt.type === 'audio') { - enqueueAudio(evt.b64, evt.mime || 'audio/wav'); + enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || ''); } else if (evt.type === 'stats') { lastStats = evt; updateStats(evt); @@ -755,6 +765,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () { const userBubble = addBubble('user', text); const assistantBubble = addTypingBubble(); + convCurrentSentenceBubble = assistantBubble; let assistantText = ''; let lastStats = null; @@ -790,15 +801,17 @@ $('s-import-voices-file')?.addEventListener('change', async function () { if (micStatus) micStatus.textContent = 'Generating reply…'; } else if (evt.type === 'token') { if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = ''; + convCurrentSentenceBubble = null; assistantText += evt.delta; assistantBubble.textContent = assistantText; chatWindow.scrollTop = chatWindow.scrollHeight; } else if (evt.type === 'llm_done') { assistantText = evt.text || assistantText; assistantBubble.textContent = assistantText; + convCurrentSentenceBubble = null; if (micStatus) micStatus.textContent = 'Synthesising speech…'; } else if (evt.type === 'audio') { - enqueueAudio(evt.b64, evt.mime || 'audio/wav'); + enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || ''); } else if (evt.type === 'stats') { lastStats = evt; updateStats(evt);