Show spoken sentence in typing bubble; fix VAD noise sensitivity
Sentence text in typing bubble:
- Backend sends 'text' field with each audio SSE event (the sentence
being synthesised)
- Frontend audio queue stores {url, text} pairs
- playNextAudio() writes the sentence text into the '...' typing bubble
when LLM tokens haven't arrived yet (convCurrentSentenceBubble)
- convCurrentSentenceBubble cleared as soon as first LLM token arrives
so normal streaming takes over seamlessly
VAD noise fixes:
- VAD_THRESHOLD: 0.01 → 0.02 (background noise no longer counts as speech)
- VAD_MIN_REC_MS: 400 → 800ms (8/10s wait before silence detection starts,
gives user time to begin speaking without initial noise triggering send)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
13c79768b4
commit
bf458283af
@ -817,6 +817,7 @@ async def conversation_turn(
|
||||
ttft_done = False
|
||||
sent_buf = ""
|
||||
tts_tasks: list[asyncio.Task] = []
|
||||
tts_texts: list[str] = [] # sentence text corresponding to each task
|
||||
tts_first_start: float | None = None
|
||||
|
||||
try:
|
||||
@ -841,6 +842,7 @@ async def conversation_turn(
|
||||
if chunk_text:
|
||||
if tts_first_start is None:
|
||||
tts_first_start = time.monotonic()
|
||||
tts_texts.append(chunk_text)
|
||||
tts_tasks.append(asyncio.create_task(
|
||||
asyncio.to_thread(_preview_request_audio, chunk_text, tts_voice, settings, "", tts_be)
|
||||
))
|
||||
@ -852,6 +854,7 @@ async def conversation_turn(
|
||||
if sent_buf.strip():
|
||||
if tts_first_start is None:
|
||||
tts_first_start = time.monotonic()
|
||||
tts_texts.append(sent_buf.strip())
|
||||
tts_tasks.append(asyncio.create_task(
|
||||
asyncio.to_thread(_preview_request_audio, sent_buf.strip(), tts_voice, settings, "", tts_be)
|
||||
))
|
||||
@ -879,7 +882,9 @@ async def conversation_turn(
|
||||
return
|
||||
if i == 0:
|
||||
tts_ms = int((time.monotonic() - tts_start) * 1000)
|
||||
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), "mime": mime})
|
||||
sentence_text = tts_texts[i] if i < len(tts_texts) else ""
|
||||
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(),
|
||||
"mime": mime, "text": sentence_text})
|
||||
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
|
||||
|
||||
@ -177,8 +177,8 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
let vadRafId = null;
|
||||
let liveInterimText = '';
|
||||
let speechRec = null;
|
||||
const VAD_THRESHOLD = 0.01;
|
||||
const VAD_MIN_REC_MS = 400;
|
||||
const VAD_THRESHOLD = 0.02; // raised to ignore background noise
|
||||
const VAD_MIN_REC_MS = 800; // wait 800ms before VAD starts checking (avoids click/noise at start)
|
||||
const VAD_SILENCE_MS = 1000;
|
||||
// Whisper hallucinations on silence/noise — discard these from the live preview
|
||||
const HALLUCINATION_RE = /^(reich|danke\s*(schön)?|vielen\s*dank|thank\s*you|thanks|you|copyright|abonnieren|untertitel|zарегистрируйтесь)[.!?,\s]*$/i;
|
||||
@ -194,6 +194,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
let vadHadSpeech = false; // true once RMS crossed threshold during this recording
|
||||
let vadLastVoiceMs = 0; // last timestamp speech was detected (for preview gate)
|
||||
let cancelNextBlob = false; // set by VAD when no speech was detected → skip STT
|
||||
let convCurrentSentenceBubble = null; // assistant bubble to update with current TTS sentence
|
||||
|
||||
// ── Populate STT backends ────────────────────────────────────────────────
|
||||
async function loadConvSttBackends() {
|
||||
@ -396,6 +397,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
if (!audioQueue.length) {
|
||||
audioQueuePlaying = false;
|
||||
convCurrentAudio = null;
|
||||
convCurrentSentenceBubble = null;
|
||||
if (audioQueueDrainCb) {
|
||||
const cb = audioQueueDrainCb;
|
||||
audioQueueDrainCb = null;
|
||||
@ -406,20 +408,27 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
return;
|
||||
}
|
||||
audioQueuePlaying = true;
|
||||
const url = audioQueue.shift();
|
||||
const el = new Audio(url);
|
||||
const item = audioQueue.shift(); // {url, text}
|
||||
// Show the sentence text in the typing bubble if it still shows "..."
|
||||
if (convCurrentSentenceBubble && item.text) {
|
||||
if (convCurrentSentenceBubble.querySelector('.conv-typing')) {
|
||||
convCurrentSentenceBubble.innerHTML = '';
|
||||
convCurrentSentenceBubble.textContent = item.text;
|
||||
}
|
||||
}
|
||||
const el = new Audio(item.url);
|
||||
convCurrentAudio = el;
|
||||
el.addEventListener('ended', () => { URL.revokeObjectURL(url); playNextAudio(); });
|
||||
el.play().catch(() => { URL.revokeObjectURL(url); playNextAudio(); });
|
||||
el.addEventListener('ended', () => { URL.revokeObjectURL(item.url); playNextAudio(); });
|
||||
el.play().catch(() => { URL.revokeObjectURL(item.url); playNextAudio(); });
|
||||
if (micStatus) micStatus.textContent = 'Speaking…';
|
||||
}
|
||||
|
||||
function enqueueAudio(b64, mime) {
|
||||
function enqueueAudio(b64, mime, text) {
|
||||
const binStr = atob(b64);
|
||||
const arr = new Uint8Array(binStr.length);
|
||||
for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
|
||||
const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' }));
|
||||
audioQueue.push(url);
|
||||
audioQueue.push({ url, text: text || '' });
|
||||
if (!audioQueuePlaying) playNextAudio();
|
||||
}
|
||||
|
||||
@ -648,6 +657,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
// Show user bubble seeded with live interim transcript (if available)
|
||||
const userBubble = addBubble('user', liveInterimText || '…');
|
||||
const assistantBubble = addTypingBubble();
|
||||
convCurrentSentenceBubble = assistantBubble; // sentence text will appear here while audio plays
|
||||
let assistantText = '';
|
||||
let lastStats = null;
|
||||
|
||||
@ -683,18 +693,18 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
userBubble.textContent = evt.text || '(empty)';
|
||||
if (micStatus) micStatus.textContent = 'Generating reply…';
|
||||
} else if (evt.type === 'token') {
|
||||
if (assistantBubble.querySelector('.conv-typing')) {
|
||||
assistantBubble.innerHTML = '';
|
||||
}
|
||||
if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
|
||||
convCurrentSentenceBubble = null; // LLM tokens take over the bubble now
|
||||
assistantText += evt.delta;
|
||||
assistantBubble.textContent = assistantText;
|
||||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||||
} else if (evt.type === 'llm_done') {
|
||||
assistantText = evt.text || assistantText;
|
||||
assistantBubble.textContent = assistantText;
|
||||
convCurrentSentenceBubble = null;
|
||||
if (micStatus) micStatus.textContent = 'Synthesising speech…';
|
||||
} else if (evt.type === 'audio') {
|
||||
enqueueAudio(evt.b64, evt.mime || 'audio/wav');
|
||||
enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
|
||||
} else if (evt.type === 'stats') {
|
||||
lastStats = evt;
|
||||
updateStats(evt);
|
||||
@ -755,6 +765,7 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
|
||||
const userBubble = addBubble('user', text);
|
||||
const assistantBubble = addTypingBubble();
|
||||
convCurrentSentenceBubble = assistantBubble;
|
||||
let assistantText = '';
|
||||
let lastStats = null;
|
||||
|
||||
@ -790,15 +801,17 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
if (micStatus) micStatus.textContent = 'Generating reply…';
|
||||
} else if (evt.type === 'token') {
|
||||
if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
|
||||
convCurrentSentenceBubble = null;
|
||||
assistantText += evt.delta;
|
||||
assistantBubble.textContent = assistantText;
|
||||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||||
} else if (evt.type === 'llm_done') {
|
||||
assistantText = evt.text || assistantText;
|
||||
assistantBubble.textContent = assistantText;
|
||||
convCurrentSentenceBubble = null;
|
||||
if (micStatus) micStatus.textContent = 'Synthesising speech…';
|
||||
} else if (evt.type === 'audio') {
|
||||
enqueueAudio(evt.b64, evt.mime || 'audio/wav');
|
||||
enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
|
||||
} else if (evt.type === 'stats') {
|
||||
lastStats = evt;
|
||||
updateStats(evt);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user