// ── Settings: Logs ──────────────────────────────────────────────────────── let _logsLiveTimer = null; let _logsActiveLevel = ''; function escLog(s) { return String(s).replace(/[&<>]/g, c => ({'&':'&','<':'<','>':'>'}[c])); } window.loadSettingsLogs = async function() { const viewer = $('s-log-viewer'); if (!viewer) return; try { const data = await fetch('/api/logs?limit=300').then(r => r.json()); const items = (data.items || []).filter(item => !_logsActiveLevel || item.level === _logsActiveLevel ); const count = $('s-log-count'); if (count) count.textContent = items.length + ' entries'; if (!items.length) { viewer.innerHTML = '
No log entries
'; return; } viewer.innerHTML = items.map(item => { const ts = item.ts ? item.ts.replace('T', ' ').replace(/\.\d+Z$/, ' UTC') : ''; return `
${escLog(ts)} ${escLog(item.level)} ${escLog(item.msg)}
`; }).join(''); } catch(e) { viewer.innerHTML = `
Failed to load logs: ${escLog(e.message)}
`; } }; $('s-logs-refresh-btn')?.addEventListener('click', () => loadSettingsLogs()); $('s-logs-clear-btn')?.addEventListener('click', async () => { await fetch('/api/logs', { method: 'DELETE' }); const viewer = $('s-log-viewer'); if (viewer) viewer.innerHTML = '
Logs cleared
'; const count = $('s-log-count'); if (count) count.textContent = ''; }); $('s-logs-live-toggle')?.addEventListener('change', function() { clearInterval(_logsLiveTimer); if (this.checked) { loadSettingsLogs(); _logsLiveTimer = setInterval(loadSettingsLogs, 3000); } }); document.querySelectorAll('.s-log-filter').forEach(btn => { btn.addEventListener('click', function() { document.querySelectorAll('.s-log-filter').forEach(b => b.classList.remove('is-active')); this.classList.add('is-active'); _logsActiveLevel = this.dataset.logLevel; loadSettingsLogs(); }); }); // ── Settings: About ─────────────────────────────────────────────────────── // Show version immediately from meta tag, then confirm from server (function loadAppVersion() { const el = $('s-about-version'); const metaVer = document.querySelector('meta[name="app-version"]')?.content; if (el && metaVer) el.textContent = 'v' + metaVer; fetch('/api/version').then(r => r.json()).then(d => { if (el && d.version) el.textContent = 'v' + d.version; }).catch(() => {}); })(); // Lazy-load changelog when the details element is opened $('about-changelog-details')?.addEventListener('toggle', async function () { if (!this.open) return; const content = $('about-changelog-content'); const status = $('about-changelog-status'); if (!content || content.textContent.trim()) return; if (status) status.textContent = 'Loading…'; try { const text = await fetch('/api/changelog').then(r => r.ok ? r.text() : Promise.reject(r.status)); content.textContent = text; if (status) status.textContent = ''; } catch(e) { content.textContent = 'Could not load changelog: ' + e; if (status) status.textContent = 'error'; } }); function renderSettingsAbout() { const el = $('s-about-backends'); if (!el) return; const available = new Set((_ttsBackends || []).filter(b => b.available).map(b => b.id)); const all = (_ttsBackends || []); if (!all.length) { el.innerHTML = ''; return; } el.innerHTML = all.map(b => { const online = available.has(b.id); return ` ${escHtml(b.label)} `; }).join(''); } // ── Voices import ────────────────────────────────────────────────────────── $('s-import-voices-file')?.addEventListener('change', async function () { const file = this.files?.[0]; if (!file) return; const st = $('s-import-status'); if (st) st.textContent = 'Uploading…'; try { const fd = new FormData(); fd.append('file', file); const r = await fetch('/api/voices/import', { method:'POST', body: fd }); if (!r.ok) { const e = await r.json().catch(()=>({})); throw new Error(e.detail || r.statusText); } const d = await r.json(); if (st) st.textContent = `Imported ${d.imported} files.`; toast(`Imported ${d.imported} voice files`, 'success'); this.value = ''; } catch(e) { if (st) st.textContent = 'Import failed.'; toast('Import failed: ' + e.message, 'error'); } }); // ── Conversation Playground ──────────────────────────────────────────────── (function initConversationPlayground() { const chatWindow = $('conv-chat-window'); const micBtn = $('conv-mic-btn'); const micIcon = $('conv-mic-icon'); const micStatus = $('conv-mic-status'); const micTimer = $('conv-mic-timer'); const textInput = $('conv-text-input'); const sendBtn = $('conv-send-btn'); const clearBtn = $('conv-clear-btn'); const sttSel = $('conv-stt-select'); const llmUrlInp = $('conv-llm-url'); const llmFetchBtn = $('conv-llm-fetch-btn'); const llmModelSel = $('conv-llm-model-select'); const ttsBkSel = $('conv-tts-backend-select'); const ttsFetchBtn = $('conv-tts-fetch-btn'); const ttsVoiceSel = $('conv-tts-voice-select'); const ttsEmotionSel = $('conv-emotion-select'); const systemPrompt = $('conv-system-prompt'); const turnHistory = $('conv-turn-history'); if (!chatWindow || !micBtn) return; // Conversation replies previously had no style/emotion control at all, // regardless of backend. Same REH_EMOTIONS quick-pick used in Try a Voice / // Read Aloud — sent to the server as a plain English phrase; the backend // decides server-side (see _conv_tts_text_and_instruct in // routes/conversation.py) whether it becomes an inline Fish [tag] or an // instruct-field phrase for style-aware backends. if (ttsEmotionSel && typeof window.REH_EMOTIONS !== 'undefined' && !ttsEmotionSel.dataset.populated) { window.REH_EMOTIONS.forEach(e => { if (!e.value) return; const o = document.createElement('option'); o.value = e.value; o.textContent = `${e.emoji} ${e.label}`; ttsEmotionSel.appendChild(o); }); ttsEmotionSel.dataset.populated = '1'; } // Warn if microphone API is unavailable (HTTP on non-localhost = insecure context) if (!navigator.mediaDevices?.getUserMedia) { if (micStatus) { micStatus.textContent = 'Mic unavailable — insecure context'; micStatus.style.color = 'var(--red, #e05)'; } if (micBtn) { micBtn.disabled = true; micBtn.title = 'Browser blocks microphone on HTTP. Use http://localhost:7890 or HTTPS.\n' + 'Chrome fix: chrome://flags/#unsafely-treat-insecure-origin-as-secure'; micBtn.style.opacity = '0.4'; } const flagsUrl = 'chrome://flags/#unsafely-treat-insecure-origin-as-secure'; const warn = document.createElement('div'); warn.style.cssText = 'background:var(--red,#c00);color:#fff;padding:12px 16px;border-radius:8px;margin:0 0 12px;font-size:13px;line-height:1.7;flex-shrink:0'; warn.innerHTML = '⚠ Microphone blocked by browser
' + 'Browsers only allow microphone access on secure contexts (HTTPS or localhost).
' + 'Quick fix: open the app at http://localhost:7890 instead of the IP address.
' + 'Remote access fix: add the URL in Chrome flags:
' + '
' + `${flagsUrl}` + `' + '
'; // Insert inside the chat window so it scrolls with the conversation and // never pushes the input bar off-screen. if (chatWindow) chatWindow.prepend(warn); } // Restore conv LLM URL from server settings — fall back to the global Active LLM endpoint if (llmUrlInp && _appSettings) llmUrlInp.value = _appSettings.conv_llm_url || _appSettings.llm_url || ''; if (llmUrlInp) llmUrlInp.addEventListener('input', () => { _patchSettings({ conv_llm_url: llmUrlInp.value }); if (_appSettings) _appSettings.conv_llm_url = llmUrlInp.value; }); let mediaRecorder = null; let recChunks = []; let recTimerInterval = null; let recStart = 0; let conversationHistory = []; let turnCount = 0; let isProcessing = false; let vadAudioCtx = null; let vadRafId = null; let liveInterimText = ''; let speechRec = null; const VAD_MIN_REC_MS = 650; // wait before VAD starts checking (avoids click/noise at start) const VAD_SILENCE_MS = 1000; const INTERRUPT_THRESHOLD = 0.04; // higher than VAD to avoid echo triggering interruption const INTERRUPT_HOLD_MS = 350; // speech must persist this long to interrupt const MIN_SPEECH_FRAMES_MS = 300; // speech must exceed threshold for at least this long const gateSlider = () => document.getElementById('conv-gate-slider'); const vadThreshold = () => { const s = gateSlider(); return s ? (parseInt(s.value, 10) / 1000) : 0.02; }; let vadSpeechStartMs = 0; // time when speech level first crossed threshold in this recording const liveAgentOn = () => liveAgentToggle ? liveAgentToggle.checked : true; const vadSilenceMs = () => liveAgentOn() ? 650 : VAD_SILENCE_MS; const interruptHoldMs = () => liveAgentOn() ? 120 : INTERRUPT_HOLD_MS; // Whisper hallucinations on silence/noise — discard these from the live preview const HALLUCINATION_RE = /^(reich|danke\s*(schön)?|vielen\s*dank|thank\s*you|thanks|you|copyright|abonnieren|untertitel|zарегистрируйтесь)[.!?,\s]*$/i; const origPlaceholder = textInput?.placeholder || ''; const vadToggle = $('conv-vad-toggle'); const handsFreeToggle = $('conv-handsfree-toggle'); const liveAgentToggle = $('conv-live-agent-toggle'); let previewTranscribing = false; let convCurrentAudio = null; let convAbortCtrl = null; let convTurnGeneration = 0; let liveMicEnabled = false; let cancelRecordingOnStop = false; let autoMicGeneration = 0; const audioQueue = []; let audioQueuePlaying = false; let audioQueueDrainCb = null; let interruptCtx = null; let interruptRafId = null; let interruptStream = null; let interruptSpeechStart = 0; let vadHadSpeech = false; // true once RMS crossed threshold during this recording let vadLastVoiceMs = 0; // last timestamp speech was detected (for preview gate) let cancelNextBlob = false; // set by VAD when no speech was detected → skip STT // Update gate marker position and persist on slider change function updateGateMarker() { const s = gateSlider(); if (!s) return; const gate = document.getElementById('conv-level-gate'); if (gate) gate.style.left = s.value + '%'; try { localStorage.setItem('ttsvc_conv_gate', s.value); } catch (_) {} } document.addEventListener('change', function (e) { if (e.target.id === 'conv-gate-slider') updateGateMarker(); }); // Restore persisted gate (function () { const s = gateSlider(); if (!s) return; try { const v = localStorage.getItem('ttsvc_conv_gate'); if (v) s.value = v; } catch (_) {} updateGateMarker(); })(); let convCurrentSentenceBubble = null; // assistant bubble to update with current TTS sentence // ── Populate STT backends ──────────────────────────────────────────────── async function loadConvSttBackends() { if (!sttSel) return; const prev = sttSel.value; try { const d = await fetch('/api/stt-backends').then(r => r.json()); const all = d.backends || []; if (!all.length) { sttSel.innerHTML = ''; return; } sttSel.innerHTML = all.map(b => { const icon = b.available ? '✓' : '✗'; const label = `${icon} ${escHtml(b.label)}`; const disabled = !b.available; return ``; }).join(''); // Restore prev selection or pick first available const opt = sttSel.querySelector(`option[value="${CSS.escape(prev)}"]`); if (opt && !opt.disabled) { sttSel.value = prev; } else { const first = sttSel.querySelector('option:not([disabled])'); if (first) sttSel.value = first.value; } } catch(_) { sttSel.innerHTML = ''; } } // ── Populate TTS backends (reuse global _ttsBackends) ─────────────────── function populateConvTtsBackends() { if (!ttsBkSel) return; const prev = ttsBkSel.value; const all = _ttsBackends || []; if (!all.length) { ttsBkSel.innerHTML = ''; return; } ttsBkSel.innerHTML = all.map(b => { const icon = b.available ? '✓' : '✗'; return ``; }).join(''); const opt = ttsBkSel.querySelector(`option[value="${CSS.escape(prev)}"]`); if (opt && !opt.disabled) { ttsBkSel.value = prev; } else { const first = ttsBkSel.querySelector('option:not([disabled])'); if (first) ttsBkSel.value = first.value; } } // ── Fetch LLM models ───────────────────────────────────────────────────── async function fetchLlmModels() { if (!llmModelSel) return; const url = llmUrlInp?.value.trim() || ''; llmFetchBtn.disabled = true; try { const d = await fetch('/api/conversation/llm-models' + (url ? '?url=' + encodeURIComponent(url) : '')).then(r => r.json()); const models = d.models || []; llmModelSel.innerHTML = models.length ? models.map(m => ``).join('') : ''; // Default to the global Active LLM model when available (still overridable here) const want = _appSettings && _appSettings.llm_model; if (want && models.includes(want)) llmModelSel.value = want; } catch(e) { llmModelSel.innerHTML = ''; } finally { llmFetchBtn.disabled = false; } } // ── Fetch TTS voices ───────────────────────────────────────────────────── async function fetchConvTtsVoices() { if (!ttsVoiceSel || !ttsBkSel) return; const backend = ttsBkSel.value; if (!backend) return; const prev = ttsVoiceSel.value; ttsFetchBtn.disabled = true; try { const rawVoices = await fetch('/api/tts-voices?backend=' + encodeURIComponent(backend)).then(r => r.json()); let voices = Array.isArray(rawVoices) ? rawVoices : []; if (typeof shouldFilterBackendVoices === 'function' && shouldFilterBackendVoices(backend)) { const activeIds = await activeLibraryVoiceIds(); voices = voices.filter(v => activeIds.has(backendVoiceId(v))); } const ids = voices.map(backendVoiceId).filter(Boolean); if (window.VoicePicker) { VoicePicker.upgrade('conv-tts-voice-select'); VoicePicker.populate('conv-tts-voice-select', ids); if (prev && ids.includes(prev)) VoicePicker.setValue('conv-tts-voice-select', prev); else if (ids.length) VoicePicker.setValue('conv-tts-voice-select', ids[0]); } else { ttsVoiceSel.innerHTML = ids.length ? ids.map(id => ``).join('') : ''; if (prev && ids.includes(prev)) ttsVoiceSel.value = prev; } } catch(e) { ttsVoiceSel.innerHTML = ''; if (window.VoicePicker) { VoicePicker.upgrade('conv-tts-voice-select'); VoicePicker.populate('conv-tts-voice-select', []); } } finally { ttsFetchBtn.disabled = false; } } // ── Chat bubble helpers ────────────────────────────────────────────────── function timeStr() { const now = new Date(); return now.getHours().toString().padStart(2,'0') + ':' + now.getMinutes().toString().padStart(2,'0'); } function removeWelcome() { const w = chatWindow.querySelector('.conv-chat-welcome'); if (w) w.remove(); } function addBubble(role, text) { removeWelcome(); const wrap = document.createElement('div'); wrap.className = `conv-bubble-wrap conv-bubble-wrap--${role}`; const bubble = document.createElement('div'); bubble.className = `conv-bubble conv-bubble--${role}`; bubble.textContent = text || ''; const meta = document.createElement('div'); meta.className = 'conv-bubble-meta'; meta.textContent = timeStr(); wrap.appendChild(bubble); wrap.appendChild(meta); chatWindow.appendChild(wrap); chatWindow.scrollTop = chatWindow.scrollHeight; return bubble; } function addTypingBubble() { removeWelcome(); const wrap = document.createElement('div'); wrap.className = 'conv-bubble-wrap conv-bubble-wrap--assistant'; wrap.id = 'conv-typing-wrap'; const bubble = document.createElement('div'); bubble.className = 'conv-bubble conv-bubble--assistant'; bubble.innerHTML = ''; wrap.appendChild(bubble); chatWindow.appendChild(wrap); chatWindow.scrollTop = chatWindow.scrollHeight; return bubble; } // Lazily builds the thinking-panel + answer-text structure inside an // assistant bubble the first time either a 'thinking' or 'token' event // arrives, replacing the "..." typing dots. function ensureAssistantParts(bubble) { if (bubble.querySelector('.conv-answer-text')) return; bubble.innerHTML = '' + ''; bubble.querySelector('.conv-think-toggle').addEventListener('click', function () { const open = this.getAttribute('aria-expanded') === 'true'; this.setAttribute('aria-expanded', String(!open)); bubble.querySelector('.conv-think-body').hidden = open; }); } function appendThinking(bubble, delta) { ensureAssistantParts(bubble); const wrap = bubble.querySelector('.conv-think-wrap'); const toggle = bubble.querySelector('.conv-think-toggle'); const body = bubble.querySelector('.conv-think-body'); wrap.hidden = false; body.textContent += delta; // Auto-open the panel the first time reasoning arrives so the user can see // the model thinking live. After that, preserve the user's manual toggle. if (!bubble.dataset.thinkingSeen) { bubble.dataset.thinkingSeen = '1'; toggle.setAttribute('aria-expanded', 'true'); body.hidden = false; } } function setAnswerText(bubble, text) { ensureAssistantParts(bubble); bubble.dataset.answerStarted = '1'; bubble.querySelector('.conv-answer-text').textContent = text; } function addErrorBubble(msg) { removeWelcome(); const wrap = document.createElement('div'); wrap.className = 'conv-bubble-wrap conv-bubble-wrap--assistant'; const bubble = document.createElement('div'); bubble.className = 'conv-bubble conv-bubble--error'; bubble.innerHTML = ` ${escHtml(msg)}`; wrap.appendChild(bubble); chatWindow.appendChild(wrap); chatWindow.scrollTop = chatWindow.scrollHeight; } // ── Stats panel ────────────────────────────────────────────────────────── function fmtMs(ms) { return ms == null ? '—' : ms >= 1000 ? (ms/1000).toFixed(2)+'s' : ms+'ms'; } function updateStatBar(id, val, maxVal) { const fill = $(id); if (fill) fill.style.width = maxVal > 0 ? Math.min(100, (val / maxVal) * 100) + '%' : '0%'; } function updateStats(stats) { const { stt_ms, llm_ttft_ms, llm_total_ms, tts_ms, total_ms } = stats; const max = total_ms || 1; const set = (valId, fillId, ms) => { const el = $(valId); if (el) el.textContent = fmtMs(ms); updateStatBar(fillId, ms || 0, max); }; set('cpv-stt', 'cpf-stt', stt_ms); set('cpv-ttft', 'cpf-ttft', llm_ttft_ms); set('cpv-llm', 'cpf-llm', llm_total_ms); set('cpv-tts', 'cpf-tts', tts_ms); set('cpv-total', 'cpf-total', total_ms); } function addHistoryItem(n, totalMs, ok) { const empty = turnHistory?.querySelector('.conv-history-empty'); if (empty) empty.remove(); const item = document.createElement('div'); item.className = 'conv-hist-item'; const cls = ok ? 'conv-hist-ok' : 'conv-hist-err'; const icon = ok ? 'mdi-check-circle-outline' : 'mdi-alert-outline'; item.innerHTML = `#${n} ${fmtMs(totalMs)}`; turnHistory.insertBefore(item, turnHistory.firstChild); } // ── Recording ──────────────────────────────────────────────────────────── function startRecTimer() { recStart = Date.now(); recTimerInterval = setInterval(() => { const s = Math.floor((Date.now() - recStart) / 1000); if (micTimer) micTimer.textContent = s + 's'; }, 500); } function stopRecTimer() { clearInterval(recTimerInterval); if (micTimer) micTimer.textContent = ''; } // ── Audio queue — plays multi-chunk TTS responses sequentially ────────────── function clearAudio() { if (convCurrentAudio) { try { convCurrentAudio.pause(); } catch(_){} convCurrentAudio = null; } audioQueue.forEach(item => { if (item?.url) try { URL.revokeObjectURL(item.url); } catch(_){} }); audioQueue.length = 0; audioQueuePlaying = false; audioQueueDrainCb = null; stopInterruptMonitor(); } function agentBusy() { return !!(isProcessing || audioQueuePlaying || audioQueue.length || convCurrentAudio || convAbortCtrl); } function isRecording() { return !!(mediaRecorder && mediaRecorder.state === 'recording'); } function setLiveMicUi(statusText) { micBtn.classList.toggle('live', liveMicEnabled); micBtn.title = liveMicEnabled ? 'Microphone is always listening - click to turn off' : 'Click to turn live microphone on'; if (!isRecording() && !isProcessing) micIcon.className = 'mdi mdi-microphone'; if (micStatus && statusText) micStatus.textContent = statusText; } function resetConversationInput(statusText = 'Ready') { liveInterimText = ''; previewTranscribing = false; isProcessing = false; micBtn.classList.remove('processing', 'recording'); micIcon.className = 'mdi mdi-microphone'; setLiveMicUi(); if (sendBtn) sendBtn.disabled = false; if (textInput) { textInput.disabled = false; textInput.readOnly = false; textInput.classList.remove('listening'); textInput.placeholder = origPlaceholder; textInput.value = ''; } if (micStatus) micStatus.textContent = statusText; } function abortCurrentTurn() { convTurnGeneration++; if (convAbortCtrl) { try { convAbortCtrl.abort(); } catch(_){} } convAbortCtrl = null; } async function interruptAgentAndListen() { liveMicEnabled = true; autoMicGeneration++; abortCurrentTurn(); clearAudio(); resetConversationInput('Interrupted - listening...'); await startRecording(); } function stopInterruptMonitor() { cancelAnimationFrame(interruptRafId); interruptRafId = null; if (interruptCtx) { try { interruptCtx.close(); } catch(_){} interruptCtx = null; } if (interruptStream) { interruptStream.getTracks().forEach(t => t.stop()); interruptStream = null; } interruptSpeechStart = 0; } async function startInterruptMonitor() { if (interruptCtx || !navigator.mediaDevices?.getUserMedia) return; try { interruptStream = await navigator.mediaDevices.getUserMedia({ audio: true }); interruptCtx = new (window.AudioContext || window.webkitAudioContext)(); await interruptCtx.resume(); // may be suspended when created outside a user gesture const src = interruptCtx.createMediaStreamSource(interruptStream); const analyser = interruptCtx.createAnalyser(); analyser.fftSize = 256; src.connect(analyser); const buf = new Float32Array(analyser.fftSize); function tick() { // Keep listening during live-agent processing too, not only while audio is playing. const processingTurn = (liveAgentOn() || liveMicEnabled) && (isProcessing || !!convAbortCtrl); if (!audioQueuePlaying && !audioQueue.length && !convCurrentAudio && !processingTurn) { stopInterruptMonitor(); return; } analyser.getFloatTimeDomainData(buf); let rms = 0; for (const s of buf) rms += s * s; rms = Math.sqrt(rms / buf.length); if (rms > INTERRUPT_THRESHOLD) { if (!interruptSpeechStart) interruptSpeechStart = Date.now(); if (Date.now() - interruptSpeechStart >= interruptHoldMs()) { // User is talking - interrupt the AI and listen immediately. interruptAgentAndListen().catch(() => {}); return; } } else { interruptSpeechStart = 0; } interruptRafId = requestAnimationFrame(tick); } interruptRafId = requestAnimationFrame(tick); } catch (_) { stopInterruptMonitor(); } } function playNextAudio() { if (!audioQueue.length) { audioQueuePlaying = false; convCurrentAudio = null; convCurrentSentenceBubble = null; if (audioQueueDrainCb) { const cb = audioQueueDrainCb; audioQueueDrainCb = null; setTimeout(cb, 150); } else if (micStatus && !isProcessing) { micStatus.textContent = 'Ready'; } return; } audioQueuePlaying = true; const item = audioQueue.shift(); // {url, text} // Show the sentence text in the typing bubble if no answer text has landed yet if (convCurrentSentenceBubble && item.text) { const answerSpan = convCurrentSentenceBubble.querySelector('.conv-answer-text'); if (convCurrentSentenceBubble.querySelector('.conv-typing') || (answerSpan && !answerSpan.textContent)) { setAnswerText(convCurrentSentenceBubble, item.text); } } const el = new Audio(item.url); convCurrentAudio = el; el.addEventListener('ended', () => { URL.revokeObjectURL(item.url); playNextAudio(); }); el.play().catch(() => { URL.revokeObjectURL(item.url); playNextAudio(); }); if (micStatus) micStatus.textContent = 'Speaking…'; startInterruptMonitor(); } function enqueueAudio(b64, mime, text) { const binStr = atob(b64); const arr = new Uint8Array(binStr.length); for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i); const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' })); audioQueue.push({ url, text: text || '' }); if (!audioQueuePlaying) playNextAudio(); } // ── Chunked Whisper preview (fallback for browsers without SpeechRecognition) ── async function transcribeForPreview() { if (previewTranscribing || !recChunks.length) return; // When VAD is on, gate on detected speech to avoid transcribing silence. // When VAD is off the user controls recording manually — always transcribe. if (vadToggle?.checked && (!vadHadSpeech || Date.now() - vadLastVoiceMs > 5000)) return; previewTranscribing = true; try { const mime = (mediaRecorder && mediaRecorder.mimeType) || 'audio/webm'; const ext = mime.includes('ogg') ? '.ogg' : '.webm'; const blob = new Blob(recChunks, { type: mime }); const fd = new FormData(); fd.append('file', new File([blob], 'preview' + ext, { type: mime })); fd.append('backend', sttSel?.value || 'configured'); const r = await fetch('/api/transcribe-bytes', { method: 'POST', body: fd }); if (r.ok) { const d = await r.json(); const txt = (d.text || '').trim(); if (txt && txt.length > 3 && !HALLUCINATION_RE.test(txt) && mediaRecorder && mediaRecorder.state === 'recording') { liveInterimText = txt; if (textInput) textInput.value = txt; } } } catch (_) {} finally { previewTranscribing = false; } } // ── Auto-restart mic after agent finishes speaking ─────────────────────── function scheduleAutoMic() { if (!liveMicEnabled && !handsFreeToggle?.checked) return; const gen = ++autoMicGeneration; function tryStart() { if (gen !== autoMicGeneration || isProcessing) return; if (!liveMicEnabled && !handsFreeToggle?.checked) return; startRecording().catch(() => {}); } if (audioQueuePlaying || audioQueue.length > 0) { // Audio still queued — trigger after queue drains audioQueueDrainCb = tryStart; } else if (convCurrentAudio && !convCurrentAudio.ended) { convCurrentAudio.addEventListener('ended', () => setTimeout(tryStart, 150), { once: true }); } else { setTimeout(tryStart, 300); } } async function startRecording() { if (isProcessing) return; stopInterruptMonitor(); // release mic stream before opening a recording stream if (!navigator.mediaDevices?.getUserMedia) { toast('Microphone unavailable — browser requires a secure context (HTTPS or localhost). ' + 'Access the app via http://localhost:7890 or enable it in chrome://flags/#unsafely-treat-insecure-origin-as-secure', 'error', 8000); return; } let stream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); } catch(e) { toast('Microphone access denied: ' + e.message, 'error'); return; } liveInterimText = ''; previewTranscribing = false; vadHadSpeech = false; vadLastVoiceMs = 0; vadSpeechStartMs = 0; cancelNextBlob = false; recChunks = []; mediaRecorder = new MediaRecorder(stream); mediaRecorder.ondataavailable = e => { if (e.data.size > 0) { recChunks.push(e.data); // Only run preview when speech was detected (prevents "reich" on silence chunks) if (mediaRecorder.state === 'recording' && vadHadSpeech) transcribeForPreview(); } }; mediaRecorder.onstop = () => { stream.getTracks().forEach(t => t.stop()); if (cancelNextBlob || cancelRecordingOnStop) { // VAD fired with no speech, or the user turned the hot mic off. cancelNextBlob = false; cancelRecordingOnStop = false; if (textInput) { textInput.readOnly = false; textInput.classList.remove('listening'); textInput.placeholder = origPlaceholder; textInput.value = ''; } if (sendBtn) sendBtn.disabled = false; micBtn.classList.remove('processing', 'recording'); micIcon.className = 'mdi mdi-microphone'; if (micStatus) micStatus.textContent = liveMicEnabled ? 'Listening…' : 'Ready'; isProcessing = false; setLiveMicUi(); if (liveMicEnabled) startRecording().catch(() => {}); return; } const blob = new Blob(recChunks, { type: mediaRecorder.mimeType || 'audio/webm' }); processBlob(blob); }; // timeslice: 750ms when VAD is off (manual recording) for faster Whisper preview; // 1500ms with VAD on (chunks are gated anyway, smaller slices waste CPU). mediaRecorder.start(vadToggle?.checked ? (liveAgentOn() ? 900 : 1500) : 750); micBtn.classList.add('recording'); setLiveMicUi(); micIcon.className = liveMicEnabled ? 'mdi mdi-microphone' : 'mdi mdi-stop'; if (micStatus) micStatus.textContent = liveMicEnabled ? 'Listening…' : 'Recording…'; startRecTimer(); // Set input to listening mode (read-only; Whisper preview text will appear here) if (textInput) { textInput.readOnly = true; textInput.value = ''; textInput.placeholder = 'Listening…'; textInput.classList.add('listening'); } if (sendBtn) sendBtn.disabled = true; // Also try Web Speech API for faster interim results (works on HTTPS / localhost) const SpeechRec = window.SpeechRecognition || window.webkitSpeechRecognition; if (SpeechRec) { try { speechRec = new SpeechRec(); speechRec.continuous = true; speechRec.interimResults = true; speechRec.onresult = e => { let final = '', interim = ''; for (let i = 0; i < e.results.length; i++) { if (e.results[i].isFinal) final += e.results[i][0].transcript; else interim += e.results[i][0].transcript; } liveInterimText = final + interim; if (textInput) textInput.value = liveInterimText; }; speechRec.onerror = () => { speechRec = null; }; speechRec.start(); } catch(_) { speechRec = null; } } // VAD: auto-stop on silence using AudioContext if (vadToggle?.checked) { try { vadAudioCtx = new AudioContext(); const src = vadAudioCtx.createMediaStreamSource(stream); const analyser = vadAudioCtx.createAnalyser(); analyser.fftSize = 1024; src.connect(analyser); const vadBuf = new Float32Array(analyser.fftSize); let vadLastVoice = Date.now(); const levelWrap = $('conv-level-wrap'); const levelFill = $('conv-level-fill'); if (levelWrap) levelWrap.classList.add('active'); function vadTick() { if (!mediaRecorder || mediaRecorder.state !== 'recording') return; analyser.getFloatTimeDomainData(vadBuf); let rms = 0; for (const s of vadBuf) rms += s * s; rms = Math.sqrt(rms / vadBuf.length); const thr = vadThreshold(); if (levelFill) levelFill.style.width = Math.min(100, rms / thr * 40) + '%'; if (rms > thr) { const now = Date.now(); if (!vadSpeechStartMs) vadSpeechStartMs = now; vadLastVoice = now; vadLastVoiceMs = now; // Only mark as valid speech once it persists long enough if (!vadHadSpeech && (now - vadSpeechStartMs) >= MIN_SPEECH_FRAMES_MS) vadHadSpeech = true; } else { vadSpeechStartMs = 0; // reset burst timer on silence } const elapsed = Date.now() - recStart; const silence = Date.now() - vadLastVoice; if (elapsed > VAD_MIN_REC_MS) { const silenceLimit = vadSilenceMs(); const remaining = silenceLimit - silence; if (micStatus) { micStatus.textContent = liveMicEnabled && !vadHadSpeech ? 'Listening…' : remaining < silenceLimit * 0.7 ? `Sending in ${(Math.max(0, remaining) / 1000).toFixed(1)}s…` : (liveMicEnabled ? 'Listening…' : 'Recording…'); } if (silence >= silenceLimit) { if (!vadHadSpeech) { // Silence with no speech → cancel this segment (don't call STT). // In hot-mic mode onstop restarts recording with a FRESH // MediaRecorder, so every utterance keeps its own webm/EBML // header and stays decodable. (Clearing recChunks in place would // drop the header chunk → "EBML header parsing failed".) cancelNextBlob = true; if (micStatus) micStatus.textContent = liveMicEnabled ? 'Listening…' : 'Ready'; } stopRecording(); return; } } vadRafId = requestAnimationFrame(vadTick); } vadRafId = requestAnimationFrame(vadTick); } catch(_) { vadAudioCtx = null; } } } function stopRecording() { if (!mediaRecorder || mediaRecorder.state === 'inactive') return; // Clean up VAD cancelAnimationFrame(vadRafId); vadRafId = null; if (vadAudioCtx) { try { vadAudioCtx.close(); } catch(_){} vadAudioCtx = null; } const levelWrap = $('conv-level-wrap'); const levelFill = $('conv-level-fill'); if (levelWrap) levelWrap.classList.remove('active'); if (levelFill) levelFill.style.width = '0%'; // Clean up live speech recognition if (speechRec) { try { speechRec.stop(); } catch(_){} speechRec = null; } previewTranscribing = false; mediaRecorder.stop(); stopRecTimer(); micBtn.classList.remove('recording'); micBtn.classList.add('processing'); micIcon.className = 'mdi mdi-dots-horizontal'; if (micStatus) micStatus.textContent = 'Processing…'; isProcessing = true; } // ── Send turn via SSE ───────────────────────────────────────────────────── // Decode a recorded blob (webm/opus, ogg, …) in the browser and re-encode it // as a 16 kHz mono WAV. This sidesteps server-side ffmpeg, whose webm/EBML // parser fails on some platforms (ARM64): "EBML header parsing failed". // Returns a WAV Blob, or null if decoding isn't possible (caller falls back). async function blobToWav16k(blob) { try { if (!blob || !blob.size) return null; const AC = window.AudioContext || window.webkitAudioContext; if (!AC) return null; const buf = await blob.arrayBuffer(); const ctx = new AC(); let audio; try { audio = await ctx.decodeAudioData(buf.slice(0)); } finally { try { ctx.close(); } catch (_) {} } if (!audio || !audio.length) return null; const targetRate = 16000; const inRate = audio.sampleRate; const chs = audio.numberOfChannels; // Mix down to mono const mono = new Float32Array(audio.length); for (let c = 0; c < chs; c++) { const d = audio.getChannelData(c); for (let i = 0; i < d.length; i++) mono[i] += d[i] / chs; } // Linear resample to 16 kHz const outLen = Math.max(1, Math.round(mono.length * targetRate / inRate)); const out = new Float32Array(outLen); const ratio = mono.length / outLen; for (let i = 0; i < outLen; i++) { const pos = i * ratio, i0 = Math.floor(pos), i1 = Math.min(i0 + 1, mono.length - 1); const f = pos - i0; out[i] = mono[i0] * (1 - f) + mono[i1] * f; } // Encode 16-bit PCM WAV const bytes = 44 + out.length * 2; const ab = new ArrayBuffer(bytes); const view = new DataView(ab); const ws = (off, s) => { for (let i = 0; i < s.length; i++) view.setUint8(off + i, s.charCodeAt(i)); }; ws(0, 'RIFF'); view.setUint32(4, bytes - 8, true); ws(8, 'WAVE'); ws(12, 'fmt '); view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true); view.setUint32(24, targetRate, true); view.setUint32(28, targetRate * 2, true); view.setUint16(32, 2, true); view.setUint16(34, 16, true); ws(36, 'data'); view.setUint32(40, out.length * 2, true); let off = 44; for (let i = 0; i < out.length; i++) { let s = Math.max(-1, Math.min(1, out[i])); view.setInt16(off, s < 0 ? s * 0x8000 : s * 0x7FFF, true); off += 2; } return new Blob([ab], { type: 'audio/wav' }); } catch (_) { return null; } } async function processBlob(blob) { autoMicGeneration++; // cancel any pending auto-mic from previous turn clearAudio(); turnCount++; const turnN = turnCount; const t0 = Date.now(); // Transition input from listening/readOnly state to processing/disabled state if (textInput) { textInput.readOnly = false; textInput.classList.remove('listening'); textInput.placeholder = origPlaceholder; textInput.value = ''; textInput.disabled = true; } if (sendBtn) sendBtn.disabled = true; // Show user bubble seeded with live interim transcript (if available) const userBubble = addBubble('user', liveInterimText || '…'); const assistantBubble = addTypingBubble(); convCurrentSentenceBubble = assistantBubble; // sentence text will appear here while audio plays let assistantText = ''; let lastStats = null; // Convert to WAV in-browser so the server never has to ffmpeg-decode webm. const wav = await blobToWav16k(blob); const upBlob = wav || blob; const upName = wav ? 'audio.wav' : 'audio.webm'; const form = new FormData(); form.append('audio', upBlob, upName); form.append('stt_backend', sttSel?.value || 'configured'); form.append('llm_url', llmUrlInp?.value.trim() || ''); form.append('llm_model', llmModelSel?.value || ''); form.append('tts_backend', ttsBkSel?.value || 'voice_clone'); form.append('tts_voice', ttsVoiceSel?.value || ''); form.append('tts_emotion', ttsEmotionSel?.value || ''); form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.'); form.append('history', JSON.stringify(conversationHistory.slice(-20))); const turnGen = ++convTurnGeneration; if (convAbortCtrl) { try { convAbortCtrl.abort(); } catch(_){} } convAbortCtrl = new AbortController(); if (liveMicEnabled || liveAgentOn()) startInterruptMonitor(); try { const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form, signal: convAbortCtrl.signal }); if (!resp.ok) { let _d=''; try { const _j=await resp.json(); _d=JSON.stringify(_j.detail||_j); } catch(_){} throw new Error('Server error ' + resp.status + (_d ? ': ' + _d : '')); } const reader = resp.body.getReader(); const dec = new TextDecoder(); let buf = ''; while (true) { const { done, value } = await reader.read(); if (turnGen !== convTurnGeneration) return; if (done) break; buf += dec.decode(value, { stream: true }); const lines = buf.split('\n'); buf = lines.pop(); for (const line of lines) { if (turnGen !== convTurnGeneration) return; if (!line.startsWith('data:')) continue; let evt; try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; } if (evt.type === 'transcript') { userBubble.textContent = evt.text || '(empty)'; if (micStatus) micStatus.textContent = 'Generating reply…'; } else if (evt.type === 'thinking') { appendThinking(assistantBubble, evt.delta); } else if (evt.type === 'token') { convCurrentSentenceBubble = null; // LLM tokens take over the bubble now assistantText += evt.delta; setAnswerText(assistantBubble, assistantText); chatWindow.scrollTop = chatWindow.scrollHeight; } else if (evt.type === 'llm_done') { assistantText = evt.text || assistantText; setAnswerText(assistantBubble, assistantText); convCurrentSentenceBubble = null; if (micStatus) micStatus.textContent = 'Synthesising speech…'; } else if (evt.type === 'audio') { enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || ''); } else if (evt.type === 'stats') { lastStats = evt; updateStats(evt); } else if (evt.type === 'done') { conversationHistory.push({ role: 'user', content: userBubble.textContent }); conversationHistory.push({ role: 'assistant', content: assistantText }); addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true); scheduleAutoMic(); // triggers after audio queue drains } else if (evt.type === 'error') { clearAudio(); const wrap = assistantBubble.closest('.conv-bubble-wrap'); if (wrap) wrap.remove(); const stage = evt.stage?.toUpperCase() || 'ERR'; let msg = evt.message || 'Unknown error'; const detailMatch = msg.match(/HTTP \d+:\s*(.+)/s); if (detailMatch) msg = detailMatch[1].trim(); if (msg.length > 300) msg = msg.slice(0, 300) + '…'; addErrorBubble(`[${stage}] ${msg}`); addHistoryItem(turnN, Date.now() - t0, false); if (micStatus) micStatus.textContent = 'Error — ready'; } } } } catch(e) { if (e?.name !== 'AbortError' && turnGen === convTurnGeneration) { clearAudio(); const wrap = assistantBubble.closest('.conv-bubble-wrap'); if (wrap) wrap.remove(); addErrorBubble(e.message); addHistoryItem(turnN, Date.now() - t0, false); if (micStatus) micStatus.textContent = 'Error — ready'; } } finally { if (turnGen === convTurnGeneration) { convAbortCtrl = null; liveInterimText = ''; previewTranscribing = false; isProcessing = false; micBtn.classList.remove('processing'); micIcon.className = 'mdi mdi-microphone'; setLiveMicUi(); if (sendBtn) sendBtn.disabled = false; if (textInput) { textInput.disabled = false; textInput.value = ''; } if (audioQueuePlaying || audioQueue.length || convCurrentAudio) startInterruptMonitor(); else if (liveMicEnabled) scheduleAutoMic(); } } } // ── Text-input turn (skips STT, sends text directly) ──────────────────── async function processText(text) { text = text.trim(); if (!text || isProcessing) return; autoMicGeneration++; // cancel any pending auto-mic clearAudio(); isProcessing = true; if (sendBtn) sendBtn.disabled = true; if (textInput) { textInput.disabled = true; textInput.value = ''; } micBtn.classList.add('processing'); micIcon.className = 'mdi mdi-dots-horizontal'; if (micStatus) micStatus.textContent = 'Processing…'; turnCount++; const turnN = turnCount; const t0 = Date.now(); const userBubble = addBubble('user', text); const assistantBubble = addTypingBubble(); convCurrentSentenceBubble = assistantBubble; let assistantText = ''; let lastStats = null; const form = new FormData(); form.append('text', text); form.append('stt_backend', sttSel?.value || 'configured'); form.append('llm_url', llmUrlInp?.value.trim() || ''); form.append('llm_model', llmModelSel?.value || ''); form.append('tts_backend', ttsBkSel?.value || 'voice_clone'); form.append('tts_voice', ttsVoiceSel?.value || ''); form.append('tts_emotion', ttsEmotionSel?.value || ''); form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.'); form.append('history', JSON.stringify(conversationHistory.slice(-20))); const turnGen = ++convTurnGeneration; if (convAbortCtrl) { try { convAbortCtrl.abort(); } catch(_){} } convAbortCtrl = new AbortController(); if (liveMicEnabled || liveAgentOn()) startInterruptMonitor(); try { const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form, signal: convAbortCtrl.signal }); if (!resp.ok) { let _d=''; try { const _j=await resp.json(); _d=JSON.stringify(_j.detail||_j); } catch(_){} throw new Error('Server error ' + resp.status + (_d ? ': ' + _d : '')); } const reader = resp.body.getReader(); const dec = new TextDecoder(); let buf = ''; while (true) { const { done, value } = await reader.read(); if (turnGen !== convTurnGeneration) return; if (done) break; buf += dec.decode(value, { stream: true }); const lines = buf.split('\n'); buf = lines.pop(); for (const line of lines) { if (turnGen !== convTurnGeneration) return; if (!line.startsWith('data:')) continue; let evt; try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; } if (evt.type === 'transcript') { if (micStatus) micStatus.textContent = 'Generating reply…'; } else if (evt.type === 'thinking') { appendThinking(assistantBubble, evt.delta); } else if (evt.type === 'token') { convCurrentSentenceBubble = null; assistantText += evt.delta; setAnswerText(assistantBubble, assistantText); chatWindow.scrollTop = chatWindow.scrollHeight; } else if (evt.type === 'llm_done') { assistantText = evt.text || assistantText; setAnswerText(assistantBubble, assistantText); convCurrentSentenceBubble = null; if (micStatus) micStatus.textContent = 'Synthesising speech…'; } else if (evt.type === 'audio') { enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || ''); } else if (evt.type === 'stats') { lastStats = evt; updateStats(evt); } else if (evt.type === 'done') { conversationHistory.push({ role: 'user', content: text }); conversationHistory.push({ role: 'assistant', content: assistantText }); addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true); scheduleAutoMic(); // triggers after audio queue drains } else if (evt.type === 'error') { clearAudio(); const wrap = assistantBubble.closest('.conv-bubble-wrap'); if (wrap) wrap.remove(); addErrorBubble(`[${evt.stage?.toUpperCase() || 'ERR'}] ${evt.message || 'Unknown error'}`); addHistoryItem(turnN, Date.now() - t0, false); if (micStatus) micStatus.textContent = 'Error — ready'; } } } } catch(e) { if (e?.name !== 'AbortError' && turnGen === convTurnGeneration) { clearAudio(); const wrap = assistantBubble.closest('.conv-bubble-wrap'); if (wrap) wrap.remove(); addErrorBubble(e.message); addHistoryItem(turnN, Date.now() - t0, false); if (micStatus) micStatus.textContent = 'Error — ready'; } } finally { if (turnGen === convTurnGeneration) { convAbortCtrl = null; isProcessing = false; micBtn.classList.remove('processing'); micIcon.className = 'mdi mdi-microphone'; setLiveMicUi(); if (sendBtn) sendBtn.disabled = false; if (textInput) textInput.disabled = false; if (audioQueuePlaying || audioQueue.length || convCurrentAudio) startInterruptMonitor(); else if (liveMicEnabled) scheduleAutoMic(); } } } async function enableLiveMic() { liveMicEnabled = true; autoMicGeneration++; if (vadToggle) vadToggle.checked = true; if (handsFreeToggle) handsFreeToggle.checked = true; if (liveAgentToggle) liveAgentToggle.checked = true; setLiveMicUi('Listening…'); if (isRecording()) return; if (agentBusy()) await interruptAgentAndListen(); else await startRecording(); } function disableLiveMic() { liveMicEnabled = false; if (handsFreeToggle) handsFreeToggle.checked = false; autoMicGeneration++; abortCurrentTurn(); clearAudio(); if (isRecording()) { cancelRecordingOnStop = true; stopRecording(); } else { resetConversationInput('Mic off'); } setLiveMicUi('Mic off'); } // ── Wire up events ─────────────────────────────────────────────────────── micBtn.addEventListener('click', () => { if (liveMicEnabled) disableLiveMic(); else enableLiveMic().catch(() => {}); }); // Text input — Enter key or Send button sendBtn?.addEventListener('click', () => processText(textInput?.value || '')); textInput?.addEventListener('keydown', e => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); processText(textInput.value); } }); clearBtn?.addEventListener('click', () => { conversationHistory = []; turnCount = 0; chatWindow.innerHTML = '

Type a message or press the microphone button below to start.

'; if (turnHistory) turnHistory.innerHTML = '
No turns yet.
'; ['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; }); ['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; }); }); llmFetchBtn?.addEventListener('click', fetchLlmModels); ttsFetchBtn?.addEventListener('click', fetchConvTtsVoices); // Re-populate TTS when backend changes ttsBkSel?.addEventListener('change', () => { ttsVoiceSel.innerHTML = ''; if (window.VoicePicker) { VoicePicker.upgrade('conv-tts-voice-select'); VoicePicker.populate('conv-tts-voice-select', []); } }); liveAgentToggle?.addEventListener('change', () => { if (liveAgentToggle.checked) { if (vadToggle) vadToggle.checked = true; if (handsFreeToggle) handsFreeToggle.checked = true; toast('Live agent mode on: faster silence cut-off and barge-in', 'success'); } else { toast('Live agent mode off', 'info'); } }); // ── Init ───────────────────────────────────────────────────────────────── loadConvSttBackends(); populateConvTtsBackends(); if (window.VoicePicker) VoicePicker.upgrade('conv-tts-voice-select'); setLiveMicUi(); // Keep TTS backend select in sync after global backend refresh window._ttsRefreshHooks = window._ttsRefreshHooks || []; window._ttsRefreshHooks.push(populateConvTtsBackends); // ── Stats panel collapse ───────────────────────────────────────────────── (function () { const panel = document.getElementById('conv-stats-panel'); const colBtn = document.getElementById('conv-stats-collapse-btn'); const expBtn = document.getElementById('conv-stats-expand-btn'); const KEY = 'ttsvc_conv_stats_collapsed'; function setCollapsed(on) { if (!panel) return; panel.classList.toggle('collapsed', on); if (expBtn) expBtn.style.display = on ? 'flex' : 'none'; try { localStorage.setItem(KEY, on ? '1' : '0'); } catch (_) {} } if (colBtn) colBtn.addEventListener('click', function () { setCollapsed(true); }); if (expBtn) expBtn.addEventListener('click', function () { setCollapsed(false); }); try { if (localStorage.getItem(KEY) === '1') setCollapsed(true); } catch (_) {} })(); })();