tts-voice-creator-clone-and.../static/js/conversation.js
mARTin-B78 40e42590cc Release v1.6.0: a11y (WCAG AA), i18n (DE), PWA, perf, tests, Cast UX
Cast: card/list views, sort & filter, online voice picker, "Hear a line"
sample button, AI character notes, import auto-save.

Platform: WCAG 2.1 AA accessibility pass; German UI translation + language
picker; installable PWA with offline shell; GZip + content-visibility
virtualization + lazy images + Rehearser PCM memory cap (mobile stability);
Playwright suite (desktop + iPhone); opt-in minified bundle build.

Fixes: screenplay parser false characters; Fish-Speech inline-tag tones;
narrator/voice pickers list full library; clone GUI rework; fish.audio
import dedup; voice-ID rename; bulk-delete modal.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-03 14:23:35 +02:00

978 lines
42 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// ── Settings: Logs ────────────────────────────────────────────────────────
let _logsLiveTimer = null;
let _logsActiveLevel = '';
function escLog(s) { return String(s).replace(/[&<>]/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;'}[c])); }
window.loadSettingsLogs = async function() {
const viewer = $('s-log-viewer');
if (!viewer) return;
try {
const data = await fetch('/api/logs?limit=300').then(r => r.json());
const items = (data.items || []).filter(item =>
!_logsActiveLevel || item.level === _logsActiveLevel
);
const count = $('s-log-count');
if (count) count.textContent = items.length + ' entries';
if (!items.length) {
viewer.innerHTML = '<div class="s-log-empty">No log entries</div>';
return;
}
viewer.innerHTML = items.map(item => {
const ts = item.ts ? item.ts.replace('T', ' ').replace(/\.\d+Z$/, ' UTC') : '';
return `<div class="s-log-row">
<span class="s-log-ts">${escLog(ts)}</span>
<span class="s-log-level ${escLog(item.level)}">${escLog(item.level)}</span>
<span class="s-log-msg">${escLog(item.msg)}</span>
</div>`;
}).join('');
} catch(e) {
viewer.innerHTML = `<div class="s-log-empty">Failed to load logs: ${escLog(e.message)}</div>`;
}
};
$('s-logs-refresh-btn')?.addEventListener('click', () => loadSettingsLogs());
$('s-logs-clear-btn')?.addEventListener('click', async () => {
await fetch('/api/logs', { method: 'DELETE' });
const viewer = $('s-log-viewer');
if (viewer) viewer.innerHTML = '<div class="s-log-empty">Logs cleared</div>';
const count = $('s-log-count');
if (count) count.textContent = '';
});
$('s-logs-live-toggle')?.addEventListener('change', function() {
clearInterval(_logsLiveTimer);
if (this.checked) {
loadSettingsLogs();
_logsLiveTimer = setInterval(loadSettingsLogs, 3000);
}
});
document.querySelectorAll('.s-log-filter').forEach(btn => {
btn.addEventListener('click', function() {
document.querySelectorAll('.s-log-filter').forEach(b => b.classList.remove('is-active'));
this.classList.add('is-active');
_logsActiveLevel = this.dataset.logLevel;
loadSettingsLogs();
});
});
// ── Settings: About ───────────────────────────────────────────────────────
// Fetch and display the server version in the About page
(async function loadAppVersion() {
try {
const d = await fetch('/api/version').then(r => r.json());
const el = $('s-about-version');
if (el && d.version) el.textContent = 'v' + d.version;
} catch (_) {}
})();
// Lazy-load changelog when the details element is opened
$('about-changelog-details')?.addEventListener('toggle', async function () {
if (!this.open) return;
const content = $('about-changelog-content');
const status = $('about-changelog-status');
if (!content || content.textContent.trim()) return;
if (status) status.textContent = 'Loading…';
try {
const text = await fetch('/api/changelog').then(r => r.ok ? r.text() : Promise.reject(r.status));
content.textContent = text;
if (status) status.textContent = '';
} catch(e) {
content.textContent = 'Could not load changelog: ' + e;
if (status) status.textContent = 'error';
}
});
function renderSettingsAbout() {
const el = $('s-about-backends');
if (!el) return;
const available = new Set((_ttsBackends || []).filter(b => b.available).map(b => b.id));
const all = (_ttsBackends || []);
if (!all.length) { el.innerHTML = ''; return; }
el.innerHTML = all.map(b => {
const online = available.has(b.id);
return `<span class="s-about-backend ${online ? 'online' : 'offline'}">
<span class="mdi mdi-speaker-outline"></span>${escHtml(b.label)}
<span class="mdi ${online ? 'mdi-check-circle-outline' : 'mdi-close-circle-outline'}"></span>
</span>`;
}).join('');
}
// ── Voices import ──────────────────────────────────────────────────────────
$('s-import-voices-file')?.addEventListener('change', async function () {
const file = this.files?.[0];
if (!file) return;
const st = $('s-import-status');
if (st) st.textContent = 'Uploading…';
try {
const fd = new FormData();
fd.append('file', file);
const r = await fetch('/api/voices/import', { method:'POST', body: fd });
if (!r.ok) { const e = await r.json().catch(()=>({})); throw new Error(e.detail || r.statusText); }
const d = await r.json();
if (st) st.textContent = `Imported ${d.imported} files.`;
toast(`Imported ${d.imported} voice files`, 'success');
this.value = '';
} catch(e) {
if (st) st.textContent = 'Import failed.';
toast('Import failed: ' + e.message, 'error');
}
});
// ── Conversation Playground ────────────────────────────────────────────────
(function initConversationPlayground() {
const chatWindow = $('conv-chat-window');
const micBtn = $('conv-mic-btn');
const micIcon = $('conv-mic-icon');
const micStatus = $('conv-mic-status');
const micTimer = $('conv-mic-timer');
const textInput = $('conv-text-input');
const sendBtn = $('conv-send-btn');
const clearBtn = $('conv-clear-btn');
const sttSel = $('conv-stt-select');
const llmUrlInp = $('conv-llm-url');
const llmFetchBtn = $('conv-llm-fetch-btn');
const llmModelSel = $('conv-llm-model-select');
const ttsBkSel = $('conv-tts-backend-select');
const ttsFetchBtn = $('conv-tts-fetch-btn');
const ttsVoiceSel = $('conv-tts-voice-select');
const systemPrompt = $('conv-system-prompt');
const turnHistory = $('conv-turn-history');
if (!chatWindow || !micBtn) return;
// Warn if microphone API is unavailable (HTTP on non-localhost = insecure context)
if (!navigator.mediaDevices?.getUserMedia) {
if (micStatus) {
micStatus.textContent = 'Mic unavailable — insecure context';
micStatus.style.color = 'var(--red, #e05)';
}
if (micBtn) {
micBtn.disabled = true;
micBtn.title = 'Browser blocks microphone on HTTP. Use http://localhost:7890 or HTTPS.\n' +
'Chrome fix: chrome://flags/#unsafely-treat-insecure-origin-as-secure';
micBtn.style.opacity = '0.4';
}
const flagsUrl = 'chrome://flags/#unsafely-treat-insecure-origin-as-secure';
const warn = document.createElement('div');
warn.style.cssText = 'background:var(--red,#c00);color:#fff;padding:12px 16px;border-radius:8px;margin:0 0 12px;font-size:13px;line-height:1.7;flex-shrink:0';
warn.innerHTML =
'<strong>⚠ Microphone blocked by browser</strong><br>' +
'Browsers only allow microphone access on <b>secure contexts</b> (HTTPS or localhost).<br>' +
'Quick fix: open the app at <b>http://localhost:7890</b> instead of the IP address.<br>' +
'Remote access fix: add the URL in Chrome flags:<br>' +
'<div style="display:flex;align-items:center;gap:8px;margin-top:6px">' +
`<code style="flex:1;background:rgba(0,0,0,.35);color:#fff;border-radius:4px;padding:4px 8px;font-size:12px;word-break:break-all;font-family:monospace">${flagsUrl}</code>` +
`<button onclick="navigator.clipboard?.writeText('${flagsUrl}').then(()=>{this.textContent='Copied!';setTimeout(()=>this.textContent='Copy',1500)})" ` +
'style="flex-shrink:0;background:rgba(255,255,255,.2);border:1px solid rgba(255,255,255,.45);color:#fff;border-radius:4px;padding:4px 10px;font-size:12px;cursor:pointer;font-weight:600">Copy</button>' +
'</div>';
// Insert inside the chat window so it scrolls with the conversation and
// never pushes the input bar off-screen.
if (chatWindow) chatWindow.prepend(warn);
}
// Restore conv LLM URL from server settings — fall back to the global Active LLM endpoint
if (llmUrlInp && _appSettings) llmUrlInp.value = _appSettings.conv_llm_url || _appSettings.llm_url || '';
if (llmUrlInp) llmUrlInp.addEventListener('input', () => {
_patchSettings({ conv_llm_url: llmUrlInp.value });
if (_appSettings) _appSettings.conv_llm_url = llmUrlInp.value;
});
let mediaRecorder = null;
let recChunks = [];
let recTimerInterval = null;
let recStart = 0;
let conversationHistory = [];
let turnCount = 0;
let isProcessing = false;
let vadAudioCtx = null;
let vadRafId = null;
let liveInterimText = '';
let speechRec = null;
const VAD_THRESHOLD = 0.02; // raised to ignore background noise
const VAD_MIN_REC_MS = 800; // wait 800ms before VAD starts checking (avoids click/noise at start)
const VAD_SILENCE_MS = 1000;
const INTERRUPT_THRESHOLD = 0.04; // higher than VAD to avoid echo triggering interruption
const INTERRUPT_HOLD_MS = 350; // speech must persist this long to interrupt
// Whisper hallucinations on silence/noise — discard these from the live preview
const HALLUCINATION_RE = /^(reich|danke\s*(schön)?|vielen\s*dank|thank\s*you|thanks|you|copyright|abonnieren|untertitel|zарегистрируйтесь)[.!?,\s]*$/i;
const origPlaceholder = textInput?.placeholder || '';
const vadToggle = $('conv-vad-toggle');
const handsFreeToggle = $('conv-handsfree-toggle');
let previewTranscribing = false;
let convCurrentAudio = null;
let autoMicGeneration = 0;
const audioQueue = [];
let audioQueuePlaying = false;
let audioQueueDrainCb = null;
let interruptCtx = null;
let interruptRafId = null;
let interruptStream = null;
let interruptSpeechStart = 0;
let vadHadSpeech = false; // true once RMS crossed threshold during this recording
let vadLastVoiceMs = 0; // last timestamp speech was detected (for preview gate)
let cancelNextBlob = false; // set by VAD when no speech was detected → skip STT
let convCurrentSentenceBubble = null; // assistant bubble to update with current TTS sentence
// ── Populate STT backends ────────────────────────────────────────────────
async function loadConvSttBackends() {
if (!sttSel) return;
const prev = sttSel.value;
try {
const d = await fetch('/api/stt-backends').then(r => r.json());
const all = d.backends || [];
if (!all.length) {
sttSel.innerHTML = '<option value="configured">Default (configured)</option>';
return;
}
sttSel.innerHTML = all.map(b => {
const icon = b.available ? '✓' : '✗';
const label = `${icon} ${escHtml(b.label)}`;
const disabled = !b.available;
return `<option value="${escHtml(b.id)}"${disabled ? ' disabled' : ''}>${label}</option>`;
}).join('');
// Restore prev selection or pick first available
const opt = sttSel.querySelector(`option[value="${CSS.escape(prev)}"]`);
if (opt && !opt.disabled) { sttSel.value = prev; }
else {
const first = sttSel.querySelector('option:not([disabled])');
if (first) sttSel.value = first.value;
}
} catch(_) {
sttSel.innerHTML = '<option value="configured">Default (configured)</option>';
}
}
// ── Populate TTS backends (reuse global _ttsBackends) ───────────────────
function populateConvTtsBackends() {
if (!ttsBkSel) return;
const prev = ttsBkSel.value;
const all = _ttsBackends || [];
if (!all.length) {
ttsBkSel.innerHTML = '<option value="">No TTS backend available</option>';
return;
}
ttsBkSel.innerHTML = all.map(b => {
const icon = b.available ? '✓' : '✗';
return `<option value="${escHtml(b.id)}"${!b.available ? ' disabled' : ''}>${icon} ${escHtml(b.label)}</option>`;
}).join('');
const opt = ttsBkSel.querySelector(`option[value="${CSS.escape(prev)}"]`);
if (opt && !opt.disabled) { ttsBkSel.value = prev; }
else {
const first = ttsBkSel.querySelector('option:not([disabled])');
if (first) ttsBkSel.value = first.value;
}
}
// ── Fetch LLM models ─────────────────────────────────────────────────────
async function fetchLlmModels() {
if (!llmModelSel) return;
const url = llmUrlInp?.value.trim() || '';
llmFetchBtn.disabled = true;
try {
const d = await fetch('/api/conversation/llm-models' + (url ? '?url=' + encodeURIComponent(url) : '')).then(r => r.json());
const models = d.models || [];
llmModelSel.innerHTML = models.length
? models.map(m => `<option value="${escHtml(m)}">${escHtml(m)}</option>`).join('')
: '<option value="">No models found</option>';
// Default to the global Active LLM model when available (still overridable here)
const want = _appSettings && _appSettings.llm_model;
if (want && models.includes(want)) llmModelSel.value = want;
} catch(e) {
llmModelSel.innerHTML = '<option value="">Fetch failed</option>';
} finally {
llmFetchBtn.disabled = false;
}
}
// ── Fetch TTS voices ─────────────────────────────────────────────────────
async function fetchConvTtsVoices() {
if (!ttsVoiceSel || !ttsBkSel) return;
const backend = ttsBkSel.value;
if (!backend) return;
ttsFetchBtn.disabled = true;
try {
const rawVoices = await fetch('/api/tts-voices?backend=' + encodeURIComponent(backend)).then(r => r.json());
ttsVoiceSel.innerHTML = rawVoices.length
? rawVoices.map(v => { const id = backendVoiceId(v); return `<option value="${escHtml(id)}">${escHtml(id)}</option>`; }).join('')
: '<option value="">No voices</option>';
} catch(e) {
ttsVoiceSel.innerHTML = '<option value="">Fetch failed</option>';
} finally {
ttsFetchBtn.disabled = false;
}
}
// ── Chat bubble helpers ──────────────────────────────────────────────────
function timeStr() {
const now = new Date();
return now.getHours().toString().padStart(2,'0') + ':' + now.getMinutes().toString().padStart(2,'0');
}
function removeWelcome() {
const w = chatWindow.querySelector('.conv-chat-welcome');
if (w) w.remove();
}
function addBubble(role, text) {
removeWelcome();
const wrap = document.createElement('div');
wrap.className = `conv-bubble-wrap conv-bubble-wrap--${role}`;
const bubble = document.createElement('div');
bubble.className = `conv-bubble conv-bubble--${role}`;
bubble.textContent = text || '';
const meta = document.createElement('div');
meta.className = 'conv-bubble-meta';
meta.textContent = timeStr();
wrap.appendChild(bubble);
wrap.appendChild(meta);
chatWindow.appendChild(wrap);
chatWindow.scrollTop = chatWindow.scrollHeight;
return bubble;
}
function addTypingBubble() {
removeWelcome();
const wrap = document.createElement('div');
wrap.className = 'conv-bubble-wrap conv-bubble-wrap--assistant';
wrap.id = 'conv-typing-wrap';
const bubble = document.createElement('div');
bubble.className = 'conv-bubble conv-bubble--assistant';
bubble.innerHTML = '<span class="conv-typing"><span></span><span></span><span></span></span>';
wrap.appendChild(bubble);
chatWindow.appendChild(wrap);
chatWindow.scrollTop = chatWindow.scrollHeight;
return bubble;
}
function addErrorBubble(msg) {
removeWelcome();
const wrap = document.createElement('div');
wrap.className = 'conv-bubble-wrap conv-bubble-wrap--assistant';
const bubble = document.createElement('div');
bubble.className = 'conv-bubble conv-bubble--error';
bubble.innerHTML = `<span class="mdi mdi-alert-outline"></span> ${escHtml(msg)}`;
wrap.appendChild(bubble);
chatWindow.appendChild(wrap);
chatWindow.scrollTop = chatWindow.scrollHeight;
}
// ── Stats panel ──────────────────────────────────────────────────────────
function fmtMs(ms) { return ms == null ? '—' : ms >= 1000 ? (ms/1000).toFixed(2)+'s' : ms+'ms'; }
function updateStatBar(id, val, maxVal) {
const fill = $(id);
if (fill) fill.style.width = maxVal > 0 ? Math.min(100, (val / maxVal) * 100) + '%' : '0%';
}
function updateStats(stats) {
const { stt_ms, llm_ttft_ms, llm_total_ms, tts_ms, total_ms } = stats;
const max = total_ms || 1;
const set = (valId, fillId, ms) => {
const el = $(valId); if (el) el.textContent = fmtMs(ms);
updateStatBar(fillId, ms || 0, max);
};
set('cpv-stt', 'cpf-stt', stt_ms);
set('cpv-ttft', 'cpf-ttft', llm_ttft_ms);
set('cpv-llm', 'cpf-llm', llm_total_ms);
set('cpv-tts', 'cpf-tts', tts_ms);
set('cpv-total', 'cpf-total', total_ms);
}
function addHistoryItem(n, totalMs, ok) {
const empty = turnHistory?.querySelector('.conv-history-empty');
if (empty) empty.remove();
const item = document.createElement('div');
item.className = 'conv-hist-item';
const cls = ok ? 'conv-hist-ok' : 'conv-hist-err';
const icon = ok ? 'mdi-check-circle-outline' : 'mdi-alert-outline';
item.innerHTML = `<span class="conv-hist-num">#${n}</span>
<span class="${cls}"><span class="mdi ${icon}"></span></span>
<span class="conv-hist-time ${cls}">${fmtMs(totalMs)}</span>`;
turnHistory.insertBefore(item, turnHistory.firstChild);
}
// ── Recording ────────────────────────────────────────────────────────────
function startRecTimer() {
recStart = Date.now();
recTimerInterval = setInterval(() => {
const s = Math.floor((Date.now() - recStart) / 1000);
if (micTimer) micTimer.textContent = s + 's';
}, 500);
}
function stopRecTimer() {
clearInterval(recTimerInterval);
if (micTimer) micTimer.textContent = '';
}
// ── Audio queue — plays multi-chunk TTS responses sequentially ──────────────
function clearAudio() {
if (convCurrentAudio) { try { convCurrentAudio.pause(); } catch(_){} convCurrentAudio = null; }
audioQueue.length = 0;
audioQueuePlaying = false;
audioQueueDrainCb = null;
stopInterruptMonitor();
}
function stopInterruptMonitor() {
cancelAnimationFrame(interruptRafId); interruptRafId = null;
if (interruptCtx) { try { interruptCtx.close(); } catch(_){} interruptCtx = null; }
if (interruptStream) { interruptStream.getTracks().forEach(t => t.stop()); interruptStream = null; }
interruptSpeechStart = 0;
}
async function startInterruptMonitor() {
if (interruptCtx || !navigator.mediaDevices?.getUserMedia) return;
try {
interruptStream = await navigator.mediaDevices.getUserMedia({ audio: true });
interruptCtx = new (window.AudioContext || window.webkitAudioContext)();
await interruptCtx.resume(); // may be suspended when created outside a user gesture
const src = interruptCtx.createMediaStreamSource(interruptStream);
const analyser = interruptCtx.createAnalyser();
analyser.fftSize = 256;
src.connect(analyser);
const buf = new Float32Array(analyser.fftSize);
function tick() {
// Stop monitoring once nothing is playing and queue is empty
if (!audioQueuePlaying && !audioQueue.length && !convCurrentAudio) {
stopInterruptMonitor();
return;
}
analyser.getFloatTimeDomainData(buf);
let rms = 0;
for (const s of buf) rms += s * s;
rms = Math.sqrt(rms / buf.length);
if (rms > INTERRUPT_THRESHOLD) {
if (!interruptSpeechStart) interruptSpeechStart = Date.now();
if (Date.now() - interruptSpeechStart >= INTERRUPT_HOLD_MS) {
// User is talking — interrupt the AI.
// Force-release isProcessing so startRecording's guard doesn't block us.
stopInterruptMonitor();
clearAudio();
autoMicGeneration++;
isProcessing = false;
micBtn.classList.remove('processing');
micIcon.className = 'mdi mdi-microphone';
if (sendBtn) sendBtn.disabled = false;
if (textInput) textInput.disabled = false;
startRecording().catch(() => {});
return;
}
} else {
interruptSpeechStart = 0;
}
interruptRafId = requestAnimationFrame(tick);
}
interruptRafId = requestAnimationFrame(tick);
} catch (_) { stopInterruptMonitor(); }
}
function playNextAudio() {
if (!audioQueue.length) {
audioQueuePlaying = false;
convCurrentAudio = null;
convCurrentSentenceBubble = null;
if (audioQueueDrainCb) {
const cb = audioQueueDrainCb;
audioQueueDrainCb = null;
setTimeout(cb, 150);
} else if (micStatus && !isProcessing) {
micStatus.textContent = 'Ready';
}
return;
}
audioQueuePlaying = true;
const item = audioQueue.shift(); // {url, text}
// Show the sentence text in the typing bubble if it still shows "..."
if (convCurrentSentenceBubble && item.text) {
if (convCurrentSentenceBubble.querySelector('.conv-typing')) {
convCurrentSentenceBubble.innerHTML = '';
convCurrentSentenceBubble.textContent = item.text;
}
}
const el = new Audio(item.url);
convCurrentAudio = el;
el.addEventListener('ended', () => { URL.revokeObjectURL(item.url); playNextAudio(); });
el.play().catch(() => { URL.revokeObjectURL(item.url); playNextAudio(); });
if (micStatus) micStatus.textContent = 'Speaking…';
startInterruptMonitor();
}
function enqueueAudio(b64, mime, text) {
const binStr = atob(b64);
const arr = new Uint8Array(binStr.length);
for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' }));
audioQueue.push({ url, text: text || '' });
if (!audioQueuePlaying) playNextAudio();
}
// ── Chunked Whisper preview (fallback for browsers without SpeechRecognition) ──
async function transcribeForPreview() {
if (previewTranscribing || !recChunks.length) return;
// When VAD is on, gate on detected speech to avoid transcribing silence.
// When VAD is off the user controls recording manually — always transcribe.
if (vadToggle?.checked && (!vadHadSpeech || Date.now() - vadLastVoiceMs > 5000)) return;
previewTranscribing = true;
try {
const mime = (mediaRecorder && mediaRecorder.mimeType) || 'audio/webm';
const ext = mime.includes('ogg') ? '.ogg' : '.webm';
const blob = new Blob(recChunks, { type: mime });
const fd = new FormData();
fd.append('file', new File([blob], 'preview' + ext, { type: mime }));
fd.append('backend', sttSel?.value || 'configured');
const r = await fetch('/api/transcribe-bytes', { method: 'POST', body: fd });
if (r.ok) {
const d = await r.json();
const txt = (d.text || '').trim();
if (txt && txt.length > 3 && !HALLUCINATION_RE.test(txt)
&& mediaRecorder && mediaRecorder.state === 'recording') {
liveInterimText = txt;
if (textInput) textInput.value = txt;
}
}
} catch (_) {}
finally { previewTranscribing = false; }
}
// ── Auto-restart mic after agent finishes speaking ───────────────────────
function scheduleAutoMic() {
if (!handsFreeToggle?.checked) return;
const gen = ++autoMicGeneration;
function tryStart() {
if (gen !== autoMicGeneration || isProcessing) return;
startRecording().catch(() => {});
}
if (audioQueuePlaying || audioQueue.length > 0) {
// Audio still queued — trigger after queue drains
audioQueueDrainCb = tryStart;
} else if (convCurrentAudio && !convCurrentAudio.ended) {
convCurrentAudio.addEventListener('ended', () => setTimeout(tryStart, 150), { once: true });
} else {
setTimeout(tryStart, 300);
}
}
async function startRecording() {
if (isProcessing) return;
stopInterruptMonitor(); // release mic stream before opening a recording stream
if (!navigator.mediaDevices?.getUserMedia) {
toast('Microphone unavailable — browser requires a secure context (HTTPS or localhost). ' +
'Access the app via http://localhost:7890 or enable it in chrome://flags/#unsafely-treat-insecure-origin-as-secure', 'error', 8000);
return;
}
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
} catch(e) {
toast('Microphone access denied: ' + e.message, 'error');
return;
}
liveInterimText = '';
previewTranscribing = false;
vadHadSpeech = false;
vadLastVoiceMs = 0;
cancelNextBlob = false;
recChunks = [];
mediaRecorder = new MediaRecorder(stream);
mediaRecorder.ondataavailable = e => {
if (e.data.size > 0) {
recChunks.push(e.data);
// Only run preview when speech was detected (prevents "reich" on silence chunks)
if (mediaRecorder.state === 'recording' && vadHadSpeech) transcribeForPreview();
}
};
mediaRecorder.onstop = () => {
stream.getTracks().forEach(t => t.stop());
if (cancelNextBlob) {
// VAD fired but no speech was detected — reset without calling STT
cancelNextBlob = false;
if (textInput) { textInput.readOnly = false; textInput.classList.remove('listening'); textInput.placeholder = origPlaceholder; textInput.value = ''; }
if (sendBtn) sendBtn.disabled = false;
micBtn.classList.remove('processing');
micIcon.className = 'mdi mdi-microphone';
if (micStatus) micStatus.textContent = 'Ready';
isProcessing = false;
return;
}
const blob = new Blob(recChunks, { type: mediaRecorder.mimeType || 'audio/webm' });
processBlob(blob);
};
// timeslice: 750ms when VAD is off (manual recording) for faster Whisper preview;
// 1500ms with VAD on (chunks are gated anyway, smaller slices waste CPU).
mediaRecorder.start(vadToggle?.checked ? 1500 : 750);
micBtn.classList.add('recording');
micIcon.className = 'mdi mdi-stop';
if (micStatus) micStatus.textContent = 'Recording…';
startRecTimer();
// Set input to listening mode (read-only; Whisper preview text will appear here)
if (textInput) {
textInput.readOnly = true;
textInput.value = '';
textInput.placeholder = 'Listening…';
textInput.classList.add('listening');
}
if (sendBtn) sendBtn.disabled = true;
// Also try Web Speech API for faster interim results (works on HTTPS / localhost)
const SpeechRec = window.SpeechRecognition || window.webkitSpeechRecognition;
if (SpeechRec) {
try {
speechRec = new SpeechRec();
speechRec.continuous = true;
speechRec.interimResults = true;
speechRec.onresult = e => {
let final = '', interim = '';
for (let i = 0; i < e.results.length; i++) {
if (e.results[i].isFinal) final += e.results[i][0].transcript;
else interim += e.results[i][0].transcript;
}
liveInterimText = final + interim;
if (textInput) textInput.value = liveInterimText;
};
speechRec.onerror = () => { speechRec = null; };
speechRec.start();
} catch(_) { speechRec = null; }
}
// VAD: auto-stop on silence using AudioContext
if (vadToggle?.checked) {
try {
vadAudioCtx = new AudioContext();
const src = vadAudioCtx.createMediaStreamSource(stream);
const analyser = vadAudioCtx.createAnalyser();
analyser.fftSize = 1024;
src.connect(analyser);
const vadBuf = new Float32Array(analyser.fftSize);
let vadLastVoice = Date.now();
const levelWrap = $('conv-level-wrap');
const levelFill = $('conv-level-fill');
if (levelWrap) levelWrap.classList.add('active');
function vadTick() {
if (!mediaRecorder || mediaRecorder.state !== 'recording') return;
analyser.getFloatTimeDomainData(vadBuf);
let rms = 0;
for (const s of vadBuf) rms += s * s;
rms = Math.sqrt(rms / vadBuf.length);
if (levelFill) levelFill.style.width = Math.min(100, rms * 5000) + '%';
if (rms > VAD_THRESHOLD) {
vadLastVoice = Date.now();
vadLastVoiceMs = Date.now();
vadHadSpeech = true;
}
const elapsed = Date.now() - recStart;
const silence = Date.now() - vadLastVoice;
if (elapsed > VAD_MIN_REC_MS) {
const remaining = VAD_SILENCE_MS - silence;
if (micStatus) {
micStatus.textContent = remaining < VAD_SILENCE_MS * 0.7
? `Sending in ${(Math.max(0, remaining) / 1000).toFixed(1)}s…`
: 'Recording…';
}
if (silence >= VAD_SILENCE_MS) {
if (!vadHadSpeech) {
// Silence the whole time — cancel without calling STT
cancelNextBlob = true;
if (micStatus) micStatus.textContent = 'Ready';
}
stopRecording();
return;
}
}
vadRafId = requestAnimationFrame(vadTick);
}
vadRafId = requestAnimationFrame(vadTick);
} catch(_) { vadAudioCtx = null; }
}
}
function stopRecording() {
if (!mediaRecorder || mediaRecorder.state === 'inactive') return;
// Clean up VAD
cancelAnimationFrame(vadRafId);
vadRafId = null;
if (vadAudioCtx) { try { vadAudioCtx.close(); } catch(_){} vadAudioCtx = null; }
const levelWrap = $('conv-level-wrap');
const levelFill = $('conv-level-fill');
if (levelWrap) levelWrap.classList.remove('active');
if (levelFill) levelFill.style.width = '0%';
// Clean up live speech recognition
if (speechRec) { try { speechRec.stop(); } catch(_){} speechRec = null; }
previewTranscribing = false;
mediaRecorder.stop();
stopRecTimer();
micBtn.classList.remove('recording');
micBtn.classList.add('processing');
micIcon.className = 'mdi mdi-dots-horizontal';
if (micStatus) micStatus.textContent = 'Processing…';
isProcessing = true;
}
// ── Send turn via SSE ─────────────────────────────────────────────────────
async function processBlob(blob) {
autoMicGeneration++; // cancel any pending auto-mic from previous turn
clearAudio();
turnCount++;
const turnN = turnCount;
const t0 = Date.now();
// Transition input from listening/readOnly state to processing/disabled state
if (textInput) {
textInput.readOnly = false;
textInput.classList.remove('listening');
textInput.placeholder = origPlaceholder;
textInput.value = '';
textInput.disabled = true;
}
if (sendBtn) sendBtn.disabled = true;
// Show user bubble seeded with live interim transcript (if available)
const userBubble = addBubble('user', liveInterimText || '…');
const assistantBubble = addTypingBubble();
convCurrentSentenceBubble = assistantBubble; // sentence text will appear here while audio plays
let assistantText = '';
let lastStats = null;
const form = new FormData();
form.append('audio', blob, 'audio.webm');
form.append('stt_backend', sttSel?.value || 'configured');
form.append('llm_url', llmUrlInp?.value.trim() || '');
form.append('llm_model', llmModelSel?.value || '');
form.append('tts_backend', ttsBkSel?.value || 'voice_clone');
form.append('tts_voice', ttsVoiceSel?.value || '');
form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.');
form.append('history', JSON.stringify(conversationHistory.slice(-20)));
try {
const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form });
if (!resp.ok) { let _d=''; try { const _j=await resp.json(); _d=JSON.stringify(_j.detail||_j); } catch(_){} throw new Error('Server error ' + resp.status + (_d ? ': ' + _d : '')); }
const reader = resp.body.getReader();
const dec = new TextDecoder();
let buf = '';
while (true) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
const lines = buf.split('\n');
buf = lines.pop();
for (const line of lines) {
if (!line.startsWith('data:')) continue;
let evt;
try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; }
if (evt.type === 'transcript') {
userBubble.textContent = evt.text || '(empty)';
if (micStatus) micStatus.textContent = 'Generating reply…';
} else if (evt.type === 'token') {
if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
convCurrentSentenceBubble = null; // LLM tokens take over the bubble now
assistantText += evt.delta;
assistantBubble.textContent = assistantText;
chatWindow.scrollTop = chatWindow.scrollHeight;
} else if (evt.type === 'llm_done') {
assistantText = evt.text || assistantText;
assistantBubble.textContent = assistantText;
convCurrentSentenceBubble = null;
if (micStatus) micStatus.textContent = 'Synthesising speech…';
} else if (evt.type === 'audio') {
enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
} else if (evt.type === 'stats') {
lastStats = evt;
updateStats(evt);
} else if (evt.type === 'done') {
conversationHistory.push({ role: 'user', content: userBubble.textContent });
conversationHistory.push({ role: 'assistant', content: assistantText });
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
scheduleAutoMic(); // triggers after audio queue drains
} else if (evt.type === 'error') {
clearAudio();
const wrap = assistantBubble.closest('.conv-bubble-wrap');
if (wrap) wrap.remove();
const stage = evt.stage?.toUpperCase() || 'ERR';
let msg = evt.message || 'Unknown error';
const detailMatch = msg.match(/HTTP \d+:\s*(.+)/s);
if (detailMatch) msg = detailMatch[1].trim();
if (msg.length > 300) msg = msg.slice(0, 300) + '…';
addErrorBubble(`[${stage}] ${msg}`);
addHistoryItem(turnN, Date.now() - t0, false);
if (micStatus) micStatus.textContent = 'Error — ready';
}
}
}
} catch(e) {
clearAudio();
const wrap = assistantBubble.closest('.conv-bubble-wrap');
if (wrap) wrap.remove();
addErrorBubble(e.message);
addHistoryItem(turnN, Date.now() - t0, false);
if (micStatus) micStatus.textContent = 'Error — ready';
} finally {
liveInterimText = '';
previewTranscribing = false;
isProcessing = false;
micBtn.classList.remove('processing');
micIcon.className = 'mdi mdi-microphone';
if (sendBtn) sendBtn.disabled = false;
if (textInput) { textInput.disabled = false; textInput.value = ''; }
if (audioQueuePlaying || audioQueue.length || convCurrentAudio) startInterruptMonitor();
}
}
// ── Text-input turn (skips STT, sends text directly) ────────────────────
async function processText(text) {
text = text.trim();
if (!text || isProcessing) return;
autoMicGeneration++; // cancel any pending auto-mic
clearAudio();
isProcessing = true;
if (sendBtn) sendBtn.disabled = true;
if (textInput) { textInput.disabled = true; textInput.value = ''; }
micBtn.classList.add('processing');
micIcon.className = 'mdi mdi-dots-horizontal';
if (micStatus) micStatus.textContent = 'Processing…';
turnCount++;
const turnN = turnCount;
const t0 = Date.now();
const userBubble = addBubble('user', text);
const assistantBubble = addTypingBubble();
convCurrentSentenceBubble = assistantBubble;
let assistantText = '';
let lastStats = null;
const form = new FormData();
form.append('text', text);
form.append('stt_backend', sttSel?.value || 'configured');
form.append('llm_url', llmUrlInp?.value.trim() || '');
form.append('llm_model', llmModelSel?.value || '');
form.append('tts_backend', ttsBkSel?.value || 'voice_clone');
form.append('tts_voice', ttsVoiceSel?.value || '');
form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.');
form.append('history', JSON.stringify(conversationHistory.slice(-20)));
try {
const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form });
if (!resp.ok) { let _d=''; try { const _j=await resp.json(); _d=JSON.stringify(_j.detail||_j); } catch(_){} throw new Error('Server error ' + resp.status + (_d ? ': ' + _d : '')); }
const reader = resp.body.getReader();
const dec = new TextDecoder();
let buf = '';
while (true) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
const lines = buf.split('\n');
buf = lines.pop();
for (const line of lines) {
if (!line.startsWith('data:')) continue;
let evt;
try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; }
if (evt.type === 'transcript') {
if (micStatus) micStatus.textContent = 'Generating reply…';
} else if (evt.type === 'token') {
if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
convCurrentSentenceBubble = null;
assistantText += evt.delta;
assistantBubble.textContent = assistantText;
chatWindow.scrollTop = chatWindow.scrollHeight;
} else if (evt.type === 'llm_done') {
assistantText = evt.text || assistantText;
assistantBubble.textContent = assistantText;
convCurrentSentenceBubble = null;
if (micStatus) micStatus.textContent = 'Synthesising speech…';
} else if (evt.type === 'audio') {
enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
} else if (evt.type === 'stats') {
lastStats = evt;
updateStats(evt);
} else if (evt.type === 'done') {
conversationHistory.push({ role: 'user', content: text });
conversationHistory.push({ role: 'assistant', content: assistantText });
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
scheduleAutoMic(); // triggers after audio queue drains
} else if (evt.type === 'error') {
clearAudio();
const wrap = assistantBubble.closest('.conv-bubble-wrap');
if (wrap) wrap.remove();
addErrorBubble(`[${evt.stage?.toUpperCase() || 'ERR'}] ${evt.message || 'Unknown error'}`);
addHistoryItem(turnN, Date.now() - t0, false);
if (micStatus) micStatus.textContent = 'Error — ready';
}
}
}
} catch(e) {
clearAudio();
const wrap = assistantBubble.closest('.conv-bubble-wrap');
if (wrap) wrap.remove();
addErrorBubble(e.message);
addHistoryItem(turnN, Date.now() - t0, false);
if (micStatus) micStatus.textContent = 'Error — ready';
} finally {
isProcessing = false;
micBtn.classList.remove('processing');
micIcon.className = 'mdi mdi-microphone';
if (sendBtn) sendBtn.disabled = false;
if (textInput) textInput.disabled = false;
if (audioQueuePlaying || audioQueue.length || convCurrentAudio) startInterruptMonitor();
}
}
// ── Wire up events ───────────────────────────────────────────────────────
micBtn.addEventListener('click', () => {
if (isProcessing) return;
autoMicGeneration++; // cancel any pending hands-free auto-restart
if (mediaRecorder && mediaRecorder.state === 'recording') {
stopRecording();
} else {
clearAudio(); // stop agent if still speaking
startRecording();
}
});
// Text input — Enter key or Send button
sendBtn?.addEventListener('click', () => processText(textInput?.value || ''));
textInput?.addEventListener('keydown', e => {
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); processText(textInput.value); }
});
clearBtn?.addEventListener('click', () => {
conversationHistory = [];
turnCount = 0;
chatWindow.innerHTML = '<div class="conv-chat-welcome"><span class="mdi mdi-forum-outline" style="font-size:32px;opacity:.25"></span><p>Type a message or press the microphone button below to start.</p></div>';
if (turnHistory) turnHistory.innerHTML = '<div class="conv-history-empty">No turns yet.</div>';
['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; });
['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; });
});
llmFetchBtn?.addEventListener('click', fetchLlmModels);
ttsFetchBtn?.addEventListener('click', fetchConvTtsVoices);
// Re-populate TTS when backend changes
ttsBkSel?.addEventListener('change', () => { ttsVoiceSel.innerHTML = '<option value="">— fetch voices —</option>'; });
// ── Init ─────────────────────────────────────────────────────────────────
loadConvSttBackends();
populateConvTtsBackends();
// Keep TTS backend select in sync after global backend refresh
window._ttsRefreshHooks = window._ttsRefreshHooks || [];
window._ttsRefreshHooks.push(populateConvTtsBackends);
})();