mARTin-B78
5fecbf06d4
Fish-Speech emotion tags were silently ignored on non-English books: per-line emotions are LLM-generated in the book's own language, but Fish-Speech only recognizes English [tag] markers, and a double-tagging bug was stacking a broken server-derived tag on top of the client's own. Added a DE->EN translation table and removed the double-tagging. Also wires the existing book-profile context and race_species field into character portrait prompts (previously only used for voice design), adds a recast-until-threshold loop for casting, and adds backend-aware emotion quick-picks to Read Aloud, Try a Voice, and Conversation. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1292 lines
56 KiB
JavaScript
1292 lines
56 KiB
JavaScript
// ── Settings: Logs ────────────────────────────────────────────────────────
|
||
|
||
let _logsLiveTimer = null;
|
||
let _logsActiveLevel = '';
|
||
|
||
function escLog(s) { return String(s).replace(/[&<>]/g, c => ({'&':'&','<':'<','>':'>'}[c])); }
|
||
|
||
window.loadSettingsLogs = async function() {
|
||
const viewer = $('s-log-viewer');
|
||
if (!viewer) return;
|
||
try {
|
||
const data = await fetch('/api/logs?limit=300').then(r => r.json());
|
||
const items = (data.items || []).filter(item =>
|
||
!_logsActiveLevel || item.level === _logsActiveLevel
|
||
);
|
||
const count = $('s-log-count');
|
||
if (count) count.textContent = items.length + ' entries';
|
||
if (!items.length) {
|
||
viewer.innerHTML = '<div class="s-log-empty">No log entries</div>';
|
||
return;
|
||
}
|
||
viewer.innerHTML = items.map(item => {
|
||
const ts = item.ts ? item.ts.replace('T', ' ').replace(/\.\d+Z$/, ' UTC') : '';
|
||
return `<div class="s-log-row">
|
||
<span class="s-log-ts">${escLog(ts)}</span>
|
||
<span class="s-log-level ${escLog(item.level)}">${escLog(item.level)}</span>
|
||
<span class="s-log-msg">${escLog(item.msg)}</span>
|
||
</div>`;
|
||
}).join('');
|
||
} catch(e) {
|
||
viewer.innerHTML = `<div class="s-log-empty">Failed to load logs: ${escLog(e.message)}</div>`;
|
||
}
|
||
};
|
||
|
||
$('s-logs-refresh-btn')?.addEventListener('click', () => loadSettingsLogs());
|
||
|
||
$('s-logs-clear-btn')?.addEventListener('click', async () => {
|
||
await fetch('/api/logs', { method: 'DELETE' });
|
||
const viewer = $('s-log-viewer');
|
||
if (viewer) viewer.innerHTML = '<div class="s-log-empty">Logs cleared</div>';
|
||
const count = $('s-log-count');
|
||
if (count) count.textContent = '';
|
||
});
|
||
|
||
$('s-logs-live-toggle')?.addEventListener('change', function() {
|
||
clearInterval(_logsLiveTimer);
|
||
if (this.checked) {
|
||
loadSettingsLogs();
|
||
_logsLiveTimer = setInterval(loadSettingsLogs, 3000);
|
||
}
|
||
});
|
||
|
||
document.querySelectorAll('.s-log-filter').forEach(btn => {
|
||
btn.addEventListener('click', function() {
|
||
document.querySelectorAll('.s-log-filter').forEach(b => b.classList.remove('is-active'));
|
||
this.classList.add('is-active');
|
||
_logsActiveLevel = this.dataset.logLevel;
|
||
loadSettingsLogs();
|
||
});
|
||
});
|
||
|
||
// ── Settings: About ───────────────────────────────────────────────────────
|
||
|
||
// Show version immediately from meta tag, then confirm from server
|
||
(function loadAppVersion() {
|
||
const el = $('s-about-version');
|
||
const metaVer = document.querySelector('meta[name="app-version"]')?.content;
|
||
if (el && metaVer) el.textContent = 'v' + metaVer;
|
||
fetch('/api/version').then(r => r.json()).then(d => {
|
||
if (el && d.version) el.textContent = 'v' + d.version;
|
||
}).catch(() => {});
|
||
})();
|
||
|
||
// Lazy-load changelog when the details element is opened
|
||
$('about-changelog-details')?.addEventListener('toggle', async function () {
|
||
if (!this.open) return;
|
||
const content = $('about-changelog-content');
|
||
const status = $('about-changelog-status');
|
||
if (!content || content.textContent.trim()) return;
|
||
if (status) status.textContent = 'Loading…';
|
||
try {
|
||
const text = await fetch('/api/changelog').then(r => r.ok ? r.text() : Promise.reject(r.status));
|
||
content.textContent = text;
|
||
if (status) status.textContent = '';
|
||
} catch(e) {
|
||
content.textContent = 'Could not load changelog: ' + e;
|
||
if (status) status.textContent = 'error';
|
||
}
|
||
});
|
||
|
||
function renderSettingsAbout() {
|
||
const el = $('s-about-backends');
|
||
if (!el) return;
|
||
const available = new Set((_ttsBackends || []).filter(b => b.available).map(b => b.id));
|
||
const all = (_ttsBackends || []);
|
||
if (!all.length) { el.innerHTML = ''; return; }
|
||
el.innerHTML = all.map(b => {
|
||
const online = available.has(b.id);
|
||
return `<span class="s-about-backend ${online ? 'online' : 'offline'}">
|
||
<span class="mdi mdi-speaker-outline"></span>${escHtml(b.label)}
|
||
<span class="mdi ${online ? 'mdi-check-circle-outline' : 'mdi-close-circle-outline'}"></span>
|
||
</span>`;
|
||
}).join('');
|
||
}
|
||
|
||
// ── Voices import ──────────────────────────────────────────────────────────
|
||
|
||
$('s-import-voices-file')?.addEventListener('change', async function () {
|
||
const file = this.files?.[0];
|
||
if (!file) return;
|
||
const st = $('s-import-status');
|
||
if (st) st.textContent = 'Uploading…';
|
||
try {
|
||
const fd = new FormData();
|
||
fd.append('file', file);
|
||
const r = await fetch('/api/voices/import', { method:'POST', body: fd });
|
||
if (!r.ok) { const e = await r.json().catch(()=>({})); throw new Error(e.detail || r.statusText); }
|
||
const d = await r.json();
|
||
if (st) st.textContent = `Imported ${d.imported} files.`;
|
||
toast(`Imported ${d.imported} voice files`, 'success');
|
||
this.value = '';
|
||
} catch(e) {
|
||
if (st) st.textContent = 'Import failed.';
|
||
toast('Import failed: ' + e.message, 'error');
|
||
}
|
||
});
|
||
|
||
// ── Conversation Playground ────────────────────────────────────────────────
|
||
|
||
(function initConversationPlayground() {
|
||
const chatWindow = $('conv-chat-window');
|
||
const micBtn = $('conv-mic-btn');
|
||
const micIcon = $('conv-mic-icon');
|
||
const micStatus = $('conv-mic-status');
|
||
const micTimer = $('conv-mic-timer');
|
||
const textInput = $('conv-text-input');
|
||
const sendBtn = $('conv-send-btn');
|
||
const clearBtn = $('conv-clear-btn');
|
||
const sttSel = $('conv-stt-select');
|
||
const llmUrlInp = $('conv-llm-url');
|
||
const llmFetchBtn = $('conv-llm-fetch-btn');
|
||
const llmModelSel = $('conv-llm-model-select');
|
||
const ttsBkSel = $('conv-tts-backend-select');
|
||
const ttsFetchBtn = $('conv-tts-fetch-btn');
|
||
const ttsVoiceSel = $('conv-tts-voice-select');
|
||
const ttsEmotionSel = $('conv-emotion-select');
|
||
const systemPrompt = $('conv-system-prompt');
|
||
const turnHistory = $('conv-turn-history');
|
||
if (!chatWindow || !micBtn) return;
|
||
|
||
// Conversation replies previously had no style/emotion control at all,
|
||
// regardless of backend. Same REH_EMOTIONS quick-pick used in Try a Voice /
|
||
// Read Aloud — sent to the server as a plain English phrase; the backend
|
||
// decides server-side (see _conv_tts_text_and_instruct in
|
||
// routes/conversation.py) whether it becomes an inline Fish [tag] or an
|
||
// instruct-field phrase for style-aware backends.
|
||
if (ttsEmotionSel && typeof REH_EMOTIONS !== 'undefined' && !ttsEmotionSel.dataset.populated) {
|
||
REH_EMOTIONS.forEach(e => {
|
||
if (!e.value) return;
|
||
const o = document.createElement('option');
|
||
o.value = e.value;
|
||
o.textContent = `${e.emoji} ${e.label}`;
|
||
ttsEmotionSel.appendChild(o);
|
||
});
|
||
ttsEmotionSel.dataset.populated = '1';
|
||
}
|
||
|
||
// Warn if microphone API is unavailable (HTTP on non-localhost = insecure context)
|
||
if (!navigator.mediaDevices?.getUserMedia) {
|
||
if (micStatus) {
|
||
micStatus.textContent = 'Mic unavailable — insecure context';
|
||
micStatus.style.color = 'var(--red, #e05)';
|
||
}
|
||
if (micBtn) {
|
||
micBtn.disabled = true;
|
||
micBtn.title = 'Browser blocks microphone on HTTP. Use http://localhost:7890 or HTTPS.\n' +
|
||
'Chrome fix: chrome://flags/#unsafely-treat-insecure-origin-as-secure';
|
||
micBtn.style.opacity = '0.4';
|
||
}
|
||
const flagsUrl = 'chrome://flags/#unsafely-treat-insecure-origin-as-secure';
|
||
const warn = document.createElement('div');
|
||
warn.style.cssText = 'background:var(--red,#c00);color:#fff;padding:12px 16px;border-radius:8px;margin:0 0 12px;font-size:13px;line-height:1.7;flex-shrink:0';
|
||
warn.innerHTML =
|
||
'<strong>⚠ Microphone blocked by browser</strong><br>' +
|
||
'Browsers only allow microphone access on <b>secure contexts</b> (HTTPS or localhost).<br>' +
|
||
'Quick fix: open the app at <b>http://localhost:7890</b> instead of the IP address.<br>' +
|
||
'Remote access fix: add the URL in Chrome flags:<br>' +
|
||
'<div style="display:flex;align-items:center;gap:8px;margin-top:6px">' +
|
||
`<code style="flex:1;background:rgba(0,0,0,.35);color:#fff;border-radius:4px;padding:4px 8px;font-size:12px;word-break:break-all;font-family:monospace">${flagsUrl}</code>` +
|
||
`<button onclick="navigator.clipboard?.writeText('${flagsUrl}').then(()=>{this.textContent='Copied!';setTimeout(()=>this.textContent='Copy',1500)})" ` +
|
||
'style="flex-shrink:0;background:rgba(255,255,255,.2);border:1px solid rgba(255,255,255,.45);color:#fff;border-radius:4px;padding:4px 10px;font-size:12px;cursor:pointer;font-weight:600">Copy</button>' +
|
||
'</div>';
|
||
// Insert inside the chat window so it scrolls with the conversation and
|
||
// never pushes the input bar off-screen.
|
||
if (chatWindow) chatWindow.prepend(warn);
|
||
}
|
||
|
||
// Restore conv LLM URL from server settings — fall back to the global Active LLM endpoint
|
||
if (llmUrlInp && _appSettings) llmUrlInp.value = _appSettings.conv_llm_url || _appSettings.llm_url || '';
|
||
if (llmUrlInp) llmUrlInp.addEventListener('input', () => {
|
||
_patchSettings({ conv_llm_url: llmUrlInp.value });
|
||
if (_appSettings) _appSettings.conv_llm_url = llmUrlInp.value;
|
||
});
|
||
|
||
let mediaRecorder = null;
|
||
let recChunks = [];
|
||
let recTimerInterval = null;
|
||
let recStart = 0;
|
||
let conversationHistory = [];
|
||
let turnCount = 0;
|
||
let isProcessing = false;
|
||
let vadAudioCtx = null;
|
||
let vadRafId = null;
|
||
let liveInterimText = '';
|
||
let speechRec = null;
|
||
const VAD_MIN_REC_MS = 650; // wait before VAD starts checking (avoids click/noise at start)
|
||
const VAD_SILENCE_MS = 1000;
|
||
const INTERRUPT_THRESHOLD = 0.04; // higher than VAD to avoid echo triggering interruption
|
||
const INTERRUPT_HOLD_MS = 350; // speech must persist this long to interrupt
|
||
const MIN_SPEECH_FRAMES_MS = 300; // speech must exceed threshold for at least this long
|
||
const gateSlider = () => document.getElementById('conv-gate-slider');
|
||
const vadThreshold = () => { const s = gateSlider(); return s ? (parseInt(s.value, 10) / 1000) : 0.02; };
|
||
let vadSpeechStartMs = 0; // time when speech level first crossed threshold in this recording
|
||
const liveAgentOn = () => liveAgentToggle ? liveAgentToggle.checked : true;
|
||
const vadSilenceMs = () => liveAgentOn() ? 650 : VAD_SILENCE_MS;
|
||
const interruptHoldMs = () => liveAgentOn() ? 120 : INTERRUPT_HOLD_MS;
|
||
// Whisper hallucinations on silence/noise — discard these from the live preview
|
||
const HALLUCINATION_RE = /^(reich|danke\s*(schön)?|vielen\s*dank|thank\s*you|thanks|you|copyright|abonnieren|untertitel|zарегистрируйтесь)[.!?,\s]*$/i;
|
||
const origPlaceholder = textInput?.placeholder || '';
|
||
const vadToggle = $('conv-vad-toggle');
|
||
const handsFreeToggle = $('conv-handsfree-toggle');
|
||
const liveAgentToggle = $('conv-live-agent-toggle');
|
||
let previewTranscribing = false;
|
||
let convCurrentAudio = null;
|
||
let convAbortCtrl = null;
|
||
let convTurnGeneration = 0;
|
||
let liveMicEnabled = false;
|
||
let cancelRecordingOnStop = false;
|
||
let autoMicGeneration = 0;
|
||
const audioQueue = [];
|
||
let audioQueuePlaying = false;
|
||
let audioQueueDrainCb = null;
|
||
let interruptCtx = null;
|
||
let interruptRafId = null;
|
||
let interruptStream = null;
|
||
let interruptSpeechStart = 0;
|
||
let vadHadSpeech = false; // true once RMS crossed threshold during this recording
|
||
let vadLastVoiceMs = 0; // last timestamp speech was detected (for preview gate)
|
||
let cancelNextBlob = false; // set by VAD when no speech was detected → skip STT
|
||
|
||
// Update gate marker position and persist on slider change
|
||
function updateGateMarker() {
|
||
const s = gateSlider(); if (!s) return;
|
||
const gate = document.getElementById('conv-level-gate');
|
||
if (gate) gate.style.left = s.value + '%';
|
||
try { localStorage.setItem('ttsvc_conv_gate', s.value); } catch (_) {}
|
||
}
|
||
document.addEventListener('change', function (e) {
|
||
if (e.target.id === 'conv-gate-slider') updateGateMarker();
|
||
});
|
||
// Restore persisted gate
|
||
(function () {
|
||
const s = gateSlider(); if (!s) return;
|
||
try { const v = localStorage.getItem('ttsvc_conv_gate'); if (v) s.value = v; } catch (_) {}
|
||
updateGateMarker();
|
||
})();
|
||
let convCurrentSentenceBubble = null; // assistant bubble to update with current TTS sentence
|
||
|
||
// ── Populate STT backends ────────────────────────────────────────────────
|
||
async function loadConvSttBackends() {
|
||
if (!sttSel) return;
|
||
const prev = sttSel.value;
|
||
try {
|
||
const d = await fetch('/api/stt-backends').then(r => r.json());
|
||
const all = d.backends || [];
|
||
if (!all.length) {
|
||
sttSel.innerHTML = '<option value="configured">Default (configured)</option>';
|
||
return;
|
||
}
|
||
sttSel.innerHTML = all.map(b => {
|
||
const icon = b.available ? '✓' : '✗';
|
||
const label = `${icon} ${escHtml(b.label)}`;
|
||
const disabled = !b.available;
|
||
return `<option value="${escHtml(b.id)}"${disabled ? ' disabled' : ''}>${label}</option>`;
|
||
}).join('');
|
||
// Restore prev selection or pick first available
|
||
const opt = sttSel.querySelector(`option[value="${CSS.escape(prev)}"]`);
|
||
if (opt && !opt.disabled) { sttSel.value = prev; }
|
||
else {
|
||
const first = sttSel.querySelector('option:not([disabled])');
|
||
if (first) sttSel.value = first.value;
|
||
}
|
||
} catch(_) {
|
||
sttSel.innerHTML = '<option value="configured">Default (configured)</option>';
|
||
}
|
||
}
|
||
|
||
// ── Populate TTS backends (reuse global _ttsBackends) ───────────────────
|
||
function populateConvTtsBackends() {
|
||
if (!ttsBkSel) return;
|
||
const prev = ttsBkSel.value;
|
||
const all = _ttsBackends || [];
|
||
if (!all.length) {
|
||
ttsBkSel.innerHTML = '<option value="">No TTS backend available</option>';
|
||
return;
|
||
}
|
||
ttsBkSel.innerHTML = all.map(b => {
|
||
const icon = b.available ? '✓' : '✗';
|
||
return `<option value="${escHtml(b.id)}"${!b.available ? ' disabled' : ''}>${icon} ${escHtml(b.label)}</option>`;
|
||
}).join('');
|
||
const opt = ttsBkSel.querySelector(`option[value="${CSS.escape(prev)}"]`);
|
||
if (opt && !opt.disabled) { ttsBkSel.value = prev; }
|
||
else {
|
||
const first = ttsBkSel.querySelector('option:not([disabled])');
|
||
if (first) ttsBkSel.value = first.value;
|
||
}
|
||
}
|
||
|
||
// ── Fetch LLM models ─────────────────────────────────────────────────────
|
||
async function fetchLlmModels() {
|
||
if (!llmModelSel) return;
|
||
const url = llmUrlInp?.value.trim() || '';
|
||
llmFetchBtn.disabled = true;
|
||
try {
|
||
const d = await fetch('/api/conversation/llm-models' + (url ? '?url=' + encodeURIComponent(url) : '')).then(r => r.json());
|
||
const models = d.models || [];
|
||
llmModelSel.innerHTML = models.length
|
||
? models.map(m => `<option value="${escHtml(m)}">${escHtml(m)}</option>`).join('')
|
||
: '<option value="">No models found</option>';
|
||
// Default to the global Active LLM model when available (still overridable here)
|
||
const want = _appSettings && _appSettings.llm_model;
|
||
if (want && models.includes(want)) llmModelSel.value = want;
|
||
} catch(e) {
|
||
llmModelSel.innerHTML = '<option value="">Fetch failed</option>';
|
||
} finally {
|
||
llmFetchBtn.disabled = false;
|
||
}
|
||
}
|
||
|
||
// ── Fetch TTS voices ─────────────────────────────────────────────────────
|
||
async function fetchConvTtsVoices() {
|
||
if (!ttsVoiceSel || !ttsBkSel) return;
|
||
const backend = ttsBkSel.value;
|
||
if (!backend) return;
|
||
const prev = ttsVoiceSel.value;
|
||
ttsFetchBtn.disabled = true;
|
||
try {
|
||
const rawVoices = await fetch('/api/tts-voices?backend=' + encodeURIComponent(backend)).then(r => r.json());
|
||
let voices = Array.isArray(rawVoices) ? rawVoices : [];
|
||
if (typeof shouldFilterBackendVoices === 'function' && shouldFilterBackendVoices(backend)) {
|
||
const activeIds = await activeLibraryVoiceIds();
|
||
voices = voices.filter(v => activeIds.has(backendVoiceId(v)));
|
||
}
|
||
const ids = voices.map(backendVoiceId).filter(Boolean);
|
||
if (window.VoicePicker) {
|
||
VoicePicker.upgrade('conv-tts-voice-select');
|
||
VoicePicker.populate('conv-tts-voice-select', ids);
|
||
if (prev && ids.includes(prev)) VoicePicker.setValue('conv-tts-voice-select', prev);
|
||
else if (ids.length) VoicePicker.setValue('conv-tts-voice-select', ids[0]);
|
||
} else {
|
||
ttsVoiceSel.innerHTML = ids.length
|
||
? ids.map(id => `<option value="${escHtml(id)}">${escHtml(id)}</option>`).join('')
|
||
: '<option value="">No voices</option>';
|
||
if (prev && ids.includes(prev)) ttsVoiceSel.value = prev;
|
||
}
|
||
} catch(e) {
|
||
ttsVoiceSel.innerHTML = '<option value="">Fetch failed</option>';
|
||
if (window.VoicePicker) {
|
||
VoicePicker.upgrade('conv-tts-voice-select');
|
||
VoicePicker.populate('conv-tts-voice-select', []);
|
||
}
|
||
} finally {
|
||
ttsFetchBtn.disabled = false;
|
||
}
|
||
}
|
||
|
||
// ── Chat bubble helpers ──────────────────────────────────────────────────
|
||
function timeStr() {
|
||
const now = new Date();
|
||
return now.getHours().toString().padStart(2,'0') + ':' + now.getMinutes().toString().padStart(2,'0');
|
||
}
|
||
|
||
function removeWelcome() {
|
||
const w = chatWindow.querySelector('.conv-chat-welcome');
|
||
if (w) w.remove();
|
||
}
|
||
|
||
function addBubble(role, text) {
|
||
removeWelcome();
|
||
const wrap = document.createElement('div');
|
||
wrap.className = `conv-bubble-wrap conv-bubble-wrap--${role}`;
|
||
const bubble = document.createElement('div');
|
||
bubble.className = `conv-bubble conv-bubble--${role}`;
|
||
bubble.textContent = text || '';
|
||
const meta = document.createElement('div');
|
||
meta.className = 'conv-bubble-meta';
|
||
meta.textContent = timeStr();
|
||
wrap.appendChild(bubble);
|
||
wrap.appendChild(meta);
|
||
chatWindow.appendChild(wrap);
|
||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||
return bubble;
|
||
}
|
||
|
||
function addTypingBubble() {
|
||
removeWelcome();
|
||
const wrap = document.createElement('div');
|
||
wrap.className = 'conv-bubble-wrap conv-bubble-wrap--assistant';
|
||
wrap.id = 'conv-typing-wrap';
|
||
const bubble = document.createElement('div');
|
||
bubble.className = 'conv-bubble conv-bubble--assistant';
|
||
bubble.innerHTML = '<span class="conv-typing"><span></span><span></span><span></span></span>';
|
||
wrap.appendChild(bubble);
|
||
chatWindow.appendChild(wrap);
|
||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||
return bubble;
|
||
}
|
||
|
||
// Lazily builds the thinking-panel + answer-text structure inside an
|
||
// assistant bubble the first time either a 'thinking' or 'token' event
|
||
// arrives, replacing the "..." typing dots.
|
||
function ensureAssistantParts(bubble) {
|
||
if (bubble.querySelector('.conv-answer-text')) return;
|
||
bubble.innerHTML =
|
||
'<div class="conv-think-wrap" hidden>' +
|
||
'<button type="button" class="conv-think-toggle" aria-expanded="false">' +
|
||
'<span class="mdi mdi-chevron-right"></span> Thinking' +
|
||
'</button>' +
|
||
'<div class="conv-think-body" hidden></div>' +
|
||
'</div>' +
|
||
'<span class="conv-answer-text"></span>';
|
||
bubble.querySelector('.conv-think-toggle').addEventListener('click', function () {
|
||
const open = this.getAttribute('aria-expanded') === 'true';
|
||
this.setAttribute('aria-expanded', String(!open));
|
||
bubble.querySelector('.conv-think-body').hidden = open;
|
||
});
|
||
}
|
||
|
||
function appendThinking(bubble, delta) {
|
||
ensureAssistantParts(bubble);
|
||
const wrap = bubble.querySelector('.conv-think-wrap');
|
||
const toggle = bubble.querySelector('.conv-think-toggle');
|
||
const body = bubble.querySelector('.conv-think-body');
|
||
wrap.hidden = false;
|
||
body.textContent += delta;
|
||
|
||
// Auto-open the panel the first time reasoning arrives so the user can see
|
||
// the model thinking live. After that, preserve the user's manual toggle.
|
||
if (!bubble.dataset.thinkingSeen) {
|
||
bubble.dataset.thinkingSeen = '1';
|
||
toggle.setAttribute('aria-expanded', 'true');
|
||
body.hidden = false;
|
||
}
|
||
}
|
||
|
||
function setAnswerText(bubble, text) {
|
||
ensureAssistantParts(bubble);
|
||
bubble.dataset.answerStarted = '1';
|
||
bubble.querySelector('.conv-answer-text').textContent = text;
|
||
}
|
||
|
||
function addErrorBubble(msg) {
|
||
removeWelcome();
|
||
const wrap = document.createElement('div');
|
||
wrap.className = 'conv-bubble-wrap conv-bubble-wrap--assistant';
|
||
const bubble = document.createElement('div');
|
||
bubble.className = 'conv-bubble conv-bubble--error';
|
||
bubble.innerHTML = `<span class="mdi mdi-alert-outline"></span> ${escHtml(msg)}`;
|
||
wrap.appendChild(bubble);
|
||
chatWindow.appendChild(wrap);
|
||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||
}
|
||
|
||
// ── Stats panel ──────────────────────────────────────────────────────────
|
||
function fmtMs(ms) { return ms == null ? '—' : ms >= 1000 ? (ms/1000).toFixed(2)+'s' : ms+'ms'; }
|
||
|
||
function updateStatBar(id, val, maxVal) {
|
||
const fill = $(id);
|
||
if (fill) fill.style.width = maxVal > 0 ? Math.min(100, (val / maxVal) * 100) + '%' : '0%';
|
||
}
|
||
|
||
function updateStats(stats) {
|
||
const { stt_ms, llm_ttft_ms, llm_total_ms, tts_ms, total_ms } = stats;
|
||
const max = total_ms || 1;
|
||
const set = (valId, fillId, ms) => {
|
||
const el = $(valId); if (el) el.textContent = fmtMs(ms);
|
||
updateStatBar(fillId, ms || 0, max);
|
||
};
|
||
set('cpv-stt', 'cpf-stt', stt_ms);
|
||
set('cpv-ttft', 'cpf-ttft', llm_ttft_ms);
|
||
set('cpv-llm', 'cpf-llm', llm_total_ms);
|
||
set('cpv-tts', 'cpf-tts', tts_ms);
|
||
set('cpv-total', 'cpf-total', total_ms);
|
||
}
|
||
|
||
function addHistoryItem(n, totalMs, ok) {
|
||
const empty = turnHistory?.querySelector('.conv-history-empty');
|
||
if (empty) empty.remove();
|
||
const item = document.createElement('div');
|
||
item.className = 'conv-hist-item';
|
||
const cls = ok ? 'conv-hist-ok' : 'conv-hist-err';
|
||
const icon = ok ? 'mdi-check-circle-outline' : 'mdi-alert-outline';
|
||
item.innerHTML = `<span class="conv-hist-num">#${n}</span>
|
||
<span class="${cls}"><span class="mdi ${icon}"></span></span>
|
||
<span class="conv-hist-time ${cls}">${fmtMs(totalMs)}</span>`;
|
||
turnHistory.insertBefore(item, turnHistory.firstChild);
|
||
}
|
||
|
||
// ── Recording ────────────────────────────────────────────────────────────
|
||
function startRecTimer() {
|
||
recStart = Date.now();
|
||
recTimerInterval = setInterval(() => {
|
||
const s = Math.floor((Date.now() - recStart) / 1000);
|
||
if (micTimer) micTimer.textContent = s + 's';
|
||
}, 500);
|
||
}
|
||
|
||
function stopRecTimer() {
|
||
clearInterval(recTimerInterval);
|
||
if (micTimer) micTimer.textContent = '';
|
||
}
|
||
|
||
// ── Audio queue — plays multi-chunk TTS responses sequentially ──────────────
|
||
function clearAudio() {
|
||
if (convCurrentAudio) { try { convCurrentAudio.pause(); } catch(_){} convCurrentAudio = null; }
|
||
audioQueue.forEach(item => { if (item?.url) try { URL.revokeObjectURL(item.url); } catch(_){} });
|
||
audioQueue.length = 0;
|
||
audioQueuePlaying = false;
|
||
audioQueueDrainCb = null;
|
||
stopInterruptMonitor();
|
||
}
|
||
|
||
function agentBusy() {
|
||
return !!(isProcessing || audioQueuePlaying || audioQueue.length || convCurrentAudio || convAbortCtrl);
|
||
}
|
||
|
||
function isRecording() {
|
||
return !!(mediaRecorder && mediaRecorder.state === 'recording');
|
||
}
|
||
|
||
function setLiveMicUi(statusText) {
|
||
micBtn.classList.toggle('live', liveMicEnabled);
|
||
micBtn.title = liveMicEnabled ? 'Microphone is always listening - click to turn off' : 'Click to turn live microphone on';
|
||
if (!isRecording() && !isProcessing) micIcon.className = 'mdi mdi-microphone';
|
||
if (micStatus && statusText) micStatus.textContent = statusText;
|
||
}
|
||
|
||
function resetConversationInput(statusText = 'Ready') {
|
||
liveInterimText = '';
|
||
previewTranscribing = false;
|
||
isProcessing = false;
|
||
micBtn.classList.remove('processing', 'recording');
|
||
micIcon.className = 'mdi mdi-microphone';
|
||
setLiveMicUi();
|
||
if (sendBtn) sendBtn.disabled = false;
|
||
if (textInput) {
|
||
textInput.disabled = false;
|
||
textInput.readOnly = false;
|
||
textInput.classList.remove('listening');
|
||
textInput.placeholder = origPlaceholder;
|
||
textInput.value = '';
|
||
}
|
||
if (micStatus) micStatus.textContent = statusText;
|
||
}
|
||
|
||
function abortCurrentTurn() {
|
||
convTurnGeneration++;
|
||
if (convAbortCtrl) { try { convAbortCtrl.abort(); } catch(_){} }
|
||
convAbortCtrl = null;
|
||
}
|
||
|
||
async function interruptAgentAndListen() {
|
||
liveMicEnabled = true;
|
||
autoMicGeneration++;
|
||
abortCurrentTurn();
|
||
clearAudio();
|
||
resetConversationInput('Interrupted - listening...');
|
||
await startRecording();
|
||
}
|
||
|
||
function stopInterruptMonitor() {
|
||
cancelAnimationFrame(interruptRafId); interruptRafId = null;
|
||
if (interruptCtx) { try { interruptCtx.close(); } catch(_){} interruptCtx = null; }
|
||
if (interruptStream) { interruptStream.getTracks().forEach(t => t.stop()); interruptStream = null; }
|
||
interruptSpeechStart = 0;
|
||
}
|
||
|
||
async function startInterruptMonitor() {
|
||
if (interruptCtx || !navigator.mediaDevices?.getUserMedia) return;
|
||
try {
|
||
interruptStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||
interruptCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||
await interruptCtx.resume(); // may be suspended when created outside a user gesture
|
||
const src = interruptCtx.createMediaStreamSource(interruptStream);
|
||
const analyser = interruptCtx.createAnalyser();
|
||
analyser.fftSize = 256;
|
||
src.connect(analyser);
|
||
const buf = new Float32Array(analyser.fftSize);
|
||
|
||
function tick() {
|
||
// Keep listening during live-agent processing too, not only while audio is playing.
|
||
const processingTurn = (liveAgentOn() || liveMicEnabled) && (isProcessing || !!convAbortCtrl);
|
||
if (!audioQueuePlaying && !audioQueue.length && !convCurrentAudio && !processingTurn) {
|
||
stopInterruptMonitor();
|
||
return;
|
||
}
|
||
analyser.getFloatTimeDomainData(buf);
|
||
let rms = 0;
|
||
for (const s of buf) rms += s * s;
|
||
rms = Math.sqrt(rms / buf.length);
|
||
|
||
if (rms > INTERRUPT_THRESHOLD) {
|
||
if (!interruptSpeechStart) interruptSpeechStart = Date.now();
|
||
if (Date.now() - interruptSpeechStart >= interruptHoldMs()) {
|
||
// User is talking - interrupt the AI and listen immediately.
|
||
interruptAgentAndListen().catch(() => {});
|
||
return;
|
||
}
|
||
} else {
|
||
interruptSpeechStart = 0;
|
||
}
|
||
interruptRafId = requestAnimationFrame(tick);
|
||
}
|
||
interruptRafId = requestAnimationFrame(tick);
|
||
} catch (_) { stopInterruptMonitor(); }
|
||
}
|
||
|
||
function playNextAudio() {
|
||
if (!audioQueue.length) {
|
||
audioQueuePlaying = false;
|
||
convCurrentAudio = null;
|
||
convCurrentSentenceBubble = null;
|
||
if (audioQueueDrainCb) {
|
||
const cb = audioQueueDrainCb;
|
||
audioQueueDrainCb = null;
|
||
setTimeout(cb, 150);
|
||
} else if (micStatus && !isProcessing) {
|
||
micStatus.textContent = 'Ready';
|
||
}
|
||
return;
|
||
}
|
||
audioQueuePlaying = true;
|
||
const item = audioQueue.shift(); // {url, text}
|
||
// Show the sentence text in the typing bubble if no answer text has landed yet
|
||
if (convCurrentSentenceBubble && item.text) {
|
||
const answerSpan = convCurrentSentenceBubble.querySelector('.conv-answer-text');
|
||
if (convCurrentSentenceBubble.querySelector('.conv-typing') || (answerSpan && !answerSpan.textContent)) {
|
||
setAnswerText(convCurrentSentenceBubble, item.text);
|
||
}
|
||
}
|
||
const el = new Audio(item.url);
|
||
convCurrentAudio = el;
|
||
el.addEventListener('ended', () => { URL.revokeObjectURL(item.url); playNextAudio(); });
|
||
el.play().catch(() => { URL.revokeObjectURL(item.url); playNextAudio(); });
|
||
if (micStatus) micStatus.textContent = 'Speaking…';
|
||
startInterruptMonitor();
|
||
}
|
||
|
||
function enqueueAudio(b64, mime, text) {
|
||
const binStr = atob(b64);
|
||
const arr = new Uint8Array(binStr.length);
|
||
for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
|
||
const url = URL.createObjectURL(new Blob([arr], { type: mime || 'audio/wav' }));
|
||
audioQueue.push({ url, text: text || '' });
|
||
if (!audioQueuePlaying) playNextAudio();
|
||
}
|
||
|
||
// ── Chunked Whisper preview (fallback for browsers without SpeechRecognition) ──
|
||
async function transcribeForPreview() {
|
||
if (previewTranscribing || !recChunks.length) return;
|
||
// When VAD is on, gate on detected speech to avoid transcribing silence.
|
||
// When VAD is off the user controls recording manually — always transcribe.
|
||
if (vadToggle?.checked && (!vadHadSpeech || Date.now() - vadLastVoiceMs > 5000)) return;
|
||
previewTranscribing = true;
|
||
try {
|
||
const mime = (mediaRecorder && mediaRecorder.mimeType) || 'audio/webm';
|
||
const ext = mime.includes('ogg') ? '.ogg' : '.webm';
|
||
const blob = new Blob(recChunks, { type: mime });
|
||
const fd = new FormData();
|
||
fd.append('file', new File([blob], 'preview' + ext, { type: mime }));
|
||
fd.append('backend', sttSel?.value || 'configured');
|
||
const r = await fetch('/api/transcribe-bytes', { method: 'POST', body: fd });
|
||
if (r.ok) {
|
||
const d = await r.json();
|
||
const txt = (d.text || '').trim();
|
||
if (txt && txt.length > 3 && !HALLUCINATION_RE.test(txt)
|
||
&& mediaRecorder && mediaRecorder.state === 'recording') {
|
||
liveInterimText = txt;
|
||
if (textInput) textInput.value = txt;
|
||
}
|
||
}
|
||
} catch (_) {}
|
||
finally { previewTranscribing = false; }
|
||
}
|
||
|
||
// ── Auto-restart mic after agent finishes speaking ───────────────────────
|
||
function scheduleAutoMic() {
|
||
if (!liveMicEnabled && !handsFreeToggle?.checked) return;
|
||
const gen = ++autoMicGeneration;
|
||
function tryStart() {
|
||
if (gen !== autoMicGeneration || isProcessing) return;
|
||
if (!liveMicEnabled && !handsFreeToggle?.checked) return;
|
||
startRecording().catch(() => {});
|
||
}
|
||
if (audioQueuePlaying || audioQueue.length > 0) {
|
||
// Audio still queued — trigger after queue drains
|
||
audioQueueDrainCb = tryStart;
|
||
} else if (convCurrentAudio && !convCurrentAudio.ended) {
|
||
convCurrentAudio.addEventListener('ended', () => setTimeout(tryStart, 150), { once: true });
|
||
} else {
|
||
setTimeout(tryStart, 300);
|
||
}
|
||
}
|
||
|
||
async function startRecording() {
|
||
if (isProcessing) return;
|
||
stopInterruptMonitor(); // release mic stream before opening a recording stream
|
||
if (!navigator.mediaDevices?.getUserMedia) {
|
||
toast('Microphone unavailable — browser requires a secure context (HTTPS or localhost). ' +
|
||
'Access the app via http://localhost:7890 or enable it in chrome://flags/#unsafely-treat-insecure-origin-as-secure', 'error', 8000);
|
||
return;
|
||
}
|
||
let stream;
|
||
try {
|
||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||
} catch(e) {
|
||
toast('Microphone access denied: ' + e.message, 'error');
|
||
return;
|
||
}
|
||
|
||
liveInterimText = '';
|
||
previewTranscribing = false;
|
||
vadHadSpeech = false;
|
||
vadLastVoiceMs = 0;
|
||
vadSpeechStartMs = 0;
|
||
cancelNextBlob = false;
|
||
recChunks = [];
|
||
mediaRecorder = new MediaRecorder(stream);
|
||
mediaRecorder.ondataavailable = e => {
|
||
if (e.data.size > 0) {
|
||
recChunks.push(e.data);
|
||
// Only run preview when speech was detected (prevents "reich" on silence chunks)
|
||
if (mediaRecorder.state === 'recording' && vadHadSpeech) transcribeForPreview();
|
||
}
|
||
};
|
||
mediaRecorder.onstop = () => {
|
||
stream.getTracks().forEach(t => t.stop());
|
||
if (cancelNextBlob || cancelRecordingOnStop) {
|
||
// VAD fired with no speech, or the user turned the hot mic off.
|
||
cancelNextBlob = false;
|
||
cancelRecordingOnStop = false;
|
||
if (textInput) { textInput.readOnly = false; textInput.classList.remove('listening'); textInput.placeholder = origPlaceholder; textInput.value = ''; }
|
||
if (sendBtn) sendBtn.disabled = false;
|
||
micBtn.classList.remove('processing', 'recording');
|
||
micIcon.className = 'mdi mdi-microphone';
|
||
if (micStatus) micStatus.textContent = liveMicEnabled ? 'Listening…' : 'Ready';
|
||
isProcessing = false;
|
||
setLiveMicUi();
|
||
if (liveMicEnabled) startRecording().catch(() => {});
|
||
return;
|
||
}
|
||
const blob = new Blob(recChunks, { type: mediaRecorder.mimeType || 'audio/webm' });
|
||
processBlob(blob);
|
||
};
|
||
// timeslice: 750ms when VAD is off (manual recording) for faster Whisper preview;
|
||
// 1500ms with VAD on (chunks are gated anyway, smaller slices waste CPU).
|
||
mediaRecorder.start(vadToggle?.checked ? (liveAgentOn() ? 900 : 1500) : 750);
|
||
micBtn.classList.add('recording');
|
||
setLiveMicUi();
|
||
micIcon.className = liveMicEnabled ? 'mdi mdi-microphone' : 'mdi mdi-stop';
|
||
if (micStatus) micStatus.textContent = liveMicEnabled ? 'Listening…' : 'Recording…';
|
||
startRecTimer();
|
||
|
||
// Set input to listening mode (read-only; Whisper preview text will appear here)
|
||
if (textInput) {
|
||
textInput.readOnly = true;
|
||
textInput.value = '';
|
||
textInput.placeholder = 'Listening…';
|
||
textInput.classList.add('listening');
|
||
}
|
||
if (sendBtn) sendBtn.disabled = true;
|
||
|
||
// Also try Web Speech API for faster interim results (works on HTTPS / localhost)
|
||
const SpeechRec = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||
if (SpeechRec) {
|
||
try {
|
||
speechRec = new SpeechRec();
|
||
speechRec.continuous = true;
|
||
speechRec.interimResults = true;
|
||
speechRec.onresult = e => {
|
||
let final = '', interim = '';
|
||
for (let i = 0; i < e.results.length; i++) {
|
||
if (e.results[i].isFinal) final += e.results[i][0].transcript;
|
||
else interim += e.results[i][0].transcript;
|
||
}
|
||
liveInterimText = final + interim;
|
||
if (textInput) textInput.value = liveInterimText;
|
||
};
|
||
speechRec.onerror = () => { speechRec = null; };
|
||
speechRec.start();
|
||
} catch(_) { speechRec = null; }
|
||
}
|
||
|
||
// VAD: auto-stop on silence using AudioContext
|
||
if (vadToggle?.checked) {
|
||
try {
|
||
vadAudioCtx = new AudioContext();
|
||
const src = vadAudioCtx.createMediaStreamSource(stream);
|
||
const analyser = vadAudioCtx.createAnalyser();
|
||
analyser.fftSize = 1024;
|
||
src.connect(analyser);
|
||
const vadBuf = new Float32Array(analyser.fftSize);
|
||
let vadLastVoice = Date.now();
|
||
const levelWrap = $('conv-level-wrap');
|
||
const levelFill = $('conv-level-fill');
|
||
if (levelWrap) levelWrap.classList.add('active');
|
||
|
||
function vadTick() {
|
||
if (!mediaRecorder || mediaRecorder.state !== 'recording') return;
|
||
analyser.getFloatTimeDomainData(vadBuf);
|
||
let rms = 0;
|
||
for (const s of vadBuf) rms += s * s;
|
||
rms = Math.sqrt(rms / vadBuf.length);
|
||
const thr = vadThreshold();
|
||
if (levelFill) levelFill.style.width = Math.min(100, rms / thr * 40) + '%';
|
||
if (rms > thr) {
|
||
const now = Date.now();
|
||
if (!vadSpeechStartMs) vadSpeechStartMs = now;
|
||
vadLastVoice = now;
|
||
vadLastVoiceMs = now;
|
||
// Only mark as valid speech once it persists long enough
|
||
if (!vadHadSpeech && (now - vadSpeechStartMs) >= MIN_SPEECH_FRAMES_MS) vadHadSpeech = true;
|
||
} else {
|
||
vadSpeechStartMs = 0; // reset burst timer on silence
|
||
}
|
||
const elapsed = Date.now() - recStart;
|
||
const silence = Date.now() - vadLastVoice;
|
||
if (elapsed > VAD_MIN_REC_MS) {
|
||
const silenceLimit = vadSilenceMs();
|
||
const remaining = silenceLimit - silence;
|
||
if (micStatus) {
|
||
micStatus.textContent = liveMicEnabled && !vadHadSpeech
|
||
? 'Listening…'
|
||
: remaining < silenceLimit * 0.7
|
||
? `Sending in ${(Math.max(0, remaining) / 1000).toFixed(1)}s…`
|
||
: (liveMicEnabled ? 'Listening…' : 'Recording…');
|
||
}
|
||
if (silence >= silenceLimit) {
|
||
if (!vadHadSpeech) {
|
||
// Silence with no speech → cancel this segment (don't call STT).
|
||
// In hot-mic mode onstop restarts recording with a FRESH
|
||
// MediaRecorder, so every utterance keeps its own webm/EBML
|
||
// header and stays decodable. (Clearing recChunks in place would
|
||
// drop the header chunk → "EBML header parsing failed".)
|
||
cancelNextBlob = true;
|
||
if (micStatus) micStatus.textContent = liveMicEnabled ? 'Listening…' : 'Ready';
|
||
}
|
||
stopRecording();
|
||
return;
|
||
}
|
||
}
|
||
vadRafId = requestAnimationFrame(vadTick);
|
||
}
|
||
vadRafId = requestAnimationFrame(vadTick);
|
||
} catch(_) { vadAudioCtx = null; }
|
||
}
|
||
}
|
||
|
||
function stopRecording() {
|
||
if (!mediaRecorder || mediaRecorder.state === 'inactive') return;
|
||
|
||
// Clean up VAD
|
||
cancelAnimationFrame(vadRafId);
|
||
vadRafId = null;
|
||
if (vadAudioCtx) { try { vadAudioCtx.close(); } catch(_){} vadAudioCtx = null; }
|
||
const levelWrap = $('conv-level-wrap');
|
||
const levelFill = $('conv-level-fill');
|
||
if (levelWrap) levelWrap.classList.remove('active');
|
||
if (levelFill) levelFill.style.width = '0%';
|
||
|
||
// Clean up live speech recognition
|
||
if (speechRec) { try { speechRec.stop(); } catch(_){} speechRec = null; }
|
||
previewTranscribing = false;
|
||
|
||
mediaRecorder.stop();
|
||
stopRecTimer();
|
||
micBtn.classList.remove('recording');
|
||
micBtn.classList.add('processing');
|
||
micIcon.className = 'mdi mdi-dots-horizontal';
|
||
if (micStatus) micStatus.textContent = 'Processing…';
|
||
isProcessing = true;
|
||
}
|
||
|
||
// ── Send turn via SSE ─────────────────────────────────────────────────────
|
||
// Decode a recorded blob (webm/opus, ogg, …) in the browser and re-encode it
|
||
// as a 16 kHz mono WAV. This sidesteps server-side ffmpeg, whose webm/EBML
|
||
// parser fails on some platforms (ARM64): "EBML header parsing failed".
|
||
// Returns a WAV Blob, or null if decoding isn't possible (caller falls back).
|
||
async function blobToWav16k(blob) {
|
||
try {
|
||
if (!blob || !blob.size) return null;
|
||
const AC = window.AudioContext || window.webkitAudioContext;
|
||
if (!AC) return null;
|
||
const buf = await blob.arrayBuffer();
|
||
const ctx = new AC();
|
||
let audio;
|
||
try { audio = await ctx.decodeAudioData(buf.slice(0)); }
|
||
finally { try { ctx.close(); } catch (_) {} }
|
||
if (!audio || !audio.length) return null;
|
||
|
||
const targetRate = 16000;
|
||
const inRate = audio.sampleRate;
|
||
const chs = audio.numberOfChannels;
|
||
// Mix down to mono
|
||
const mono = new Float32Array(audio.length);
|
||
for (let c = 0; c < chs; c++) {
|
||
const d = audio.getChannelData(c);
|
||
for (let i = 0; i < d.length; i++) mono[i] += d[i] / chs;
|
||
}
|
||
// Linear resample to 16 kHz
|
||
const outLen = Math.max(1, Math.round(mono.length * targetRate / inRate));
|
||
const out = new Float32Array(outLen);
|
||
const ratio = mono.length / outLen;
|
||
for (let i = 0; i < outLen; i++) {
|
||
const pos = i * ratio, i0 = Math.floor(pos), i1 = Math.min(i0 + 1, mono.length - 1);
|
||
const f = pos - i0;
|
||
out[i] = mono[i0] * (1 - f) + mono[i1] * f;
|
||
}
|
||
// Encode 16-bit PCM WAV
|
||
const bytes = 44 + out.length * 2;
|
||
const ab = new ArrayBuffer(bytes);
|
||
const view = new DataView(ab);
|
||
const ws = (off, s) => { for (let i = 0; i < s.length; i++) view.setUint8(off + i, s.charCodeAt(i)); };
|
||
ws(0, 'RIFF'); view.setUint32(4, bytes - 8, true); ws(8, 'WAVE');
|
||
ws(12, 'fmt '); view.setUint32(16, 16, true); view.setUint16(20, 1, true);
|
||
view.setUint16(22, 1, true); view.setUint32(24, targetRate, true);
|
||
view.setUint32(28, targetRate * 2, true); view.setUint16(32, 2, true); view.setUint16(34, 16, true);
|
||
ws(36, 'data'); view.setUint32(40, out.length * 2, true);
|
||
let off = 44;
|
||
for (let i = 0; i < out.length; i++) {
|
||
let s = Math.max(-1, Math.min(1, out[i]));
|
||
view.setInt16(off, s < 0 ? s * 0x8000 : s * 0x7FFF, true); off += 2;
|
||
}
|
||
return new Blob([ab], { type: 'audio/wav' });
|
||
} catch (_) { return null; }
|
||
}
|
||
|
||
async function processBlob(blob) {
|
||
autoMicGeneration++; // cancel any pending auto-mic from previous turn
|
||
clearAudio();
|
||
turnCount++;
|
||
const turnN = turnCount;
|
||
const t0 = Date.now();
|
||
|
||
// Transition input from listening/readOnly state to processing/disabled state
|
||
if (textInput) {
|
||
textInput.readOnly = false;
|
||
textInput.classList.remove('listening');
|
||
textInput.placeholder = origPlaceholder;
|
||
textInput.value = '';
|
||
textInput.disabled = true;
|
||
}
|
||
if (sendBtn) sendBtn.disabled = true;
|
||
|
||
// Show user bubble seeded with live interim transcript (if available)
|
||
const userBubble = addBubble('user', liveInterimText || '…');
|
||
const assistantBubble = addTypingBubble();
|
||
convCurrentSentenceBubble = assistantBubble; // sentence text will appear here while audio plays
|
||
let assistantText = '';
|
||
let lastStats = null;
|
||
|
||
// Convert to WAV in-browser so the server never has to ffmpeg-decode webm.
|
||
const wav = await blobToWav16k(blob);
|
||
const upBlob = wav || blob;
|
||
const upName = wav ? 'audio.wav' : 'audio.webm';
|
||
|
||
const form = new FormData();
|
||
form.append('audio', upBlob, upName);
|
||
form.append('stt_backend', sttSel?.value || 'configured');
|
||
form.append('llm_url', llmUrlInp?.value.trim() || '');
|
||
form.append('llm_model', llmModelSel?.value || '');
|
||
form.append('tts_backend', ttsBkSel?.value || 'voice_clone');
|
||
form.append('tts_voice', ttsVoiceSel?.value || '');
|
||
form.append('tts_emotion', ttsEmotionSel?.value || '');
|
||
form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.');
|
||
form.append('history', JSON.stringify(conversationHistory.slice(-20)));
|
||
|
||
const turnGen = ++convTurnGeneration;
|
||
if (convAbortCtrl) { try { convAbortCtrl.abort(); } catch(_){} }
|
||
convAbortCtrl = new AbortController();
|
||
if (liveMicEnabled || liveAgentOn()) startInterruptMonitor();
|
||
|
||
try {
|
||
const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form, signal: convAbortCtrl.signal });
|
||
if (!resp.ok) { let _d=''; try { const _j=await resp.json(); _d=JSON.stringify(_j.detail||_j); } catch(_){} throw new Error('Server error ' + resp.status + (_d ? ': ' + _d : '')); }
|
||
const reader = resp.body.getReader();
|
||
const dec = new TextDecoder();
|
||
let buf = '';
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read();
|
||
if (turnGen !== convTurnGeneration) return;
|
||
if (done) break;
|
||
buf += dec.decode(value, { stream: true });
|
||
const lines = buf.split('\n');
|
||
buf = lines.pop();
|
||
for (const line of lines) {
|
||
if (turnGen !== convTurnGeneration) return;
|
||
if (!line.startsWith('data:')) continue;
|
||
let evt;
|
||
try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; }
|
||
|
||
if (evt.type === 'transcript') {
|
||
userBubble.textContent = evt.text || '(empty)';
|
||
if (micStatus) micStatus.textContent = 'Generating reply…';
|
||
} else if (evt.type === 'thinking') {
|
||
appendThinking(assistantBubble, evt.delta);
|
||
} else if (evt.type === 'token') {
|
||
convCurrentSentenceBubble = null; // LLM tokens take over the bubble now
|
||
assistantText += evt.delta;
|
||
setAnswerText(assistantBubble, assistantText);
|
||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||
} else if (evt.type === 'llm_done') {
|
||
assistantText = evt.text || assistantText;
|
||
setAnswerText(assistantBubble, assistantText);
|
||
convCurrentSentenceBubble = null;
|
||
if (micStatus) micStatus.textContent = 'Synthesising speech…';
|
||
} else if (evt.type === 'audio') {
|
||
enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
|
||
} else if (evt.type === 'stats') {
|
||
lastStats = evt;
|
||
updateStats(evt);
|
||
} else if (evt.type === 'done') {
|
||
conversationHistory.push({ role: 'user', content: userBubble.textContent });
|
||
conversationHistory.push({ role: 'assistant', content: assistantText });
|
||
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
|
||
scheduleAutoMic(); // triggers after audio queue drains
|
||
} else if (evt.type === 'error') {
|
||
clearAudio();
|
||
const wrap = assistantBubble.closest('.conv-bubble-wrap');
|
||
if (wrap) wrap.remove();
|
||
const stage = evt.stage?.toUpperCase() || 'ERR';
|
||
let msg = evt.message || 'Unknown error';
|
||
const detailMatch = msg.match(/HTTP \d+:\s*(.+)/s);
|
||
if (detailMatch) msg = detailMatch[1].trim();
|
||
if (msg.length > 300) msg = msg.slice(0, 300) + '…';
|
||
addErrorBubble(`[${stage}] ${msg}`);
|
||
addHistoryItem(turnN, Date.now() - t0, false);
|
||
if (micStatus) micStatus.textContent = 'Error — ready';
|
||
}
|
||
}
|
||
}
|
||
} catch(e) {
|
||
if (e?.name !== 'AbortError' && turnGen === convTurnGeneration) {
|
||
clearAudio();
|
||
const wrap = assistantBubble.closest('.conv-bubble-wrap');
|
||
if (wrap) wrap.remove();
|
||
addErrorBubble(e.message);
|
||
addHistoryItem(turnN, Date.now() - t0, false);
|
||
if (micStatus) micStatus.textContent = 'Error — ready';
|
||
}
|
||
} finally {
|
||
if (turnGen === convTurnGeneration) {
|
||
convAbortCtrl = null;
|
||
liveInterimText = '';
|
||
previewTranscribing = false;
|
||
isProcessing = false;
|
||
micBtn.classList.remove('processing');
|
||
micIcon.className = 'mdi mdi-microphone';
|
||
setLiveMicUi();
|
||
if (sendBtn) sendBtn.disabled = false;
|
||
if (textInput) { textInput.disabled = false; textInput.value = ''; }
|
||
if (audioQueuePlaying || audioQueue.length || convCurrentAudio) startInterruptMonitor();
|
||
else if (liveMicEnabled) scheduleAutoMic();
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── Text-input turn (skips STT, sends text directly) ────────────────────
|
||
async function processText(text) {
|
||
text = text.trim();
|
||
if (!text || isProcessing) return;
|
||
autoMicGeneration++; // cancel any pending auto-mic
|
||
clearAudio();
|
||
isProcessing = true;
|
||
if (sendBtn) sendBtn.disabled = true;
|
||
if (textInput) { textInput.disabled = true; textInput.value = ''; }
|
||
micBtn.classList.add('processing');
|
||
micIcon.className = 'mdi mdi-dots-horizontal';
|
||
if (micStatus) micStatus.textContent = 'Processing…';
|
||
|
||
turnCount++;
|
||
const turnN = turnCount;
|
||
const t0 = Date.now();
|
||
|
||
const userBubble = addBubble('user', text);
|
||
const assistantBubble = addTypingBubble();
|
||
convCurrentSentenceBubble = assistantBubble;
|
||
let assistantText = '';
|
||
let lastStats = null;
|
||
|
||
const form = new FormData();
|
||
form.append('text', text);
|
||
form.append('stt_backend', sttSel?.value || 'configured');
|
||
form.append('llm_url', llmUrlInp?.value.trim() || '');
|
||
form.append('llm_model', llmModelSel?.value || '');
|
||
form.append('tts_backend', ttsBkSel?.value || 'voice_clone');
|
||
form.append('tts_voice', ttsVoiceSel?.value || '');
|
||
form.append('tts_emotion', ttsEmotionSel?.value || '');
|
||
form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.');
|
||
form.append('history', JSON.stringify(conversationHistory.slice(-20)));
|
||
|
||
const turnGen = ++convTurnGeneration;
|
||
if (convAbortCtrl) { try { convAbortCtrl.abort(); } catch(_){} }
|
||
convAbortCtrl = new AbortController();
|
||
if (liveMicEnabled || liveAgentOn()) startInterruptMonitor();
|
||
|
||
try {
|
||
const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form, signal: convAbortCtrl.signal });
|
||
if (!resp.ok) { let _d=''; try { const _j=await resp.json(); _d=JSON.stringify(_j.detail||_j); } catch(_){} throw new Error('Server error ' + resp.status + (_d ? ': ' + _d : '')); }
|
||
const reader = resp.body.getReader();
|
||
const dec = new TextDecoder();
|
||
let buf = '';
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read();
|
||
if (turnGen !== convTurnGeneration) return;
|
||
if (done) break;
|
||
buf += dec.decode(value, { stream: true });
|
||
const lines = buf.split('\n');
|
||
buf = lines.pop();
|
||
for (const line of lines) {
|
||
if (turnGen !== convTurnGeneration) return;
|
||
if (!line.startsWith('data:')) continue;
|
||
let evt;
|
||
try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; }
|
||
|
||
if (evt.type === 'transcript') {
|
||
if (micStatus) micStatus.textContent = 'Generating reply…';
|
||
} else if (evt.type === 'thinking') {
|
||
appendThinking(assistantBubble, evt.delta);
|
||
} else if (evt.type === 'token') {
|
||
convCurrentSentenceBubble = null;
|
||
assistantText += evt.delta;
|
||
setAnswerText(assistantBubble, assistantText);
|
||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||
} else if (evt.type === 'llm_done') {
|
||
assistantText = evt.text || assistantText;
|
||
setAnswerText(assistantBubble, assistantText);
|
||
convCurrentSentenceBubble = null;
|
||
if (micStatus) micStatus.textContent = 'Synthesising speech…';
|
||
} else if (evt.type === 'audio') {
|
||
enqueueAudio(evt.b64, evt.mime || 'audio/wav', evt.text || '');
|
||
} else if (evt.type === 'stats') {
|
||
lastStats = evt;
|
||
updateStats(evt);
|
||
} else if (evt.type === 'done') {
|
||
conversationHistory.push({ role: 'user', content: text });
|
||
conversationHistory.push({ role: 'assistant', content: assistantText });
|
||
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
|
||
scheduleAutoMic(); // triggers after audio queue drains
|
||
} else if (evt.type === 'error') {
|
||
clearAudio();
|
||
const wrap = assistantBubble.closest('.conv-bubble-wrap');
|
||
if (wrap) wrap.remove();
|
||
addErrorBubble(`[${evt.stage?.toUpperCase() || 'ERR'}] ${evt.message || 'Unknown error'}`);
|
||
addHistoryItem(turnN, Date.now() - t0, false);
|
||
if (micStatus) micStatus.textContent = 'Error — ready';
|
||
}
|
||
}
|
||
}
|
||
} catch(e) {
|
||
if (e?.name !== 'AbortError' && turnGen === convTurnGeneration) {
|
||
clearAudio();
|
||
const wrap = assistantBubble.closest('.conv-bubble-wrap');
|
||
if (wrap) wrap.remove();
|
||
addErrorBubble(e.message);
|
||
addHistoryItem(turnN, Date.now() - t0, false);
|
||
if (micStatus) micStatus.textContent = 'Error — ready';
|
||
}
|
||
} finally {
|
||
if (turnGen === convTurnGeneration) {
|
||
convAbortCtrl = null;
|
||
isProcessing = false;
|
||
micBtn.classList.remove('processing');
|
||
micIcon.className = 'mdi mdi-microphone';
|
||
setLiveMicUi();
|
||
if (sendBtn) sendBtn.disabled = false;
|
||
if (textInput) textInput.disabled = false;
|
||
if (audioQueuePlaying || audioQueue.length || convCurrentAudio) startInterruptMonitor();
|
||
else if (liveMicEnabled) scheduleAutoMic();
|
||
}
|
||
}
|
||
}
|
||
|
||
async function enableLiveMic() {
|
||
liveMicEnabled = true;
|
||
autoMicGeneration++;
|
||
if (vadToggle) vadToggle.checked = true;
|
||
if (handsFreeToggle) handsFreeToggle.checked = true;
|
||
if (liveAgentToggle) liveAgentToggle.checked = true;
|
||
setLiveMicUi('Listening…');
|
||
if (isRecording()) return;
|
||
if (agentBusy()) await interruptAgentAndListen();
|
||
else await startRecording();
|
||
}
|
||
|
||
function disableLiveMic() {
|
||
liveMicEnabled = false;
|
||
if (handsFreeToggle) handsFreeToggle.checked = false;
|
||
autoMicGeneration++;
|
||
abortCurrentTurn();
|
||
clearAudio();
|
||
if (isRecording()) {
|
||
cancelRecordingOnStop = true;
|
||
stopRecording();
|
||
} else {
|
||
resetConversationInput('Mic off');
|
||
}
|
||
setLiveMicUi('Mic off');
|
||
}
|
||
|
||
// ── Wire up events ───────────────────────────────────────────────────────
|
||
micBtn.addEventListener('click', () => {
|
||
if (liveMicEnabled) disableLiveMic();
|
||
else enableLiveMic().catch(() => {});
|
||
});
|
||
|
||
// Text input — Enter key or Send button
|
||
sendBtn?.addEventListener('click', () => processText(textInput?.value || ''));
|
||
textInput?.addEventListener('keydown', e => {
|
||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); processText(textInput.value); }
|
||
});
|
||
|
||
clearBtn?.addEventListener('click', () => {
|
||
conversationHistory = [];
|
||
turnCount = 0;
|
||
chatWindow.innerHTML = '<div class="conv-chat-welcome"><span class="mdi mdi-forum-outline" style="font-size:32px;opacity:.25"></span><p>Type a message or press the microphone button below to start.</p></div>';
|
||
if (turnHistory) turnHistory.innerHTML = '<div class="conv-history-empty">No turns yet.</div>';
|
||
['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; });
|
||
['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; });
|
||
});
|
||
|
||
llmFetchBtn?.addEventListener('click', fetchLlmModels);
|
||
ttsFetchBtn?.addEventListener('click', fetchConvTtsVoices);
|
||
|
||
// Re-populate TTS when backend changes
|
||
ttsBkSel?.addEventListener('change', () => {
|
||
ttsVoiceSel.innerHTML = '<option value="">— fetch voices —</option>';
|
||
if (window.VoicePicker) {
|
||
VoicePicker.upgrade('conv-tts-voice-select');
|
||
VoicePicker.populate('conv-tts-voice-select', []);
|
||
}
|
||
});
|
||
liveAgentToggle?.addEventListener('change', () => {
|
||
if (liveAgentToggle.checked) {
|
||
if (vadToggle) vadToggle.checked = true;
|
||
if (handsFreeToggle) handsFreeToggle.checked = true;
|
||
toast('Live agent mode on: faster silence cut-off and barge-in', 'success');
|
||
} else {
|
||
toast('Live agent mode off', 'info');
|
||
}
|
||
});
|
||
|
||
// ── Init ─────────────────────────────────────────────────────────────────
|
||
loadConvSttBackends();
|
||
populateConvTtsBackends();
|
||
if (window.VoicePicker) VoicePicker.upgrade('conv-tts-voice-select');
|
||
setLiveMicUi();
|
||
|
||
// Keep TTS backend select in sync after global backend refresh
|
||
window._ttsRefreshHooks = window._ttsRefreshHooks || [];
|
||
window._ttsRefreshHooks.push(populateConvTtsBackends);
|
||
|
||
// ── Stats panel collapse ─────────────────────────────────────────────────
|
||
(function () {
|
||
const panel = document.getElementById('conv-stats-panel');
|
||
const colBtn = document.getElementById('conv-stats-collapse-btn');
|
||
const expBtn = document.getElementById('conv-stats-expand-btn');
|
||
const KEY = 'ttsvc_conv_stats_collapsed';
|
||
function setCollapsed(on) {
|
||
if (!panel) return;
|
||
panel.classList.toggle('collapsed', on);
|
||
if (expBtn) expBtn.style.display = on ? 'flex' : 'none';
|
||
try { localStorage.setItem(KEY, on ? '1' : '0'); } catch (_) {}
|
||
}
|
||
if (colBtn) colBtn.addEventListener('click', function () { setCollapsed(true); });
|
||
if (expBtn) expBtn.addEventListener('click', function () { setCollapsed(false); });
|
||
try { if (localStorage.getItem(KEY) === '1') setCollapsed(true); } catch (_) {}
|
||
})();
|
||
})();
|