From 0a5b607fa5dd6fe65f60660010e1a20543059d7b Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Fri, 29 May 2026 13:12:27 +0200 Subject: [PATCH] Add text input to Conversation Playground (bypass mic / STT) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frontend: - Add pill-shaped text input + send button (→) to the left of the mic button - Enter key or → click sends text directly without recording audio - Input is disabled while a turn is processing; cleared on submit - Welcome message updated to mention both input methods - New CSS: .conv-input-bar, .conv-text-row, .conv-text-inp, .conv-send-btn, .conv-divider (visual separator between text and mic sections) Backend: - /api/conversation/turn: audio is now optional (UploadFile | None) - New text form field — when provided, STT step is skipped and text is used as the transcript directly; SSE emits transcript event with stt_ms=null - Raises 400 if neither audio nor text is supplied Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 7 ++ routes/conversation.py | 70 ++++++++++------- static/js/conversation.js | 114 +++++++++++++++++++++++++++- static/sections/s-conversation.html | 23 ++++-- static/style.css | 35 ++++++++- 5 files changed, 208 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c38d56b..54a7467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- **Text input in Conversation Playground** — a pill-shaped text field and send + button (→) now sit left of the mic button. Typing a message and pressing Enter + or → skips STT entirely and sends text directly through LLM → TTS. Makes the + playground fully usable without a microphone (HTTP context, no mic permission, + remote access). Backend `/api/conversation/turn` now accepts an optional `text` + form field; when set, the STT step is skipped and the STT latency row shows `—`. + - **Container name field on all engine cards** — every TTS and STT engine card (Docker stack cards *and* static "Other Local" cards) now always shows the Docker container name input row. Previously absent/not-installed cards hid it; diff --git a/routes/conversation.py b/routes/conversation.py index e0f57db..625baca 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -659,7 +659,8 @@ async def conversation_llm_models(url: str = ""): @router.post("/api/conversation/turn") async def conversation_turn( - audio: UploadFile = File(...), + audio: UploadFile | None = File(None), + text: str = Form(""), stt_backend: str = Form("configured"), llm_url: str = Form(""), llm_model: str = Form(""), @@ -668,21 +669,31 @@ async def conversation_turn( system_prompt: str = Form("You are a helpful voice assistant. Keep replies short and conversational."), history: str = Form("[]"), ): - """Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events.""" + """Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events. + Pass either an audio file (runs STT first) or a plain text string (skips STT). + """ + direct_text = text.strip() + if not direct_text and (audio is None or not getattr(audio, "filename", None)): + raise HTTPException(400, "Provide either an audio file or a text field") + settings = _load_settings() eff_llm_url = (llm_url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/") - suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm" - if suffix not in _UPLOAD_EXTS: - suffix = ".webm" - tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}" - try: - with tmp.open("wb") as f: - _copy_limited(audio.file, f, _MAX_UPLOAD_BYTES) - wav_tmp = _to_wav_16k(tmp) - except Exception as e: - tmp.unlink(missing_ok=True) - raise HTTPException(400, f"Audio upload failed: {e}") + # Prepare audio temp files only when an audio upload was provided + _tmp: Path | None = None + _wav: Path | None = None + if not direct_text and audio is not None: + suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm" + if suffix not in _UPLOAD_EXTS: + suffix = ".webm" + _tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}" + try: + with _tmp.open("wb") as f: + _copy_limited(audio.file, f, _MAX_UPLOAD_BYTES) + _wav = _to_wav_16k(_tmp) + except Exception as e: + _tmp.unlink(missing_ok=True) + raise HTTPException(400, f"Audio upload failed: {e}") try: hist = json.loads(history) if history else [] @@ -693,7 +704,6 @@ async def conversation_turn( stt_be = _clean_stt_backend(stt_backend) tts_be = _clean_preview_backend(tts_backend) - _tmp, _wav = tmp, wav_tmp async def generate(): t0 = time.monotonic() @@ -703,21 +713,23 @@ async def conversation_turn( def sse(obj: dict) -> str: return f"data: {json.dumps(obj)}\n\n" - # 1. STT - try: - t_stt = time.monotonic() - transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be) - stt_ms = int((time.monotonic() - t_stt) * 1000) - yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms}) - except Exception as e: - yield sse({"type": "error", "stage": "stt", "message": str(e)}) - return - finally: - for p in {_tmp, _wav}: - try: - p.unlink(missing_ok=True) - except Exception: - pass + # 1. STT — skipped when caller sends direct text + if direct_text: + transcript = direct_text + yield sse({"type": "transcript", "text": transcript, "stt_ms": None}) + else: + try: + t_stt = time.monotonic() + transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be) + stt_ms = int((time.monotonic() - t_stt) * 1000) + yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms}) + except Exception as e: + yield sse({"type": "error", "stage": "stt", "message": str(e)}) + return + finally: + for p in filter(None, {_tmp, _wav}): + with contextlib.suppress(Exception): + p.unlink(missing_ok=True) if not transcript.strip(): yield sse({"type": "error", "stage": "stt", "message": "No speech detected."}) diff --git a/static/js/conversation.js b/static/js/conversation.js index 4c9b2dd..56c517f 100644 --- a/static/js/conversation.js +++ b/static/js/conversation.js @@ -106,6 +106,8 @@ $('s-import-voices-file')?.addEventListener('change', async function () { const micIcon = $('conv-mic-icon'); const micStatus = $('conv-mic-status'); const micTimer = $('conv-mic-timer'); + const textInput = $('conv-text-input'); + const sendBtn = $('conv-send-btn'); const clearBtn = $('conv-clear-btn'); const sttSel = $('conv-stt-select'); const llmUrlInp = $('conv-llm-url'); @@ -489,6 +491,110 @@ $('s-import-voices-file')?.addEventListener('change', async function () { isProcessing = false; micBtn.classList.remove('processing'); micIcon.className = 'mdi mdi-microphone'; + if (sendBtn) sendBtn.disabled = false; + if (textInput) textInput.disabled = false; + } + } + + // ── Text-input turn (skips STT, sends text directly) ──────────────────── + async function processText(text) { + text = text.trim(); + if (!text || isProcessing) return; + isProcessing = true; + if (sendBtn) sendBtn.disabled = true; + if (textInput) { textInput.disabled = true; textInput.value = ''; } + micBtn.classList.add('processing'); + micIcon.className = 'mdi mdi-dots-horizontal'; + if (micStatus) micStatus.textContent = 'Processing…'; + + turnCount++; + const turnN = turnCount; + const t0 = Date.now(); + + const userBubble = addBubble('user', text); + const assistantBubble = addTypingBubble(); + let assistantText = ''; + let lastStats = null; + + const form = new FormData(); + form.append('text', text); + form.append('stt_backend', sttSel?.value || 'configured'); + form.append('llm_url', llmUrlInp?.value.trim() || ''); + form.append('llm_model', llmModelSel?.value || ''); + form.append('tts_backend', ttsBkSel?.value || 'voice_clone'); + form.append('tts_voice', ttsVoiceSel?.value || ''); + form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.'); + form.append('history', JSON.stringify(conversationHistory.slice(-20))); + + try { + const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form }); + if (!resp.ok) throw new Error('Server error ' + resp.status); + const reader = resp.body.getReader(); + const dec = new TextDecoder(); + let buf = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += dec.decode(value, { stream: true }); + const lines = buf.split('\n'); + buf = lines.pop(); + for (const line of lines) { + if (!line.startsWith('data:')) continue; + let evt; + try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; } + + if (evt.type === 'transcript') { + if (micStatus) micStatus.textContent = 'Generating reply…'; + } else if (evt.type === 'token') { + if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = ''; + assistantText += evt.delta; + assistantBubble.textContent = assistantText; + chatWindow.scrollTop = chatWindow.scrollHeight; + } else if (evt.type === 'llm_done') { + assistantText = evt.text || assistantText; + assistantBubble.textContent = assistantText; + if (micStatus) micStatus.textContent = 'Synthesising speech…'; + } else if (evt.type === 'audio') { + const mime = evt.mime || 'audio/wav'; + const binStr = atob(evt.b64); + const arr = new Uint8Array(binStr.length); + for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i); + const audioBlob = new Blob([arr], { type: mime }); + const url = URL.createObjectURL(audioBlob); + const audio = new Audio(url); + audio.onended = () => URL.revokeObjectURL(url); + audio.play().catch(() => {}); + if (micStatus) micStatus.textContent = 'Speaking…'; + } else if (evt.type === 'stats') { + lastStats = evt; + updateStats(evt); + } else if (evt.type === 'done') { + conversationHistory.push({ role: 'user', content: text }); + conversationHistory.push({ role: 'assistant', content: assistantText }); + addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true); + if (micStatus) micStatus.textContent = 'Ready'; + } else if (evt.type === 'error') { + const wrap = assistantBubble.closest('.conv-bubble-wrap'); + if (wrap) wrap.remove(); + addErrorBubble(`[${evt.stage?.toUpperCase() || 'ERR'}] ${evt.message || 'Unknown error'}`); + addHistoryItem(turnN, Date.now() - t0, false); + if (micStatus) micStatus.textContent = 'Error — ready'; + } + } + } + } catch(e) { + const wrap = assistantBubble.closest('.conv-bubble-wrap'); + if (wrap) wrap.remove(); + addErrorBubble(e.message); + addHistoryItem(turnN, Date.now() - t0, false); + if (micStatus) micStatus.textContent = 'Error — ready'; + } finally { + isProcessing = false; + micBtn.classList.remove('processing'); + micIcon.className = 'mdi mdi-microphone'; + if (sendBtn) sendBtn.disabled = false; + if (textInput) textInput.disabled = false; } } @@ -502,10 +608,16 @@ $('s-import-voices-file')?.addEventListener('change', async function () { } }); + // Text input — Enter key or Send button + sendBtn?.addEventListener('click', () => processText(textInput?.value || '')); + textInput?.addEventListener('keydown', e => { + if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); processText(textInput.value); } + }); + clearBtn?.addEventListener('click', () => { conversationHistory = []; turnCount = 0; - chatWindow.innerHTML = '

Press the microphone button below and start talking.

'; + chatWindow.innerHTML = '

Type a message or press the microphone button below to start.

'; if (turnHistory) turnHistory.innerHTML = '
No turns yet.
'; ['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; }); ['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; }); diff --git a/static/sections/s-conversation.html b/static/sections/s-conversation.html index e333979..1fcea0f 100644 --- a/static/sections/s-conversation.html +++ b/static/sections/s-conversation.html @@ -47,17 +47,26 @@
-

Press the microphone button below and start talking.

+

Type a message or press the microphone button below to start.

- -
+ +
Ready
- -
+
+ + +
+ +
+
diff --git a/static/style.css b/static/style.css index 00dec2b..afb9b1d 100644 --- a/static/style.css +++ b/static/style.css @@ -2053,14 +2053,41 @@ code { background: var(--panel); border-radius: 4px; padding: 1px 5px; font-fami .conv-typing span:nth-child(3) { animation-delay: .4s; } @keyframes convDot { 0%,80%,100% { transform: scale(.7); opacity:.3; } 40% { transform: scale(1); opacity:.9; } } -/* Mic bar */ -.conv-mic-bar { background: var(--surface); border: 1px solid var(--border); border-top: none; border-radius: 0 0 var(--radius) var(--radius); padding: 12px 16px; display: flex; align-items: center; gap: 14px; } -.conv-mic-btn { width: 52px; height: 52px; border-radius: 50%; border: 2px solid var(--accent); background: var(--accent); color: #fff; font-size: 22px; cursor: pointer; display: flex; align-items: center; justify-content: center; transition: background .15s, transform .1s, box-shadow .15s; flex-shrink: 0; } +/* ── Conversation input bar (text field + send + mic) ───────────────────── */ +.conv-input-bar { + background: var(--surface); border: 1px solid var(--border); border-top: none; + border-radius: 0 0 var(--radius) var(--radius); padding: 10px 16px 12px; + display: flex; flex-direction: column; gap: 8px; +} +.conv-mic-status { font-size: 13px; color: var(--subtext); min-height: 18px; } +.conv-text-row { display: flex; align-items: center; gap: 8px; } +.conv-text-inp { + flex: 1; min-width: 0; padding: 10px 14px; font-size: 14px; font-family: inherit; + border: 1.5px solid var(--border); border-radius: 22px; + background: var(--panel); color: var(--text); outline: none; + transition: border-color .15s; +} +.conv-text-inp:focus { border-color: var(--accent); } +.conv-text-inp::placeholder { color: var(--subtext); opacity: .7; } +.conv-send-btn { + flex-shrink: 0; width: 40px; height: 40px; border-radius: 50%; + border: none; background: var(--accent); color: #fff; + font-size: 17px; cursor: pointer; display: flex; align-items: center; + justify-content: center; transition: background .15s, transform .1s; +} +.conv-send-btn:hover:not(:disabled) { background: #1d4ed8; transform: scale(1.05); } +.conv-send-btn:disabled { opacity: .4; cursor: default; } +.conv-divider { width: 1px; height: 32px; background: var(--border); flex-shrink: 0; } +.conv-mic-btn { + flex-shrink: 0; width: 52px; height: 52px; border-radius: 50%; + border: 2px solid var(--accent); background: var(--accent); color: #fff; + font-size: 22px; cursor: pointer; display: flex; align-items: center; + justify-content: center; transition: background .15s, transform .1s, box-shadow .15s; +} .conv-mic-btn:hover { background: #1d4ed8; } .conv-mic-btn.recording { background: var(--red); border-color: var(--red); animation: convPulse 1s infinite; } .conv-mic-btn.processing { background: var(--yellow); border-color: var(--yellow); cursor: default; } @keyframes convPulse { 0%,100% { box-shadow: 0 0 0 0 rgba(220,38,38,.4); } 50% { box-shadow: 0 0 0 8px rgba(220,38,38,0); } } -.conv-mic-status { flex: 1; font-size: 13px; color: var(--subtext); } .conv-mic-timer { font-size: 13px; font-variant-numeric: tabular-nums; color: var(--red); min-width: 36px; text-align: right; } /* Stats panel */