Add text input to Conversation Playground (bypass mic / STT)
Frontend: - Add pill-shaped text input + send button (→) to the left of the mic button - Enter key or → click sends text directly without recording audio - Input is disabled while a turn is processing; cleared on submit - Welcome message updated to mention both input methods - New CSS: .conv-input-bar, .conv-text-row, .conv-text-inp, .conv-send-btn, .conv-divider (visual separator between text and mic sections) Backend: - /api/conversation/turn: audio is now optional (UploadFile | None) - New text form field — when provided, STT step is skipped and text is used as the transcript directly; SSE emits transcript event with stt_ms=null - Raises 400 if neither audio nor text is supplied Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
a9d4c45a3c
commit
0a5b607fa5
@ -9,6 +9,13 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
||||
|
||||
### Added
|
||||
|
||||
- **Text input in Conversation Playground** — a pill-shaped text field and send
|
||||
button (→) now sit left of the mic button. Typing a message and pressing Enter
|
||||
or → skips STT entirely and sends text directly through LLM → TTS. Makes the
|
||||
playground fully usable without a microphone (HTTP context, no mic permission,
|
||||
remote access). Backend `/api/conversation/turn` now accepts an optional `text`
|
||||
form field; when set, the STT step is skipped and the STT latency row shows `—`.
|
||||
|
||||
- **Container name field on all engine cards** — every TTS and STT engine card
|
||||
(Docker stack cards *and* static "Other Local" cards) now always shows the
|
||||
Docker container name input row. Previously absent/not-installed cards hid it;
|
||||
|
||||
@ -659,7 +659,8 @@ async def conversation_llm_models(url: str = ""):
|
||||
|
||||
@router.post("/api/conversation/turn")
|
||||
async def conversation_turn(
|
||||
audio: UploadFile = File(...),
|
||||
audio: UploadFile | None = File(None),
|
||||
text: str = Form(""),
|
||||
stt_backend: str = Form("configured"),
|
||||
llm_url: str = Form(""),
|
||||
llm_model: str = Form(""),
|
||||
@ -668,21 +669,31 @@ async def conversation_turn(
|
||||
system_prompt: str = Form("You are a helpful voice assistant. Keep replies short and conversational."),
|
||||
history: str = Form("[]"),
|
||||
):
|
||||
"""Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events."""
|
||||
"""Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events.
|
||||
Pass either an audio file (runs STT first) or a plain text string (skips STT).
|
||||
"""
|
||||
direct_text = text.strip()
|
||||
if not direct_text and (audio is None or not getattr(audio, "filename", None)):
|
||||
raise HTTPException(400, "Provide either an audio file or a text field")
|
||||
|
||||
settings = _load_settings()
|
||||
eff_llm_url = (llm_url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||||
|
||||
suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm"
|
||||
if suffix not in _UPLOAD_EXTS:
|
||||
suffix = ".webm"
|
||||
tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}"
|
||||
try:
|
||||
with tmp.open("wb") as f:
|
||||
_copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
|
||||
wav_tmp = _to_wav_16k(tmp)
|
||||
except Exception as e:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise HTTPException(400, f"Audio upload failed: {e}")
|
||||
# Prepare audio temp files only when an audio upload was provided
|
||||
_tmp: Path | None = None
|
||||
_wav: Path | None = None
|
||||
if not direct_text and audio is not None:
|
||||
suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm"
|
||||
if suffix not in _UPLOAD_EXTS:
|
||||
suffix = ".webm"
|
||||
_tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}"
|
||||
try:
|
||||
with _tmp.open("wb") as f:
|
||||
_copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
|
||||
_wav = _to_wav_16k(_tmp)
|
||||
except Exception as e:
|
||||
_tmp.unlink(missing_ok=True)
|
||||
raise HTTPException(400, f"Audio upload failed: {e}")
|
||||
|
||||
try:
|
||||
hist = json.loads(history) if history else []
|
||||
@ -693,7 +704,6 @@ async def conversation_turn(
|
||||
|
||||
stt_be = _clean_stt_backend(stt_backend)
|
||||
tts_be = _clean_preview_backend(tts_backend)
|
||||
_tmp, _wav = tmp, wav_tmp
|
||||
|
||||
async def generate():
|
||||
t0 = time.monotonic()
|
||||
@ -703,21 +713,23 @@ async def conversation_turn(
|
||||
def sse(obj: dict) -> str:
|
||||
return f"data: {json.dumps(obj)}\n\n"
|
||||
|
||||
# 1. STT
|
||||
try:
|
||||
t_stt = time.monotonic()
|
||||
transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be)
|
||||
stt_ms = int((time.monotonic() - t_stt) * 1000)
|
||||
yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms})
|
||||
except Exception as e:
|
||||
yield sse({"type": "error", "stage": "stt", "message": str(e)})
|
||||
return
|
||||
finally:
|
||||
for p in {_tmp, _wav}:
|
||||
try:
|
||||
p.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
# 1. STT — skipped when caller sends direct text
|
||||
if direct_text:
|
||||
transcript = direct_text
|
||||
yield sse({"type": "transcript", "text": transcript, "stt_ms": None})
|
||||
else:
|
||||
try:
|
||||
t_stt = time.monotonic()
|
||||
transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be)
|
||||
stt_ms = int((time.monotonic() - t_stt) * 1000)
|
||||
yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms})
|
||||
except Exception as e:
|
||||
yield sse({"type": "error", "stage": "stt", "message": str(e)})
|
||||
return
|
||||
finally:
|
||||
for p in filter(None, {_tmp, _wav}):
|
||||
with contextlib.suppress(Exception):
|
||||
p.unlink(missing_ok=True)
|
||||
|
||||
if not transcript.strip():
|
||||
yield sse({"type": "error", "stage": "stt", "message": "No speech detected."})
|
||||
|
||||
@ -106,6 +106,8 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
const micIcon = $('conv-mic-icon');
|
||||
const micStatus = $('conv-mic-status');
|
||||
const micTimer = $('conv-mic-timer');
|
||||
const textInput = $('conv-text-input');
|
||||
const sendBtn = $('conv-send-btn');
|
||||
const clearBtn = $('conv-clear-btn');
|
||||
const sttSel = $('conv-stt-select');
|
||||
const llmUrlInp = $('conv-llm-url');
|
||||
@ -489,6 +491,110 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
isProcessing = false;
|
||||
micBtn.classList.remove('processing');
|
||||
micIcon.className = 'mdi mdi-microphone';
|
||||
if (sendBtn) sendBtn.disabled = false;
|
||||
if (textInput) textInput.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Text-input turn (skips STT, sends text directly) ────────────────────
|
||||
async function processText(text) {
|
||||
text = text.trim();
|
||||
if (!text || isProcessing) return;
|
||||
isProcessing = true;
|
||||
if (sendBtn) sendBtn.disabled = true;
|
||||
if (textInput) { textInput.disabled = true; textInput.value = ''; }
|
||||
micBtn.classList.add('processing');
|
||||
micIcon.className = 'mdi mdi-dots-horizontal';
|
||||
if (micStatus) micStatus.textContent = 'Processing…';
|
||||
|
||||
turnCount++;
|
||||
const turnN = turnCount;
|
||||
const t0 = Date.now();
|
||||
|
||||
const userBubble = addBubble('user', text);
|
||||
const assistantBubble = addTypingBubble();
|
||||
let assistantText = '';
|
||||
let lastStats = null;
|
||||
|
||||
const form = new FormData();
|
||||
form.append('text', text);
|
||||
form.append('stt_backend', sttSel?.value || 'configured');
|
||||
form.append('llm_url', llmUrlInp?.value.trim() || '');
|
||||
form.append('llm_model', llmModelSel?.value || '');
|
||||
form.append('tts_backend', ttsBkSel?.value || 'voice_clone');
|
||||
form.append('tts_voice', ttsVoiceSel?.value || '');
|
||||
form.append('system_prompt', systemPrompt?.value.trim() || 'You are a helpful voice assistant.');
|
||||
form.append('history', JSON.stringify(conversationHistory.slice(-20)));
|
||||
|
||||
try {
|
||||
const resp = await fetch('/api/conversation/turn', { method: 'POST', body: form });
|
||||
if (!resp.ok) throw new Error('Server error ' + resp.status);
|
||||
const reader = resp.body.getReader();
|
||||
const dec = new TextDecoder();
|
||||
let buf = '';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buf += dec.decode(value, { stream: true });
|
||||
const lines = buf.split('\n');
|
||||
buf = lines.pop();
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith('data:')) continue;
|
||||
let evt;
|
||||
try { evt = JSON.parse(line.slice(5).trim()); } catch(_) { continue; }
|
||||
|
||||
if (evt.type === 'transcript') {
|
||||
if (micStatus) micStatus.textContent = 'Generating reply…';
|
||||
} else if (evt.type === 'token') {
|
||||
if (assistantBubble.querySelector('.conv-typing')) assistantBubble.innerHTML = '';
|
||||
assistantText += evt.delta;
|
||||
assistantBubble.textContent = assistantText;
|
||||
chatWindow.scrollTop = chatWindow.scrollHeight;
|
||||
} else if (evt.type === 'llm_done') {
|
||||
assistantText = evt.text || assistantText;
|
||||
assistantBubble.textContent = assistantText;
|
||||
if (micStatus) micStatus.textContent = 'Synthesising speech…';
|
||||
} else if (evt.type === 'audio') {
|
||||
const mime = evt.mime || 'audio/wav';
|
||||
const binStr = atob(evt.b64);
|
||||
const arr = new Uint8Array(binStr.length);
|
||||
for (let i = 0; i < binStr.length; i++) arr[i] = binStr.charCodeAt(i);
|
||||
const audioBlob = new Blob([arr], { type: mime });
|
||||
const url = URL.createObjectURL(audioBlob);
|
||||
const audio = new Audio(url);
|
||||
audio.onended = () => URL.revokeObjectURL(url);
|
||||
audio.play().catch(() => {});
|
||||
if (micStatus) micStatus.textContent = 'Speaking…';
|
||||
} else if (evt.type === 'stats') {
|
||||
lastStats = evt;
|
||||
updateStats(evt);
|
||||
} else if (evt.type === 'done') {
|
||||
conversationHistory.push({ role: 'user', content: text });
|
||||
conversationHistory.push({ role: 'assistant', content: assistantText });
|
||||
addHistoryItem(turnN, lastStats?.total_ms ?? (Date.now() - t0), true);
|
||||
if (micStatus) micStatus.textContent = 'Ready';
|
||||
} else if (evt.type === 'error') {
|
||||
const wrap = assistantBubble.closest('.conv-bubble-wrap');
|
||||
if (wrap) wrap.remove();
|
||||
addErrorBubble(`[${evt.stage?.toUpperCase() || 'ERR'}] ${evt.message || 'Unknown error'}`);
|
||||
addHistoryItem(turnN, Date.now() - t0, false);
|
||||
if (micStatus) micStatus.textContent = 'Error — ready';
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch(e) {
|
||||
const wrap = assistantBubble.closest('.conv-bubble-wrap');
|
||||
if (wrap) wrap.remove();
|
||||
addErrorBubble(e.message);
|
||||
addHistoryItem(turnN, Date.now() - t0, false);
|
||||
if (micStatus) micStatus.textContent = 'Error — ready';
|
||||
} finally {
|
||||
isProcessing = false;
|
||||
micBtn.classList.remove('processing');
|
||||
micIcon.className = 'mdi mdi-microphone';
|
||||
if (sendBtn) sendBtn.disabled = false;
|
||||
if (textInput) textInput.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
@ -502,10 +608,16 @@ $('s-import-voices-file')?.addEventListener('change', async function () {
|
||||
}
|
||||
});
|
||||
|
||||
// Text input — Enter key or Send button
|
||||
sendBtn?.addEventListener('click', () => processText(textInput?.value || ''));
|
||||
textInput?.addEventListener('keydown', e => {
|
||||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); processText(textInput.value); }
|
||||
});
|
||||
|
||||
clearBtn?.addEventListener('click', () => {
|
||||
conversationHistory = [];
|
||||
turnCount = 0;
|
||||
chatWindow.innerHTML = '<div class="conv-chat-welcome"><span class="mdi mdi-forum-outline" style="font-size:32px;opacity:.25"></span><p>Press the microphone button below and start talking.</p></div>';
|
||||
chatWindow.innerHTML = '<div class="conv-chat-welcome"><span class="mdi mdi-forum-outline" style="font-size:32px;opacity:.25"></span><p>Type a message or press the microphone button below to start.</p></div>';
|
||||
if (turnHistory) turnHistory.innerHTML = '<div class="conv-history-empty">No turns yet.</div>';
|
||||
['cpv-stt','cpv-ttft','cpv-llm','cpv-tts','cpv-total'].forEach(id => { const el = $(id); if(el) el.textContent='—'; });
|
||||
['cpf-stt','cpf-ttft','cpf-llm','cpf-tts','cpf-total'].forEach(id => { const el = $(id); if(el) el.style.width='0%'; });
|
||||
|
||||
@ -47,17 +47,26 @@
|
||||
<div class="conv-chat-window" id="conv-chat-window">
|
||||
<div class="conv-chat-welcome">
|
||||
<span class="mdi mdi-forum-outline" style="font-size:32px;opacity:.25"></span>
|
||||
<p>Press the microphone button below and start talking.</p>
|
||||
<p>Type a message or press the microphone button below to start.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Mic bar -->
|
||||
<div class="conv-mic-bar">
|
||||
<!-- Input bar: text field + send + mic -->
|
||||
<div class="conv-input-bar">
|
||||
<div class="conv-mic-status" id="conv-mic-status">Ready</div>
|
||||
<button class="conv-mic-btn" id="conv-mic-btn" title="Click to record">
|
||||
<span class="mdi mdi-microphone" id="conv-mic-icon"></span>
|
||||
</button>
|
||||
<div class="conv-mic-timer" id="conv-mic-timer"></div>
|
||||
<div class="conv-text-row">
|
||||
<input id="conv-text-input" class="conv-text-inp"
|
||||
type="text" placeholder="Type a message and press Enter or →"
|
||||
spellcheck="false" autocomplete="off">
|
||||
<button class="conv-send-btn" id="conv-send-btn" title="Send text (Enter)">
|
||||
<span class="mdi mdi-send"></span>
|
||||
</button>
|
||||
<div class="conv-divider" title="or use microphone"></div>
|
||||
<button class="conv-mic-btn" id="conv-mic-btn" title="Click to record via microphone">
|
||||
<span class="mdi mdi-microphone" id="conv-mic-icon"></span>
|
||||
</button>
|
||||
<div class="conv-mic-timer" id="conv-mic-timer"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@ -2053,14 +2053,41 @@ code { background: var(--panel); border-radius: 4px; padding: 1px 5px; font-fami
|
||||
.conv-typing span:nth-child(3) { animation-delay: .4s; }
|
||||
@keyframes convDot { 0%,80%,100% { transform: scale(.7); opacity:.3; } 40% { transform: scale(1); opacity:.9; } }
|
||||
|
||||
/* Mic bar */
|
||||
.conv-mic-bar { background: var(--surface); border: 1px solid var(--border); border-top: none; border-radius: 0 0 var(--radius) var(--radius); padding: 12px 16px; display: flex; align-items: center; gap: 14px; }
|
||||
.conv-mic-btn { width: 52px; height: 52px; border-radius: 50%; border: 2px solid var(--accent); background: var(--accent); color: #fff; font-size: 22px; cursor: pointer; display: flex; align-items: center; justify-content: center; transition: background .15s, transform .1s, box-shadow .15s; flex-shrink: 0; }
|
||||
/* ── Conversation input bar (text field + send + mic) ───────────────────── */
|
||||
.conv-input-bar {
|
||||
background: var(--surface); border: 1px solid var(--border); border-top: none;
|
||||
border-radius: 0 0 var(--radius) var(--radius); padding: 10px 16px 12px;
|
||||
display: flex; flex-direction: column; gap: 8px;
|
||||
}
|
||||
.conv-mic-status { font-size: 13px; color: var(--subtext); min-height: 18px; }
|
||||
.conv-text-row { display: flex; align-items: center; gap: 8px; }
|
||||
.conv-text-inp {
|
||||
flex: 1; min-width: 0; padding: 10px 14px; font-size: 14px; font-family: inherit;
|
||||
border: 1.5px solid var(--border); border-radius: 22px;
|
||||
background: var(--panel); color: var(--text); outline: none;
|
||||
transition: border-color .15s;
|
||||
}
|
||||
.conv-text-inp:focus { border-color: var(--accent); }
|
||||
.conv-text-inp::placeholder { color: var(--subtext); opacity: .7; }
|
||||
.conv-send-btn {
|
||||
flex-shrink: 0; width: 40px; height: 40px; border-radius: 50%;
|
||||
border: none; background: var(--accent); color: #fff;
|
||||
font-size: 17px; cursor: pointer; display: flex; align-items: center;
|
||||
justify-content: center; transition: background .15s, transform .1s;
|
||||
}
|
||||
.conv-send-btn:hover:not(:disabled) { background: #1d4ed8; transform: scale(1.05); }
|
||||
.conv-send-btn:disabled { opacity: .4; cursor: default; }
|
||||
.conv-divider { width: 1px; height: 32px; background: var(--border); flex-shrink: 0; }
|
||||
.conv-mic-btn {
|
||||
flex-shrink: 0; width: 52px; height: 52px; border-radius: 50%;
|
||||
border: 2px solid var(--accent); background: var(--accent); color: #fff;
|
||||
font-size: 22px; cursor: pointer; display: flex; align-items: center;
|
||||
justify-content: center; transition: background .15s, transform .1s, box-shadow .15s;
|
||||
}
|
||||
.conv-mic-btn:hover { background: #1d4ed8; }
|
||||
.conv-mic-btn.recording { background: var(--red); border-color: var(--red); animation: convPulse 1s infinite; }
|
||||
.conv-mic-btn.processing { background: var(--yellow); border-color: var(--yellow); cursor: default; }
|
||||
@keyframes convPulse { 0%,100% { box-shadow: 0 0 0 0 rgba(220,38,38,.4); } 50% { box-shadow: 0 0 0 8px rgba(220,38,38,0); } }
|
||||
.conv-mic-status { flex: 1; font-size: 13px; color: var(--subtext); }
|
||||
.conv-mic-timer { font-size: 13px; font-variant-numeric: tabular-nums; color: var(--red); min-width: 36px; text-align: right; }
|
||||
|
||||
/* Stats panel */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user