mARTin-B78
40e42590cc
Cast: card/list views, sort & filter, online voice picker, "Hear a line" sample button, AI character notes, import auto-save. Platform: WCAG 2.1 AA accessibility pass; German UI translation + language picker; installable PWA with offline shell; GZip + content-visibility virtualization + lazy images + Rehearser PCM memory cap (mobile stability); Playwright suite (desktop + iPhone); opt-in minified bundle build. Fixes: screenplay parser false characters; Fish-Speech inline-tag tones; narrator/voice pickers list full library; clone GUI rework; fish.audio import dedup; voice-ID rename; bulk-delete modal. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1134 lines
46 KiB
Python
1134 lines
46 KiB
Python
"""Conversation playground, LLM refinement, audio effects, export/import, speak, MCP."""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import base64
|
||
import contextlib
|
||
import io
|
||
import json
|
||
import re
|
||
import threading
|
||
import time
|
||
import uuid
|
||
import wave
|
||
from pathlib import Path
|
||
|
||
import requests
|
||
from typing import Optional
|
||
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
||
from fastapi.responses import Response, StreamingResponse
|
||
|
||
from core.config import _load_settings, _save_settings, _clean_preview_backend
|
||
from core.constants import _VOICES_DIR_DEFAULT, _MAX_UPLOAD_BYTES
|
||
from core.registry import _registry_get, TEMP_DIR
|
||
from core.validation import _copy_limited
|
||
from core.audio import _to_wav_16k
|
||
from core.voice import (
|
||
_AUDIO_EXTS, _UPLOAD_EXTS, _PICTURE_EXTS,
|
||
_find_voice_audio, _load_meta, _active_voices_dir,
|
||
_voice_audio_files, _is_internal_voice_file,
|
||
)
|
||
from core.tts_helpers import _preview_request_audio
|
||
from routes.stt import _transcribe_audio, _clean_stt_backend
|
||
|
||
router = APIRouter()
|
||
|
||
# ── STT hallucination filter ──────────────────────────────────────────────────
|
||
# Whisper commonly hallucinates these phrases on silence/noise.
|
||
# Treat them as "no speech detected" rather than passing them to the LLM.
|
||
_HALLUCINATIONS: frozenset[str] = frozenset([
|
||
"reich", "danke", "danke schön", "danke schoen", "vielen dank",
|
||
"thank you", "thank you.", "thanks", "thanks.", "you", "you.",
|
||
"copyright", "abonnieren", "untertitel", "subscribe", "subscribing",
|
||
])
|
||
|
||
def _is_hallucination(text: str) -> bool:
|
||
t = text.strip().lower().rstrip(".!?,;:-").strip()
|
||
return len(t) <= 2 or t in _HALLUCINATIONS
|
||
|
||
# ── Sentence-boundary helpers for pipelined TTS ───────────────────────────────
|
||
|
||
_SENT_RE = re.compile(r'(?<=[.!?])\s+')
|
||
_MIN_SENTENCE = 30 # min chars in buffer before we split
|
||
|
||
def _sentence_split(buf: str) -> int:
|
||
"""Return the index after the first sentence boundary, or -1."""
|
||
if len(buf) < _MIN_SENTENCE:
|
||
return -1
|
||
for m in _SENT_RE.finditer(buf):
|
||
if m.end() >= _MIN_SENTENCE:
|
||
return m.end()
|
||
return -1
|
||
|
||
# ── LLM helpers ───────────────────────────────────────────────────────────────
|
||
|
||
def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
|
||
"""Inline synchronous persona rewrite; raises RuntimeError on failure."""
|
||
system = (
|
||
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
||
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
||
"Return ONLY the rephrased text — no quotes, no explanation."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": text},
|
||
],
|
||
"temperature": 0.3,
|
||
"max_tokens": 512,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
resp = requests.post(
|
||
f"{llm_url.rstrip('/')}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": "Bearer no-key"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
result = resp.json()["choices"][0]["message"]["content"].strip()
|
||
if result.startswith('"') and result.endswith('"'):
|
||
result = result[1:-1].strip()
|
||
return result
|
||
|
||
|
||
def _resolve_speak_voice(settings: dict, client_id: str, explicit_voice: str) -> str:
|
||
if explicit_voice:
|
||
return explicit_voice
|
||
bindings: dict = settings.get("client_voice_bindings") or {}
|
||
if client_id and client_id in bindings:
|
||
return bindings[client_id]
|
||
return settings.get("captures_default_voice") or ""
|
||
|
||
|
||
# ── Routes ────────────────────────────────────────────────────────────────────
|
||
|
||
@router.post("/api/refine-text")
|
||
async def refine_text(request: Request):
|
||
"""Clean up raw STT transcription using a local OpenAI-compatible LLM."""
|
||
data = await request.json()
|
||
settings = _load_settings()
|
||
text: str = (data.get("text") or "").strip()
|
||
llm_url: str = (data.get("llm_url") or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or settings.get("refine_model") or settings.get("llm_model") or "").strip()
|
||
toggles: dict = data.get("toggles") or {}
|
||
|
||
if not text:
|
||
raise HTTPException(400, "No text to refine")
|
||
|
||
rules = []
|
||
if toggles.get("fillers", True):
|
||
rules.append("Remove filler words (um, uh, like, you know, basically, literally, I mean, so, right, etc.)")
|
||
if toggles.get("repetitions", True):
|
||
rules.append("Remove repeated words and false starts (e.g. 'the the dog' → 'the dog', 'I was- I was going' → 'I was going')")
|
||
if toggles.get("corrections", True):
|
||
rules.append("Remove self-corrections and restarts, keeping only the final intended phrasing")
|
||
if toggles.get("punctuation", True):
|
||
rules.append("Fix punctuation, capitalisation, and sentence boundaries")
|
||
if not rules:
|
||
return {"text": text, "original": text}
|
||
|
||
system = (
|
||
"You are a transcription cleanup assistant. "
|
||
"Apply ONLY the following rules to the user's text. "
|
||
"Return ONLY the cleaned text — no explanations, no quotes, no markdown:\n"
|
||
+ "\n".join(f"- {r}" for r in rules)
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": text},
|
||
],
|
||
"temperature": 0.1,
|
||
"max_tokens": 2048,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": "Bearer no-key"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
refined = resp.json()["choices"][0]["message"]["content"].strip()
|
||
if refined.startswith('"') and refined.endswith('"'):
|
||
refined = refined[1:-1].strip()
|
||
return {"text": refined, "original": text}
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM refinement failed: {e}")
|
||
|
||
|
||
@router.post("/api/rewrite-with-persona")
|
||
async def rewrite_with_persona(request: Request):
|
||
"""Rewrite user text in a voice persona's character using a local LLM."""
|
||
data = await request.json()
|
||
settings = _load_settings()
|
||
text: str = (data.get("text") or "").strip()
|
||
persona: str = (data.get("persona") or "").strip()
|
||
llm_url: str = (data.get("llm_url") or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or settings.get("llm_model") or "").strip()
|
||
mode: str = (data.get("mode") or "rewrite").strip()
|
||
|
||
if not persona:
|
||
raise HTTPException(400, "No persona defined for this voice")
|
||
if not text and mode != "compose":
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
if mode == "compose":
|
||
system = (
|
||
f"You are a voice assistant with this character: {persona}\n"
|
||
"Write a single natural utterance in this character's voice about the topic given. "
|
||
"Return ONLY the utterance — no quotes, no explanation."
|
||
)
|
||
user_msg = text or "Introduce yourself briefly."
|
||
temp = 0.9
|
||
else:
|
||
system = (
|
||
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
||
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
||
"Return ONLY the rephrased text — no quotes, no explanation."
|
||
)
|
||
user_msg = text
|
||
temp = 0.3
|
||
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user_msg},
|
||
],
|
||
"temperature": temp,
|
||
"max_tokens": 512,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": "Bearer no-key"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
result = resp.json()["choices"][0]["message"]["content"].strip()
|
||
if result.startswith('"') and result.endswith('"'):
|
||
result = result[1:-1].strip()
|
||
return {"text": result, "original": text, "persona": persona}
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM persona rewrite failed: {e}")
|
||
|
||
|
||
@router.post("/api/analyze-characters")
|
||
async def analyze_characters(request: Request):
|
||
"""Analyze a script with an LLM and return per-character voice descriptions.
|
||
|
||
Used by the Script Rehearser to auto-design voices that match each role.
|
||
Returns: {"characters": [{name, gender, language, age, description}, ...]}
|
||
"""
|
||
data = await request.json()
|
||
script: str = (data.get("script") or "").strip()
|
||
names: list = data.get("names") or []
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not script:
|
||
raise HTTPException(400, "No script provided")
|
||
if not names:
|
||
raise HTTPException(400, "No character names provided")
|
||
|
||
# Truncate script to leave room for output within typical model context windows.
|
||
# German/non-English text tokenises at ~2.5–3 chars/token, so 6000 chars ≈ 2000–2400 tokens.
|
||
if len(script) > 6000:
|
||
script = script[:6000]
|
||
|
||
lang_hint = f" The script language is {language}." if language else ""
|
||
|
||
def _build_system() -> str:
|
||
return (
|
||
"You are a casting director and TTS voice-design expert. "
|
||
"Your job is to read a script, understand each character deeply from their "
|
||
"dialogue, role, and context, then write a voice description that a "
|
||
"text-to-speech model can use to generate a matching voice.\n"
|
||
f"{lang_hint}\n"
|
||
"For EVERY character in the provided list output:\n"
|
||
"- name: exact name as given\n"
|
||
"- gender: M, F, or N\n"
|
||
"- language: spoken language of this character\n"
|
||
"- age: estimated age range (e.g. 20s, 40s, elderly)\n"
|
||
"- description: 2–3 sentences covering pitch (high/mid/low), pace, "
|
||
"timbre, accent/dialect, emotional default, and any distinctive speech trait "
|
||
"that fits the character's personality and role.\n"
|
||
"Characters with few lines: infer from their role name and context.\n"
|
||
"Respond with STRICT JSON only — no markdown, no explanation:\n"
|
||
'{"characters":[{"name":"NAME","gender":"M|F|N","language":"LANG",'
|
||
'"age":"30s","description":"voice description"}]}\n'
|
||
"Use the exact character names provided.\n/no-think"
|
||
)
|
||
|
||
def _call_llm(batch: list[str]) -> list[dict]:
|
||
"""Call the LLM for one batch of character names; return list of character dicts."""
|
||
user_msg = (
|
||
"Character names: " + ", ".join(str(n) for n in batch) + "\n\n"
|
||
"Script:\n" + script
|
||
)
|
||
# 150 tokens per character output, capped at 3072 to stay within 16K context
|
||
mt = min(3072, max(512, len(batch) * 150))
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": _build_system()},
|
||
{"role": "user", "content": user_msg},
|
||
],
|
||
"temperature": 0.4,
|
||
"max_tokens": mt,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": "Bearer no-key"},
|
||
timeout=180,
|
||
)
|
||
resp.raise_for_status()
|
||
raw = resp.json()["choices"][0]["message"]["content"].strip()
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
for candidate in (content, _extract_json_block(content)):
|
||
if not candidate:
|
||
continue
|
||
try:
|
||
parsed = json.loads(candidate)
|
||
if isinstance(parsed, dict) and "characters" in parsed:
|
||
return parsed["characters"]
|
||
except Exception:
|
||
continue
|
||
return []
|
||
|
||
# Process cast in batches of 10 to stay well within 16 K context
|
||
BATCH = 10
|
||
all_characters: list[dict] = []
|
||
try:
|
||
for i in range(0, len(names), BATCH):
|
||
batch = names[i:i + BATCH]
|
||
all_characters.extend(_call_llm(batch))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM character analysis failed: {e}")
|
||
|
||
if not all_characters:
|
||
raise HTTPException(502, "LLM did not return valid character JSON")
|
||
|
||
return {"characters": all_characters}
|
||
|
||
|
||
@router.post("/api/match-characters-voices")
|
||
async def match_characters_voices(request: Request):
|
||
"""Pick the best EXISTING library voice for each character (instead of designing new ones).
|
||
|
||
Body: {script, names:[...], voices:[{id, gender, language, tags, description}], llm_url, model, language}
|
||
Returns: {"assignments": [{name, voice_id, reason}]}
|
||
"""
|
||
data = await request.json()
|
||
script: str = (data.get("script") or "").strip()
|
||
names: list = data.get("names") or []
|
||
voices: list = data.get("voices") or []
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not names:
|
||
raise HTTPException(400, "No character names provided")
|
||
if not voices:
|
||
raise HTTPException(400, "No candidate voices provided")
|
||
if len(script) > 5000:
|
||
script = script[:5000]
|
||
|
||
def _vline(v: dict) -> str:
|
||
meta = [str(v.get(k)) for k in ("gender", "language") if v.get(k)]
|
||
if v.get("tags"):
|
||
meta.append("tags:" + str(v["tags"]))
|
||
desc = str(v.get("description") or v.get("name") or "")[:120]
|
||
return f"- {v.get('id')} [{', '.join(meta)}] {desc}".rstrip()
|
||
|
||
catalogue = "\n".join(_vline(v) for v in voices[:300])
|
||
valid_ids = {str(v.get("id")) for v in voices if v.get("id")}
|
||
|
||
system = (
|
||
"You are a casting director assigning existing TTS voices to script characters. "
|
||
f"{('Script language: ' + language + '. ') if language else ''}"
|
||
"For EACH character, choose the single BEST voice_id from the CATALOGUE. "
|
||
"RULE 1 — GENDER FIRST: the voice's gender MUST match the character's gender whenever the "
|
||
"character's gender is clear from the script; only pick a different gender if no same-gender "
|
||
"voice exists in the catalogue. "
|
||
"RULE 2 — then match apparent age, language, personality, and the voice's description/tags. "
|
||
"You MUST choose a voice_id that appears verbatim in the catalogue — never invent one. "
|
||
"Reuse a voice for two characters only if no better distinct option exists. "
|
||
"Respond with STRICT JSON only, no markdown:\n"
|
||
'{"assignments":[{"name":"NAME","voice_id":"ID","reason":"short reason"}]}\n/no-think'
|
||
)
|
||
user = (
|
||
"Characters to cast: " + ", ".join(str(n) for n in names) + "\n\n"
|
||
"CATALOGUE (voice_id [gender, language, tags] description):\n" + catalogue + "\n\n"
|
||
"Script excerpt:\n" + script
|
||
)
|
||
payload: dict = {
|
||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||
"temperature": 0.3,
|
||
"max_tokens": min(2048, max(512, len(names) * 60)),
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
import time as _time
|
||
raw = None
|
||
last_err: Exception | None = None
|
||
for attempt in range(3):
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": "Bearer no-key"}, timeout=180,
|
||
)
|
||
resp.raise_for_status()
|
||
raw = resp.json()["choices"][0]["message"]["content"].strip()
|
||
break
|
||
except requests.exceptions.ConnectionError as e:
|
||
# llama-swap (and similar) often drop the first request while swapping/loading
|
||
# the model — wait and retry rather than failing the cast.
|
||
last_err = e
|
||
_time.sleep(4 + attempt * 3)
|
||
except Exception as e:
|
||
last_err = e
|
||
break
|
||
if raw is None:
|
||
raise HTTPException(502, f"LLM voice matching failed: {last_err}")
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
assignments: list = []
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("assignments"), list):
|
||
assignments = parsed["assignments"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
# Keep only assignments that reference a real catalogue voice
|
||
clean = [a for a in assignments if isinstance(a, dict) and str(a.get("voice_id")) in valid_ids]
|
||
return {"assignments": clean}
|
||
|
||
|
||
def _extract_json_block(text: str) -> str:
|
||
"""Pull the first {...} JSON object out of an LLM response."""
|
||
text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
|
||
if text.startswith("```"):
|
||
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
|
||
text = re.sub(r"\n?```$", "", text).strip()
|
||
start = text.find("{")
|
||
end = text.rfind("}")
|
||
if start != -1 and end != -1 and end > start:
|
||
return text[start:end + 1]
|
||
return ""
|
||
|
||
|
||
# ── Audio effects ─────────────────────────────────────────────────────────────
|
||
|
||
def _apply_audio_effects(audio_bytes: bytes, effects: list) -> bytes:
|
||
try:
|
||
from pedalboard import Pedalboard, Reverb, Chorus, Delay, Compressor, Gain, HighpassFilter, LowpassFilter, PitchShift # type: ignore
|
||
import numpy as np # type: ignore
|
||
except ImportError:
|
||
raise RuntimeError("pedalboard is not installed — run: pip install pedalboard numpy")
|
||
|
||
with io.BytesIO(audio_bytes) as buf:
|
||
with wave.open(buf, "rb") as wf:
|
||
n_channels = wf.getnchannels()
|
||
sample_rate = wf.getframerate()
|
||
n_frames = wf.getnframes()
|
||
raw = wf.readframes(n_frames)
|
||
sampwidth = wf.getsampwidth()
|
||
|
||
import numpy as np # noqa: F811
|
||
dtype = {1: np.int8, 2: np.int16, 4: np.int32}.get(sampwidth, np.int16)
|
||
samples = np.frombuffer(raw, dtype=dtype).astype(np.float32) / float(np.iinfo(dtype).max)
|
||
samples = samples.reshape(1, -1) if n_channels == 1 else samples.reshape(-1, n_channels).T
|
||
|
||
board = []
|
||
for fx in effects:
|
||
t = fx.get("type", "")
|
||
p = fx.get("params", {})
|
||
if t == "reverb":
|
||
board.append(Reverb(
|
||
room_size=float(p.get("room_size", 0.35)),
|
||
damping=float(p.get("damping", 0.5)),
|
||
wet_level=float(p.get("wet", 0.25)),
|
||
dry_level=float(p.get("dry", 0.8)),
|
||
))
|
||
elif t == "chorus":
|
||
board.append(Chorus(
|
||
rate_hz=float(p.get("rate_hz", 1.0)),
|
||
depth=float(p.get("depth", 0.25)),
|
||
mix=float(p.get("mix", 0.5)),
|
||
))
|
||
elif t == "delay":
|
||
board.append(Delay(
|
||
delay_seconds=float(p.get("delay_s", 0.25)),
|
||
feedback=float(p.get("feedback", 0.3)),
|
||
mix=float(p.get("mix", 0.4)),
|
||
))
|
||
elif t == "compressor":
|
||
board.append(Compressor(
|
||
threshold_db=float(p.get("threshold_db", -20.0)),
|
||
ratio=float(p.get("ratio", 4.0)),
|
||
attack_ms=float(p.get("attack_ms", 10.0)),
|
||
release_ms=float(p.get("release_ms", 100.0)),
|
||
))
|
||
elif t == "gain":
|
||
board.append(Gain(gain_db=float(p.get("gain_db", 0.0))))
|
||
elif t == "highpass":
|
||
board.append(HighpassFilter(cutoff_frequency_hz=float(p.get("cutoff_hz", 80.0))))
|
||
elif t == "lowpass":
|
||
board.append(LowpassFilter(cutoff_frequency_hz=float(p.get("cutoff_hz", 8000.0))))
|
||
elif t == "pitch_shift":
|
||
board.append(PitchShift(semitones=float(p.get("semitones", 0.0))))
|
||
|
||
if board:
|
||
samples = Pedalboard(board)(samples, sample_rate)
|
||
|
||
out = np.clip(samples, -1.0, 1.0)
|
||
pcm = ((out[0] if out.shape[0] == 1 else out.T.reshape(-1)) * 32767).astype(np.int16).tobytes()
|
||
buf_out = io.BytesIO()
|
||
with wave.open(buf_out, "wb") as wf:
|
||
wf.setnchannels(n_channels)
|
||
wf.setsampwidth(2)
|
||
wf.setframerate(sample_rate)
|
||
wf.writeframes(pcm)
|
||
return buf_out.getvalue()
|
||
|
||
|
||
@router.post("/api/audio/effects")
|
||
async def audio_effects(request: Request):
|
||
"""Apply an effects chain to a WAV file."""
|
||
form = await request.form()
|
||
audio_file = form.get("audio")
|
||
effects_json = str(form.get("effects") or "[]")
|
||
|
||
if audio_file is None:
|
||
raise HTTPException(400, "No audio file provided")
|
||
audio_bytes = await audio_file.read()
|
||
try:
|
||
effects = json.loads(effects_json)
|
||
except Exception:
|
||
raise HTTPException(400, "Invalid effects JSON")
|
||
|
||
try:
|
||
result = await asyncio.to_thread(_apply_audio_effects, audio_bytes, effects)
|
||
return Response(content=result, media_type="audio/wav")
|
||
except RuntimeError as e:
|
||
raise HTTPException(501, str(e))
|
||
except Exception as e:
|
||
raise HTTPException(500, f"Effects processing failed: {e}")
|
||
|
||
|
||
# ── Voices export / import ────────────────────────────────────────────────────
|
||
|
||
_EXPORT_SKIP_KEYS = {"groq_api_key", "whisper_api_key", "tts_api_key", "voice_design_api_key", "elevenlabs_api_key"}
|
||
_IMPORT_ALLOWED_SUFFIXES = set(_AUDIO_EXTS + [".reference.txt", ".meta.json", ".jpg", ".jpeg", ".png", ".webp"])
|
||
|
||
|
||
@router.get("/api/voices/export")
|
||
async def voices_export():
|
||
"""Export all voices + non-sensitive settings as a ZIP archive."""
|
||
import zipfile
|
||
from datetime import datetime
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
||
safe_settings = {k: v for k, v in settings.items() if k not in _EXPORT_SKIP_KEYS}
|
||
zf.writestr("settings.json", json.dumps(safe_settings, indent=2))
|
||
if scan_dir.exists():
|
||
for f in scan_dir.rglob("*"):
|
||
if f.is_file():
|
||
try:
|
||
zf.write(f, str(f.relative_to(scan_dir.parent)))
|
||
except Exception:
|
||
pass
|
||
buf.seek(0)
|
||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
return Response(
|
||
content=buf.read(),
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f'attachment; filename="voices_export_{ts}.zip"'},
|
||
)
|
||
|
||
|
||
@router.post("/api/voices/import")
|
||
async def voices_import(file: UploadFile = File(...)):
|
||
"""Import voices from a ZIP archive (skips settings.json and unsafe paths)."""
|
||
import zipfile
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
scan_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
content = await file.read()
|
||
if len(content) > _MAX_UPLOAD_BYTES:
|
||
raise HTTPException(413, "ZIP file too large")
|
||
|
||
try:
|
||
imported = 0
|
||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||
for info in zf.infolist():
|
||
if info.is_dir():
|
||
continue
|
||
parts = Path(info.filename).parts
|
||
if any(p in ("..", "") for p in parts) or Path(info.filename).name == "settings.json":
|
||
continue
|
||
if Path(info.filename).suffix.lower() not in _IMPORT_ALLOWED_SUFFIXES:
|
||
continue
|
||
rel = parts[1:] if len(parts) > 1 else parts
|
||
dest = scan_dir / Path(*rel)
|
||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||
dest.write_bytes(zf.read(info))
|
||
imported += 1
|
||
except zipfile.BadZipFile:
|
||
raise HTTPException(400, "Not a valid ZIP file")
|
||
except Exception as e:
|
||
raise HTTPException(500, f"Import failed: {e}")
|
||
|
||
return {"ok": True, "imported": imported}
|
||
|
||
|
||
# ── /speak REST endpoint ──────────────────────────────────────────────────────
|
||
|
||
@router.post("/speak")
|
||
async def speak(request: Request):
|
||
data = await request.json()
|
||
text: str = str(data.get("text") or "").strip()
|
||
if not text:
|
||
raise HTTPException(400, "text is required")
|
||
|
||
explicit_voice: str = str(data.get("voice") or data.get("profile_id") or data.get("profile") or "").strip()
|
||
backend: str = _clean_preview_backend(str(data.get("backend") or "voice_clone"))
|
||
apply_persona: bool = bool(data.get("apply_persona") or data.get("personality"))
|
||
client_id: str = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
||
|
||
settings = _load_settings()
|
||
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
||
if not voice:
|
||
raise HTTPException(400, "No voice specified and no default voice configured")
|
||
|
||
if apply_persona:
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
wav = _find_voice_audio(voice, scan_dir)
|
||
if wav:
|
||
persona = _load_meta(wav).get("persona", "")
|
||
if persona:
|
||
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
llm_model = settings.get("llm_model") or ""
|
||
try:
|
||
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url, llm_model)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Persona rewrite failed: {e}")
|
||
|
||
instruct = str(data.get("instruct") or "")
|
||
try:
|
||
audio, media_type = await asyncio.to_thread(
|
||
_preview_request_audio, text, voice, settings, instruct, backend
|
||
)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"TTS error: {e}")
|
||
|
||
return Response(content=audio, media_type=media_type)
|
||
|
||
|
||
@router.get("/speak/bindings")
|
||
async def get_speak_bindings():
|
||
s = _load_settings()
|
||
return {"bindings": s.get("client_voice_bindings") or {}}
|
||
|
||
|
||
@router.put("/speak/bindings/{client_id}")
|
||
async def put_speak_binding(client_id: str, request: Request):
|
||
data = await request.json()
|
||
voice = str(data.get("voice") or "").strip()
|
||
if not voice:
|
||
raise HTTPException(400, "voice is required")
|
||
s = _load_settings()
|
||
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
||
bindings[client_id] = voice
|
||
s["client_voice_bindings"] = bindings
|
||
_save_settings(s)
|
||
return {"ok": True, "client_id": client_id, "voice": voice}
|
||
|
||
|
||
@router.delete("/speak/bindings/{client_id}")
|
||
async def delete_speak_binding(client_id: str):
|
||
s = _load_settings()
|
||
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
||
if client_id not in bindings:
|
||
raise HTTPException(404, f"No binding for client '{client_id}'")
|
||
del bindings[client_id]
|
||
s["client_voice_bindings"] = bindings
|
||
_save_settings(s)
|
||
return {"ok": True, "client_id": client_id}
|
||
|
||
|
||
# ── MCP JSON-RPC 2.0 server ───────────────────────────────────────────────────
|
||
|
||
_MCP_SERVER_INFO = {"name": "tts-voice-creator", "version": "1.0.0"}
|
||
|
||
_MCP_TOOLS = [
|
||
{
|
||
"name": "speak",
|
||
"description": "Generate speech audio from text using a cloned voice. Returns a data URI with the WAV audio.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"text": {"type": "string", "description": "Text to synthesize"},
|
||
"voice": {"type": "string", "description": "Voice ID (optional — uses per-client binding or default if omitted)"},
|
||
"apply_persona": {"type": "boolean", "description": "Rewrite text through the voice's persona before synthesis"},
|
||
},
|
||
"required": ["text"],
|
||
},
|
||
},
|
||
{
|
||
"name": "transcribe",
|
||
"description": "Transcribe base64-encoded WAV audio to text.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"audio_base64": {"type": "string", "description": "Base64-encoded audio bytes (WAV preferred)"},
|
||
},
|
||
"required": ["audio_base64"],
|
||
},
|
||
},
|
||
{
|
||
"name": "list_captures",
|
||
"description": "List the 20 most recently generated audio files.",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
{
|
||
"name": "list_profiles",
|
||
"description": "List all available voice profiles with their language, persona, and enabled state.",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
]
|
||
|
||
|
||
async def _mcp_tool_speak(args: dict, client_id: str) -> dict:
|
||
text = str(args.get("text") or "").strip()
|
||
if not text:
|
||
raise ValueError("text is required")
|
||
explicit_voice = str(args.get("voice") or "").strip()
|
||
apply_persona = bool(args.get("apply_persona", False))
|
||
settings = _load_settings()
|
||
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
||
if not voice:
|
||
raise ValueError("No voice specified and no default voice configured")
|
||
if apply_persona:
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
wav = _find_voice_audio(voice, scan_dir)
|
||
if wav:
|
||
persona = _load_meta(wav).get("persona", "")
|
||
if persona:
|
||
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
llm_model = settings.get("llm_model") or ""
|
||
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url, llm_model)
|
||
audio, _media_type = await asyncio.to_thread(
|
||
_preview_request_audio, text, voice, settings, "", "voice_clone"
|
||
)
|
||
audio_b64 = base64.b64encode(audio).decode()
|
||
return {
|
||
"content": [
|
||
{"type": "text", "text": f"Generated {len(audio)} bytes of speech audio for voice '{voice}'."},
|
||
{"type": "resource", "resource": {"uri": f"data:audio/wav;base64,{audio_b64}", "mimeType": "audio/wav"}},
|
||
]
|
||
}
|
||
|
||
|
||
async def _mcp_tool_transcribe(args: dict) -> dict:
|
||
raw = args.get("audio_base64") or ""
|
||
try:
|
||
audio_bytes = base64.b64decode(raw)
|
||
except Exception:
|
||
raise ValueError("audio_base64 is not valid base64")
|
||
tmp = TEMP_DIR / f"{uuid.uuid4().hex}_mcp_transcribe.wav"
|
||
tmp.write_bytes(audio_bytes)
|
||
settings = _load_settings()
|
||
try:
|
||
text, used_backend = await asyncio.to_thread(_transcribe_audio, tmp, settings, "configured")
|
||
finally:
|
||
try:
|
||
tmp.unlink(missing_ok=True)
|
||
except Exception:
|
||
pass
|
||
return {"content": [{"type": "text", "text": text}], "backend": used_backend}
|
||
|
||
|
||
async def _mcp_tool_list_captures() -> dict:
|
||
settings = _load_settings()
|
||
out_dir = _active_voices_dir(settings)
|
||
if not out_dir.exists():
|
||
return {"content": [{"type": "text", "text": "[]"}]}
|
||
files = sorted(
|
||
(p for p in out_dir.rglob("*.wav") if not _is_internal_voice_file(p)),
|
||
key=lambda p: p.stat().st_mtime,
|
||
reverse=True,
|
||
)[:20]
|
||
result = [{"name": p.stem, "path": str(p), "mtime": p.stat().st_mtime} for p in files]
|
||
return {"content": [{"type": "text", "text": json.dumps(result)}]}
|
||
|
||
|
||
async def _mcp_tool_list_profiles() -> dict:
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
profiles = []
|
||
seen: set[str] = set()
|
||
for audio in sorted(_voice_audio_files(scan_dir), key=lambda p: p.stem.lower()):
|
||
if audio.stem in seen:
|
||
continue
|
||
seen.add(audio.stem)
|
||
meta = _load_meta(audio)
|
||
parts = audio.stem.split("_", 2)
|
||
lang = parts[0].upper() if parts else ""
|
||
profiles.append({
|
||
"id": audio.stem,
|
||
"name": audio.stem,
|
||
"lang": lang,
|
||
"persona": meta.get("persona", ""),
|
||
"enabled": meta.get("enabled", True),
|
||
})
|
||
return {"content": [{"type": "text", "text": json.dumps(profiles)}]}
|
||
|
||
|
||
def _mcp_error_response(code: int, message: str, rpc_id) -> Response:
|
||
import logging
|
||
body = {"jsonrpc": "2.0", "error": {"code": code, "message": message}, "id": rpc_id}
|
||
return Response(content=json.dumps(body), media_type="application/json")
|
||
|
||
|
||
@router.post("/mcp")
|
||
async def mcp_jsonrpc(request: Request):
|
||
import logging
|
||
logger = logging.getLogger("uvicorn.error")
|
||
try:
|
||
body = await request.json()
|
||
except Exception:
|
||
return _mcp_error_response(-32700, "Parse error", None)
|
||
|
||
rpc_id = body.get("id")
|
||
method = body.get("method", "")
|
||
params = body.get("params") or {}
|
||
client_id = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
||
|
||
try:
|
||
if method == "initialize":
|
||
result = {
|
||
"protocolVersion": "2024-11-05",
|
||
"serverInfo": _MCP_SERVER_INFO,
|
||
"capabilities": {"tools": {}},
|
||
}
|
||
elif method == "notifications/initialized":
|
||
return Response(status_code=204)
|
||
elif method == "tools/list":
|
||
result = {"tools": _MCP_TOOLS}
|
||
elif method == "tools/call":
|
||
tool_name = str(params.get("name") or "")
|
||
tool_args = params.get("arguments") or {}
|
||
if tool_name == "speak":
|
||
result = await _mcp_tool_speak(tool_args, client_id)
|
||
elif tool_name == "transcribe":
|
||
result = await _mcp_tool_transcribe(tool_args)
|
||
elif tool_name == "list_captures":
|
||
result = await _mcp_tool_list_captures()
|
||
elif tool_name == "list_profiles":
|
||
result = await _mcp_tool_list_profiles()
|
||
else:
|
||
return _mcp_error_response(-32601, f"Unknown tool: {tool_name}", rpc_id)
|
||
else:
|
||
return _mcp_error_response(-32601, f"Method not found: {method}", rpc_id)
|
||
except ValueError as e:
|
||
return _mcp_error_response(-32602, str(e), rpc_id)
|
||
except Exception as e:
|
||
logger.exception("MCP tool error in method %s", method)
|
||
return _mcp_error_response(-32000, f"Server error: {e}", rpc_id)
|
||
|
||
return Response(
|
||
content=json.dumps({"jsonrpc": "2.0", "result": result, "id": rpc_id}),
|
||
media_type="application/json",
|
||
)
|
||
|
||
|
||
@router.get("/mcp")
|
||
async def mcp_sse(request: Request):
|
||
"""SSE keep-alive stream (satisfies MCP spec GET /mcp requirement)."""
|
||
from typing import AsyncGenerator
|
||
|
||
async def _keepalive() -> AsyncGenerator[str, None]:
|
||
yield ": mcp-sse-ready\n\n"
|
||
while True:
|
||
await asyncio.sleep(15)
|
||
yield ": ping\n\n"
|
||
return StreamingResponse(
|
||
_keepalive(),
|
||
media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||
)
|
||
|
||
|
||
# ── Conversation playground ───────────────────────────────────────────────────
|
||
|
||
@router.get("/api/conversation/llm-models")
|
||
async def conversation_llm_models(url: str = ""):
|
||
"""List models from a local LLM endpoint (Ollama / vLLM / LM Studio)."""
|
||
from core.validation import _validate_http_url
|
||
settings = _load_settings()
|
||
base = (url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
try:
|
||
base = _validate_http_url(base, allow_private=True)
|
||
r = requests.get(f"{base}/models", timeout=5, headers={"Authorization": "Bearer no-key"})
|
||
if r.status_code == 200:
|
||
payload = r.json()
|
||
data = payload.get("data", []) if isinstance(payload, dict) else []
|
||
models = [
|
||
str(item["id"]) if isinstance(item, dict) and item.get("id") else str(item)
|
||
for item in data if item
|
||
]
|
||
return {"models": models, "url": base}
|
||
except Exception:
|
||
pass
|
||
return {"models": [], "url": base}
|
||
|
||
|
||
def _make_tts_task(
|
||
text: str,
|
||
voice: str,
|
||
settings: dict,
|
||
backend: str,
|
||
sem: "asyncio.Semaphore | None",
|
||
) -> "asyncio.Task":
|
||
if sem is None:
|
||
return asyncio.create_task(
|
||
asyncio.to_thread(_preview_request_audio, text, voice, settings, "", backend)
|
||
)
|
||
|
||
async def _guarded() -> tuple[bytes, str]:
|
||
async with sem:
|
||
return await asyncio.to_thread(_preview_request_audio, text, voice, settings, "", backend)
|
||
|
||
return asyncio.create_task(_guarded())
|
||
|
||
|
||
@router.post("/api/conversation/turn")
|
||
async def conversation_turn(
|
||
audio: Optional[UploadFile] = None,
|
||
text: str = Form(""),
|
||
stt_backend: str = Form("configured"),
|
||
llm_url: str = Form(""),
|
||
llm_model: str = Form(""),
|
||
tts_backend: str = Form("voice_clone"),
|
||
tts_voice: str = Form(""),
|
||
system_prompt: str = Form("You are a helpful voice assistant. Keep replies short and conversational."),
|
||
history: str = Form("[]"),
|
||
):
|
||
"""Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events.
|
||
Pass either an audio file (runs STT first) or a plain text string (skips STT).
|
||
"""
|
||
direct_text = text.strip()
|
||
if not direct_text and (audio is None or not getattr(audio, "filename", None)):
|
||
raise HTTPException(400, "Provide either an audio file or a text field")
|
||
|
||
settings = _load_settings()
|
||
eff_llm_url = (llm_url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
|
||
# Prepare audio temp files only when an audio upload was provided
|
||
_tmp: Path | None = None
|
||
_wav: Path | None = None
|
||
if not direct_text and audio is not None:
|
||
suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm"
|
||
if suffix not in _UPLOAD_EXTS:
|
||
suffix = ".webm"
|
||
_tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}"
|
||
try:
|
||
with _tmp.open("wb") as f:
|
||
_copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
|
||
_wav = _to_wav_16k(_tmp)
|
||
except Exception as e:
|
||
_tmp.unlink(missing_ok=True)
|
||
raise HTTPException(400, f"Audio upload failed: {e}")
|
||
|
||
try:
|
||
hist = json.loads(history) if history else []
|
||
if not isinstance(hist, list):
|
||
hist = []
|
||
except Exception:
|
||
hist = []
|
||
|
||
stt_be = _clean_stt_backend(stt_backend)
|
||
tts_be = _clean_preview_backend(tts_backend)
|
||
|
||
async def generate():
|
||
t0 = time.monotonic()
|
||
stt_ms = llm_ttft_ms = llm_total_ms = tts_ms = None
|
||
transcript = llm_text = ""
|
||
|
||
def sse(obj: dict) -> str:
|
||
return f"data: {json.dumps(obj)}\n\n"
|
||
|
||
# 1. STT — skipped when caller sends direct text
|
||
if direct_text:
|
||
transcript = direct_text
|
||
yield sse({"type": "transcript", "text": transcript, "stt_ms": None})
|
||
else:
|
||
try:
|
||
t_stt = time.monotonic()
|
||
transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be)
|
||
stt_ms = int((time.monotonic() - t_stt) * 1000)
|
||
yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms})
|
||
except Exception as e:
|
||
yield sse({"type": "error", "stage": "stt", "message": str(e)})
|
||
return
|
||
finally:
|
||
for p in filter(None, {_tmp, _wav}):
|
||
with contextlib.suppress(Exception):
|
||
p.unlink(missing_ok=True)
|
||
|
||
if not transcript.strip() or _is_hallucination(transcript):
|
||
yield sse({"type": "error", "stage": "stt", "message": "No speech detected."})
|
||
return
|
||
|
||
# 2. LLM stream — runs in a thread; tokens arrive via asyncio.Queue
|
||
# so the event loop is never blocked and TTS can start on sentence 1
|
||
# while the LLM is still generating sentences 2, 3, …
|
||
messages = [{"role": "system", "content": system_prompt}]
|
||
messages.extend(hist[-20:])
|
||
messages.append({"role": "user", "content": transcript})
|
||
llm_payload: dict = {"messages": messages, "stream": True, "temperature": 0.7, "max_tokens": 512}
|
||
eff_llm_model = llm_model or settings.get("llm_model") or ""
|
||
if eff_llm_model:
|
||
llm_payload["model"] = eff_llm_model
|
||
|
||
_loop = asyncio.get_event_loop()
|
||
_token_q: asyncio.Queue[str | None] = asyncio.Queue()
|
||
|
||
def _llm_thread() -> None:
|
||
try:
|
||
resp = requests.post(
|
||
f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||
headers={"Authorization": "Bearer no-key"}, stream=True, timeout=120,
|
||
)
|
||
resp.raise_for_status()
|
||
for raw_line in resp.iter_lines():
|
||
if not raw_line:
|
||
continue
|
||
line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else str(raw_line)
|
||
if not line.startswith("data:"):
|
||
continue
|
||
payload_str = line[5:].strip()
|
||
if payload_str == "[DONE]":
|
||
break
|
||
try:
|
||
obj = json.loads(payload_str)
|
||
d_obj = ((obj.get("choices") or [{}])[0].get("delta") or {})
|
||
delta = d_obj.get("content") or d_obj.get("reasoning_content") or ""
|
||
if not delta and isinstance(obj.get("message"), dict):
|
||
delta = obj["message"].get("content") or ""
|
||
if delta:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, delta)
|
||
except Exception:
|
||
continue
|
||
except Exception as exc:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, f"\x00ERR:{exc}")
|
||
finally:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, None)
|
||
|
||
threading.Thread(target=_llm_thread, daemon=True).start()
|
||
|
||
# Streaming backend (8023) has per-process voice state — serialize to prevent
|
||
# concurrent requests from clobbering each other's voice context.
|
||
tts_sem: asyncio.Semaphore | None = asyncio.Semaphore(1) if tts_be == "streaming" else None
|
||
|
||
t_llm = time.monotonic()
|
||
llm_ttft_ms: int | None = None
|
||
ttft_done = False
|
||
sent_buf = ""
|
||
tts_tasks: list[asyncio.Task] = []
|
||
tts_texts: list[str] = [] # sentence text corresponding to each task
|
||
tts_first_start: float | None = None
|
||
|
||
try:
|
||
while True:
|
||
delta = await _token_q.get()
|
||
if delta is None:
|
||
break
|
||
if delta.startswith("\x00ERR:"):
|
||
yield sse({"type": "error", "stage": "llm", "message": delta[5:]})
|
||
return
|
||
if not ttft_done:
|
||
llm_ttft_ms = int((time.monotonic() - t_llm) * 1000)
|
||
ttft_done = True
|
||
llm_text += delta
|
||
sent_buf += delta
|
||
yield sse({"type": "token", "delta": delta})
|
||
# Fire TTS on sentence boundary — runs concurrently with LLM
|
||
split = _sentence_split(sent_buf)
|
||
if split > 0:
|
||
chunk_text = sent_buf[:split].strip()
|
||
sent_buf = sent_buf[split:]
|
||
if chunk_text:
|
||
if tts_first_start is None:
|
||
tts_first_start = time.monotonic()
|
||
tts_texts.append(chunk_text)
|
||
tts_tasks.append(_make_tts_task(chunk_text, tts_voice, settings, tts_be, tts_sem))
|
||
except Exception as exc:
|
||
yield sse({"type": "error", "stage": "llm", "message": str(exc)})
|
||
return
|
||
|
||
# Flush any remaining text as a final TTS task
|
||
if sent_buf.strip():
|
||
if tts_first_start is None:
|
||
tts_first_start = time.monotonic()
|
||
tts_texts.append(sent_buf.strip())
|
||
tts_tasks.append(_make_tts_task(sent_buf.strip(), tts_voice, settings, tts_be, tts_sem))
|
||
|
||
llm_total_ms = int((time.monotonic() - t_llm) * 1000)
|
||
yield sse({"type": "llm_done", "text": llm_text,
|
||
"llm_ttft_ms": llm_ttft_ms, "llm_total_ms": llm_total_ms})
|
||
|
||
if not llm_text.strip():
|
||
yield sse({"type": "error", "stage": "llm",
|
||
"message": "LLM returned empty response. "
|
||
"If using a Qwen3 model, try adding /no-think to your system prompt "
|
||
"or pick a non-thinking model in the Language Model dropdown."})
|
||
return
|
||
|
||
# 3. Stream audio chunks in order — each chunk's TTS ran concurrently
|
||
# with LLM generation, so first audio arrives much sooner than
|
||
# waiting for the full response.
|
||
tts_start = tts_first_start or time.monotonic()
|
||
for i, task in enumerate(tts_tasks):
|
||
try:
|
||
audio_bytes, mime = await task
|
||
except Exception as exc:
|
||
yield sse({"type": "error", "stage": "tts", "message": str(exc)})
|
||
return
|
||
if i == 0:
|
||
tts_ms = int((time.monotonic() - tts_start) * 1000)
|
||
sentence_text = tts_texts[i] if i < len(tts_texts) else ""
|
||
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(),
|
||
"mime": mime, "text": sentence_text})
|
||
|
||
total_ms = int((time.monotonic() - t0) * 1000)
|
||
yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
|
||
"llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms})
|
||
yield sse({"type": "done"})
|
||
|
||
return StreamingResponse(generate(), media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|