mARTin-B78
628bd75a82
Table view was rendering as stacked blocks instead of columns: rows reused .lib-char-card for its event wiring, but that class's display:flex;flex-direction:column turned every <tr> into a flex column. Reset to display:table-row and stripped the leaked-in card chrome. Split /api/character-generate-prompts into four independent per-field LLM calls (from two paired calls) and added a `fields` filter, so the UI can offer one Generate button per prompt box instead of a single button that always regenerated all four - cheaper, and further shrinks each response to reduce truncation risk. Added a Sort dropdown (Role/Alphabet/Lines/Gender/Voice assigned) to the Characters/Cast list, persisted like the Cards/Table toggle. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2016 lines
95 KiB
Python
2016 lines
95 KiB
Python
"""Conversation playground, LLM refinement, audio effects, export/import, speak, MCP."""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import base64
|
||
import contextlib
|
||
import io
|
||
import json
|
||
import re
|
||
import threading
|
||
import time
|
||
import uuid
|
||
import wave
|
||
from pathlib import Path
|
||
|
||
import requests
|
||
from typing import Optional
|
||
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
||
from fastapi.responses import Response, StreamingResponse
|
||
|
||
from core.config import _load_settings, _save_settings, _clean_preview_backend
|
||
from core.constants import _VOICES_DIR_DEFAULT, _MAX_UPLOAD_BYTES
|
||
from core.registry import _registry_get, TEMP_DIR
|
||
from core.validation import _copy_limited
|
||
from core.audio import _to_wav_16k
|
||
from core.voice import (
|
||
_AUDIO_EXTS, _UPLOAD_EXTS, _PICTURE_EXTS,
|
||
_find_voice_audio, _load_meta, _active_voices_dir,
|
||
_voice_audio_files, _is_internal_voice_file,
|
||
)
|
||
from core.voice_index import rebuild_voice_index
|
||
from core.tts_helpers import _preview_request_audio
|
||
from routes.stt import _transcribe_audio, _clean_stt_backend
|
||
|
||
router = APIRouter()
|
||
|
||
# Serializes outbound calls to the audiobook-attribution LLM. A client-side
|
||
# timeout aborts the browser fetch but can't kill the backend's blocking
|
||
# requests.post thread, so a slow/cold model leaves "ghost" requests still
|
||
# occupying the LLM's single processing slot. Without this lock, the next
|
||
# chunk (or the retry-in-halves) fires into that busy slot and the LLM
|
||
# answers with 429s that cascade until the ghosts drain.
|
||
# threading.Lock (not asyncio.Lock) because the streaming endpoint's SSE
|
||
# generator runs in a plain thread (StreamingResponse iterates a sync
|
||
# generator via a threadpool), so both the blocking and streaming endpoints
|
||
# must share ONE lock object to actually serialize on the LLM's single slot —
|
||
# two separate locks would let a stream call and its own blocking fallback
|
||
# fire into that slot concurrently, which is exactly the "ghost request"
|
||
# scenario this lock exists to prevent.
|
||
_attribution_llm_lock = threading.Lock()
|
||
|
||
# ── STT hallucination filter ──────────────────────────────────────────────────
|
||
# Whisper commonly hallucinates these phrases on silence/noise.
|
||
# Treat them as "no speech detected" rather than passing them to the LLM.
|
||
_HALLUCINATIONS: frozenset[str] = frozenset([
|
||
"reich", "danke", "danke schön", "danke schoen", "vielen dank",
|
||
"thank you", "thank you.", "thanks", "thanks.", "you", "you.",
|
||
"copyright", "abonnieren", "untertitel", "subscribe", "subscribing",
|
||
])
|
||
|
||
def _is_hallucination(text: str) -> bool:
|
||
t = text.strip().lower().rstrip(".!?,;:-").strip()
|
||
return len(t) <= 2 or t in _HALLUCINATIONS
|
||
|
||
# ── Sentence-boundary helpers for pipelined TTS ───────────────────────────────
|
||
|
||
_SENT_RE = re.compile(r'(?<=[.!?])\s+')
|
||
_MIN_SENTENCE = 30 # min chars in buffer before we split
|
||
|
||
def _sentence_split(buf: str) -> int:
|
||
"""Return the index after the first sentence boundary, or -1."""
|
||
if len(buf) < _MIN_SENTENCE:
|
||
return -1
|
||
for m in _SENT_RE.finditer(buf):
|
||
if m.end() >= _MIN_SENTENCE:
|
||
return m.end()
|
||
return -1
|
||
|
||
# ── LLM helpers ───────────────────────────────────────────────────────────────
|
||
|
||
def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
|
||
"""Inline synchronous persona rewrite; raises RuntimeError on failure."""
|
||
settings = _load_settings()
|
||
system = (
|
||
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
||
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
||
"Return ONLY the rephrased text — no quotes, no explanation."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": text},
|
||
],
|
||
"temperature": 0.3,
|
||
"max_tokens": 512,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
resp = requests.post(
|
||
f"{llm_url.rstrip('/')}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
result = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
if result.startswith('"') and result.endswith('"'):
|
||
result = result[1:-1].strip()
|
||
return result
|
||
|
||
|
||
def _request_timeout_seconds(value, default: float = 600.0, minimum: float = 5.0, maximum: float = 600.0) -> float:
|
||
"""Clamp caller-provided LLM timeouts so UI recovery cannot hang indefinitely."""
|
||
try:
|
||
timeout = float(value)
|
||
except Exception:
|
||
timeout = default
|
||
return max(minimum, min(maximum, timeout))
|
||
|
||
|
||
def _is_router_model_alias(model: str) -> bool:
|
||
return bool(re.match(r"^\s*auto[-_ ]?router", model or "", re.I))
|
||
|
||
|
||
def _response_error_text(resp) -> str:
|
||
try:
|
||
text = resp.text or ""
|
||
except Exception:
|
||
text = ""
|
||
return text.strip()[:2000] or getattr(resp, "reason", "") or "Unknown upstream error"
|
||
|
||
|
||
def _llm_chat_completion_urls(llm_url: str) -> list[str]:
|
||
base = (llm_url or "http://localhost:11434/v1").rstrip("/")
|
||
urls: list[str] = []
|
||
if not re.search(r"/v\d+(?:\.\d+)?$", base):
|
||
urls.append(f"{base}/v1/chat/completions")
|
||
urls.append(f"{base}/chat/completions")
|
||
return list(dict.fromkeys(urls))
|
||
|
||
|
||
def _post_llm_chat_completion(llm_url: str, payload: dict, headers: dict, timeout: float):
|
||
last_resp = None
|
||
last_exc = None
|
||
for url in _llm_chat_completion_urls(llm_url):
|
||
try:
|
||
resp = requests.post(url, json=payload, headers=headers, timeout=timeout)
|
||
last_resp = resp
|
||
if resp.status_code in (404, 405) and url != _llm_chat_completion_urls(llm_url)[-1]:
|
||
continue
|
||
return resp
|
||
except Exception as exc:
|
||
last_exc = exc
|
||
if last_resp is not None:
|
||
return last_resp
|
||
if last_exc:
|
||
raise last_exc
|
||
raise RuntimeError("LLM request failed")
|
||
|
||
|
||
def _fallback_attribute_response(text: str) -> dict:
|
||
"""Deterministic fallback for attribution when an upstream LLM fails."""
|
||
quote_re = re.compile(
|
||
r"»([^«]+)«|«([^»]+)»|„([^“”]+)[“”]|“([^”]+)”|\"([^\"]+)\"|「([^」]+)」|『([^』]+)』",
|
||
re.DOTALL,
|
||
)
|
||
segments = []
|
||
pos = 0
|
||
for match in quote_re.finditer(text):
|
||
if match.start() > pos:
|
||
narr = text[pos:match.start()].strip()
|
||
if narr:
|
||
segments.append({"speaker": "Narrator", "type": "narration", "text": narr, "emotion": ""})
|
||
spoken = next((g for g in match.groups() if g), "")
|
||
if spoken.strip():
|
||
segments.append({"speaker": "Unknown", "type": "dialogue", "text": spoken.strip(), "emotion": ""})
|
||
pos = match.end()
|
||
tail = text[pos:].strip()
|
||
if tail:
|
||
segments.append({"speaker": "Narrator", "type": "narration", "text": tail, "emotion": ""})
|
||
if not segments:
|
||
segments = [{"speaker": "Narrator", "type": "narration", "text": text, "emotion": ""}]
|
||
chars = []
|
||
for seg in segments:
|
||
sp = seg.get("speaker")
|
||
if seg.get("type") == "dialogue" and sp and sp not in ("Narrator", "Unknown") and sp not in chars:
|
||
chars.append(sp)
|
||
return {"segments": segments, "characters": chars, "fallback": True}
|
||
|
||
|
||
def _resolve_speak_voice(settings: dict, client_id: str, explicit_voice: str) -> str:
|
||
if explicit_voice:
|
||
return explicit_voice
|
||
bindings: dict = settings.get("client_voice_bindings") or {}
|
||
if client_id and client_id in bindings:
|
||
return bindings[client_id]
|
||
return settings.get("captures_default_voice") or ""
|
||
|
||
|
||
# ── Routes ────────────────────────────────────────────────────────────────────
|
||
|
||
@router.post("/api/refine-text")
|
||
async def refine_text(request: Request):
|
||
"""Clean up raw STT transcription using a local OpenAI-compatible LLM."""
|
||
data = await request.json()
|
||
settings = _load_settings()
|
||
text: str = (data.get("text") or "").strip()
|
||
llm_url: str = (data.get("llm_url") or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or settings.get("refine_model") or settings.get("llm_model") or "").strip()
|
||
toggles: dict = data.get("toggles") or {}
|
||
|
||
if not text:
|
||
raise HTTPException(400, "No text to refine")
|
||
|
||
rules = []
|
||
if toggles.get("fillers", True):
|
||
rules.append("Remove filler words (um, uh, like, you know, basically, literally, I mean, so, right, etc.)")
|
||
if toggles.get("repetitions", True):
|
||
rules.append("Remove repeated words and false starts (e.g. 'the the dog' → 'the dog', 'I was- I was going' → 'I was going')")
|
||
if toggles.get("corrections", True):
|
||
rules.append("Remove self-corrections and restarts, keeping only the final intended phrasing")
|
||
if toggles.get("punctuation", True):
|
||
rules.append("Fix punctuation, capitalisation, and sentence boundaries")
|
||
if not rules:
|
||
return {"text": text, "original": text}
|
||
|
||
system = (
|
||
"You are a transcription cleanup assistant. "
|
||
"Apply ONLY the following rules to the user's text. "
|
||
"Return ONLY the cleaned text — no explanations, no quotes, no markdown:\n"
|
||
+ "\n".join(f"- {r}" for r in rules)
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": text},
|
||
],
|
||
"temperature": 0.1,
|
||
"max_tokens": 2048,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
refined = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
if refined.startswith('"') and refined.endswith('"'):
|
||
refined = refined[1:-1].strip()
|
||
return {"text": refined, "original": text}
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM refinement failed: {e}")
|
||
|
||
|
||
@router.post("/api/rewrite-with-persona")
|
||
async def rewrite_with_persona(request: Request):
|
||
"""Rewrite user text in a voice persona's character using a local LLM."""
|
||
data = await request.json()
|
||
settings = _load_settings()
|
||
text: str = (data.get("text") or "").strip()
|
||
persona: str = (data.get("persona") or "").strip()
|
||
llm_url: str = (data.get("llm_url") or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or settings.get("llm_model") or "").strip()
|
||
mode: str = (data.get("mode") or "rewrite").strip()
|
||
|
||
if not persona:
|
||
raise HTTPException(400, "No persona defined for this voice")
|
||
if not text and mode != "compose":
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
if mode == "compose":
|
||
system = (
|
||
f"You are a voice assistant with this character: {persona}\n"
|
||
"Write a single natural utterance in this character's voice about the topic given. "
|
||
"Return ONLY the utterance — no quotes, no explanation."
|
||
)
|
||
user_msg = text or "Introduce yourself briefly."
|
||
temp = 0.9
|
||
else:
|
||
system = (
|
||
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
||
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
||
"Return ONLY the rephrased text — no quotes, no explanation."
|
||
)
|
||
user_msg = text
|
||
temp = 0.3
|
||
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user_msg},
|
||
],
|
||
"temperature": temp,
|
||
"max_tokens": 512,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
result = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
if result.startswith('"') and result.endswith('"'):
|
||
result = result[1:-1].strip()
|
||
return {"text": result, "original": text, "persona": persona}
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM persona rewrite failed: {e}")
|
||
|
||
|
||
@router.post("/api/analyze-characters")
|
||
async def analyze_characters(request: Request):
|
||
"""Analyze a script with an LLM and return per-character voice descriptions.
|
||
|
||
Used by the Script Rehearser to auto-design voices that match each role.
|
||
Returns: {"characters": [{name, gender, language, age, description}, ...]}
|
||
"""
|
||
data = await request.json()
|
||
script: str = (data.get("script") or "").strip()
|
||
names: list = data.get("names") or []
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not script:
|
||
raise HTTPException(400, "No script provided")
|
||
if not names:
|
||
raise HTTPException(400, "No character names provided")
|
||
|
||
# Truncate script to leave room for output within typical model context windows.
|
||
# German/non-English text tokenises at ~2.5–3 chars/token, so 6000 chars ≈ 2000–2400 tokens.
|
||
if len(script) > 6000:
|
||
script = script[:6000]
|
||
|
||
lang_hint = f" The script language is {language}." if language else ""
|
||
|
||
def _build_system() -> str:
|
||
return (
|
||
"You are a casting director and TTS voice-design expert. "
|
||
"Your job is to read a script, understand each character deeply from their "
|
||
"dialogue, role, and context, then write a voice description that a "
|
||
"text-to-speech model can use to generate a matching voice.\n"
|
||
f"{lang_hint}\n"
|
||
"For EVERY character in the provided list output:\n"
|
||
"- name: exact name as given\n"
|
||
"- gender: M, F, or N\n"
|
||
"- language: spoken language of this character\n"
|
||
"- age: estimated age range (e.g. 20s, 40s, elderly)\n"
|
||
"- description: 2–3 sentences covering pitch (high/mid/low), pace, "
|
||
"timbre, accent/dialect, emotional default, and any distinctive speech trait "
|
||
"that fits the character's personality and role.\n"
|
||
"Characters with few lines: infer from their role name and context.\n"
|
||
"Respond with STRICT JSON only — no markdown, no explanation:\n"
|
||
'{"characters":[{"name":"NAME","gender":"M|F|N","language":"LANG",'
|
||
'"age":"30s","description":"voice description"}]}\n'
|
||
"Use the exact character names provided.\n/no-think"
|
||
)
|
||
|
||
def _call_llm(batch: list[str]) -> list[dict]:
|
||
"""Call the LLM for one batch of character names; return list of character dicts."""
|
||
user_msg = (
|
||
"Character names: " + ", ".join(str(n) for n in batch) + "\n\n"
|
||
"Script:\n" + script
|
||
)
|
||
# 150 tokens per character output, capped at 3072 to stay within 16K context
|
||
mt = min(3072, max(512, len(batch) * 150))
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": _build_system()},
|
||
{"role": "user", "content": user_msg},
|
||
],
|
||
"temperature": 0.4,
|
||
"max_tokens": mt,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
for candidate in (content, _extract_json_block(content)):
|
||
if not candidate:
|
||
continue
|
||
try:
|
||
parsed = json.loads(candidate)
|
||
if isinstance(parsed, dict) and "characters" in parsed:
|
||
return parsed["characters"]
|
||
except Exception:
|
||
continue
|
||
return []
|
||
|
||
# Process cast in batches of 10 to stay well within 16 K context
|
||
BATCH = 10
|
||
all_characters: list[dict] = []
|
||
try:
|
||
for i in range(0, len(names), BATCH):
|
||
batch = names[i:i + BATCH]
|
||
all_characters.extend(_call_llm(batch))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM character analysis failed: {e}")
|
||
|
||
if not all_characters:
|
||
raise HTTPException(502, "LLM did not return valid character JSON")
|
||
|
||
return {"characters": all_characters}
|
||
|
||
|
||
@router.post("/api/match-characters-voices")
|
||
async def match_characters_voices(request: Request):
|
||
"""Pick the best EXISTING library voice for each character (instead of designing new ones).
|
||
|
||
Body: {script, names:[...], voices:[{id, gender, language, tags, description}], llm_url, model, language}
|
||
Returns: {"assignments": [{name, voice_id, reason}]}
|
||
"""
|
||
data = await request.json()
|
||
script: str = (data.get("script") or "").strip()
|
||
names: list = data.get("names") or []
|
||
voices: list = data.get("voices") or []
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not names:
|
||
raise HTTPException(400, "No character names provided")
|
||
if not voices:
|
||
raise HTTPException(400, "No candidate voices provided")
|
||
if len(script) > 5000:
|
||
script = script[:5000]
|
||
|
||
def _vline(v: dict) -> str:
|
||
meta = [str(v.get(k)) for k in ("gender", "language") if v.get(k)]
|
||
if v.get("tags"):
|
||
meta.append("tags:" + str(v["tags"]))
|
||
desc = str(v.get("description") or v.get("name") or "")[:120]
|
||
return f"- {v.get('id')} [{', '.join(meta)}] {desc}".rstrip()
|
||
|
||
catalogue = "\n".join(_vline(v) for v in voices[:300])
|
||
valid_ids = {str(v.get("id")) for v in voices if v.get("id")}
|
||
|
||
system = (
|
||
"You are a casting director assigning existing TTS voices to script characters. "
|
||
f"{('Script language: ' + language + '. ') if language else ''}"
|
||
"For EACH character, choose the single BEST voice_id from the CATALOGUE. "
|
||
"RULE 1 — GENDER FIRST: the voice's gender MUST match the character's gender whenever the "
|
||
"character's gender is clear from the script; only pick a different gender if no same-gender "
|
||
"voice exists in the catalogue. "
|
||
"RULE 2 — then match apparent age, language, personality, and the voice's description/tags. "
|
||
"You MUST choose a voice_id that appears verbatim in the catalogue — never invent one. "
|
||
"Reuse a voice for two characters only if no better distinct option exists. "
|
||
"Respond with STRICT JSON only, no markdown:\n"
|
||
'{"assignments":[{"name":"NAME","voice_id":"ID","reason":"short reason"}]}\n/no-think'
|
||
)
|
||
user = (
|
||
"Characters to cast: " + ", ".join(str(n) for n in names) + "\n\n"
|
||
"CATALOGUE (voice_id [gender, language, tags] description):\n" + catalogue + "\n\n"
|
||
"Script excerpt:\n" + script
|
||
)
|
||
payload: dict = {
|
||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||
"temperature": 0.3,
|
||
"max_tokens": min(2048, max(512, len(names) * 60)),
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
import time as _time
|
||
raw = None
|
||
last_err: Exception | None = None
|
||
for attempt in range(3):
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
break
|
||
except requests.exceptions.ConnectionError as e:
|
||
# llama-swap (and similar) often drop the first request while swapping/loading
|
||
# the model — wait and retry rather than failing the cast.
|
||
last_err = e
|
||
_time.sleep(4 + attempt * 3)
|
||
except Exception as e:
|
||
last_err = e
|
||
break
|
||
if raw is None:
|
||
raise HTTPException(502, f"LLM voice matching failed: {last_err}")
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
assignments: list = []
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("assignments"), list):
|
||
assignments = parsed["assignments"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
# Keep only assignments that reference a real catalogue voice
|
||
clean = [a for a in assignments if isinstance(a, dict) and str(a.get("voice_id")) in valid_ids]
|
||
return {"assignments": clean}
|
||
|
||
|
||
def _extract_json_block(text: str) -> str:
|
||
"""Pull the first {...} JSON object out of an LLM response."""
|
||
text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
|
||
if text.startswith("```"):
|
||
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
|
||
text = re.sub(r"\n?```$", "", text).strip()
|
||
start = text.find("{")
|
||
end = text.rfind("}")
|
||
if start != -1 and end != -1 and end > start:
|
||
return text[start:end + 1]
|
||
return ""
|
||
|
||
|
||
@router.post("/api/character-sheets")
|
||
async def character_sheets(request: Request):
|
||
"""Extract actor-facing RPG-style character sheets from a passage.
|
||
|
||
Body: {text, known_characters:[...], language, llm_url, model}
|
||
The text may contain "[p.N]" page markers so the model can cite sources.
|
||
Returns: {sheets:[{name, aliases, first_name, last_name, full_name, title, archetype, physical, alignment,
|
||
attribute_high, attribute_low, skills, inventory:[...], secret,
|
||
conflict_style, win_condition, tier:"main"|"supporting",
|
||
sources:[{page, quote}]}], characters:[names]}
|
||
Deduced (not explicit) values are marked with a trailing " *".
|
||
"""
|
||
data = await request.json()
|
||
text: str = (data.get("text") or "").strip()
|
||
known: list = data.get("known_characters") or []
|
||
target_mode: bool = bool(data.get("target_mode"))
|
||
existing: str = (data.get("existing") or "").strip() # partial sheets so far (progressive fill)
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
timeout_seconds = _request_timeout_seconds(data.get("timeout_seconds"), 600.0)
|
||
if not text:
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
lang_instruction = (
|
||
f"⚠️ SPRACHE / LANGUAGE — OBLIGATORISCH / MANDATORY:\n"
|
||
f"The source text is in {language}.\n"
|
||
f"YOU MUST write EVERY descriptive field value in {language}. This is non-negotiable.\n"
|
||
f"Fields that MUST be in {language}: physical, clothing, alignment, arc_note, skills, "
|
||
f"capabilities, backstory, relationships, motivation, fears, mannerisms, voice_pattern, secret, "
|
||
f"conflict_style, win_condition, archetype, aliases, first_name, last_name, full_name, title.\n"
|
||
f"voice_design_prompt and image_prompt should be concise English tool prompts for generation tools.\n"
|
||
f"JSON field KEYS stay in English. Character names stay exactly as they appear in the text.\n"
|
||
f"If you write any descriptive value in English instead of {language}, your response is WRONG.\n\n"
|
||
) if language else ""
|
||
target_instruction = (
|
||
"CAST TARGET MODE:\n"
|
||
"- The known-characters list is the already recognized cast roster from the audiobook pass. "
|
||
"Treat it as TARGET PROFILES to fill, not as a list of aliases and not as proof that those names are the same person.\n"
|
||
"- Prefer outputting sheets for those target names when this passage reveals usable profile details. "
|
||
"Do not output a blank sheet just because a target is listed.\n"
|
||
"- If the passage proves that a target has another name/title/nickname, keep ONE profile using the best known target name "
|
||
"and put only the proven alternate form in aliases/title/full_name.\n"
|
||
"- You may add a brand-new character only when the passage clearly introduces one that is not in the target roster.\n"
|
||
"- NEVER copy the known-characters roster into one character's aliases, relationships, title, or description fields.\n\n"
|
||
) if target_mode and known else ""
|
||
system = (
|
||
f"{lang_instruction}"
|
||
f"{target_instruction}"
|
||
"You are an expert dramaturge, developmental editor, and tabletop RPG game master building rich "
|
||
"character sheets passage by passage as a book is read. Extract playable, action-oriented sheets "
|
||
"an actor can use to immediately know how to PLAY the character.\n"
|
||
"PROGRESSIVE FILLING: you may be given the sheets built so far. For returning characters, ADD any "
|
||
"NEW detail this passage reveals and refine vague fields; do not contradict solid earlier facts or "
|
||
"blank out a field you cannot improve. Add brand-new characters as they appear. Leave a field empty "
|
||
"if the book genuinely hasn't shown it yet (a later passage can fill it). Extrapolate from dialogue "
|
||
"and actions when reasonable, and mark any deduced value with a trailing ' *'.\n"
|
||
"For each character output these fields:\n"
|
||
"- name: canonical display name for this one character. Use the real personal name if known; otherwise use the most stable role/title.\n"
|
||
"- aliases: ONLY alternate names, roles, epithets, mistranscriptions, and titles proven to refer to the SAME character, comma-separated (max 6 items; e.g. 'Henker, Vampir, Zerwas der Henker'). Leave empty when uncertain.\n"
|
||
"- first_name, last_name, full_name, title: split the character identity when known. Leave unknown parts empty. Put noble/office/role labels in title (e.g. 'Henker', 'Vampir', 'Graf').\n"
|
||
"- archetype: a two-word role summary (e.g. 'Ruthless Scholar')\n"
|
||
"- gender: 'male', 'female', or 'nonbinary' — as apparent from the text (pronouns, roles, physical description). Leave empty if genuinely indeterminable.\n"
|
||
"- physical: age, height, build, hair, eyes, skin, posture, gait, vocal quality. Use ONLY metric system.\n"
|
||
"- clothing: distinctive clothing, armour, accessories — as observed in the text\n"
|
||
"- alignment: strict moral code + the one line they will never cross\n"
|
||
"- moral_alignment_score: integer 0–100. 100 = purely good/heroic, 0 = purely evil/villainous, 50 = neutral/ambiguous\n"
|
||
"- arc_direction: one of: 'stable-good', 'stable-bad', 'neutral', 'good-to-bad', 'bad-to-good', 'complex'\n"
|
||
"- arc_note: one sentence explaining the arc or moral position visible so far\n"
|
||
"- attribute_high / attribute_low: highest and lowest natural attribute (Charisma, Intelligence, Wisdom, Agility…)\n"
|
||
"- skills: what they are demonstrably good at in the story\n"
|
||
"- capabilities: combat, magic, social, technical, or other demonstrated abilities\n"
|
||
"- backstory: origin, formative background and history revealed in the text\n"
|
||
"- relationships: key allies, family, rivals and enemies — name them and how they relate\n"
|
||
"- motivation: the inner drive — WHY they pursue what they pursue (distinct from the win condition)\n"
|
||
"- fears: their deepest fears, phobias or dread\n"
|
||
"- mannerisms: habitual gestures, tics, body language, habits and quirks\n"
|
||
"- voice_pattern: speech style — accent, pacing, vocabulary, register and verbal tics (for voice casting)\n"
|
||
"- voice_design_prompt: concise English Qwen voice-design prompt (15-45 words). Include age impression, gender/androgyny if inferable, pitch, timbre, pace, accent/register, emotional baseline and suitability for audiobook dialogue. Do NOT mention plot spoilers.\n"
|
||
"- image_prompt: detailed English image-generation prompt for this character. Include face, age impression, build, hair/eyes/skin if known, clothing, posture, props, mood, genre/style, and visible symbols. Mark inferred traits with '*'.\n"
|
||
"- inventory: 1-3 defining items/props/clothing (array of short strings)\n"
|
||
"- secret: dark secret or fatal flaw\n"
|
||
"- conflict_style: fight, flight, or manipulate — how they act when cornered\n"
|
||
"- win_condition: the specific event that would make them feel they have won\n"
|
||
"- tier: 'main' or 'supporting'\n"
|
||
"- sources: array of {page, quote, line_hint} — the page number from the nearest [p.N] marker, "
|
||
"a short verbatim quote that supports the sheet (1-12 entries), and a brief label (e.g. 'physical', 'clothing', 'relationships', 'motivation'). "
|
||
"Use null page if unknown. line_hint MUST name the supported field when possible: physical, clothing, relationships, motivation, fears, mannerisms, voice_pattern, backstory, alignment, skills, capabilities, secret, conflict_style, or win_condition.\n"
|
||
"IDENTITY MERGING: A character may appear under multiple names in the book (first name, last name, title, role, alias, nickname). "
|
||
"Examples: 'Zerwas', 'Henker', and 'Vampir' may all refer to ONE profile if context shows they are the same person. "
|
||
"Do NOT create separate sheets for aliases/titles of the same person; put the alternate forms in aliases/title/full_name and keep one canonical name.\n"
|
||
"Reuse the EXACT names from the known-characters list for returning characters when they are the canonical name or an alias of this character. "
|
||
"Do not merge characters merely because their names appear near each other, in the known-character list, or in relationships.\n"
|
||
"Respond with STRICT JSON only:\n"
|
||
'{"sheets":[{"name":"","aliases":"","first_name":"","last_name":"","full_name":"","title":"","archetype":"","gender":"","physical":"","clothing":"",'
|
||
'"alignment":"","moral_alignment_score":50,"arc_direction":"neutral","arc_note":"",'
|
||
'"attribute_high":"","attribute_low":"","skills":"","capabilities":"",'
|
||
'"backstory":"","relationships":"","motivation":"","fears":"","mannerisms":"","voice_pattern":"","voice_design_prompt":"","image_prompt":"",'
|
||
'"inventory":[],"secret":"","conflict_style":"","win_condition":"",'
|
||
'"tier":"main","sources":[{"page":1,"quote":"","line_hint":""}]}]}\n/no-think'
|
||
)
|
||
lang_reminder = (
|
||
f"⚠️ WICHTIG: Alle beschreibenden Feldwerte MÜSSEN auf {language} geschrieben werden. "
|
||
f"Kein einziges beschreibendes Feld darf auf Englisch sein. Nur JSON-Schlüssel bleiben Englisch.\n\n"
|
||
) if language else ""
|
||
user = (
|
||
("Known characters so far: " + ", ".join(str(n) for n in known) + "\n\n" if known else "")
|
||
+ ("Sheets so far (fill gaps / refine; keep solid facts):\n" + existing + "\n\n" if existing else "")
|
||
+ lang_reminder
|
||
+ "Passage:\n" + text
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.4,
|
||
"max_tokens": 4096,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
try:
|
||
resp = await asyncio.to_thread(
|
||
_post_llm_chat_completion,
|
||
llm_url, payload,
|
||
{"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout_seconds,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM character-sheet generation failed: {e}")
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
sheets = []
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("sheets"), list):
|
||
sheets = parsed["sheets"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
clean, names = [], []
|
||
alias_splitter = re.compile(r"[,;/|]|\baka\b|\baka\.\b|\balias(?:es)?\b|\bgenannt\b|\bnamens\b|\bcalled\b|\bknown as\b", re.I)
|
||
for s in sheets:
|
||
if not isinstance(s, dict):
|
||
continue
|
||
name = str(s.get("name") or "").strip()
|
||
if not name:
|
||
continue
|
||
alias_raw = str(s.get("aliases") or "").strip()
|
||
alias_parts = [p.strip() for p in alias_splitter.split(alias_raw) if p.strip()]
|
||
alias_parts = [p for p in alias_parts if len(p) <= 80 and not re.match(r"^(needs?:|complete$)", p, re.I)]
|
||
if len(alias_raw) > 500 or len(alias_parts) > 12:
|
||
aliases = ""
|
||
else:
|
||
aliases = ", ".join(alias_parts[:6])
|
||
inv = s.get("inventory")
|
||
if isinstance(inv, str):
|
||
inv = [x.strip() for x in inv.split(",") if x.strip()]
|
||
elif not isinstance(inv, list):
|
||
inv = []
|
||
src_raw = s.get("sources") if isinstance(s.get("sources"), list) else []
|
||
src = []
|
||
for item in src_raw:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
quote = str(item.get("quote") or "").strip()
|
||
if not quote:
|
||
continue
|
||
try:
|
||
page = int(item.get("page")) if item.get("page") is not None else None
|
||
except (TypeError, ValueError):
|
||
page = None
|
||
src.append({
|
||
"page": page,
|
||
"quote": quote[:240],
|
||
"line_hint": str(item.get("line_hint") or "").strip()[:60],
|
||
})
|
||
# Clamp moral alignment score
|
||
try:
|
||
mas = int(s.get("moral_alignment_score") or 50)
|
||
mas = max(0, min(100, mas))
|
||
except (TypeError, ValueError):
|
||
mas = 50
|
||
arc = str(s.get("arc_direction") or "neutral").strip()
|
||
if arc not in ("stable-good", "stable-bad", "neutral", "good-to-bad", "bad-to-good", "complex"):
|
||
arc = "neutral"
|
||
gender = str(s.get("gender") or "").strip().lower()
|
||
if gender not in ("male", "female", "nonbinary"):
|
||
gender = ""
|
||
s.update({
|
||
"name": name, "aliases": aliases, "inventory": inv[:3],
|
||
"tier": "main" if str(s.get("tier") or "").lower().startswith("main") else "supporting",
|
||
"sources": src[:12],
|
||
"moral_alignment_score": mas,
|
||
"arc_direction": arc,
|
||
"gender": gender,
|
||
})
|
||
clean.append(s)
|
||
names.append(name)
|
||
return {"sheets": clean, "characters": names}
|
||
|
||
|
||
@router.post("/api/character-deep-analysis")
|
||
async def character_deep_analysis(request: Request):
|
||
"""Run a deep 5-area psychological analysis of a single character.
|
||
|
||
Body: {name, role, goal, summary, text_excerpt, language, llm_url, model}
|
||
Returns: {analysis: {core_flaw, agency, dialogue_voice, narrative_arc, paradox}}
|
||
"""
|
||
data = await request.json()
|
||
name: str = (data.get("name") or "").strip()
|
||
role: str = (data.get("role") or "unknown").strip()
|
||
goal: str = (data.get("goal") or "").strip()
|
||
summary: str = (data.get("summary") or "").strip()
|
||
excerpt: str = (data.get("text_excerpt") or "").strip()
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not name:
|
||
raise HTTPException(400, "Character name required")
|
||
|
||
lang_note = f" Write the analysis in {language}." if language else ""
|
||
|
||
system = (
|
||
f"You are an expert developmental editor, literary coach, and psychologist specializing in "
|
||
f"profound character studies.{lang_note} Provide a deep, multi-layered psychological and "
|
||
f"narrative analysis of the character in exactly this JSON structure:\n"
|
||
'{"core_flaw":"<analysis of primary flaw, its psychological origin, and conflict with their desire>",'
|
||
'"agency":"<are they active or reactive? where does their momentum stall? what harder choices could force growth?>",'
|
||
'"dialogue_voice":"<speech pattern analysis: how does their dialogue reflect background, anxieties, hidden motives? how do they sound distinct?>",'
|
||
'"narrative_arc":"<how does their worldview shift? what is the lie they believe vs the truth they must accept?>",'
|
||
'"paradox":"<3 specific contradictions in behaviour/personality that make them feel authentic and unpredictable>"}\n'
|
||
"Be ruthlessly comprehensive and honest. Use concrete examples from the text. "
|
||
"Map where character development might break down or feel cliché. "
|
||
"Respond with STRICT JSON only — no markdown, no explanation./no-think"
|
||
)
|
||
user = (
|
||
f'CHARACTER: "{name}"\n'
|
||
f"Role in story: {role}\n"
|
||
f"Core goal: {goal or 'not specified'}\n"
|
||
f"Background/key actions: {summary or 'not specified'}\n\n"
|
||
+ (f"Text excerpt for analysis:\n{excerpt[:4000]}\n\n" if excerpt else "")
|
||
+ "Provide the deep psychological analysis now."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.6,
|
||
"max_tokens": 2048,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM deep analysis failed: {e}")
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
analysis = {}
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and "core_flaw" in parsed:
|
||
analysis = parsed
|
||
break
|
||
except Exception:
|
||
continue
|
||
if not analysis:
|
||
analysis = {"core_flaw": raw, "agency": "", "dialogue_voice": "", "narrative_arc": "", "paradox": ""}
|
||
return {"name": name, "analysis": analysis}
|
||
|
||
|
||
@router.post("/api/character-generate-prompts")
|
||
async def character_generate_prompts(request: Request):
|
||
"""Turn an already-extracted character sheet into four ready-to-use external
|
||
prompts, in one LLM call over the character's full profile (rather than
|
||
generating them incrementally passage-by-passage, where the model only
|
||
sees a fraction of the character at a time).
|
||
|
||
Body: {name, book, sheet: {...character sheet fields...}, language, llm_url, model}
|
||
Returns: {voice_design_prompt, image_prompt, silly_tavern_prompt, concept_art_prompt}
|
||
"""
|
||
data = await request.json()
|
||
name: str = (data.get("name") or "").strip()
|
||
book: str = (data.get("book") or "").strip()
|
||
sheet: dict = data.get("sheet") or {}
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
if not name:
|
||
raise HTTPException(400, "Character name required")
|
||
|
||
def _f(key: str) -> str:
|
||
v = sheet.get(key)
|
||
if isinstance(v, list):
|
||
return ", ".join(str(x) for x in v if x)
|
||
return str(v or "").strip()
|
||
|
||
profile = "\n".join(f"{label}: {v}" for label, v in [
|
||
("Name", name), ("Aliases/titles", _f("aliases") or _f("title")),
|
||
("Archetype", _f("archetype")), ("Physical", _f("physical")), ("Clothing", _f("clothing")),
|
||
("Mannerisms", _f("mannerisms")), ("Voice/speech pattern", _f("voice_pattern")),
|
||
("Backstory", _f("backstory")), ("Motivation", _f("motivation")), ("Fears", _f("fears")),
|
||
("Relationships", _f("relationships")), ("Conflict style", _f("conflict_style")),
|
||
("Secret/flaw", _f("secret")), ("Inventory", _f("inventory")),
|
||
] if v)
|
||
|
||
lang_note = f" Write every prompt in {language} EXCEPT where told to use English." if language else ""
|
||
preamble = (
|
||
"You are a prompt engineer who turns a fiction character's profile into ready-to-paste prompts for "
|
||
"other tools. Use ONLY details present in the profile below; mark anything you must reasonably infer "
|
||
f"with a trailing '*'. Never invent plot spoilers not implied by the profile.{lang_note}\n\n"
|
||
)
|
||
user = f"BOOK: {book or 'unspecified'}\n\nCHARACTER PROFILE:\n{profile or name}\n\n"
|
||
|
||
# One independent call per field (not one four-field JSON object, nor even
|
||
# the two-field pairing this used briefly) — the UI now has a Generate
|
||
# button per prompt box, so a click on just one must not also burn tokens
|
||
# regenerating the other three. Singling them out also further shrinks
|
||
# each response, since a four-field JSON object reliably truncated the
|
||
# LAST fields regardless of the token ceiling.
|
||
all_groups = {
|
||
"voice_design_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- voice_design_prompt: an English prompt for Qwen3 TTS Voice Design (15-45 words, one paragraph, "
|
||
"no markdown). Cover: apparent age, gender/androgyny if inferable, pitch, timbre/texture, pace, "
|
||
"accent or register, emotional baseline, and suitability for audiobook dialogue delivery. Do not "
|
||
"mention plot events — describe only how the voice should SOUND.\n\n"
|
||
'Respond with STRICT JSON only: {"voice_design_prompt":""}/no-think'
|
||
),
|
||
"image_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- image_prompt: a detailed English image-generation prompt for a character profile picture/portrait. "
|
||
"Include face and expression typical of this character, age impression, build, hair/eyes/skin if known, "
|
||
"clothing, signature tools/weapons/props, an environment typical for them, mood, and an art style "
|
||
"(e.g. 'detailed digital painting, dramatic lighting'). One dense paragraph, comma-separated descriptors "
|
||
"are fine.\n\n"
|
||
'Respond with STRICT JSON only: {"image_prompt":""}/no-think'
|
||
),
|
||
"silly_tavern_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- silly_tavern_prompt: character-card content for SillyTavern, formatted as labelled sections on their "
|
||
"own lines: 'Description:' (physical + personality summary), 'Personality:' (a compact trait list), "
|
||
"'Scenario:' (the situation/setting they're typically found in), 'First message:' (one in-character "
|
||
"greeting line in their own voice/speech pattern), and 'Example dialogue:' (2-3 short in-character "
|
||
"lines showing their manner of speech). Keep each section a few lines at most.\n\n"
|
||
'Respond with STRICT JSON only: {"silly_tavern_prompt":""}/no-think'
|
||
),
|
||
"concept_art_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- concept_art_prompt: an English prompt for a character CONCEPT SHEET (not a single portrait) — "
|
||
"a turnaround/reference sheet with multiple views and expressions: front view, side or back view, "
|
||
"2-3 facial expressions, and a close-up of a signature prop/costume detail, all on one clean sheet, "
|
||
"in a character-design-sheet art style (e.g. 'character turnaround, model sheet, flat lighting, "
|
||
"white background').\n\n"
|
||
'Respond with STRICT JSON only: {"concept_art_prompt":""}/no-think'
|
||
),
|
||
}
|
||
requested = [f for f in (data.get("fields") or []) if f in all_groups]
|
||
field_groups = [((f,), all_groups[f]) for f in (requested or all_groups.keys())]
|
||
|
||
def _call_group(fields: tuple, system: str) -> dict:
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user + f"Generate the {' and '.join(fields)} now."},
|
||
],
|
||
"temperature": 0.7,
|
||
"max_tokens": 1536,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _msg.get("reasoning_content") or "").strip()
|
||
except Exception as e:
|
||
print(f"[character-generate-prompts] LLM call failed for {fields}: {e}")
|
||
return {}
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
block = _extract_json_block(content)
|
||
# If the model still got cut off mid-string, closing the dangling string +
|
||
# object often salvages the field that DID complete.
|
||
candidates = [content, block]
|
||
if block:
|
||
candidates.extend([block + "\"}", block + "}"])
|
||
for cand in candidates:
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and fields[0] in parsed:
|
||
return parsed
|
||
except Exception:
|
||
continue
|
||
return {}
|
||
|
||
results = await asyncio.gather(*[
|
||
asyncio.to_thread(_call_group, fields, system) for fields, system in field_groups
|
||
])
|
||
out = {}
|
||
for (fields, _), result in zip(field_groups, results):
|
||
for f in fields:
|
||
out[f] = str(result.get(f) or "").strip()
|
||
if not any(out.values()):
|
||
# A silent all-empty response looked identical to success in the UI —
|
||
# fail loudly so the client can show a real error instead.
|
||
raise HTTPException(502, "LLM returned no parseable prompts — try again (or a different model)")
|
||
return out
|
||
|
||
|
||
def _attribution_prepare(data: dict) -> dict:
|
||
"""Resolve settings and build the chat payload for one attribution request.
|
||
Shared by the blocking endpoint and the streaming (watch-the-LLM-think)
|
||
endpoint so the prompt logic can never drift between the two."""
|
||
text: str = (data.get("text") or "").strip()
|
||
known: list = data.get("known_characters") or []
|
||
recent: str = (data.get("recent") or "").strip() # last few attributed lines, for continuity
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
timeout_seconds = _request_timeout_seconds(data.get("timeout_seconds"), 600.0)
|
||
if not text:
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
lang_hint = f" The text language is {language}; keep names and wording in that language." if language else ""
|
||
|
||
prompt_override = data.get("audiobook_prompt")
|
||
if not isinstance(prompt_override, str):
|
||
prompt_override = data.get("prompt")
|
||
base_prompt = (prompt_override if isinstance(prompt_override, str) else "") or _settings.get("audiobook_prompt") or ""
|
||
# A user-saved custom prompt can predate (or have replaced) the deduction
|
||
# rules that cut down false Unknown/Narrator attributions — the client-side
|
||
# prompt migration only upgrades prompts still containing the original
|
||
# anchor text. Guarantee the rules ride along regardless of what prompt is
|
||
# in play, so attribution quality never silently depends on prompt history.
|
||
if base_prompt.strip() and "Doppelpunkt-Regel" not in base_prompt:
|
||
base_prompt += (
|
||
"\n\nANALYSE-REGELN FÜR DIE ZUORDNUNG DES SPRECHERS (zusätzlich, immer anwenden):\n"
|
||
"1. Pronomen (er/sie) IMMER zum zuletzt genannten Charakter passenden Geschlechts auflösen.\n"
|
||
"2. Doppelpunkt-Regel: Endet ein Erzählersatz mit \":\", spricht dessen Subjekt das folgende Zitat "
|
||
"(z.B. \"Dann richtete er sich auf und rief in die Runde:\" → der zuvor genannte Charakter spricht).\n"
|
||
"3. Nachgestellte Zuordnung: Der Erzählersatz NACH einem Zitat verrät oft den Sprecher — auch bei "
|
||
"unpersönlicher Formel (\"»Was machst du da?« ertönte es über ihm. Karyla war herübergekommen.\" → Karyla sprach).\n"
|
||
"4. Adressaten-Regel: \"X wandte sich an Y\" / \"X sah Y an\" → X spricht das nächste Zitat, Y antwortet.\n"
|
||
"5. Ping-Pong: Zwei Personen im Gespräch wechseln sich strikt ab, auch über viele Zitate ohne Tags. "
|
||
"In einer Zwei-Personen-Szene ist 'Unknown' fast immer falsch.\n"
|
||
"6. Rollenbezeichnungen sind gültige Sprecher ('Ork', 'Der Fremde', 'Nachbar', 'Wächter') — nutze sie statt 'Unknown'.\n"
|
||
"7. 'Unknown' NUR, wenn eine Zuordnung trotz aller Regeln absolut unmöglich ist."
|
||
)
|
||
if not base_prompt.strip():
|
||
base_prompt = (
|
||
"You attribute dialogue in prose fiction for a multi-voice audiobook. "
|
||
"Split the passage into consecutive segments in reading order. For each segment output:\n"
|
||
"- speaker: 'Narrator' for narration/description, or the character's name for spoken dialogue. "
|
||
"Use the exact English word 'Unknown' ONLY as an absolute last resort.\n"
|
||
"- type: 'narration' or 'dialogue'\n"
|
||
"- text: the verbatim spoken words for dialogue (WITHOUT the surrounding quotation marks), or the verbatim prose for narration\n"
|
||
"- emotion: for dialogue, one or two words in the SAME LANGUAGE as the text (e.g. for German: wütend, traurig, flüsternd); '' for narration\n"
|
||
f"{lang_hint}\n"
|
||
"QUOTATION STYLES — books mark speech in many ways; treat ALL of these as spoken dialogue:\n"
|
||
" English straight \"...\" and curly “...”; German »...« (guillemets pointing inward) and „...“; "
|
||
"French «...» (pointing outward); single ‘...’; CJK 「...」 『...』; and em-dash speech where a line "
|
||
"starts with — or – (Spanish/French/Polish style).\n"
|
||
"PARAGRAPH/CHAPTER STRUCTURE: a blank line marks a real paragraph break or a chapter/scene start. "
|
||
"A very short standalone line right before a blank line (e.g. 'Chapter 1', 'Prologue', a bare number) "
|
||
"is a chapter heading — always narration/'Narrator', never dialogue. Keep blank lines as part of the "
|
||
"surrounding narration segment or their own narration segment; never invent dialogue from them and "
|
||
"never drop them from the reconstructed text.\n"
|
||
"German guillemets are the MOST IMPORTANT to detect: »Was schaust du dir an?« is a spoken line.\n"
|
||
"ATTRIBUTING THE SPEAKER (this is the hard, important part — be decisive):\n"
|
||
"1. If there is a dialogue tag ('sagte Riskan', 'fragte sie', 'Peter said'), use it. Resolve pronouns "
|
||
"(er/sie/he/she) to the actual name from nearby context.\n"
|
||
"2. UNTAGGED lines: use **conversational turn-taking**. In a two-person exchange the speaker ALTERNATES "
|
||
"every line — if Riskan just spoke, the next untagged quote is the other person, then back to Riskan, and so on.\n"
|
||
"3. Use the scene context, action beats around a quote (the person doing the action usually speaks), the "
|
||
"'Recent dialogue' below (continue the same conversation/alternation across the passage boundary), and the "
|
||
"known-characters list. Reuse the EXACT known names.\n"
|
||
"4. Only output 'Unknown' if the speaker is genuinely indeterminable even after applying turn-taking and "
|
||
"context — this should be rare. Prefer the most likely named character over 'Unknown'.\n"
|
||
"RULES:\n"
|
||
"- Put dialogue tags and action beats in a NARRATION segment, never inside the dialogue text.\n"
|
||
"- GRAMMAR CHECK FOR NARRATION: if a fragment is a reporting clause about speech "
|
||
"(finite speech verb + speaker/pronoun/name, e.g. 'murmelte er mit erstickter Stimme', "
|
||
"', entgegnete Marcian kalt', 'asked Peter quietly'), it is narration, never dialogue.\n"
|
||
"- Action beats are narration: a character looks, walks, laughs, stays silent, raises a hand, "
|
||
"turns away, etc. Only the actual quoted words are dialogue.\n"
|
||
"- If a quote is interrupted by a tag (»Die Pause«, sagte Peter, »ist vorbei.«), stitch the spoken parts "
|
||
"into ONE dialogue segment ('Die Pause ist vorbei.') with the tag as a separate narration segment.\n"
|
||
"- Strip the quotation marks/guillemets from dialogue text. Keep every word otherwise, in order.\n"
|
||
"- DO NOT hallucinate, summarize, or alter the text. The combined text of your segments MUST exactly match the original passage, word for word, except for dropped quotation marks."
|
||
)
|
||
else:
|
||
if lang_hint:
|
||
base_prompt += f"\n\n{lang_hint}"
|
||
|
||
# Only the streaming ("watch it think") call asks for a reasoning preamble —
|
||
# it costs extra tokens/latency, which the blocking/fallback path can't
|
||
# afford when it's already the retry-in-halves path for a slow model.
|
||
if data.get("want_reasoning"):
|
||
system = (
|
||
f"{base_prompt}\n\n"
|
||
"First, in a <think>...</think> block, briefly reason (2-4 short sentences) about who speaks each "
|
||
"untagged or ambiguous quote in this passage — mention the names/pronouns you're resolving. Keep it short.\n"
|
||
"Then, after </think>, respond with a single, valid JSON object containing a 'segments' array. "
|
||
"Nothing else outside the <think> block and the JSON object:\n"
|
||
'{"segments":[{"speaker":"Narrator","type":"narration","text":"...","emotion":""}]}'
|
||
)
|
||
else:
|
||
system = (
|
||
f"{base_prompt}\n\n"
|
||
"You MUST respond with a single, valid JSON object containing a 'segments' array.\n"
|
||
"Do NOT output any reasoning, chain of thought, or conversational text. Output ONLY the raw JSON object:\n"
|
||
'{"segments":[{"speaker":"Narrator","type":"narration","text":"...","emotion":""}]}'
|
||
)
|
||
user = (
|
||
("Known characters so far: " + ", ".join(str(n) for n in known) + "\n\n" if known else "")
|
||
+ ("Recent dialogue (the immediately preceding lines — continue the same conversation/turn-taking):\n" + recent + "\n\n" if recent else "")
|
||
+ "Passage to attribute:\n" + text + "\n\n"
|
||
+ "Return ONLY valid JSON. Do not include markdown blocks or any other text."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.2,
|
||
# Dialogue-dense passages need much more output than input: every quoted
|
||
# line becomes a full JSON object ({"speaker":...,"type":...,"text":...,
|
||
# "emotion":...}), so budgeting off raw character count alone (as a
|
||
# tighter cap here once did) truncates mid-JSON on exchange-heavy scenes
|
||
# — which the parser can't always repair, degrading the whole chunk to
|
||
# naive quote-splitting with every speaker labelled "Unknown".
|
||
"max_tokens": min(8192, max(2048, len(text) + 1000)) + (250 if data.get("want_reasoning") else 0),
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
return {
|
||
"payload": payload, "llm_url": llm_url, "model": model,
|
||
"api_key": _settings.get("llm_api_key") or "sk-dummy-key",
|
||
"timeout_seconds": timeout_seconds, "text": text,
|
||
"fallback_model": (_settings.get("llm_model") or "").strip(),
|
||
}
|
||
|
||
|
||
@router.post("/api/attribute-dialogue")
|
||
async def attribute_dialogue(request: Request):
|
||
"""Split a prose passage into attributed segments for a multi-voice audiobook.
|
||
|
||
Body: {text, known_characters:[...], language, llm_url, model}
|
||
Returns: {segments:[{speaker, type:"narration"|"dialogue", text, emotion}], characters:[names]}
|
||
The frontend calls this per chunk, passing the running character roster so the
|
||
same speaker keeps the same name across the whole book.
|
||
"""
|
||
data = await request.json()
|
||
prep = _attribution_prepare(data)
|
||
payload, llm_url, model = prep["payload"], prep["llm_url"], prep["model"]
|
||
timeout_seconds, text = prep["timeout_seconds"], prep["text"]
|
||
api_key = prep["api_key"]
|
||
try:
|
||
if not await asyncio.to_thread(_attribution_llm_lock.acquire, True, timeout_seconds):
|
||
return _fallback_attribute_response(text)
|
||
try:
|
||
resp = await asyncio.to_thread(
|
||
_post_llm_chat_completion,
|
||
llm_url, payload,
|
||
{"Authorization": f"Bearer {api_key}"}, timeout_seconds,
|
||
)
|
||
fallback_model = prep["fallback_model"]
|
||
if (
|
||
resp.status_code >= 400
|
||
and _is_router_model_alias(model)
|
||
and fallback_model
|
||
and fallback_model != model
|
||
and not _is_router_model_alias(fallback_model)
|
||
):
|
||
retry_payload = dict(payload)
|
||
retry_payload["model"] = fallback_model
|
||
resp = await asyncio.to_thread(
|
||
_post_llm_chat_completion,
|
||
llm_url, retry_payload,
|
||
{"Authorization": f"Bearer {api_key}"}, timeout_seconds,
|
||
)
|
||
payload = retry_payload
|
||
finally:
|
||
_attribution_llm_lock.release()
|
||
if resp.status_code >= 400:
|
||
print(
|
||
f"[attribute-dialogue] LLM failed ({resp.status_code}) for model "
|
||
f"{payload.get('model') or '(default)'}: {_response_error_text(resp)}"
|
||
)
|
||
return _fallback_attribute_response(text)
|
||
msg = resp.json()["choices"][0]["message"]
|
||
raw = (msg.get("content") or msg.get("reasoning_content") or "").strip()
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[attribute-dialogue] LLM failed, using deterministic fallback: {e}")
|
||
return _fallback_attribute_response(text)
|
||
|
||
return _attribution_parse(raw, text)
|
||
|
||
|
||
def _attribution_parse(raw: str, text: str) -> dict:
|
||
"""Turn the LLM's raw answer into normalised {segments, characters}."""
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
segments = []
|
||
|
||
block = _extract_json_block(content)
|
||
candidates = [content, block]
|
||
if block:
|
||
# If the JSON was truncated by token limits, appending ]} to the last complete object often salvages it.
|
||
candidates.append(block + "]}")
|
||
|
||
for cand in candidates:
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("segments"), list):
|
||
segments = parsed["segments"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
|
||
if not segments:
|
||
# If the LLM completely failed to output JSON despite the strict system prompt,
|
||
# it means it dropped into a conversational/reasoning hallucination.
|
||
# Do NOT try to parse its reasoning as a script. Abort and preserve the book text.
|
||
segments = [{"speaker": "Narrator", "type": "narration", "text": text, "emotion": ""}]
|
||
|
||
# Normalise + collect speaker roster
|
||
clean, chars = [], []
|
||
for seg in segments:
|
||
if not isinstance(seg, dict):
|
||
continue
|
||
t = str(seg.get("text") or "").strip()
|
||
if not t:
|
||
continue
|
||
sp = str(seg.get("speaker") or "Narrator").strip() or "Narrator"
|
||
typ = "dialogue" if str(seg.get("type") or "").lower().startswith("dial") else "narration"
|
||
if typ == "narration":
|
||
sp = "Narrator"
|
||
emo = str(seg.get("emotion") or "").strip()
|
||
clean.append({"speaker": sp, "type": typ, "text": t, "emotion": emo})
|
||
if typ == "dialogue" and sp.lower() != "narrator" and sp not in chars:
|
||
chars.append(sp)
|
||
return {"segments": clean, "characters": chars}
|
||
|
||
|
||
@router.post("/api/attribute-dialogue/stream")
|
||
async def attribute_dialogue_stream(request: Request):
|
||
"""Same job as /api/attribute-dialogue, but streams the LLM's live output
|
||
(reasoning + the JSON being written) as SSE `{"t": "..."}` events so the UI
|
||
can show what the model is thinking, ending with `{"done": true, "result"}`.
|
||
On upstream failure it emits `{"error": "..."}` — the client then falls
|
||
back to the blocking endpoint."""
|
||
data = await request.json()
|
||
prep = _attribution_prepare(data)
|
||
payload = dict(prep["payload"])
|
||
payload["stream"] = True
|
||
headers = {"Authorization": f"Bearer {prep['api_key']}"}
|
||
|
||
def gen():
|
||
if not _attribution_llm_lock.acquire(timeout=prep["timeout_seconds"]):
|
||
yield 'data: {"error": "Attribution engine busy"}\n\n'
|
||
return
|
||
upstream = None
|
||
try:
|
||
urls = _llm_chat_completion_urls(prep["llm_url"])
|
||
last_err = None
|
||
for u in urls:
|
||
try:
|
||
upstream = requests.post(
|
||
u, json=payload, headers=headers,
|
||
timeout=(15, prep["timeout_seconds"]), stream=True,
|
||
)
|
||
if upstream.status_code in (404, 405) and u != urls[-1]:
|
||
upstream.close()
|
||
upstream = None
|
||
continue
|
||
break
|
||
except Exception as exc:
|
||
last_err = exc
|
||
upstream = None
|
||
if upstream is None:
|
||
print(f"[attribute-dialogue/stream] LLM connect failed: {last_err}")
|
||
yield f'data: {json.dumps({"error": str(last_err or "LLM connect failed")})}\n\n'
|
||
return
|
||
if upstream.status_code >= 400:
|
||
print(
|
||
f"[attribute-dialogue/stream] LLM failed ({upstream.status_code}) for model "
|
||
f"{payload.get('model') or '(default)'}: {_response_error_text(upstream)[:300]}"
|
||
)
|
||
yield f'data: {json.dumps({"error": f"HTTP {upstream.status_code}: {_response_error_text(upstream)[:300]}"})}\n\n'
|
||
return
|
||
raw_parts = []
|
||
# The LLM's SSE response usually omits a charset on its Content-Type,
|
||
# so requests falls back to Latin-1 (the old HTTP default) instead of
|
||
# UTF-8 when decode_unicode=True guesses the encoding — every umlaut
|
||
# then comes out mojibake'd ("Häfen" -> "Häfen"). Force UTF-8.
|
||
upstream.encoding = "utf-8"
|
||
for line in upstream.iter_lines(decode_unicode=True):
|
||
if not line or not line.startswith("data:"):
|
||
continue
|
||
chunk = line[5:].strip()
|
||
if chunk == "[DONE]":
|
||
break
|
||
try:
|
||
delta = json.loads(chunk)["choices"][0]["delta"]
|
||
except Exception:
|
||
continue
|
||
# Reasoning models emit thoughts via reasoning_content; the
|
||
# answer JSON arrives via content. Forward both to the viewer,
|
||
# but only content counts toward the parseable answer.
|
||
t = delta.get("reasoning_content") or delta.get("content") or ""
|
||
if delta.get("content"):
|
||
raw_parts.append(delta["content"])
|
||
if t:
|
||
yield f'data: {json.dumps({"t": t})}\n\n'
|
||
result = _attribution_parse("".join(raw_parts), prep["text"])
|
||
yield f'data: {json.dumps({"done": True, "result": result})}\n\n'
|
||
except GeneratorExit:
|
||
raise
|
||
except Exception as e:
|
||
yield f'data: {json.dumps({"error": str(e)})}\n\n'
|
||
finally:
|
||
try:
|
||
if upstream is not None:
|
||
upstream.close()
|
||
except Exception:
|
||
pass
|
||
_attribution_llm_lock.release()
|
||
|
||
return StreamingResponse(gen(), media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||
|
||
|
||
# ── Audio effects ─────────────────────────────────────────────────────────────
|
||
|
||
def _apply_audio_effects(audio_bytes: bytes, effects: list) -> bytes:
|
||
try:
|
||
from pedalboard import Pedalboard, Reverb, Chorus, Delay, Compressor, Gain, HighpassFilter, LowpassFilter, PitchShift # type: ignore
|
||
import numpy as np # type: ignore
|
||
except ImportError:
|
||
raise RuntimeError("pedalboard is not installed — run: pip install pedalboard numpy")
|
||
|
||
with io.BytesIO(audio_bytes) as buf:
|
||
with wave.open(buf, "rb") as wf:
|
||
n_channels = wf.getnchannels()
|
||
sample_rate = wf.getframerate()
|
||
n_frames = wf.getnframes()
|
||
raw = wf.readframes(n_frames)
|
||
sampwidth = wf.getsampwidth()
|
||
|
||
import numpy as np # noqa: F811
|
||
dtype = {1: np.int8, 2: np.int16, 4: np.int32}.get(sampwidth, np.int16)
|
||
samples = np.frombuffer(raw, dtype=dtype).astype(np.float32) / float(np.iinfo(dtype).max)
|
||
samples = samples.reshape(1, -1) if n_channels == 1 else samples.reshape(-1, n_channels).T
|
||
|
||
board = []
|
||
for fx in effects:
|
||
t = fx.get("type", "")
|
||
p = fx.get("params", {})
|
||
if t == "reverb":
|
||
board.append(Reverb(
|
||
room_size=float(p.get("room_size", 0.35)),
|
||
damping=float(p.get("damping", 0.5)),
|
||
wet_level=float(p.get("wet", 0.25)),
|
||
dry_level=float(p.get("dry", 0.8)),
|
||
))
|
||
elif t == "chorus":
|
||
board.append(Chorus(
|
||
rate_hz=float(p.get("rate_hz", 1.0)),
|
||
depth=float(p.get("depth", 0.25)),
|
||
mix=float(p.get("mix", 0.5)),
|
||
))
|
||
elif t == "delay":
|
||
board.append(Delay(
|
||
delay_seconds=float(p.get("delay_s", 0.25)),
|
||
feedback=float(p.get("feedback", 0.3)),
|
||
mix=float(p.get("mix", 0.4)),
|
||
))
|
||
elif t == "compressor":
|
||
board.append(Compressor(
|
||
threshold_db=float(p.get("threshold_db", -20.0)),
|
||
ratio=float(p.get("ratio", 4.0)),
|
||
attack_ms=float(p.get("attack_ms", 10.0)),
|
||
release_ms=float(p.get("release_ms", 100.0)),
|
||
))
|
||
elif t == "gain":
|
||
board.append(Gain(gain_db=float(p.get("gain_db", 0.0))))
|
||
elif t == "highpass":
|
||
board.append(HighpassFilter(cutoff_frequency_hz=float(p.get("cutoff_hz", 80.0))))
|
||
elif t == "lowpass":
|
||
board.append(LowpassFilter(cutoff_frequency_hz=float(p.get("cutoff_hz", 8000.0))))
|
||
elif t == "pitch_shift":
|
||
board.append(PitchShift(semitones=float(p.get("semitones", 0.0))))
|
||
|
||
if board:
|
||
samples = Pedalboard(board)(samples, sample_rate)
|
||
|
||
out = np.clip(samples, -1.0, 1.0)
|
||
pcm = ((out[0] if out.shape[0] == 1 else out.T.reshape(-1)) * 32767).astype(np.int16).tobytes()
|
||
buf_out = io.BytesIO()
|
||
with wave.open(buf_out, "wb") as wf:
|
||
wf.setnchannels(n_channels)
|
||
wf.setsampwidth(2)
|
||
wf.setframerate(sample_rate)
|
||
wf.writeframes(pcm)
|
||
return buf_out.getvalue()
|
||
|
||
|
||
@router.post("/api/audio/effects")
|
||
async def audio_effects(request: Request):
|
||
"""Apply an effects chain to a WAV file."""
|
||
form = await request.form()
|
||
audio_file = form.get("audio")
|
||
effects_json = str(form.get("effects") or "[]")
|
||
|
||
if audio_file is None:
|
||
raise HTTPException(400, "No audio file provided")
|
||
audio_bytes = await audio_file.read()
|
||
try:
|
||
effects = json.loads(effects_json)
|
||
except Exception:
|
||
raise HTTPException(400, "Invalid effects JSON")
|
||
|
||
try:
|
||
result = await asyncio.to_thread(_apply_audio_effects, audio_bytes, effects)
|
||
return Response(content=result, media_type="audio/wav")
|
||
except RuntimeError as e:
|
||
raise HTTPException(501, str(e))
|
||
except Exception as e:
|
||
raise HTTPException(500, f"Effects processing failed: {e}")
|
||
|
||
|
||
# ── Voices export / import ────────────────────────────────────────────────────
|
||
|
||
_EXPORT_SKIP_KEYS = {
|
||
"groq_api_key", "whisper_api_key", "tts_api_key", "voice_design_api_key",
|
||
"elevenlabs_api_key", "llm_api_key", "engine_api_keys",
|
||
}
|
||
_IMPORT_ALLOWED_SUFFIXES = set(_AUDIO_EXTS + [".reference.txt", ".meta.json", ".jpg", ".jpeg", ".png", ".webp"])
|
||
|
||
|
||
@router.get("/api/voices/export")
|
||
async def voices_export():
|
||
"""Export all voices + non-sensitive settings as a ZIP archive."""
|
||
import zipfile
|
||
from datetime import datetime
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
||
safe_settings = {k: v for k, v in settings.items() if k not in _EXPORT_SKIP_KEYS}
|
||
zf.writestr("settings.json", json.dumps(safe_settings, indent=2))
|
||
if scan_dir.exists():
|
||
for f in scan_dir.rglob("*"):
|
||
if f.is_file():
|
||
try:
|
||
zf.write(f, str(f.relative_to(scan_dir.parent)))
|
||
except Exception:
|
||
pass
|
||
buf.seek(0)
|
||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
return Response(
|
||
content=buf.read(),
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f'attachment; filename="voices_export_{ts}.zip"'},
|
||
)
|
||
|
||
|
||
@router.post("/api/voices/import")
|
||
async def voices_import(file: UploadFile = File(...)):
|
||
"""Import voices from a ZIP archive (skips settings.json and unsafe paths)."""
|
||
import zipfile
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
scan_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
content = await file.read()
|
||
if len(content) > _MAX_UPLOAD_BYTES:
|
||
raise HTTPException(413, "ZIP file too large")
|
||
|
||
try:
|
||
imported = 0
|
||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||
for info in zf.infolist():
|
||
if info.is_dir():
|
||
continue
|
||
parts = Path(info.filename).parts
|
||
if any(p in ("..", "") for p in parts) or Path(info.filename).name == "settings.json":
|
||
continue
|
||
if Path(info.filename).suffix.lower() not in _IMPORT_ALLOWED_SUFFIXES:
|
||
continue
|
||
rel = parts[1:] if len(parts) > 1 else parts
|
||
dest = scan_dir / Path(*rel)
|
||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||
dest.write_bytes(zf.read(info))
|
||
imported += 1
|
||
except zipfile.BadZipFile:
|
||
raise HTTPException(400, "Not a valid ZIP file")
|
||
except Exception as e:
|
||
raise HTTPException(500, f"Import failed: {e}")
|
||
|
||
await asyncio.to_thread(rebuild_voice_index, settings)
|
||
return {"ok": True, "imported": imported}
|
||
|
||
|
||
# ── /speak REST endpoint ──────────────────────────────────────────────────────
|
||
|
||
@router.post("/speak")
|
||
async def speak(request: Request):
|
||
data = await request.json()
|
||
text: str = str(data.get("text") or "").strip()
|
||
if not text:
|
||
raise HTTPException(400, "text is required")
|
||
|
||
explicit_voice: str = str(data.get("voice") or data.get("profile_id") or data.get("profile") or "").strip()
|
||
backend: str = _clean_preview_backend(str(data.get("backend") or "voice_clone"))
|
||
apply_persona: bool = bool(data.get("apply_persona") or data.get("personality"))
|
||
client_id: str = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
||
|
||
settings = _load_settings()
|
||
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
||
if not voice:
|
||
raise HTTPException(400, "No voice specified and no default voice configured")
|
||
|
||
if apply_persona:
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
wav = _find_voice_audio(voice, scan_dir)
|
||
if wav:
|
||
persona = _load_meta(wav).get("persona", "")
|
||
if persona:
|
||
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
llm_model = settings.get("llm_model") or ""
|
||
try:
|
||
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url, llm_model)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Persona rewrite failed: {e}")
|
||
|
||
instruct = str(data.get("instruct") or "")
|
||
try:
|
||
audio, media_type = await asyncio.to_thread(
|
||
_preview_request_audio, text, voice, settings, instruct, backend
|
||
)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"TTS error: {e}")
|
||
|
||
return Response(content=audio, media_type=media_type)
|
||
|
||
|
||
@router.get("/speak/bindings")
|
||
async def get_speak_bindings():
|
||
s = _load_settings()
|
||
return {"bindings": s.get("client_voice_bindings") or {}}
|
||
|
||
|
||
@router.put("/speak/bindings/{client_id}")
|
||
async def put_speak_binding(client_id: str, request: Request):
|
||
data = await request.json()
|
||
voice = str(data.get("voice") or "").strip()
|
||
if not voice:
|
||
raise HTTPException(400, "voice is required")
|
||
s = _load_settings()
|
||
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
||
bindings[client_id] = voice
|
||
s["client_voice_bindings"] = bindings
|
||
_save_settings(s)
|
||
return {"ok": True, "client_id": client_id, "voice": voice}
|
||
|
||
|
||
@router.delete("/speak/bindings/{client_id}")
|
||
async def delete_speak_binding(client_id: str):
|
||
s = _load_settings()
|
||
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
||
if client_id not in bindings:
|
||
raise HTTPException(404, f"No binding for client '{client_id}'")
|
||
del bindings[client_id]
|
||
s["client_voice_bindings"] = bindings
|
||
_save_settings(s)
|
||
return {"ok": True, "client_id": client_id}
|
||
|
||
|
||
# ── MCP JSON-RPC 2.0 server ───────────────────────────────────────────────────
|
||
|
||
_MCP_SERVER_INFO = {"name": "tts-voice-creator", "version": "1.0.0"}
|
||
|
||
_MCP_TOOLS = [
|
||
{
|
||
"name": "speak",
|
||
"description": "Generate speech audio from text using a cloned voice. Returns a data URI with the WAV audio.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"text": {"type": "string", "description": "Text to synthesize"},
|
||
"voice": {"type": "string", "description": "Voice ID (optional — uses per-client binding or default if omitted)"},
|
||
"apply_persona": {"type": "boolean", "description": "Rewrite text through the voice's persona before synthesis"},
|
||
},
|
||
"required": ["text"],
|
||
},
|
||
},
|
||
{
|
||
"name": "transcribe",
|
||
"description": "Transcribe base64-encoded WAV audio to text.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"audio_base64": {"type": "string", "description": "Base64-encoded audio bytes (WAV preferred)"},
|
||
},
|
||
"required": ["audio_base64"],
|
||
},
|
||
},
|
||
{
|
||
"name": "list_captures",
|
||
"description": "List the 20 most recently generated audio files.",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
{
|
||
"name": "list_profiles",
|
||
"description": "List all available voice profiles with their language, persona, and enabled state.",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
]
|
||
|
||
|
||
async def _mcp_tool_speak(args: dict, client_id: str) -> dict:
|
||
text = str(args.get("text") or "").strip()
|
||
if not text:
|
||
raise ValueError("text is required")
|
||
explicit_voice = str(args.get("voice") or "").strip()
|
||
apply_persona = bool(args.get("apply_persona", False))
|
||
settings = _load_settings()
|
||
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
||
if not voice:
|
||
raise ValueError("No voice specified and no default voice configured")
|
||
if apply_persona:
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
wav = _find_voice_audio(voice, scan_dir)
|
||
if wav:
|
||
persona = _load_meta(wav).get("persona", "")
|
||
if persona:
|
||
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
llm_model = settings.get("llm_model") or ""
|
||
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url, llm_model)
|
||
audio, _media_type = await asyncio.to_thread(
|
||
_preview_request_audio, text, voice, settings, "", "voice_clone"
|
||
)
|
||
audio_b64 = base64.b64encode(audio).decode()
|
||
return {
|
||
"content": [
|
||
{"type": "text", "text": f"Generated {len(audio)} bytes of speech audio for voice '{voice}'."},
|
||
{"type": "resource", "resource": {"uri": f"data:audio/wav;base64,{audio_b64}", "mimeType": "audio/wav"}},
|
||
]
|
||
}
|
||
|
||
|
||
async def _mcp_tool_transcribe(args: dict) -> dict:
|
||
raw = args.get("audio_base64") or ""
|
||
try:
|
||
audio_bytes = base64.b64decode(raw)
|
||
except Exception:
|
||
raise ValueError("audio_base64 is not valid base64")
|
||
tmp = TEMP_DIR / f"{uuid.uuid4().hex}_mcp_transcribe.wav"
|
||
tmp.write_bytes(audio_bytes)
|
||
settings = _load_settings()
|
||
try:
|
||
text, used_backend = await asyncio.to_thread(_transcribe_audio, tmp, settings, "configured")
|
||
finally:
|
||
try:
|
||
tmp.unlink(missing_ok=True)
|
||
except Exception:
|
||
pass
|
||
return {"content": [{"type": "text", "text": text}], "backend": used_backend}
|
||
|
||
|
||
async def _mcp_tool_list_captures() -> dict:
|
||
settings = _load_settings()
|
||
out_dir = _active_voices_dir(settings)
|
||
if not out_dir.exists():
|
||
return {"content": [{"type": "text", "text": "[]"}]}
|
||
files = sorted(
|
||
(p for p in out_dir.rglob("*.wav") if not _is_internal_voice_file(p)),
|
||
key=lambda p: p.stat().st_mtime,
|
||
reverse=True,
|
||
)[:20]
|
||
result = [{"name": p.stem, "path": str(p), "mtime": p.stat().st_mtime} for p in files]
|
||
return {"content": [{"type": "text", "text": json.dumps(result)}]}
|
||
|
||
|
||
async def _mcp_tool_list_profiles() -> dict:
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
profiles = []
|
||
seen: set[str] = set()
|
||
for audio in sorted(_voice_audio_files(scan_dir), key=lambda p: p.stem.lower()):
|
||
if audio.stem in seen:
|
||
continue
|
||
seen.add(audio.stem)
|
||
meta = _load_meta(audio)
|
||
parts = audio.stem.split("_", 2)
|
||
lang = parts[0].upper() if parts else ""
|
||
profiles.append({
|
||
"id": audio.stem,
|
||
"name": audio.stem,
|
||
"lang": lang,
|
||
"persona": meta.get("persona", ""),
|
||
"enabled": meta.get("enabled", True),
|
||
})
|
||
return {"content": [{"type": "text", "text": json.dumps(profiles)}]}
|
||
|
||
|
||
def _mcp_error_response(code: int, message: str, rpc_id) -> Response:
|
||
import logging
|
||
body = {"jsonrpc": "2.0", "error": {"code": code, "message": message}, "id": rpc_id}
|
||
return Response(content=json.dumps(body), media_type="application/json")
|
||
|
||
|
||
@router.post("/mcp")
|
||
async def mcp_jsonrpc(request: Request):
|
||
import logging
|
||
logger = logging.getLogger("uvicorn.error")
|
||
try:
|
||
body = await request.json()
|
||
except Exception:
|
||
return _mcp_error_response(-32700, "Parse error", None)
|
||
|
||
rpc_id = body.get("id")
|
||
method = body.get("method", "")
|
||
params = body.get("params") or {}
|
||
client_id = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
||
|
||
try:
|
||
if method == "initialize":
|
||
result = {
|
||
"protocolVersion": "2024-11-05",
|
||
"serverInfo": _MCP_SERVER_INFO,
|
||
"capabilities": {"tools": {}},
|
||
}
|
||
elif method == "notifications/initialized":
|
||
return Response(status_code=204)
|
||
elif method == "tools/list":
|
||
result = {"tools": _MCP_TOOLS}
|
||
elif method == "tools/call":
|
||
tool_name = str(params.get("name") or "")
|
||
tool_args = params.get("arguments") or {}
|
||
if tool_name == "speak":
|
||
result = await _mcp_tool_speak(tool_args, client_id)
|
||
elif tool_name == "transcribe":
|
||
result = await _mcp_tool_transcribe(tool_args)
|
||
elif tool_name == "list_captures":
|
||
result = await _mcp_tool_list_captures()
|
||
elif tool_name == "list_profiles":
|
||
result = await _mcp_tool_list_profiles()
|
||
else:
|
||
return _mcp_error_response(-32601, f"Unknown tool: {tool_name}", rpc_id)
|
||
else:
|
||
return _mcp_error_response(-32601, f"Method not found: {method}", rpc_id)
|
||
except ValueError as e:
|
||
return _mcp_error_response(-32602, str(e), rpc_id)
|
||
except Exception as e:
|
||
logger.exception("MCP tool error in method %s", method)
|
||
return _mcp_error_response(-32000, f"Server error: {e}", rpc_id)
|
||
|
||
return Response(
|
||
content=json.dumps({"jsonrpc": "2.0", "result": result, "id": rpc_id}),
|
||
media_type="application/json",
|
||
)
|
||
|
||
|
||
@router.get("/mcp")
|
||
async def mcp_sse(request: Request):
|
||
"""SSE keep-alive stream (satisfies MCP spec GET /mcp requirement)."""
|
||
from typing import AsyncGenerator
|
||
|
||
async def _keepalive() -> AsyncGenerator[str, None]:
|
||
yield ": mcp-sse-ready\n\n"
|
||
while True:
|
||
await asyncio.sleep(15)
|
||
yield ": ping\n\n"
|
||
return StreamingResponse(
|
||
_keepalive(),
|
||
media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||
)
|
||
|
||
|
||
# ── Conversation playground ───────────────────────────────────────────────────
|
||
|
||
@router.get("/api/conversation/llm-models")
|
||
async def conversation_llm_models(url: str = "", api_key: str = ""):
|
||
"""List models from a local LLM endpoint (Ollama / vLLM / LM Studio)."""
|
||
from core.validation import _validate_http_url
|
||
settings = _load_settings()
|
||
base = (url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
key_to_use = api_key or settings.get("llm_api_key") or "sk-dummy-key"
|
||
try:
|
||
base = _validate_http_url(base, allow_private=True)
|
||
r = requests.get(f"{base}/models", timeout=5, headers={"Authorization": f"Bearer {key_to_use}"})
|
||
|
||
# Fallback for gateways like LiteLLM that might prefer /models over /v1/models
|
||
if r.status_code != 200 and base.endswith("/v1"):
|
||
fallback_base = base[:-3]
|
||
r2 = requests.get(f"{fallback_base}/models", timeout=5, headers={"Authorization": f"Bearer {key_to_use}"})
|
||
if r2.status_code == 200:
|
||
r = r2
|
||
|
||
if r.status_code == 200:
|
||
payload = r.json()
|
||
data = payload.get("data", []) if isinstance(payload, dict) else []
|
||
models = [
|
||
str(item["id"]) if isinstance(item, dict) and item.get("id") else str(item)
|
||
for item in data if item
|
||
]
|
||
return {"models": models, "url": base}
|
||
else:
|
||
return {"models": [], "url": base, "error": r.text}
|
||
except Exception as e:
|
||
return {"models": [], "url": base, "error": str(e)}
|
||
|
||
|
||
def _make_tts_task(
|
||
text: str,
|
||
voice: str,
|
||
settings: dict,
|
||
backend: str,
|
||
sem: "asyncio.Semaphore | None",
|
||
) -> "asyncio.Task":
|
||
if sem is None:
|
||
return asyncio.create_task(
|
||
asyncio.to_thread(_preview_request_audio, text, voice, settings, "", backend)
|
||
)
|
||
|
||
async def _guarded() -> tuple[bytes, str]:
|
||
async with sem:
|
||
return await asyncio.to_thread(_preview_request_audio, text, voice, settings, "", backend)
|
||
|
||
return asyncio.create_task(_guarded())
|
||
|
||
|
||
@router.post("/api/conversation/turn")
|
||
async def conversation_turn(
|
||
audio: Optional[UploadFile] = None,
|
||
text: str = Form(""),
|
||
stt_backend: str = Form("configured"),
|
||
llm_url: str = Form(""),
|
||
llm_model: str = Form(""),
|
||
tts_backend: str = Form("voice_clone"),
|
||
tts_voice: str = Form(""),
|
||
system_prompt: str = Form("You are a helpful voice assistant. Keep replies short and conversational."),
|
||
history: str = Form("[]"),
|
||
):
|
||
"""Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events.
|
||
Pass either an audio file (runs STT first) or a plain text string (skips STT).
|
||
"""
|
||
direct_text = text.strip()
|
||
if not direct_text and (audio is None or not getattr(audio, "filename", None)):
|
||
raise HTTPException(400, "Provide either an audio file or a text field")
|
||
|
||
settings = _load_settings()
|
||
eff_llm_url = (llm_url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
|
||
# Prepare audio temp files only when an audio upload was provided
|
||
_tmp: Path | None = None
|
||
_wav: Path | None = None
|
||
if not direct_text and audio is not None:
|
||
suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm"
|
||
if suffix not in _UPLOAD_EXTS:
|
||
suffix = ".webm"
|
||
_tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}"
|
||
try:
|
||
with _tmp.open("wb") as f:
|
||
_copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
|
||
_wav = _to_wav_16k(_tmp)
|
||
except Exception as e:
|
||
_tmp.unlink(missing_ok=True)
|
||
raise HTTPException(400, f"Audio upload failed: {e}")
|
||
|
||
try:
|
||
hist = json.loads(history) if history else []
|
||
if not isinstance(hist, list):
|
||
hist = []
|
||
except Exception:
|
||
hist = []
|
||
|
||
stt_be = _clean_stt_backend(stt_backend)
|
||
tts_be = _clean_preview_backend(tts_backend)
|
||
|
||
async def generate():
|
||
t0 = time.monotonic()
|
||
stt_ms = llm_ttft_ms = llm_total_ms = tts_ms = None
|
||
transcript = llm_text = ""
|
||
|
||
def sse(obj: dict) -> str:
|
||
return f"data: {json.dumps(obj)}\n\n"
|
||
|
||
# 1. STT — skipped when caller sends direct text
|
||
if direct_text:
|
||
transcript = direct_text
|
||
yield sse({"type": "transcript", "text": transcript, "stt_ms": None})
|
||
else:
|
||
try:
|
||
t_stt = time.monotonic()
|
||
transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be)
|
||
stt_ms = int((time.monotonic() - t_stt) * 1000)
|
||
yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms})
|
||
except Exception as e:
|
||
yield sse({"type": "error", "stage": "stt", "message": str(e)})
|
||
return
|
||
finally:
|
||
for p in filter(None, {_tmp, _wav}):
|
||
with contextlib.suppress(Exception):
|
||
p.unlink(missing_ok=True)
|
||
|
||
if not transcript.strip() or _is_hallucination(transcript):
|
||
yield sse({"type": "error", "stage": "stt", "message": "No speech detected."})
|
||
return
|
||
|
||
# 2. LLM stream — runs in a thread; tokens arrive via asyncio.Queue
|
||
# so the event loop is never blocked and TTS can start on sentence 1
|
||
# while the LLM is still generating sentences 2, 3, …
|
||
messages = [{"role": "system", "content": system_prompt}]
|
||
messages.extend(hist[-20:])
|
||
messages.append({"role": "user", "content": transcript})
|
||
llm_payload: dict = {"messages": messages, "stream": True, "temperature": 0.7, "max_tokens": 512}
|
||
eff_llm_model = llm_model or settings.get("llm_model") or ""
|
||
if eff_llm_model:
|
||
llm_payload["model"] = eff_llm_model
|
||
|
||
_loop = asyncio.get_event_loop()
|
||
_token_q: asyncio.Queue[str | None] = asyncio.Queue()
|
||
|
||
def _llm_thread() -> None:
|
||
try:
|
||
resp = requests.post(
|
||
f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}, stream=True, timeout=120,
|
||
)
|
||
resp.raise_for_status()
|
||
for raw_line in resp.iter_lines():
|
||
if not raw_line:
|
||
continue
|
||
line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else str(raw_line)
|
||
if not line.startswith("data:"):
|
||
continue
|
||
payload_str = line[5:].strip()
|
||
if payload_str == "[DONE]":
|
||
break
|
||
try:
|
||
obj = json.loads(payload_str)
|
||
d_obj = ((obj.get("choices") or [{}])[0].get("delta") or {})
|
||
delta = d_obj.get("content") or d_obj.get("reasoning_content") or ""
|
||
if not delta and isinstance(obj.get("message"), dict):
|
||
delta = obj["message"].get("content") or ""
|
||
if delta:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, delta)
|
||
except Exception:
|
||
continue
|
||
except Exception as exc:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, f"\x00ERR:{exc}")
|
||
finally:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, None)
|
||
|
||
threading.Thread(target=_llm_thread, daemon=True).start()
|
||
|
||
# Streaming backend (8023) has per-process voice state — serialize to prevent
|
||
# concurrent requests from clobbering each other's voice context.
|
||
tts_sem: asyncio.Semaphore | None = asyncio.Semaphore(1) if tts_be == "streaming" else None
|
||
|
||
t_llm = time.monotonic()
|
||
llm_ttft_ms: int | None = None
|
||
ttft_done = False
|
||
sent_buf = ""
|
||
tts_tasks: list[asyncio.Task] = []
|
||
tts_texts: list[str] = [] # sentence text corresponding to each task
|
||
tts_first_start: float | None = None
|
||
|
||
try:
|
||
while True:
|
||
delta = await _token_q.get()
|
||
if delta is None:
|
||
break
|
||
if delta.startswith("\x00ERR:"):
|
||
yield sse({"type": "error", "stage": "llm", "message": delta[5:]})
|
||
return
|
||
if not ttft_done:
|
||
llm_ttft_ms = int((time.monotonic() - t_llm) * 1000)
|
||
ttft_done = True
|
||
llm_text += delta
|
||
sent_buf += delta
|
||
yield sse({"type": "token", "delta": delta})
|
||
# Fire TTS on sentence boundary — runs concurrently with LLM
|
||
split = _sentence_split(sent_buf)
|
||
if split > 0:
|
||
chunk_text = sent_buf[:split].strip()
|
||
sent_buf = sent_buf[split:]
|
||
if chunk_text:
|
||
if tts_first_start is None:
|
||
tts_first_start = time.monotonic()
|
||
tts_texts.append(chunk_text)
|
||
tts_tasks.append(_make_tts_task(chunk_text, tts_voice, settings, tts_be, tts_sem))
|
||
except Exception as exc:
|
||
yield sse({"type": "error", "stage": "llm", "message": str(exc)})
|
||
return
|
||
|
||
# Flush any remaining text as a final TTS task
|
||
if sent_buf.strip():
|
||
if tts_first_start is None:
|
||
tts_first_start = time.monotonic()
|
||
tts_texts.append(sent_buf.strip())
|
||
tts_tasks.append(_make_tts_task(sent_buf.strip(), tts_voice, settings, tts_be, tts_sem))
|
||
|
||
llm_total_ms = int((time.monotonic() - t_llm) * 1000)
|
||
yield sse({"type": "llm_done", "text": llm_text,
|
||
"llm_ttft_ms": llm_ttft_ms, "llm_total_ms": llm_total_ms})
|
||
|
||
if not llm_text.strip():
|
||
yield sse({"type": "error", "stage": "llm",
|
||
"message": "LLM returned empty response. "
|
||
"If using a Qwen3 model, try adding /no-think to your system prompt "
|
||
"or pick a non-thinking model in the Language Model dropdown."})
|
||
return
|
||
|
||
# 3. Stream audio chunks in order — each chunk's TTS ran concurrently
|
||
# with LLM generation, so first audio arrives much sooner than
|
||
# waiting for the full response.
|
||
tts_start = tts_first_start or time.monotonic()
|
||
for i, task in enumerate(tts_tasks):
|
||
try:
|
||
audio_bytes, mime = await task
|
||
except Exception as exc:
|
||
yield sse({"type": "error", "stage": "tts", "message": str(exc)})
|
||
return
|
||
if i == 0:
|
||
tts_ms = int((time.monotonic() - tts_start) * 1000)
|
||
sentence_text = tts_texts[i] if i < len(tts_texts) else ""
|
||
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(),
|
||
"mime": mime, "text": sentence_text})
|
||
|
||
total_ms = int((time.monotonic() - t0) * 1000)
|
||
yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
|
||
"llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms})
|
||
yield sse({"type": "done"})
|
||
|
||
return StreamingResponse(generate(), media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|