mARTin-B78
5fecbf06d4
Fish-Speech emotion tags were silently ignored on non-English books: per-line emotions are LLM-generated in the book's own language, but Fish-Speech only recognizes English [tag] markers, and a double-tagging bug was stacking a broken server-derived tag on top of the client's own. Added a DE->EN translation table and removed the double-tagging. Also wires the existing book-profile context and race_species field into character portrait prompts (previously only used for voice design), adds a recast-until-threshold loop for casting, and adds backend-aware emotion quick-picks to Read Aloud, Try a Voice, and Conversation. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
3084 lines
156 KiB
Python
3084 lines
156 KiB
Python
"""Conversation playground, LLM refinement, audio effects, export/import, speak, MCP."""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import base64
|
||
import contextlib
|
||
import copy
|
||
import io
|
||
import json
|
||
import re
|
||
import threading
|
||
import time
|
||
import uuid
|
||
import wave
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
import requests
|
||
from typing import Optional
|
||
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
||
from fastapi.responses import Response, StreamingResponse
|
||
|
||
from core.config import _load_settings, _save_settings, _clean_preview_backend
|
||
from core.constants import _VOICES_DIR_DEFAULT, _MAX_UPLOAD_BYTES
|
||
from core.registry import _registry_get, TEMP_DIR
|
||
from core.validation import _copy_limited
|
||
from core.audio import _to_wav_16k
|
||
from core.voice import (
|
||
_AUDIO_EXTS, _UPLOAD_EXTS, _PICTURE_EXTS,
|
||
_find_voice_audio, _load_meta, _active_voices_dir,
|
||
_voice_audio_files, _is_internal_voice_file,
|
||
)
|
||
from core.voice_index import rebuild_voice_index
|
||
from core.tts_helpers import _preview_request_audio
|
||
from routes.stt import _transcribe_audio, _clean_stt_backend
|
||
|
||
router = APIRouter()
|
||
|
||
# Serializes outbound calls to the audiobook-attribution LLM. A client-side
|
||
# timeout aborts the browser fetch but can't kill the backend's blocking
|
||
# requests.post thread, so a slow/cold model leaves "ghost" requests still
|
||
# occupying the LLM's single processing slot. Without this lock, the next
|
||
# chunk (or the retry-in-halves) fires into that busy slot and the LLM
|
||
# answers with 429s that cascade until the ghosts drain.
|
||
# threading.Lock (not asyncio.Lock) because the streaming endpoint's SSE
|
||
# generator runs in a plain thread (StreamingResponse iterates a sync
|
||
# generator via a threadpool), so both the blocking and streaming endpoints
|
||
# must share ONE lock object to actually serialize on the LLM's single slot —
|
||
# two separate locks would let a stream call and its own blocking fallback
|
||
# fire into that slot concurrently, which is exactly the "ghost request"
|
||
# scenario this lock exists to prevent.
|
||
_attribution_llm_lock = threading.Lock()
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _watchdog_close(resp, timeout_seconds):
|
||
"""Force-close `resp`'s connection if the wrapped block hasn't finished
|
||
within timeout_seconds.
|
||
|
||
requests' own `timeout=` on a stream=True call only guards the connect
|
||
+ first byte — NOT the gaps between subsequent body reads. If the LLM
|
||
backend goes silent mid-stream (no more chunks, connection left open),
|
||
the blocking socket recv() inside iter_lines() can hang forever, and
|
||
since that's a native blocking call rather than a Python-level `yield`
|
||
point, neither a wall-clock check in the loop body nor GeneratorExit from
|
||
a disconnected client can interrupt it — both only take effect at the
|
||
next bytecode boundary, which never arrives. Observed in production: a
|
||
stuck stream held the shared attribution lock for 11+ minutes, silently
|
||
starving every other passage. Closing the connection from a separate
|
||
watchdog thread forces the blocked recv() to raise, unblocking the
|
||
generator so the normal except/finally cleanup (incl. releasing the
|
||
lock) actually runs.
|
||
"""
|
||
done = threading.Event()
|
||
|
||
def _kill():
|
||
if not done.wait(timeout_seconds):
|
||
try:
|
||
resp.close()
|
||
except Exception:
|
||
pass
|
||
|
||
t = threading.Thread(target=_kill, daemon=True)
|
||
t.start()
|
||
try:
|
||
yield
|
||
finally:
|
||
done.set()
|
||
|
||
# ── STT hallucination filter ──────────────────────────────────────────────────
|
||
# Whisper commonly hallucinates these phrases on silence/noise.
|
||
# Treat them as "no speech detected" rather than passing them to the LLM.
|
||
_HALLUCINATIONS: frozenset[str] = frozenset([
|
||
"reich", "danke", "danke schön", "danke schoen", "vielen dank",
|
||
"thank you", "thank you.", "thanks", "thanks.", "you", "you.",
|
||
"copyright", "abonnieren", "untertitel", "subscribe", "subscribing",
|
||
])
|
||
|
||
def _is_hallucination(text: str) -> bool:
|
||
t = text.strip().lower().rstrip(".!?,;:-").strip()
|
||
return len(t) <= 2 or t in _HALLUCINATIONS
|
||
|
||
# ── Sentence-boundary helpers for pipelined TTS ───────────────────────────────
|
||
|
||
_SENT_RE = re.compile(r'(?<=[.!?])\s+')
|
||
_MIN_SENTENCE = 30 # min chars in buffer before we split
|
||
|
||
def _sentence_split(buf: str) -> int:
|
||
"""Return the index after the first sentence boundary, or -1."""
|
||
if len(buf) < _MIN_SENTENCE:
|
||
return -1
|
||
for m in _SENT_RE.finditer(buf):
|
||
if m.end() >= _MIN_SENTENCE:
|
||
return m.end()
|
||
return -1
|
||
|
||
# ── LLM helpers ───────────────────────────────────────────────────────────────
|
||
|
||
def _reasoning_text(obj: dict) -> str:
|
||
"""Extract chain-of-thought text from a message or delta dict. Most
|
||
OpenAI-compatible backends use `reasoning_content`, but some vLLM builds
|
||
(observed: vllm-0.23.1rc1) use a plain `reasoning` key instead — check both."""
|
||
return obj.get("reasoning_content") or obj.get("reasoning") or ""
|
||
|
||
|
||
def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
|
||
"""Inline synchronous persona rewrite; raises RuntimeError on failure."""
|
||
settings = _load_settings()
|
||
system = (
|
||
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
||
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
||
"Return ONLY the rephrased text — no quotes, no explanation."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": text},
|
||
],
|
||
"temperature": 0.3,
|
||
"max_tokens": 512,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
resp = requests.post(
|
||
f"{llm_url.rstrip('/')}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
result = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
if result.startswith('"') and result.endswith('"'):
|
||
result = result[1:-1].strip()
|
||
return result
|
||
|
||
|
||
def _request_timeout_seconds(value, default: float = 600.0, minimum: float = 5.0, maximum: float = 600.0) -> float:
|
||
"""Clamp caller-provided LLM timeouts so UI recovery cannot hang indefinitely."""
|
||
try:
|
||
timeout = float(value)
|
||
except Exception:
|
||
timeout = default
|
||
return max(minimum, min(maximum, timeout))
|
||
|
||
|
||
def _is_router_model_alias(model: str) -> bool:
|
||
return bool(re.match(r"^\s*auto[-_ ]?router", model or "", re.I))
|
||
|
||
|
||
def _response_error_text(resp) -> str:
|
||
try:
|
||
text = resp.text or ""
|
||
except Exception:
|
||
text = ""
|
||
return text.strip()[:2000] or getattr(resp, "reason", "") or "Unknown upstream error"
|
||
|
||
|
||
def _llm_chat_completion_urls(llm_url: str) -> list[str]:
|
||
base = (llm_url or "http://localhost:11434/v1").rstrip("/")
|
||
urls: list[str] = []
|
||
if not re.search(r"/v\d+(?:\.\d+)?$", base):
|
||
urls.append(f"{base}/v1/chat/completions")
|
||
urls.append(f"{base}/chat/completions")
|
||
return list(dict.fromkeys(urls))
|
||
|
||
|
||
def _post_llm_chat_completion(llm_url: str, payload: dict, headers: dict, timeout: float):
|
||
last_resp = None
|
||
last_exc = None
|
||
for url in _llm_chat_completion_urls(llm_url):
|
||
try:
|
||
resp = requests.post(url, json=payload, headers=headers, timeout=timeout)
|
||
last_resp = resp
|
||
if resp.status_code in (404, 405) and url != _llm_chat_completion_urls(llm_url)[-1]:
|
||
continue
|
||
return resp
|
||
except Exception as exc:
|
||
last_exc = exc
|
||
if last_resp is not None:
|
||
return last_resp
|
||
if last_exc:
|
||
raise last_exc
|
||
raise RuntimeError("LLM request failed")
|
||
|
||
|
||
def _fallback_attribute_response(text: str) -> dict:
|
||
"""Deterministic fallback for attribution when an upstream LLM fails."""
|
||
quote_re = re.compile(
|
||
r"»([^«]+)«|«([^»]+)»|„([^“”]+)[“”]|“([^”]+)”|\"([^\"]+)\"|「([^」]+)」|『([^』]+)』",
|
||
re.DOTALL,
|
||
)
|
||
segments = []
|
||
pos = 0
|
||
for match in quote_re.finditer(text):
|
||
if match.start() > pos:
|
||
narr = text[pos:match.start()].strip()
|
||
if narr:
|
||
segments.append({"speaker": "Narrator", "type": "narration", "text": narr, "emotion": ""})
|
||
spoken = next((g for g in match.groups() if g), "")
|
||
if spoken.strip():
|
||
segments.append({"speaker": "Unknown", "type": "dialogue", "text": spoken.strip(), "emotion": ""})
|
||
pos = match.end()
|
||
tail = text[pos:].strip()
|
||
if tail:
|
||
segments.append({"speaker": "Narrator", "type": "narration", "text": tail, "emotion": ""})
|
||
if not segments:
|
||
segments = [{"speaker": "Narrator", "type": "narration", "text": text, "emotion": ""}]
|
||
chars = []
|
||
for seg in segments:
|
||
sp = seg.get("speaker")
|
||
if seg.get("type") == "dialogue" and sp and sp not in ("Narrator", "Unknown") and sp not in chars:
|
||
chars.append(sp)
|
||
return {"segments": segments, "characters": chars, "fallback": True}
|
||
|
||
|
||
def _resolve_speak_voice(settings: dict, client_id: str, explicit_voice: str) -> str:
|
||
if explicit_voice:
|
||
return explicit_voice
|
||
bindings: dict = settings.get("client_voice_bindings") or {}
|
||
if client_id and client_id in bindings:
|
||
return bindings[client_id]
|
||
return settings.get("captures_default_voice") or ""
|
||
|
||
|
||
# ── Routes ────────────────────────────────────────────────────────────────────
|
||
|
||
@router.post("/api/refine-text")
|
||
async def refine_text(request: Request):
|
||
"""Clean up raw STT transcription using a local OpenAI-compatible LLM."""
|
||
data = await request.json()
|
||
settings = _load_settings()
|
||
text: str = (data.get("text") or "").strip()
|
||
llm_url: str = (data.get("llm_url") or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or settings.get("refine_model") or settings.get("llm_model") or "").strip()
|
||
toggles: dict = data.get("toggles") or {}
|
||
|
||
if not text:
|
||
raise HTTPException(400, "No text to refine")
|
||
|
||
rules = []
|
||
if toggles.get("fillers", True):
|
||
rules.append("Remove filler words (um, uh, like, you know, basically, literally, I mean, so, right, etc.)")
|
||
if toggles.get("repetitions", True):
|
||
rules.append("Remove repeated words and false starts (e.g. 'the the dog' → 'the dog', 'I was- I was going' → 'I was going')")
|
||
if toggles.get("corrections", True):
|
||
rules.append("Remove self-corrections and restarts, keeping only the final intended phrasing")
|
||
if toggles.get("punctuation", True):
|
||
rules.append("Fix punctuation, capitalisation, and sentence boundaries")
|
||
if not rules:
|
||
return {"text": text, "original": text}
|
||
|
||
system = (
|
||
"You are a transcription cleanup assistant. "
|
||
"Apply ONLY the following rules to the user's text. "
|
||
"Return ONLY the cleaned text — no explanations, no quotes, no markdown:\n"
|
||
+ "\n".join(f"- {r}" for r in rules)
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": text},
|
||
],
|
||
"temperature": 0.1,
|
||
"max_tokens": 2048,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=60,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
refined = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
if refined.startswith('"') and refined.endswith('"'):
|
||
refined = refined[1:-1].strip()
|
||
return {"text": refined, "original": text}
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM refinement failed: {e}")
|
||
|
||
|
||
@router.post("/api/rewrite-with-persona")
|
||
async def rewrite_with_persona(request: Request):
|
||
"""Rewrite user text in a voice persona's character using a local LLM."""
|
||
data = await request.json()
|
||
settings = _load_settings()
|
||
text: str = (data.get("text") or "").strip()
|
||
persona: str = (data.get("persona") or "").strip()
|
||
llm_url: str = (data.get("llm_url") or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or settings.get("llm_model") or "").strip()
|
||
mode: str = (data.get("mode") or "rewrite").strip()
|
||
|
||
if not persona:
|
||
raise HTTPException(400, "No persona defined for this voice")
|
||
if not text and mode != "compose":
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
if mode == "compose":
|
||
system = (
|
||
f"You are a voice assistant with this character: {persona}\n"
|
||
"Write a single natural utterance in this character's voice about the topic given. "
|
||
"Return ONLY the utterance — no quotes, no explanation."
|
||
)
|
||
user_msg = text or "Introduce yourself briefly."
|
||
temp = 0.9
|
||
else:
|
||
system = (
|
||
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
||
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
||
"Return ONLY the rephrased text — no quotes, no explanation."
|
||
)
|
||
user_msg = text
|
||
temp = 0.3
|
||
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user_msg},
|
||
],
|
||
"temperature": temp,
|
||
"max_tokens": 512,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
result = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
if result.startswith('"') and result.endswith('"'):
|
||
result = result[1:-1].strip()
|
||
return {"text": result, "original": text, "persona": persona}
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM persona rewrite failed: {e}")
|
||
|
||
|
||
@router.post("/api/analyze-characters")
|
||
async def analyze_characters(request: Request):
|
||
"""Analyze a script with an LLM and return per-character voice descriptions.
|
||
|
||
Used by the Script Rehearser to auto-design voices that match each role.
|
||
Returns: {"characters": [{name, gender, language, age, description}, ...]}
|
||
"""
|
||
data = await request.json()
|
||
script: str = (data.get("script") or "").strip()
|
||
names: list = data.get("names") or []
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not script:
|
||
raise HTTPException(400, "No script provided")
|
||
if not names:
|
||
raise HTTPException(400, "No character names provided")
|
||
|
||
# Truncate script to leave room for output within typical model context windows.
|
||
# German/non-English text tokenises at ~2.5–3 chars/token, so 6000 chars ≈ 2000–2400 tokens.
|
||
if len(script) > 6000:
|
||
script = script[:6000]
|
||
|
||
lang_hint = f" The script language is {language}." if language else ""
|
||
|
||
def _build_system() -> str:
|
||
return (
|
||
"You are a casting director and TTS voice-design expert. "
|
||
"Your job is to read a script, understand each character deeply from their "
|
||
"dialogue, role, and context, then write a voice description that a "
|
||
"text-to-speech model can use to generate a matching voice.\n"
|
||
f"{lang_hint}\n"
|
||
"For EVERY character in the provided list output:\n"
|
||
"- name: exact name as given\n"
|
||
"- gender: M, F, or N\n"
|
||
"- language: spoken language of this character\n"
|
||
"- age: estimated age range (e.g. 20s, 40s, elderly)\n"
|
||
"- description: 2–3 sentences covering pitch (high/mid/low), pace, "
|
||
"timbre, accent/dialect, emotional default, and any distinctive speech trait "
|
||
"that fits the character's personality and role.\n"
|
||
"Characters with few lines: infer from their role name and context.\n"
|
||
"Respond with STRICT JSON only — no markdown, no explanation:\n"
|
||
'{"characters":[{"name":"NAME","gender":"M|F|N","language":"LANG",'
|
||
'"age":"30s","description":"voice description"}]}\n'
|
||
"Use the exact character names provided.\n/no-think"
|
||
)
|
||
|
||
def _call_llm(batch: list[str]) -> list[dict]:
|
||
"""Call the LLM for one batch of character names; return list of character dicts."""
|
||
user_msg = (
|
||
"Character names: " + ", ".join(str(n) for n in batch) + "\n\n"
|
||
"Script:\n" + script
|
||
)
|
||
# 150 tokens per character output, capped at 3072 to stay within 16K context
|
||
mt = min(3072, max(512, len(batch) * 150))
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": _build_system()},
|
||
{"role": "user", "content": user_msg},
|
||
],
|
||
"temperature": 0.4,
|
||
"max_tokens": mt,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions",
|
||
json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"},
|
||
timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
for candidate in (content, _extract_json_block(content)):
|
||
if not candidate:
|
||
continue
|
||
try:
|
||
parsed = json.loads(candidate)
|
||
if isinstance(parsed, dict) and "characters" in parsed:
|
||
return parsed["characters"]
|
||
except Exception:
|
||
continue
|
||
return []
|
||
|
||
# Process cast in batches of 10 to stay well within 16 K context
|
||
BATCH = 10
|
||
all_characters: list[dict] = []
|
||
try:
|
||
for i in range(0, len(names), BATCH):
|
||
batch = names[i:i + BATCH]
|
||
all_characters.extend(_call_llm(batch))
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM character analysis failed: {e}")
|
||
|
||
if not all_characters:
|
||
raise HTTPException(502, "LLM did not return valid character JSON")
|
||
|
||
return {"characters": all_characters}
|
||
|
||
|
||
@router.post("/api/match-characters-voices")
|
||
async def match_characters_voices(request: Request):
|
||
"""Pick the best EXISTING library voice for each character (instead of designing new ones).
|
||
|
||
Body: {script, names:[...], voices:[{id, gender, language, tags, description}], llm_url, model, language}
|
||
Returns: {"assignments": [{name, voice_id, reason}]}
|
||
"""
|
||
data = await request.json()
|
||
script: str = (data.get("script") or "").strip()
|
||
names: list = data.get("names") or []
|
||
voices: list = data.get("voices") or []
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not names:
|
||
raise HTTPException(400, "No character names provided")
|
||
if not voices:
|
||
raise HTTPException(400, "No candidate voices provided")
|
||
if len(script) > 5000:
|
||
script = script[:5000]
|
||
|
||
def _vline(v: dict) -> str:
|
||
meta = [str(v.get(k)) for k in ("gender", "language") if v.get(k)]
|
||
if v.get("tags"):
|
||
meta.append("tags:" + str(v["tags"]))
|
||
desc = str(v.get("description") or v.get("name") or "")[:120]
|
||
return f"- {v.get('id')} [{', '.join(meta)}] {desc}".rstrip()
|
||
|
||
catalogue = "\n".join(_vline(v) for v in voices[:300])
|
||
valid_ids = {str(v.get("id")) for v in voices if v.get("id")}
|
||
|
||
system = (
|
||
"You are a casting director assigning existing TTS voices to script characters. "
|
||
f"{('Script language: ' + language + '. ') if language else ''}"
|
||
"For EACH character, choose the single BEST voice_id from the CATALOGUE. "
|
||
"RULE 1 — GENDER FIRST: the voice's gender MUST match the character's gender whenever the "
|
||
"character's gender is clear from the script; only pick a different gender if no same-gender "
|
||
"voice exists in the catalogue. "
|
||
"RULE 2 — then match apparent age, language, personality, and the voice's description/tags. "
|
||
"You MUST choose a voice_id that appears verbatim in the catalogue — never invent one. "
|
||
"Reuse a voice for two characters only if no better distinct option exists. "
|
||
"Respond with STRICT JSON only, no markdown:\n"
|
||
'{"assignments":[{"name":"NAME","voice_id":"ID","reason":"short reason"}]}\n/no-think'
|
||
)
|
||
user = (
|
||
"Characters to cast: " + ", ".join(str(n) for n in names) + "\n\n"
|
||
"CATALOGUE (voice_id [gender, language, tags] description):\n" + catalogue + "\n\n"
|
||
"Script excerpt:\n" + script
|
||
)
|
||
payload: dict = {
|
||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||
"temperature": 0.3,
|
||
"max_tokens": min(2048, max(512, len(names) * 60)),
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
import time as _time
|
||
raw = None
|
||
last_err: Exception | None = None
|
||
for attempt in range(3):
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
break
|
||
except requests.exceptions.ConnectionError as e:
|
||
# llama-swap (and similar) often drop the first request while swapping/loading
|
||
# the model — wait and retry rather than failing the cast.
|
||
last_err = e
|
||
_time.sleep(4 + attempt * 3)
|
||
except Exception as e:
|
||
last_err = e
|
||
break
|
||
if raw is None:
|
||
raise HTTPException(502, f"LLM voice matching failed: {last_err}")
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
assignments: list = []
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("assignments"), list):
|
||
assignments = parsed["assignments"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
# Keep only assignments that reference a real catalogue voice
|
||
clean = [a for a in assignments if isinstance(a, dict) and str(a.get("voice_id")) in valid_ids]
|
||
return {"assignments": clean}
|
||
|
||
|
||
def _extract_json_block(text: str) -> str:
|
||
"""Pull the first {...} JSON object out of an LLM response."""
|
||
text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
|
||
if text.startswith("```"):
|
||
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
|
||
text = re.sub(r"\n?```$", "", text).strip()
|
||
start = text.find("{")
|
||
end = text.rfind("}")
|
||
if start != -1 and end != -1 and end > start:
|
||
return text[start:end + 1]
|
||
return ""
|
||
|
||
|
||
def _charsheets_prepare(data: dict) -> dict:
|
||
"""Resolve settings and build the chat payload for one character-sheets
|
||
extraction request. Shared by the blocking endpoint and the streaming
|
||
(watch-it-fill-out) endpoint so the prompt can never drift between them."""
|
||
text: str = (data.get("text") or "").strip()
|
||
known: list = data.get("known_characters") or []
|
||
target_mode: bool = bool(data.get("target_mode"))
|
||
existing: str = (data.get("existing") or "").strip() # partial sheets so far (progressive fill)
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
timeout_seconds = _request_timeout_seconds(data.get("timeout_seconds"), 600.0)
|
||
if not text:
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
lang_instruction = (
|
||
f"⚠️ SPRACHE / LANGUAGE — OBLIGATORISCH / MANDATORY:\n"
|
||
f"The source text is in {language}.\n"
|
||
f"YOU MUST write EVERY descriptive field value in {language}. This is non-negotiable.\n"
|
||
f"Fields that MUST be in {language}: physical, clothing, alignment, arc_note, skills, "
|
||
f"capabilities, backstory, relationships, motivation, fears, mannerisms, communication_style, voice_pattern, "
|
||
f"secret, conflict_style, win_condition, archetype, aliases, first_name, last_name, title, age_estimate, "
|
||
f"race_species, languages, nationality_background, social_class, profession, reputation, religious_beliefs, notes.\n"
|
||
f"voice_design_prompt and image_prompt should be concise English tool prompts for generation tools.\n"
|
||
f"JSON field KEYS stay in English. Character names stay exactly as they appear in the text.\n"
|
||
f"If you write any descriptive value in English instead of {language}, your response is WRONG.\n\n"
|
||
) if language else ""
|
||
target_instruction = (
|
||
"CAST TARGET MODE:\n"
|
||
"- The known-characters list is the already recognized cast roster from the audiobook pass. "
|
||
"Treat it as TARGET PROFILES to fill, not as a list of aliases and not as proof that those names are the same person.\n"
|
||
"- Prefer outputting sheets for those target names when this passage reveals usable profile details. "
|
||
"Do not output a blank sheet just because a target is listed.\n"
|
||
"- If the passage proves that a target has another name/title/nickname, keep ONE profile using the best known target name "
|
||
"and put only the proven alternate form in aliases/title/full_name.\n"
|
||
"- Do NOT invent brand-new profiles in this pass. Only refine the already-casted roster; ignore places, institutions, and other non-person entities.\n"
|
||
"- NEVER copy the known-characters roster into one character's aliases, relationships, title, or description fields.\n"
|
||
"- Before deciding a passage is about someone else: check whether the person could BE one of the targets "
|
||
"under an alias, title, role, or nickname instead of their literal listed name — check both this passage's "
|
||
"own context (action beats, who's being addressed, profession mentioned) AND the 'existing sheets so far' "
|
||
"below, which may already record that alias for a target (e.g. a target's existing sheet says "
|
||
"'aliases: der Schmied' — a passage about 'der Schmied' with no other name given IS that target, attribute "
|
||
"it there, do not skip it just because the passage never says the target's literal name).\n"
|
||
"- Only if, after that check, the passage is clearly about a DIFFERENT person who is NOT one of the listed "
|
||
"targets and NOT an alias/role already recorded for one of them (a scene that doesn't actually involve any "
|
||
"target), output NO sheet for this passage at all — an empty 'sheets' array is correct and expected. Never "
|
||
"force unrelated content about someone else into a target's profile just because a target happens to be "
|
||
"listed and you can't create a new one — but do not use this as an excuse to skip a passage that genuinely "
|
||
"is about a target under an alias; skipping a real match is just as wrong as gluing a fact onto the wrong "
|
||
"character.\n\n"
|
||
) if target_mode and known else ""
|
||
|
||
# A user-editable prompt (client-side "Prompt" panel, mirroring the
|
||
# audiobook casting one) replaces the extraction-instructions/schema body
|
||
# below wholesale — the language/target-mode prefixes above still apply
|
||
# regardless, since they depend on per-request state the user's saved
|
||
# prompt text can't know about.
|
||
prompt_override = data.get("character_sheets_prompt")
|
||
if isinstance(prompt_override, str) and prompt_override.strip():
|
||
system = f"{lang_instruction}{target_instruction}{prompt_override.strip()}"
|
||
else:
|
||
system = (
|
||
f"{lang_instruction}"
|
||
f"{target_instruction}"
|
||
"You are an expert dramaturge, developmental editor, and tabletop RPG game master building rich "
|
||
"character sheets passage by passage as a book is read. Extract playable, action-oriented sheets "
|
||
"an actor can use to immediately know how to PLAY the character.\n"
|
||
"PROGRESSIVE FILLING: you may be given the sheets built so far. For returning characters, ADD any "
|
||
"NEW detail this passage reveals and refine vague fields; do not contradict solid earlier facts or "
|
||
"blank out a field you cannot improve. In this cast-character pass, only refine the already-casted "
|
||
"roster and do NOT invent new profiles, places, or institutions. The passage may be a focused "
|
||
"evidence window around a character mention, so use nearby paragraphs as context. Leave a field "
|
||
"empty if the book genuinely hasn't shown it yet (a later passage can fill it). Extrapolate from "
|
||
"dialogue and actions when reasonable, and mark any deduced value with a trailing ' *'.\n"
|
||
"For each character output these fields:\n"
|
||
"- name: canonical display name for this one character. Use the real personal name if known; otherwise use the most stable role/title.\n"
|
||
"- aliases: ONLY alternate names, roles, epithets, mistranscriptions, and titles proven to refer to the SAME character, comma-separated (max 6 items; e.g. 'the Executioner, Bloodfang, Marcus the Executioner'). Leave empty when uncertain.\n"
|
||
"- first_name, last_name: split the character identity when known. Leave unknown parts empty.\n"
|
||
"- title: nobility title only, if the text explicitly gives one (e.g. 'Graf', 'Baron', 'Ritter').\n"
|
||
"- profession: occupation / job / role in the story (e.g. 'Inquisitor', 'Soldier', 'Merchant', 'Priest').\n"
|
||
"- age_estimate: estimated age or age range, if inferable\n"
|
||
"- race_species: species, race, or kind (human, elf, ork, vampire, etc.) if relevant\n"
|
||
"- languages: spoken languages / dialects / tongues, comma-separated if multiple\n"
|
||
"- nationality_background: homeland, culture, origin, or social background if known\n"
|
||
"- social_class: rank or class if the text makes it clear (noble, soldier, slave, merchant, priesthood, etc.)\n"
|
||
"- archetype: a two-word role summary (e.g. 'Ruthless Scholar')\n"
|
||
"- gender: 'male', 'female', or 'nonbinary' — as apparent from the text (pronouns, roles, physical description). Leave empty if genuinely indeterminable.\n"
|
||
"- physical: height, weight, build, hair, eyes, skin, posture, gait, distinguishing features, physical disabilities, fantasy-specific extras, and any other bodily appearance details. Use metric units when size/weight is known.\n"
|
||
"- clothing: day-to-day wear, work attire, formal wear, sleepwear, undergarments, accessories, and visible weapons/gear if they define the look\n"
|
||
"- alignment: strict moral code + the one line they will never cross\n"
|
||
"- moral_alignment_score: integer 0–100. 100 = purely good/heroic, 0 = purely evil/villainous, 50 = neutral/ambiguous\n"
|
||
"- arc_direction: one of: 'stable-good', 'stable-bad', 'neutral', 'good-to-bad', 'bad-to-good', 'complex'\n"
|
||
"- arc_note: one sentence explaining the arc or moral position visible so far\n"
|
||
"- attribute_high / attribute_low: highest and lowest natural attribute (Charisma, Intelligence, Wisdom, Agility…)\n"
|
||
"- skills: what they are demonstrably good at in the story\n"
|
||
"- capabilities: combat, magic, social, technical, or other demonstrated abilities\n"
|
||
"- backstory: origin, formative background and history revealed in the text\n"
|
||
"- relationships: key allies, family, rivals and enemies — name them and how they relate\n"
|
||
"- motivation: the inner drive — WHY they pursue what they pursue (distinct from the win condition)\n"
|
||
"- fears: their deepest fears, phobias or dread\n"
|
||
"- mannerisms: habitual gestures, tics, body language, habits and quirks\n"
|
||
"- communication_style: how they communicate socially — blunt, formal, warm, guarded, sarcastic, etc.\n"
|
||
"- voice_pattern: speech style — accent, pacing, vocabulary, register and verbal tics (for voice casting)\n"
|
||
"- voice_design_prompt: concise English Qwen voice-design prompt (15-45 words). Include age impression, gender/androgyny if inferable, pitch, timbre, pace, accent/register, emotional baseline and suitability for audiobook dialogue. "
|
||
+ ("State the accent explicitly as an authentic native " + language + " accent — never American-accented English, even though the prompt itself is written in English. " if language and language.lower() != "english" else "State the accent explicitly as a neutral British or international English accent — never American/US-accented. ")
|
||
+ "Do NOT mention plot spoilers.\n"
|
||
"- image_prompt: detailed English image-generation prompt for this character. Include face, age impression, build, hair/eyes/skin if known, clothing, posture, props, mood, genre/style, and visible symbols. Mark inferred traits with '*'.\n"
|
||
"- inventory: 1-3 defining items/props/clothing (array of short strings)\n"
|
||
"- secret: dark secret or fatal flaw\n"
|
||
"- conflict_style: fight, flight, or manipulate — how they act when cornered\n"
|
||
"- win_condition: the specific event that would make them feel they have won\n"
|
||
"- reputation: how other characters or society see them\n"
|
||
"- religious_beliefs: faith, religion, worship, or lack of belief if the text shows it\n"
|
||
"- notes: brief catch-all notes for useful details that do not fit elsewhere\n"
|
||
"- tier: 'main' or 'supporting'\n"
|
||
"- sources: array of {page, quote, line_hint} — the page number from the nearest [p.N] marker, "
|
||
"a short verbatim quote that supports the sheet (1-12 entries), and a brief label (e.g. 'physical', 'clothing', 'relationships', 'motivation'). "
|
||
"Use null page if unknown. line_hint MUST name the supported field when possible: physical, clothing, relationships, motivation, fears, mannerisms, voice_pattern, backstory, alignment, skills, capabilities, secret, conflict_style, or win_condition.\n"
|
||
"IDENTITY MERGING: A character may appear under multiple names in the book (first name, last name, title, role, alias, nickname). "
|
||
"Examples: 'Marcus', 'the Executioner', and 'Bloodfang' may all refer to ONE profile if context shows they are the same person. "
|
||
"Do NOT create separate sheets for aliases/titles of the same person; put the alternate forms in aliases/title and keep one canonical name.\n"
|
||
"Reuse the EXACT names from the known-characters list for returning characters when they are the canonical name or an alias of this character. "
|
||
"Do not merge characters merely because their names appear near each other, in the known-character list, or in relationships.\n"
|
||
"Respond with STRICT JSON only:\n"
|
||
'{"sheets":[{"name":"","aliases":"","first_name":"","last_name":"","title":"","profession":"","age_estimate":"","race_species":"","languages":"","nationality_background":"","social_class":"","archetype":"","gender":"","physical":"","clothing":"",'
|
||
'"alignment":"","moral_alignment_score":50,"arc_direction":"neutral","arc_note":"",'
|
||
'"attribute_high":"","attribute_low":"","skills":"","capabilities":"",'
|
||
'"backstory":"","relationships":"","motivation":"","fears":"","mannerisms":"","communication_style":"","voice_pattern":"","voice_design_prompt":"","image_prompt":"",'
|
||
'"inventory":[],"secret":"","conflict_style":"","win_condition":"","reputation":"","religious_beliefs":"","notes":"",'
|
||
'"tier":"main","sources":[{"page":1,"quote":"","line_hint":""}]}]}\n/no-think'
|
||
)
|
||
lang_reminder = (
|
||
f"⚠️ WICHTIG: Alle beschreibenden Feldwerte MÜSSEN auf {language} geschrieben werden. "
|
||
f"Kein einziges beschreibendes Feld darf auf Englisch sein. Nur JSON-Schlüssel bleiben Englisch.\n\n"
|
||
) if language else ""
|
||
user = (
|
||
("Known characters so far: " + ", ".join(str(n) for n in known) + "\n\n" if known else "")
|
||
+ ("Sheets so far (fill gaps / refine; keep solid facts):\n" + existing + "\n\n" if existing else "")
|
||
+ lang_reminder
|
||
+ "Passage:\n" + text
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.4,
|
||
"max_tokens": 4096,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
return {
|
||
"payload": payload, "llm_url": llm_url, "model": model,
|
||
"api_key": _settings.get("llm_api_key") or "sk-dummy-key",
|
||
"timeout_seconds": timeout_seconds,
|
||
}
|
||
|
||
|
||
def _charsheets_parse(raw: str) -> dict:
|
||
"""Turn the LLM's raw answer into normalised {sheets, characters}."""
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
sheets = []
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("sheets"), list):
|
||
sheets = parsed["sheets"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
clean, names = [], []
|
||
alias_splitter = re.compile(r"[,;/|]|\baka\b|\baka\.\b|\balias(?:es)?\b|\bgenannt\b|\bnamens\b|\bcalled\b|\bknown as\b", re.I)
|
||
for s in sheets:
|
||
if not isinstance(s, dict):
|
||
continue
|
||
name = str(s.get("name") or "").strip()
|
||
if not name:
|
||
continue
|
||
alias_raw = str(s.get("aliases") or "").strip()
|
||
alias_parts = [p.strip() for p in alias_splitter.split(alias_raw) if p.strip()]
|
||
alias_parts = [p for p in alias_parts if len(p) <= 80 and not re.match(r"^(needs?:|complete$)", p, re.I)]
|
||
if len(alias_raw) > 500 or len(alias_parts) > 12:
|
||
aliases = ""
|
||
else:
|
||
aliases = ", ".join(alias_parts[:6])
|
||
inv = s.get("inventory")
|
||
if isinstance(inv, str):
|
||
inv = [x.strip() for x in inv.split(",") if x.strip()]
|
||
elif not isinstance(inv, list):
|
||
inv = []
|
||
src_raw = s.get("sources") if isinstance(s.get("sources"), list) else []
|
||
src = []
|
||
for item in src_raw:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
quote = str(item.get("quote") or "").strip()
|
||
if not quote:
|
||
continue
|
||
try:
|
||
page = int(item.get("page")) if item.get("page") is not None else None
|
||
except (TypeError, ValueError):
|
||
page = None
|
||
src.append({
|
||
"page": page,
|
||
"quote": quote[:240],
|
||
"line_hint": str(item.get("line_hint") or "").strip()[:60],
|
||
})
|
||
# Clamp moral alignment score
|
||
try:
|
||
mas = int(s.get("moral_alignment_score") or 50)
|
||
mas = max(0, min(100, mas))
|
||
except (TypeError, ValueError):
|
||
mas = 50
|
||
arc = str(s.get("arc_direction") or "neutral").strip()
|
||
if arc not in ("stable-good", "stable-bad", "neutral", "good-to-bad", "bad-to-good", "complex"):
|
||
arc = "neutral"
|
||
# The model answers in whatever language the source text is in (this
|
||
# app is used heavily with German books), so a strict English-only
|
||
# whitelist silently discarded valid answers like "weiblich"/"männlich"
|
||
# instead of normalizing them — every non-German-speaking character's
|
||
# gender was quietly wiped to blank.
|
||
gender_raw = str(s.get("gender") or "").strip().lower()
|
||
_GENDER_NORMALIZE = {
|
||
"male": "male", "m": "male", "man": "male", "mann": "male", "männlich": "male",
|
||
"female": "female", "f": "female", "woman": "female", "frau": "female", "weiblich": "female",
|
||
"nonbinary": "nonbinary", "non-binary": "nonbinary", "n": "nonbinary",
|
||
"nichtbinär": "nonbinary", "nicht-binär": "nonbinary", "divers": "nonbinary",
|
||
}
|
||
gender = _GENDER_NORMALIZE.get(gender_raw, "")
|
||
s.update({
|
||
"name": name, "aliases": aliases, "inventory": inv[:3],
|
||
"tier": "main" if str(s.get("tier") or "").lower().startswith("main") else "supporting",
|
||
"sources": src[:12],
|
||
"moral_alignment_score": mas,
|
||
"arc_direction": arc,
|
||
"gender": gender,
|
||
})
|
||
clean.append(s)
|
||
names.append(name)
|
||
return {"sheets": clean, "characters": names}
|
||
|
||
|
||
@router.post("/api/character-sheets")
|
||
async def character_sheets(request: Request):
|
||
"""Extract actor-facing RPG-style character sheets from a passage.
|
||
|
||
Body: {text, known_characters:[...], language, llm_url, model}
|
||
The text may contain "[p.N]" page markers so the model can cite sources.
|
||
Returns: {sheets:[{name, aliases, first_name, last_name, full_name, title, archetype, physical, alignment,
|
||
attribute_high, attribute_low, skills, inventory:[...], secret,
|
||
conflict_style, win_condition, tier:"main"|"supporting",
|
||
sources:[{page, quote}]}], characters:[names]}
|
||
Deduced (not explicit) values are marked with a trailing " *".
|
||
"""
|
||
data = await request.json()
|
||
prep = _charsheets_prepare(data)
|
||
try:
|
||
if not await asyncio.to_thread(_attribution_llm_lock.acquire, True, prep["timeout_seconds"]):
|
||
raise HTTPException(503, "Attribution engine busy")
|
||
try:
|
||
resp = await asyncio.to_thread(
|
||
_post_llm_chat_completion,
|
||
prep["llm_url"], prep["payload"],
|
||
{"Authorization": f"Bearer {prep['api_key']}"}, prep["timeout_seconds"],
|
||
)
|
||
finally:
|
||
_attribution_llm_lock.release()
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM character-sheet generation failed: {e}")
|
||
return _charsheets_parse(raw)
|
||
|
||
|
||
@router.post("/api/character-sheets/stream")
|
||
async def character_sheets_stream(request: Request):
|
||
"""Same job as /api/character-sheets, but streams the LLM's raw output
|
||
(its JSON answer being written field by field) as SSE `{"t": "..."}`
|
||
events, ending with `{"done": true, "result": {...}}` — lets the UI show
|
||
the sheet actually filling out passage by passage instead of just a
|
||
progress bar. On upstream failure it emits `{"error": "..."}` and the
|
||
client falls back to the blocking endpoint."""
|
||
data = await request.json()
|
||
prep = _charsheets_prepare(data)
|
||
payload = dict(prep["payload"])
|
||
payload["stream"] = True
|
||
headers = {"Authorization": f"Bearer {prep['api_key']}"}
|
||
|
||
def gen():
|
||
if not _attribution_llm_lock.acquire(timeout=prep["timeout_seconds"]):
|
||
yield 'data: {"error": "Attribution engine busy"}\n\n'
|
||
return
|
||
upstream = None
|
||
try:
|
||
urls = _llm_chat_completion_urls(prep["llm_url"])
|
||
last_err = None
|
||
for u in urls:
|
||
try:
|
||
upstream = requests.post(
|
||
u, json=payload, headers=headers,
|
||
timeout=(15, prep["timeout_seconds"]), stream=True,
|
||
)
|
||
if upstream.status_code in (404, 405) and u != urls[-1]:
|
||
upstream.close()
|
||
upstream = None
|
||
continue
|
||
break
|
||
except Exception as exc:
|
||
last_err = exc
|
||
upstream = None
|
||
if upstream is None:
|
||
print(f"[character-sheets/stream] LLM connect failed: {last_err}")
|
||
yield f'data: {json.dumps({"error": str(last_err or "LLM connect failed")})}\n\n'
|
||
return
|
||
if upstream.status_code >= 400:
|
||
print(
|
||
f"[character-sheets/stream] LLM failed ({upstream.status_code}) for model "
|
||
f"{payload.get('model') or '(default)'}: {_response_error_text(upstream)[:300]}"
|
||
)
|
||
yield f'data: {json.dumps({"error": f"HTTP {upstream.status_code}: {_response_error_text(upstream)[:300]}"})}\n\n'
|
||
return
|
||
raw_parts = []
|
||
upstream.encoding = "utf-8"
|
||
# Two layers against a stuck stream: the wall-clock check catches
|
||
# a model that keeps trickling chunks past its budget, and
|
||
# _watchdog_close catches the connection going fully silent (which
|
||
# the in-loop check can't see, since a blocked socket read never
|
||
# reaches it — see _watchdog_close's docstring for why).
|
||
deadline = time.monotonic() + prep["timeout_seconds"]
|
||
with _watchdog_close(upstream, prep["timeout_seconds"]):
|
||
for line in upstream.iter_lines(decode_unicode=True):
|
||
if time.monotonic() > deadline:
|
||
print(f"[character-sheets/stream] wall-clock deadline hit after {prep['timeout_seconds']}s, aborting stream")
|
||
yield 'data: {"error": "LLM stream exceeded timeout"}\n\n'
|
||
return
|
||
if not line or not line.startswith("data:"):
|
||
continue
|
||
chunk = line[5:].strip()
|
||
if chunk == "[DONE]":
|
||
break
|
||
try:
|
||
delta = json.loads(chunk)["choices"][0]["delta"]
|
||
except Exception:
|
||
continue
|
||
t = _reasoning_text(delta) or delta.get("content") or ""
|
||
if delta.get("content"):
|
||
raw_parts.append(delta["content"])
|
||
if t:
|
||
yield f'data: {json.dumps({"t": t})}\n\n'
|
||
result = _charsheets_parse("".join(raw_parts))
|
||
yield f'data: {json.dumps({"done": True, "result": result})}\n\n'
|
||
except GeneratorExit:
|
||
raise
|
||
except Exception as e:
|
||
yield f'data: {json.dumps({"error": str(e)})}\n\n'
|
||
finally:
|
||
try:
|
||
if upstream is not None:
|
||
upstream.close()
|
||
except Exception:
|
||
pass
|
||
_attribution_llm_lock.release()
|
||
|
||
return StreamingResponse(gen(), media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||
|
||
|
||
@router.post("/api/character-deep-analysis")
|
||
async def character_deep_analysis(request: Request):
|
||
"""Run a deep 5-area psychological analysis of a single character.
|
||
|
||
Body: {name, role, goal, summary, text_excerpt, language, llm_url, model}
|
||
Returns: {analysis: {core_flaw, agency, dialogue_voice, narrative_arc, paradox}}
|
||
"""
|
||
data = await request.json()
|
||
name: str = (data.get("name") or "").strip()
|
||
role: str = (data.get("role") or "unknown").strip()
|
||
goal: str = (data.get("goal") or "").strip()
|
||
summary: str = (data.get("summary") or "").strip()
|
||
excerpt: str = (data.get("text_excerpt") or "").strip()
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
|
||
if not name:
|
||
raise HTTPException(400, "Character name required")
|
||
|
||
lang_note = f" Write the analysis in {language}." if language else ""
|
||
|
||
system = (
|
||
f"You are an expert developmental editor, literary coach, and psychologist specializing in "
|
||
f"profound character studies.{lang_note} Provide a deep, multi-layered psychological and "
|
||
f"narrative analysis of the character in exactly this JSON structure:\n"
|
||
'{"core_flaw":"<analysis of primary flaw, its psychological origin, and conflict with their desire>",'
|
||
'"agency":"<are they active or reactive? where does their momentum stall? what harder choices could force growth?>",'
|
||
'"dialogue_voice":"<speech pattern analysis: how does their dialogue reflect background, anxieties, hidden motives? how do they sound distinct?>",'
|
||
'"narrative_arc":"<how does their worldview shift? what is the lie they believe vs the truth they must accept?>",'
|
||
'"paradox":"<3 specific contradictions in behaviour/personality that make them feel authentic and unpredictable>"}\n'
|
||
"Be ruthlessly comprehensive and honest. Use concrete examples from the text. "
|
||
"Map where character development might break down or feel cliché. "
|
||
"Respond with STRICT JSON only — no markdown, no explanation./no-think"
|
||
)
|
||
user = (
|
||
f'CHARACTER: "{name}"\n'
|
||
f"Role in story: {role}\n"
|
||
f"Core goal: {goal or 'not specified'}\n"
|
||
f"Background/key actions: {summary or 'not specified'}\n\n"
|
||
+ (f"Text excerpt for analysis:\n{excerpt[:4000]}\n\n" if excerpt else "")
|
||
+ "Provide the deep psychological analysis now."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.6,
|
||
"max_tokens": 2048,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
except Exception as e:
|
||
raise HTTPException(502, f"LLM deep analysis failed: {e}")
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
analysis = {}
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and "core_flaw" in parsed:
|
||
analysis = parsed
|
||
break
|
||
except Exception:
|
||
continue
|
||
if not analysis:
|
||
analysis = {"core_flaw": raw, "agency": "", "dialogue_voice": "", "narrative_arc": "", "paradox": ""}
|
||
return {"name": name, "analysis": analysis}
|
||
|
||
|
||
def _silly_tavern_prompt_instruction(sheet: dict) -> str:
|
||
"""Full SillyTavern character-card template (card + scenario + first
|
||
message), adapted from a user-supplied prompt originally written for
|
||
building a card from scratch via internet research (fandom/wikipedia).
|
||
Here the character's profile is already fully known from the book, so
|
||
the internet-research instructions are dropped and replaced with
|
||
'use ONLY the profile provided' (already stated in the shared preamble
|
||
this gets appended to) — everything else (exact field structure, the
|
||
generic-but-deliberately-open Scenario block, the First Message rules)
|
||
is kept close to the original since it's a well-tested format."""
|
||
gender = str((sheet or {}).get("gender") or "").strip().lower()
|
||
is_female = gender.startswith("f")
|
||
is_male = gender.startswith("m")
|
||
|
||
if is_female:
|
||
appearance = (
|
||
'hair: [COLOR, PICK FROM:straight/wavy/curly, PICK FROM:long (mid-back length)/long (waist-length)/'
|
||
'long (arms-length)/short (chin-length)], eyes: COLOR, height: HEIGHT cm, weight: WEIGHT kg, '
|
||
'body: [PICK FROM:slim/curvy, PICK FROM:perfect figure/sensual/abs, PICK FROM:light skin/tanned skin/'
|
||
'brown skin/green skin/blue skin/red skin], breasts: [SIZE, CUP, PICK FROM:big areolas/medium-sized '
|
||
'areolas/small areolas, PICK FROM:cherry-tan nipples/cherry-pink nipples/honey-tan nipples/'
|
||
'golden-brown nipples/dark-brown nipples], armpit hair: PICK FROM:shaved/natural, '
|
||
'pubic hair: PICK FROM:shaved/natural, fingernails: PICK FROM:natural/painted (color), '
|
||
'toenails: PICK FROM:natural/painted (color)'
|
||
)
|
||
outfits = (
|
||
'{{"Main Outfit"}}:{DESCRIBE TOP (COLOR), DESCRIBE BOTTOM (COLOR), DESCRIBE LEGS (COLOR), '
|
||
'DESCRIBE SHOES (COLOR), lingerie: [lace bra (COLOR), lace thong (COLOR)]}\n'
|
||
'{{"Formal Outfit"}}:{DESCRIBE TOP (COLOR), DESCRIBE BOTTOM (COLOR), DESCRIBE LEGS (COLOR), '
|
||
'DESCRIBE SHOES (COLOR), lingerie: [lace bra (color), lace thong (color)]}\n'
|
||
'{{"Sleeping Outfit"}}:{nightgown (COLOR), thong (COLOR), soft slippers (white)}\n'
|
||
'{{"Running Outfit"}}:{sports bra (COLOR), leggings (COLOR), sports shoes (white), lingerie: thong (COLOR)}\n'
|
||
'{{"Exercise Outfit"}}:{sports bra (COLOR), leggings (COLOR), bare feet, lingerie: lace thong (COLOR)}\n'
|
||
'{{"Swimsuit"}}:{PICK FROM: bikini/one-piece (COLOR), DESCRIBE SHOES (COLOR)}'
|
||
)
|
||
elif is_male:
|
||
appearance = (
|
||
'hair: [COLOR, PICK FROM:straight/wavy/curly, PICK FROM:long (mid-back length)/long (waist-length)/'
|
||
'long (arms-length)/short (chin-length)], facial hair: PICK FROM:beard/goatie/beard & moustache/'
|
||
'moustache/clean-shaven, eyes: COLOR, height: HEIGHT cm, weight: WEIGHT kg, '
|
||
'body: [PICK FROM:slim/muscular/bulky/fat, PICK FROM:light skin/tanned skin/brown skin/green skin/'
|
||
'blue skin/red skin], penis: [SIZE, LENGTH cm, PICK FROM:big balls/medium-sized balls/small balls, '
|
||
'PICK FROM:circumcised/uncircumcised], armpit hair: PICK FROM:shaved/natural, '
|
||
'pubic hair: PICK FROM:shaved/natural'
|
||
)
|
||
outfits = (
|
||
'{{"Main Outfit"}}:{DESCRIBE TOP (color), DESCRIBE BOTTOM (color), DESCRIBE SHOES (COLOR), '
|
||
'lingerie: DESCRIBE LINGERIE (COLOR)}\n'
|
||
'{{"Formal Outfit"}}:{DESCRIBE TOP (COLOR), DESCRIBE BOTTOM (COLOR), DESCRIBE LEGS (COLOR), '
|
||
'DESCRIBE SHOES (COLOR), lingerie: DESCRIBE LINGERIE (COLOR)}\n'
|
||
'{{"Sleeping Outfit"}}:{DESCRIBE TOP, DESCRIBE BOTTOM, soft slippers (white)}\n'
|
||
'{{"Running Outfit"}}:{DESCRIBE TOP, DESCRIBE BOTTOM, sports shoes (white), lingerie: DESCRIBE LINGERIE (COLOR)}\n'
|
||
'{{"Exercise Outfit"}}:{DESCRIBE TOP, DESCRIBE BOTTOM, bare feet, lingerie: DESCRIBE LINGERIE (COLOR)}\n'
|
||
'{{"Swimsuit"}}:{DESCRIBE BOTTOM, DESCRIBE SHOES (COLOR)}'
|
||
)
|
||
else:
|
||
# Gender unknown/non-binary/narrator role — keep the same card
|
||
# shape but skip the anatomy-specific appearance fields entirely
|
||
# rather than guessing a binary that doesn't fit.
|
||
appearance = (
|
||
'hair: [COLOR, STYLE, LENGTH], eyes: COLOR, height: HEIGHT cm, weight: WEIGHT kg, '
|
||
'body: [BUILD, SKIN TONE], distinguishing features: DESCRIBE'
|
||
)
|
||
outfits = (
|
||
'{{"Main Outfit"}}:{DESCRIBE TOP (COLOR), DESCRIBE BOTTOM (COLOR), DESCRIBE SHOES (COLOR)}\n'
|
||
'{{"Formal Outfit"}}:{DESCRIBE TOP (COLOR), DESCRIBE BOTTOM (COLOR), DESCRIBE SHOES (COLOR)}\n'
|
||
'{{"Sleeping Outfit"}}:{DESCRIBE SLEEPWEAR, soft slippers (white)}'
|
||
)
|
||
|
||
return (
|
||
"Produce exactly this field:\n"
|
||
"- silly_tavern_prompt: a complete SillyTavern character card for this character, in the EXACT format "
|
||
"below — a card body, then a Scenario block, then a First Message. Fill every field from the character "
|
||
"profile above; only invent a value when the profile truly has nothing for it, and mark anything invented "
|
||
"with a trailing '*'. Do not add bullet points, extra spaces, or commentary — follow the formatting "
|
||
"exactly. Do not replace '{{char}}' with the character's actual name — keep it literal. Keep every '{', "
|
||
"'}', '[', ']', '(', ')' character exactly as shown.\n\n"
|
||
"{{char}}:\n"
|
||
"{\n"
|
||
'{{"Personal Information"}}:{name: NAME, surname: SURNAME, race: PICK FROM PROFILE OR INFER, '
|
||
"nationality: NATIONALITY, gender: GENDER, age: AGE, profession: PROFESSION, "
|
||
"residence: [PLACE, TYPE OF DWELLING], marital status: MARITAL STATUS}\n\n"
|
||
f'{{{{"Appearance"}}}}:{{{appearance}}}\n\n'
|
||
'{{"Personality"}}:{A DETAILED, SPECIFIC DESCRIPTION OF THIS CHARACTER\'S OWN PERSONALITY, SPEECH PATTERN '
|
||
"AND QUIRKS FROM THE PROFILE ABOVE — NOT A GENERIC PERSONALITY TYPE. BE SPECIFIC TO THIS CHARACTER.}\n\n"
|
||
'{{"Likes"}}:{LIST FROM PROFILE, INFER IF NEEDED}\n\n'
|
||
'{{"Dislikes"}}:{LIST FROM PROFILE, INFER IF NEEDED}\n\n'
|
||
'{{"Goals"}}:{LIST FROM PROFILE, INFER IF NEEDED}\n\n'
|
||
'{{"Skills"}}:{LIST FROM PROFILE, INFER IF NEEDED}\n\n'
|
||
'{{"Weapons"}}:{LIST ONLY IF THIS CHARACTER PLAUSIBLY CARRIES ONE PER THE PROFILE — OMIT THIS FIELD '
|
||
"ENTIRELY OTHERWISE}\n\n"
|
||
f"{outfits}\n"
|
||
"}\n\n"
|
||
"Then, in the SAME string, add a scenario block as clear instructions/definitions for the LLM, not "
|
||
"narration — {{char}}'s relationship with {{user}}, everyday routine, current mood, current plans. Keep "
|
||
"it open-ended (many different stories could start from it) rather than building one specific scene. "
|
||
"Use this exact structure:\n\n"
|
||
'{{"Scenario"}}:{"{{char}} is living everyday life","{{char}} and {{user}} keep crossing each other\'s '
|
||
'paths as {{char}} and {{user}} relationship develops","everyday routine":["mornings":"{{char}} GENERATE",'
|
||
'"days":"{{char}} GENERATE","evenings":"{{char}} GENERATE"],"current mood":"{{char}} GENERATE"]}\n\n'
|
||
"Then add a section literally titled 'First Message:' on its own line, followed by the message itself: "
|
||
"maximum 3 paragraphs, balancing narration with {{char}} dialogue, true to the profile's personality and "
|
||
"the scenario above. Never decide what {{user}} does or says. Avoid describing eyes. Use direct speech "
|
||
"with no markdown for dialogue, and *asterisks* for narration.\n\n"
|
||
"The finished silly_tavern_prompt string MUST contain all three parts, in this order: the {{char}} card "
|
||
"block, the {{\"Scenario\"}} block, and the 'First Message:' section — never stop after the card or the "
|
||
"Scenario alone.\n\n"
|
||
'Respond with STRICT JSON only: {"silly_tavern_prompt":""}/no-think'
|
||
)
|
||
|
||
|
||
@router.post("/api/character-generate-prompts")
|
||
async def character_generate_prompts(request: Request):
|
||
"""Turn an already-extracted character sheet into four ready-to-use external
|
||
prompts, in one LLM call over the character's full profile (rather than
|
||
generating them incrementally passage-by-passage, where the model only
|
||
sees a fraction of the character at a time).
|
||
|
||
Body: {name, book, sheet: {...character sheet fields...}, language, llm_url, model}
|
||
Returns: {voice_design_prompt, image_prompt, silly_tavern_prompt, concept_art_prompt}
|
||
"""
|
||
data = await request.json()
|
||
name: str = (data.get("name") or "").strip()
|
||
book: str = (data.get("book") or "").strip()
|
||
sheet: dict = data.get("sheet") or {}
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
if not name:
|
||
raise HTTPException(400, "Character name required")
|
||
|
||
def _f(key: str) -> str:
|
||
v = sheet.get(key)
|
||
if isinstance(v, list):
|
||
return ", ".join(str(x) for x in v if x)
|
||
return str(v or "").strip()
|
||
|
||
profile = "\n".join(f"{label}: {v}" for label, v in [
|
||
("Name", name), ("Aliases / noble title", _f("aliases") or _f("title")),
|
||
("Occupation", _f("profession")), ("Archetype", _f("archetype")), ("Physical", _f("physical")), ("Clothing", _f("clothing")),
|
||
("Mannerisms", _f("mannerisms")), ("Voice/speech pattern", _f("voice_pattern")),
|
||
("Backstory", _f("backstory")), ("Motivation", _f("motivation")), ("Fears", _f("fears")),
|
||
("Relationships", _f("relationships")), ("Conflict style", _f("conflict_style")),
|
||
("Secret/flaw", _f("secret")), ("Inventory", _f("inventory")),
|
||
] if v)
|
||
|
||
lang_note = f" Write every prompt in {language} EXCEPT where told to use English." if language else ""
|
||
preamble = (
|
||
"You are a prompt engineer who turns a fiction character's profile into ready-to-paste prompts for "
|
||
"other tools. Use ONLY details present in the profile below; mark anything you must reasonably infer "
|
||
f"with a trailing '*'. Never invent plot spoilers not implied by the profile.{lang_note}\n\n"
|
||
)
|
||
user = f"BOOK: {book or 'unspecified'}\n\nCHARACTER PROFILE:\n{profile or name}\n\n"
|
||
|
||
# One independent call per field (not one four-field JSON object, nor even
|
||
# the two-field pairing this used briefly) — the UI now has a Generate
|
||
# button per prompt box, so a click on just one must not also burn tokens
|
||
# regenerating the other three. Singling them out also further shrinks
|
||
# each response, since a four-field JSON object reliably truncated the
|
||
# LAST fields regardless of the token ceiling.
|
||
all_groups = {
|
||
"voice_design_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- voice_design_prompt: an English prompt for Qwen3 TTS Voice Design (25-60 words, one paragraph, "
|
||
"no markdown). Cover: apparent age, gender/androgyny if inferable, pitch, timbre/texture, pace, "
|
||
"accent or register, emotional baseline, and suitability for audiobook dialogue delivery. "
|
||
+ ("State the accent explicitly as an authentic native " + language + " accent — never American-accented English, even though this prompt itself is written in English. " if language and language.lower() != "english" else "State the accent explicitly as a neutral British or international English accent — never American/US-accented. ")
|
||
+
|
||
"Generic category labels alone ('young female voice, energetic tone, clear pitch') describe a whole "
|
||
"demographic, not a person, and different characters who happen to share an age/gender end up "
|
||
"sounding like the same person — confirmed live as a real problem with several young-female "
|
||
"characters in one book. Every one of these fields needs a SPECIFIC, CONCRETE choice, not the safe "
|
||
"generic default: pick a distinctive timbre (breathy/husky/bright/nasal/silvery/gravelly/reedy, not "
|
||
"just 'clear'), a specific pace/rhythm quirk (clipped consonants, unhurried drawl, rapid-fire, "
|
||
"deliberate pauses before key words), and a specific emotional baseline drawn from THIS character's "
|
||
"own personality/backstory (guarded warmth, brittle confidence, weary sarcasm — not just 'friendly' "
|
||
"or 'energetic'). Two characters with the same age and gender in your profile should still end up "
|
||
"with visibly different prompts once you've done this. Do not mention plot events — describe only "
|
||
"how the voice should SOUND.\n\n"
|
||
'Respond with STRICT JSON only: {"voice_design_prompt":""}/no-think'
|
||
),
|
||
"image_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- image_prompt: a detailed English image-generation prompt for a full CHARACTER REFERENCE SHEET "
|
||
"(a single composite image, like a game/animation production turnaround), not just one portrait. "
|
||
"FIRST, from the book title and profile, work out the story's genre, setting, and era (e.g. "
|
||
"'medieval European-inspired high fantasy', 'grimdark low fantasy', 'space opera sci-fi', "
|
||
"'contemporary urban fantasy') — an image model has no idea what a book title implies and will "
|
||
"default to generic modern/real-world imagery unless told explicitly, which is exactly wrong for "
|
||
"an occupation like 'Admiral' or 'General' in a fantasy world (it will draw a 20th-century military "
|
||
"uniform instead of that world's actual equivalent). State that genre/setting/era explicitly, as "
|
||
"its own descriptor near the START of the prompt, and make clear the world has no real-world 20th "
|
||
"or 21st century technology, uniforms, or clothing unless the story is actually confirmed "
|
||
"contemporary/near-future — every other visual choice (armor, dress, rank insignia, weapons) must "
|
||
"fit THAT world, not the real one. Then ask the image model for: (1) a full-body front-view "
|
||
"illustration as the anchor, (2) a turnaround panel with side and back views, (3) a small "
|
||
"expression sheet with 3-5 headshots showing this character's typical emotional range "
|
||
"(calm/determined/etc. — pick expressions that fit their personality), (4) a color palette swatch "
|
||
"panel for hair/eyes/outfit, (5) callouts for their signature props, tools, or clothing details "
|
||
"with short labels. Include age impression, build, hair/eyes/skin if known, and clothing "
|
||
"appropriate to the established setting. Specify a clean production-design/concept-art layout "
|
||
"with a plain neutral background, and explicitly note this is an ORIGINAL character, not based on "
|
||
"any copyrighted character. One dense paragraph, comma-separated descriptors are fine.\n\n"
|
||
'Respond with STRICT JSON only: {"image_prompt":""}/no-think'
|
||
),
|
||
"silly_tavern_prompt": preamble + _silly_tavern_prompt_instruction(sheet),
|
||
"concept_art_prompt": (
|
||
preamble
|
||
+ "Produce exactly this field:\n"
|
||
"- concept_art_prompt: an English image-generation prompt for a production-ready character/NPC "
|
||
"reference sheet, written as labelled clauses in this EXACT order — Task, Subject, Context, Style, "
|
||
"Composition, Lighting, Constraints, Output — each a single sentence, all as one dense paragraph "
|
||
"(not a list). This mirrors a well-tested prompt-engineering pattern for these sheets; follow it "
|
||
"precisely rather than writing free-form:\n"
|
||
" Task: name the sheet type, e.g. 'Generate a character/NPC design sheet.'\n"
|
||
" Subject: 'an original adult [role/archetype from the profile]' plus its 3-6 most visually "
|
||
"distinctive, ALREADY-ESTABLISHED traits (skin/hair, signature garment layers, and — only if the "
|
||
"profile actually gives this character a prop, weapon, or tool — its exact count, e.g. 'exactly one "
|
||
"quiver' or 'exactly two throwing knives'; omit props entirely if the profile has none).\n"
|
||
" Context: one clause on what the sheet is for and the story's genre/setting/era — infer this from "
|
||
"the book/profile the same way you would for a portrait (a fantasy Admiral is NOT a real-world 20th-"
|
||
"century Admiral) and state it explicitly, since an image model defaults to generic modern imagery "
|
||
"otherwise.\n"
|
||
" Style: a concept-art style matching that genre/setting (painterly game concept art / detailed "
|
||
"semi-realistic concept art / hand-painted concept art — pick what fits), grounded materials, clear "
|
||
"shape language.\n"
|
||
" Composition: full-body front, side, and back views across the top; below them, ONE clean row of "
|
||
"isolated callouts for this character's established props/costume components ONLY (skip this row "
|
||
"entirely if the profile establishes no distinct props/costume pieces worth separating out) — state "
|
||
"the exact number of callouts and name each one.\n"
|
||
" Lighting: neutral studio/museum-style light, no cinematic color cast.\n"
|
||
" Constraints: face/silhouette/garment/prop consistency across every view; the exact prop count "
|
||
"restated; no duplicate gear, no extra limbs, no readable text/logos/watermark, no real-world/"
|
||
"franchise references, this is an ORIGINAL character not based on any copyrighted one.\n"
|
||
" Output: one production-ready 3:2 reference sheet.\n\n"
|
||
'Respond with STRICT JSON only: {"concept_art_prompt":""}/no-think'
|
||
),
|
||
}
|
||
requested = [f for f in (data.get("fields") or []) if f in all_groups]
|
||
field_groups = [((f,), all_groups[f]) for f in (requested or all_groups.keys())]
|
||
# The SillyTavern card is a full structured card + scenario + first
|
||
# message now, not a few short labelled lines — needs a much bigger
|
||
# budget than the other three (single-paragraph) prompt fields or it
|
||
# reliably truncates mid-card.
|
||
_field_max_tokens = {"silly_tavern_prompt": 3000}
|
||
|
||
def _call_group(fields: tuple, system: str) -> dict:
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user + f"Generate the {' and '.join(fields)} now."},
|
||
],
|
||
"temperature": 0.7,
|
||
"max_tokens": _field_max_tokens.get(fields[0], 1536),
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
try:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
_msg = resp.json()["choices"][0]["message"]
|
||
raw = (_msg.get("content") or _reasoning_text(_msg) or "").strip()
|
||
except Exception as e:
|
||
print(f"[character-generate-prompts] LLM call failed for {fields}: {e}")
|
||
return {}
|
||
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
block = _extract_json_block(content)
|
||
# If the model still got cut off mid-string, closing the dangling string +
|
||
# object often salvages the field that DID complete.
|
||
candidates = [content, block]
|
||
if block:
|
||
candidates.extend([block + "\"}", block + "}"])
|
||
for cand in candidates:
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and fields[0] in parsed:
|
||
return parsed
|
||
except Exception:
|
||
continue
|
||
return {}
|
||
|
||
results = await asyncio.gather(*[
|
||
asyncio.to_thread(_call_group, fields, system) for fields, system in field_groups
|
||
])
|
||
out = {}
|
||
for (fields, _), result in zip(field_groups, results):
|
||
for f in fields:
|
||
out[f] = str(result.get(f) or "").strip()
|
||
if not any(out.values()):
|
||
# A silent all-empty response looked identical to success in the UI —
|
||
# fail loudly so the client can show a real error instead.
|
||
raise HTTPException(502, "LLM returned no parseable prompts — try again (or a different model)")
|
||
return out
|
||
|
||
|
||
@router.post("/api/audiobook-consistency-check")
|
||
async def audiobook_consistency_check(request: Request):
|
||
"""Check whether every line already attributed to ONE character across the
|
||
whole book actually sounds like them — using the character's OWN other
|
||
lines as the reference, not a chunk-local judgement. Unlike the casting/
|
||
verification passes (which re-read the source text chunk by chunk), this
|
||
works purely over already-attributed lines gathered from anywhere in the
|
||
book, so it can catch a character who briefly "borrows" someone else's
|
||
voice in a way no single passage-sized chunk would ever expose.
|
||
|
||
Body: {character, lines: [{index, text}], known_characters: [...], llm_url, model}
|
||
Returns: {outliers: [{index, reason, suggested_speaker}]}
|
||
"""
|
||
data = await request.json()
|
||
character: str = (data.get("character") or "").strip()
|
||
lines: list = data.get("lines") or []
|
||
known: list = data.get("known_characters") or []
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
if not character:
|
||
raise HTTPException(400, "Character name required")
|
||
if not lines:
|
||
raise HTTPException(400, "No lines to check")
|
||
|
||
numbered = "\n".join(
|
||
f"[{l.get('index')}] {str(l.get('text') or '').strip()}"
|
||
for l in lines if str(l.get("text") or "").strip()
|
||
)
|
||
known_note = (
|
||
f"\n\nOther characters already recognized in this book (only suggest one of these, or 'Unknown' — "
|
||
f"never invent a new name): {', '.join(str(n) for n in known)}"
|
||
) if known else ""
|
||
|
||
system = (
|
||
f"You are a dramaturge auditing dialogue attribution in a novel already cast for audiobook production. "
|
||
f"Below are ALL the lines currently attributed to ONE character, '{character}', gathered from across the "
|
||
f"whole book (not necessarily consecutive). Your job: read them as a whole and judge whether each line "
|
||
f"actually sounds like the SAME person speaking — same tone, vocabulary, register, and personality — or "
|
||
f"whether one or more lines sound like they were misattributed from someone else (a different tone, "
|
||
f"formality, vocabulary, or a statement that contradicts what the rest of {character}'s lines establish "
|
||
f"about them).\n\n"
|
||
f"Be conservative: most lines are correctly attributed. Only flag a line if it genuinely reads like a "
|
||
f"different voice compared to the REST of {character}'s own lines here — not just because it's short, "
|
||
f"blunt, or otherwise unremarkable.{known_note}\n\n"
|
||
'Respond with STRICT JSON only: {"outliers":[{"index":0,"quote":"","reason":"","suggested_speaker":""}]}\n'
|
||
"index: the EXACT number shown in [square brackets] right before the flagged line below — copy that "
|
||
"number verbatim, do NOT count lines yourself or renumber them (the brackets are the book's real line "
|
||
"numbers, not a 0/1/2/3 sequence). quote: the first few words of the flagged line, verbatim, so the "
|
||
"index can be double-checked. reason: one short sentence explaining why this line doesn't fit. "
|
||
"suggested_speaker: your best guess at who actually said it (exact name from the list above), or "
|
||
"'Unknown' if you can't tell. Omit any line that isn't an outlier — do not list every line.\n/no-think"
|
||
)
|
||
user = f"{character}'s lines (index — text):\n{numbered}"
|
||
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.3,
|
||
"max_tokens": 2048,
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
|
||
def _call() -> dict:
|
||
resp = requests.post(
|
||
f"{llm_url}/chat/completions", json=payload,
|
||
headers={"Authorization": f"Bearer {_settings.get('llm_api_key') or 'sk-dummy-key'}"}, timeout=600,
|
||
)
|
||
resp.raise_for_status()
|
||
msg = resp.json()["choices"][0]["message"]
|
||
raw = (msg.get("content") or _reasoning_text(msg) or "").strip()
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
for cand in (content, _extract_json_block(content)):
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("outliers"), list):
|
||
return parsed
|
||
except Exception:
|
||
continue
|
||
return {"outliers": []}
|
||
|
||
try:
|
||
result = await asyncio.to_thread(_call)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Consistency check failed: {e}")
|
||
|
||
clean = []
|
||
for o in result.get("outliers", []):
|
||
if not isinstance(o, dict):
|
||
continue
|
||
try:
|
||
idx = int(o.get("index"))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
clean.append({
|
||
"index": idx,
|
||
"quote": str(o.get("quote") or "").strip()[:120],
|
||
"reason": str(o.get("reason") or "").strip()[:200],
|
||
"suggested_speaker": str(o.get("suggested_speaker") or "").strip(),
|
||
})
|
||
return {"outliers": clean}
|
||
|
||
|
||
def _comfyui_run(comfyui_url: str, workflow_json: str, prompt_node_id: str,
|
||
prompt_field: str, output_node_id: str, prompt: str) -> str:
|
||
"""Submits a pre-exported ComfyUI API-format workflow with the given
|
||
prompt text injected into one designated node/field, waits for it to
|
||
finish, and returns the resulting image as a data: URI.
|
||
|
||
The workflow itself is opaque to us — the user exports it from ComfyUI's
|
||
own UI (Workflow > Export (API Format)), since converting the editor's
|
||
graph format ourselves risks silently mis-wiring virtual-routing addons
|
||
(e.g. rgthree Get/Set nodes) that don't show up as real graph edges.
|
||
"""
|
||
if not comfyui_url:
|
||
raise HTTPException(400, "ComfyUI URL not set (Settings > Engines > Image Generation)")
|
||
if not workflow_json:
|
||
raise HTTPException(400, "No ComfyUI workflow configured — paste an API-format workflow export in Settings > Engines > Image Generation")
|
||
if not prompt_node_id or not output_node_id:
|
||
raise HTTPException(400, "ComfyUI prompt/output node IDs not set (Settings > Engines > Image Generation)")
|
||
try:
|
||
workflow = json.loads(workflow_json)
|
||
except Exception as e:
|
||
raise HTTPException(400, f"Saved ComfyUI workflow isn't valid JSON: {e}")
|
||
if prompt_node_id not in workflow:
|
||
raise HTTPException(400, f"Prompt node id '{prompt_node_id}' not found in the saved workflow")
|
||
if output_node_id not in workflow:
|
||
raise HTTPException(400, f"Output node id '{output_node_id}' not found in the saved workflow")
|
||
|
||
graph = copy.deepcopy(workflow)
|
||
graph[prompt_node_id].setdefault("inputs", {})[prompt_field or "text"] = prompt
|
||
client_id = str(uuid.uuid4())
|
||
base = comfyui_url.rstrip("/")
|
||
|
||
try:
|
||
resp = requests.post(f"{base}/prompt", json={"prompt": graph, "client_id": client_id}, timeout=30)
|
||
if not resp.ok:
|
||
detail = resp.text[:500]
|
||
try:
|
||
detail = resp.json().get("error", {}).get("message", detail)
|
||
except Exception:
|
||
pass
|
||
raise HTTPException(resp.status_code, f"ComfyUI rejected the workflow: {detail}")
|
||
prompt_id = resp.json().get("prompt_id")
|
||
if not prompt_id:
|
||
raise HTTPException(502, "ComfyUI didn't return a prompt_id")
|
||
except HTTPException:
|
||
raise
|
||
except requests.exceptions.ConnectionError:
|
||
raise HTTPException(502, f"Cannot reach ComfyUI at {base} — is the container running?")
|
||
except Exception as e:
|
||
raise HTTPException(502, f"ComfyUI submission failed: {e}")
|
||
|
||
# Poll history — generation on a real workflow (multi-sampler, upscale,
|
||
# etc.) can take minutes, so this waits longer than a typical API call.
|
||
deadline = time.time() + 300
|
||
history = None
|
||
while time.time() < deadline:
|
||
try:
|
||
hr = requests.get(f"{base}/history/{prompt_id}", timeout=10)
|
||
if hr.ok:
|
||
data = hr.json()
|
||
if prompt_id in data:
|
||
history = data[prompt_id]
|
||
status = history.get("status", {})
|
||
if status.get("completed") is True or status.get("status_str") == "success":
|
||
break
|
||
if status.get("status_str") == "error":
|
||
msgs = status.get("messages", [])
|
||
raise HTTPException(502, f"ComfyUI generation failed: {msgs[-1] if msgs else 'unknown error'}")
|
||
except HTTPException:
|
||
raise
|
||
except Exception:
|
||
pass
|
||
time.sleep(2)
|
||
if not history:
|
||
raise HTTPException(504, "ComfyUI generation timed out after 5 minutes")
|
||
|
||
outputs = history.get("outputs", {}).get(output_node_id, {})
|
||
images = outputs.get("images") or []
|
||
if not images:
|
||
raise HTTPException(502, f"Output node '{output_node_id}' produced no images — check it's the right SaveImage/PreviewImage node id")
|
||
img = images[0]
|
||
try:
|
||
vr = requests.get(f"{base}/view", params={
|
||
"filename": img.get("filename", ""), "subfolder": img.get("subfolder", ""), "type": img.get("type", "output"),
|
||
}, timeout=30)
|
||
vr.raise_for_status()
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Fetching the generated image from ComfyUI failed: {e}")
|
||
b64 = base64.b64encode(vr.content).decode("ascii")
|
||
mime = vr.headers.get("Content-Type", "image/png")
|
||
return f"data:{mime};base64,{b64}"
|
||
|
||
|
||
@router.post("/api/character-image-from-url")
|
||
async def character_image_from_url(request: Request):
|
||
"""Download an image from a user-supplied URL server-side and return it
|
||
as a data: URI, so pasting a link works the same as an upload — fetching
|
||
an arbitrary third-party image directly from the browser would usually
|
||
fail on CORS, since most image hosts don't send permissive headers.
|
||
|
||
Body: {url}
|
||
Returns: {image: "data:image/...;base64,...."}
|
||
"""
|
||
data = await request.json()
|
||
url: str = (data.get("url") or "").strip()
|
||
if not url:
|
||
raise HTTPException(400, "Image URL required")
|
||
try:
|
||
resp = requests.get(url, timeout=30, headers={"User-Agent": "TTS-Voice-Creator/image-fetch"}, stream=True)
|
||
resp.raise_for_status()
|
||
ctype = resp.headers.get("Content-Type", "")
|
||
if not ctype.startswith("image/"):
|
||
raise HTTPException(400, f"That URL didn't return an image (got {ctype or 'unknown content type'})")
|
||
content = resp.raw.read(15 * 1024 * 1024, decode_content=True)
|
||
if not content:
|
||
raise HTTPException(502, "Empty response from that URL")
|
||
b64 = base64.b64encode(content).decode("ascii")
|
||
return {"image": f"data:{ctype};base64,{b64}"}
|
||
except HTTPException:
|
||
raise
|
||
except requests.exceptions.RequestException as e:
|
||
raise HTTPException(502, f"Couldn't fetch that URL: {e}")
|
||
|
||
|
||
@router.post("/api/character-generate-image")
|
||
async def character_generate_image(request: Request):
|
||
"""Generate a character profile picture from a text prompt via a cloud
|
||
image-gen provider (OpenAI, Google, OpenRouter) or a local ComfyUI
|
||
workflow.
|
||
|
||
Body: {prompt, provider?, model?} — provider/model default to the
|
||
Settings > Engines > Image Generation choice if not passed explicitly.
|
||
Returns: {image: "data:image/png;base64,...."}
|
||
"""
|
||
data = await request.json()
|
||
prompt: str = (data.get("prompt") or "").strip()
|
||
if not prompt:
|
||
raise HTTPException(400, "Image prompt required")
|
||
|
||
_settings = _load_settings()
|
||
keys: dict = _settings.get("engine_api_keys") or {}
|
||
provider: str = (data.get("provider") or _settings.get("image_gen_provider") or "").strip().lower()
|
||
model: str = (data.get("model") or _settings.get("image_gen_model") or "").strip()
|
||
|
||
if not provider:
|
||
raise HTTPException(400, "No image generation provider configured — set one in Settings > Engines > Image Generation")
|
||
|
||
# Every branch below does blocking requests.post/get (plus, for ComfyUI, a
|
||
# polling loop with time.sleep for up to 5 minutes on a real multi-stage
|
||
# workflow) with no asyncio.to_thread wrapper — unlike every other
|
||
# blocking call in this file. On a single-worker uvicorn process (see
|
||
# server.py) that blocked the ENTIRE event loop: every other user's
|
||
# request (TTS, page loads, /api/characters) would hang unresponsive for
|
||
# as long as one image generation took, which for a bulk "auto-generate
|
||
# images" run across a whole cast could be tens of minutes of site-wide
|
||
# freeze with no error, just silence. Runs in a worker thread instead.
|
||
return await asyncio.to_thread(_character_generate_image_sync, provider, model, prompt, keys, _settings)
|
||
|
||
|
||
def _character_generate_image_sync(provider: str, model: str, prompt: str, keys: dict, _settings: dict) -> dict:
|
||
if provider == "openai":
|
||
api_key = (keys.get("openai_image") or "").strip()
|
||
if not api_key:
|
||
raise HTTPException(400, "OpenAI API key not set (Settings > Engines > Image Generation)")
|
||
try:
|
||
resp = requests.post(
|
||
"https://api.openai.com/v1/images/generations",
|
||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
json={"model": model or "gpt-image-1", "prompt": prompt, "size": "1024x1024", "n": 1},
|
||
timeout=120,
|
||
)
|
||
if not resp.ok:
|
||
detail = resp.text[:400]
|
||
try:
|
||
detail = resp.json().get("error", {}).get("message", detail)
|
||
except Exception:
|
||
pass
|
||
raise HTTPException(resp.status_code, f"OpenAI image generation failed: {detail}")
|
||
b64 = resp.json()["data"][0]["b64_json"]
|
||
return {"image": f"data:image/png;base64,{b64}"}
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(502, f"OpenAI image generation failed: {e}")
|
||
|
||
if provider == "google":
|
||
api_key = (keys.get("google_image") or "").strip()
|
||
if not api_key:
|
||
raise HTTPException(400, "Google API key not set (Settings > Engines > Image Generation)")
|
||
gmodel = model or "gemini-2.5-flash-image"
|
||
try:
|
||
resp = requests.post(
|
||
f"https://generativelanguage.googleapis.com/v1beta/models/{gmodel}:generateContent",
|
||
params={"key": api_key},
|
||
json={"contents": [{"parts": [{"text": prompt}]}]},
|
||
timeout=120,
|
||
)
|
||
if not resp.ok:
|
||
detail = resp.text[:400]
|
||
try:
|
||
detail = resp.json().get("error", {}).get("message", detail)
|
||
except Exception:
|
||
pass
|
||
raise HTTPException(resp.status_code, f"Google image generation failed: {detail}")
|
||
parts = (resp.json().get("candidates") or [{}])[0].get("content", {}).get("parts", [])
|
||
inline = next((p.get("inlineData") for p in parts if p.get("inlineData")), None)
|
||
if not inline:
|
||
raise HTTPException(502, "Google returned no image data — try again or rephrase the prompt")
|
||
mime = inline.get("mimeType", "image/png")
|
||
return {"image": f"data:{mime};base64,{inline['data']}"}
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Google image generation failed: {e}")
|
||
|
||
if provider == "openrouter":
|
||
# Reuses the same key as the OpenRouter LLM card (Settings > Engines >
|
||
# Language Models) — OpenRouter serves image-output models through
|
||
# the same chat/completions endpoint and account as text models,
|
||
# unlike OpenAI/Google where images are a separate API surface with
|
||
# their own key.
|
||
api_key = (keys.get("openrouter") or "").strip()
|
||
if not api_key:
|
||
raise HTTPException(400, "OpenRouter API key not set (Settings > Engines > Language Models > OpenRouter)")
|
||
omodel = model or "google/gemini-2.5-flash-image-preview:free"
|
||
try:
|
||
resp = requests.post(
|
||
"https://openrouter.ai/api/v1/chat/completions",
|
||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
json={"model": omodel, "messages": [{"role": "user", "content": prompt}], "modalities": ["image", "text"]},
|
||
timeout=120,
|
||
)
|
||
if not resp.ok:
|
||
detail = resp.text[:400]
|
||
try:
|
||
detail = resp.json().get("error", {}).get("message", detail)
|
||
except Exception:
|
||
pass
|
||
raise HTTPException(resp.status_code, f"OpenRouter image generation failed: {detail}")
|
||
msg = (resp.json().get("choices") or [{}])[0].get("message", {})
|
||
# OpenRouter returns generated images in message.images (not the
|
||
# OpenAI Images API shape) — each entry is
|
||
# {"type": "image_url", "image_url": {"url": "data:...;base64,..."}}.
|
||
images = msg.get("images") or []
|
||
url = next((im.get("image_url", {}).get("url") for im in images if im.get("image_url", {}).get("url")), None)
|
||
if not url:
|
||
# Some models inline the image as a data URI in the text content instead.
|
||
content = msg.get("content")
|
||
text = content if isinstance(content, str) else " ".join(
|
||
p.get("text", "") for p in (content or []) if isinstance(p, dict)
|
||
)
|
||
m = re.search(r"data:image/\w+;base64,[A-Za-z0-9+/=]+", text or "")
|
||
url = m.group(0) if m else None
|
||
if not url:
|
||
raise HTTPException(502, "OpenRouter returned no image data — this model may not support image output, try another")
|
||
return {"image": url}
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(502, f"OpenRouter image generation failed: {e}")
|
||
|
||
if provider == "comfyui":
|
||
image = _comfyui_run(
|
||
(_settings.get("comfyui_url") or "").strip(),
|
||
_settings.get("comfyui_workflow") or "",
|
||
(_settings.get("comfyui_prompt_node_id") or "").strip(),
|
||
(_settings.get("comfyui_prompt_field") or "text").strip(),
|
||
(_settings.get("comfyui_output_node_id") or "").strip(),
|
||
prompt,
|
||
)
|
||
return {"image": image}
|
||
|
||
if provider == "pollinations":
|
||
# Pollinations.ai — free, no API key, no account. A GET request that
|
||
# returns the image directly. Third-party public service: no SLA, no
|
||
# control over the model behind it, prompts leave the server — fine
|
||
# as a free stopgap, not a permanent guarantee.
|
||
pmodel = model or "flux"
|
||
last_err = ""
|
||
# The free queue (50 concurrent slots server-wide, shared across all
|
||
# of Pollinations' users) fills up under load and rejects with a
|
||
# transient error rather than queueing — a short retry clears most
|
||
# of these without the user having to click "Generate" again.
|
||
for attempt in range(3):
|
||
try:
|
||
resp = requests.get(
|
||
f"https://image.pollinations.ai/prompt/{quote(prompt)}",
|
||
params={"model": pmodel, "width": 1024, "height": 1024, "nologo": "true"},
|
||
timeout=120,
|
||
)
|
||
if resp.ok and resp.content:
|
||
b64 = base64.b64encode(resp.content).decode("ascii")
|
||
mime = resp.headers.get("Content-Type", "image/jpeg")
|
||
return {"image": f"data:{mime};base64,{b64}"}
|
||
last_err = resp.text[:300]
|
||
except Exception as e:
|
||
last_err = str(e)
|
||
if attempt < 2:
|
||
time.sleep(4)
|
||
raise HTTPException(502, f"Pollinations.ai request failed after retries: {last_err}")
|
||
|
||
raise HTTPException(400, f"Unknown image generation provider: {provider}")
|
||
|
||
|
||
def _attribution_prepare(data: dict) -> dict:
|
||
"""Resolve settings and build the chat payload for one attribution request.
|
||
Shared by the blocking endpoint and the streaming (watch-the-LLM-think)
|
||
endpoint so the prompt logic can never drift between the two."""
|
||
text: str = (data.get("text") or "").strip()
|
||
known: list = data.get("known_characters") or []
|
||
recent: str = (data.get("recent") or "").strip() # last few attributed lines, for continuity
|
||
language: str = (data.get("language") or "").strip()
|
||
_settings = _load_settings()
|
||
llm_url: str = (data.get("llm_url") or _settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
model: str = (data.get("model") or _settings.get("llm_model") or "").strip()
|
||
timeout_seconds = _request_timeout_seconds(data.get("timeout_seconds"), 600.0)
|
||
if not text:
|
||
raise HTTPException(400, "No text provided")
|
||
|
||
lang_hint = f" The text language is {language}; keep names and wording in that language." if language else ""
|
||
|
||
prompt_override = data.get("audiobook_prompt")
|
||
if not isinstance(prompt_override, str):
|
||
prompt_override = data.get("prompt")
|
||
base_prompt = (prompt_override if isinstance(prompt_override, str) else "") or _settings.get("audiobook_prompt") or ""
|
||
# A user-saved custom prompt can predate (or have replaced) the deduction
|
||
# rules that cut down false Unknown/Narrator attributions — the client-side
|
||
# prompt migration only upgrades prompts still containing the original
|
||
# anchor text. Guarantee the rules ride along regardless of what prompt is
|
||
# in play, so attribution quality never silently depends on prompt history.
|
||
if base_prompt.strip() and "Doppelpunkt-Regel" not in base_prompt:
|
||
base_prompt += (
|
||
"\n\nANALYSE-REGELN FÜR DIE ZUORDNUNG DES SPRECHERS (zusätzlich, immer anwenden):\n"
|
||
"1. Pronomen (er/sie) IMMER zum zuletzt genannten Charakter passenden Geschlechts auflösen.\n"
|
||
"2. Doppelpunkt-Regel: Endet ein Erzählersatz mit \":\", spricht dessen Subjekt das folgende Zitat "
|
||
"(z.B. \"Dann richtete er sich auf und rief in die Runde:\" → der zuvor genannte Charakter spricht).\n"
|
||
"3. Nachgestellte Zuordnung: Der Erzählersatz NACH einem Zitat verrät oft den Sprecher — auch bei "
|
||
"unpersönlicher Formel (\"»Was machst du da?« ertönte es über ihm. Karyla war herübergekommen.\" → Karyla sprach).\n"
|
||
"4. Adressaten-Regel: \"X wandte sich an Y\" / \"X sah Y an\" → X spricht das nächste Zitat, Y antwortet.\n"
|
||
"5. Ping-Pong: Zwei Personen im Gespräch wechseln sich strikt ab, auch über viele Zitate ohne Tags. "
|
||
"In einer Zwei-Personen-Szene ist 'Unknown' fast immer falsch.\n"
|
||
"6. Rollenbezeichnungen sind gültige Sprecher ('Ork', 'Der Fremde', 'Nachbar', 'Wächter') — nutze sie statt 'Unknown'.\n"
|
||
"7. 'Unknown' NUR, wenn eine Zuordnung trotz aller Regeln absolut unmöglich ist.\n"
|
||
"8. Gedankenstrich-Pause-Regel: Ein \" - \" MITTEN in einem Zitat ist eine Sprechpause DESSELBEN Sprechers, "
|
||
"kein Zitatende — die Rede desselben Charakters geht danach unverändert weiter, bis das tatsächliche "
|
||
"schließende Anführungszeichen erscheint.\n"
|
||
"9. Stimm-Ankündigung: Erwähnt ein Erzählersatz kurz vor einer noch nicht zugeordneten Zeile explizit die "
|
||
"Stimme oder das (beginnende) Sprechen einer bestimmten Person — auch in indirekter/idiomatischer Form, "
|
||
"nicht nur mit einem wörtlichen Sprechverb (z.B. \"Marcians Stimme wirkte nicht mehr so fest\", \"X setzte "
|
||
"zum Sprechen an\", \"X fand als erster seine Stimme wieder\", \"ihre ersten Worte waren\", \"X brach das "
|
||
"Schweigen\"), gehört die folgende Zeile dieser Person — nicht 'Unknown'.\n"
|
||
"10. Selbstvorstellung: Nennt eine Zitat-Zeile selbst den Namen der sprechenden Person als Vorstellung "
|
||
"(z.B. \"Man nennt mich Andra\", \"Ich bin X\", \"Mein Name ist X\", \"Ich heiße X\"), ist diese genannte "
|
||
"Person die Sprecherin dieser Zeile — nicht 'Unknown', auch ohne separaten Sprecher-Tag."
|
||
)
|
||
if not base_prompt.strip():
|
||
base_prompt = (
|
||
"You attribute dialogue in prose fiction for a multi-voice audiobook. "
|
||
"Split the passage into consecutive segments in reading order. For each segment output:\n"
|
||
"- speaker: 'Narrator' for narration/description, or the character's name for spoken dialogue. "
|
||
"Use the exact English word 'Unknown' ONLY as an absolute last resort.\n"
|
||
"- type: 'narration' or 'dialogue'\n"
|
||
"- text: the verbatim spoken words for dialogue (WITHOUT the surrounding quotation marks), or the verbatim prose for narration\n"
|
||
"- emotion: for dialogue, one or two words in the SAME LANGUAGE as the text (e.g. for German: wütend, traurig, flüsternd); '' for narration\n"
|
||
f"{lang_hint}\n"
|
||
"QUOTATION STYLES — books mark speech in many ways; treat ALL of these as spoken dialogue:\n"
|
||
" English straight \"...\" and curly “...”; German »...« (guillemets pointing inward) and „...“; "
|
||
"French «...» (pointing outward); single ‘...’; CJK 「...」 『...』; and em-dash speech where a line "
|
||
"starts with — or – (Spanish/French/Polish style).\n"
|
||
"PARAGRAPH/CHAPTER STRUCTURE: a blank line marks a real paragraph break or a chapter/scene start. "
|
||
"A very short standalone line right before a blank line (e.g. 'Chapter 1', 'Prologue', a bare number) "
|
||
"is a chapter heading — always narration/'Narrator', never dialogue. Keep blank lines as part of the "
|
||
"surrounding narration segment or their own narration segment; never invent dialogue from them and "
|
||
"never drop them from the reconstructed text.\n"
|
||
"German guillemets are the MOST IMPORTANT to detect: »Was schaust du dir an?« is a spoken line.\n"
|
||
"ATTRIBUTING THE SPEAKER (this is the hard, important part — be decisive):\n"
|
||
"1. If there is a dialogue tag ('sagte Riskan', 'fragte sie', 'Peter said'), use it. Resolve pronouns "
|
||
"(er/sie/he/she) to the actual name from nearby context.\n"
|
||
"2. UNTAGGED lines: use **conversational turn-taking**. In a two-person exchange the speaker ALTERNATES "
|
||
"every line — if Riskan just spoke, the next untagged quote is the other person, then back to Riskan, and so on.\n"
|
||
"3. Use the scene context, action beats around a quote (the person doing the action usually speaks), the "
|
||
"'Recent dialogue' below (continue the same conversation/alternation across the passage boundary), and the "
|
||
"known-characters list. Reuse the EXACT known names.\n"
|
||
"4. Only output 'Unknown' if the speaker is genuinely indeterminable even after applying turn-taking and "
|
||
"context — this should be rare. Prefer the most likely named character over 'Unknown'.\n"
|
||
"RULES:\n"
|
||
"- Put dialogue tags and action beats in a NARRATION segment, never inside the dialogue text.\n"
|
||
"- GRAMMAR CHECK FOR NARRATION: if a fragment is a reporting clause about speech "
|
||
"(finite speech verb + speaker/pronoun/name, e.g. 'murmelte er mit erstickter Stimme', "
|
||
"', entgegnete Marcian kalt', 'asked Peter quietly'), it is narration, never dialogue.\n"
|
||
"- Action beats are narration: a character looks, walks, laughs, stays silent, raises a hand, "
|
||
"turns away, etc. Only the actual quoted words are dialogue.\n"
|
||
"- If a quote is interrupted by a tag (»Die Pause«, sagte Peter, »ist vorbei.«), stitch the spoken parts "
|
||
"into ONE dialogue segment ('Die Pause ist vorbei.') with the tag as a separate narration segment.\n"
|
||
"- Strip the quotation marks/guillemets from dialogue text. Keep every word otherwise, in order.\n"
|
||
"- DO NOT hallucinate, summarize, or alter the text. The combined text of your segments MUST exactly match the original passage, word for word, except for dropped quotation marks.\n"
|
||
"- MID-QUOTE DASH RULE: a \" - \" (em-dash/hyphen used as a pause) in the MIDDLE of a quotation does NOT "
|
||
"end it — the SAME speaker's line continues unchanged after the dash, until the actual closing quotation "
|
||
"mark appears. Do not split it into narration or a new speaker at the dash.\n"
|
||
"- VOICE-ANNOUNCEMENT RULE: if narration mentions a specific character's voice or that they are about to "
|
||
"speak, shortly before an unattributed line, attribute that line to that character instead of 'Unknown' "
|
||
"— this includes indirect/idiomatic phrasing, not just a literal speech verb (e.g. \"Marcian's voice "
|
||
"sounded...\", \"X began to say\", \"X found his voice first\", \"her first words were\", \"X broke the "
|
||
"silence\").\n"
|
||
"- SELF-INTRODUCTION RULE: if a quoted line itself names the speaker as an introduction (e.g. \"They "
|
||
"call me Andra\", \"I am X\", \"My name is X\"), that named person is the speaker of THIS line — never "
|
||
"'Unknown', even with no separate speaker tag."
|
||
)
|
||
else:
|
||
if lang_hint:
|
||
base_prompt += f"\n\n{lang_hint}"
|
||
|
||
# Only the streaming ("watch it think") call asks for a reasoning preamble —
|
||
# it costs extra tokens/latency, which the blocking/fallback path can't
|
||
# afford when it's already the retry-in-halves path for a slow model.
|
||
if data.get("want_reasoning"):
|
||
system = (
|
||
f"{base_prompt}\n\n"
|
||
"First, in a <think>...</think> block, briefly reason (2-4 short sentences) about who speaks each "
|
||
"untagged or ambiguous quote in this passage — mention the names/pronouns you're resolving. Keep it short.\n"
|
||
"Then, after </think>, respond with a single, valid JSON object containing a 'segments' array. "
|
||
"Nothing else outside the <think> block and the JSON object:\n"
|
||
'{"segments":[{"speaker":"Narrator","type":"narration","text":"...","emotion":""}]}'
|
||
)
|
||
else:
|
||
system = (
|
||
f"{base_prompt}\n\n"
|
||
"You MUST respond with a single, valid JSON object containing a 'segments' array.\n"
|
||
"Do NOT output any reasoning, chain of thought, or conversational text. Output ONLY the raw JSON object:\n"
|
||
'{"segments":[{"speaker":"Narrator","type":"narration","text":"...","emotion":""}]}'
|
||
)
|
||
user = (
|
||
("Known characters so far: " + ", ".join(str(n) for n in known) + "\n\n" if known else "")
|
||
+ ("Recent dialogue (the immediately preceding lines — continue the same conversation/turn-taking):\n" + recent + "\n\n" if recent else "")
|
||
+ "Passage to attribute:\n" + text + "\n\n"
|
||
+ "Return ONLY valid JSON. Do not include markdown blocks or any other text."
|
||
)
|
||
payload: dict = {
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
],
|
||
"temperature": 0.2,
|
||
# Dialogue-dense passages need much more output than input: every quoted
|
||
# line becomes a full JSON object ({"speaker":...,"type":...,"text":...,
|
||
# "emotion":...}), so budgeting off raw character count alone (as a
|
||
# tighter cap here once did) truncates mid-JSON on exchange-heavy scenes
|
||
# — which the parser can't always repair, degrading the whole chunk to
|
||
# naive quote-splitting with every speaker labelled "Unknown".
|
||
"max_tokens": min(8192, max(2048, len(text) + 1000)) + (250 if data.get("want_reasoning") else 0),
|
||
}
|
||
if model:
|
||
payload["model"] = model
|
||
return {
|
||
"payload": payload, "llm_url": llm_url, "model": model,
|
||
"api_key": _settings.get("llm_api_key") or "sk-dummy-key",
|
||
"timeout_seconds": timeout_seconds, "text": text,
|
||
"fallback_model": (_settings.get("llm_model") or "").strip(),
|
||
}
|
||
|
||
|
||
@router.post("/api/attribute-dialogue")
|
||
async def attribute_dialogue(request: Request):
|
||
"""Split a prose passage into attributed segments for a multi-voice audiobook.
|
||
|
||
Body: {text, known_characters:[...], language, llm_url, model}
|
||
Returns: {segments:[{speaker, type:"narration"|"dialogue", text, emotion}], characters:[names]}
|
||
The frontend calls this per chunk, passing the running character roster so the
|
||
same speaker keeps the same name across the whole book.
|
||
"""
|
||
data = await request.json()
|
||
prep = _attribution_prepare(data)
|
||
payload, llm_url, model = prep["payload"], prep["llm_url"], prep["model"]
|
||
timeout_seconds, text = prep["timeout_seconds"], prep["text"]
|
||
api_key = prep["api_key"]
|
||
try:
|
||
if not await asyncio.to_thread(_attribution_llm_lock.acquire, True, timeout_seconds):
|
||
return _fallback_attribute_response(text)
|
||
try:
|
||
resp = await asyncio.to_thread(
|
||
_post_llm_chat_completion,
|
||
llm_url, payload,
|
||
{"Authorization": f"Bearer {api_key}"}, timeout_seconds,
|
||
)
|
||
fallback_model = prep["fallback_model"]
|
||
if (
|
||
resp.status_code >= 400
|
||
and _is_router_model_alias(model)
|
||
and fallback_model
|
||
and fallback_model != model
|
||
and not _is_router_model_alias(fallback_model)
|
||
):
|
||
retry_payload = dict(payload)
|
||
retry_payload["model"] = fallback_model
|
||
resp = await asyncio.to_thread(
|
||
_post_llm_chat_completion,
|
||
llm_url, retry_payload,
|
||
{"Authorization": f"Bearer {api_key}"}, timeout_seconds,
|
||
)
|
||
payload = retry_payload
|
||
finally:
|
||
_attribution_llm_lock.release()
|
||
if resp.status_code >= 400:
|
||
print(
|
||
f"[attribute-dialogue] LLM failed ({resp.status_code}) for model "
|
||
f"{payload.get('model') or '(default)'}: {_response_error_text(resp)}"
|
||
)
|
||
return _fallback_attribute_response(text)
|
||
msg = resp.json()["choices"][0]["message"]
|
||
raw = (msg.get("content") or _reasoning_text(msg) or "").strip()
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[attribute-dialogue] LLM failed, using deterministic fallback: {e}")
|
||
return _fallback_attribute_response(text)
|
||
|
||
return _attribution_parse(raw, text)
|
||
|
||
|
||
def _attribution_parse(raw: str, text: str) -> dict:
|
||
"""Turn the LLM's raw answer into normalised {segments, characters}."""
|
||
content = re.sub(r"<think>.*?</think>", "", raw, flags=re.DOTALL).strip() or raw
|
||
segments = []
|
||
|
||
block = _extract_json_block(content)
|
||
candidates = [content, block]
|
||
if block:
|
||
# If the JSON was truncated by token limits, appending ]} to the last complete object often salvages it.
|
||
candidates.append(block + "]}")
|
||
|
||
for cand in candidates:
|
||
if not cand:
|
||
continue
|
||
try:
|
||
parsed = json.loads(cand)
|
||
if isinstance(parsed, dict) and isinstance(parsed.get("segments"), list):
|
||
segments = parsed["segments"]
|
||
break
|
||
except Exception:
|
||
continue
|
||
|
||
if not segments:
|
||
# If the LLM completely failed to output JSON despite the strict system prompt,
|
||
# it means it dropped into a conversational/reasoning hallucination.
|
||
# Do NOT try to parse its reasoning as a script. Abort and preserve the book text.
|
||
segments = [{"speaker": "Narrator", "type": "narration", "text": text, "emotion": ""}]
|
||
|
||
# Normalise + collect speaker roster
|
||
clean, chars = [], []
|
||
for seg in segments:
|
||
if not isinstance(seg, dict):
|
||
continue
|
||
t = str(seg.get("text") or "").strip()
|
||
if not t:
|
||
continue
|
||
sp = str(seg.get("speaker") or "Narrator").strip() or "Narrator"
|
||
typ = "dialogue" if str(seg.get("type") or "").lower().startswith("dial") else "narration"
|
||
if typ == "narration":
|
||
sp = "Narrator"
|
||
emo = str(seg.get("emotion") or "").strip()
|
||
clean.append({"speaker": sp, "type": typ, "text": t, "emotion": emo})
|
||
if typ == "dialogue" and sp.lower() != "narrator" and sp not in chars:
|
||
chars.append(sp)
|
||
return {"segments": clean, "characters": chars}
|
||
|
||
|
||
@router.post("/api/attribute-dialogue/stream")
|
||
async def attribute_dialogue_stream(request: Request):
|
||
"""Same job as /api/attribute-dialogue, but streams the LLM's live output
|
||
(reasoning + the JSON being written) as SSE `{"t": "..."}` events so the UI
|
||
can show what the model is thinking, ending with `{"done": true, "result"}`.
|
||
On upstream failure it emits `{"error": "..."}` — the client then falls
|
||
back to the blocking endpoint."""
|
||
data = await request.json()
|
||
prep = _attribution_prepare(data)
|
||
payload = dict(prep["payload"])
|
||
payload["stream"] = True
|
||
headers = {"Authorization": f"Bearer {prep['api_key']}"}
|
||
|
||
def gen():
|
||
if not _attribution_llm_lock.acquire(timeout=prep["timeout_seconds"]):
|
||
yield 'data: {"error": "Attribution engine busy"}\n\n'
|
||
return
|
||
upstream = None
|
||
try:
|
||
urls = _llm_chat_completion_urls(prep["llm_url"])
|
||
last_err = None
|
||
for u in urls:
|
||
try:
|
||
upstream = requests.post(
|
||
u, json=payload, headers=headers,
|
||
timeout=(15, prep["timeout_seconds"]), stream=True,
|
||
)
|
||
if upstream.status_code in (404, 405) and u != urls[-1]:
|
||
upstream.close()
|
||
upstream = None
|
||
continue
|
||
break
|
||
except Exception as exc:
|
||
last_err = exc
|
||
upstream = None
|
||
if upstream is None:
|
||
print(f"[attribute-dialogue/stream] LLM connect failed: {last_err}")
|
||
yield f'data: {json.dumps({"error": str(last_err or "LLM connect failed")})}\n\n'
|
||
return
|
||
if upstream.status_code >= 400:
|
||
print(
|
||
f"[attribute-dialogue/stream] LLM failed ({upstream.status_code}) for model "
|
||
f"{payload.get('model') or '(default)'}: {_response_error_text(upstream)[:300]}"
|
||
)
|
||
yield f'data: {json.dumps({"error": f"HTTP {upstream.status_code}: {_response_error_text(upstream)[:300]}"})}\n\n'
|
||
return
|
||
raw_parts = []
|
||
# The LLM's SSE response usually omits a charset on its Content-Type,
|
||
# so requests falls back to Latin-1 (the old HTTP default) instead of
|
||
# UTF-8 when decode_unicode=True guesses the encoding — every umlaut
|
||
# then comes out mojibake'd ("Häfen" -> "Häfen"). Force UTF-8.
|
||
upstream.encoding = "utf-8"
|
||
# Two layers against a stuck stream: the wall-clock check catches
|
||
# a model that keeps trickling chunks past its budget, and
|
||
# _watchdog_close catches the connection going fully silent (which
|
||
# the in-loop check can't see, since a blocked socket read never
|
||
# reaches it — see _watchdog_close's docstring for why). Observed
|
||
# in production without the watchdog: a stuck stream held the
|
||
# shared lock for 11+ minutes, silently starving every other
|
||
# passage/request.
|
||
deadline = time.monotonic() + prep["timeout_seconds"]
|
||
with _watchdog_close(upstream, prep["timeout_seconds"]):
|
||
for line in upstream.iter_lines(decode_unicode=True):
|
||
if time.monotonic() > deadline:
|
||
print(f"[attribute-dialogue/stream] wall-clock deadline hit after {prep['timeout_seconds']}s, aborting stream")
|
||
yield 'data: {"error": "LLM stream exceeded timeout"}\n\n'
|
||
return
|
||
if not line or not line.startswith("data:"):
|
||
continue
|
||
chunk = line[5:].strip()
|
||
if chunk == "[DONE]":
|
||
break
|
||
try:
|
||
delta = json.loads(chunk)["choices"][0]["delta"]
|
||
except Exception:
|
||
continue
|
||
# Reasoning models emit thoughts via reasoning_content (or, on
|
||
# some vLLM builds, a plain "reasoning" key — see _reasoning_text);
|
||
# the answer JSON arrives via content. Forward both to the
|
||
# viewer, but only content counts toward the parseable answer.
|
||
t = _reasoning_text(delta) or delta.get("content") or ""
|
||
if delta.get("content"):
|
||
raw_parts.append(delta["content"])
|
||
if t:
|
||
yield f'data: {json.dumps({"t": t})}\n\n'
|
||
result = _attribution_parse("".join(raw_parts), prep["text"])
|
||
yield f'data: {json.dumps({"done": True, "result": result})}\n\n'
|
||
except GeneratorExit:
|
||
raise
|
||
except Exception as e:
|
||
yield f'data: {json.dumps({"error": str(e)})}\n\n'
|
||
finally:
|
||
try:
|
||
if upstream is not None:
|
||
upstream.close()
|
||
except Exception:
|
||
pass
|
||
_attribution_llm_lock.release()
|
||
|
||
return StreamingResponse(gen(), media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||
|
||
|
||
# ── Audio effects ─────────────────────────────────────────────────────────────
|
||
|
||
def _apply_audio_effects(audio_bytes: bytes, effects: list) -> bytes:
|
||
try:
|
||
from pedalboard import Pedalboard, Reverb, Chorus, Delay, Compressor, Gain, HighpassFilter, LowpassFilter, PitchShift, Limiter # type: ignore
|
||
import numpy as np # type: ignore
|
||
except ImportError:
|
||
raise RuntimeError("pedalboard is not installed — run: pip install pedalboard numpy")
|
||
|
||
with io.BytesIO(audio_bytes) as buf:
|
||
with wave.open(buf, "rb") as wf:
|
||
n_channels = wf.getnchannels()
|
||
sample_rate = wf.getframerate()
|
||
n_frames = wf.getnframes()
|
||
raw = wf.readframes(n_frames)
|
||
sampwidth = wf.getsampwidth()
|
||
|
||
import numpy as np # noqa: F811
|
||
dtype = {1: np.int8, 2: np.int16, 4: np.int32}.get(sampwidth, np.int16)
|
||
samples = np.frombuffer(raw, dtype=dtype).astype(np.float32) / float(np.iinfo(dtype).max)
|
||
samples = samples.reshape(1, -1) if n_channels == 1 else samples.reshape(-1, n_channels).T
|
||
|
||
board = []
|
||
for fx in effects:
|
||
t = fx.get("type", "")
|
||
p = fx.get("params", {})
|
||
if t == "reverb":
|
||
board.append(Reverb(
|
||
room_size=float(p.get("room_size", 0.35)),
|
||
damping=float(p.get("damping", 0.5)),
|
||
wet_level=float(p.get("wet", 0.25)),
|
||
dry_level=float(p.get("dry", 0.8)),
|
||
))
|
||
elif t == "chorus":
|
||
board.append(Chorus(
|
||
rate_hz=float(p.get("rate_hz", 1.0)),
|
||
depth=float(p.get("depth", 0.25)),
|
||
mix=float(p.get("mix", 0.5)),
|
||
))
|
||
elif t == "delay":
|
||
board.append(Delay(
|
||
delay_seconds=float(p.get("delay_s", 0.25)),
|
||
feedback=float(p.get("feedback", 0.3)),
|
||
mix=float(p.get("mix", 0.4)),
|
||
))
|
||
elif t == "compressor":
|
||
threshold_db = float(p.get("threshold_db", -20.0))
|
||
ratio = float(p.get("ratio", 4.0))
|
||
board.append(Compressor(
|
||
threshold_db=threshold_db,
|
||
ratio=ratio,
|
||
attack_ms=float(p.get("attack_ms", 10.0)),
|
||
release_ms=float(p.get("release_ms", 100.0)),
|
||
))
|
||
# A pure compressor with no makeup gain only ever shaves peaks
|
||
# quieter — it never produces the louder, "punchier" sound people
|
||
# actually associate with compression (broadcast/telephone/radio
|
||
# effects). Confirmed live: applying the "Telephone" preset
|
||
# (threshold -10dB, ratio 8:1) to normal TTS speech (normalized to
|
||
# ~-20dBFS) changed the output by less than 0.1% RMS — with
|
||
# nothing pushing the result back up, a compressor engaging only
|
||
# on brief peaks is nearly imperceptible over a whole clip. Makeup
|
||
# gain restores perceived loudness to roughly what an
|
||
# unprocessed signal peaking at the threshold would have, which
|
||
# is the standard way compressors are actually used.
|
||
makeup_db = float(p.get("makeup_db", -threshold_db * (1 - 1 / max(ratio, 1.0)) * 0.5))
|
||
if makeup_db:
|
||
board.append(Gain(gain_db=makeup_db))
|
||
# Verified live: makeup gain routinely pushed peaks to ~1.9
|
||
# (well past ±1.0), and the hard np.clip() at the end of this
|
||
# function turned that into audible digital clipping —
|
||
# broadband harmonic distortion that swamped whatever the
|
||
# rest of the chain (e.g. telephone bandpass) was supposed to
|
||
# sound like. A limiter catches the overshoot smoothly
|
||
# instead of slicing it off.
|
||
board.append(Limiter(threshold_db=-1.0, release_ms=100.0))
|
||
elif t == "gain":
|
||
board.append(Gain(gain_db=float(p.get("gain_db", 0.0))))
|
||
elif t == "highpass":
|
||
# pedalboard's HighpassFilter is a single-pole design (~6 dB/octave)
|
||
# — confirmed live it was too gentle to meaningfully shape a full-
|
||
# bandwidth voice recording (e.g. barely touched content an octave
|
||
# above the cutoff). Cascading 3 independent stages gives a much
|
||
# steeper, actually audible roll-off (~18 dB/octave).
|
||
cutoff = float(p.get("cutoff_hz", 80.0))
|
||
board.extend(HighpassFilter(cutoff_frequency_hz=cutoff) for _ in range(3))
|
||
elif t == "lowpass":
|
||
cutoff = float(p.get("cutoff_hz", 8000.0))
|
||
board.extend(LowpassFilter(cutoff_frequency_hz=cutoff) for _ in range(3))
|
||
elif t == "pitch_shift":
|
||
board.append(PitchShift(semitones=float(p.get("semitones", 0.0))))
|
||
|
||
if board:
|
||
samples = Pedalboard(board)(samples, sample_rate)
|
||
|
||
out = np.clip(samples, -1.0, 1.0)
|
||
pcm = ((out[0] if out.shape[0] == 1 else out.T.reshape(-1)) * 32767).astype(np.int16).tobytes()
|
||
buf_out = io.BytesIO()
|
||
with wave.open(buf_out, "wb") as wf:
|
||
wf.setnchannels(n_channels)
|
||
wf.setsampwidth(2)
|
||
wf.setframerate(sample_rate)
|
||
wf.writeframes(pcm)
|
||
return buf_out.getvalue()
|
||
|
||
|
||
@router.post("/api/audio/effects")
|
||
async def audio_effects(request: Request):
|
||
"""Apply an effects chain to a WAV file."""
|
||
form = await request.form()
|
||
audio_file = form.get("audio")
|
||
effects_json = str(form.get("effects") or "[]")
|
||
|
||
if audio_file is None:
|
||
raise HTTPException(400, "No audio file provided")
|
||
audio_bytes = await audio_file.read()
|
||
try:
|
||
effects = json.loads(effects_json)
|
||
except Exception:
|
||
raise HTTPException(400, "Invalid effects JSON")
|
||
|
||
try:
|
||
result = await asyncio.to_thread(_apply_audio_effects, audio_bytes, effects)
|
||
return Response(content=result, media_type="audio/wav")
|
||
except RuntimeError as e:
|
||
raise HTTPException(501, str(e))
|
||
except Exception as e:
|
||
raise HTTPException(500, f"Effects processing failed: {e}")
|
||
|
||
|
||
# ── Voices export / import ────────────────────────────────────────────────────
|
||
|
||
_EXPORT_SKIP_KEYS = {
|
||
"groq_api_key", "whisper_api_key", "tts_api_key", "voice_design_api_key",
|
||
"elevenlabs_api_key", "llm_api_key", "engine_api_keys",
|
||
}
|
||
_IMPORT_ALLOWED_SUFFIXES = set(_AUDIO_EXTS + [".reference.txt", ".meta.json", ".jpg", ".jpeg", ".png", ".webp"])
|
||
|
||
|
||
@router.get("/api/voices/export")
|
||
async def voices_export():
|
||
"""Export all voices + non-sensitive settings as a ZIP archive."""
|
||
import zipfile
|
||
from datetime import datetime
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
||
safe_settings = {k: v for k, v in settings.items() if k not in _EXPORT_SKIP_KEYS}
|
||
zf.writestr("settings.json", json.dumps(safe_settings, indent=2))
|
||
if scan_dir.exists():
|
||
for f in scan_dir.rglob("*"):
|
||
if f.is_file():
|
||
try:
|
||
zf.write(f, str(f.relative_to(scan_dir.parent)))
|
||
except Exception:
|
||
pass
|
||
buf.seek(0)
|
||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
return Response(
|
||
content=buf.read(),
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f'attachment; filename="voices_export_{ts}.zip"'},
|
||
)
|
||
|
||
|
||
@router.post("/api/voices/import")
|
||
async def voices_import(file: UploadFile = File(...)):
|
||
"""Import voices from a ZIP archive (skips settings.json and unsafe paths)."""
|
||
import zipfile
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
scan_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
content = await file.read()
|
||
if len(content) > _MAX_UPLOAD_BYTES:
|
||
raise HTTPException(413, "ZIP file too large")
|
||
|
||
try:
|
||
imported = 0
|
||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||
for info in zf.infolist():
|
||
if info.is_dir():
|
||
continue
|
||
parts = Path(info.filename).parts
|
||
if any(p in ("..", "") for p in parts) or Path(info.filename).name == "settings.json":
|
||
continue
|
||
if Path(info.filename).suffix.lower() not in _IMPORT_ALLOWED_SUFFIXES:
|
||
continue
|
||
rel = parts[1:] if len(parts) > 1 else parts
|
||
dest = scan_dir / Path(*rel)
|
||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||
dest.write_bytes(zf.read(info))
|
||
imported += 1
|
||
except zipfile.BadZipFile:
|
||
raise HTTPException(400, "Not a valid ZIP file")
|
||
except Exception as e:
|
||
raise HTTPException(500, f"Import failed: {e}")
|
||
|
||
await asyncio.to_thread(rebuild_voice_index, settings)
|
||
return {"ok": True, "imported": imported}
|
||
|
||
|
||
# ── /speak REST endpoint ──────────────────────────────────────────────────────
|
||
|
||
@router.post("/speak")
|
||
async def speak(request: Request):
|
||
data = await request.json()
|
||
text: str = str(data.get("text") or "").strip()
|
||
if not text:
|
||
raise HTTPException(400, "text is required")
|
||
|
||
explicit_voice: str = str(data.get("voice") or data.get("profile_id") or data.get("profile") or "").strip()
|
||
backend: str = _clean_preview_backend(str(data.get("backend") or "voice_clone"))
|
||
apply_persona: bool = bool(data.get("apply_persona") or data.get("personality"))
|
||
client_id: str = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
||
|
||
settings = _load_settings()
|
||
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
||
if not voice:
|
||
raise HTTPException(400, "No voice specified and no default voice configured")
|
||
|
||
if apply_persona:
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
wav = _find_voice_audio(voice, scan_dir)
|
||
if wav:
|
||
persona = _load_meta(wav).get("persona", "")
|
||
if persona:
|
||
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
llm_model = settings.get("llm_model") or ""
|
||
try:
|
||
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url, llm_model)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"Persona rewrite failed: {e}")
|
||
|
||
instruct = str(data.get("instruct") or "")
|
||
try:
|
||
audio, media_type = await asyncio.to_thread(
|
||
_preview_request_audio, text, voice, settings, instruct, backend
|
||
)
|
||
except Exception as e:
|
||
raise HTTPException(502, f"TTS error: {e}")
|
||
|
||
return Response(content=audio, media_type=media_type)
|
||
|
||
|
||
@router.get("/speak/bindings")
|
||
async def get_speak_bindings():
|
||
s = _load_settings()
|
||
return {"bindings": s.get("client_voice_bindings") or {}}
|
||
|
||
|
||
@router.put("/speak/bindings/{client_id}")
|
||
async def put_speak_binding(client_id: str, request: Request):
|
||
data = await request.json()
|
||
voice = str(data.get("voice") or "").strip()
|
||
if not voice:
|
||
raise HTTPException(400, "voice is required")
|
||
s = _load_settings()
|
||
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
||
bindings[client_id] = voice
|
||
s["client_voice_bindings"] = bindings
|
||
_save_settings(s)
|
||
return {"ok": True, "client_id": client_id, "voice": voice}
|
||
|
||
|
||
@router.delete("/speak/bindings/{client_id}")
|
||
async def delete_speak_binding(client_id: str):
|
||
s = _load_settings()
|
||
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
||
if client_id not in bindings:
|
||
raise HTTPException(404, f"No binding for client '{client_id}'")
|
||
del bindings[client_id]
|
||
s["client_voice_bindings"] = bindings
|
||
_save_settings(s)
|
||
return {"ok": True, "client_id": client_id}
|
||
|
||
|
||
# ── MCP JSON-RPC 2.0 server ───────────────────────────────────────────────────
|
||
|
||
_MCP_SERVER_INFO = {"name": "tts-voice-creator", "version": "1.0.0"}
|
||
|
||
_MCP_TOOLS = [
|
||
{
|
||
"name": "speak",
|
||
"description": "Generate speech audio from text using a cloned voice. Returns a data URI with the WAV audio.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"text": {"type": "string", "description": "Text to synthesize"},
|
||
"voice": {"type": "string", "description": "Voice ID (optional — uses per-client binding or default if omitted)"},
|
||
"apply_persona": {"type": "boolean", "description": "Rewrite text through the voice's persona before synthesis"},
|
||
},
|
||
"required": ["text"],
|
||
},
|
||
},
|
||
{
|
||
"name": "transcribe",
|
||
"description": "Transcribe base64-encoded WAV audio to text.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"audio_base64": {"type": "string", "description": "Base64-encoded audio bytes (WAV preferred)"},
|
||
},
|
||
"required": ["audio_base64"],
|
||
},
|
||
},
|
||
{
|
||
"name": "list_captures",
|
||
"description": "List the 20 most recently generated audio files.",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
{
|
||
"name": "list_profiles",
|
||
"description": "List all available voice profiles with their language, persona, and enabled state.",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
{
|
||
"name": "list_books",
|
||
"description": "List all Read Aloud books/documents (title, id, character count metadata).",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
{
|
||
"name": "list_characters",
|
||
"description": "List character library records, optionally filtered to one book/production.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {"book": {"type": "string", "description": "Book/production title to filter by (optional — omit for every character across every book)"}},
|
||
},
|
||
},
|
||
{
|
||
"name": "get_character",
|
||
"description": "Get one character's full record (sheet, voice, image, tags) by id.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {"id": {"type": "string", "description": "Character record id"}},
|
||
"required": ["id"],
|
||
},
|
||
},
|
||
{
|
||
"name": "update_character",
|
||
"description": "Update fields on a character's sheet (e.g. backstory, archetype, gender) or top-level record fields (name, voice, tags). Merges with the existing record — only send the fields you want to change.",
|
||
"inputSchema": {
|
||
"type": "object",
|
||
"properties": {
|
||
"id": {"type": "string", "description": "Character record id"},
|
||
"sheet": {"type": "object", "description": "Partial sheet fields to merge in (e.g. {\"backstory\": \"...\", \"archetype\": \"...\"})"},
|
||
"voice": {"type": "string", "description": "Voice id to assign (shortcut for updating just the voice)"},
|
||
"name": {"type": "string", "description": "Rename the character (rare — usually leave unset)"},
|
||
},
|
||
"required": ["id"],
|
||
},
|
||
},
|
||
{
|
||
"name": "list_rehearsals",
|
||
"description": "List all saved Script Rehearser sessions (title, id, character/line counts).",
|
||
"inputSchema": {"type": "object", "properties": {}},
|
||
},
|
||
]
|
||
|
||
|
||
async def _mcp_tool_speak(args: dict, client_id: str) -> dict:
|
||
text = str(args.get("text") or "").strip()
|
||
if not text:
|
||
raise ValueError("text is required")
|
||
explicit_voice = str(args.get("voice") or "").strip()
|
||
apply_persona = bool(args.get("apply_persona", False))
|
||
settings = _load_settings()
|
||
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
||
if not voice:
|
||
raise ValueError("No voice specified and no default voice configured")
|
||
if apply_persona:
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
wav = _find_voice_audio(voice, scan_dir)
|
||
if wav:
|
||
persona = _load_meta(wav).get("persona", "")
|
||
if persona:
|
||
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
llm_model = settings.get("llm_model") or ""
|
||
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url, llm_model)
|
||
audio, _media_type = await asyncio.to_thread(
|
||
_preview_request_audio, text, voice, settings, "", "voice_clone"
|
||
)
|
||
audio_b64 = base64.b64encode(audio).decode()
|
||
return {
|
||
"content": [
|
||
{"type": "text", "text": f"Generated {len(audio)} bytes of speech audio for voice '{voice}'."},
|
||
{"type": "resource", "resource": {"uri": f"data:audio/wav;base64,{audio_b64}", "mimeType": "audio/wav"}},
|
||
]
|
||
}
|
||
|
||
|
||
async def _mcp_tool_transcribe(args: dict) -> dict:
|
||
raw = args.get("audio_base64") or ""
|
||
try:
|
||
audio_bytes = base64.b64decode(raw)
|
||
except Exception:
|
||
raise ValueError("audio_base64 is not valid base64")
|
||
tmp = TEMP_DIR / f"{uuid.uuid4().hex}_mcp_transcribe.wav"
|
||
tmp.write_bytes(audio_bytes)
|
||
settings = _load_settings()
|
||
try:
|
||
text, used_backend = await asyncio.to_thread(_transcribe_audio, tmp, settings, "configured")
|
||
finally:
|
||
try:
|
||
tmp.unlink(missing_ok=True)
|
||
except Exception:
|
||
pass
|
||
return {"content": [{"type": "text", "text": text}], "backend": used_backend}
|
||
|
||
|
||
async def _mcp_tool_list_captures() -> dict:
|
||
settings = _load_settings()
|
||
out_dir = _active_voices_dir(settings)
|
||
if not out_dir.exists():
|
||
return {"content": [{"type": "text", "text": "[]"}]}
|
||
files = sorted(
|
||
(p for p in out_dir.rglob("*.wav") if not _is_internal_voice_file(p)),
|
||
key=lambda p: p.stat().st_mtime,
|
||
reverse=True,
|
||
)[:20]
|
||
result = [{"name": p.stem, "path": str(p), "mtime": p.stat().st_mtime} for p in files]
|
||
return {"content": [{"type": "text", "text": json.dumps(result)}]}
|
||
|
||
|
||
async def _mcp_tool_list_profiles() -> dict:
|
||
settings = _load_settings()
|
||
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
||
profiles = []
|
||
seen: set[str] = set()
|
||
for audio in sorted(_voice_audio_files(scan_dir), key=lambda p: p.stem.lower()):
|
||
if audio.stem in seen:
|
||
continue
|
||
seen.add(audio.stem)
|
||
meta = _load_meta(audio)
|
||
parts = audio.stem.split("_", 2)
|
||
lang = parts[0].upper() if parts else ""
|
||
profiles.append({
|
||
"id": audio.stem,
|
||
"name": audio.stem,
|
||
"lang": lang,
|
||
"persona": meta.get("persona", ""),
|
||
"enabled": meta.get("enabled", True),
|
||
})
|
||
return {"content": [{"type": "text", "text": json.dumps(profiles)}]}
|
||
|
||
|
||
async def _mcp_tool_list_books() -> dict:
|
||
from routes.reader import reader_list_docs
|
||
data = await reader_list_docs()
|
||
return {"content": [{"type": "text", "text": json.dumps(data.get("docs", []))}]}
|
||
|
||
|
||
async def _mcp_tool_list_characters(args: dict) -> dict:
|
||
from core.database import char_get_all
|
||
book = str(args.get("book") or "").strip().lower()
|
||
recs = char_get_all()
|
||
if book:
|
||
recs = [r for r in recs if str(r.get("book") or "").strip().lower() == book
|
||
or book in [t.strip().lower() for t in str(r.get("tags") or "").split(",")]]
|
||
return {"content": [{"type": "text", "text": json.dumps(recs)}]}
|
||
|
||
|
||
async def _mcp_tool_get_character(args: dict) -> dict:
|
||
from core.database import char_get
|
||
char_id = str(args.get("id") or "").strip()
|
||
if not char_id:
|
||
raise ValueError("id is required")
|
||
rec = char_get(char_id)
|
||
if rec is None:
|
||
raise ValueError(f"Character '{char_id}' not found")
|
||
return {"content": [{"type": "text", "text": json.dumps(rec)}]}
|
||
|
||
|
||
async def _mcp_tool_update_character(args: dict) -> dict:
|
||
from core.database import char_get, char_put
|
||
char_id = str(args.get("id") or "").strip()
|
||
if not char_id:
|
||
raise ValueError("id is required")
|
||
rec = char_get(char_id)
|
||
if rec is None:
|
||
raise ValueError(f"Character '{char_id}' not found")
|
||
if "sheet" in args and isinstance(args["sheet"], dict):
|
||
rec["sheet"] = {**(rec.get("sheet") or {}), **args["sheet"]}
|
||
if "voice" in args:
|
||
rec["voice"] = args["voice"]
|
||
if "name" in args:
|
||
rec["name"] = args["name"]
|
||
rec["id"] = char_id
|
||
updated = char_put(rec)
|
||
return {"content": [{"type": "text", "text": json.dumps(updated)}]}
|
||
|
||
|
||
async def _mcp_tool_list_rehearsals() -> dict:
|
||
from core.database import reh_get_all, reh_compact_titles
|
||
reh_compact_titles()
|
||
return {"content": [{"type": "text", "text": json.dumps(reh_get_all())}]}
|
||
|
||
|
||
def _mcp_error_response(code: int, message: str, rpc_id) -> Response:
|
||
import logging
|
||
body = {"jsonrpc": "2.0", "error": {"code": code, "message": message}, "id": rpc_id}
|
||
return Response(content=json.dumps(body), media_type="application/json")
|
||
|
||
|
||
@router.post("/mcp")
|
||
async def mcp_jsonrpc(request: Request):
|
||
import logging
|
||
logger = logging.getLogger("uvicorn.error")
|
||
try:
|
||
body = await request.json()
|
||
except Exception:
|
||
return _mcp_error_response(-32700, "Parse error", None)
|
||
|
||
rpc_id = body.get("id")
|
||
method = body.get("method", "")
|
||
params = body.get("params") or {}
|
||
client_id = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
||
|
||
try:
|
||
if method == "initialize":
|
||
result = {
|
||
"protocolVersion": "2024-11-05",
|
||
"serverInfo": _MCP_SERVER_INFO,
|
||
"capabilities": {"tools": {}},
|
||
}
|
||
elif method == "notifications/initialized":
|
||
return Response(status_code=204)
|
||
elif method == "tools/list":
|
||
result = {"tools": _MCP_TOOLS}
|
||
elif method == "tools/call":
|
||
tool_name = str(params.get("name") or "")
|
||
tool_args = params.get("arguments") or {}
|
||
if tool_name == "speak":
|
||
result = await _mcp_tool_speak(tool_args, client_id)
|
||
elif tool_name == "transcribe":
|
||
result = await _mcp_tool_transcribe(tool_args)
|
||
elif tool_name == "list_captures":
|
||
result = await _mcp_tool_list_captures()
|
||
elif tool_name == "list_profiles":
|
||
result = await _mcp_tool_list_profiles()
|
||
elif tool_name == "list_books":
|
||
result = await _mcp_tool_list_books()
|
||
elif tool_name == "list_characters":
|
||
result = await _mcp_tool_list_characters(tool_args)
|
||
elif tool_name == "get_character":
|
||
result = await _mcp_tool_get_character(tool_args)
|
||
elif tool_name == "update_character":
|
||
result = await _mcp_tool_update_character(tool_args)
|
||
elif tool_name == "list_rehearsals":
|
||
result = await _mcp_tool_list_rehearsals()
|
||
else:
|
||
return _mcp_error_response(-32601, f"Unknown tool: {tool_name}", rpc_id)
|
||
else:
|
||
return _mcp_error_response(-32601, f"Method not found: {method}", rpc_id)
|
||
except ValueError as e:
|
||
return _mcp_error_response(-32602, str(e), rpc_id)
|
||
except Exception as e:
|
||
logger.exception("MCP tool error in method %s", method)
|
||
return _mcp_error_response(-32000, f"Server error: {e}", rpc_id)
|
||
|
||
return Response(
|
||
content=json.dumps({"jsonrpc": "2.0", "result": result, "id": rpc_id}),
|
||
media_type="application/json",
|
||
)
|
||
|
||
|
||
@router.get("/mcp")
|
||
async def mcp_sse(request: Request):
|
||
"""SSE keep-alive stream (satisfies MCP spec GET /mcp requirement)."""
|
||
from typing import AsyncGenerator
|
||
|
||
async def _keepalive() -> AsyncGenerator[str, None]:
|
||
yield ": mcp-sse-ready\n\n"
|
||
while True:
|
||
await asyncio.sleep(15)
|
||
yield ": ping\n\n"
|
||
return StreamingResponse(
|
||
_keepalive(),
|
||
media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||
)
|
||
|
||
|
||
# ── Conversation playground ───────────────────────────────────────────────────
|
||
|
||
@router.get("/api/conversation/llm-models")
|
||
async def conversation_llm_models(url: str = "", api_key: str = ""):
|
||
"""List models from a local LLM endpoint (Ollama / vLLM / LM Studio)."""
|
||
from core.validation import _validate_http_url
|
||
settings = _load_settings()
|
||
base = (url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
key_to_use = api_key or settings.get("llm_api_key") or "sk-dummy-key"
|
||
try:
|
||
base = _validate_http_url(base, allow_private=True)
|
||
r = requests.get(f"{base}/models", timeout=5, headers={"Authorization": f"Bearer {key_to_use}"})
|
||
|
||
# Fallback for gateways like LiteLLM that might prefer /models over /v1/models
|
||
if r.status_code != 200 and base.endswith("/v1"):
|
||
fallback_base = base[:-3]
|
||
r2 = requests.get(f"{fallback_base}/models", timeout=5, headers={"Authorization": f"Bearer {key_to_use}"})
|
||
if r2.status_code == 200:
|
||
r = r2
|
||
|
||
if r.status_code == 200:
|
||
payload = r.json()
|
||
data = payload.get("data", []) if isinstance(payload, dict) else []
|
||
models = [
|
||
str(item["id"]) if isinstance(item, dict) and item.get("id") else str(item)
|
||
for item in data if item
|
||
]
|
||
return {"models": models, "url": base}
|
||
else:
|
||
return {"models": [], "url": base, "error": r.text}
|
||
except Exception as e:
|
||
return {"models": [], "url": base, "error": str(e)}
|
||
|
||
|
||
# Conversation had no style/emotion control at all before this — every reply
|
||
# synthesized flat regardless of backend. Mirrors the same backend-aware split
|
||
# used for the Rehearser/Studio pipeline: Fish-Speech only reacts to an inline
|
||
# [tag] in the text itself (the instruct field is ignored), other backends
|
||
# take the descriptive phrase directly as instruct. `emotion` here is always
|
||
# already-English (the REH_EMOTIONS quick-pick list), so no translation table
|
||
# is needed the way the German-templated audiobook instruct sentences needed one.
|
||
def _conv_tts_text_and_instruct(text: str, emotion: str, backend: str) -> tuple[str, str]:
|
||
emotion = (emotion or "").strip()
|
||
if not emotion:
|
||
return text, ""
|
||
if re.search(r"fish", backend or "", re.I):
|
||
tag = emotion.split(",")[0].strip().lower()
|
||
if re.fullmatch(r"[a-z\- ]+", tag):
|
||
return f"[{tag}] {text}", ""
|
||
return text, ""
|
||
return text, emotion
|
||
|
||
|
||
def _make_tts_task(
|
||
text: str,
|
||
voice: str,
|
||
settings: dict,
|
||
backend: str,
|
||
sem: "asyncio.Semaphore | None",
|
||
emotion: str = "",
|
||
) -> "asyncio.Task":
|
||
tts_text, instruct = _conv_tts_text_and_instruct(text, emotion, backend)
|
||
if sem is None:
|
||
return asyncio.create_task(
|
||
asyncio.to_thread(_preview_request_audio, tts_text, voice, settings, instruct, backend)
|
||
)
|
||
|
||
async def _guarded() -> tuple[bytes, str]:
|
||
async with sem:
|
||
return await asyncio.to_thread(_preview_request_audio, tts_text, voice, settings, instruct, backend)
|
||
|
||
return asyncio.create_task(_guarded())
|
||
|
||
|
||
@router.post("/api/conversation/turn")
|
||
async def conversation_turn(
|
||
audio: Optional[UploadFile] = None,
|
||
text: str = Form(""),
|
||
stt_backend: str = Form("configured"),
|
||
llm_url: str = Form(""),
|
||
llm_model: str = Form(""),
|
||
tts_backend: str = Form("voice_clone"),
|
||
tts_voice: str = Form(""),
|
||
tts_emotion: str = Form(""),
|
||
system_prompt: str = Form("You are a helpful voice assistant. Keep replies short and conversational."),
|
||
history: str = Form("[]"),
|
||
):
|
||
"""Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events.
|
||
Pass either an audio file (runs STT first) or a plain text string (skips STT).
|
||
"""
|
||
direct_text = text.strip()
|
||
if not direct_text and (audio is None or not getattr(audio, "filename", None)):
|
||
raise HTTPException(400, "Provide either an audio file or a text field")
|
||
|
||
settings = _load_settings()
|
||
eff_llm_url = (llm_url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
||
|
||
# Prepare audio temp files only when an audio upload was provided
|
||
_tmp: Path | None = None
|
||
_wav: Path | None = None
|
||
if not direct_text and audio is not None:
|
||
suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm"
|
||
if suffix not in _UPLOAD_EXTS:
|
||
suffix = ".webm"
|
||
_tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}"
|
||
try:
|
||
with _tmp.open("wb") as f:
|
||
_copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
|
||
_wav = _to_wav_16k(_tmp)
|
||
except Exception as e:
|
||
_tmp.unlink(missing_ok=True)
|
||
raise HTTPException(400, f"Audio upload failed: {e}")
|
||
|
||
try:
|
||
hist = json.loads(history) if history else []
|
||
if not isinstance(hist, list):
|
||
hist = []
|
||
except Exception:
|
||
hist = []
|
||
|
||
stt_be = _clean_stt_backend(stt_backend)
|
||
tts_be = _clean_preview_backend(tts_backend)
|
||
|
||
async def generate():
|
||
t0 = time.monotonic()
|
||
stt_ms = llm_ttft_ms = llm_total_ms = tts_ms = None
|
||
transcript = llm_text = ""
|
||
|
||
def sse(obj: dict) -> str:
|
||
return f"data: {json.dumps(obj)}\n\n"
|
||
|
||
# 1. STT — skipped when caller sends direct text
|
||
if direct_text:
|
||
transcript = direct_text
|
||
yield sse({"type": "transcript", "text": transcript, "stt_ms": None})
|
||
else:
|
||
try:
|
||
t_stt = time.monotonic()
|
||
transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be)
|
||
stt_ms = int((time.monotonic() - t_stt) * 1000)
|
||
yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms})
|
||
except Exception as e:
|
||
yield sse({"type": "error", "stage": "stt", "message": str(e)})
|
||
return
|
||
finally:
|
||
for p in filter(None, {_tmp, _wav}):
|
||
with contextlib.suppress(Exception):
|
||
p.unlink(missing_ok=True)
|
||
|
||
if not transcript.strip() or _is_hallucination(transcript):
|
||
yield sse({"type": "error", "stage": "stt", "message": "No speech detected."})
|
||
return
|
||
|
||
# 2. LLM stream — runs in a thread; tokens arrive via asyncio.Queue
|
||
# so the event loop is never blocked and TTS can start on sentence 1
|
||
# while the LLM is still generating sentences 2, 3, …
|
||
messages = [{"role": "system", "content": system_prompt}]
|
||
messages.extend(hist[-20:])
|
||
messages.append({"role": "user", "content": transcript})
|
||
llm_payload: dict = {"messages": messages, "stream": True, "temperature": 0.7, "max_tokens": 512}
|
||
eff_llm_model = llm_model or settings.get("llm_model") or ""
|
||
if eff_llm_model:
|
||
llm_payload["model"] = eff_llm_model
|
||
|
||
_loop = asyncio.get_event_loop()
|
||
_token_q: asyncio.Queue[tuple[str, str] | None] = asyncio.Queue()
|
||
|
||
def _llm_thread() -> None:
|
||
try:
|
||
resp = requests.post(
|
||
f"{eff_llm_url}/chat/completions", json=llm_payload,
|
||
headers={"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}, stream=True, timeout=120,
|
||
)
|
||
resp.raise_for_status()
|
||
for raw_line in resp.iter_lines():
|
||
if not raw_line:
|
||
continue
|
||
line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else str(raw_line)
|
||
if not line.startswith("data:"):
|
||
continue
|
||
payload_str = line[5:].strip()
|
||
if payload_str == "[DONE]":
|
||
break
|
||
try:
|
||
obj = json.loads(payload_str)
|
||
d_obj = ((obj.get("choices") or [{}])[0].get("delta") or {})
|
||
reasoning = _reasoning_text(d_obj)
|
||
content = d_obj.get("content") or ""
|
||
if not content and not reasoning and isinstance(obj.get("message"), dict):
|
||
content = obj["message"].get("content") or ""
|
||
if reasoning:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, ("reasoning", reasoning))
|
||
if content:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, ("content", content))
|
||
except Exception:
|
||
continue
|
||
except Exception as exc:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, ("error", str(exc)))
|
||
finally:
|
||
_loop.call_soon_threadsafe(_token_q.put_nowait, None)
|
||
|
||
threading.Thread(target=_llm_thread, daemon=True).start()
|
||
|
||
# Some models (e.g. Qwen3 in "thinking" mode) don't use a separate
|
||
# reasoning_content field — they emit a <think>...</think> block inline
|
||
# inside content, split across arbitrary delta chunks. Many chat
|
||
# templates also inject the opening "<think>" as a fixed prompt prefix,
|
||
# so it never appears in the generated stream at all — only the closing
|
||
# "</think>" does. This tracks both explicit-tag and implicit-open
|
||
# styles so the answer stays clean for chat history/TTS and the
|
||
# reasoning can be shown in a collapsible "thinking" panel instead.
|
||
_THINK_SNIFF_LIMIT = 300 # chars buffered before assuming "not a reasoning response"
|
||
_think_state = {"buf": "", "resolved": False, "in_think": False}
|
||
|
||
def _explicit_tag_split(text: str) -> tuple[str, str]:
|
||
# Assumes _think_state["resolved"] is True — scans for <think>/</think>
|
||
# tags that may be split across delta chunks.
|
||
_think_state["buf"] += text
|
||
thinking_parts: list[str] = []
|
||
answer_parts: list[str] = []
|
||
buf = _think_state["buf"]
|
||
while True:
|
||
tag = "</think>" if _think_state["in_think"] else "<think>"
|
||
idx = buf.find(tag)
|
||
if idx == -1:
|
||
break
|
||
before = buf[:idx]
|
||
(thinking_parts if _think_state["in_think"] else answer_parts).append(before)
|
||
buf = buf[idx + len(tag):]
|
||
_think_state["in_think"] = not _think_state["in_think"]
|
||
# Hold back a short suffix that could be the start of a split tag
|
||
hold = 0
|
||
for cut in range(1, min(8, len(buf)) + 1):
|
||
suffix = buf[-cut:]
|
||
if "<think>".startswith(suffix) or "</think>".startswith(suffix):
|
||
hold = cut
|
||
flush = buf[:len(buf) - hold] if hold else buf
|
||
_think_state["buf"] = buf[len(buf) - hold:] if hold else ""
|
||
(thinking_parts if _think_state["in_think"] else answer_parts).append(flush)
|
||
return "".join(thinking_parts), "".join(answer_parts)
|
||
|
||
def _split_think(text: str) -> tuple[str, str]:
|
||
if _think_state["resolved"]:
|
||
return _explicit_tag_split(text)
|
||
_think_state["buf"] += text
|
||
buf = _think_state["buf"]
|
||
open_idx = buf.find("<think>")
|
||
close_idx = buf.find("</think>")
|
||
if close_idx != -1 and (open_idx == -1 or close_idx < open_idx):
|
||
# No opening tag before this close — the model (or its chat
|
||
# template) started inside a think block implicitly.
|
||
thinking_text = buf[:close_idx]
|
||
rest = buf[close_idx + len("</think>"):]
|
||
_think_state["resolved"] = True
|
||
_think_state["in_think"] = False
|
||
_think_state["buf"] = ""
|
||
_, rest_answer = _explicit_tag_split(rest) if rest else ("", "")
|
||
return thinking_text, rest_answer
|
||
if open_idx != -1:
|
||
before = buf[:open_idx]
|
||
rest = buf[open_idx:]
|
||
_think_state["resolved"] = True
|
||
_think_state["in_think"] = False
|
||
_think_state["buf"] = ""
|
||
thinking_rest, answer_rest = _explicit_tag_split(rest) if rest else ("", "")
|
||
return thinking_rest, before + answer_rest
|
||
if len(buf) >= _THINK_SNIFF_LIMIT:
|
||
# No think tag within the sniff window — treat as a plain,
|
||
# non-reasoning response from here on.
|
||
_think_state["resolved"] = True
|
||
_think_state["in_think"] = False
|
||
_think_state["buf"] = ""
|
||
return "", buf
|
||
return "", "" # still buffering — undecided
|
||
|
||
def _finish_split_think() -> tuple[str, str]:
|
||
if not _think_state["buf"]:
|
||
return "", ""
|
||
leftover = _think_state["buf"]
|
||
_think_state["buf"] = ""
|
||
if not _think_state["resolved"]:
|
||
return "", leftover # stream ended before we saw any think tag
|
||
return (leftover, "") if _think_state["in_think"] else ("", leftover)
|
||
|
||
# Streaming backend (8023) has per-process voice state — serialize to prevent
|
||
# concurrent requests from clobbering each other's voice context.
|
||
tts_sem: asyncio.Semaphore | None = asyncio.Semaphore(1) if tts_be == "streaming" else None
|
||
|
||
t_llm = time.monotonic()
|
||
llm_ttft_ms: int | None = None
|
||
ttft_done = False
|
||
sent_buf = ""
|
||
tts_tasks: list[asyncio.Task] = []
|
||
tts_texts: list[str] = [] # sentence text corresponding to each task
|
||
tts_first_start: float | None = None
|
||
|
||
try:
|
||
while True:
|
||
item = await _token_q.get()
|
||
if item is None:
|
||
break
|
||
kind, text = item
|
||
if kind == "error":
|
||
yield sse({"type": "error", "stage": "llm", "message": text})
|
||
return
|
||
if not ttft_done:
|
||
llm_ttft_ms = int((time.monotonic() - t_llm) * 1000)
|
||
ttft_done = True
|
||
thinking_piece, answer_piece = (text, "") if kind == "reasoning" else _split_think(text)
|
||
if thinking_piece:
|
||
yield sse({"type": "thinking", "delta": thinking_piece})
|
||
if not answer_piece:
|
||
continue
|
||
llm_text += answer_piece
|
||
sent_buf += answer_piece
|
||
yield sse({"type": "token", "delta": answer_piece})
|
||
# Fire TTS on sentence boundary — runs concurrently with LLM
|
||
split = _sentence_split(sent_buf)
|
||
if split > 0:
|
||
chunk_text = sent_buf[:split].strip()
|
||
sent_buf = sent_buf[split:]
|
||
if chunk_text:
|
||
if tts_first_start is None:
|
||
tts_first_start = time.monotonic()
|
||
tts_texts.append(chunk_text)
|
||
tts_tasks.append(_make_tts_task(chunk_text, tts_voice, settings, tts_be, tts_sem, tts_emotion))
|
||
except Exception as exc:
|
||
yield sse({"type": "error", "stage": "llm", "message": str(exc)})
|
||
return
|
||
|
||
# Flush any text still held back (partial tag, or sniff buffer never resolved)
|
||
_thinking_tail, _answer_tail = _finish_split_think()
|
||
if _thinking_tail:
|
||
yield sse({"type": "thinking", "delta": _thinking_tail})
|
||
if _answer_tail:
|
||
llm_text += _answer_tail
|
||
sent_buf += _answer_tail
|
||
yield sse({"type": "token", "delta": _answer_tail})
|
||
|
||
# Flush any remaining text as a final TTS task
|
||
if sent_buf.strip():
|
||
if tts_first_start is None:
|
||
tts_first_start = time.monotonic()
|
||
tts_texts.append(sent_buf.strip())
|
||
tts_tasks.append(_make_tts_task(sent_buf.strip(), tts_voice, settings, tts_be, tts_sem, tts_emotion))
|
||
|
||
llm_total_ms = int((time.monotonic() - t_llm) * 1000)
|
||
yield sse({"type": "llm_done", "text": llm_text,
|
||
"llm_ttft_ms": llm_ttft_ms, "llm_total_ms": llm_total_ms})
|
||
|
||
if not llm_text.strip():
|
||
yield sse({"type": "error", "stage": "llm",
|
||
"message": "LLM returned empty response. "
|
||
"If using a Qwen3 model, try adding /no-think to your system prompt "
|
||
"or pick a non-thinking model in the Language Model dropdown."})
|
||
return
|
||
|
||
# 3. Stream audio chunks in order — each chunk's TTS ran concurrently
|
||
# with LLM generation, so first audio arrives much sooner than
|
||
# waiting for the full response.
|
||
tts_start = tts_first_start or time.monotonic()
|
||
for i, task in enumerate(tts_tasks):
|
||
try:
|
||
audio_bytes, mime = await task
|
||
except Exception as exc:
|
||
yield sse({"type": "error", "stage": "tts", "message": str(exc)})
|
||
return
|
||
if i == 0:
|
||
tts_ms = int((time.monotonic() - tts_start) * 1000)
|
||
sentence_text = tts_texts[i] if i < len(tts_texts) else ""
|
||
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(),
|
||
"mime": mime, "text": sentence_text})
|
||
|
||
total_ms = int((time.monotonic() - t0) * 1000)
|
||
yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
|
||
"llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms})
|
||
yield sse({"type": "done"})
|
||
|
||
return StreamingResponse(generate(), media_type="text/event-stream",
|
||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|