mARTin-B78
b3a408bfd9
UploadFile | None = File(None) with from __future__ import annotations caused FastAPI to still treat audio as required when omitted. Changed to Optional[UploadFile] = None (no File() wrapper) so the field is genuinely optional for text-input turns. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
806 lines
32 KiB
Python
806 lines
32 KiB
Python
"""Conversation playground, LLM refinement, audio effects, export/import, speak, MCP."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import io
|
|
import json
|
|
import time
|
|
import uuid
|
|
import wave
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from typing import Optional
|
|
from fastapi import APIRouter, Form, HTTPException, Request, UploadFile
|
|
from fastapi.responses import Response, StreamingResponse
|
|
|
|
from core.config import _load_settings, _save_settings, _clean_preview_backend
|
|
from core.constants import _VOICES_DIR_DEFAULT, _MAX_UPLOAD_BYTES
|
|
from core.registry import _registry_get, TEMP_DIR
|
|
from core.validation import _copy_limited
|
|
from core.audio import _to_wav_16k
|
|
from core.voice import (
|
|
_AUDIO_EXTS, _UPLOAD_EXTS, _PICTURE_EXTS,
|
|
_find_voice_audio, _load_meta, _active_voices_dir,
|
|
_voice_audio_files, _is_internal_voice_file,
|
|
)
|
|
from core.tts_helpers import _preview_request_audio
|
|
from routes.stt import _transcribe_audio, _clean_stt_backend
|
|
|
|
router = APIRouter()
|
|
|
|
# ── LLM helpers ───────────────────────────────────────────────────────────────
|
|
|
|
def _rewrite_with_persona_sync(text: str, persona: str, llm_url: str, model: str = "") -> str:
|
|
"""Inline synchronous persona rewrite; raises RuntimeError on failure."""
|
|
system = (
|
|
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
|
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
|
"Return ONLY the rephrased text — no quotes, no explanation."
|
|
)
|
|
payload: dict = {
|
|
"messages": [
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": text},
|
|
],
|
|
"temperature": 0.3,
|
|
"max_tokens": 512,
|
|
}
|
|
if model:
|
|
payload["model"] = model
|
|
resp = requests.post(
|
|
f"{llm_url.rstrip('/')}/chat/completions",
|
|
json=payload,
|
|
headers={"Authorization": "Bearer no-key"},
|
|
timeout=60,
|
|
)
|
|
resp.raise_for_status()
|
|
result = resp.json()["choices"][0]["message"]["content"].strip()
|
|
if result.startswith('"') and result.endswith('"'):
|
|
result = result[1:-1].strip()
|
|
return result
|
|
|
|
|
|
def _resolve_speak_voice(settings: dict, client_id: str, explicit_voice: str) -> str:
|
|
if explicit_voice:
|
|
return explicit_voice
|
|
bindings: dict = settings.get("client_voice_bindings") or {}
|
|
if client_id and client_id in bindings:
|
|
return bindings[client_id]
|
|
return settings.get("captures_default_voice") or ""
|
|
|
|
|
|
# ── Routes ────────────────────────────────────────────────────────────────────
|
|
|
|
@router.post("/api/refine-text")
|
|
async def refine_text(request: Request):
|
|
"""Clean up raw STT transcription using a local OpenAI-compatible LLM."""
|
|
data = await request.json()
|
|
text: str = (data.get("text") or "").strip()
|
|
llm_url: str = (data.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
|
model: str = (data.get("model") or "").strip()
|
|
toggles: dict = data.get("toggles") or {}
|
|
|
|
if not text:
|
|
raise HTTPException(400, "No text to refine")
|
|
|
|
rules = []
|
|
if toggles.get("fillers", True):
|
|
rules.append("Remove filler words (um, uh, like, you know, basically, literally, I mean, so, right, etc.)")
|
|
if toggles.get("repetitions", True):
|
|
rules.append("Remove repeated words and false starts (e.g. 'the the dog' → 'the dog', 'I was- I was going' → 'I was going')")
|
|
if toggles.get("corrections", True):
|
|
rules.append("Remove self-corrections and restarts, keeping only the final intended phrasing")
|
|
if toggles.get("punctuation", True):
|
|
rules.append("Fix punctuation, capitalisation, and sentence boundaries")
|
|
if not rules:
|
|
return {"text": text, "original": text}
|
|
|
|
system = (
|
|
"You are a transcription cleanup assistant. "
|
|
"Apply ONLY the following rules to the user's text. "
|
|
"Return ONLY the cleaned text — no explanations, no quotes, no markdown:\n"
|
|
+ "\n".join(f"- {r}" for r in rules)
|
|
)
|
|
payload: dict = {
|
|
"messages": [
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": text},
|
|
],
|
|
"temperature": 0.1,
|
|
"max_tokens": 2048,
|
|
}
|
|
if model:
|
|
payload["model"] = model
|
|
|
|
try:
|
|
resp = requests.post(
|
|
f"{llm_url}/chat/completions",
|
|
json=payload,
|
|
headers={"Authorization": "Bearer no-key"},
|
|
timeout=60,
|
|
)
|
|
resp.raise_for_status()
|
|
refined = resp.json()["choices"][0]["message"]["content"].strip()
|
|
if refined.startswith('"') and refined.endswith('"'):
|
|
refined = refined[1:-1].strip()
|
|
return {"text": refined, "original": text}
|
|
except Exception as e:
|
|
raise HTTPException(502, f"LLM refinement failed: {e}")
|
|
|
|
|
|
@router.post("/api/rewrite-with-persona")
|
|
async def rewrite_with_persona(request: Request):
|
|
"""Rewrite user text in a voice persona's character using a local LLM."""
|
|
data = await request.json()
|
|
text: str = (data.get("text") or "").strip()
|
|
persona: str = (data.get("persona") or "").strip()
|
|
llm_url: str = (data.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
|
model: str = (data.get("model") or "").strip()
|
|
mode: str = (data.get("mode") or "rewrite").strip()
|
|
|
|
if not persona:
|
|
raise HTTPException(400, "No persona defined for this voice")
|
|
if not text and mode != "compose":
|
|
raise HTTPException(400, "No text provided")
|
|
|
|
if mode == "compose":
|
|
system = (
|
|
f"You are a voice assistant with this character: {persona}\n"
|
|
"Write a single natural utterance in this character's voice about the topic given. "
|
|
"Return ONLY the utterance — no quotes, no explanation."
|
|
)
|
|
user_msg = text or "Introduce yourself briefly."
|
|
temp = 0.9
|
|
else:
|
|
system = (
|
|
f"Rephrase the user's text as if spoken by this character: {persona}\n"
|
|
"Keep the same meaning but adapt vocabulary, tone, and style to the character. "
|
|
"Return ONLY the rephrased text — no quotes, no explanation."
|
|
)
|
|
user_msg = text
|
|
temp = 0.3
|
|
|
|
payload: dict = {
|
|
"messages": [
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": user_msg},
|
|
],
|
|
"temperature": temp,
|
|
"max_tokens": 512,
|
|
}
|
|
if model:
|
|
payload["model"] = model
|
|
|
|
try:
|
|
resp = requests.post(
|
|
f"{llm_url}/chat/completions",
|
|
json=payload,
|
|
headers={"Authorization": "Bearer no-key"},
|
|
timeout=60,
|
|
)
|
|
resp.raise_for_status()
|
|
result = resp.json()["choices"][0]["message"]["content"].strip()
|
|
if result.startswith('"') and result.endswith('"'):
|
|
result = result[1:-1].strip()
|
|
return {"text": result, "original": text, "persona": persona}
|
|
except Exception as e:
|
|
raise HTTPException(502, f"LLM persona rewrite failed: {e}")
|
|
|
|
|
|
# ── Audio effects ─────────────────────────────────────────────────────────────
|
|
|
|
def _apply_audio_effects(audio_bytes: bytes, effects: list) -> bytes:
|
|
try:
|
|
from pedalboard import Pedalboard, Reverb, Chorus, Delay, Compressor, Gain, HighpassFilter, LowpassFilter, PitchShift # type: ignore
|
|
import numpy as np # type: ignore
|
|
except ImportError:
|
|
raise RuntimeError("pedalboard is not installed — run: pip install pedalboard numpy")
|
|
|
|
with io.BytesIO(audio_bytes) as buf:
|
|
with wave.open(buf, "rb") as wf:
|
|
n_channels = wf.getnchannels()
|
|
sample_rate = wf.getframerate()
|
|
n_frames = wf.getnframes()
|
|
raw = wf.readframes(n_frames)
|
|
sampwidth = wf.getsampwidth()
|
|
|
|
import numpy as np # noqa: F811
|
|
dtype = {1: np.int8, 2: np.int16, 4: np.int32}.get(sampwidth, np.int16)
|
|
samples = np.frombuffer(raw, dtype=dtype).astype(np.float32) / float(np.iinfo(dtype).max)
|
|
samples = samples.reshape(1, -1) if n_channels == 1 else samples.reshape(-1, n_channels).T
|
|
|
|
board = []
|
|
for fx in effects:
|
|
t = fx.get("type", "")
|
|
p = fx.get("params", {})
|
|
if t == "reverb":
|
|
board.append(Reverb(
|
|
room_size=float(p.get("room_size", 0.35)),
|
|
damping=float(p.get("damping", 0.5)),
|
|
wet_level=float(p.get("wet", 0.25)),
|
|
dry_level=float(p.get("dry", 0.8)),
|
|
))
|
|
elif t == "chorus":
|
|
board.append(Chorus(
|
|
rate_hz=float(p.get("rate_hz", 1.0)),
|
|
depth=float(p.get("depth", 0.25)),
|
|
mix=float(p.get("mix", 0.5)),
|
|
))
|
|
elif t == "delay":
|
|
board.append(Delay(
|
|
delay_seconds=float(p.get("delay_s", 0.25)),
|
|
feedback=float(p.get("feedback", 0.3)),
|
|
mix=float(p.get("mix", 0.4)),
|
|
))
|
|
elif t == "compressor":
|
|
board.append(Compressor(
|
|
threshold_db=float(p.get("threshold_db", -20.0)),
|
|
ratio=float(p.get("ratio", 4.0)),
|
|
attack_ms=float(p.get("attack_ms", 10.0)),
|
|
release_ms=float(p.get("release_ms", 100.0)),
|
|
))
|
|
elif t == "gain":
|
|
board.append(Gain(gain_db=float(p.get("gain_db", 0.0))))
|
|
elif t == "highpass":
|
|
board.append(HighpassFilter(cutoff_frequency_hz=float(p.get("cutoff_hz", 80.0))))
|
|
elif t == "lowpass":
|
|
board.append(LowpassFilter(cutoff_frequency_hz=float(p.get("cutoff_hz", 8000.0))))
|
|
elif t == "pitch_shift":
|
|
board.append(PitchShift(semitones=float(p.get("semitones", 0.0))))
|
|
|
|
if board:
|
|
samples = Pedalboard(board)(samples, sample_rate)
|
|
|
|
out = np.clip(samples, -1.0, 1.0)
|
|
pcm = ((out[0] if out.shape[0] == 1 else out.T.reshape(-1)) * 32767).astype(np.int16).tobytes()
|
|
buf_out = io.BytesIO()
|
|
with wave.open(buf_out, "wb") as wf:
|
|
wf.setnchannels(n_channels)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(sample_rate)
|
|
wf.writeframes(pcm)
|
|
return buf_out.getvalue()
|
|
|
|
|
|
@router.post("/api/audio/effects")
|
|
async def audio_effects(request: Request):
|
|
"""Apply an effects chain to a WAV file."""
|
|
form = await request.form()
|
|
audio_file = form.get("audio")
|
|
effects_json = str(form.get("effects") or "[]")
|
|
|
|
if audio_file is None:
|
|
raise HTTPException(400, "No audio file provided")
|
|
audio_bytes = await audio_file.read()
|
|
try:
|
|
effects = json.loads(effects_json)
|
|
except Exception:
|
|
raise HTTPException(400, "Invalid effects JSON")
|
|
|
|
try:
|
|
result = await asyncio.to_thread(_apply_audio_effects, audio_bytes, effects)
|
|
return Response(content=result, media_type="audio/wav")
|
|
except RuntimeError as e:
|
|
raise HTTPException(501, str(e))
|
|
except Exception as e:
|
|
raise HTTPException(500, f"Effects processing failed: {e}")
|
|
|
|
|
|
# ── Voices export / import ────────────────────────────────────────────────────
|
|
|
|
_EXPORT_SKIP_KEYS = {"groq_api_key", "whisper_api_key", "tts_api_key", "voice_design_api_key", "elevenlabs_api_key"}
|
|
_IMPORT_ALLOWED_SUFFIXES = set(_AUDIO_EXTS + [".reference.txt", ".meta.json", ".jpg", ".jpeg", ".png", ".webp"])
|
|
|
|
|
|
@router.get("/api/voices/export")
|
|
async def voices_export():
|
|
"""Export all voices + non-sensitive settings as a ZIP archive."""
|
|
import zipfile
|
|
from datetime import datetime
|
|
settings = _load_settings()
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
safe_settings = {k: v for k, v in settings.items() if k not in _EXPORT_SKIP_KEYS}
|
|
zf.writestr("settings.json", json.dumps(safe_settings, indent=2))
|
|
if scan_dir.exists():
|
|
for f in scan_dir.rglob("*"):
|
|
if f.is_file():
|
|
try:
|
|
zf.write(f, str(f.relative_to(scan_dir.parent)))
|
|
except Exception:
|
|
pass
|
|
buf.seek(0)
|
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
return Response(
|
|
content=buf.read(),
|
|
media_type="application/zip",
|
|
headers={"Content-Disposition": f'attachment; filename="voices_export_{ts}.zip"'},
|
|
)
|
|
|
|
|
|
@router.post("/api/voices/import")
|
|
async def voices_import(file: UploadFile = File(...)):
|
|
"""Import voices from a ZIP archive (skips settings.json and unsafe paths)."""
|
|
import zipfile
|
|
settings = _load_settings()
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
scan_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
content = await file.read()
|
|
if len(content) > _MAX_UPLOAD_BYTES:
|
|
raise HTTPException(413, "ZIP file too large")
|
|
|
|
try:
|
|
imported = 0
|
|
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
|
for info in zf.infolist():
|
|
if info.is_dir():
|
|
continue
|
|
parts = Path(info.filename).parts
|
|
if any(p in ("..", "") for p in parts) or Path(info.filename).name == "settings.json":
|
|
continue
|
|
if Path(info.filename).suffix.lower() not in _IMPORT_ALLOWED_SUFFIXES:
|
|
continue
|
|
rel = parts[1:] if len(parts) > 1 else parts
|
|
dest = scan_dir / Path(*rel)
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(zf.read(info))
|
|
imported += 1
|
|
except zipfile.BadZipFile:
|
|
raise HTTPException(400, "Not a valid ZIP file")
|
|
except Exception as e:
|
|
raise HTTPException(500, f"Import failed: {e}")
|
|
|
|
return {"ok": True, "imported": imported}
|
|
|
|
|
|
# ── /speak REST endpoint ──────────────────────────────────────────────────────
|
|
|
|
@router.post("/speak")
|
|
async def speak(request: Request):
|
|
data = await request.json()
|
|
text: str = str(data.get("text") or "").strip()
|
|
if not text:
|
|
raise HTTPException(400, "text is required")
|
|
|
|
explicit_voice: str = str(data.get("voice") or data.get("profile_id") or data.get("profile") or "").strip()
|
|
backend: str = _clean_preview_backend(str(data.get("backend") or "voice_clone"))
|
|
apply_persona: bool = bool(data.get("apply_persona") or data.get("personality"))
|
|
client_id: str = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
|
|
|
settings = _load_settings()
|
|
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
|
if not voice:
|
|
raise HTTPException(400, "No voice specified and no default voice configured")
|
|
|
|
if apply_persona:
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
wav = _find_voice_audio(voice, scan_dir)
|
|
if wav:
|
|
persona = _load_meta(wav).get("persona", "")
|
|
if persona:
|
|
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
|
try:
|
|
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url)
|
|
except Exception as e:
|
|
raise HTTPException(502, f"Persona rewrite failed: {e}")
|
|
|
|
instruct = str(data.get("instruct") or "")
|
|
try:
|
|
audio, media_type = await asyncio.to_thread(
|
|
_preview_request_audio, text, voice, settings, instruct, backend
|
|
)
|
|
except Exception as e:
|
|
raise HTTPException(502, f"TTS error: {e}")
|
|
|
|
return Response(content=audio, media_type=media_type)
|
|
|
|
|
|
@router.get("/speak/bindings")
|
|
async def get_speak_bindings():
|
|
s = _load_settings()
|
|
return {"bindings": s.get("client_voice_bindings") or {}}
|
|
|
|
|
|
@router.put("/speak/bindings/{client_id}")
|
|
async def put_speak_binding(client_id: str, request: Request):
|
|
data = await request.json()
|
|
voice = str(data.get("voice") or "").strip()
|
|
if not voice:
|
|
raise HTTPException(400, "voice is required")
|
|
s = _load_settings()
|
|
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
|
bindings[client_id] = voice
|
|
s["client_voice_bindings"] = bindings
|
|
_save_settings(s)
|
|
return {"ok": True, "client_id": client_id, "voice": voice}
|
|
|
|
|
|
@router.delete("/speak/bindings/{client_id}")
|
|
async def delete_speak_binding(client_id: str):
|
|
s = _load_settings()
|
|
bindings: dict = dict(s.get("client_voice_bindings") or {})
|
|
if client_id not in bindings:
|
|
raise HTTPException(404, f"No binding for client '{client_id}'")
|
|
del bindings[client_id]
|
|
s["client_voice_bindings"] = bindings
|
|
_save_settings(s)
|
|
return {"ok": True, "client_id": client_id}
|
|
|
|
|
|
# ── MCP JSON-RPC 2.0 server ───────────────────────────────────────────────────
|
|
|
|
_MCP_SERVER_INFO = {"name": "tts-voice-creator", "version": "1.0.0"}
|
|
|
|
_MCP_TOOLS = [
|
|
{
|
|
"name": "speak",
|
|
"description": "Generate speech audio from text using a cloned voice. Returns a data URI with the WAV audio.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"text": {"type": "string", "description": "Text to synthesize"},
|
|
"voice": {"type": "string", "description": "Voice ID (optional — uses per-client binding or default if omitted)"},
|
|
"apply_persona": {"type": "boolean", "description": "Rewrite text through the voice's persona before synthesis"},
|
|
},
|
|
"required": ["text"],
|
|
},
|
|
},
|
|
{
|
|
"name": "transcribe",
|
|
"description": "Transcribe base64-encoded WAV audio to text.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"audio_base64": {"type": "string", "description": "Base64-encoded audio bytes (WAV preferred)"},
|
|
},
|
|
"required": ["audio_base64"],
|
|
},
|
|
},
|
|
{
|
|
"name": "list_captures",
|
|
"description": "List the 20 most recently generated audio files.",
|
|
"inputSchema": {"type": "object", "properties": {}},
|
|
},
|
|
{
|
|
"name": "list_profiles",
|
|
"description": "List all available voice profiles with their language, persona, and enabled state.",
|
|
"inputSchema": {"type": "object", "properties": {}},
|
|
},
|
|
]
|
|
|
|
|
|
async def _mcp_tool_speak(args: dict, client_id: str) -> dict:
|
|
text = str(args.get("text") or "").strip()
|
|
if not text:
|
|
raise ValueError("text is required")
|
|
explicit_voice = str(args.get("voice") or "").strip()
|
|
apply_persona = bool(args.get("apply_persona", False))
|
|
settings = _load_settings()
|
|
voice = _resolve_speak_voice(settings, client_id, explicit_voice)
|
|
if not voice:
|
|
raise ValueError("No voice specified and no default voice configured")
|
|
if apply_persona:
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
wav = _find_voice_audio(voice, scan_dir)
|
|
if wav:
|
|
persona = _load_meta(wav).get("persona", "")
|
|
if persona:
|
|
llm_url = (settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
|
text = await asyncio.to_thread(_rewrite_with_persona_sync, text, persona, llm_url)
|
|
audio, _media_type = await asyncio.to_thread(
|
|
_preview_request_audio, text, voice, settings, "", "voice_clone"
|
|
)
|
|
audio_b64 = base64.b64encode(audio).decode()
|
|
return {
|
|
"content": [
|
|
{"type": "text", "text": f"Generated {len(audio)} bytes of speech audio for voice '{voice}'."},
|
|
{"type": "resource", "resource": {"uri": f"data:audio/wav;base64,{audio_b64}", "mimeType": "audio/wav"}},
|
|
]
|
|
}
|
|
|
|
|
|
async def _mcp_tool_transcribe(args: dict) -> dict:
|
|
raw = args.get("audio_base64") or ""
|
|
try:
|
|
audio_bytes = base64.b64decode(raw)
|
|
except Exception:
|
|
raise ValueError("audio_base64 is not valid base64")
|
|
tmp = TEMP_DIR / f"{uuid.uuid4().hex}_mcp_transcribe.wav"
|
|
tmp.write_bytes(audio_bytes)
|
|
settings = _load_settings()
|
|
try:
|
|
text, used_backend = await asyncio.to_thread(_transcribe_audio, tmp, settings, "configured")
|
|
finally:
|
|
try:
|
|
tmp.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
return {"content": [{"type": "text", "text": text}], "backend": used_backend}
|
|
|
|
|
|
async def _mcp_tool_list_captures() -> dict:
|
|
settings = _load_settings()
|
|
out_dir = _active_voices_dir(settings)
|
|
if not out_dir.exists():
|
|
return {"content": [{"type": "text", "text": "[]"}]}
|
|
files = sorted(
|
|
(p for p in out_dir.rglob("*.wav") if not _is_internal_voice_file(p)),
|
|
key=lambda p: p.stat().st_mtime,
|
|
reverse=True,
|
|
)[:20]
|
|
result = [{"name": p.stem, "path": str(p), "mtime": p.stat().st_mtime} for p in files]
|
|
return {"content": [{"type": "text", "text": json.dumps(result)}]}
|
|
|
|
|
|
async def _mcp_tool_list_profiles() -> dict:
|
|
settings = _load_settings()
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
profiles = []
|
|
seen: set[str] = set()
|
|
for audio in sorted(_voice_audio_files(scan_dir), key=lambda p: p.stem.lower()):
|
|
if audio.stem in seen:
|
|
continue
|
|
seen.add(audio.stem)
|
|
meta = _load_meta(audio)
|
|
parts = audio.stem.split("_", 2)
|
|
lang = parts[0].upper() if parts else ""
|
|
profiles.append({
|
|
"id": audio.stem,
|
|
"name": audio.stem,
|
|
"lang": lang,
|
|
"persona": meta.get("persona", ""),
|
|
"enabled": meta.get("enabled", True),
|
|
})
|
|
return {"content": [{"type": "text", "text": json.dumps(profiles)}]}
|
|
|
|
|
|
def _mcp_error_response(code: int, message: str, rpc_id) -> Response:
|
|
import logging
|
|
body = {"jsonrpc": "2.0", "error": {"code": code, "message": message}, "id": rpc_id}
|
|
return Response(content=json.dumps(body), media_type="application/json")
|
|
|
|
|
|
@router.post("/mcp")
|
|
async def mcp_jsonrpc(request: Request):
|
|
import logging
|
|
logger = logging.getLogger("uvicorn.error")
|
|
try:
|
|
body = await request.json()
|
|
except Exception:
|
|
return _mcp_error_response(-32700, "Parse error", None)
|
|
|
|
rpc_id = body.get("id")
|
|
method = body.get("method", "")
|
|
params = body.get("params") or {}
|
|
client_id = request.headers.get("X-Voice-Creator-Client-Id", "").strip()
|
|
|
|
try:
|
|
if method == "initialize":
|
|
result = {
|
|
"protocolVersion": "2024-11-05",
|
|
"serverInfo": _MCP_SERVER_INFO,
|
|
"capabilities": {"tools": {}},
|
|
}
|
|
elif method == "notifications/initialized":
|
|
return Response(status_code=204)
|
|
elif method == "tools/list":
|
|
result = {"tools": _MCP_TOOLS}
|
|
elif method == "tools/call":
|
|
tool_name = str(params.get("name") or "")
|
|
tool_args = params.get("arguments") or {}
|
|
if tool_name == "speak":
|
|
result = await _mcp_tool_speak(tool_args, client_id)
|
|
elif tool_name == "transcribe":
|
|
result = await _mcp_tool_transcribe(tool_args)
|
|
elif tool_name == "list_captures":
|
|
result = await _mcp_tool_list_captures()
|
|
elif tool_name == "list_profiles":
|
|
result = await _mcp_tool_list_profiles()
|
|
else:
|
|
return _mcp_error_response(-32601, f"Unknown tool: {tool_name}", rpc_id)
|
|
else:
|
|
return _mcp_error_response(-32601, f"Method not found: {method}", rpc_id)
|
|
except ValueError as e:
|
|
return _mcp_error_response(-32602, str(e), rpc_id)
|
|
except Exception as e:
|
|
logger.exception("MCP tool error in method %s", method)
|
|
return _mcp_error_response(-32000, f"Server error: {e}", rpc_id)
|
|
|
|
return Response(
|
|
content=json.dumps({"jsonrpc": "2.0", "result": result, "id": rpc_id}),
|
|
media_type="application/json",
|
|
)
|
|
|
|
|
|
@router.get("/mcp")
|
|
async def mcp_sse(request: Request):
|
|
"""SSE keep-alive stream (satisfies MCP spec GET /mcp requirement)."""
|
|
from typing import AsyncGenerator
|
|
|
|
async def _keepalive() -> AsyncGenerator[str, None]:
|
|
yield ": mcp-sse-ready\n\n"
|
|
while True:
|
|
await asyncio.sleep(15)
|
|
yield ": ping\n\n"
|
|
return StreamingResponse(
|
|
_keepalive(),
|
|
media_type="text/event-stream",
|
|
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
|
)
|
|
|
|
|
|
# ── Conversation playground ───────────────────────────────────────────────────
|
|
|
|
@router.get("/api/conversation/llm-models")
|
|
async def conversation_llm_models(url: str = ""):
|
|
"""List models from a local LLM endpoint (Ollama / vLLM / LM Studio)."""
|
|
from core.validation import _validate_http_url
|
|
settings = _load_settings()
|
|
base = (url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
|
try:
|
|
base = _validate_http_url(base, allow_private=True)
|
|
r = requests.get(f"{base}/models", timeout=5, headers={"Authorization": "Bearer no-key"})
|
|
if r.status_code == 200:
|
|
payload = r.json()
|
|
data = payload.get("data", []) if isinstance(payload, dict) else []
|
|
models = [
|
|
str(item["id"]) if isinstance(item, dict) and item.get("id") else str(item)
|
|
for item in data if item
|
|
]
|
|
return {"models": models, "url": base}
|
|
except Exception:
|
|
pass
|
|
return {"models": [], "url": base}
|
|
|
|
|
|
@router.post("/api/conversation/turn")
|
|
async def conversation_turn(
|
|
audio: Optional[UploadFile] = None,
|
|
text: str = Form(""),
|
|
stt_backend: str = Form("configured"),
|
|
llm_url: str = Form(""),
|
|
llm_model: str = Form(""),
|
|
tts_backend: str = Form("voice_clone"),
|
|
tts_voice: str = Form(""),
|
|
system_prompt: str = Form("You are a helpful voice assistant. Keep replies short and conversational."),
|
|
history: str = Form("[]"),
|
|
):
|
|
"""Stream a full conversation turn (STT → LLM → TTS) as Server-Sent Events.
|
|
Pass either an audio file (runs STT first) or a plain text string (skips STT).
|
|
"""
|
|
direct_text = text.strip()
|
|
if not direct_text and (audio is None or not getattr(audio, "filename", None)):
|
|
raise HTTPException(400, "Provide either an audio file or a text field")
|
|
|
|
settings = _load_settings()
|
|
eff_llm_url = (llm_url or settings.get("llm_url") or "http://localhost:11434/v1").rstrip("/")
|
|
|
|
# Prepare audio temp files only when an audio upload was provided
|
|
_tmp: Path | None = None
|
|
_wav: Path | None = None
|
|
if not direct_text and audio is not None:
|
|
suffix = Path(audio.filename or "audio.webm").suffix.lower() or ".webm"
|
|
if suffix not in _UPLOAD_EXTS:
|
|
suffix = ".webm"
|
|
_tmp = TEMP_DIR / f"{uuid.uuid4().hex}_conv{suffix}"
|
|
try:
|
|
with _tmp.open("wb") as f:
|
|
_copy_limited(audio.file, f, _MAX_UPLOAD_BYTES)
|
|
_wav = _to_wav_16k(_tmp)
|
|
except Exception as e:
|
|
_tmp.unlink(missing_ok=True)
|
|
raise HTTPException(400, f"Audio upload failed: {e}")
|
|
|
|
try:
|
|
hist = json.loads(history) if history else []
|
|
if not isinstance(hist, list):
|
|
hist = []
|
|
except Exception:
|
|
hist = []
|
|
|
|
stt_be = _clean_stt_backend(stt_backend)
|
|
tts_be = _clean_preview_backend(tts_backend)
|
|
|
|
async def generate():
|
|
t0 = time.monotonic()
|
|
stt_ms = llm_ttft_ms = llm_total_ms = tts_ms = None
|
|
transcript = llm_text = ""
|
|
|
|
def sse(obj: dict) -> str:
|
|
return f"data: {json.dumps(obj)}\n\n"
|
|
|
|
# 1. STT — skipped when caller sends direct text
|
|
if direct_text:
|
|
transcript = direct_text
|
|
yield sse({"type": "transcript", "text": transcript, "stt_ms": None})
|
|
else:
|
|
try:
|
|
t_stt = time.monotonic()
|
|
transcript, _ = await asyncio.to_thread(_transcribe_audio, _wav, settings, stt_be)
|
|
stt_ms = int((time.monotonic() - t_stt) * 1000)
|
|
yield sse({"type": "transcript", "text": transcript, "stt_ms": stt_ms})
|
|
except Exception as e:
|
|
yield sse({"type": "error", "stage": "stt", "message": str(e)})
|
|
return
|
|
finally:
|
|
for p in filter(None, {_tmp, _wav}):
|
|
with contextlib.suppress(Exception):
|
|
p.unlink(missing_ok=True)
|
|
|
|
if not transcript.strip():
|
|
yield sse({"type": "error", "stage": "stt", "message": "No speech detected."})
|
|
return
|
|
|
|
# 2. LLM stream
|
|
messages = [{"role": "system", "content": system_prompt}]
|
|
messages.extend(hist[-20:])
|
|
messages.append({"role": "user", "content": transcript})
|
|
llm_payload: dict = {"messages": messages, "stream": True, "temperature": 0.7, "max_tokens": 512}
|
|
if llm_model:
|
|
llm_payload["model"] = llm_model
|
|
try:
|
|
t_llm = time.monotonic()
|
|
llm_resp = await asyncio.to_thread(lambda: requests.post(
|
|
f"{eff_llm_url}/chat/completions", json=llm_payload,
|
|
headers={"Authorization": "Bearer no-key"}, stream=True, timeout=120,
|
|
))
|
|
llm_resp.raise_for_status()
|
|
ttft_done = False
|
|
for raw_line in llm_resp.iter_lines():
|
|
if not raw_line:
|
|
continue
|
|
line = raw_line.decode("utf-8") if isinstance(raw_line, bytes) else str(raw_line)
|
|
if not line.startswith("data:"):
|
|
continue
|
|
chunk = line[5:].strip()
|
|
if chunk == "[DONE]":
|
|
break
|
|
try:
|
|
obj = json.loads(chunk)
|
|
delta = ((obj.get("choices") or [{}])[0].get("delta") or {}).get("content") or ""
|
|
if not delta and isinstance(obj.get("message"), dict):
|
|
delta = obj["message"].get("content") or ""
|
|
if delta:
|
|
if not ttft_done:
|
|
llm_ttft_ms = int((time.monotonic() - t_llm) * 1000)
|
|
ttft_done = True
|
|
llm_text += delta
|
|
yield sse({"type": "token", "delta": delta})
|
|
except Exception:
|
|
continue
|
|
llm_total_ms = int((time.monotonic() - t_llm) * 1000)
|
|
yield sse({"type": "llm_done", "text": llm_text,
|
|
"llm_ttft_ms": llm_ttft_ms, "llm_total_ms": llm_total_ms})
|
|
except Exception as e:
|
|
yield sse({"type": "error", "stage": "llm", "message": str(e)})
|
|
return
|
|
|
|
if not llm_text.strip():
|
|
yield sse({"type": "error", "stage": "llm", "message": "LLM returned empty response."})
|
|
return
|
|
|
|
# 3. TTS
|
|
try:
|
|
t_tts = time.monotonic()
|
|
audio_bytes, mime = await asyncio.to_thread(
|
|
_preview_request_audio, llm_text, tts_voice, settings, "", tts_be
|
|
)
|
|
tts_ms = int((time.monotonic() - t_tts) * 1000)
|
|
total_ms = int((time.monotonic() - t0) * 1000)
|
|
yield sse({"type": "audio", "b64": base64.b64encode(audio_bytes).decode(), "mime": mime})
|
|
yield sse({"type": "stats", "stt_ms": stt_ms, "llm_ttft_ms": llm_ttft_ms,
|
|
"llm_total_ms": llm_total_ms, "tts_ms": tts_ms, "total_ms": total_ms})
|
|
except Exception as e:
|
|
yield sse({"type": "error", "stage": "tts", "message": str(e)})
|
|
return
|
|
|
|
yield sse({"type": "done"})
|
|
|
|
return StreamingResponse(generate(), media_type="text/event-stream",
|
|
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|