mARTin-B78
ea50267c30
Introduces the new Studio section (Source -> Characters -> Voices -> Perform & Export) that reuses the existing Read Aloud/Library/Script Rehearsal code via DOM reparenting instead of duplicating it, and rolls up a long tail of bugs found while producing a real audiobook through it: umlaut-eating name sanitizers, a voice picker that mispositioned itself and capped results at 60, PDF pagination silently breaking on trimmed \f markers, a race letting stale audio keep playing after a new line was clicked, an alias-overlap bug that could silently redirect a voice/image save onto the wrong character, voice design failing outright during brief TTS backend restarts instead of retrying, sparse cast entries defaulting to English/wrong gender, and a reassigned voice never reaching an already-open Stage session or invalidating its cached audio. Also adds a persistent per-line audio cache, audiobook export browsing/download, and an inline voice-design prompt editor. Full details in CHANGELOG.md. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
691 lines
29 KiB
Python
691 lines
29 KiB
Python
"""TTS request helpers: config, audio request, streaming, voice design, NVIDIA, benchmark."""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import struct
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from pydub import AudioSegment
|
|
|
|
from core.config import (
|
|
_clean_preview_backend,
|
|
_preview_backend_base_url,
|
|
_tts_extra_params,
|
|
_apply_tts_extra_params,
|
|
_post_tts_with_fallback,
|
|
)
|
|
from core.constants import (
|
|
_VOICES_DIR_DEFAULT,
|
|
_VOICE_DESIGN_DEFAULT,
|
|
_VOICE_DESIGN_MODEL,
|
|
_MAX_TTS_OUTPUT_SECONDS,
|
|
)
|
|
from core.audio import _to_wav_24k, _duration as _dur
|
|
from core.voice import _find_voice_audio, _read_reference_text
|
|
|
|
|
|
# ── Language helpers ──────────────────────────────────────────────────────────
|
|
|
|
_NVIDIA_LANGUAGE_CODES = {
|
|
"EN": "en-US", "DE": "de-DE", "ES": "es-ES", "FR": "fr-FR",
|
|
"IT": "it-IT", "PT": "pt-PT", "NL": "nl-NL", "PL": "pl-PL",
|
|
}
|
|
|
|
# Maps 2-letter prefix from voice ID (e.g. "DE_M_…") to full language name
|
|
# used by the faster-qwen3-tts clone server's `language` field.
|
|
_VOICE_LANG_NAMES = {
|
|
"EN": "English", "DE": "German", "ES": "Spanish", "FR": "French",
|
|
"IT": "Italian", "PT": "Portuguese", "NL": "Dutch", "PL": "Polish",
|
|
"ZH": "Chinese", "JA": "Japanese", "KO": "Korean", "RU": "Russian",
|
|
"AR": "Arabic", "HI": "Hindi", "TR": "Turkish", "SV": "Swedish",
|
|
}
|
|
|
|
|
|
def _voice_language_name(voice: str) -> str:
|
|
"""Return the full language name inferred from the voice ID prefix (e.g. DE_M_ → German)."""
|
|
prefix = str(voice or "").split("_", 1)[0].upper()
|
|
return _VOICE_LANG_NAMES.get(prefix, "")
|
|
|
|
|
|
def _nvidia_clone_language_code(voice: str, language: str = "") -> str:
|
|
raw = str(language or "").strip()
|
|
if raw and raw.lower() not in {"auto", "*"}:
|
|
return raw
|
|
prefix = str(voice or "").split("_", 1)[0].upper()
|
|
return _NVIDIA_LANGUAGE_CODES.get(prefix, "en-US")
|
|
|
|
|
|
def _form_value(value) -> str:
|
|
if isinstance(value, bool):
|
|
return "true" if value else "false"
|
|
return str(value)
|
|
|
|
|
|
# ── WAV helpers ───────────────────────────────────────────────────────────────
|
|
|
|
def _wav_data_offset(data: bytes) -> int | None:
|
|
if len(data) < 12:
|
|
return None
|
|
if data[:4] != b"RIFF" or data[8:12] != b"WAVE":
|
|
return 0
|
|
pos = 12
|
|
while pos + 8 <= len(data):
|
|
chunk_sz = struct.unpack_from("<I", data, pos + 4)[0]
|
|
if data[pos:pos + 4] == b"data":
|
|
return pos + 8
|
|
pos += 8 + chunk_sz + (chunk_sz % 2)
|
|
return None
|
|
|
|
|
|
def _audio_duration_from_bytes(audio: bytes, media_type: str) -> float | None:
|
|
try:
|
|
source_format = "wav" if audio[:4] == b"RIFF" or "wav" in media_type.lower() else None
|
|
return len(AudioSegment.from_file(io.BytesIO(audio), format=source_format)) / 1000.0
|
|
except Exception:
|
|
offset = _wav_data_offset(audio)
|
|
if offset is not None and offset > 0 and len(audio) > offset:
|
|
return (len(audio) - offset) / (24000 * 2)
|
|
return None
|
|
|
|
|
|
# ── TTS request config & audio ────────────────────────────────────────────────
|
|
|
|
def _tts_request_config(
|
|
text: str,
|
|
voice: str,
|
|
settings: dict,
|
|
response_format: str = "wav",
|
|
instruct: str = "",
|
|
url_override: str = "",
|
|
api_key_override: str | None = None,
|
|
backend_override: str = "",
|
|
extra_backend: str = "",
|
|
) -> tuple[str, dict, dict]:
|
|
from core.constants import _TTS_DEFAULT
|
|
from core.validation import _validate_http_url
|
|
|
|
tts_url = _validate_http_url(url_override or settings.get("tts_url", _TTS_DEFAULT), allow_private=True).rstrip("/")
|
|
backend = backend_override or settings.get("tts_backend", "openai")
|
|
tts_key = (api_key_override if api_key_override is not None else settings.get("tts_api_key", "")).strip()
|
|
tts_hdrs = {"Authorization": f"Bearer {tts_key}"} if tts_key else {}
|
|
|
|
if backend == "localai":
|
|
endpoint, payload = f"{tts_url}/tts", {"input": text, "model": voice, "response_format": response_format}
|
|
elif backend == "pocket":
|
|
endpoint, payload = f"{tts_url}/v1/audio/speech", {"input": text, "voice": voice, "response_format": response_format}
|
|
else:
|
|
endpoint, payload = f"{tts_url}/v1/audio/speech", {"model": "tts-1", "input": text, "voice": voice, "response_format": response_format}
|
|
if instruct.strip():
|
|
payload["instruct"] = instruct.strip()
|
|
# Include language derived from voice ID prefix so the TTS server knows what
|
|
# language to synthesise (e.g. DE_M_… → German). Only set when the backend
|
|
# is the local faster-qwen3-tts clone server — skip for OpenAI / cloud APIs.
|
|
_eff_backend = extra_backend or backend_override or "voice_clone"
|
|
if _eff_backend in ("voice_clone", "streaming", "customvoice"):
|
|
lang = _voice_language_name(voice)
|
|
if lang:
|
|
payload["language"] = lang
|
|
_apply_tts_extra_params(payload, settings, _eff_backend)
|
|
return endpoint, payload, tts_hdrs
|
|
|
|
|
|
def _tts_request_audio(
|
|
text: str,
|
|
voice: str,
|
|
settings: dict,
|
|
instruct: str = "",
|
|
url_override: str = "",
|
|
api_key_override: str | None = None,
|
|
backend_override: str = "",
|
|
extra_backend: str = "",
|
|
) -> tuple[bytes, str]:
|
|
endpoint, payload, tts_hdrs = _tts_request_config(
|
|
text, voice, settings, "wav", instruct, url_override, api_key_override, backend_override, extra_backend
|
|
)
|
|
resp = _post_tts_with_fallback(endpoint, payload, tts_hdrs, timeout=120)
|
|
resp.raise_for_status()
|
|
audio = resp.content
|
|
if not audio:
|
|
raise RuntimeError("backend returned empty audio")
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return audio, media_type
|
|
|
|
|
|
# ── TTS streaming helpers ─────────────────────────────────────────────────────
|
|
|
|
def _tts_stream_request_config(text: str, voice: str, settings: dict, instruct: str = "") -> tuple[str, dict, dict]:
|
|
from core.constants import _TTS_STREAM_DEFAULT
|
|
from core.validation import _validate_http_url
|
|
|
|
stream_url = settings.get("tts_stream_url") or settings.get("tts_url") or _TTS_STREAM_DEFAULT
|
|
tts_url = _validate_http_url(stream_url, allow_private=True).rstrip("/")
|
|
tts_key = settings.get("tts_api_key", "").strip()
|
|
tts_hdrs = {"Authorization": f"Bearer {tts_key}"} if tts_key else {}
|
|
payload = {"model": "tts-1", "input": text, "voice": voice, "response_format": "wav"}
|
|
if instruct.strip():
|
|
payload["instruct"] = instruct.strip()
|
|
_apply_tts_extra_params(payload, settings, "streaming")
|
|
return f"{tts_url}/v1/audio/speech", payload, tts_hdrs
|
|
|
|
|
|
def _open_tts_stream_response(text: str, voice: str, settings: dict, instruct: str = "") -> requests.Response:
|
|
endpoint, payload, tts_hdrs = _tts_stream_request_config(text, voice, settings, instruct)
|
|
resp = _post_tts_with_fallback(endpoint, payload, tts_hdrs, stream=True, timeout=(10, 900))
|
|
try:
|
|
resp.raise_for_status()
|
|
except Exception as exc:
|
|
detail = ""
|
|
try:
|
|
detail = resp.text[:500]
|
|
except Exception:
|
|
pass
|
|
resp.close()
|
|
raise RuntimeError(f"streaming backend error: {exc}{(': ' + detail) if detail else ''}") from exc
|
|
return resp
|
|
|
|
|
|
def _read_tts_stream_response(resp: requests.Response) -> tuple[bytes, str]:
|
|
try:
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return b"".join(chunk for chunk in resp.iter_content(chunk_size=64 * 1024) if chunk), media_type
|
|
finally:
|
|
resp.close()
|
|
|
|
|
|
def _iter_tts_stream_response(resp: requests.Response):
|
|
try:
|
|
for chunk in resp.iter_content(chunk_size=64 * 1024):
|
|
if chunk:
|
|
yield chunk
|
|
finally:
|
|
resp.close()
|
|
|
|
|
|
# ── Voice design request helpers ──────────────────────────────────────────────
|
|
|
|
def _apply_voice_design_gender(instruct: str, gender: str) -> str:
|
|
gender = (gender or "").strip().upper()
|
|
if gender == "F":
|
|
prefix = (
|
|
"MANDATORY SPEAKER IDENTITY: Female speaker / woman.\n"
|
|
"gender: Female.\n"
|
|
"Use a clearly feminine vocal timbre, light-to-medium resonance, and soprano or mezzo-soprano pitch range.\n"
|
|
"Avoid male baritone, bass, chest-heavy, or masculine vocal qualities."
|
|
)
|
|
elif gender == "M":
|
|
prefix = (
|
|
"MANDATORY SPEAKER IDENTITY: Male speaker / man.\n"
|
|
"gender: Male.\n"
|
|
"Use a clearly masculine vocal timbre, medium-to-deep resonance, and tenor, baritone, or bass pitch range.\n"
|
|
"Avoid feminine soprano or mezzo-soprano vocal qualities."
|
|
)
|
|
else:
|
|
return instruct
|
|
return f"{prefix}\n\n{instruct.strip()}"
|
|
|
|
|
|
def _voice_design_request_audio(
|
|
instruct: str,
|
|
text: str,
|
|
language: str,
|
|
settings: dict,
|
|
gender: str = "",
|
|
) -> tuple[bytes, str]:
|
|
from core.validation import _validate_http_url
|
|
vd_url = _validate_http_url(settings.get("voice_design_url") or _VOICE_DESIGN_DEFAULT, allow_private=True).rstrip("/")
|
|
vd_key = (settings.get("voice_design_api_key") or settings.get("tts_api_key", "")).strip()
|
|
vd_hdrs = {"Authorization": f"Bearer {vd_key}"} if vd_key else {}
|
|
payload = {
|
|
"model": _VOICE_DESIGN_MODEL,
|
|
"input": text,
|
|
"instruct": _apply_voice_design_gender(instruct, gender),
|
|
"language": language,
|
|
"response_format": "wav",
|
|
}
|
|
_apply_tts_extra_params(payload, settings, "voice_design")
|
|
resp = _post_tts_with_fallback(f"{vd_url}/v1/audio/speech", payload, vd_hdrs, timeout=180)
|
|
resp.raise_for_status()
|
|
audio = resp.content
|
|
if not audio:
|
|
raise RuntimeError("backend returned empty audio")
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return audio, media_type
|
|
|
|
|
|
def _voice_design_voice_request_audio(
|
|
voice: str,
|
|
text: str,
|
|
settings: dict,
|
|
instruct: str = "",
|
|
language: str = "Auto",
|
|
) -> tuple[bytes, str]:
|
|
from core.validation import _validate_http_url
|
|
vd_url = _validate_http_url(settings.get("voice_design_url") or _VOICE_DESIGN_DEFAULT, allow_private=True).rstrip("/")
|
|
vd_key = (settings.get("voice_design_api_key") or settings.get("tts_api_key", "")).strip()
|
|
vd_hdrs = {"Authorization": f"Bearer {vd_key}"} if vd_key else {}
|
|
payload = {
|
|
"model": _VOICE_DESIGN_MODEL,
|
|
"input": text,
|
|
"voice": voice,
|
|
"response_format": "wav",
|
|
}
|
|
if instruct.strip():
|
|
payload["instruct"] = instruct.strip()
|
|
if language and language != "Auto":
|
|
payload["language"] = language
|
|
_apply_tts_extra_params(payload, settings, "voice_design")
|
|
resp = _post_tts_with_fallback(f"{vd_url}/v1/audio/speech", payload, vd_hdrs, timeout=180)
|
|
resp.raise_for_status()
|
|
audio = resp.content
|
|
if not audio:
|
|
raise RuntimeError("backend returned empty audio")
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return audio, media_type
|
|
|
|
|
|
# ── NVIDIA clone request ──────────────────────────────────────────────────────
|
|
|
|
def _nvidia_clone_request_audio(
|
|
text: str,
|
|
voice: str,
|
|
settings: dict,
|
|
mode: str = "zeroshot",
|
|
reference_transcript: str = "",
|
|
language: str = "",
|
|
) -> tuple[bytes, str]:
|
|
from core.validation import _validate_http_url
|
|
|
|
mode = "flow" if str(mode).lower().endswith("flow") else "zeroshot"
|
|
backend = "nvidia_flow" if mode == "flow" else "nvidia_zeroshot"
|
|
base_url = _validate_http_url(_preview_backend_base_url(settings, backend), allow_private=True).rstrip("/")
|
|
key = (settings.get("voice_design_api_key") or settings.get("tts_api_key", "")).strip()
|
|
headers = {"Authorization": f"Bearer {key}"} if key else {}
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
audio_path = _find_voice_audio(voice, scan_dir)
|
|
if audio_path is None:
|
|
raise RuntimeError(f"reference voice not found in library: {voice}")
|
|
prompt_wav = _to_wav_24k(audio_path)
|
|
has_ref, saved_transcript = _read_reference_text(audio_path)
|
|
prompt_transcript = str(reference_transcript or saved_transcript or "").strip()
|
|
if mode == "flow" and not prompt_transcript:
|
|
raise RuntimeError("NVIDIA Magpie Flow requires the selected voice to have an exact saved reference transcript")
|
|
data = {
|
|
"language": _nvidia_clone_language_code(voice, language),
|
|
"text": text,
|
|
}
|
|
if mode == "flow":
|
|
data["audio_prompt_transcript"] = prompt_transcript
|
|
params = _tts_extra_params(settings, backend)
|
|
for k, v in params.items():
|
|
if k in {"audio_prompt", "audio_prompt_transcript", "text", "language"}:
|
|
continue
|
|
data[k] = _form_value(v)
|
|
endpoint = f"{base_url}/v1/audio/synthesize"
|
|
with prompt_wav.open("rb") as f:
|
|
files = {"audio_prompt": ("prompt.wav", f, "audio/wav")}
|
|
resp = requests.post(endpoint, data=data, files=files, headers=headers, timeout=180)
|
|
if resp.status_code in {400, 404, 415, 422} and params:
|
|
try:
|
|
resp.close()
|
|
except Exception:
|
|
pass
|
|
fallback = {k: v for k, v in data.items() if k in {"language", "text", "audio_prompt_transcript"}}
|
|
with prompt_wav.open("rb") as f:
|
|
files = {"audio_prompt": ("prompt.wav", f, "audio/wav")}
|
|
resp = requests.post(endpoint, data=fallback, files=files, headers=headers, timeout=180)
|
|
resp.raise_for_status()
|
|
audio = resp.content
|
|
if not audio:
|
|
raise RuntimeError("NVIDIA clone backend returned empty audio")
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return audio, media_type
|
|
|
|
|
|
# ── Fish-Speech request (clone from saved WAV + inline emotion markers) ──────
|
|
|
|
def _fishspeech_emotion_prefix(instruct: str) -> str:
|
|
"""Turn the per-line style instruction into a Fish-Speech inline emotion marker.
|
|
|
|
The Rehearser sends ``"Speak in a {emotion} manner. {persona}"`` — the persona is
|
|
already carried by the cloned reference WAV, so we only forward the emotion as a
|
|
``(emotion)`` tag, which Fish-Speech honours for per-line tone control.
|
|
"""
|
|
import re
|
|
s = (instruct or "").strip()
|
|
if not s:
|
|
return ""
|
|
m = re.search(r"speak(?:ing)?\s+in\s+(?:a|an)\s+([a-z\- ]+?)\s+manner", s, re.I)
|
|
if m:
|
|
return f"({m.group(1).strip().lower()}) "
|
|
return f"({s}) " if len(s) <= 40 else ""
|
|
|
|
|
|
def _fishspeech_request_audio(
|
|
text: str,
|
|
voice: str,
|
|
settings: dict,
|
|
instruct: str = "",
|
|
language: str = "",
|
|
) -> tuple[bytes, str]:
|
|
"""Synthesize via Fish-Speech: clone the voice's saved reference WAV (consistent
|
|
speaker identity) while applying the line's tone as an inline emotion marker."""
|
|
import base64
|
|
import hashlib
|
|
from core.validation import _validate_http_url
|
|
base_url = _validate_http_url(_preview_backend_base_url(settings, "fishspeech"), allow_private=True).rstrip("/")
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
audio_path = _find_voice_audio(voice, scan_dir)
|
|
if audio_path is None:
|
|
raise RuntimeError(f"reference voice not found in library: {voice}")
|
|
wav = _to_wav_24k(audio_path)
|
|
_has_ref, ref_text = _read_reference_text(audio_path)
|
|
audio_b64 = base64.b64encode(wav.read_bytes()).decode("ascii")
|
|
# Stable per-voice seed → reduces run-to-run drift on top of the reference clone.
|
|
seed = int(hashlib.md5(voice.encode("utf-8")).hexdigest()[:8], 16)
|
|
|
|
payload = {
|
|
"text": _fishspeech_emotion_prefix(instruct) + text,
|
|
"format": "wav",
|
|
"references": [{"audio": audio_b64, "text": ref_text or ""}],
|
|
"seed": seed,
|
|
"use_memory_cache": "on",
|
|
"chunk_length": 200,
|
|
"normalize": True,
|
|
}
|
|
resp = requests.post(f"{base_url}/v1/tts", json=payload, timeout=180)
|
|
resp.raise_for_status()
|
|
audio = resp.content
|
|
if not audio or len(audio) < 256:
|
|
raise RuntimeError("Fish-Speech returned empty audio")
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return audio, media_type
|
|
|
|
|
|
# ── VibeVoice request ─────────────────────────────────────────────────────────
|
|
|
|
def _vibevoice_request_audio(text: str, settings: dict) -> tuple[bytes, str]:
|
|
url = _preview_backend_base_url(settings, "vibevoice").rstrip("/")
|
|
resp = requests.post(f"{url}/tts", json={"text": text}, timeout=60)
|
|
resp.raise_for_status()
|
|
audio = resp.content
|
|
if not audio:
|
|
raise RuntimeError("VibeVoice backend returned empty audio")
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
return audio, media_type
|
|
|
|
|
|
# ── Preview request dispatcher ────────────────────────────────────────────────
|
|
|
|
def _preview_request_audio(
|
|
text: str,
|
|
voice: str,
|
|
settings: dict,
|
|
instruct: str = "",
|
|
backend: str = "voice_clone",
|
|
) -> tuple[bytes, str]:
|
|
backend = _clean_preview_backend(backend)
|
|
if backend == "streaming":
|
|
resp = _open_tts_stream_response(text, voice, settings, instruct)
|
|
audio, media_type = _read_tts_stream_response(resp)
|
|
if not audio:
|
|
raise RuntimeError("backend returned empty audio")
|
|
return audio, media_type
|
|
if backend == "customvoice":
|
|
return _tts_request_audio(
|
|
text, voice, settings, instruct,
|
|
_preview_backend_base_url(settings, "customvoice"),
|
|
settings.get("tts_api_key", ""),
|
|
"openai",
|
|
"customvoice",
|
|
)
|
|
if backend == "voice_design":
|
|
return _voice_design_voice_request_audio(voice, text, settings, instruct)
|
|
if backend == "nvidia_magpie":
|
|
return _tts_request_audio(
|
|
text, voice, settings, instruct,
|
|
_preview_backend_base_url(settings, "nvidia_magpie"),
|
|
settings.get("tts_api_key", ""),
|
|
"nvidia_magpie",
|
|
"nvidia_magpie",
|
|
)
|
|
if backend == "nvidia_zeroshot":
|
|
return _nvidia_clone_request_audio(text, voice, settings, "zeroshot")
|
|
if backend == "nvidia_flow":
|
|
return _nvidia_clone_request_audio(text, voice, settings, "flow")
|
|
if backend == "kokoro":
|
|
return _tts_request_audio(
|
|
text, voice, settings, instruct,
|
|
url_override=_preview_backend_base_url(settings, "kokoro"),
|
|
api_key_override=settings.get("tts_api_key", ""),
|
|
backend_override="openai",
|
|
extra_backend="kokoro",
|
|
)
|
|
if backend == "vibevoice":
|
|
return _vibevoice_request_audio(text, settings)
|
|
if backend == "fishspeech":
|
|
return _fishspeech_request_audio(text, voice, settings, instruct)
|
|
if backend == "xtts":
|
|
return _tts_request_audio(
|
|
text, voice, settings, instruct,
|
|
url_override=_preview_backend_base_url(settings, "xtts"),
|
|
api_key_override=settings.get("tts_api_key", ""),
|
|
backend_override="openai",
|
|
extra_backend="xtts",
|
|
)
|
|
return _tts_request_audio(text, voice, settings, instruct)
|
|
|
|
|
|
# ── Response format helpers ───────────────────────────────────────────────────
|
|
|
|
def _requested_response_format(data: dict) -> str:
|
|
requested = str(data.get("response_format") or data.get("format") or "wav").strip().lower()
|
|
if requested in {"mp3", "mpeg"}:
|
|
return "mp3"
|
|
if requested in {"wav", "pcm"}:
|
|
return "wav"
|
|
return "wav"
|
|
|
|
|
|
def _audio_ext_media(response_format: str) -> tuple[str, str]:
|
|
if response_format == "mp3":
|
|
return "mp3", "audio/mpeg"
|
|
return "wav", "audio/wav"
|
|
|
|
|
|
def _prepare_proxy_audio(audio: bytes, media_type: str, response_format: str) -> tuple[bytes, str, str, float | None, bool]:
|
|
"""Rewrite backend audio so clients receive playable headers and requested format."""
|
|
try:
|
|
source_format = "wav" if audio[:4] == b"RIFF" or "wav" in media_type.lower() else None
|
|
segment = AudioSegment.from_file(io.BytesIO(audio), format=source_format)
|
|
except Exception:
|
|
ext, wanted_media = _audio_ext_media(response_format)
|
|
return audio, media_type or wanted_media, ext, None, False
|
|
|
|
duration = len(segment) / 1000.0
|
|
clipped = False
|
|
if _MAX_TTS_OUTPUT_SECONDS > 0 and duration > _MAX_TTS_OUTPUT_SECONDS:
|
|
segment = segment[:int(_MAX_TTS_OUTPUT_SECONDS * 1000)]
|
|
duration = len(segment) / 1000.0
|
|
clipped = True
|
|
ext, wanted_media = _audio_ext_media(response_format)
|
|
out = io.BytesIO()
|
|
export_format = "mp3" if response_format == "mp3" else "wav"
|
|
segment.export(out, format=export_format)
|
|
return out.getvalue(), wanted_media, ext, duration, clipped
|
|
|
|
|
|
# ── Route sound helpers ───────────────────────────────────────────────────────
|
|
|
|
def _route_sound_path(settings: dict, value: str) -> Path | None:
|
|
from core.validation import _safe_child_path
|
|
from core.voice import _AUDIO_EXTS
|
|
|
|
value = str(value or "").strip()
|
|
if not value:
|
|
return None
|
|
scan_dir = Path(settings.get("voices_scan_dir", _VOICES_DIR_DEFAULT))
|
|
candidate = Path(value)
|
|
if not candidate.is_absolute():
|
|
candidate = scan_dir / candidate
|
|
path = _safe_child_path(scan_dir, candidate)
|
|
if not path.exists() or not path.is_file() or path.suffix.lower() not in _AUDIO_EXTS:
|
|
raise RuntimeError(f"Route sound not found or unsupported: {value}")
|
|
return path
|
|
|
|
|
|
def _sound_segment(path: Path) -> AudioSegment:
|
|
return AudioSegment.from_file(str(path)).set_channels(1).set_sample_width(2).set_frame_rate(24000)
|
|
|
|
|
|
def _apply_route_sounds(audio: bytes, media_type: str, route: dict | None, settings: dict) -> tuple[bytes, str, list[str]]:
|
|
if not route:
|
|
return audio, media_type, []
|
|
before = _route_sound_path(settings, str(route.get("before_sound", "")))
|
|
after = _route_sound_path(settings, str(route.get("after_sound", "")))
|
|
if not before and not after:
|
|
return audio, media_type, []
|
|
|
|
source_format = "wav" if audio[:4] == b"RIFF" or "wav" in media_type.lower() else None
|
|
speech = AudioSegment.from_file(io.BytesIO(audio), format=source_format)
|
|
speech = speech.set_channels(1).set_sample_width(2).set_frame_rate(24000)
|
|
combined = AudioSegment.empty()
|
|
applied = []
|
|
if before:
|
|
combined += _sound_segment(before)
|
|
applied.append(f"before:{before.name}")
|
|
combined += speech
|
|
if after:
|
|
combined += _sound_segment(after)
|
|
applied.append(f"after:{after.name}")
|
|
|
|
out = io.BytesIO()
|
|
combined.export(out, format="wav")
|
|
return out.getvalue(), "audio/wav", applied
|
|
|
|
|
|
# ── Voice design dialogue helpers ─────────────────────────────────────────────
|
|
|
|
def _parse_voice_design_dialogue(instruct: str, script: str) -> tuple[dict[str, str], list[tuple[str, str]]] | None:
|
|
import re
|
|
speakers: dict[str, str] = {}
|
|
for raw in instruct.splitlines():
|
|
line = raw.strip()
|
|
if not line:
|
|
continue
|
|
match = re.match(r'^"?([^":]+)"?\s*:\s*"?(.+?)"?$', line)
|
|
if match:
|
|
speakers[match.group(1).strip()] = match.group(2).strip()
|
|
|
|
turns: list[tuple[str, str]] = []
|
|
for raw in script.splitlines():
|
|
line = raw.strip()
|
|
if not line:
|
|
continue
|
|
match = re.match(r"^([^:]{1,40}):\s*(.+)$", line)
|
|
if match:
|
|
speaker = match.group(1).strip()
|
|
text = match.group(2).strip()
|
|
if speaker in speakers and text:
|
|
turns.append((speaker, text))
|
|
|
|
if len(speakers) < 2 or len(turns) < 2:
|
|
return None
|
|
if len({speaker for speaker, _text in turns}) < 2:
|
|
return None
|
|
return speakers, turns
|
|
|
|
|
|
def _infer_voice_design_gender(description: str) -> str:
|
|
import re
|
|
text = f" {description.lower()} "
|
|
if re.search(r"\b(female|woman|girl|feminine|soprano|mezzo-soprano|mezzo)\b", text):
|
|
return "F"
|
|
if re.search(r"\b(male|man|boy|masculine|tenor|baritone|bass)\b", text):
|
|
return "M"
|
|
return ""
|
|
|
|
|
|
def _voice_design_dialogue_request_audio(
|
|
speakers: dict[str, str],
|
|
turns: list[tuple[str, str]],
|
|
language: str,
|
|
settings: dict,
|
|
) -> tuple[bytes, str]:
|
|
from core.audio import _audio_segment_from_wav, _wav_bytes_from_segment
|
|
combined = AudioSegment.silent(duration=120, frame_rate=24000).set_channels(1).set_sample_width(2)
|
|
pause = AudioSegment.silent(duration=180, frame_rate=24000).set_channels(1).set_sample_width(2)
|
|
|
|
for speaker, text in turns:
|
|
description = speakers[speaker]
|
|
turn_instruct = f'Speaker "{speaker}".\n{description}'
|
|
audio, _media_type = _voice_design_request_audio(
|
|
turn_instruct,
|
|
text,
|
|
language,
|
|
settings,
|
|
gender=_infer_voice_design_gender(description),
|
|
)
|
|
combined += _audio_segment_from_wav(audio) + pause
|
|
|
|
return _wav_bytes_from_segment(combined), "audio/wav"
|
|
|
|
|
|
# ── Benchmark request ─────────────────────────────────────────────────────────
|
|
|
|
def _tts_benchmark_request(text: str, voice: str, settings: dict, label: str, is_designed: bool = False) -> dict:
|
|
start = time.perf_counter()
|
|
first_audio_at = None
|
|
raw = bytearray()
|
|
media_type = "audio/wav"
|
|
|
|
if is_designed:
|
|
# A designed voice has no reference WAV to clone from — it can only
|
|
# ever be synthesized through the voice_design engine (same dispatch
|
|
# as /api/tts-preview's backend=='voice_design' branch), never the
|
|
# generic voice_clone-style request this function otherwise builds.
|
|
# Previously every voice benchmarked through the one fixed tts_url
|
|
# regardless of origin, so a designed voice's benchmark only ever
|
|
# "worked" by coincidence when that unrelated clone engine happened
|
|
# to also be reachable — confirmed live: with it down, EVERY voice
|
|
# in an all-designed batch failed the benchmark even though the
|
|
# voice_design engine itself was reachable the whole time.
|
|
audio_bytes, media_type = _voice_design_voice_request_audio(voice, text, settings)
|
|
raw.extend(audio_bytes)
|
|
first_audio_at = time.perf_counter()
|
|
else:
|
|
endpoint, payload, tts_hdrs = _tts_request_config(text, voice, settings, "wav")
|
|
with _post_tts_with_fallback(endpoint, payload, tts_hdrs, stream=True, timeout=180) as resp:
|
|
resp.raise_for_status()
|
|
media_type = resp.headers.get("content-type", "audio/wav").split(";", 1)[0] or "audio/wav"
|
|
for chunk in resp.iter_content(chunk_size=512):
|
|
if not chunk:
|
|
continue
|
|
raw.extend(chunk)
|
|
if first_audio_at is None:
|
|
if "wav" in media_type.lower():
|
|
offset = _wav_data_offset(bytes(raw))
|
|
if offset is not None and len(raw) > offset:
|
|
first_audio_at = time.perf_counter()
|
|
else:
|
|
first_audio_at = time.perf_counter()
|
|
|
|
total = time.perf_counter() - start
|
|
if not raw:
|
|
raise RuntimeError("backend returned empty audio")
|
|
audio_sec = _audio_duration_from_bytes(bytes(raw), media_type)
|
|
rtf = total / audio_sec if audio_sec and audio_sec > 0 else None
|
|
speed = audio_sec / total if audio_sec and total > 0 else None
|
|
return {
|
|
"ok": True,
|
|
"label": label,
|
|
"text": text,
|
|
"ttfa_ms": round(((first_audio_at or time.perf_counter()) - start) * 1000, 1),
|
|
"total_sec": round(total, 3),
|
|
"audio_sec": round(audio_sec, 3) if audio_sec is not None else None,
|
|
"rtf": round(rtf, 3) if rtf is not None else None,
|
|
"speed": round(speed, 3) if speed is not None else None,
|
|
"bytes": len(raw),
|
|
}
|