mARTin-B78/tts-dgx-spark-faster-qwen3-tts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync remote URLs: make GitHub primary, Gitea backup

Browse Source
This commit is contained in:
mARTin-B78 2026-06-26 11:24:32 +02:00
parent 529599d98c
commit dbcbddedb4
1 changed files with 165 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
patches/openai_server.patch
View File

@ -1,8 +1,21 @@
diff --git a/examples/openai_server.py b/examples/openai_server.py
index 7845b2f..d10065f 100644
index 2199e14..d10065f 100644
--- a/examples/openai_server.py
+++ b/examples/openai_server.py
@@ -72,6 +72,24 @@ last_voices_mtime: float = 0.0
@@ -36,6 +36,7 @@ API usage:
"""
import argparse
import asyncio
+import hashlib
import io
import json
import logging
@@ -66,9 +67,46 @@ app = FastAPI(title="faster-qwen3-tts OpenAI-compatible API")
tts_model = None
voices: dict = {}
+voices_file_path: Optional[str] = None
+last_voices_mtime: float = 0.0
default_voice: Optional[str] = None
SAMPLE_RATE = 24000 # updated once the model loads
_model_lock = threading.Lock() # prevent concurrent GPU inference
@ -24,10 +37,27 @@ index 7845b2f..d10065f 100644
+ )
+ logger.info("Aligner loaded.")
+ return aligner_model
+
+
+def _voice_seed(voice_name: str) -> int:
+ """Return a stable per-voice seed derived from the voice name.
+
+ Used as the default when no explicit 'seed' is set in voices.json.
+ MD5 is used only for its stable byte output — not for security.
+ """
+ return int(hashlib.md5(voice_name.encode()).hexdigest(), 16) % (2 ** 31)
+
+
+def _seed_rng(seed: int) -> None:
+ """Seed PyTorch CPU and CUDA RNGs for reproducible sampling."""
+ torch.manual_seed(seed)
+ if torch.cuda.is_available():
+ torch.cuda.manual_seed_all(seed)
+
def _voice_seed(voice_name: str) -> int:
@@ -99,8 +117,8 @@ class SpeechRequest(BaseModel):
# ---------------------------------------------------------------------------
# Request / response models
@@ -79,8 +117,8 @@ class SpeechRequest(BaseModel):
model: str = "tts-1"
input: str
voice: str = "alloy"
@ -38,16 +68,78 @@ index 7845b2f..d10065f 100644
# ---------------------------------------------------------------------------
@@ -243,7 +261,7 @@ def _load_voice_clone_prompt(voice_cfg: dict, voice_name: str, tts_model):
return None
@@ -145,6 +183,21 @@ def _to_mp3_bytes(pcm: np.ndarray, sample_rate: int) -> bytes:
def resolve_voice(voice_name: str) -> dict:
"""Return voice config dict or fall back to default, else raise 400."""
+ global voices, last_voices_mtime
+ voice_name = voice_name.strip()
+
+ # Hot-reload voices.json if it was modified
+ if voices_file_path and os.path.exists(voices_file_path):
+ try:
+ current_mtime = os.path.getmtime(voices_file_path)
+ if current_mtime > last_voices_mtime:
+ with open(voices_file_path, "r", encoding="utf-8") as f:
+ voices = json.load(f)
+ last_voices_mtime = current_mtime
+ logger.info("Hot-reloaded %d voices from %s", len(voices), voices_file_path)
+ except Exception as e:
+ logger.warning("Failed to hot-reload voices.json: %s", e)
+
if voice_name in voices:
return voices[voice_name]
if default_voice and default_voice in voices:
@@ -168,7 +221,47 @@ def resolve_voice(voice_name: str) -> dict:
# ---------------------------------------------------------------------------
-async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGenerator[bytes, None]:
-async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, None]:
+def _load_voice_clone_prompt(voice_cfg: dict, voice_name: str, tts_model):
+ spk_emb_path = voice_cfg.get("speaker_embeddings") or voice_cfg.get("speaker embeddings")
+ if spk_emb_path and os.path.isfile(spk_emb_path):
+ try:
+ return torch.load(spk_emb_path, map_location="cpu", weights_only=False)
+ except Exception as e:
+ logger.error("Failed to load speaker embeddings from %s: %s", spk_emb_path, e)
+
+ # Auto-generate if missing
+ ref_audio = voice_cfg.get("ref_audio")
+ if not ref_audio or not os.path.isfile(ref_audio):
+ return None
+
+ logger.info("Precomputing and saving speaker embedding for voice %r...", voice_name)
+ try:
+ ref_text = voice_cfg.get("ref_text", "")
+ # generate prompt using the model's built-in helper
+ prompt_items = tts_model.model.create_voice_clone_prompt(ref_audio, [ref_text])
+ vcp = tts_model.model._prompt_items_to_voice_clone_prompt(prompt_items)
+
+ # save it to the speakers directory
+ # If the file path is already in voice_cfg but doesn't exist, use that, otherwise generate a path
+ if spk_emb_path and not os.path.exists(spk_emb_path) and spk_emb_path.endswith('.pt'):
+ pt_path = spk_emb_path
+ else:
+ pt_path = f"/config/speakers/{voice_name}.pt"
+
+ os.makedirs(os.path.dirname(pt_path), exist_ok=True)
+ torch.save(vcp, pt_path)
+ logger.info("Saved speaker embedding to %s", pt_path)
+
+ # update in memory so future requests skip generating
+ voice_cfg["speaker_embeddings"] = pt_path
+
+ return vcp
+ except Exception as e:
+ logger.error("Failed to precompute speaker embedding: %s", e)
+ return None
+
+
+async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str, speed: float) -> AsyncGenerator[bytes, None]:
"""
Run generate_voice_clone_streaming in a background thread and yield
raw PCM bytes for each chunk as they arrive.
@@ -252,6 +270,31 @@ async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGe
@@ -177,21 +270,63 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No
_DONE = object()
def producer():
@ -78,10 +170,21 @@ index 7845b2f..d10065f 100644
+
try:
with _model_lock:
_seed_rng(voice_cfg.get("seed", _voice_seed(voice_name)))
@@ -268,11 +311,22 @@ async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGe
top_k=voice_cfg.get("top_k", 50),
top_p=voice_cfg.get("top_p", 0.9),
+ _seed_rng(voice_cfg.get("seed", _voice_seed(voice_name)))
for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming(
text=text,
language=voice_cfg.get("language", "Auto"),
- ref_audio=voice_cfg["ref_audio"],
+ ref_audio=voice_cfg.get("ref_audio"),
ref_text=voice_cfg.get("ref_text", ""),
chunk_size=voice_cfg.get("chunk_size", 12),
- non_streaming_mode=False,
+ instruct=voice_cfg.get("instruct"),
+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, voice_name, tts_model),
+ non_streaming_mode=True,
+ temperature=voice_cfg.get("temperature", 0.8),
+ top_k=voice_cfg.get("top_k", 50),
+ top_p=voice_cfg.get("top_p", 0.9),
):
- q.put(chunk)
+ raw = _to_pcm16(chunk)
@ -104,7 +207,7 @@ index 7845b2f..d10065f 100644
thread = threading.Thread(target=producer, daemon=True)
thread.start()
@@ -284,7 +338,7 @@ async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGe
@@ -203,7 +338,7 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No
break
if isinstance(item, Exception):
raise item
@ -113,7 +216,7 @@ index 7845b2f..d10065f 100644
# ---------------------------------------------------------------------------
@@ -301,7 +355,8 @@ async def health():
@@ -220,7 +355,8 @@ async def health():
async def create_speech(req: SpeechRequest):
if tts_model is None:
raise HTTPException(status_code=503, detail="Model not loaded")
@ -123,7 +226,7 @@ index 7845b2f..d10065f 100644
raise HTTPException(status_code=400, detail="'input' text is empty")
voice_cfg = resolve_voice(req.voice)
@@ -311,16 +366,17 @@ async def create_speech(req: SpeechRequest):
@@ -230,36 +366,77 @@ async def create_speech(req: SpeechRequest):
"wav": "audio/wav",
"pcm": "audio/pcm",
"mp3": "audio/mpeg",
@ -144,7 +247,21 @@ index 7845b2f..d10065f 100644
loop = asyncio.get_event_loop()
def _generate():
@@ -341,13 +397,46 @@ async def create_speech(req: SpeechRequest):
with _model_lock:
+ _seed_rng(voice_cfg.get("seed", _voice_seed(req.voice)))
return tts_model.generate_voice_clone(
text=req.input,
language=voice_cfg.get("language", "Auto"),
- ref_audio=voice_cfg["ref_audio"],
+ ref_audio=voice_cfg.get("ref_audio"),
ref_text=voice_cfg.get("ref_text", ""),
+ instruct=voice_cfg.get("instruct"),
+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, req.voice, tts_model),
+ non_streaming_mode=True,
+ temperature=voice_cfg.get("temperature", 0.8),
+ top_k=voice_cfg.get("top_k", 50),
+ top_p=voice_cfg.get("top_p", 0.9),
)
audio_arrays, sr = await loop.run_in_executor(None, _generate)
audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32)
@ -187,8 +304,38 @@ index 7845b2f..d10065f 100644
async def audio_stream():
if fmt == "wav":
yield _wav_header(SAMPLE_RATE) # stream with unknown data length
- async for raw_chunk in _stream_chunks(voice_cfg, req.input, req.voice):
- async for raw_chunk in _stream_chunks(voice_cfg, req.input):
+ async for raw_chunk in _stream_chunks(voice_cfg, req.input, req.voice, req.speed):
yield raw_chunk
return StreamingResponse(audio_stream(), media_type=content_type)
@@ -306,16 +483,20 @@ def _parse_args():
p.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)")
p.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)")
p.add_argument("--device", default="cuda", help="Torch device (default: cuda)")
+ p.add_argument("--max-seq-len", type=int, default=4096, help="Max sequence length for CUDA graph static cache (default: 4096)")
return p.parse_args()
def main():
- global tts_model, voices, default_voice, SAMPLE_RATE
+ global tts_model, voices, voices_file_path, last_voices_mtime, default_voice, SAMPLE_RATE
args = _parse_args()
# Build voice registry
if args.voices:
+ voices_file_path = args.voices
+ if os.path.exists(args.voices):
+ last_voices_mtime = os.path.getmtime(args.voices)
with open(args.voices) as f:
voices = json.load(f)
default_voice = next(iter(voices))
@@ -344,6 +525,7 @@ def main():
args.model,
device=args.device,
dtype=torch.bfloat16,
+ max_seq_len=args.max_seq_len,
)
SAMPLE_RATE = tts_model.sample_rate
logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)