119 lines
5.5 KiB
Diff
119 lines
5.5 KiB
Diff
--- /tmp/upstream_openai_server.py 2026-06-20 20:43:53.380341702 +0200
|
|
+++ build/examples/openai_server.py 2026-06-20 20:52:17.725652965 +0200
|
|
@@ -145,6 +145,7 @@
|
|
|
|
def resolve_voice(voice_name: str) -> dict:
|
|
"""Return voice config dict or fall back to default, else raise 400."""
|
|
+ voice_name = voice_name.strip()
|
|
if voice_name in voices:
|
|
return voices[voice_name]
|
|
if default_voice and default_voice in voices:
|
|
@@ -168,7 +169,47 @@
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
-async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, None]:
|
|
+def _load_voice_clone_prompt(voice_cfg: dict, voice_name: str, tts_model):
|
|
+ spk_emb_path = voice_cfg.get("speaker_embeddings") or voice_cfg.get("speaker embeddings")
|
|
+ if spk_emb_path and os.path.isfile(spk_emb_path):
|
|
+ try:
|
|
+ return torch.load(spk_emb_path, map_location="cpu", weights_only=False)
|
|
+ except Exception as e:
|
|
+ logger.error("Failed to load speaker embeddings from %s: %s", spk_emb_path, e)
|
|
+
|
|
+ # Auto-generate if missing
|
|
+ ref_audio = voice_cfg.get("ref_audio")
|
|
+ if not ref_audio or not os.path.isfile(ref_audio):
|
|
+ return None
|
|
+
|
|
+ logger.info("Precomputing and saving speaker embedding for voice %r...", voice_name)
|
|
+ try:
|
|
+ ref_text = voice_cfg.get("ref_text", "")
|
|
+ # generate prompt using the model's built-in helper
|
|
+ prompt_items = tts_model.model.create_voice_clone_prompt(ref_audio, [ref_text])
|
|
+ vcp = tts_model.model._prompt_items_to_voice_clone_prompt(prompt_items)
|
|
+
|
|
+ # save it to the speakers directory
|
|
+ # If the file path is already in voice_cfg but doesn't exist, use that, otherwise generate a path
|
|
+ if spk_emb_path and not os.path.exists(spk_emb_path) and spk_emb_path.endswith('.pt'):
|
|
+ pt_path = spk_emb_path
|
|
+ else:
|
|
+ pt_path = f"/config/speakers/{voice_name}.pt"
|
|
+
|
|
+ os.makedirs(os.path.dirname(pt_path), exist_ok=True)
|
|
+ torch.save(vcp, pt_path)
|
|
+ logger.info("Saved speaker embedding to %s", pt_path)
|
|
+
|
|
+ # update in memory so future requests skip generating
|
|
+ voice_cfg["speaker_embeddings"] = pt_path
|
|
+
|
|
+ return vcp
|
|
+ except Exception as e:
|
|
+ logger.error("Failed to precompute speaker embedding: %s", e)
|
|
+ return None
|
|
+
|
|
+
|
|
+async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGenerator[bytes, None]:
|
|
"""
|
|
Run generate_voice_clone_streaming in a background thread and yield
|
|
raw PCM bytes for each chunk as they arrive.
|
|
@@ -182,10 +223,15 @@
|
|
for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming(
|
|
text=text,
|
|
language=voice_cfg.get("language", "Auto"),
|
|
- ref_audio=voice_cfg["ref_audio"],
|
|
+ ref_audio=voice_cfg.get("ref_audio"),
|
|
ref_text=voice_cfg.get("ref_text", ""),
|
|
chunk_size=voice_cfg.get("chunk_size", 12),
|
|
- non_streaming_mode=False,
|
|
+ instruct=voice_cfg.get("instruct"),
|
|
+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, voice_name, tts_model),
|
|
+ non_streaming_mode=True,
|
|
+ temperature=voice_cfg.get("temperature", 0.8),
|
|
+ top_k=voice_cfg.get("top_k", 50),
|
|
+ top_p=voice_cfg.get("top_p", 0.9),
|
|
):
|
|
q.put(chunk)
|
|
except Exception as exc:
|
|
@@ -247,8 +293,14 @@
|
|
return tts_model.generate_voice_clone(
|
|
text=req.input,
|
|
language=voice_cfg.get("language", "Auto"),
|
|
- ref_audio=voice_cfg["ref_audio"],
|
|
+ ref_audio=voice_cfg.get("ref_audio"),
|
|
ref_text=voice_cfg.get("ref_text", ""),
|
|
+ instruct=voice_cfg.get("instruct"),
|
|
+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, req.voice, tts_model),
|
|
+ non_streaming_mode=True,
|
|
+ temperature=voice_cfg.get("temperature", 0.8),
|
|
+ top_k=voice_cfg.get("top_k", 50),
|
|
+ top_p=voice_cfg.get("top_p", 0.9),
|
|
)
|
|
|
|
audio_arrays, sr = await loop.run_in_executor(None, _generate)
|
|
@@ -259,7 +311,7 @@
|
|
async def audio_stream():
|
|
if fmt == "wav":
|
|
yield _wav_header(SAMPLE_RATE) # stream with unknown data length
|
|
- async for raw_chunk in _stream_chunks(voice_cfg, req.input):
|
|
+ async for raw_chunk in _stream_chunks(voice_cfg, req.input, req.voice):
|
|
yield raw_chunk
|
|
|
|
return StreamingResponse(audio_stream(), media_type=content_type)
|
|
@@ -306,6 +358,7 @@
|
|
p.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)")
|
|
p.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)")
|
|
p.add_argument("--device", default="cuda", help="Torch device (default: cuda)")
|
|
+ p.add_argument("--max-seq-len", type=int, default=4096, help="Max sequence length for CUDA graph static cache (default: 4096)")
|
|
return p.parse_args()
|
|
|
|
|
|
@@ -344,6 +397,7 @@
|
|
args.model,
|
|
device=args.device,
|
|
dtype=torch.bfloat16,
|
|
+ max_seq_len=args.max_seq_len,
|
|
)
|
|
SAMPLE_RATE = tts_model.sample_rate
|
|
logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)
|