--- /tmp/upstream_openai_server.py 2026-06-20 20:43:53.380341702 +0200 +++ build/examples/openai_server.py 2026-06-20 21:18:30.515352046 +0200 @@ -66,6 +66,8 @@ tts_model = None voices: dict = {} +voices_file_path: Optional[str] = None +last_voices_mtime: float = 0.0 default_voice: Optional[str] = None SAMPLE_RATE = 24000 # updated once the model loads _model_lock = threading.Lock() # prevent concurrent GPU inference @@ -145,6 +147,21 @@ def resolve_voice(voice_name: str) -> dict: """Return voice config dict or fall back to default, else raise 400.""" + global voices, last_voices_mtime + voice_name = voice_name.strip() + + # Hot-reload voices.json if it was modified + if voices_file_path and os.path.exists(voices_file_path): + try: + current_mtime = os.path.getmtime(voices_file_path) + if current_mtime > last_voices_mtime: + with open(voices_file_path, "r", encoding="utf-8") as f: + voices = json.load(f) + last_voices_mtime = current_mtime + logger.info("Hot-reloaded %d voices from %s", len(voices), voices_file_path) + except Exception as e: + logger.warning("Failed to hot-reload voices.json: %s", e) + if voice_name in voices: return voices[voice_name] if default_voice and default_voice in voices: @@ -168,7 +185,47 @@ # --------------------------------------------------------------------------- -async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, None]: +def _load_voice_clone_prompt(voice_cfg: dict, voice_name: str, tts_model): + spk_emb_path = voice_cfg.get("speaker_embeddings") or voice_cfg.get("speaker embeddings") + if spk_emb_path and os.path.isfile(spk_emb_path): + try: + return torch.load(spk_emb_path, map_location="cpu", weights_only=False) + except Exception as e: + logger.error("Failed to load speaker embeddings from %s: %s", spk_emb_path, e) + + # Auto-generate if missing + ref_audio = voice_cfg.get("ref_audio") + if not ref_audio or not os.path.isfile(ref_audio): + return None + + logger.info("Precomputing and saving speaker embedding for voice %r...", voice_name) + try: + ref_text = voice_cfg.get("ref_text", "") + # generate prompt using the model's built-in helper + prompt_items = tts_model.model.create_voice_clone_prompt(ref_audio, [ref_text]) + vcp = tts_model.model._prompt_items_to_voice_clone_prompt(prompt_items) + + # save it to the speakers directory + # If the file path is already in voice_cfg but doesn't exist, use that, otherwise generate a path + if spk_emb_path and not os.path.exists(spk_emb_path) and spk_emb_path.endswith('.pt'): + pt_path = spk_emb_path + else: + pt_path = f"/config/speakers/{voice_name}.pt" + + os.makedirs(os.path.dirname(pt_path), exist_ok=True) + torch.save(vcp, pt_path) + logger.info("Saved speaker embedding to %s", pt_path) + + # update in memory so future requests skip generating + voice_cfg["speaker_embeddings"] = pt_path + + return vcp + except Exception as e: + logger.error("Failed to precompute speaker embedding: %s", e) + return None + + +async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGenerator[bytes, None]: """ Run generate_voice_clone_streaming in a background thread and yield raw PCM bytes for each chunk as they arrive. @@ -182,10 +239,15 @@ for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming( text=text, language=voice_cfg.get("language", "Auto"), - ref_audio=voice_cfg["ref_audio"], + ref_audio=voice_cfg.get("ref_audio"), ref_text=voice_cfg.get("ref_text", ""), chunk_size=voice_cfg.get("chunk_size", 12), - non_streaming_mode=False, + instruct=voice_cfg.get("instruct"), + voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, voice_name, tts_model), + non_streaming_mode=True, + temperature=voice_cfg.get("temperature", 0.8), + top_k=voice_cfg.get("top_k", 50), + top_p=voice_cfg.get("top_p", 0.9), ): q.put(chunk) except Exception as exc: @@ -247,8 +309,14 @@ return tts_model.generate_voice_clone( text=req.input, language=voice_cfg.get("language", "Auto"), - ref_audio=voice_cfg["ref_audio"], + ref_audio=voice_cfg.get("ref_audio"), ref_text=voice_cfg.get("ref_text", ""), + instruct=voice_cfg.get("instruct"), + voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, req.voice, tts_model), + non_streaming_mode=True, + temperature=voice_cfg.get("temperature", 0.8), + top_k=voice_cfg.get("top_k", 50), + top_p=voice_cfg.get("top_p", 0.9), ) audio_arrays, sr = await loop.run_in_executor(None, _generate) @@ -259,7 +327,7 @@ async def audio_stream(): if fmt == "wav": yield _wav_header(SAMPLE_RATE) # stream with unknown data length - async for raw_chunk in _stream_chunks(voice_cfg, req.input): + async for raw_chunk in _stream_chunks(voice_cfg, req.input, req.voice): yield raw_chunk return StreamingResponse(audio_stream(), media_type=content_type) @@ -306,16 +374,20 @@ p.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)") p.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)") p.add_argument("--device", default="cuda", help="Torch device (default: cuda)") + p.add_argument("--max-seq-len", type=int, default=4096, help="Max sequence length for CUDA graph static cache (default: 4096)") return p.parse_args() def main(): - global tts_model, voices, default_voice, SAMPLE_RATE + global tts_model, voices, voices_file_path, last_voices_mtime, default_voice, SAMPLE_RATE args = _parse_args() # Build voice registry if args.voices: + voices_file_path = args.voices + if os.path.exists(args.voices): + last_voices_mtime = os.path.getmtime(args.voices) with open(args.voices) as f: voices = json.load(f) default_voice = next(iter(voices)) @@ -344,6 +416,7 @@ args.model, device=args.device, dtype=torch.bfloat16, + max_seq_len=args.max_seq_len, ) SAMPLE_RATE = tts_model.sample_rate logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)