diff --git a/README.md b/README.md index 788d11a..d825046 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ python config/auto_transcribe.py --api-url http://localhost:8010/v1/audio/transc `config/generate_voices.py` runs on container startup and creates `config/voices.json` from your speaker files. +When you start using a new voice for the first time, the server will automatically do the heavy lifting to extract the voice's acoustic fingerprint (a "speaker embedding") and save it as a `.pt` file in the `config/speakers/` directory. Future requests will instantly load this `.pt` file instead of re-analyzing the audio, which dramatically speeds up Time To First Audio (TTFA). ## VoiceDesign voices VoiceDesign does not need reference audio. Define reusable voice personalities in `config/voicedesign_voices.json`: @@ -340,6 +341,14 @@ The first request after container startup can be slower because CUDA graph captu ## Changelog +### v6.4 — 2026-06-20 +**Feature: Fully Automated Speaker Embeddings (.pt files)** + +- Integrated speaker embedding extraction directly into the API server (`openai_server.py`). +- When a new voice is requested for the first time, the server will automatically compute the speaker embedding and save it as a `.pt` file in `/config/speakers/`. +- Future requests for the same voice automatically use the `.pt` file instead of recalculating the prompt from the `.wav` or `.mp3` reference audio. +- This provides the massive TTFA speedup of precomputed embeddings without requiring any manual scripting or configuration. + ### v6.3 — 2026-06-20 **Feature: Precomputed Speaker Embeddings (.pt files)** diff --git a/patches/openai_server.patch b/patches/openai_server.patch index 8103ca1..2ad1031 100644 --- a/patches/openai_server.patch +++ b/patches/openai_server.patch @@ -1,25 +1,63 @@ -diff --git a/examples/openai_server.py b/examples/openai_server.py -index 61047ea..2f95bbd 100644 ---- a/examples/openai_server.py -+++ b/examples/openai_server.py -@@ -169,6 +169,16 @@ def resolve_voice(voice_name: str) -> dict: +--- /tmp/upstream_openai_server.py 2026-06-20 20:43:53.380341702 +0200 ++++ build/examples/openai_server.py 2026-06-20 20:52:17.725652965 +0200 +@@ -145,6 +145,7 @@ + + def resolve_voice(voice_name: str) -> dict: + """Return voice config dict or fall back to default, else raise 400.""" ++ voice_name = voice_name.strip() + if voice_name in voices: + return voices[voice_name] + if default_voice and default_voice in voices: +@@ -168,7 +169,47 @@ # --------------------------------------------------------------------------- -+def _load_voice_clone_prompt(voice_cfg: dict): +-async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, None]: ++def _load_voice_clone_prompt(voice_cfg: dict, voice_name: str, tts_model): + spk_emb_path = voice_cfg.get("speaker_embeddings") or voice_cfg.get("speaker embeddings") + if spk_emb_path and os.path.isfile(spk_emb_path): + try: + return torch.load(spk_emb_path, map_location="cpu", weights_only=False) + except Exception as e: + logger.error("Failed to load speaker embeddings from %s: %s", spk_emb_path, e) -+ return None ++ ++ # Auto-generate if missing ++ ref_audio = voice_cfg.get("ref_audio") ++ if not ref_audio or not os.path.isfile(ref_audio): ++ return None ++ ++ logger.info("Precomputing and saving speaker embedding for voice %r...", voice_name) ++ try: ++ ref_text = voice_cfg.get("ref_text", "") ++ # generate prompt using the model's built-in helper ++ prompt_items = tts_model.model.create_voice_clone_prompt(ref_audio, [ref_text]) ++ vcp = tts_model.model._prompt_items_to_voice_clone_prompt(prompt_items) ++ ++ # save it to the speakers directory ++ # If the file path is already in voice_cfg but doesn't exist, use that, otherwise generate a path ++ if spk_emb_path and not os.path.exists(spk_emb_path) and spk_emb_path.endswith('.pt'): ++ pt_path = spk_emb_path ++ else: ++ pt_path = f"/config/speakers/{voice_name}.pt" ++ ++ os.makedirs(os.path.dirname(pt_path), exist_ok=True) ++ torch.save(vcp, pt_path) ++ logger.info("Saved speaker embedding to %s", pt_path) ++ ++ # update in memory so future requests skip generating ++ voice_cfg["speaker_embeddings"] = pt_path ++ ++ return vcp ++ except Exception as e: ++ logger.error("Failed to precompute speaker embedding: %s", e) ++ return None + + - async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, None]: ++async def _stream_chunks(voice_cfg: dict, text: str, voice_name: str) -> AsyncGenerator[bytes, None]: """ Run generate_voice_clone_streaming in a background thread and yield -@@ -183,11 +193,15 @@ async def _stream_chunks(voice_cfg: dict, text: str) -> AsyncGenerator[bytes, No + raw PCM bytes for each chunk as they arrive. +@@ -182,10 +223,15 @@ for chunk, _sr, _timing in tts_model.generate_voice_clone_streaming( text=text, language=voice_cfg.get("language", "Auto"), @@ -27,9 +65,9 @@ index 61047ea..2f95bbd 100644 + ref_audio=voice_cfg.get("ref_audio"), ref_text=voice_cfg.get("ref_text", ""), chunk_size=voice_cfg.get("chunk_size", 12), - instruct=voice_cfg.get("instruct"), - non_streaming_mode=False, -+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg), ++ instruct=voice_cfg.get("instruct"), ++ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, voice_name, tts_model), + non_streaming_mode=True, + temperature=voice_cfg.get("temperature", 0.8), + top_k=voice_cfg.get("top_k", 50), @@ -37,15 +75,15 @@ index 61047ea..2f95bbd 100644 ): q.put(chunk) except Exception as exc: -@@ -249,9 +263,14 @@ async def create_speech(req: SpeechRequest): +@@ -247,8 +293,14 @@ return tts_model.generate_voice_clone( text=req.input, language=voice_cfg.get("language", "Auto"), - ref_audio=voice_cfg["ref_audio"], + ref_audio=voice_cfg.get("ref_audio"), ref_text=voice_cfg.get("ref_text", ""), - instruct=voice_cfg.get("instruct"), -+ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg), ++ instruct=voice_cfg.get("instruct"), ++ voice_clone_prompt=_load_voice_clone_prompt(voice_cfg, req.voice, tts_model), + non_streaming_mode=True, + temperature=voice_cfg.get("temperature", 0.8), + top_k=voice_cfg.get("top_k", 50), @@ -53,3 +91,28 @@ index 61047ea..2f95bbd 100644 ) audio_arrays, sr = await loop.run_in_executor(None, _generate) +@@ -259,7 +311,7 @@ + async def audio_stream(): + if fmt == "wav": + yield _wav_header(SAMPLE_RATE) # stream with unknown data length +- async for raw_chunk in _stream_chunks(voice_cfg, req.input): ++ async for raw_chunk in _stream_chunks(voice_cfg, req.input, req.voice): + yield raw_chunk + + return StreamingResponse(audio_stream(), media_type=content_type) +@@ -306,6 +358,7 @@ + p.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)") + p.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)") + p.add_argument("--device", default="cuda", help="Torch device (default: cuda)") ++ p.add_argument("--max-seq-len", type=int, default=4096, help="Max sequence length for CUDA graph static cache (default: 4096)") + return p.parse_args() + + +@@ -344,6 +397,7 @@ + args.model, + device=args.device, + dtype=torch.bfloat16, ++ max_seq_len=args.max_seq_len, + ) + SAMPLE_RATE = tts_model.sample_rate + logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)