From 9836261d282386fade118bc61ed3a327956b6944 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Wed, 29 Jul 2026 15:51:54 +0200 Subject: [PATCH] Fix stale speaker embeddings and unregistered Voice Design voices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two engine-level bugs made custom voices unusable in different ways. Stale speaker embeddings (voice clone): A .pt embedding is a cache baked from a specific pairing of reference audio and reference transcript. Re-recording or re-designing a voice replaces those sources but leaves the old .pt in place, and generate_voices.py pointed at it unconditionally — so the server kept cloning from an embedding whose audio tokens no longer matched the transcript stored alongside them. Confirmed on 73 of 155 voices: generation ignored the requested text entirely, emitting short unrelated filler ("Thank you.") or fragments of the previous reference transcript. Voices that had never been re-recorded were unaffected, which is why this looked specific to designed voices. Now an embedding older than its reference audio/transcript is treated as invalid and removed so the server recomputes it. Verified: 73 embeddings regenerated, and voices that previously returned unrelated text now transcribe back to exactly the requested input. Unregistered Voice Design voices: For the VoiceDesign model a voice's identity IS its instruct prompt, but designed voices were never written into voicedesign_voices.json, so the server only knew its 8 bundled presets. resolve_voice() silently substituted the first one, answering requests for a German male character with 'vd_british_male' — the source of the apparent gender flips between takes. - generate_voices.py now mirrors designed voices into the VoiceDesign registry (it runs in the clone container, which is the one with the voice library mounted; /config is shared with the VoiceDesign container). - The VoiceDesign server hot-reloads its registry, matching what the clone server already did, so voices designed while it is running resolve without a restart. - An unknown voice no longer becomes a different one: if the request carries its own instruct that is used, otherwise it is a 404 rather than a silent substitution. - A request instruct is now combined with the voice's registered instruct instead of replacing it. Previously any line carrying an emotion discarded the character's identity and re-rolled a voice from a few words of direction, which made a character drift between lines. - Per-voice temperature/top_p/top_k are honored. Deliberately no seed: generate_voice_design() takes no seed parameter, so designed voices cannot be pinned that way — consistency comes from low temperature/top_p. Verified end to end: three consecutive takes of the same designed voice now hold 86-91 Hz median F0 (was flipping register between takes), and emotional lines stay within 84-86 Hz instead of losing the character entirely. Co-Authored-By: Claude Opus 5 --- config/generate_voices.py | 127 +++++++++++++++++++++++++++++-- config/run_server.py | 6 +- config/run_voicedesign_server.py | 86 +++++++++++++++++++-- 3 files changed, 204 insertions(+), 15 deletions(-) diff --git a/config/generate_voices.py b/config/generate_voices.py index c8a3cfa..b440e32 100644 --- a/config/generate_voices.py +++ b/config/generate_voices.py @@ -96,12 +96,6 @@ for scan_dir in SCAN_DIRS: "chunk_size": 4, } - pt_path = os.path.join("/config/speakers", f"{voice_id}.pt") - if os.path.exists(pt_path): - entry["speaker_embeddings"] = pt_path - else: - entry["speaker_embeddings"] = "" - ref_txt = os.path.join(root, f"{base_name}.reference.txt") txt = os.path.join(root, f"{base_name}.txt") if os.path.exists(ref_txt): @@ -111,6 +105,34 @@ for scan_dir in SCAN_DIRS: with open(txt, encoding="utf-8") as f: entry["ref_text"] = f.read().strip() + # A .pt speaker embedding is a CACHE baked from a specific pairing of + # reference audio + reference transcript. Re-recording or re-designing + # a voice replaces those source files but leaves the old .pt sitting + # there, and this script used to point at it unconditionally — so the + # server kept cloning from an embedding whose internal audio tokens no + # longer matched the transcript it was stored with. Confirmed live on + # 73 voices: output ignored the requested text entirely, emitting short + # unrelated filler ("Thank you.") or fragments of the OLD reference + # transcript. Treat an embedding older than its sources as invalid so + # the server recomputes it from the current audio. + pt_path = os.path.join("/config/speakers", f"{voice_id}.pt") + entry["speaker_embeddings"] = "" + if os.path.exists(pt_path): + try: + pt_mtime = os.path.getmtime(pt_path) + newest_source = os.path.getmtime(audio_path) + for src in (ref_txt, txt): + if os.path.exists(src): + newest_source = max(newest_source, os.path.getmtime(src)) + # 1s slack absorbs filesystem timestamp granularity. + if newest_source > pt_mtime + 1: + print(f" ⚠ stale embedding for {voice_id} — regenerating from current reference") + os.remove(pt_path) + else: + entry["speaker_embeddings"] = pt_path + except OSError as exc: + print(f" ✗ could not validate embedding for {voice_id}: {exc}") + # Preserve any user-added fields from the previous voices.json # (temperature, top_k, top_p, chunk_size overrides, etc.) if voice_id in _existing: @@ -127,3 +149,96 @@ if voices != _existing: else: pass # No changes, do not update mtime + +# --------------------------------------------------------------------------- +# Voice Design registry +# --------------------------------------------------------------------------- +# For the VoiceDesign model there is no reference audio and no embedding — a +# voice's identity IS its instruct prompt. Designed voices were never written +# into voicedesign_voices.json at all, so the server only knew its 8 bundled +# presets and silently substituted one of them for every custom voice +# (confirmed live: requests for a designed male German character were answered +# by the 'vd_british_male' preset). Mirror the app's designed voices here so +# the identity resolves to the prompt it was actually created from. +# +# This runs in the voice-clone container because that is the one with the +# voice library mounted; /config is shared with the VoiceDesign container, +# which hot-reloads this file. + +VOICEDESIGN_OUTPUT = "/config/voicedesign_voices.json" +VOICEDESIGN_NOTE_PREFIX = "Voice Design:" + +_LANG_BY_FLAG = { + "EN": "English", "DE": "German", "FR": "French", "ES": "Spanish", + "IT": "Italian", "PT": "Portuguese", "NL": "Dutch", "PL": "Polish", + "ZH": "Chinese", "JA": "Japanese", "KO": "Korean", +} + + +def _build_voicedesign_registry(): + try: + with open(VOICEDESIGN_OUTPUT, encoding="utf-8") as f: + existing = json.load(f) + except (OSError, json.JSONDecodeError): + existing = {} + + # Keep the bundled vd_* presets; rebuild every app-managed entry. + registry = {k: v for k, v in existing.items() if k.startswith("vd_")} + added = 0 + + for scan_dir in SCAN_DIRS: + if not os.path.exists(scan_dir): + continue + for root, dirs, files in os.walk(scan_dir): + dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) + for filename in sorted(files): + if not filename.endswith(".meta.json"): + continue + try: + with open(os.path.join(root, filename), encoding="utf-8") as f: + meta = json.load(f) + except (OSError, json.JSONDecodeError): + continue + if meta.get("origin") != "designed": + continue + + # Prefer the full prompt. `note` is a display summary the app + # clips to 240 characters, which for older voices is the only + # copy that survives — usable (it still carries gender, accent + # and timbre) but missing the tail of the description. + instruct = str(meta.get("voice_design_prompt") or "").strip() + if not instruct: + note = str(meta.get("note") or "").strip() + if not note.startswith(VOICEDESIGN_NOTE_PREFIX): + continue + instruct = note[len(VOICEDESIGN_NOTE_PREFIX):].strip() + if not instruct: + continue + + base_name = filename[: -len(".meta.json")] + voice_id = make_voice_id(scan_dir, root, base_name) + flag = str(meta.get("flag") or "").upper() + entry = { + "instruct": instruct, + "language": _LANG_BY_FLAG.get(flag) or detect_language(base_name), + } + # Deliberately no "seed": generate_voice_design() takes no seed + # parameter, so a designed voice cannot be pinned that way. + # Consistency across takes comes from low temperature/top_p, + # which the server reads from the entry when present. Preserve + # any values a previous run or the user set by hand. + for key in ("temperature", "top_p", "top_k"): + prev = existing.get(voice_id, {}).get(key) + if prev is not None: + entry[key] = prev + registry[voice_id] = entry + added += 1 + + if registry != existing: + with open(VOICEDESIGN_OUTPUT, "w", encoding="utf-8") as f: + json.dump(registry, f, indent=2, ensure_ascii=False) + print(f"Success! Generated voicedesign_voices.json with {added} designed voices.") + + +_build_voicedesign_registry() + diff --git a/config/run_server.py b/config/run_server.py index 9ba53f7..bdc5671 100644 --- a/config/run_server.py +++ b/config/run_server.py @@ -76,7 +76,11 @@ def _precompute_all_embeddings(): spk_emb_path = voice_cfg.get("speaker_embeddings") or voice_cfg.get("speaker embeddings") if spk_emb_path and os.path.isfile(spk_emb_path): continue - + + if not voice_cfg.get("ref_text"): + logger.warning("Skipping embedding precompute for %r: no ref_text (add a transcript to voices.json)", voice_name) + continue + logger.info("Background precomputing embedding for %r...", voice_name) # We must lock the model to prevent concurrent generation with incoming requests with openai_server._model_lock: diff --git a/config/run_voicedesign_server.py b/config/run_voicedesign_server.py index 66b6e3c..136c198 100644 --- a/config/run_voicedesign_server.py +++ b/config/run_voicedesign_server.py @@ -8,6 +8,7 @@ No ref_audio needed — the instruct text fully describes the voice. """ import json import logging +import os import queue import threading import asyncio @@ -32,6 +33,8 @@ app = FastAPI() tts_model: FasterQwen3TTS = None voices: dict = {} default_voice: str = None +voices_file_path: str = None +last_voices_mtime: float = 0.0 SAMPLE_RATE = 24000 DEFAULT_MAX_NEW_TOKENS = 2048 _model_lock = threading.Lock() @@ -143,14 +146,52 @@ def _to_mp3_bytes(audio: np.ndarray, sr: int) -> bytes: return buf.getvalue() -def resolve_voice(name: str) -> dict: +def _reload_voices_if_changed(): + """Pick up voices added to the registry since startup. + + The registry used to be read once in main(), so any voice designed while + the server was up stayed invisible until a manual restart — and an unknown + voice silently became a bundled preset (see resolve_voice). The voice-clone + server already hot-reloads its registry; this brings VoiceDesign in line. + """ + global voices, last_voices_mtime + if not voices_file_path or not os.path.exists(voices_file_path): + return + try: + mtime = os.path.getmtime(voices_file_path) + if mtime > last_voices_mtime: + with open(voices_file_path, encoding="utf-8") as f: + loaded = json.load(f) + if isinstance(loaded, dict) and loaded: + voices = loaded + last_voices_mtime = mtime + _build_voice_list() + logger.info("Hot-reloaded %d voices from %s", len(voices), voices_file_path) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("Failed to hot-reload voices: %s", exc) + + +def resolve_voice(name: str, has_request_instruct: bool = False) -> dict: + _reload_voices_if_changed() cfg = voices.get(name) if cfg: return cfg - if default_voice and default_voice in voices: - logger.warning("Voice %r not found, falling back to %r", name, default_voice) - return voices[default_voice] - raise HTTPException(status_code=404, detail=f"Voice {name!r} not found") + # Falling back to another voice is meaningless here: for VoiceDesign the + # instruct IS the voice, so substituting the first registered preset does + # not degrade the result, it silently returns a completely different + # character — confirmed live as the cause of a German male character being + # read by 'vd_british_male', including apparent gender flips between takes. + if has_request_instruct: + # The caller described the voice inline, so nothing is missing. + logger.info("Voice %r not registered; using the instruct supplied with the request", name) + return {} + raise HTTPException( + status_code=404, + detail=( + f"Voice {name!r} is not registered and the request carried no 'instruct' " + f"to describe it. Known voices: {sorted(voices)}" + ), + ) # --------------------------------------------------------------------------- @@ -158,14 +199,32 @@ def resolve_voice(name: str) -> dict: # --------------------------------------------------------------------------- def _request_generation_params(req: SpeechRequest, voice_cfg: dict) -> dict: - instruct = req.instruct if req.instruct is not None else voice_cfg.get("instruct", "") + # A registered voice's instruct is its IDENTITY; an instruct sent with the + # request is a per-line direction (emotion, delivery). Replacing the former + # with the latter — the previous behaviour — meant every line that carried + # any emotion discarded the character's voice entirely and re-rolled a new + # one from a few words of direction, which is why a character's voice + # drifted between lines. Combine them, identity first. + base_instruct = str(voice_cfg.get("instruct", "") or "").strip() + line_instruct = str(req.instruct or "").strip() + if base_instruct and line_instruct and line_instruct != base_instruct: + instruct = f"{base_instruct} {line_instruct}" + else: + instruct = line_instruct or base_instruct + language = req.language or voice_cfg.get("language", "English") - return { + params = { "text": req.input, "instruct": instruct, "language": language, "max_new_tokens": req.max_new_tokens or int(voice_cfg.get("max_new_tokens", DEFAULT_MAX_NEW_TOKENS)), } + # Per-voice sampling overrides — the only way to make a designed voice + # reproducible, since generate_voice_design() accepts no seed. + for key in ("temperature", "top_p", "top_k"): + if voice_cfg.get(key) is not None: + params[key] = float(voice_cfg[key]) if key != "top_k" else int(voice_cfg[key]) + return params async def _stream_chunks(params: dict, speed: float): @@ -248,7 +307,7 @@ async def create_speech(req: SpeechRequest): if not req.input: raise HTTPException(status_code=400, detail="'input' text is empty") - voice_cfg = resolve_voice(req.voice) + voice_cfg = resolve_voice(req.voice, has_request_instruct=bool((req.instruct or "").strip())) params = _request_generation_params(req, voice_cfg) fmt = req.response_format.lower() @@ -320,18 +379,22 @@ def _build_voice_list(): @app.get("/v1/models") async def list_models(): + _reload_voices_if_changed() return _models_response @app.get("/v1/audio/voices") async def list_audio_voices(): + _reload_voices_if_changed() return _models_response @app.get("/v1/audio/models") async def list_audio_models(): + _reload_voices_if_changed() return _models_response @app.get("/speakers") async def get_speakers(): + _reload_voices_if_changed() return list(voices.keys()) @app.options("/{path:path}") @@ -345,6 +408,7 @@ async def options_handler(path: str): def main(): global voices, default_voice, SAMPLE_RATE, DEFAULT_MAX_NEW_TOKENS, _load_model_kwargs + global voices_file_path, last_voices_mtime parser = argparse.ArgumentParser() parser.add_argument("--model", default="/models/Qwen3-TTS-VoiceDesign") @@ -357,10 +421,16 @@ def main(): DEFAULT_MAX_NEW_TOKENS = args.max_seq_len _load_model_kwargs = args + voices_file_path = args.voices with open(args.voices) as f: voices = json.load(f) + try: + last_voices_mtime = os.path.getmtime(args.voices) + except OSError: + last_voices_mtime = 0.0 default_voice = next(iter(voices), None) _build_voice_list() + logger.info("Loaded %d voices from %s", len(voices), args.voices) uvicorn.run(app, host=args.host, port=args.port, log_level="info")