Add multi-source voice pipeline with VoiceDesign support

- generate_voices.py: scan /config/speakers and /voices recursively, support .ogg and .m4a (M4A auto-converted via ffmpeg), sanitise voice IDs - auto_transcribe.py: scan both host paths recursively, support all formats, use parakeet-asr on port 8010 - docker-compose.yml: mount /home/sparky/Projekte/TTS_Voices/speakers as /voices, add faster-qwen3-tts-voicedesign service on port 8021 - run_voicedesign_server.py: OpenAI-compatible server for VoiceDesign model - voicedesign_voices.json: 8 British/German VoiceDesign voices Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-12 01:47:19 +02:00 · 2026-05-12 01:47:19 +02:00 · 295de3059d
commit 295de3059d
parent b7b917db66
6 changed files with 500 additions and 133 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,21 +1,42 @@
-# Audio files (add your own voice references locally)
+# Upstream faster-qwen3-tts repo (tracked separately)
+build/
+
+# Personal voice recordings — keep only reference transcripts
 config/speakers/*.wav
 config/speakers/*.mp3
-config/speakers/originals_backup/

-# Generated at runtime
+# Private speakers — entire subtree
+config/speakers/privat/
+
+# Generated at container startup from speakers/
 config/voices.json
-config/voices.json.bak
+config/voices.json*

-# Environment
-.env
-*.pyc
-__pycache__/
+# Converted private audio (M4A → WAV, generated at runtime)
+config/converted/

-# Editor backups
+# Editor backup files
+*.py~
+*.yml~
+*.yaml~
 *~
-*.swp

-# OS
-.DS_Store
-Thumbs.db
+# Claude Code settings
+.claude/
+config/.claude/
+
+# Random generated files in project root
+json
+os
+*.txt
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+venv/
+
+# Environment files
+.env
+.env.*
--- a/config/auto_transcribe.py
+++ b/config/auto_transcribe.py
@ -1,93 +1,65 @@
-"""
-Batch-transcribe speaker reference audio files using a local Whisper-compatible API.
-
-Creates .reference.txt files alongside each audio file in the speakers directory.
-These transcriptions are used by generate_voices.py to build the voice registry.
-
-Usage:
-    python auto_transcribe.py [--api-url http://localhost:8010/v1/audio/transcriptions]
-
-IMPORTANT: Reference audio for voice cloning should be 5-15 seconds long.
-Longer files will produce poor cloning results and slow down inference.
-"""
-
 import os
-import sys
-import json
-import argparse
 import requests
+import json

-def main():
-    parser = argparse.ArgumentParser(description="Batch-transcribe speaker reference audio")
-    parser.add_argument("--api-url", default="http://localhost:8010/v1/audio/transcriptions",
-                        help="Whisper-compatible transcription API URL")
-    parser.add_argument("--speaker-dir", default="./speakers",
-                        help="Directory containing speaker audio files")
-    parser.add_argument("--model", default="whisper-1",
-                        help="Transcription model name")
-    args = parser.parse_args()
+# Host-side paths (this script runs on the host, not inside the container)
+SCAN_DIRS = [
+    "/home/sparky/Docker/faster-qwen3-tts/config/speakers",
+    "/home/sparky/Projekte/TTS_Voices/speakers",
+]
+AUDIO_EXTS = (".wav", ".mp3", ".ogg", ".m4a")
+SKIP_DIRS = {"originals_backup", "xtts_multi_voice_sets", "txt"}

-    speaker_dir = args.speaker_dir
+whisper_api_url = "http://localhost:8010/v1/audio/transcriptions"

-    # Verify API is reachable
-    try:
-        requests.get(args.api_url.rsplit('/', 2)[0], timeout=5)
-    except requests.ConnectionError:
-        print(f"Error: Cannot reach transcription API at {args.api_url}")
-        print("Make sure your Whisper/ASR service is running.")
-        sys.exit(1)
+for scan_dir in SCAN_DIRS:
+    if not os.path.exists(scan_dir):
+        print(f"Skipping {scan_dir} (not found)")
+        continue

-    if not os.path.exists(speaker_dir):
-        print(f"Error: Speaker directory not found: {speaker_dir}")
-        sys.exit(1)
+    print(f"\nScanning {scan_dir} for missing transcripts...")

-    audio_files = [f for f in os.listdir(speaker_dir)
-                   if f.endswith(('.wav', '.mp3')) and not f.startswith('.')]
+    for root, dirs, files in os.walk(scan_dir):
+        dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)

-    print(f"Found {len(audio_files)} audio files in {speaker_dir}")
+        for filename in sorted(files):
+            if not filename.lower().endswith(AUDIO_EXTS):
+                continue

-    for filename in sorted(audio_files):
-        base_name = os.path.splitext(filename)[0]
-        ref_txt_path = os.path.join(speaker_dir, f"{base_name}.reference.txt")
+            base_name = os.path.splitext(filename)[0]
+            ref_txt_path = os.path.join(root, f"{base_name}.reference.txt")
+            audio_path = os.path.join(root, filename)

-        if os.path.exists(ref_txt_path):
-            print(f"  Skipping {filename} (already transcribed)")
-            continue
+            if os.path.exists(ref_txt_path):
+                continue

-        filepath = os.path.join(speaker_dir, filename)
-        print(f"  Transcribing {filename}...", end=" ", flush=True)
+            print(f"Transcribing: {os.path.relpath(audio_path, scan_dir)}")
+            try:
+                with open(audio_path, "rb") as audio_file:
+                    response = requests.post(
+                        whisper_api_url,
+                        files={"file": (filename, audio_file)},
+                        data={"model": "large-v3", "response_format": "text"},
+                    )

-        try:
-            with open(filepath, 'rb') as f:
-                response = requests.post(
-                    args.api_url,
-                    files={"file": (filename, f)},
-                    data={"model": args.model},
-                    timeout=60,
-                )
+                if response.status_code == 200:
+                    transcript = response.text.strip()
+                    if transcript.startswith("{"):
+                        try:
+                            transcript = json.loads(transcript).get("text", transcript).strip()
+                        except json.JSONDecodeError:
+                            pass

-            if response.status_code == 200:
-                # Handle both JSON and plain text responses
-                try:
-                    text = response.json().get("text", "").strip()
-                except (json.JSONDecodeError, AttributeError):
-                    text = response.text.strip()
-
-                if text:
-                    with open(ref_txt_path, 'w', encoding='utf-8') as f:
-                        f.write(text)
-                    print(f"OK ({len(text)} chars)")
+                    with open(ref_txt_path, "w", encoding="utf-8") as f:
+                        f.write(transcript)
+                    print(f"  ✓ {transcript[:80]}")
                else:
-                    print("EMPTY (no speech detected)")
-            else:
-                print(f"FAILED (HTTP {response.status_code})")
+                    print(f"  ✗ API error {response.status_code}: {response.text}")

-        except requests.Timeout:
-            print("TIMEOUT")
-        except Exception as e:
-            print(f"ERROR: {e}")
+            except requests.exceptions.ConnectionError:
+                print(f"  ✗ Cannot reach Whisper API at {whisper_api_url}")
+                raise SystemExit(1)
+            except Exception as e:
+                print(f"  ✗ Error on {filename}: {e}")

-    print("Done.")
-
-if __name__ == "__main__":
-    main()
+print("\nBatch transcription complete.")
--- a/config/docker-compose.yml
+++ b/config/docker-compose.yml
@ -0,0 +1,68 @@
+services:
+  faster-qwen3-tts:
+    image: faster-qwen3-tts-dgx-spark:v4
+    container_name: faster-qwen3-tts
+    restart: unless-stopped
+    runtime: nvidia
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - HF_TOKEN=${HF_TOKEN}
+    ports:
+      - "8020:8000"
+    volumes:
+      - /home/sparky/LLMs/vllm/Alibaba/Qwen3-TTS-12Hz-1.7B-Base:/models/Qwen3-TTS:ro
+      - /home/sparky/Docker/faster-qwen3-tts/config:/config:rw
+      - /home/sparky/Projekte/TTS_Voices/speakers:/voices:ro
+    command: >
+      /bin/bash -c "
+      python3 /config/generate_voices.py &&
+      python3 /config/run_server.py
+      --model /models/Qwen3-TTS
+      --voices /config/voices.json
+      --port 8000
+      --max-seq-len 2048
+      "
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    networks:
+      - dgx_net
+
+  faster-qwen3-tts-voicedesign:
+    image: faster-qwen3-tts-dgx-spark:v4
+    container_name: faster-qwen3-tts-voicedesign
+    restart: unless-stopped
+    runtime: nvidia
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - HF_TOKEN=${HF_TOKEN}
+    ports:
+      - "8021:8000"
+    volumes:
+      - /home/sparky/LLMs/vllm/Alibaba/Qwen3-TTS-12Hz-1.7B-VoiceDesign:/models/Qwen3-TTS-VoiceDesign:ro
+      - /home/sparky/Docker/faster-qwen3-tts/config:/config:rw
+    command: >
+      /bin/bash -c "
+      python3 /config/run_voicedesign_server.py
+      --model /models/Qwen3-TTS-VoiceDesign
+      --voices /config/voicedesign_voices.json
+      --port 8000
+      --max-seq-len 2048
+      "
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    networks:
+      - dgx_net
+
+networks:
+  dgx_net:
+    external: true
--- a/config/generate_voices.py
+++ b/config/generate_voices.py
@ -1,65 +1,103 @@
 #generate_voices.py
-"""
-Scan the speakers directory for reference audio files and generate voices.json.
-
-Runs inside the container at startup to map all available voice reference
-audio files into a format the OpenAI-compatible TTS server understands.
-"""

 import os
 import json
+import re
+import subprocess

-# Internal container paths
-speaker_dir = "/config/speakers"
 output_file = "/config/voices.json"
+converted_dir = "/config/converted"
+
+# .m4a is converted to WAV on the fly because soundfile doesn't support AAC
+AUDIO_EXTS = (".wav", ".mp3", ".ogg", ".m4a")
+SKIP_DIRS = {"originals_backup", "xtts_multi_voice_sets", "txt"}
+
+# (base_dir_on_container, container_path_prefix)
+# /config/speakers — legacy location, writable
+# /voices          — new external mount, read-only
+SCAN_DIRS = [
+    "/config/speakers",
+    "/voices",
+]

 voices = {}

-# Ensure the directory exists just in case
-if os.path.exists(speaker_dir):
-    for filename in os.listdir(speaker_dir):
-        if filename.endswith((".wav", ".mp3")):
+
+def detect_language(base_name):
+    if base_name.startswith("EN_") or base_name.startswith("basic_ref_en"):
+        return "English"
+    if base_name.startswith("DE_"):
+        return "German"
+    if base_name.startswith("basic_ref_zh"):
+        return "Chinese"
+    return "Auto"
+
+
+
+
+def make_voice_id(base_dir, root, base_name):
+    rel = os.path.relpath(root, base_dir)
+    parts = [] if rel == "." else rel.split(os.sep)
+    parts.append(base_name)
+    raw = "_".join(parts)
+    return re.sub(r"[^\w\-]", "_", raw)
+
+
+def convert_m4a(src_path, voice_id):
+    """Convert M4A to WAV in /config/converted/. Returns the WAV path."""
+    os.makedirs(converted_dir, exist_ok=True)
+    dst_path = os.path.join(converted_dir, f"{voice_id}.wav")
+    if not os.path.exists(dst_path):
+        result = subprocess.run(
+            ["ffmpeg", "-y", "-i", src_path, "-ar", "24000", "-ac", "1", dst_path],
+            capture_output=True,
+        )
+        if result.returncode != 0:
+            print(f"  ✗ ffmpeg failed for {src_path}: {result.stderr.decode()[:200]}")
+            return None
+        print(f"  Converted: {os.path.basename(src_path)} → {dst_path}")
+    return dst_path
+
+
+for scan_dir in SCAN_DIRS:
+    if not os.path.exists(scan_dir):
+        print(f"Skipping {scan_dir} (not mounted)")
+        continue
+
+    for root, dirs, files in os.walk(scan_dir):
+        dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)
+
+        for filename in sorted(files):
+            if not filename.lower().endswith(AUDIO_EXTS):
+                continue
+
            base_name = os.path.splitext(filename)[0]
+            audio_path = os.path.join(root, filename)
+            voice_id = make_voice_id(scan_dir, root, base_name)

-            # Determine language based on file prefixes
-            lang = "Auto"
-            if base_name.startswith("EN_") or base_name.startswith("basic_ref_en"):
-                lang = "English"
-            elif base_name.startswith("DE_"):
-                lang = "German"
-            elif base_name.startswith("basic_ref_zh"):
-                lang = "Chinese"
-
-        #  # Create a clean voice ID
-        #    voice_id = base_name.lower()
-        #    prefixes_to_strip = ["en_m_", "en_f_", "de_m_", "de_f_"]
-        #    for prefix in prefixes_to_strip:
-        #        if voice_id.startswith(prefix):
-        #            voice_id = voice_id.replace(prefix, "", 1)
-        #            break
+            if filename.lower().endswith(".m4a"):
+                audio_path = convert_m4a(audio_path, voice_id)
+                if audio_path is None:
+                    continue

            entry = {
-                "ref_audio": f"/config/speakers/{filename}",
-                "language": lang,
+                "ref_audio": audio_path,
+                "language": detect_language(base_name),
                "chunk_size": 4,
            }

-            # Look for matching reference text files
-            ref_txt_path = os.path.join(speaker_dir, f"{base_name}.reference.txt")
-            txt_path = os.path.join(speaker_dir, f"{base_name}.txt")
-
-            if os.path.exists(ref_txt_path):
-                with open(ref_txt_path, 'r', encoding='utf-8') as f:
+            ref_txt = os.path.join(root, f"{base_name}.reference.txt")
+            txt = os.path.join(root, f"{base_name}.txt")
+            if os.path.exists(ref_txt):
+                with open(ref_txt, encoding="utf-8") as f:
                    entry["ref_text"] = f.read().strip()
-            elif os.path.exists(txt_path):
-                with open(txt_path, 'r', encoding='utf-8') as f:
+            elif os.path.exists(txt):
+                with open(txt, encoding="utf-8") as f:
                    entry["ref_text"] = f.read().strip()

            voices[voice_id] = entry

-    with open(output_file, 'w', encoding='utf-8') as f:
-        json.dump(voices, f, indent=2, ensure_ascii=False)
+with open(output_file, "w", encoding="utf-8") as f:
+    json.dump(voices, f, indent=2, ensure_ascii=False)

-    print(f"Success! Generated voices.json with {len(voices)} mapped voices.")
-else:
-    print(f"Warning: Directory {speaker_dir} not found. Skipping voice generation.")
+print(f"Success! Generated voices.json with {len(voices)} mapped voices.")
--- a/config/run_voicedesign_server.py
+++ b/config/run_voicedesign_server.py
@ -0,0 +1,234 @@
+"""
+OpenAI-compatible TTS server for Qwen3-TTS-12Hz-1.7B-VoiceDesign.
+
+Voices are defined in voicedesign_voices.json as:
+  { "voice_id": { "instruct": "...", "language": "..." } }
+
+No ref_audio needed — the instruct text fully describes the voice.
+"""
+import json
+import logging
+import queue
+import threading
+import asyncio
+import argparse
+import numpy as np
+import sys
+
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import Response, StreamingResponse, JSONResponse
+from pydantic import BaseModel
+
+sys.path.append("/app")
+from faster_qwen3_tts.model import FasterQwen3TTS
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+app = FastAPI()
+tts_model: FasterQwen3TTS = None
+voices: dict = {}
+default_voice: str = None
+SAMPLE_RATE = 24000
+_model_lock = threading.Lock()
+
+
+# ---------------------------------------------------------------------------
+# Request schema (OpenAI TTS compatible)
+# ---------------------------------------------------------------------------
+
+class SpeechRequest(BaseModel):
+    model: str = "tts-1"
+    input: str
+    voice: str = "vd_british_male"
+    response_format: str = "wav"
+    speed: float = 1.0
+
+
+# ---------------------------------------------------------------------------
+# Audio helpers
+# ---------------------------------------------------------------------------
+
+def _to_pcm16(audio: np.ndarray) -> bytes:
+    return (audio * 32767).clip(-32768, 32767).astype(np.int16).tobytes()
+
+
+def _wav_header(sample_rate: int) -> bytes:
+    import struct
+    return struct.pack(
+        "<4sI4s4sIHHIIHH4sI",
+        b"RIFF", 0xFFFFFFFF, b"WAVE",
+        b"fmt ", 16, 1, 1,
+        sample_rate, sample_rate * 2, 2, 16,
+        b"data", 0xFFFFFFFF,
+    )
+
+
+def _to_mp3_bytes(audio: np.ndarray, sr: int) -> bytes:
+    from pydub import AudioSegment
+    import io
+    pcm = _to_pcm16(audio)
+    seg = AudioSegment(pcm, frame_rate=sr, sample_width=2, channels=1)
+    buf = io.BytesIO()
+    seg.export(buf, format="mp3")
+    return buf.getvalue()
+
+
+def resolve_voice(name: str) -> dict:
+    cfg = voices.get(name)
+    if cfg:
+        return cfg
+    if default_voice and default_voice in voices:
+        logger.warning("Voice %r not found, falling back to %r", name, default_voice)
+        return voices[default_voice]
+    raise HTTPException(status_code=404, detail=f"Voice {name!r} not found")
+
+
+# ---------------------------------------------------------------------------
+# Generation helpers
+# ---------------------------------------------------------------------------
+
+async def _stream_chunks(voice_cfg: dict, text: str):
+    q: queue.Queue = queue.Queue()
+    _DONE = object()
+
+    def producer():
+        try:
+            with _model_lock:
+                for chunk, _sr, _timing in tts_model.generate_voice_design_streaming(
+                    text=text,
+                    instruct=voice_cfg["instruct"],
+                    language=voice_cfg.get("language", "English"),
+                ):
+                    q.put(chunk)
+        except Exception as exc:
+            q.put(exc)
+        finally:
+            q.put(_DONE)
+
+    threading.Thread(target=producer, daemon=True).start()
+    loop = asyncio.get_event_loop()
+    while True:
+        item = await loop.run_in_executor(None, q.get)
+        if item is _DONE:
+            break
+        if isinstance(item, Exception):
+            raise item
+        yield _to_pcm16(item)
+
+
+# ---------------------------------------------------------------------------
+# Endpoints
+# ---------------------------------------------------------------------------
+
+@app.get("/health")
+async def health():
+    return {"status": "ok", "model_loaded": tts_model is not None}
+
+
+@app.post("/v1/audio/speech")
+async def create_speech(req: SpeechRequest):
+    if tts_model is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    if not req.input.strip():
+        raise HTTPException(status_code=400, detail="'input' text is empty")
+
+    voice_cfg = resolve_voice(req.voice)
+    fmt = req.response_format.lower()
+
+    _CONTENT_TYPES = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"}
+    if fmt not in _CONTENT_TYPES:
+        raise HTTPException(status_code=400, detail=f"Unsupported format: {fmt!r}")
+
+    if fmt == "mp3":
+        loop = asyncio.get_event_loop()
+        def _gen():
+            with _model_lock:
+                return tts_model.generate_voice_design(
+                    text=req.input,
+                    instruct=voice_cfg["instruct"],
+                    language=voice_cfg.get("language", "English"),
+                )
+        audio_arrays, sr = await loop.run_in_executor(None, _gen)
+        audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32)
+        return Response(content=_to_mp3_bytes(audio, sr), media_type="audio/mpeg")
+
+    async def audio_stream():
+        if fmt == "wav":
+            yield _wav_header(SAMPLE_RATE)
+        async for raw in _stream_chunks(voice_cfg, req.input):
+            yield raw
+
+    return StreamingResponse(audio_stream(), media_type=_CONTENT_TYPES[fmt])
+
+
+_voice_list = None
+_models_response = None
+
+
+def _build_voice_list():
+    global _voice_list, _models_response
+    _voice_list = [{"id": v, "object": "model", "created": 1686935002, "owned_by": "qwen"} for v in voices]
+    _models_response = {"object": "list", "data": _voice_list}
+
+
+@app.get("/v1/models")
+async def list_models():
+    return _models_response
+
+@app.get("/v1/audio/voices")
+async def list_audio_voices():
+    return _models_response
+
+@app.get("/v1/audio/models")
+async def list_audio_models():
+    return _models_response
+
+@app.get("/speakers")
+async def get_speakers():
+    return list(voices.keys())
+
+@app.options("/{path:path}")
+async def options_handler(path: str):
+    return JSONResponse(content={"status": "ok"})
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+def main():
+    global tts_model, voices, default_voice, SAMPLE_RATE
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", default="/models/Qwen3-TTS-VoiceDesign")
+    parser.add_argument("--voices", default="/config/voicedesign_voices.json")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--max-seq-len", type=int, default=2048)
+    args = parser.parse_args()
+
+    with open(args.voices) as f:
+        voices = json.load(f)
+    default_voice = next(iter(voices), None)
+    _build_voice_list()
+
+    import torch
+    logger.info("Loading VoiceDesign model %s …", args.model)
+    tts_model = FasterQwen3TTS.from_pretrained(
+        args.model,
+        device=args.device,
+        dtype=torch.bfloat16,
+        attn_implementation="sdpa",
+        max_seq_len=args.max_seq_len,
+    )
+    SAMPLE_RATE = tts_model.sample_rate
+    logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)
+
+    uvicorn.run(app, host=args.host, port=args.port, log_level="info")
+
+
+if __name__ == "__main__":
+    main()
--- a/config/voicedesign_voices.json
+++ b/config/voicedesign_voices.json
@ -0,0 +1,34 @@
+{
+  "vd_british_male": {
+    "instruct": "A native British English male speaker with received pronunciation (RP) accent, clear articulation, calm and authoritative tone",
+    "language": "English"
+  },
+  "vd_british_female": {
+    "instruct": "A native British English female speaker with received pronunciation (RP) accent, warm and clearly articulated",
+    "language": "English"
+  },
+  "vd_british_male_casual": {
+    "instruct": "A young British English male speaker with a natural conversational RP accent, friendly and relaxed",
+    "language": "English"
+  },
+  "vd_british_female_warm": {
+    "instruct": "A middle-aged British English female speaker, warm southern English accent, gentle and expressive",
+    "language": "English"
+  },
+  "vd_german_male": {
+    "instruct": "A native German male speaker with standard Hochdeutsch pronunciation, clear articulation, no foreign accent, professional tone",
+    "language": "German"
+  },
+  "vd_german_female": {
+    "instruct": "A native German female speaker with standard Hochdeutsch pronunciation, warm and natural, no foreign accent",
+    "language": "German"
+  },
+  "vd_german_male_casual": {
+    "instruct": "A young native German male speaker, natural conversational Hochdeutsch, friendly and relaxed, no foreign accent",
+    "language": "German"
+  },
+  "vd_german_female_warm": {
+    "instruct": "A middle-aged native German female speaker, warm Hochdeutsch, expressive and clear, no foreign accent",
+    "language": "German"
+  }
+}