diff --git a/.gitignore b/.gitignore index 67c2f80..887f235 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,42 @@ -# Audio files (add your own voice references locally) +# Upstream faster-qwen3-tts repo (tracked separately) +build/ + +# Personal voice recordings — keep only reference transcripts config/speakers/*.wav config/speakers/*.mp3 -config/speakers/originals_backup/ -# Generated at runtime +# Private speakers — entire subtree +config/speakers/privat/ + +# Generated at container startup from speakers/ config/voices.json -config/voices.json.bak +config/voices.json* -# Environment -.env -*.pyc -__pycache__/ +# Converted private audio (M4A → WAV, generated at runtime) +config/converted/ -# Editor backups +# Editor backup files +*.py~ +*.yml~ +*.yaml~ *~ -*.swp -# OS -.DS_Store -Thumbs.db +# Claude Code settings +.claude/ +config/.claude/ + +# Random generated files in project root +json +os +*.txt + +# Python +__pycache__/ +*.pyc +*.pyo +.venv/ +venv/ + +# Environment files +.env +.env.* diff --git a/config/auto_transcribe.py b/config/auto_transcribe.py index bf64277..f2f3b32 100644 --- a/config/auto_transcribe.py +++ b/config/auto_transcribe.py @@ -1,93 +1,65 @@ -""" -Batch-transcribe speaker reference audio files using a local Whisper-compatible API. - -Creates .reference.txt files alongside each audio file in the speakers directory. -These transcriptions are used by generate_voices.py to build the voice registry. - -Usage: - python auto_transcribe.py [--api-url http://localhost:8010/v1/audio/transcriptions] - -IMPORTANT: Reference audio for voice cloning should be 5-15 seconds long. -Longer files will produce poor cloning results and slow down inference. -""" - import os -import sys -import json -import argparse import requests +import json -def main(): - parser = argparse.ArgumentParser(description="Batch-transcribe speaker reference audio") - parser.add_argument("--api-url", default="http://localhost:8010/v1/audio/transcriptions", - help="Whisper-compatible transcription API URL") - parser.add_argument("--speaker-dir", default="./speakers", - help="Directory containing speaker audio files") - parser.add_argument("--model", default="whisper-1", - help="Transcription model name") - args = parser.parse_args() +# Host-side paths (this script runs on the host, not inside the container) +SCAN_DIRS = [ + "/home/sparky/Docker/faster-qwen3-tts/config/speakers", + "/home/sparky/Projekte/TTS_Voices/speakers", +] +AUDIO_EXTS = (".wav", ".mp3", ".ogg", ".m4a") +SKIP_DIRS = {"originals_backup", "xtts_multi_voice_sets", "txt"} - speaker_dir = args.speaker_dir +whisper_api_url = "http://localhost:8010/v1/audio/transcriptions" - # Verify API is reachable - try: - requests.get(args.api_url.rsplit('/', 2)[0], timeout=5) - except requests.ConnectionError: - print(f"Error: Cannot reach transcription API at {args.api_url}") - print("Make sure your Whisper/ASR service is running.") - sys.exit(1) +for scan_dir in SCAN_DIRS: + if not os.path.exists(scan_dir): + print(f"Skipping {scan_dir} (not found)") + continue - if not os.path.exists(speaker_dir): - print(f"Error: Speaker directory not found: {speaker_dir}") - sys.exit(1) + print(f"\nScanning {scan_dir} for missing transcripts...") - audio_files = [f for f in os.listdir(speaker_dir) - if f.endswith(('.wav', '.mp3')) and not f.startswith('.')] + for root, dirs, files in os.walk(scan_dir): + dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) - print(f"Found {len(audio_files)} audio files in {speaker_dir}") + for filename in sorted(files): + if not filename.lower().endswith(AUDIO_EXTS): + continue - for filename in sorted(audio_files): - base_name = os.path.splitext(filename)[0] - ref_txt_path = os.path.join(speaker_dir, f"{base_name}.reference.txt") + base_name = os.path.splitext(filename)[0] + ref_txt_path = os.path.join(root, f"{base_name}.reference.txt") + audio_path = os.path.join(root, filename) - if os.path.exists(ref_txt_path): - print(f" Skipping {filename} (already transcribed)") - continue + if os.path.exists(ref_txt_path): + continue - filepath = os.path.join(speaker_dir, filename) - print(f" Transcribing {filename}...", end=" ", flush=True) + print(f"Transcribing: {os.path.relpath(audio_path, scan_dir)}") + try: + with open(audio_path, "rb") as audio_file: + response = requests.post( + whisper_api_url, + files={"file": (filename, audio_file)}, + data={"model": "large-v3", "response_format": "text"}, + ) - try: - with open(filepath, 'rb') as f: - response = requests.post( - args.api_url, - files={"file": (filename, f)}, - data={"model": args.model}, - timeout=60, - ) + if response.status_code == 200: + transcript = response.text.strip() + if transcript.startswith("{"): + try: + transcript = json.loads(transcript).get("text", transcript).strip() + except json.JSONDecodeError: + pass - if response.status_code == 200: - # Handle both JSON and plain text responses - try: - text = response.json().get("text", "").strip() - except (json.JSONDecodeError, AttributeError): - text = response.text.strip() - - if text: - with open(ref_txt_path, 'w', encoding='utf-8') as f: - f.write(text) - print(f"OK ({len(text)} chars)") + with open(ref_txt_path, "w", encoding="utf-8") as f: + f.write(transcript) + print(f" ✓ {transcript[:80]}") else: - print("EMPTY (no speech detected)") - else: - print(f"FAILED (HTTP {response.status_code})") + print(f" ✗ API error {response.status_code}: {response.text}") - except requests.Timeout: - print("TIMEOUT") - except Exception as e: - print(f"ERROR: {e}") + except requests.exceptions.ConnectionError: + print(f" ✗ Cannot reach Whisper API at {whisper_api_url}") + raise SystemExit(1) + except Exception as e: + print(f" ✗ Error on {filename}: {e}") - print("Done.") - -if __name__ == "__main__": - main() +print("\nBatch transcription complete.") diff --git a/config/docker-compose.yml b/config/docker-compose.yml new file mode 100644 index 0000000..d8b3a28 --- /dev/null +++ b/config/docker-compose.yml @@ -0,0 +1,68 @@ +services: + faster-qwen3-tts: + image: faster-qwen3-tts-dgx-spark:v4 + container_name: faster-qwen3-tts + restart: unless-stopped + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + - HF_TOKEN=${HF_TOKEN} + ports: + - "8020:8000" + volumes: + - /home/sparky/LLMs/vllm/Alibaba/Qwen3-TTS-12Hz-1.7B-Base:/models/Qwen3-TTS:ro + - /home/sparky/Docker/faster-qwen3-tts/config:/config:rw + - /home/sparky/Projekte/TTS_Voices/speakers:/voices:ro + command: > + /bin/bash -c " + python3 /config/generate_voices.py && + python3 /config/run_server.py + --model /models/Qwen3-TTS + --voices /config/voices.json + --port 8000 + --max-seq-len 2048 + " + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + networks: + - dgx_net + + faster-qwen3-tts-voicedesign: + image: faster-qwen3-tts-dgx-spark:v4 + container_name: faster-qwen3-tts-voicedesign + restart: unless-stopped + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + - HF_TOKEN=${HF_TOKEN} + ports: + - "8021:8000" + volumes: + - /home/sparky/LLMs/vllm/Alibaba/Qwen3-TTS-12Hz-1.7B-VoiceDesign:/models/Qwen3-TTS-VoiceDesign:ro + - /home/sparky/Docker/faster-qwen3-tts/config:/config:rw + command: > + /bin/bash -c " + python3 /config/run_voicedesign_server.py + --model /models/Qwen3-TTS-VoiceDesign + --voices /config/voicedesign_voices.json + --port 8000 + --max-seq-len 2048 + " + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + networks: + - dgx_net + +networks: + dgx_net: + external: true diff --git a/config/generate_voices.py b/config/generate_voices.py index ac7024c..45fe7d4 100644 --- a/config/generate_voices.py +++ b/config/generate_voices.py @@ -1,65 +1,103 @@ #generate_voices.py -""" -Scan the speakers directory for reference audio files and generate voices.json. - -Runs inside the container at startup to map all available voice reference -audio files into a format the OpenAI-compatible TTS server understands. -""" import os import json +import re +import subprocess -# Internal container paths -speaker_dir = "/config/speakers" output_file = "/config/voices.json" +converted_dir = "/config/converted" + +# .m4a is converted to WAV on the fly because soundfile doesn't support AAC +AUDIO_EXTS = (".wav", ".mp3", ".ogg", ".m4a") +SKIP_DIRS = {"originals_backup", "xtts_multi_voice_sets", "txt"} + +# (base_dir_on_container, container_path_prefix) +# /config/speakers — legacy location, writable +# /voices — new external mount, read-only +SCAN_DIRS = [ + "/config/speakers", + "/voices", +] voices = {} -# Ensure the directory exists just in case -if os.path.exists(speaker_dir): - for filename in os.listdir(speaker_dir): - if filename.endswith((".wav", ".mp3")): + +def detect_language(base_name): + if base_name.startswith("EN_") or base_name.startswith("basic_ref_en"): + return "English" + if base_name.startswith("DE_"): + return "German" + if base_name.startswith("basic_ref_zh"): + return "Chinese" + return "Auto" + + + + +def make_voice_id(base_dir, root, base_name): + rel = os.path.relpath(root, base_dir) + parts = [] if rel == "." else rel.split(os.sep) + parts.append(base_name) + raw = "_".join(parts) + return re.sub(r"[^\w\-]", "_", raw) + + +def convert_m4a(src_path, voice_id): + """Convert M4A to WAV in /config/converted/. Returns the WAV path.""" + os.makedirs(converted_dir, exist_ok=True) + dst_path = os.path.join(converted_dir, f"{voice_id}.wav") + if not os.path.exists(dst_path): + result = subprocess.run( + ["ffmpeg", "-y", "-i", src_path, "-ar", "24000", "-ac", "1", dst_path], + capture_output=True, + ) + if result.returncode != 0: + print(f" ✗ ffmpeg failed for {src_path}: {result.stderr.decode()[:200]}") + return None + print(f" Converted: {os.path.basename(src_path)} → {dst_path}") + return dst_path + + +for scan_dir in SCAN_DIRS: + if not os.path.exists(scan_dir): + print(f"Skipping {scan_dir} (not mounted)") + continue + + for root, dirs, files in os.walk(scan_dir): + dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS) + + for filename in sorted(files): + if not filename.lower().endswith(AUDIO_EXTS): + continue + base_name = os.path.splitext(filename)[0] + audio_path = os.path.join(root, filename) + voice_id = make_voice_id(scan_dir, root, base_name) - # Determine language based on file prefixes - lang = "Auto" - if base_name.startswith("EN_") or base_name.startswith("basic_ref_en"): - lang = "English" - elif base_name.startswith("DE_"): - lang = "German" - elif base_name.startswith("basic_ref_zh"): - lang = "Chinese" - - # # Create a clean voice ID - # voice_id = base_name.lower() - # prefixes_to_strip = ["en_m_", "en_f_", "de_m_", "de_f_"] - # for prefix in prefixes_to_strip: - # if voice_id.startswith(prefix): - # voice_id = voice_id.replace(prefix, "", 1) - # break + if filename.lower().endswith(".m4a"): + audio_path = convert_m4a(audio_path, voice_id) + if audio_path is None: + continue entry = { - "ref_audio": f"/config/speakers/{filename}", - "language": lang, + "ref_audio": audio_path, + "language": detect_language(base_name), "chunk_size": 4, } - # Look for matching reference text files - ref_txt_path = os.path.join(speaker_dir, f"{base_name}.reference.txt") - txt_path = os.path.join(speaker_dir, f"{base_name}.txt") - - if os.path.exists(ref_txt_path): - with open(ref_txt_path, 'r', encoding='utf-8') as f: + ref_txt = os.path.join(root, f"{base_name}.reference.txt") + txt = os.path.join(root, f"{base_name}.txt") + if os.path.exists(ref_txt): + with open(ref_txt, encoding="utf-8") as f: entry["ref_text"] = f.read().strip() - elif os.path.exists(txt_path): - with open(txt_path, 'r', encoding='utf-8') as f: + elif os.path.exists(txt): + with open(txt, encoding="utf-8") as f: entry["ref_text"] = f.read().strip() voices[voice_id] = entry - with open(output_file, 'w', encoding='utf-8') as f: - json.dump(voices, f, indent=2, ensure_ascii=False) +with open(output_file, "w", encoding="utf-8") as f: + json.dump(voices, f, indent=2, ensure_ascii=False) - print(f"Success! Generated voices.json with {len(voices)} mapped voices.") -else: - print(f"Warning: Directory {speaker_dir} not found. Skipping voice generation.") +print(f"Success! Generated voices.json with {len(voices)} mapped voices.") diff --git a/config/run_voicedesign_server.py b/config/run_voicedesign_server.py new file mode 100644 index 0000000..f6e58f1 --- /dev/null +++ b/config/run_voicedesign_server.py @@ -0,0 +1,234 @@ +""" +OpenAI-compatible TTS server for Qwen3-TTS-12Hz-1.7B-VoiceDesign. + +Voices are defined in voicedesign_voices.json as: + { "voice_id": { "instruct": "...", "language": "..." } } + +No ref_audio needed — the instruct text fully describes the voice. +""" +import json +import logging +import queue +import threading +import asyncio +import argparse +import numpy as np +import sys + +import uvicorn +from fastapi import FastAPI, HTTPException +from fastapi.responses import Response, StreamingResponse, JSONResponse +from pydantic import BaseModel + +sys.path.append("/app") +from faster_qwen3_tts.model import FasterQwen3TTS + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +app = FastAPI() +tts_model: FasterQwen3TTS = None +voices: dict = {} +default_voice: str = None +SAMPLE_RATE = 24000 +_model_lock = threading.Lock() + + +# --------------------------------------------------------------------------- +# Request schema (OpenAI TTS compatible) +# --------------------------------------------------------------------------- + +class SpeechRequest(BaseModel): + model: str = "tts-1" + input: str + voice: str = "vd_british_male" + response_format: str = "wav" + speed: float = 1.0 + + +# --------------------------------------------------------------------------- +# Audio helpers +# --------------------------------------------------------------------------- + +def _to_pcm16(audio: np.ndarray) -> bytes: + return (audio * 32767).clip(-32768, 32767).astype(np.int16).tobytes() + + +def _wav_header(sample_rate: int) -> bytes: + import struct + return struct.pack( + "<4sI4s4sIHHIIHH4sI", + b"RIFF", 0xFFFFFFFF, b"WAVE", + b"fmt ", 16, 1, 1, + sample_rate, sample_rate * 2, 2, 16, + b"data", 0xFFFFFFFF, + ) + + +def _to_mp3_bytes(audio: np.ndarray, sr: int) -> bytes: + from pydub import AudioSegment + import io + pcm = _to_pcm16(audio) + seg = AudioSegment(pcm, frame_rate=sr, sample_width=2, channels=1) + buf = io.BytesIO() + seg.export(buf, format="mp3") + return buf.getvalue() + + +def resolve_voice(name: str) -> dict: + cfg = voices.get(name) + if cfg: + return cfg + if default_voice and default_voice in voices: + logger.warning("Voice %r not found, falling back to %r", name, default_voice) + return voices[default_voice] + raise HTTPException(status_code=404, detail=f"Voice {name!r} not found") + + +# --------------------------------------------------------------------------- +# Generation helpers +# --------------------------------------------------------------------------- + +async def _stream_chunks(voice_cfg: dict, text: str): + q: queue.Queue = queue.Queue() + _DONE = object() + + def producer(): + try: + with _model_lock: + for chunk, _sr, _timing in tts_model.generate_voice_design_streaming( + text=text, + instruct=voice_cfg["instruct"], + language=voice_cfg.get("language", "English"), + ): + q.put(chunk) + except Exception as exc: + q.put(exc) + finally: + q.put(_DONE) + + threading.Thread(target=producer, daemon=True).start() + loop = asyncio.get_event_loop() + while True: + item = await loop.run_in_executor(None, q.get) + if item is _DONE: + break + if isinstance(item, Exception): + raise item + yield _to_pcm16(item) + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + +@app.get("/health") +async def health(): + return {"status": "ok", "model_loaded": tts_model is not None} + + +@app.post("/v1/audio/speech") +async def create_speech(req: SpeechRequest): + if tts_model is None: + raise HTTPException(status_code=503, detail="Model not loaded") + if not req.input.strip(): + raise HTTPException(status_code=400, detail="'input' text is empty") + + voice_cfg = resolve_voice(req.voice) + fmt = req.response_format.lower() + + _CONTENT_TYPES = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"} + if fmt not in _CONTENT_TYPES: + raise HTTPException(status_code=400, detail=f"Unsupported format: {fmt!r}") + + if fmt == "mp3": + loop = asyncio.get_event_loop() + def _gen(): + with _model_lock: + return tts_model.generate_voice_design( + text=req.input, + instruct=voice_cfg["instruct"], + language=voice_cfg.get("language", "English"), + ) + audio_arrays, sr = await loop.run_in_executor(None, _gen) + audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32) + return Response(content=_to_mp3_bytes(audio, sr), media_type="audio/mpeg") + + async def audio_stream(): + if fmt == "wav": + yield _wav_header(SAMPLE_RATE) + async for raw in _stream_chunks(voice_cfg, req.input): + yield raw + + return StreamingResponse(audio_stream(), media_type=_CONTENT_TYPES[fmt]) + + +_voice_list = None +_models_response = None + + +def _build_voice_list(): + global _voice_list, _models_response + _voice_list = [{"id": v, "object": "model", "created": 1686935002, "owned_by": "qwen"} for v in voices] + _models_response = {"object": "list", "data": _voice_list} + + +@app.get("/v1/models") +async def list_models(): + return _models_response + +@app.get("/v1/audio/voices") +async def list_audio_voices(): + return _models_response + +@app.get("/v1/audio/models") +async def list_audio_models(): + return _models_response + +@app.get("/speakers") +async def get_speakers(): + return list(voices.keys()) + +@app.options("/{path:path}") +async def options_handler(path: str): + return JSONResponse(content={"status": "ok"}) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main(): + global tts_model, voices, default_voice, SAMPLE_RATE + + parser = argparse.ArgumentParser() + parser.add_argument("--model", default="/models/Qwen3-TTS-VoiceDesign") + parser.add_argument("--voices", default="/config/voicedesign_voices.json") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--device", default="cuda") + parser.add_argument("--max-seq-len", type=int, default=2048) + args = parser.parse_args() + + with open(args.voices) as f: + voices = json.load(f) + default_voice = next(iter(voices), None) + _build_voice_list() + + import torch + logger.info("Loading VoiceDesign model %s …", args.model) + tts_model = FasterQwen3TTS.from_pretrained( + args.model, + device=args.device, + dtype=torch.bfloat16, + attn_implementation="sdpa", + max_seq_len=args.max_seq_len, + ) + SAMPLE_RATE = tts_model.sample_rate + logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE) + + uvicorn.run(app, host=args.host, port=args.port, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/config/voicedesign_voices.json b/config/voicedesign_voices.json new file mode 100644 index 0000000..45dcdb2 --- /dev/null +++ b/config/voicedesign_voices.json @@ -0,0 +1,34 @@ +{ + "vd_british_male": { + "instruct": "A native British English male speaker with received pronunciation (RP) accent, clear articulation, calm and authoritative tone", + "language": "English" + }, + "vd_british_female": { + "instruct": "A native British English female speaker with received pronunciation (RP) accent, warm and clearly articulated", + "language": "English" + }, + "vd_british_male_casual": { + "instruct": "A young British English male speaker with a natural conversational RP accent, friendly and relaxed", + "language": "English" + }, + "vd_british_female_warm": { + "instruct": "A middle-aged British English female speaker, warm southern English accent, gentle and expressive", + "language": "English" + }, + "vd_german_male": { + "instruct": "A native German male speaker with standard Hochdeutsch pronunciation, clear articulation, no foreign accent, professional tone", + "language": "German" + }, + "vd_german_female": { + "instruct": "A native German female speaker with standard Hochdeutsch pronunciation, warm and natural, no foreign accent", + "language": "German" + }, + "vd_german_male_casual": { + "instruct": "A young native German male speaker, natural conversational Hochdeutsch, friendly and relaxed, no foreign accent", + "language": "German" + }, + "vd_german_female_warm": { + "instruct": "A middle-aged native German female speaker, warm Hochdeutsch, expressive and clear, no foreign accent", + "language": "German" + } +}