From 60db1b181ff944e574eccdefd6f01077b5a3f63b Mon Sep 17 00:00:00 2001 From: mARTin Date: Tue, 14 Apr 2026 00:06:05 +0200 Subject: [PATCH] Initial release: Faster-Qwen3-TTS for DGX Spark GB10 Run Qwen3-TTS with CUDA graph acceleration on NVIDIA DGX Spark (ARM64 / SM 121 / CUDA 13) as an OpenAI-compatible TTS API server. - Dockerfile targeting nvidia/cuda:13.0.2-base-ubuntu24.04 with ARM64 cu130 PyTorch wheels - Patch for max-seq-len support to handle long reference audio without crashes - OpenWebUI + SillyTavern compatible API endpoints (/v1/models, /v1/audio/voices, /speakers) - Voice management: auto-generate voices.json from speaker reference audio files - Auto-transcription helper script for generating reference text from audio Co-Authored-By: Claude Opus 4.6 --- .env.example | 5 ++ .gitignore | 21 ++++++ Dockerfile | 47 +++++++++++++ LICENSE | 21 ++++++ README.md | 127 ++++++++++++++++++++++++++++++++++++ config/auto_transcribe.py | 93 ++++++++++++++++++++++++++ config/generate_voices.py | 62 ++++++++++++++++++ config/run_server.py | 57 ++++++++++++++++ config/speakers/.gitkeep | 0 docker-compose.yml | 40 ++++++++++++ patches/openai_server.patch | 20 ++++++ 11 files changed, 493 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config/auto_transcribe.py create mode 100644 config/generate_voices.py create mode 100644 config/run_server.py create mode 100644 config/speakers/.gitkeep create mode 100644 docker-compose.yml create mode 100644 patches/openai_server.patch diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..eb1baea --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# Path to your local Qwen3-TTS model weights +MODEL_PATH=/path/to/Qwen3-TTS-12Hz-1.7B-Base + +# Optional: HuggingFace token (only needed if model requires authentication) +# HF_TOKEN=hf_your_token_here diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67c2f80 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Audio files (add your own voice references locally) +config/speakers/*.wav +config/speakers/*.mp3 +config/speakers/originals_backup/ + +# Generated at runtime +config/voices.json +config/voices.json.bak + +# Environment +.env +*.pyc +__pycache__/ + +# Editor backups +*~ +*.swp + +# OS +.DS_Store +Thumbs.db diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f793d8f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# Faster-Qwen3-TTS for NVIDIA DGX Spark (GB10 / SM 121 / ARM64 / CUDA 13) +# +# Builds an OpenAI-compatible TTS server with CUDA graph acceleration. +# Clones upstream faster-qwen3-tts at build time and applies DGX Spark fixes. + +FROM nvidia/cuda:13.0.2-base-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# Install Python 3.12 and required audio/build tools +RUN apt-get update && \ + apt-get install -y python3.12 python3.12-venv python3-pip python3.12-dev \ + ffmpeg git curl sox libsox-fmt-all && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Clone the upstream faster-qwen3-tts library +ARG FASTER_QWEN3_TTS_REF=main +RUN git clone --depth 1 --branch ${FASTER_QWEN3_TTS_REF} \ + https://github.com/andimarafioti/faster-qwen3-tts.git /app + +# Apply DGX Spark patches (max-seq-len support for long reference audio) +COPY patches/openai_server.patch /tmp/ +RUN cd /app && git apply /tmp/openai_server.patch || true + +# Create virtual environment (Ubuntu 24.04 enforces PEP 668) +ENV VIRTUAL_ENV=/opt/venv +RUN python3.12 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +RUN pip install --upgrade pip + +# Install ARM64 CUDA 13 wheels for PyTorch stack +RUN pip install --no-cache-dir torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/cu130 + +# Install faster-qwen3-tts and server dependencies +RUN pip install --no-cache-dir -e ".[demo]" +RUN pip install --no-cache-dir pydub soundfile uvicorn fastapi + +EXPOSE 8000 + +CMD ["python", "examples/openai_server.py", \ + "--model", "/models/Qwen3-TTS", \ + "--voices", "/config/voices.json", \ + "--port", "8000"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..380c893 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 mARTin-B78 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..56df1c6 --- /dev/null +++ b/README.md @@ -0,0 +1,127 @@ +# Faster-Qwen3-TTS for NVIDIA DGX Spark (GB10) + +Run [faster-qwen3-tts](https://github.com/andimarafioti/faster-qwen3-tts) on the **NVIDIA DGX Spark GB10** (ARM64 / SM 121 / CUDA 13) as a Docker container with an OpenAI-compatible TTS API. + +Integrates with **OpenWebUI**, **SillyTavern**, and any OpenAI TTS-compatible client. + +## What this solves + +The DGX Spark GB10 has a unique combination of ARM64 (Grace CPU) + Blackwell GPU (SM 121) that causes issues with standard ML Docker images: + +- **torchaudio ARM64 wheels** - resolved by using PyTorch's `cu130` wheel index +- **Flash Attention** - won't compile on SM 121, but faster-qwen3-tts uses CUDA graphs instead (6-10x speedup) +- **CUDA graph capture** - works on SM 121 with max_seq_len tuned for voice cloning workloads +- **OpenWebUI voice discovery** - custom endpoints (`/v1/models`, `/v1/audio/voices`) for voice dropdown population + +## Quick Start + +### Option 1: Pull pre-built image (recommended) + +```bash +# Pull the image +docker pull martinb78/faster-qwen3-tts-dgx-spark:latest + +# Download the model +mkdir -p models +huggingface-cli download Qwen/Qwen3-TTS-12Hz-1.7B-Base --local-dir ./models/Qwen3-TTS + +# Copy .env.example to .env and set MODEL_PATH +cp .env.example .env + +# Add voice reference audio (5-15 second WAV/MP3 clips) to config/speakers/ +# See "Adding Voices" below + +# Start +docker compose up -d +``` + +### Option 2: Build from source + +```bash +docker build -t faster-qwen3-tts-dgx-spark:latest . +``` + +## Adding Voices + +Place reference audio files in `config/speakers/` using this naming convention: + +``` +EN_M_Speaker_Name.wav # English, Male +EN_F_Speaker_Name.wav # English, Female +DE_M_Speaker_Name.wav # German, Male +``` + +**Important:** Reference audio must be **5-15 seconds** long. Longer files cause slow inference and poor voice cloning quality. + +For each audio file, create a matching transcript: + +``` +EN_M_Speaker_Name.reference.txt +``` + +Or use the auto-transcription script (requires a running Whisper-compatible ASR service): + +```bash +python config/auto_transcribe.py --api-url http://localhost:8010/v1/audio/transcriptions +``` + +The `generate_voices.py` script runs automatically on container startup and creates `voices.json` from your speaker files. + +## API Endpoints + +| Endpoint | Method | Description | +|---|---|---| +| `/health` | GET | Health check | +| `/v1/audio/speech` | POST | Generate speech (OpenAI-compatible) | +| `/v1/models` | GET | List available voices | +| `/v1/audio/voices` | GET | List voices (OpenWebUI fallback) | +| `/v1/audio/models` | GET | List models (OpenWebUI fallback) | +| `/speakers` | GET | List speaker IDs (SillyTavern) | + +### Example + +```bash +curl -X POST http://localhost:8020/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model": "tts-1", "input": "Hello world!", "voice": "speaker_name", "response_format": "wav"}' \ + --output speech.wav +``` + +## OpenWebUI Configuration + +In OpenWebUI Settings > Audio > Text-to-Speech: + +| Setting | Value | +|---|---| +| Engine | OpenAI | +| URL | `http://faster-qwen3-tts:8000/v1` | +| API Key | `sk-dummy-key` | +| TTS Model | `tts-1` | +| TTS Voice | Select from dropdown | + +## Performance + +On DGX Spark GB10 with the 1.7B model: + +| Input | Audio Duration | Generation Time | RTF | +|---|---|---|---| +| Short sentence | ~2s | ~2.5s | 0.8 | +| Medium paragraph | ~7s | ~5.5s | 0.77 | + +First request is slower due to one-time CUDA graph warmup. + +## Hardware Requirements + +- NVIDIA DGX Spark GB10 (or any ARM64 + Blackwell GPU with CUDA 13) +- ~6 GB GPU memory for the 1.7B model +- CUDA driver 580+ with CUDA 13.0 support + +## Credits + +- [faster-qwen3-tts](https://github.com/andimarafioti/faster-qwen3-tts) by Andres Marafioti +- [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) by Alibaba Qwen team +- DGX Spark compatibility fixes by [mARTin-B78](https://github.com/mARTin-B78) + +## License + +MIT (same as upstream faster-qwen3-tts) diff --git a/config/auto_transcribe.py b/config/auto_transcribe.py new file mode 100644 index 0000000..bf64277 --- /dev/null +++ b/config/auto_transcribe.py @@ -0,0 +1,93 @@ +""" +Batch-transcribe speaker reference audio files using a local Whisper-compatible API. + +Creates .reference.txt files alongside each audio file in the speakers directory. +These transcriptions are used by generate_voices.py to build the voice registry. + +Usage: + python auto_transcribe.py [--api-url http://localhost:8010/v1/audio/transcriptions] + +IMPORTANT: Reference audio for voice cloning should be 5-15 seconds long. +Longer files will produce poor cloning results and slow down inference. +""" + +import os +import sys +import json +import argparse +import requests + +def main(): + parser = argparse.ArgumentParser(description="Batch-transcribe speaker reference audio") + parser.add_argument("--api-url", default="http://localhost:8010/v1/audio/transcriptions", + help="Whisper-compatible transcription API URL") + parser.add_argument("--speaker-dir", default="./speakers", + help="Directory containing speaker audio files") + parser.add_argument("--model", default="whisper-1", + help="Transcription model name") + args = parser.parse_args() + + speaker_dir = args.speaker_dir + + # Verify API is reachable + try: + requests.get(args.api_url.rsplit('/', 2)[0], timeout=5) + except requests.ConnectionError: + print(f"Error: Cannot reach transcription API at {args.api_url}") + print("Make sure your Whisper/ASR service is running.") + sys.exit(1) + + if not os.path.exists(speaker_dir): + print(f"Error: Speaker directory not found: {speaker_dir}") + sys.exit(1) + + audio_files = [f for f in os.listdir(speaker_dir) + if f.endswith(('.wav', '.mp3')) and not f.startswith('.')] + + print(f"Found {len(audio_files)} audio files in {speaker_dir}") + + for filename in sorted(audio_files): + base_name = os.path.splitext(filename)[0] + ref_txt_path = os.path.join(speaker_dir, f"{base_name}.reference.txt") + + if os.path.exists(ref_txt_path): + print(f" Skipping {filename} (already transcribed)") + continue + + filepath = os.path.join(speaker_dir, filename) + print(f" Transcribing {filename}...", end=" ", flush=True) + + try: + with open(filepath, 'rb') as f: + response = requests.post( + args.api_url, + files={"file": (filename, f)}, + data={"model": args.model}, + timeout=60, + ) + + if response.status_code == 200: + # Handle both JSON and plain text responses + try: + text = response.json().get("text", "").strip() + except (json.JSONDecodeError, AttributeError): + text = response.text.strip() + + if text: + with open(ref_txt_path, 'w', encoding='utf-8') as f: + f.write(text) + print(f"OK ({len(text)} chars)") + else: + print("EMPTY (no speech detected)") + else: + print(f"FAILED (HTTP {response.status_code})") + + except requests.Timeout: + print("TIMEOUT") + except Exception as e: + print(f"ERROR: {e}") + + print("Done.") + +if __name__ == "__main__": + main() diff --git a/config/generate_voices.py b/config/generate_voices.py new file mode 100644 index 0000000..288bedf --- /dev/null +++ b/config/generate_voices.py @@ -0,0 +1,62 @@ +""" +Scan the speakers directory for reference audio files and generate voices.json. + +Runs inside the container at startup to map all available voice reference +audio files into a format the OpenAI-compatible TTS server understands. +""" + +import os +import json + +# Internal container paths +speaker_dir = "/config/speakers" +output_file = "/config/voices.json" + +voices = {} + +if os.path.exists(speaker_dir): + for filename in os.listdir(speaker_dir): + if filename.endswith((".wav", ".mp3")): + base_name = os.path.splitext(filename)[0] + + # Determine language based on file prefixes + lang = "Auto" + if base_name.startswith("EN_") or base_name.startswith("basic_ref_en"): + lang = "English" + elif base_name.startswith("DE_"): + lang = "German" + elif base_name.startswith("basic_ref_zh"): + lang = "Chinese" + + # Create a clean voice ID + voice_id = base_name.lower() + prefixes_to_strip = ["en_m_", "en_f_", "de_m_", "de_f_"] + for prefix in prefixes_to_strip: + if voice_id.startswith(prefix): + voice_id = voice_id.replace(prefix, "", 1) + break + + entry = { + "ref_audio": f"/config/speakers/{filename}", + "language": lang + } + + # Look for matching reference text files + ref_txt_path = os.path.join(speaker_dir, f"{base_name}.reference.txt") + txt_path = os.path.join(speaker_dir, f"{base_name}.txt") + + if os.path.exists(ref_txt_path): + with open(ref_txt_path, 'r', encoding='utf-8') as f: + entry["ref_text"] = f.read().strip() + elif os.path.exists(txt_path): + with open(txt_path, 'r', encoding='utf-8') as f: + entry["ref_text"] = f.read().strip() + + voices[voice_id] = entry + + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(voices, f, indent=2, ensure_ascii=False) + + print(f"Success! Generated voices.json with {len(voices)} mapped voices.") +else: + print(f"Warning: Directory {speaker_dir} not found. Skipping voice generation.") diff --git a/config/run_server.py b/config/run_server.py new file mode 100644 index 0000000..6f592a1 --- /dev/null +++ b/config/run_server.py @@ -0,0 +1,57 @@ +""" +Wrapper around faster-qwen3-tts's openai_server.py that injects additional +API endpoints for compatibility with OpenWebUI and SillyTavern. + +Endpoints added: + GET /v1/models - Lists available voices (OpenWebUI primary discovery) + GET /v1/audio/voices - Lists available voices (OpenWebUI fallback) + GET /v1/audio/models - Lists available voices (OpenWebUI fallback) + GET /speakers - Lists speaker IDs (SillyTavern) + OPTIONS /{path} - Pre-flight CORS handler +""" + +import sys +import json +from fastapi.responses import JSONResponse + +# Point Python to the app directory inside the container +sys.path.append("/app/examples") +import openai_server + +# Load generated voices +try: + with open('/config/voices.json', 'r') as f: + voices_data = json.load(f) +except FileNotFoundError: + voices_data = {} + +# Build reusable response payloads +_voice_list = [{'id': v, 'object': 'model', 'created': 1686935002, 'owned_by': 'qwen'} for v in voices_data.keys()] +_models_response = {'object': 'list', 'data': _voice_list} + +# OpenWebUI model discovery (primary) +@openai_server.app.get('/v1/models') +async def list_models(): + return _models_response + +# OpenWebUI voice discovery fallbacks +@openai_server.app.get('/v1/audio/voices') +async def list_audio_voices(): + return _models_response + +@openai_server.app.get('/v1/audio/models') +async def list_audio_models(): + return _models_response + +# SillyTavern speaker endpoint +@openai_server.app.get('/speakers') +async def get_speakers(): + return list(voices_data.keys()) + +# Pre-flight OPTIONS handler to prevent 404s +@openai_server.app.options('/{path:path}') +async def options_handler(path: str): + return JSONResponse(content={'status': 'ok'}) + +if __name__ == '__main__': + openai_server.main() diff --git a/config/speakers/.gitkeep b/config/speakers/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6b1603c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +services: + faster-qwen3-tts: + image: martinb78/faster-qwen3-tts-dgx-spark:latest + container_name: faster-qwen3-tts + restart: unless-stopped + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + # Optional: set HF_TOKEN if model needs downloading from HuggingFace + # - HF_TOKEN=${HF_TOKEN} + ports: + - "8020:8000" + volumes: + # Mount your local Qwen3-TTS model (download first, see README) + - ${MODEL_PATH:-./models/Qwen3-TTS}:/models/Qwen3-TTS:ro + # Config directory (voices.json generated on startup) + - ./config:/config:rw + # Speaker reference audio files + - ./config/speakers:/voices:ro + command: > + /bin/bash -c " + python3 /config/generate_voices.py && + python3 /config/run_server.py + --model /models/Qwen3-TTS + --voices /config/voices.json + --port 8000 + " + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + networks: + - dgx_net + +networks: + dgx_net: + external: true diff --git a/patches/openai_server.patch b/patches/openai_server.patch new file mode 100644 index 0000000..aadd4af --- /dev/null +++ b/patches/openai_server.patch @@ -0,0 +1,20 @@ +diff --git a/examples/openai_server.py b/examples/openai_server.py +index 2199e14..d38a684 100644 +--- a/examples/openai_server.py ++++ b/examples/openai_server.py +@@ -306,6 +306,7 @@ def _parse_args(): + p.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)") + p.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)") + p.add_argument("--device", default="cuda", help="Torch device (default: cuda)") ++ p.add_argument("--max-seq-len", type=int, default=4096, help="Max sequence length for CUDA graph static cache (default: 4096)") + return p.parse_args() + + +@@ -344,6 +345,7 @@ def main(): + args.model, + device=args.device, + dtype=torch.bfloat16, ++ max_seq_len=args.max_seq_len, + ) + SAMPLE_RATE = tts_model.sample_rate + logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)