mARTin-B78/tts-dgx-spark-faster-qwen3-tts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CustomVoice server, benchmark tool, and VoiceDesign API improvements

Browse Source 
- Add run_customvoice_server.py: OpenAI-compatible TTS server for Qwen3-TTS
  CustomVoice models with speaker-based voice selection and per-request
  language/instruct/max_new_tokens overrides
- Add customvoice_voices.json: voice config for 9 built-in speakers across
  English, Chinese, Japanese, and Korean
- Add benchmark_api.py: API benchmarking tool reporting TTFA, total time,
  RTF, and speed multiplier across short/medium/long sentences
- Refactor run_voicedesign_server.py: extract _request_generation_params()
  helper, add per-request language/instruct/max_new_tokens override support,
  and wire DEFAULT_MAX_NEW_TOKENS to --max-seq-len arg
- Add faster-qwen3-tts.code-workspace for VS Code project config

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
mARTin-B78 2026-05-18 18:18:26 +02:00
parent 295de3059d
commit 8363dc7eb1
4 changed files with 311 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
config/customvoice_voices.json Normal file
View File

@ -0,0 +1,47 @@
{
"Ryan": {
"speaker": "Ryan",
"language": "English",
"instruct": ""
},
"Aiden": {
"speaker": "Aiden",
"language": "English",
"instruct": ""
},
"Vivian": {
"speaker": "Vivian",
"language": "Chinese",
"instruct": ""
},
"Serena": {
"speaker": "Serena",
"language": "Chinese",
"instruct": ""
},
"Uncle_Fu": {
"speaker": "Uncle_Fu",
"language": "Chinese",
"instruct": ""
},
"Dylan": {
"speaker": "Dylan",
"language": "Chinese",
"instruct": ""
},
"Eric": {
"speaker": "Eric",
"language": "Chinese",
"instruct": ""
},
"Ono_Anna": {
"speaker": "Ono_Anna",
"language": "Japanese",
"instruct": ""
},
"Sohee": {
"speaker": "Sohee",
"language": "Korean",
"instruct": ""
}
}

233
config/run_customvoice_server.py Normal file
View File

@ -0,0 +1,233 @@
"""
OpenAI-compatible TTS server for Qwen3-TTS CustomVoice models.
Voices are defined in customvoice_voices.json as:
{ "voice_id": { "speaker": "Ryan", "language": "English", "instruct": "" } }
The request body may also include "language" and "instruct" fields to override
the configured defaults for a single generation.
"""
import argparse
import asyncio
import json
import logging
import queue
import sys
import threading
from typing import Optional
import numpy as np
import uvicorn
from fastapi import FastAPI, HTTPException
from fastapi.responses import JSONResponse, Response, StreamingResponse
from pydantic import BaseModel
sys.path.append("/app")
from faster_qwen3_tts.model import FasterQwen3TTS
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
app = FastAPI()
tts_model: FasterQwen3TTS = None
voices: dict = {}
default_voice: str = None
SAMPLE_RATE = 24000
DEFAULT_MAX_NEW_TOKENS = 2048
_model_lock = threading.Lock()
class SpeechRequest(BaseModel):
model: str = "tts-1"
input: str
voice: str = "Ryan"
response_format: str = "wav"
speed: float = 1.0
language: Optional[str] = None
instruct: Optional[str] = None
max_new_tokens: Optional[int] = None
def _to_pcm16(audio: np.ndarray) -> bytes:
return (audio * 32767).clip(-32768, 32767).astype(np.int16).tobytes()
def _wav_header(sample_rate: int) -> bytes:
import struct
return struct.pack(
"<4sI4s4sIHHIIHH4sI",
b"RIFF", 0xFFFFFFFF, b"WAVE",
b"fmt ", 16, 1, 1,
sample_rate, sample_rate * 2, 2, 16,
b"data", 0xFFFFFFFF,
)
def _to_mp3_bytes(audio: np.ndarray, sr: int) -> bytes:
import io
from pydub import AudioSegment
pcm = _to_pcm16(audio)
seg = AudioSegment(pcm, frame_rate=sr, sample_width=2, channels=1)
buf = io.BytesIO()
seg.export(buf, format="mp3")
return buf.getvalue()
def resolve_voice(name: str) -> dict:
cfg = voices.get(name)
if cfg:
return cfg
if default_voice and default_voice in voices:
logger.warning("Voice %r not found, falling back to %r", name, default_voice)
return voices[default_voice]
raise HTTPException(status_code=404, detail=f"Voice {name!r} not found")
def _request_generation_params(req: SpeechRequest, voice_cfg: dict) -> dict:
return {
"text": req.input,
"speaker": voice_cfg.get("speaker") or req.voice,
"language": req.language or voice_cfg.get("language", "Auto"),
"instruct": req.instruct if req.instruct is not None else voice_cfg.get("instruct") or None,
"max_new_tokens": req.max_new_tokens or int(voice_cfg.get("max_new_tokens", DEFAULT_MAX_NEW_TOKENS)),
}
async def _stream_chunks(params: dict):
q: queue.Queue = queue.Queue()
done = object()
def producer():
try:
with _model_lock:
for chunk, _sr, _timing in tts_model.generate_custom_voice_streaming(**params):
q.put(chunk)
except Exception as exc:
q.put(exc)
finally:
q.put(done)
threading.Thread(target=producer, daemon=True).start()
loop = asyncio.get_event_loop()
while True:
item = await loop.run_in_executor(None, q.get)
if item is done:
break
if isinstance(item, Exception):
raise item
yield _to_pcm16(item)
@app.get("/health")
async def health():
return {"status": "ok", "model_loaded": tts_model is not None}
@app.post("/v1/audio/speech")
async def create_speech(req: SpeechRequest):
if tts_model is None:
raise HTTPException(status_code=503, detail="Model not loaded")
if not req.input.strip():
raise HTTPException(status_code=400, detail="'input' text is empty")
voice_cfg = resolve_voice(req.voice)
params = _request_generation_params(req, voice_cfg)
fmt = req.response_format.lower()
content_types = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"}
if fmt not in content_types:
raise HTTPException(status_code=400, detail=f"Unsupported format: {fmt!r}")
if fmt == "mp3":
loop = asyncio.get_event_loop()
def generate():
with _model_lock:
return tts_model.generate_custom_voice(**params)
audio_arrays, sr = await loop.run_in_executor(None, generate)
audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32)
return Response(content=_to_mp3_bytes(audio, sr), media_type="audio/mpeg")
async def audio_stream():
if fmt == "wav":
yield _wav_header(SAMPLE_RATE)
async for raw in _stream_chunks(params):
yield raw
return StreamingResponse(audio_stream(), media_type=content_types[fmt])
_voice_list = None
_models_response = None
def _build_voice_list():
global _voice_list, _models_response
_voice_list = [{"id": v, "object": "model", "created": 1686935002, "owned_by": "qwen"} for v in voices]
_models_response = {"object": "list", "data": _voice_list}
@app.get("/v1/models")
async def list_models():
return _models_response
@app.get("/v1/audio/voices")
async def list_audio_voices():
return _models_response
@app.get("/v1/audio/models")
async def list_audio_models():
return _models_response
@app.get("/speakers")
async def get_speakers():
return list(voices.keys())
@app.options("/{path:path}")
async def options_handler(path: str):
return JSONResponse(content={"status": "ok"})
def main():
global tts_model, voices, default_voice, SAMPLE_RATE, DEFAULT_MAX_NEW_TOKENS
parser = argparse.ArgumentParser()
parser.add_argument("--model", default="/models/Qwen3-TTS-CustomVoice")
parser.add_argument("--voices", default="/config/customvoice_voices.json")
parser.add_argument("--port", type=int, default=8000)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--device", default="cuda")
parser.add_argument("--max-seq-len", type=int, default=2048)
args = parser.parse_args()
DEFAULT_MAX_NEW_TOKENS = args.max_seq_len
with open(args.voices) as f:
voices = json.load(f)
default_voice = next(iter(voices), None)
_build_voice_list()
import torch
logger.info("Loading CustomVoice model %s ...", args.model)
tts_model = FasterQwen3TTS.from_pretrained(
args.model,
device=args.device,
dtype=torch.bfloat16,
attn_implementation="sdpa",
max_seq_len=args.max_seq_len,
)
SAMPLE_RATE = tts_model.sample_rate
logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
if __name__ == "__main__":
main()

36
config/run_voicedesign_server.py
View File

@ -14,6 +14,7 @@ import asyncio
import argparse
import numpy as np
import sys
from typing import Optional
import uvicorn
from fastapi import FastAPI, HTTPException
@ -31,6 +32,7 @@ tts_model: FasterQwen3TTS = None
voices: dict = {}
default_voice: str = None
SAMPLE_RATE = 24000
DEFAULT_MAX_NEW_TOKENS = 2048
_model_lock = threading.Lock()
@ -44,6 +46,9 @@ class SpeechRequest(BaseModel):
voice: str = "vd_british_male"
response_format: str = "wav"
speed: float = 1.0
language: Optional[str] = None
instruct: Optional[str] = None
max_new_tokens: Optional[int] = None
# ---------------------------------------------------------------------------
@ -89,18 +94,25 @@ def resolve_voice(name: str) -> dict:
# Generation helpers
# ---------------------------------------------------------------------------
async def _stream_chunks(voice_cfg: dict, text: str):
def _request_generation_params(req: SpeechRequest, voice_cfg: dict) -> dict:
instruct = req.instruct if req.instruct is not None else voice_cfg.get("instruct", "")
language = req.language or voice_cfg.get("language", "English")
return {
"text": req.input,
"instruct": instruct,
"language": language,
"max_new_tokens": req.max_new_tokens or int(voice_cfg.get("max_new_tokens", DEFAULT_MAX_NEW_TOKENS)),
}
async def _stream_chunks(params: dict):
q: queue.Queue = queue.Queue()
_DONE = object()
def producer():
try:
with _model_lock:
for chunk, _sr, _timing in tts_model.generate_voice_design_streaming(
text=text,
instruct=voice_cfg["instruct"],
language=voice_cfg.get("language", "English"),
):
for chunk, _sr, _timing in tts_model.generate_voice_design_streaming(**params):
q.put(chunk)
except Exception as exc:
q.put(exc)
@ -135,6 +147,7 @@ async def create_speech(req: SpeechRequest):
raise HTTPException(status_code=400, detail="'input' text is empty")
voice_cfg = resolve_voice(req.voice)
params = _request_generation_params(req, voice_cfg)
fmt = req.response_format.lower()
_CONTENT_TYPES = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"}
@ -145,11 +158,7 @@ async def create_speech(req: SpeechRequest):
loop = asyncio.get_event_loop()
def _gen():
with _model_lock:
return tts_model.generate_voice_design(
text=req.input,
instruct=voice_cfg["instruct"],
language=voice_cfg.get("language", "English"),
)
return tts_model.generate_voice_design(**params)
audio_arrays, sr = await loop.run_in_executor(None, _gen)
audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32)
return Response(content=_to_mp3_bytes(audio, sr), media_type="audio/mpeg")
@ -157,7 +166,7 @@ async def create_speech(req: SpeechRequest):
async def audio_stream():
if fmt == "wav":
yield _wav_header(SAMPLE_RATE)
async for raw in _stream_chunks(voice_cfg, req.input):
async for raw in _stream_chunks(params):
yield raw
return StreamingResponse(audio_stream(), media_type=_CONTENT_TYPES[fmt])
@ -199,7 +208,7 @@ async def options_handler(path: str):
# ---------------------------------------------------------------------------
def main():
global tts_model, voices, default_voice, SAMPLE_RATE
global tts_model, voices, default_voice, SAMPLE_RATE, DEFAULT_MAX_NEW_TOKENS
parser = argparse.ArgumentParser()
parser.add_argument("--model", default="/models/Qwen3-TTS-VoiceDesign")
@ -209,6 +218,7 @@ def main():
parser.add_argument("--device", default="cuda")
parser.add_argument("--max-seq-len", type=int, default=2048)
args = parser.parse_args()
DEFAULT_MAX_NEW_TOKENS = args.max_seq_len
with open(args.voices) as f:
voices = json.load(f)

8
faster-qwen3-tts.code-workspace Normal file
View File

@ -0,0 +1,8 @@
{
"folders": [
{
"path": "."
}
],
"settings": {}
}