Add CustomVoice server, benchmark tool, and VoiceDesign API improvements

- Add run_customvoice_server.py: OpenAI-compatible TTS server for Qwen3-TTS
  CustomVoice models with speaker-based voice selection and per-request
  language/instruct/max_new_tokens overrides
- Add customvoice_voices.json: voice config for 9 built-in speakers across
  English, Chinese, Japanese, and Korean
- Add benchmark_api.py: API benchmarking tool reporting TTFA, total time,
  RTF, and speed multiplier across short/medium/long sentences
- Refactor run_voicedesign_server.py: extract _request_generation_params()
  helper, add per-request language/instruct/max_new_tokens override support,
  and wire DEFAULT_MAX_NEW_TOKENS to --max-seq-len arg
- Add faster-qwen3-tts.code-workspace for VS Code project config

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

config/customvoice_voices.json

@@ -0,0 +1,47 @@
+{
+  "Ryan": {
+    "speaker": "Ryan",
+    "language": "English",
+    "instruct": ""
+  },
+  "Aiden": {
+    "speaker": "Aiden",
+    "language": "English",
+    "instruct": ""
+  },
+  "Vivian": {
+    "speaker": "Vivian",
+    "language": "Chinese",
+    "instruct": ""
+  },
+  "Serena": {
+    "speaker": "Serena",
+    "language": "Chinese",
+    "instruct": ""
+  },
+  "Uncle_Fu": {
+    "speaker": "Uncle_Fu",
+    "language": "Chinese",
+    "instruct": ""
+  },
+  "Dylan": {
+    "speaker": "Dylan",
+    "language": "Chinese",
+    "instruct": ""
+  },
+  "Eric": {
+    "speaker": "Eric",
+    "language": "Chinese",
+    "instruct": ""
+  },
+  "Ono_Anna": {
+    "speaker": "Ono_Anna",
+    "language": "Japanese",
+    "instruct": ""
+  },
+  "Sohee": {
+    "speaker": "Sohee",
+    "language": "Korean",
+    "instruct": ""
+  }
+}

config/run_customvoice_server.py

@@ -0,0 +1,233 @@
+"""
+OpenAI-compatible TTS server for Qwen3-TTS CustomVoice models.
+
+Voices are defined in customvoice_voices.json as:
+  { "voice_id": { "speaker": "Ryan", "language": "English", "instruct": "" } }
+
+The request body may also include "language" and "instruct" fields to override
+the configured defaults for a single generation.
+"""
+import argparse
+import asyncio
+import json
+import logging
+import queue
+import sys
+import threading
+from typing import Optional
+
+import numpy as np
+import uvicorn
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse, Response, StreamingResponse
+from pydantic import BaseModel
+
+sys.path.append("/app")
+from faster_qwen3_tts.model import FasterQwen3TTS
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+app = FastAPI()
+tts_model: FasterQwen3TTS = None
+voices: dict = {}
+default_voice: str = None
+SAMPLE_RATE = 24000
+DEFAULT_MAX_NEW_TOKENS = 2048
+_model_lock = threading.Lock()
+
+
+class SpeechRequest(BaseModel):
+    model: str = "tts-1"
+    input: str
+    voice: str = "Ryan"
+    response_format: str = "wav"
+    speed: float = 1.0
+    language: Optional[str] = None
+    instruct: Optional[str] = None
+    max_new_tokens: Optional[int] = None
+
+
+def _to_pcm16(audio: np.ndarray) -> bytes:
+    return (audio * 32767).clip(-32768, 32767).astype(np.int16).tobytes()
+
+
+def _wav_header(sample_rate: int) -> bytes:
+    import struct
+
+    return struct.pack(
+        "<4sI4s4sIHHIIHH4sI",
+        b"RIFF", 0xFFFFFFFF, b"WAVE",
+        b"fmt ", 16, 1, 1,
+        sample_rate, sample_rate * 2, 2, 16,
+        b"data", 0xFFFFFFFF,
+    )
+
+
+def _to_mp3_bytes(audio: np.ndarray, sr: int) -> bytes:
+    import io
+    from pydub import AudioSegment
+
+    pcm = _to_pcm16(audio)
+    seg = AudioSegment(pcm, frame_rate=sr, sample_width=2, channels=1)
+    buf = io.BytesIO()
+    seg.export(buf, format="mp3")
+    return buf.getvalue()
+
+
+def resolve_voice(name: str) -> dict:
+    cfg = voices.get(name)
+    if cfg:
+        return cfg
+    if default_voice and default_voice in voices:
+        logger.warning("Voice %r not found, falling back to %r", name, default_voice)
+        return voices[default_voice]
+    raise HTTPException(status_code=404, detail=f"Voice {name!r} not found")
+
+
+def _request_generation_params(req: SpeechRequest, voice_cfg: dict) -> dict:
+    return {
+        "text": req.input,
+        "speaker": voice_cfg.get("speaker") or req.voice,
+        "language": req.language or voice_cfg.get("language", "Auto"),
+        "instruct": req.instruct if req.instruct is not None else voice_cfg.get("instruct") or None,
+        "max_new_tokens": req.max_new_tokens or int(voice_cfg.get("max_new_tokens", DEFAULT_MAX_NEW_TOKENS)),
+    }
+
+
+async def _stream_chunks(params: dict):
+    q: queue.Queue = queue.Queue()
+    done = object()
+
+    def producer():
+        try:
+            with _model_lock:
+                for chunk, _sr, _timing in tts_model.generate_custom_voice_streaming(**params):
+                    q.put(chunk)
+        except Exception as exc:
+            q.put(exc)
+        finally:
+            q.put(done)
+
+    threading.Thread(target=producer, daemon=True).start()
+    loop = asyncio.get_event_loop()
+    while True:
+        item = await loop.run_in_executor(None, q.get)
+        if item is done:
+            break
+        if isinstance(item, Exception):
+            raise item
+        yield _to_pcm16(item)
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok", "model_loaded": tts_model is not None}
+
+
+@app.post("/v1/audio/speech")
+async def create_speech(req: SpeechRequest):
+    if tts_model is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    if not req.input.strip():
+        raise HTTPException(status_code=400, detail="'input' text is empty")
+
+    voice_cfg = resolve_voice(req.voice)
+    params = _request_generation_params(req, voice_cfg)
+    fmt = req.response_format.lower()
+
+    content_types = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"}
+    if fmt not in content_types:
+        raise HTTPException(status_code=400, detail=f"Unsupported format: {fmt!r}")
+
+    if fmt == "mp3":
+        loop = asyncio.get_event_loop()
+
+        def generate():
+            with _model_lock:
+                return tts_model.generate_custom_voice(**params)
+
+        audio_arrays, sr = await loop.run_in_executor(None, generate)
+        audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32)
+        return Response(content=_to_mp3_bytes(audio, sr), media_type="audio/mpeg")
+
+    async def audio_stream():
+        if fmt == "wav":
+            yield _wav_header(SAMPLE_RATE)
+        async for raw in _stream_chunks(params):
+            yield raw
+
+    return StreamingResponse(audio_stream(), media_type=content_types[fmt])
+
+
+_voice_list = None
+_models_response = None
+
+
+def _build_voice_list():
+    global _voice_list, _models_response
+    _voice_list = [{"id": v, "object": "model", "created": 1686935002, "owned_by": "qwen"} for v in voices]
+    _models_response = {"object": "list", "data": _voice_list}
+
+
+@app.get("/v1/models")
+async def list_models():
+    return _models_response
+
+
+@app.get("/v1/audio/voices")
+async def list_audio_voices():
+    return _models_response
+
+
+@app.get("/v1/audio/models")
+async def list_audio_models():
+    return _models_response
+
+
+@app.get("/speakers")
+async def get_speakers():
+    return list(voices.keys())
+
+
+@app.options("/{path:path}")
+async def options_handler(path: str):
+    return JSONResponse(content={"status": "ok"})
+
+
+def main():
+    global tts_model, voices, default_voice, SAMPLE_RATE, DEFAULT_MAX_NEW_TOKENS
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", default="/models/Qwen3-TTS-CustomVoice")
+    parser.add_argument("--voices", default="/config/customvoice_voices.json")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--device", default="cuda")
+    parser.add_argument("--max-seq-len", type=int, default=2048)
+    args = parser.parse_args()
+    DEFAULT_MAX_NEW_TOKENS = args.max_seq_len
+
+    with open(args.voices) as f:
+        voices = json.load(f)
+    default_voice = next(iter(voices), None)
+    _build_voice_list()
+
+    import torch
+
+    logger.info("Loading CustomVoice model %s ...", args.model)
+    tts_model = FasterQwen3TTS.from_pretrained(
+        args.model,
+        device=args.device,
+        dtype=torch.bfloat16,
+        attn_implementation="sdpa",
+        max_seq_len=args.max_seq_len,
+    )
+    SAMPLE_RATE = tts_model.sample_rate
+    logger.info("Model ready. Sample rate: %d Hz", SAMPLE_RATE)
+
+    uvicorn.run(app, host=args.host, port=args.port, log_level="info")
+
+
+if __name__ == "__main__":
+    main()

config/run_voicedesign_server.py

@@ -14,6 +14,7 @@ import asyncio
 import argparse
 import numpy as np
 import sys
+from typing import Optional
 
 import uvicorn
 from fastapi import FastAPI, HTTPException
@@ -31,6 +32,7 @@ tts_model: FasterQwen3TTS = None
 voices: dict = {}
 default_voice: str = None
 SAMPLE_RATE = 24000
+DEFAULT_MAX_NEW_TOKENS = 2048
 _model_lock = threading.Lock()
 
 
@@ -44,6 +46,9 @@ class SpeechRequest(BaseModel):
     voice: str = "vd_british_male"
     response_format: str = "wav"
     speed: float = 1.0
+    language: Optional[str] = None
+    instruct: Optional[str] = None
+    max_new_tokens: Optional[int] = None
 
 
 # ---------------------------------------------------------------------------
@@ -89,18 +94,25 @@ def resolve_voice(name: str) -> dict:
 # Generation helpers
 # ---------------------------------------------------------------------------
 
-async def _stream_chunks(voice_cfg: dict, text: str):
+def _request_generation_params(req: SpeechRequest, voice_cfg: dict) -> dict:
+    instruct = req.instruct if req.instruct is not None else voice_cfg.get("instruct", "")
+    language = req.language or voice_cfg.get("language", "English")
+    return {
+        "text": req.input,
+        "instruct": instruct,
+        "language": language,
+        "max_new_tokens": req.max_new_tokens or int(voice_cfg.get("max_new_tokens", DEFAULT_MAX_NEW_TOKENS)),
+    }
+
+
+async def _stream_chunks(params: dict):
     q: queue.Queue = queue.Queue()
     _DONE = object()
 
     def producer():
         try:
             with _model_lock:
-                for chunk, _sr, _timing in tts_model.generate_voice_design_streaming(
+                for chunk, _sr, _timing in tts_model.generate_voice_design_streaming(**params):
-                    text=text,
-                    instruct=voice_cfg["instruct"],
-                    language=voice_cfg.get("language", "English"),
-                ):
                     q.put(chunk)
         except Exception as exc:
             q.put(exc)
@@ -135,6 +147,7 @@ async def create_speech(req: SpeechRequest):
         raise HTTPException(status_code=400, detail="'input' text is empty")
 
     voice_cfg = resolve_voice(req.voice)
+    params = _request_generation_params(req, voice_cfg)
     fmt = req.response_format.lower()
 
     _CONTENT_TYPES = {"wav": "audio/wav", "pcm": "audio/pcm", "mp3": "audio/mpeg"}
@@ -145,11 +158,7 @@ async def create_speech(req: SpeechRequest):
         loop = asyncio.get_event_loop()
         def _gen():
             with _model_lock:
-                return tts_model.generate_voice_design(
+                return tts_model.generate_voice_design(**params)
-                    text=req.input,
-                    instruct=voice_cfg["instruct"],
-                    language=voice_cfg.get("language", "English"),
-                )
         audio_arrays, sr = await loop.run_in_executor(None, _gen)
         audio = audio_arrays[0] if audio_arrays else np.zeros(1, dtype=np.float32)
         return Response(content=_to_mp3_bytes(audio, sr), media_type="audio/mpeg")
@@ -157,7 +166,7 @@ async def create_speech(req: SpeechRequest):
     async def audio_stream():
         if fmt == "wav":
             yield _wav_header(SAMPLE_RATE)
-        async for raw in _stream_chunks(voice_cfg, req.input):
+        async for raw in _stream_chunks(params):
             yield raw
 
     return StreamingResponse(audio_stream(), media_type=_CONTENT_TYPES[fmt])
@@ -199,7 +208,7 @@ async def options_handler(path: str):
 # ---------------------------------------------------------------------------
 
 def main():
-    global tts_model, voices, default_voice, SAMPLE_RATE
+    global tts_model, voices, default_voice, SAMPLE_RATE, DEFAULT_MAX_NEW_TOKENS
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--model", default="/models/Qwen3-TTS-VoiceDesign")
@@ -209,6 +218,7 @@ def main():
     parser.add_argument("--device", default="cuda")
     parser.add_argument("--max-seq-len", type=int, default=2048)
     args = parser.parse_args()
+    DEFAULT_MAX_NEW_TOKENS = args.max_seq_len
 
     with open(args.voices) as f:
         voices = json.load(f)

faster-qwen3-tts.code-workspace

@@ -0,0 +1,8 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {}
+}