Reduce latency: CUDA warmup, chunk_size=4, max-seq-len 2048

- run_server.py: warm up CUDA graphs at server startup via lifespan
  event so the first real request does not pay the 7-8s graph-
  compilation penalty; uses modern lifespan API instead of deprecated
  on_event
- generate_voices.py: set chunk_size=4 per voice so streaming clients
  receive first audio after ~333ms instead of ~1s
- docker-compose.yml: add --max-seq-len 2048 (halves static KV cache,
  reduces VRAM and graph-capture time)
- config/benchmark_api.py: new script to measure TTFA, RTF and speed
  against the live API endpoint

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

config/benchmark_api.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+Benchmark the faster-qwen3-tts API server.
+
+Measures:
+  - TTFA  : time-to-first-audio (ms) — latency before playback can start
+  - Total : wall-clock time until full audio received (s)
+  - RTF   : real-time factor = generation_time / audio_duration  (lower = faster)
+  - Speed : audio_duration / generation_time  (higher = faster, e.g. 3× real-time)
+
+Usage:
+    python benchmark_api.py [--host localhost] [--port 8020] [--voice alloy]
+                            [--runs 3] [--format wav]
+"""
+import argparse
+import struct
+import sys
+import time
+
+import requests
+
+# ── Test sentences of increasing length ─────────────────────────────────────
+SENTENCES = [
+    ("short",  "Hello, how are you today?"),
+    ("medium", "The quick brown fox jumps over the lazy dog near the river bank."),
+    ("long",   "Artificial intelligence is transforming the way we interact with "
+               "technology. From voice assistants to autonomous vehicles, machine "
+               "learning models are becoming an integral part of everyday life."),
+]
+
+SAMPLE_RATE = 24000
+BYTES_PER_SAMPLE = 2  # 16-bit PCM
+
+
+def parse_wav_header(data: bytes) -> int:
+    """Return the data-chunk offset so we can skip the header bytes."""
+    # WAV: RIFF(4) + size(4) + WAVE(4) + fmt (8+16) + data(4) + size(4) = 44 bytes
+    if data[:4] != b"RIFF" or data[8:12] != b"WAVE":
+        return 0
+    # Walk chunks after WAVE marker
+    pos = 12
+    while pos + 8 <= len(data):
+        chunk_id = data[pos:pos+4]
+        chunk_sz = struct.unpack_from("<I", data, pos+4)[0]
+        if chunk_id == b"data":
+            return pos + 8
+        pos += 8 + chunk_sz
+    return 44  # fallback
+
+
+def benchmark_request(url: str, payload: dict, label: str, run: int) -> dict:
+    t0 = time.perf_counter()
+    first_byte_t = None
+    raw = bytearray()
+
+    with requests.post(url, json=payload, stream=True, timeout=120) as resp:
+        resp.raise_for_status()
+        for chunk in resp.iter_content(chunk_size=512):
+            if chunk:
+                if first_byte_t is None:
+                    first_byte_t = time.perf_counter()
+                raw.extend(chunk)
+
+    t1 = time.perf_counter()
+
+    ttfa_ms    = (first_byte_t - t0) * 1000 if first_byte_t else float("nan")
+    total_s    = t1 - t0
+
+    # Strip WAV header to count actual PCM bytes
+    if payload.get("response_format", "wav") == "wav":
+        data_offset = parse_wav_header(bytes(raw))
+        pcm_bytes = len(raw) - data_offset
+    else:
+        pcm_bytes = len(raw)
+
+    audio_s  = pcm_bytes / (SAMPLE_RATE * BYTES_PER_SAMPLE)
+    rtf      = total_s / audio_s if audio_s > 0 else float("nan")
+    speedup  = audio_s / total_s if total_s > 0 else float("nan")
+
+    return {
+        "label":    label,
+        "run":      run,
+        "ttfa_ms":  ttfa_ms,
+        "total_s":  total_s,
+        "audio_s":  audio_s,
+        "rtf":      rtf,
+        "speedup":  speedup,
+    }
+
+
+def main():
+    p = argparse.ArgumentParser(description="Benchmark faster-qwen3-tts API")
+    p.add_argument("--host",   default="localhost")
+    p.add_argument("--port",   type=int, default=8020)
+    p.add_argument("--voice",  default=None,
+                   help="Voice name (default: first available voice)")
+    p.add_argument("--runs",   type=int, default=3,
+                   help="Repetitions per sentence (default: 3)")
+    p.add_argument("--format", default="wav", choices=["wav", "pcm"],
+                   help="Response format (default: wav)")
+    args = p.parse_args()
+
+    base = f"http://{args.host}:{args.port}"
+    url  = f"{base}/v1/audio/speech"
+
+    # Auto-detect first voice if not specified
+    voice = args.voice
+    if not voice:
+        try:
+            models = requests.get(f"{base}/v1/models", timeout=5).json()
+            voice = models["data"][0]["id"]
+            print(f"Auto-selected voice: {voice!r}")
+        except Exception:
+            voice = "alloy"
+            print(f"Could not auto-detect voice, using {voice!r}")
+
+    print(f"\n{'─'*65}")
+    print(f"  faster-qwen3-tts API Benchmark")
+    print(f"  URL: {url}  voice={voice}  runs={args.runs}  fmt={args.format}")
+    print(f"{'─'*65}\n")
+
+    header = f"{'Label':<10}  {'Run':>3}  {'TTFA*':>9}  {'Total':>7}  {'Audio':>7}  {'RTF':>6}  {'Speed':>8}"
+    print(header)
+    print("─" * len(header))
+    print("  * TTFA = time to first PCM byte (WAV header excluded)")
+    print()
+
+    all_results = []
+
+    for label, text in SENTENCES:
+        payload = {
+            "model": "tts-1",
+            "input": text,
+            "voice": voice,
+            "response_format": args.format,
+        }
+        for run in range(1, args.runs + 1):
+            try:
+                r = benchmark_request(url, payload, label, run)
+                all_results.append(r)
+                print(
+                    f"{r['label']:<10}  {r['run']:>3}  "
+                    f"{r['ttfa_ms']:>7.0f}ms  "
+                    f"{r['total_s']:>6.2f}s  "
+                    f"{r['audio_s']:>6.2f}s  "
+                    f"{r['rtf']:>5.2f}x  "
+                    f"{r['speedup']:>6.1f}x rt"
+                )
+                sys.stdout.flush()
+            except Exception as e:
+                print(f"{label:<10}  run {run}  ERROR: {e}")
+
+    # ── Summary ──────────────────────────────────────────────────────────────
+    if all_results:
+        print(f"\n{'─'*65}")
+        print("  Summary (averages across all runs)")
+        print(f"{'─'*65}")
+        for label, _ in SENTENCES:
+            rows = [r for r in all_results if r["label"] == label]
+            if not rows:
+                continue
+            avg_ttfa  = sum(r["ttfa_ms"] for r in rows) / len(rows)
+            avg_total = sum(r["total_s"] for r in rows) / len(rows)
+            avg_rtf   = sum(r["rtf"]     for r in rows) / len(rows)
+            avg_speed = sum(r["speedup"] for r in rows) / len(rows)
+            print(
+                f"  {label:<10}  TTFA={avg_ttfa:.0f}ms  "
+                f"total={avg_total:.2f}s  RTF={avg_rtf:.2f}x  "
+                f"speed={avg_speed:.1f}× real-time"
+            )
+        print()
+
+
+if __name__ == "__main__":
+    main()

config/generate_voices.py

@@ -40,7 +40,8 @@ if os.path.exists(speaker_dir):
 
             entry = {
                 "ref_audio": f"/config/speakers/{filename}",
-                "language": lang
+                "language": lang,
+                "chunk_size": 4,
             }
 
             # Look for matching reference text files

config/run_server.py

@@ -8,16 +8,69 @@ Endpoints added:
   GET /v1/audio/models  - Lists available voices (OpenWebUI fallback)
   GET /speakers         - Lists speaker IDs (SillyTavern)
   OPTIONS /{path}       - Pre-flight CORS handler
+
+Startup:
+  CUDA graphs are warmed up on server start so the first real request
+  does not pay the 7-8s graph-compilation penalty.
 """
 
+import asyncio
+import logging
 import sys
 import json
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
 from fastapi.responses import JSONResponse
 
 # Point Python to the app directory inside the container
 sys.path.append("/app/examples")
 import openai_server
 
+logger = logging.getLogger(__name__)
+
+
+def _do_warmup():
+    """Run one short generation to compile CUDA graphs before serving requests."""
+    model = openai_server.tts_model
+    voices = openai_server.voices
+    default_voice = openai_server.default_voice
+
+    if model is None or not voices or default_voice is None:
+        logger.warning("Warmup skipped: model or voices not ready")
+        return
+
+    voice_cfg = voices.get(default_voice, {})
+    ref_audio = voice_cfg.get("ref_audio")
+    if not ref_audio:
+        logger.warning("Warmup skipped: no ref_audio on default voice")
+        return
+
+    logger.info("Warming up CUDA graphs (first request will be fast)...")
+    try:
+        for _ in model.generate_voice_clone_streaming(
+            text="Warmup.",
+            language=voice_cfg.get("language", "Auto"),
+            ref_audio=ref_audio,
+            ref_text=voice_cfg.get("ref_text", ""),
+            chunk_size=12,
+        ):
+            pass
+        logger.info("CUDA warmup complete — server ready.")
+    except Exception as exc:
+        logger.warning("Warmup failed (non-fatal): %s", exc)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    loop = asyncio.get_event_loop()
+    await loop.run_in_executor(None, _do_warmup)
+    yield
+
+
+# Attach lifespan to the existing FastAPI app
+openai_server.app.router.lifespan_context = lifespan
+
 # Load generated voices
 try:
     with open('/config/voices.json', 'r') as f:

docker-compose.yml

@@ -24,6 +24,7 @@ services:
       --model /models/Qwen3-TTS
       --voices /config/voices.json
       --port 8000
+      --max-seq-len 2048
       "
     deploy:
       resources: