From 92e2381d0b9a479326d18e3a556f913e8eabca72 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Mon, 11 May 2026 20:43:21 +0200 Subject: [PATCH] Reduce latency: CUDA warmup, chunk_size=4, max-seq-len 2048 - run_server.py: warm up CUDA graphs at server startup via lifespan event so the first real request does not pay the 7-8s graph- compilation penalty; uses modern lifespan API instead of deprecated on_event - generate_voices.py: set chunk_size=4 per voice so streaming clients receive first audio after ~333ms instead of ~1s - docker-compose.yml: add --max-seq-len 2048 (halves static KV cache, reduces VRAM and graph-capture time) - config/benchmark_api.py: new script to measure TTFA, RTF and speed against the live API endpoint Co-Authored-By: Claude Sonnet 4.6 --- config/benchmark_api.py | 175 ++++++++++++++++++++++++++++++++++++++ config/generate_voices.py | 3 +- config/run_server.py | 53 ++++++++++++ docker-compose.yml | 1 + 4 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 config/benchmark_api.py diff --git a/config/benchmark_api.py b/config/benchmark_api.py new file mode 100644 index 0000000..635e5b4 --- /dev/null +++ b/config/benchmark_api.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +""" +Benchmark the faster-qwen3-tts API server. + +Measures: + - TTFA : time-to-first-audio (ms) — latency before playback can start + - Total : wall-clock time until full audio received (s) + - RTF : real-time factor = generation_time / audio_duration (lower = faster) + - Speed : audio_duration / generation_time (higher = faster, e.g. 3× real-time) + +Usage: + python benchmark_api.py [--host localhost] [--port 8020] [--voice alloy] + [--runs 3] [--format wav] +""" +import argparse +import struct +import sys +import time + +import requests + +# ── Test sentences of increasing length ───────────────────────────────────── +SENTENCES = [ + ("short", "Hello, how are you today?"), + ("medium", "The quick brown fox jumps over the lazy dog near the river bank."), + ("long", "Artificial intelligence is transforming the way we interact with " + "technology. From voice assistants to autonomous vehicles, machine " + "learning models are becoming an integral part of everyday life."), +] + +SAMPLE_RATE = 24000 +BYTES_PER_SAMPLE = 2 # 16-bit PCM + + +def parse_wav_header(data: bytes) -> int: + """Return the data-chunk offset so we can skip the header bytes.""" + # WAV: RIFF(4) + size(4) + WAVE(4) + fmt (8+16) + data(4) + size(4) = 44 bytes + if data[:4] != b"RIFF" or data[8:12] != b"WAVE": + return 0 + # Walk chunks after WAVE marker + pos = 12 + while pos + 8 <= len(data): + chunk_id = data[pos:pos+4] + chunk_sz = struct.unpack_from(" dict: + t0 = time.perf_counter() + first_byte_t = None + raw = bytearray() + + with requests.post(url, json=payload, stream=True, timeout=120) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=512): + if chunk: + if first_byte_t is None: + first_byte_t = time.perf_counter() + raw.extend(chunk) + + t1 = time.perf_counter() + + ttfa_ms = (first_byte_t - t0) * 1000 if first_byte_t else float("nan") + total_s = t1 - t0 + + # Strip WAV header to count actual PCM bytes + if payload.get("response_format", "wav") == "wav": + data_offset = parse_wav_header(bytes(raw)) + pcm_bytes = len(raw) - data_offset + else: + pcm_bytes = len(raw) + + audio_s = pcm_bytes / (SAMPLE_RATE * BYTES_PER_SAMPLE) + rtf = total_s / audio_s if audio_s > 0 else float("nan") + speedup = audio_s / total_s if total_s > 0 else float("nan") + + return { + "label": label, + "run": run, + "ttfa_ms": ttfa_ms, + "total_s": total_s, + "audio_s": audio_s, + "rtf": rtf, + "speedup": speedup, + } + + +def main(): + p = argparse.ArgumentParser(description="Benchmark faster-qwen3-tts API") + p.add_argument("--host", default="localhost") + p.add_argument("--port", type=int, default=8020) + p.add_argument("--voice", default=None, + help="Voice name (default: first available voice)") + p.add_argument("--runs", type=int, default=3, + help="Repetitions per sentence (default: 3)") + p.add_argument("--format", default="wav", choices=["wav", "pcm"], + help="Response format (default: wav)") + args = p.parse_args() + + base = f"http://{args.host}:{args.port}" + url = f"{base}/v1/audio/speech" + + # Auto-detect first voice if not specified + voice = args.voice + if not voice: + try: + models = requests.get(f"{base}/v1/models", timeout=5).json() + voice = models["data"][0]["id"] + print(f"Auto-selected voice: {voice!r}") + except Exception: + voice = "alloy" + print(f"Could not auto-detect voice, using {voice!r}") + + print(f"\n{'─'*65}") + print(f" faster-qwen3-tts API Benchmark") + print(f" URL: {url} voice={voice} runs={args.runs} fmt={args.format}") + print(f"{'─'*65}\n") + + header = f"{'Label':<10} {'Run':>3} {'TTFA*':>9} {'Total':>7} {'Audio':>7} {'RTF':>6} {'Speed':>8}" + print(header) + print("─" * len(header)) + print(" * TTFA = time to first PCM byte (WAV header excluded)") + print() + + all_results = [] + + for label, text in SENTENCES: + payload = { + "model": "tts-1", + "input": text, + "voice": voice, + "response_format": args.format, + } + for run in range(1, args.runs + 1): + try: + r = benchmark_request(url, payload, label, run) + all_results.append(r) + print( + f"{r['label']:<10} {r['run']:>3} " + f"{r['ttfa_ms']:>7.0f}ms " + f"{r['total_s']:>6.2f}s " + f"{r['audio_s']:>6.2f}s " + f"{r['rtf']:>5.2f}x " + f"{r['speedup']:>6.1f}x rt" + ) + sys.stdout.flush() + except Exception as e: + print(f"{label:<10} run {run} ERROR: {e}") + + # ── Summary ────────────────────────────────────────────────────────────── + if all_results: + print(f"\n{'─'*65}") + print(" Summary (averages across all runs)") + print(f"{'─'*65}") + for label, _ in SENTENCES: + rows = [r for r in all_results if r["label"] == label] + if not rows: + continue + avg_ttfa = sum(r["ttfa_ms"] for r in rows) / len(rows) + avg_total = sum(r["total_s"] for r in rows) / len(rows) + avg_rtf = sum(r["rtf"] for r in rows) / len(rows) + avg_speed = sum(r["speedup"] for r in rows) / len(rows) + print( + f" {label:<10} TTFA={avg_ttfa:.0f}ms " + f"total={avg_total:.2f}s RTF={avg_rtf:.2f}x " + f"speed={avg_speed:.1f}× real-time" + ) + print() + + +if __name__ == "__main__": + main() diff --git a/config/generate_voices.py b/config/generate_voices.py index e15c0bc..ac7024c 100644 --- a/config/generate_voices.py +++ b/config/generate_voices.py @@ -40,7 +40,8 @@ if os.path.exists(speaker_dir): entry = { "ref_audio": f"/config/speakers/{filename}", - "language": lang + "language": lang, + "chunk_size": 4, } # Look for matching reference text files diff --git a/config/run_server.py b/config/run_server.py index 6f592a1..2460e96 100644 --- a/config/run_server.py +++ b/config/run_server.py @@ -8,16 +8,69 @@ Endpoints added: GET /v1/audio/models - Lists available voices (OpenWebUI fallback) GET /speakers - Lists speaker IDs (SillyTavern) OPTIONS /{path} - Pre-flight CORS handler + +Startup: + CUDA graphs are warmed up on server start so the first real request + does not pay the 7-8s graph-compilation penalty. """ +import asyncio +import logging import sys import json +from contextlib import asynccontextmanager + +from fastapi import FastAPI from fastapi.responses import JSONResponse # Point Python to the app directory inside the container sys.path.append("/app/examples") import openai_server +logger = logging.getLogger(__name__) + + +def _do_warmup(): + """Run one short generation to compile CUDA graphs before serving requests.""" + model = openai_server.tts_model + voices = openai_server.voices + default_voice = openai_server.default_voice + + if model is None or not voices or default_voice is None: + logger.warning("Warmup skipped: model or voices not ready") + return + + voice_cfg = voices.get(default_voice, {}) + ref_audio = voice_cfg.get("ref_audio") + if not ref_audio: + logger.warning("Warmup skipped: no ref_audio on default voice") + return + + logger.info("Warming up CUDA graphs (first request will be fast)...") + try: + for _ in model.generate_voice_clone_streaming( + text="Warmup.", + language=voice_cfg.get("language", "Auto"), + ref_audio=ref_audio, + ref_text=voice_cfg.get("ref_text", ""), + chunk_size=12, + ): + pass + logger.info("CUDA warmup complete — server ready.") + except Exception as exc: + logger.warning("Warmup failed (non-fatal): %s", exc) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, _do_warmup) + yield + + +# Attach lifespan to the existing FastAPI app +openai_server.app.router.lifespan_context = lifespan + # Load generated voices try: with open('/config/voices.json', 'r') as f: diff --git a/docker-compose.yml b/docker-compose.yml index 6b1603c..43ec30c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: --model /models/Qwen3-TTS --voices /config/voices.json --port 8000 + --max-seq-len 2048 " deploy: resources: