Reduce latency: CUDA warmup, chunk_size=4, max-seq-len 2048
- run_server.py: warm up CUDA graphs at server startup via lifespan event so the first real request does not pay the 7-8s graph- compilation penalty; uses modern lifespan API instead of deprecated on_event - generate_voices.py: set chunk_size=4 per voice so streaming clients receive first audio after ~333ms instead of ~1s - docker-compose.yml: add --max-seq-len 2048 (halves static KV cache, reduces VRAM and graph-capture time) - config/benchmark_api.py: new script to measure TTFA, RTF and speed against the live API endpoint Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3f723b4032
commit
92e2381d0b
175
config/benchmark_api.py
Normal file
175
config/benchmark_api.py
Normal file
@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark the faster-qwen3-tts API server.
|
||||
|
||||
Measures:
|
||||
- TTFA : time-to-first-audio (ms) — latency before playback can start
|
||||
- Total : wall-clock time until full audio received (s)
|
||||
- RTF : real-time factor = generation_time / audio_duration (lower = faster)
|
||||
- Speed : audio_duration / generation_time (higher = faster, e.g. 3× real-time)
|
||||
|
||||
Usage:
|
||||
python benchmark_api.py [--host localhost] [--port 8020] [--voice alloy]
|
||||
[--runs 3] [--format wav]
|
||||
"""
|
||||
import argparse
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
# ── Test sentences of increasing length ─────────────────────────────────────
|
||||
SENTENCES = [
|
||||
("short", "Hello, how are you today?"),
|
||||
("medium", "The quick brown fox jumps over the lazy dog near the river bank."),
|
||||
("long", "Artificial intelligence is transforming the way we interact with "
|
||||
"technology. From voice assistants to autonomous vehicles, machine "
|
||||
"learning models are becoming an integral part of everyday life."),
|
||||
]
|
||||
|
||||
SAMPLE_RATE = 24000
|
||||
BYTES_PER_SAMPLE = 2 # 16-bit PCM
|
||||
|
||||
|
||||
def parse_wav_header(data: bytes) -> int:
|
||||
"""Return the data-chunk offset so we can skip the header bytes."""
|
||||
# WAV: RIFF(4) + size(4) + WAVE(4) + fmt (8+16) + data(4) + size(4) = 44 bytes
|
||||
if data[:4] != b"RIFF" or data[8:12] != b"WAVE":
|
||||
return 0
|
||||
# Walk chunks after WAVE marker
|
||||
pos = 12
|
||||
while pos + 8 <= len(data):
|
||||
chunk_id = data[pos:pos+4]
|
||||
chunk_sz = struct.unpack_from("<I", data, pos+4)[0]
|
||||
if chunk_id == b"data":
|
||||
return pos + 8
|
||||
pos += 8 + chunk_sz
|
||||
return 44 # fallback
|
||||
|
||||
|
||||
def benchmark_request(url: str, payload: dict, label: str, run: int) -> dict:
|
||||
t0 = time.perf_counter()
|
||||
first_byte_t = None
|
||||
raw = bytearray()
|
||||
|
||||
with requests.post(url, json=payload, stream=True, timeout=120) as resp:
|
||||
resp.raise_for_status()
|
||||
for chunk in resp.iter_content(chunk_size=512):
|
||||
if chunk:
|
||||
if first_byte_t is None:
|
||||
first_byte_t = time.perf_counter()
|
||||
raw.extend(chunk)
|
||||
|
||||
t1 = time.perf_counter()
|
||||
|
||||
ttfa_ms = (first_byte_t - t0) * 1000 if first_byte_t else float("nan")
|
||||
total_s = t1 - t0
|
||||
|
||||
# Strip WAV header to count actual PCM bytes
|
||||
if payload.get("response_format", "wav") == "wav":
|
||||
data_offset = parse_wav_header(bytes(raw))
|
||||
pcm_bytes = len(raw) - data_offset
|
||||
else:
|
||||
pcm_bytes = len(raw)
|
||||
|
||||
audio_s = pcm_bytes / (SAMPLE_RATE * BYTES_PER_SAMPLE)
|
||||
rtf = total_s / audio_s if audio_s > 0 else float("nan")
|
||||
speedup = audio_s / total_s if total_s > 0 else float("nan")
|
||||
|
||||
return {
|
||||
"label": label,
|
||||
"run": run,
|
||||
"ttfa_ms": ttfa_ms,
|
||||
"total_s": total_s,
|
||||
"audio_s": audio_s,
|
||||
"rtf": rtf,
|
||||
"speedup": speedup,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Benchmark faster-qwen3-tts API")
|
||||
p.add_argument("--host", default="localhost")
|
||||
p.add_argument("--port", type=int, default=8020)
|
||||
p.add_argument("--voice", default=None,
|
||||
help="Voice name (default: first available voice)")
|
||||
p.add_argument("--runs", type=int, default=3,
|
||||
help="Repetitions per sentence (default: 3)")
|
||||
p.add_argument("--format", default="wav", choices=["wav", "pcm"],
|
||||
help="Response format (default: wav)")
|
||||
args = p.parse_args()
|
||||
|
||||
base = f"http://{args.host}:{args.port}"
|
||||
url = f"{base}/v1/audio/speech"
|
||||
|
||||
# Auto-detect first voice if not specified
|
||||
voice = args.voice
|
||||
if not voice:
|
||||
try:
|
||||
models = requests.get(f"{base}/v1/models", timeout=5).json()
|
||||
voice = models["data"][0]["id"]
|
||||
print(f"Auto-selected voice: {voice!r}")
|
||||
except Exception:
|
||||
voice = "alloy"
|
||||
print(f"Could not auto-detect voice, using {voice!r}")
|
||||
|
||||
print(f"\n{'─'*65}")
|
||||
print(f" faster-qwen3-tts API Benchmark")
|
||||
print(f" URL: {url} voice={voice} runs={args.runs} fmt={args.format}")
|
||||
print(f"{'─'*65}\n")
|
||||
|
||||
header = f"{'Label':<10} {'Run':>3} {'TTFA*':>9} {'Total':>7} {'Audio':>7} {'RTF':>6} {'Speed':>8}"
|
||||
print(header)
|
||||
print("─" * len(header))
|
||||
print(" * TTFA = time to first PCM byte (WAV header excluded)")
|
||||
print()
|
||||
|
||||
all_results = []
|
||||
|
||||
for label, text in SENTENCES:
|
||||
payload = {
|
||||
"model": "tts-1",
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"response_format": args.format,
|
||||
}
|
||||
for run in range(1, args.runs + 1):
|
||||
try:
|
||||
r = benchmark_request(url, payload, label, run)
|
||||
all_results.append(r)
|
||||
print(
|
||||
f"{r['label']:<10} {r['run']:>3} "
|
||||
f"{r['ttfa_ms']:>7.0f}ms "
|
||||
f"{r['total_s']:>6.2f}s "
|
||||
f"{r['audio_s']:>6.2f}s "
|
||||
f"{r['rtf']:>5.2f}x "
|
||||
f"{r['speedup']:>6.1f}x rt"
|
||||
)
|
||||
sys.stdout.flush()
|
||||
except Exception as e:
|
||||
print(f"{label:<10} run {run} ERROR: {e}")
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────────────
|
||||
if all_results:
|
||||
print(f"\n{'─'*65}")
|
||||
print(" Summary (averages across all runs)")
|
||||
print(f"{'─'*65}")
|
||||
for label, _ in SENTENCES:
|
||||
rows = [r for r in all_results if r["label"] == label]
|
||||
if not rows:
|
||||
continue
|
||||
avg_ttfa = sum(r["ttfa_ms"] for r in rows) / len(rows)
|
||||
avg_total = sum(r["total_s"] for r in rows) / len(rows)
|
||||
avg_rtf = sum(r["rtf"] for r in rows) / len(rows)
|
||||
avg_speed = sum(r["speedup"] for r in rows) / len(rows)
|
||||
print(
|
||||
f" {label:<10} TTFA={avg_ttfa:.0f}ms "
|
||||
f"total={avg_total:.2f}s RTF={avg_rtf:.2f}x "
|
||||
f"speed={avg_speed:.1f}× real-time"
|
||||
)
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -40,7 +40,8 @@ if os.path.exists(speaker_dir):
|
||||
|
||||
entry = {
|
||||
"ref_audio": f"/config/speakers/{filename}",
|
||||
"language": lang
|
||||
"language": lang,
|
||||
"chunk_size": 4,
|
||||
}
|
||||
|
||||
# Look for matching reference text files
|
||||
|
||||
@ -8,16 +8,69 @@ Endpoints added:
|
||||
GET /v1/audio/models - Lists available voices (OpenWebUI fallback)
|
||||
GET /speakers - Lists speaker IDs (SillyTavern)
|
||||
OPTIONS /{path} - Pre-flight CORS handler
|
||||
|
||||
Startup:
|
||||
CUDA graphs are warmed up on server start so the first real request
|
||||
does not pay the 7-8s graph-compilation penalty.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
# Point Python to the app directory inside the container
|
||||
sys.path.append("/app/examples")
|
||||
import openai_server
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_warmup():
|
||||
"""Run one short generation to compile CUDA graphs before serving requests."""
|
||||
model = openai_server.tts_model
|
||||
voices = openai_server.voices
|
||||
default_voice = openai_server.default_voice
|
||||
|
||||
if model is None or not voices or default_voice is None:
|
||||
logger.warning("Warmup skipped: model or voices not ready")
|
||||
return
|
||||
|
||||
voice_cfg = voices.get(default_voice, {})
|
||||
ref_audio = voice_cfg.get("ref_audio")
|
||||
if not ref_audio:
|
||||
logger.warning("Warmup skipped: no ref_audio on default voice")
|
||||
return
|
||||
|
||||
logger.info("Warming up CUDA graphs (first request will be fast)...")
|
||||
try:
|
||||
for _ in model.generate_voice_clone_streaming(
|
||||
text="Warmup.",
|
||||
language=voice_cfg.get("language", "Auto"),
|
||||
ref_audio=ref_audio,
|
||||
ref_text=voice_cfg.get("ref_text", ""),
|
||||
chunk_size=12,
|
||||
):
|
||||
pass
|
||||
logger.info("CUDA warmup complete — server ready.")
|
||||
except Exception as exc:
|
||||
logger.warning("Warmup failed (non-fatal): %s", exc)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, _do_warmup)
|
||||
yield
|
||||
|
||||
|
||||
# Attach lifespan to the existing FastAPI app
|
||||
openai_server.app.router.lifespan_context = lifespan
|
||||
|
||||
# Load generated voices
|
||||
try:
|
||||
with open('/config/voices.json', 'r') as f:
|
||||
|
||||
@ -24,6 +24,7 @@ services:
|
||||
--model /models/Qwen3-TTS
|
||||
--voices /config/voices.json
|
||||
--port 8000
|
||||
--max-seq-len 2048
|
||||
"
|
||||
deploy:
|
||||
resources:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user