tts-dgx-spark-faster-qwen3-tts/config/run_server.py

"""
Wrapper around faster-qwen3-tts's openai_server.py that injects additional
API endpoints for compatibility with OpenWebUI and SillyTavern.

Endpoints added:
  GET /v1/models        - Lists available voices (OpenWebUI primary discovery)
  GET /v1/audio/voices  - Lists available voices (OpenWebUI fallback)
  GET /v1/audio/models  - Lists available voices (OpenWebUI fallback)
  GET /speakers         - Lists speaker IDs (SillyTavern)
  OPTIONS /{path}       - Pre-flight CORS handler

Startup:
  CUDA graphs are warmed up on server start so the first real request
  does not pay the 7-8s graph-compilation penalty.
"""

import asyncio
import logging
import sys
import os
import json
import threading
from contextlib import asynccontextmanager

from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse

# Point Python to the app directory inside the container
sys.path.append("/app/examples")
import openai_server

logger = logging.getLogger(__name__)


def _do_warmup():
    """Run one short generation to compile CUDA graphs before serving requests."""
    model = openai_server.tts_model
    voices = openai_server.voices
    default_voice = openai_server.default_voice

    if model is None or not voices or default_voice is None:
        logger.warning("Warmup skipped: model or voices not ready")
        return

    voice_cfg = voices.get(default_voice, {})
    ref_audio = voice_cfg.get("ref_audio")
    if not ref_audio:
        logger.warning("Warmup skipped: no ref_audio on default voice")
        return

    logger.info("Warming up CUDA graphs (first request will be fast)...")
    try:
        for _ in model.generate_voice_clone_streaming(
            text="Warmup.",
            language=voice_cfg.get("language", "Auto"),
            ref_audio=ref_audio,
            ref_text=voice_cfg.get("ref_text", ""),
            chunk_size=12,
            non_streaming_mode=True,
        ):
            pass
        logger.info("CUDA warmup complete — server ready.")
    except Exception as exc:
        logger.warning("Warmup failed (non-fatal): %s", exc)


def _precompute_all_embeddings():
    """Background task to precompute all missing embeddings to avoid lazy-load delay."""
    model = openai_server.tts_model
    voices = openai_server.voices
    if not model or not voices:
        return
        
    for voice_name, voice_cfg in list(voices.items()):
        # skip if already precomputed
        spk_emb_path = voice_cfg.get("speaker_embeddings") or voice_cfg.get("speaker embeddings")
        if spk_emb_path and os.path.isfile(spk_emb_path):
            continue
            
        logger.info("Background precomputing embedding for %r...", voice_name)
        # We must lock the model to prevent concurrent generation with incoming requests
        with openai_server._model_lock:
            try:
                # _load_voice_clone_prompt updates voice_cfg in place and saves the .pt
                openai_server._load_voice_clone_prompt(voice_cfg, voice_name, model)
            except Exception as e:
                logger.error("Failed to background precompute for %r: %s", voice_name, e)


@asynccontextmanager
async def lifespan(app: FastAPI):
    loop = asyncio.get_event_loop()
    await loop.run_in_executor(None, _do_warmup)
    loop.run_in_executor(None, _precompute_all_embeddings)
    yield


# Attach lifespan to the existing FastAPI app
openai_server.app.router.lifespan_context = lifespan

# Load generated voices
try:
    with open('/config/voices.json', 'r') as f:
        voices_data = json.load(f)
except FileNotFoundError:
    voices_data = {}

# Build reusable response payloads
_voice_list = [{'id': v, 'object': 'model', 'created': 1686935002, 'owned_by': 'qwen'} for v in voices_data.keys()]
_models_response = {'object': 'list', 'data': _voice_list}

# OpenWebUI model discovery (primary)
@openai_server.app.get('/v1/models')
async def list_models():
    return _models_response

# OpenWebUI voice discovery fallbacks
@openai_server.app.get('/v1/audio/voices')
async def list_audio_voices():
    return _models_response

@openai_server.app.get('/v1/audio/models')
async def list_audio_models():
    return _models_response

# SillyTavern speaker endpoint
@openai_server.app.get('/speakers')
async def get_speakers():
    return list(voices_data.keys())

# Pre-flight OPTIONS handler to prevent 404s
@openai_server.app.options('/{path:path}')
async def options_handler(path: str):
    return JSONResponse(content={'status': 'ok'})

_voices_lock = threading.Lock()


@openai_server.app.post('/voice-seed')
async def set_voice_seed(request: Request):
    """Set (or clear) the seed for a voice in voices.json.

    Body: {"voice": "EN_F_NatashaNeural", "seed": 7}
    To remove a seed: {"voice": "EN_F_NatashaNeural", "seed": null}
    """
    data = await request.json()
    voice_name = data.get("voice")
    seed = data.get("seed")

    if not voice_name:
        raise HTTPException(status_code=400, detail="'voice' field is required")

    voices_path = '/config/voices.json'
    with _voices_lock:
        try:
            with open(voices_path, 'r', encoding='utf-8') as f:
                voices = json.load(f)
        except FileNotFoundError:
            raise HTTPException(status_code=404, detail="voices.json not found")

        if voice_name not in voices:
            raise HTTPException(status_code=404, detail=f"Voice {voice_name!r} not found")

        if seed is None:
            voices[voice_name].pop("seed", None)
        else:
            try:
                voices[voice_name]["seed"] = int(seed)
            except (TypeError, ValueError):
                raise HTTPException(status_code=400, detail="'seed' must be an integer or null")

        with open(voices_path, 'w', encoding='utf-8') as f:
            json.dump(voices, f, indent=2, ensure_ascii=False)

    return JSONResponse({"ok": True, "voice": voice_name, "seed": seed})


if __name__ == '__main__':
    openai_server.main()