mARTin-B78
1a2ea421c4
Some checks are pending
Test and SonarQube / quality (push) Waiting to run
Connect and "Use as LLM" on the LiteLLM Gateway card now resolve that card's own saved key (engine_api_keys[name]) server-side in routes/docker.py, ahead of the old global llm_api_key fallback. The browser can no longer reliably forward a saved key at all -- it's masked to "__configured__" in every /api/settings response the moment it's saved -- so relying on a frontend-forwarded header alone left Connect silently using a stale global key after LiteLLM's own key was rotated, surfacing a misleading /api/version 404 instead of the real 401. "Use as LLM" now goes through a dedicated /use-as-llm endpoint that saves the exact credential Connect just verified (llm_url normalized to .../v1, llm_api_key copied from the card), rather than a separately-typed value that could drift again. Also documents per-user provider-key storage as an explicit item for the deferred multi-user/identity planning pass. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
262 lines
19 KiB
Python
262 lines
19 KiB
Python
"""Local Docker container management and URL probe endpoints."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
import requests
|
|
from fastapi import APIRouter, Header, HTTPException, Request
|
|
|
|
from core.docker_client import _docker_available, _docker_get_json, _docker_post
|
|
from core.config import _load_settings, _save_settings, _normalize_settings
|
|
from core.validation import _normalize_service_url
|
|
from core.provider_health import probe_provider
|
|
|
|
router = APIRouter()
|
|
|
|
_LOCAL_CONTAINER_DEFS: list[dict] = [
|
|
{"name": "litellm", "label": "LiteLLM Gateway", "role": "llm", "port": 14000, "stack": "dgx-spark_lite-llm_llama-swap_vllm_llama-cpp_ollama", "image": "ghcr.io/berriai/litellm:main-latest", "repo": "https://github.com/BerriAI/litellm", "description": "OpenAI-compatible LLM gateway. Use for character analysis, casting, and text refinement."},
|
|
{"name": "llama.cpp", "label": "llama.cpp Server", "role": "llm", "port": 19000, "stack": "dgx-spark_lite-llm_llama-swap_vllm_llama-cpp_ollama", "image": "ghcr.io/ggml-org/llama.cpp:server", "repo": "https://github.com/ggml-org/llama.cpp", "description": "OpenAI-compatible local LLM server when launched with its server API."},
|
|
{"name": "ollama", "label": "Ollama", "role": "llm", "port": 11434, "stack": "dgx-spark_lite-llm_llama-swap_vllm_llama-cpp_ollama", "image": "ollama/ollama", "repo": "https://ollama.com", "description": "Local LLM runtime. The app uses its OpenAI-compatible /v1 endpoint."},
|
|
{"name": "llama-swap", "label": "llama-swap", "role": "llm", "port": 28080, "stack": "dgx-spark_lite-llm_llama-swap_vllm_llama-cpp_ollama", "image": "ghcr.io/mostlygeek/llama-swap:cuda", "repo": "https://github.com/mostlygeek/llama-swap", "description": "Model swapping proxy for local LLMs. Useful as the default Rehearser LLM endpoint."},
|
|
{"name": "llama-qwen35-4b", "label": "Qwen3.5 4B LLM", "role": "llm", "port": 19001, "stack": "dgx-spark_lite-llm_llama-swap_vllm_llama-cpp_ollama", "image": "local", "repo": "", "description": "Dedicated Qwen LLM container. Use if it exposes an OpenAI-compatible /v1 API."},
|
|
{"name": "vllm", "label": "vLLM", "role": "llm", "port": 8000, "stack": "dgx-spark_lite-llm_llama-swap_vllm_llama-cpp_ollama", "image": "vllm/vllm-openai:latest", "repo": "https://github.com/vllm-project/vllm", "description": "High-throughput GPU inference. Serve any HuggingFace model with PagedAttention."},
|
|
{"name": "lmstudio", "label": "LM Studio", "role": "llm", "port": 1234, "stack": "", "image": "local", "repo": "https://lmstudio.ai", "description": "GUI app for running quantized models. Built-in model browser, chat UI, and a local server with OpenAI API."},
|
|
{"name": "tts-faster-qwen3-tts-voiceclone", "label": "Qwen3 TTS · Voice Clone", "role": "tts", "port": 8020, "stack": "faster-qwen3-tts-dgx-spark", "image": "martinb78/faster-qwen3-tts-dgx-spark:v4", "repo": "https://github.com/mARTin-B78/dgx-spark-faster-qwen3-tts", "description": "WAV voice cloning. Scans active_voices at startup; restart after adding or editing voices."},
|
|
{"name": "tts-faster-qwen3-tts-voicedesign", "label": "Qwen3 TTS · Voice Design", "role": "tts", "port": 8021, "stack": "faster-qwen3-tts-dgx-spark", "image": "martinb78/faster-qwen3-tts-dgx-spark:v4", "repo": "https://github.com/mARTin-B78/dgx-spark-faster-qwen3-tts", "description": "Instruction-based voice design. Best Qwen path for tone, style, and character prompts."},
|
|
{"name": "tts-faster-qwen3-tts-customvoice", "label": "Qwen3 TTS · Custom Voice", "role": "tts", "port": 8022, "stack": "faster-qwen3-tts-dgx-spark", "image": "martinb78/faster-qwen3-tts-dgx-spark:v4", "repo": "https://github.com/mARTin-B78/dgx-spark-faster-qwen3-tts", "description": "CustomVoice speakers with per-request instruct overrides for tone/style."},
|
|
{"name": "tts-faster-qwen3-tts-streaming", "label": "Qwen3 TTS · Streaming", "role": "tts", "port": 8023, "stack": "faster-qwen3-tts-dgx-spark", "image": "martinb78/faster-qwen3-tts-dgx-spark:streaming", "repo": "https://github.com/mARTin-B78/dgx-spark-faster-qwen3-tts", "description": "Low-latency progressive WAV streaming for voice clone voices."},
|
|
{"name": "faster-whisper", "label": "faster-whisper", "role": "stt", "port": 8010, "stack": "faster-whisper-dgx-spark", "image": "local", "repo": "https://github.com/SYSTRAN/faster-whisper", "description": "Fast local Whisper transcription endpoint."},
|
|
{"name": "whisper-cpp", "label": "whisper.cpp", "role": "stt", "port": 8085, "stack": "", "image": "local", "repo": "https://github.com/ggml-org/whisper.cpp", "description": "Minimal C++ Whisper with a built-in HTTP server. CPU or Metal/CUDA. Low memory, fast on consumer hardware."},
|
|
{"name": "fish-speech-api", "label": "Fish-Speech API", "role": "tts", "port": 38080, "stack": "fish-speech-dgx-spark", "image": "local", "repo": "https://github.com/fishaudio/fish-speech", "description": "Tone-aware cloning backend. Recommended for Rehearser character voices when you need both identity and emotion."},
|
|
{"name": "nemotron-asr-streaming", "label": "Nemotron ASR Streaming", "role": "stt", "port": 8006, "stack": "nemotron-asr-streaming", "image": "local", "repo": "https://github.com/pipecat-ai/nemotron-january-2026", "description": "Streaming ASR service. Use as a custom STT endpoint if it exposes OpenAI transcription or /transcribe."},
|
|
{"name": "parakeet-stt-multi-9011", "label": "NVIDIA Parakeet STT", "role": "stt", "port": 8007, "stack": "nvidia-parakeet-voice-recognition", "image": "local", "repo": "https://github.com/mARTin-B78/dgx-spark-parakeet-asr", "description": "Parakeet STT container on port 8007."},
|
|
{"name": "nvidia-magpie-tts", "label": "NVIDIA Magpie TTS", "role": "tts", "port": 8091, "stack": "nvidia-speech-gateway", "image": "nvcr.io/nim/nvidia/magpie-tts-multilingual:latest", "repo": "https://github.com/mARTin-B78/dgx-spark-parakeet-asr", "description": "NVIDIA neural TTS. Fixed speaker voices, fast multilingual synthesis."},
|
|
{"name": "nvidia-parakeet-asr", "label": "NVIDIA Parakeet Router ASR", "role": "stt", "port": 8090, "stack": "nvidia-speech-gateway", "image": "parakeet-tdt-v3-spark:latest", "repo": "https://github.com/mARTin-B78/dgx-spark-parakeet-asr", "description": "NVIDIA Parakeet speech router endpoint."},
|
|
{"name": "nvidia-parakeet-rnnt-nim", "label": "NVIDIA Parakeet RNNT NIM", "role": "stt", "port": 8092, "stack": "nvidia-speech-gateway", "image": "nvcr.io/nim/nvidia/parakeet-1b-rnnt-multilingual:latest", "repo": "https://github.com/mARTin-B78/dgx-spark-parakeet-asr", "description": "Parakeet RNNT NIM multilingual ASR on port 8092."},
|
|
{"name": "kokoro-tts", "label": "Kokoro FastAPI", "role": "tts", "port": 8880, "stack": "kokoro-tts", "image": "local", "repo": "", "description": "OpenAI-compatible TTS. Built-in voices: af_bella, bf_emma, am_adam... ~300 MB RAM."},
|
|
{"name": "vibevoice-msft-tts", "label": "VibeVoice TTS", "role": "tts", "port": 8027, "stack": "vibevoice", "image": "local", "repo": "", "description": "VibeVoice REST service. Simple text-to-speech backend."},
|
|
{"name": "nvidia-zeroshot-nim", "label": "NVIDIA Zeroshot NIM", "role": "tts", "port": 8093, "stack": "nvidia-speech-gateway", "image": "local", "repo": "", "description": "Magpie Zeroshot clone endpoint with audio_prompt."},
|
|
{"name": "nvidia-flow-nim", "label": "NVIDIA Flow NIM", "role": "tts", "port": 8094, "stack": "nvidia-speech-gateway", "image": "local", "repo": "", "description": "Magpie Flow clone endpoint with audio_prompt and transcript."},
|
|
{"name": "voicebox", "label": "Voicebox", "role": "llm", "port": 17493, "stack": "voicebox", "image": "local", "repo": "", "description": "Companion voice app/control plane. Not used as a normal TTS engine unless wrapped as an API."},
|
|
{"name": "stt-whisperx-blackwell", "label": "WhisperX · faster-whisper (Blackwell GPU)", "role": "stt", "port": 8003, "stack": "whisperx-blackwell", "image": "local", "repo": "https://github.com/m-bain/whisperX", "description": "WhisperX transcription/alignment service using faster-whisper on the Blackwell GPU. Use as custom STT if its API is OpenAI-compatible or /transcribe."},
|
|
{"name": "xtts-dgx-spark", "label": "XTTS v2", "role": "tts", "port": 8005, "stack": "xtts_v2", "image": "local", "repo": "https://github.com/daswer123/xtts-api-server", "description": "XTTS v2 OpenAI-compatible speaker endpoint."},
|
|
{"name": "ocr-kdl-frontier-parser-nano", "label": "KDL Frontier Parser nano", "role": "ocr", "port": 8099, "stack": "", "image": "local", "repo": "https://huggingface.co/KDLAI/KDL-Frontier-Parser-nano", "description": "Local 1.2B document parser for layout detection, OCR, tables, figures, and reading order."},
|
|
{"name": "ocr-baidu-unlimited-ocr", "label": "Baidu Unlimited-OCR", "role": "ocr", "port": 8098, "stack": "", "image": "local", "repo": "https://github.com/baidu/Unlimited-OCR", "description": "Local long-document OCR and layout parser with single-image and multi-page/PDF workflows."},
|
|
]
|
|
|
|
|
|
def _container_status(name: str) -> dict:
|
|
try:
|
|
code, data = _docker_get_json(f"/containers/{quote(name, safe='')}/json")
|
|
if code == 404:
|
|
return {"status": "not_found"}
|
|
if code == 200 and isinstance(data, dict):
|
|
state = data.get("State", {})
|
|
return {
|
|
"status": state.get("Status", "unknown"),
|
|
"running": state.get("Running", False),
|
|
"paused": state.get("Paused", False),
|
|
"started_at": state.get("StartedAt", ""),
|
|
"image": data.get("Config", {}).get("Image", ""),
|
|
}
|
|
return {"status": "error", "detail": f"HTTP {code}"}
|
|
except Exception as e:
|
|
return {"status": "error", "detail": str(e)}
|
|
|
|
|
|
@router.get("/api/local-containers")
|
|
async def get_local_containers():
|
|
sock_ok = _docker_available()
|
|
|
|
async def _check(defn: dict) -> dict:
|
|
entry = {k: v for k, v in defn.items()}
|
|
if sock_ok:
|
|
status = await asyncio.to_thread(_container_status, defn["name"])
|
|
entry.update(status)
|
|
else:
|
|
entry["status"] = "no_socket"
|
|
return entry
|
|
|
|
results = await asyncio.gather(*[_check(d) for d in _LOCAL_CONTAINER_DEFS])
|
|
return {"containers": list(results), "socket_available": sock_ok}
|
|
|
|
|
|
@router.post("/api/local-containers/{name}/start")
|
|
async def start_local_container(name: str):
|
|
if not any(c["name"] == name for c in _LOCAL_CONTAINER_DEFS):
|
|
raise HTTPException(404, f"Unknown container: {name}")
|
|
try:
|
|
code, _ = await asyncio.to_thread(_docker_post, f"/containers/{quote(name, safe='')}/start")
|
|
except Exception as e:
|
|
raise HTTPException(502, f"Docker start failed: {e}")
|
|
if code not in (204, 304):
|
|
raise HTTPException(502, f"Docker API returned HTTP {code}")
|
|
return {"ok": True, "name": name, **_container_status(name)}
|
|
|
|
|
|
@router.post("/api/local-containers/{name}/stop")
|
|
async def stop_local_container(name: str):
|
|
if not any(c["name"] == name for c in _LOCAL_CONTAINER_DEFS):
|
|
raise HTTPException(404, f"Unknown container: {name}")
|
|
try:
|
|
code, _ = await asyncio.to_thread(_docker_post, f"/containers/{quote(name, safe='')}/stop?t=10")
|
|
except Exception as e:
|
|
raise HTTPException(502, f"Docker stop failed: {e}")
|
|
if code not in (204, 304):
|
|
raise HTTPException(502, f"Docker API returned HTTP {code}")
|
|
return {"ok": True, "name": name, **_container_status(name)}
|
|
|
|
|
|
@router.post("/api/local-containers/{name}/restart")
|
|
async def restart_local_container(name: str):
|
|
if not any(c["name"] == name for c in _LOCAL_CONTAINER_DEFS):
|
|
raise HTTPException(404, f"Unknown container: {name}")
|
|
try:
|
|
code, _ = await asyncio.to_thread(_docker_post, f"/containers/{quote(name, safe='')}/restart?t=10")
|
|
except Exception as e:
|
|
raise HTTPException(502, f"Docker restart failed: {e}")
|
|
if code not in (204, 304):
|
|
raise HTTPException(502, f"Docker API returned HTTP {code}")
|
|
return {"ok": True, "name": name, **_container_status(name)}
|
|
|
|
|
|
@router.get("/api/probe-url")
|
|
async def probe_url(url: str, type: str = "", name: str = "", provider_api_key: str = Header("", alias="X-Provider-API-Key")):
|
|
"""Server-side API probe — checks service-specific endpoints and validates JSON responses."""
|
|
base = _normalize_service_url(url).rstrip("/")
|
|
for _suffix in ("/v1", "/api/v1"):
|
|
if base.endswith(_suffix):
|
|
base = base[: -len(_suffix)]
|
|
break
|
|
settings = _load_settings()
|
|
# Key precedence:
|
|
# 1. A key supplied explicitly by the browser for this probe (e.g. the
|
|
# user just typed/edited it and it isn't saved yet).
|
|
# 2. The key saved for this specific engine card (settings.engine_api_keys[name]).
|
|
# Card values are masked to "__configured__" before ever reaching the
|
|
# browser, so on a normal page load the browser has no real secret to
|
|
# forward — the backend must resolve it itself from `name`.
|
|
# 3. The old global llm_api_key, which may belong to a different
|
|
# gateway after that card's own key was rotated.
|
|
card_key = (settings.get("engine_api_keys") or {}).get(name) if name else None
|
|
key_to_use = provider_api_key or card_key or {
|
|
"anthropic": settings.get("llm_api_key"),
|
|
"llm": settings.get("llm_api_key"),
|
|
"openai": settings.get("llm_api_key"),
|
|
"stt": settings.get("groq_api_key") or settings.get("whisper_api_key"),
|
|
"tts": settings.get("tts_api_key"),
|
|
}.get(type) or "sk-dummy-key"
|
|
# Anthropic doesn't speak the OpenAI-style Bearer auth every other card
|
|
# here uses — it needs x-api-key + an anthropic-version header, so the
|
|
# generic Bearer probe below would 401 even with a perfectly valid key.
|
|
if type == "anthropic":
|
|
hdrs = {"User-Agent": "TTS-Voice-Creator/probe", "x-api-key": key_to_use, "anthropic-version": "2023-06-01"}
|
|
else:
|
|
hdrs = {"User-Agent": "TTS-Voice-Creator/probe", "Authorization": f"Bearer {key_to_use}"}
|
|
|
|
if type == "anthropic":
|
|
checks = [("/v1/models", "data")]
|
|
elif type == "llm":
|
|
checks = [("/v1/models", "data"), ("/api/tags", "models"), ("/api/version", None)]
|
|
elif type == "stt":
|
|
checks = [("/health", None), ("/v1/models", "data"), ("/v1/audio/transcriptions", None)]
|
|
elif type == "tts":
|
|
checks = [("/health", None), ("/v1/health", None), ("/v1/audio/voices", None), ("/speakers", None), ("/voices", None)]
|
|
elif type == "ocr":
|
|
checks = [("/health", None), ("/v1/models", "data"), ("/v1/chat/completions", None), ("/parse", None), ("/ocr", None)]
|
|
elif type == "comfyui":
|
|
checks = [("/system_stats", "system")]
|
|
else:
|
|
checks = [("", None)]
|
|
|
|
result = probe_provider(base, checks, headers=hdrs, timeout=5)
|
|
return {key: value for key, value in result.items() if key != 'payload'}
|
|
|
|
|
|
@router.post("/api/local-containers/{name}/use-as-llm")
|
|
async def use_as_llm(name: str, request: Request, provider_api_key: str = Header("", alias="X-Provider-API-Key")):
|
|
"""Make an engine card the active LLM, saving the same credential Connect
|
|
just verified — not a possibly-stale global llm_api_key.
|
|
|
|
The URL always comes from the request body. The key comes from (in
|
|
order): an explicit header (a freshly typed, not-yet-saved key), then
|
|
this card's own saved engine_api_keys[name] — the browser cannot supply
|
|
that value itself once saved, since it is masked before ever reaching
|
|
the browser.
|
|
"""
|
|
body = await request.json()
|
|
url = _normalize_service_url(str(body.get("url") or "")).strip()
|
|
if not url:
|
|
raise HTTPException(400, "URL is required")
|
|
|
|
s = _load_settings()
|
|
card_key = (s.get("engine_api_keys") or {}).get(name)
|
|
key_to_use = provider_api_key or card_key or ""
|
|
|
|
s["llm_url"] = url
|
|
if key_to_use:
|
|
s["llm_api_key"] = key_to_use
|
|
if provider_api_key:
|
|
# Keep this card's own saved key in sync with what was just typed.
|
|
engine_keys = dict(s.get("engine_api_keys") or {})
|
|
engine_keys[name] = provider_api_key
|
|
s["engine_api_keys"] = engine_keys
|
|
s = _normalize_settings(s)
|
|
_save_settings(s)
|
|
return {"ok": True, "llm_url": url, "key_set": bool(key_to_use)}
|
|
|
|
|
|
@router.get("/api/engine-models")
|
|
async def engine_models(url: str, type: str = "", provider_api_key: str = Header("", alias="X-Provider-API-Key")):
|
|
"""Return model IDs from an OpenAI-style engine URL without exposing host-only URLs to the browser."""
|
|
base = _normalize_service_url(url).rstrip("/")
|
|
if not base:
|
|
raise HTTPException(400, "URL is required")
|
|
candidates: list[str]
|
|
if base.endswith("/v1"):
|
|
candidates = [base + "/models", base[:-3].rstrip("/") + "/v1/models"]
|
|
else:
|
|
candidates = [base.rstrip("/") + "/v1/models", base.rstrip("/") + "/models"]
|
|
if type == "ollama":
|
|
candidates.append(base.rstrip("/") + "/api/tags")
|
|
last_err = "No response"
|
|
seen: set[str] = set()
|
|
models: list[str] = []
|
|
settings = _load_settings()
|
|
key_to_use = settings.get("llm_api_key") or provider_api_key or "sk-dummy-key"
|
|
for endpoint in candidates:
|
|
try:
|
|
r = requests.get(endpoint, timeout=6, headers={"Authorization": f"Bearer {key_to_use}", "User-Agent": "TTS-Voice-Creator/models"})
|
|
if r.status_code >= 400:
|
|
last_err = f"HTTP {r.status_code} from {endpoint}"
|
|
continue
|
|
payload = r.json()
|
|
data = payload.get("data") if isinstance(payload, dict) else None
|
|
if data is None and isinstance(payload, dict):
|
|
data = payload.get("models")
|
|
if data is None and isinstance(payload, list):
|
|
data = payload
|
|
if not isinstance(data, list):
|
|
last_err = f"Unexpected model response from {endpoint}"
|
|
continue
|
|
for item in data:
|
|
mid = ""
|
|
if isinstance(item, str):
|
|
mid = item
|
|
elif isinstance(item, dict):
|
|
mid = str(item.get("id") or item.get("name") or item.get("model") or "")
|
|
mid = mid.strip()
|
|
if mid and mid not in seen:
|
|
seen.add(mid)
|
|
models.append(mid)
|
|
if models:
|
|
return {"ok": True, "url": base, "endpoint": endpoint, "models": models}
|
|
except requests.exceptions.Timeout:
|
|
last_err = "Timeout fetching models"
|
|
except Exception as e:
|
|
last_err = str(e)
|
|
return {"ok": False, "url": base, "models": models, "error": last_err}
|