diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f461ed..42befef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi ## [Unreleased] ### Changed +- **Shared provider-health probes.** TTS, STT, and container connectivity checks now use one bounded helper with + normalized connection, timeout, HTTP-status, and JSON-shape handling while retaining provider-specific endpoints. - **Section-scoped feature loading.** The shared module manifest now builds a 17-module foundation bundle and loads Rehearser, Reader/Audiobook, Library/Characters, and Studio code only when their sections are opened. Repeated navigation reuses loaded modules, while deep links load their group before activation. diff --git a/TODO.md b/TODO.md index 7d7415a..b322042 100644 --- a/TODO.md +++ b/TODO.md @@ -157,7 +157,9 @@ browser image/profile-cache eviction. bounded retries from stripped in-memory inputs; retry data is never returned or persisted, and restart correctly requires reopening the source workflow. - [x] Prevent duplicate generation when a user clicks twice for image, voice-design, and casting submissions. -- [ ] Centralize provider health checks and error handling. +- [x] Centralize provider health checks and error handling. TTS, STT, and container URL probes now share the bounded + `core.provider_health.probe_provider` mechanics and normalized connection/timeout/HTTP/JSON-shape errors; their + provider-specific endpoint lists and authentication policies remain at the route layer. - [x] Limit concurrency to 1–2 image generations, 1–2 voice generations, and 1 STT request per user. Consistent portrait/reference rendering, preview/style/seed-finder synthesis, and the OpenAI-compatible speech proxy are capped at two active jobs; STT transcription and benchmarks are capped at one active job. diff --git a/VERSION b/VERSION index 0f089f4..d14ef08 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.24.16 +1.24.17 diff --git a/core/provider_health.py b/core/provider_health.py new file mode 100644 index 0000000..4bb916e --- /dev/null +++ b/core/provider_health.py @@ -0,0 +1,47 @@ +"""Shared, bounded provider liveness probes.""" +from __future__ import annotations + +from typing import Any, Iterable + +import requests + + +def probe_provider( + base_url: str, + checks: Iterable[tuple[str, str | None]], + *, + headers: dict[str, str] | None = None, + timeout: float = 3.0, +) -> dict[str, Any]: + """Try provider endpoints in order with normalized, body-free results.""" + base = str(base_url or '').rstrip('/') + limit = max(0.5, float(timeout)) + last_error = 'No response' + for path, json_key in checks: + endpoint = path or '/' + try: + response = requests.get(base + path, timeout=limit, headers=headers or {}, allow_redirects=True) + if response.status_code >= 500: + last_error = f'HTTP {response.status_code} on {endpoint}' + continue + if response.status_code >= 400: + last_error = f'HTTP {response.status_code} on {endpoint}' + continue + payload = None + if json_key: + try: + payload = response.json() + except Exception: + last_error = f'{endpoint} returned non-JSON (HTTP {response.status_code})' + continue + if not isinstance(payload, dict) or json_key not in payload: + last_error = f"Unexpected response from {endpoint} (missing '{json_key}')" + continue + return {'ok': True, 'status': response.status_code, 'endpoint': endpoint, 'payload': payload} + except requests.exceptions.ConnectionError: + last_error = 'Connection refused' + except requests.exceptions.Timeout: + last_error = f'Timeout ({limit:g} s)' + except Exception as exc: + last_error = str(exc) + return {'ok': False, 'error': last_error} diff --git a/routes/docker.py b/routes/docker.py index 34e2bba..1178b09 100644 --- a/routes/docker.py +++ b/routes/docker.py @@ -12,6 +12,7 @@ from fastapi import APIRouter, Header, HTTPException from core.docker_client import _docker_available, _docker_get_json, _docker_post from core.config import _load_settings from core.validation import _normalize_service_url +from core.provider_health import probe_provider router = APIRouter() @@ -157,32 +158,8 @@ async def probe_url(url: str, type: str = "", provider_api_key: str = Header("", else: checks = [("", None)] - last_err = "No response" - for path, json_key in checks: - try: - r = requests.get(base + path, timeout=5, headers=hdrs, allow_redirects=True) - if r.status_code >= 500: - last_err = f"HTTP {r.status_code} on {path or '/'}" - continue - if r.status_code >= 400 and path: - continue - if json_key: - try: - data = r.json() - if json_key not in data: - last_err = f"Unexpected response from {path} (missing '{json_key}')" - continue - except Exception: - last_err = f"{path} returned non-JSON (HTTP {r.status_code})" - continue - return {"ok": True, "status": r.status_code, "endpoint": path or "/"} - except requests.exceptions.ConnectionError: - return {"ok": False, "error": "Connection refused"} - except requests.exceptions.Timeout: - return {"ok": False, "error": "Timeout (5 s)"} - except Exception as e: - last_err = str(e) - return {"ok": False, "error": last_err} + result = probe_provider(base, checks, headers=hdrs, timeout=5) + return {key: value for key, value in result.items() if key != 'payload'} @router.get("/api/engine-models") async def engine_models(url: str, type: str = "", provider_api_key: str = Header("", alias="X-Provider-API-Key")): diff --git a/routes/stt.py b/routes/stt.py index d284fc4..6240dc9 100644 --- a/routes/stt.py +++ b/routes/stt.py @@ -20,6 +20,7 @@ from core.constants import ( ) from core.registry import _registry_get, TEMP_DIR from core.validation import _validate_http_url, _copy_limited, _validated_upload_suffix +from core.provider_health import probe_provider from core.audio import _to_wav_16k from core.database import background_job_create, background_job_get, background_job_update from core.voice import _AUDIO_EXTS, _UPLOAD_EXTS @@ -234,25 +235,19 @@ def _stt_backend_health(url: str) -> tuple[bool, list[str]]: # `/v1/v1/models`, incorrectly marking a healthy WhisperX service down. api_base = base[:-3].rstrip("/") if base.endswith("/v1") else base models: list[str] = [] - ok = False - try: - r = requests.get(f"{api_base}/health", timeout=2) - ok = r.status_code == 200 - except Exception: - pass - try: - r = requests.get(f"{api_base}/v1/models", timeout=3, headers={"Authorization": "Bearer sk-dummy-key"}) - if r.status_code == 200: - ok = True - payload = r.json() - data = payload.get("data", []) if isinstance(payload, dict) else [] - for item in data: - if isinstance(item, dict) and item.get("id"): - models.append(str(item["id"])) - elif isinstance(item, str): - models.append(item) - except Exception: - pass + health = probe_provider(api_base, [("/health", None)], timeout=2) + models_result = probe_provider( + api_base, [("/v1/models", "data")], + headers={"Authorization": "Bearer sk-dummy-key"}, timeout=3, + ) + ok = bool(health['ok'] or models_result['ok']) + payload = models_result.get('payload') + data = payload.get("data", []) if isinstance(payload, dict) else [] + for item in data: + if isinstance(item, dict) and item.get("id"): + models.append(str(item["id"])) + elif isinstance(item, str): + models.append(item) if not ok: return False, models # Do not send a real audio transcription as a health probe. WhisperX's diff --git a/routes/tts.py b/routes/tts.py index 108a8d2..a9c672d 100644 --- a/routes/tts.py +++ b/routes/tts.py @@ -38,6 +38,7 @@ from core.voice import ( ) from core.voice_index import indexed_voices, refresh_voice_index_background from core.validation import _validate_http_url +from core.provider_health import probe_provider from core.registry import _registry_put, TEMP_DIR from core.tts_helpers import ( _tts_request_audio, _open_tts_stream_response, _read_tts_stream_response, @@ -321,15 +322,7 @@ def _backend_capabilities(backend: str) -> dict: def _backend_health(url: str) -> bool: - base = url.rstrip("/") - for ep in ("/health", "/v1/health", "/v1/audio/list_voices"): - try: - r = requests.get(f"{base}{ep}", timeout=2) - if r.status_code == 200: - return True - except Exception: - continue - return False + return bool(probe_provider(url, [("/health", None), ("/v1/health", None), ("/v1/audio/list_voices", None)], timeout=2)['ok']) def _backend_available(backend: str, voices: list, health: bool) -> bool: diff --git a/tests/test_provider_health.py b/tests/test_provider_health.py new file mode 100644 index 0000000..2a98468 --- /dev/null +++ b/tests/test_provider_health.py @@ -0,0 +1,46 @@ +import unittest +from unittest.mock import Mock, patch + +from core.provider_health import probe_provider + + +class ProviderHealthTests(unittest.TestCase): + @patch('core.provider_health.requests.get') + def test_falls_back_to_next_endpoint_and_validates_json_shape(self, get): + first = Mock(status_code=503) + second = Mock(status_code=200) + second.json.return_value = {'data': [{'id': 'model-a'}]} + get.side_effect = [first, second] + + result = probe_provider('http://engine', [('/health', None), ('/v1/models', 'data')]) + + self.assertTrue(result['ok']) + self.assertEqual(result['endpoint'], '/v1/models') + self.assertEqual(result['payload']['data'][0]['id'], 'model-a') + self.assertEqual(get.call_args_list[1].kwargs['timeout'], 3.0) + + @patch('core.provider_health.requests.get') + def test_normalizes_connection_and_timeout_errors(self, get): + import requests + get.side_effect = requests.exceptions.ConnectionError() + result = probe_provider('http://engine', [('/health', None)]) + self.assertFalse(result['ok']) + self.assertEqual(result['error'], 'Connection refused') + + get.side_effect = requests.exceptions.Timeout() + result = probe_provider('http://engine', [('/health', None)], timeout=2) + self.assertFalse(result['ok']) + self.assertEqual(result['error'], 'Timeout (2 s)') + + @patch('core.provider_health.requests.get') + def test_does_not_accept_missing_json_key(self, get): + response = Mock(status_code=200) + response.json.return_value = {'models': []} + get.return_value = response + result = probe_provider('http://engine', [('/v1/models', 'data')]) + self.assertFalse(result['ok']) + self.assertIn("missing 'data'", result['error']) + + +if __name__ == '__main__': + unittest.main()