refactor: centralize provider health probes

CHANGELOG.md

@@ -8,6 +8,8 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
 ## [Unreleased]
 
 ### Changed
+- **Shared provider-health probes.** TTS, STT, and container connectivity checks now use one bounded helper with
+  normalized connection, timeout, HTTP-status, and JSON-shape handling while retaining provider-specific endpoints.
 - **Section-scoped feature loading.** The shared module manifest now builds a 17-module foundation bundle and loads
   Rehearser, Reader/Audiobook, Library/Characters, and Studio code only when their sections are opened. Repeated
   navigation reuses loaded modules, while deep links load their group before activation.

TODO.md

@@ -157,7 +157,9 @@ browser image/profile-cache eviction.
   bounded retries from stripped in-memory inputs; retry data is never returned or persisted, and restart correctly
   requires reopening the source workflow.
 - [x] Prevent duplicate generation when a user clicks twice for image, voice-design, and casting submissions.
-- [ ] Centralize provider health checks and error handling.
+- [x] Centralize provider health checks and error handling. TTS, STT, and container URL probes now share the bounded
+  `core.provider_health.probe_provider` mechanics and normalized connection/timeout/HTTP/JSON-shape errors; their
+  provider-specific endpoint lists and authentication policies remain at the route layer.
 - [x] Limit concurrency to 1–2 image generations, 1–2 voice generations, and 1 STT request per user. Consistent
   portrait/reference rendering, preview/style/seed-finder synthesis, and the OpenAI-compatible speech proxy are
   capped at two active jobs; STT transcription and benchmarks are capped at one active job.

VERSION

@@ -1 +1 @@
-1.24.16
+1.24.17

core/provider_health.py

@@ -0,0 +1,47 @@
+"""Shared, bounded provider liveness probes."""
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+import requests
+
+
+def probe_provider(
+    base_url: str,
+    checks: Iterable[tuple[str, str | None]],
+    *,
+    headers: dict[str, str] | None = None,
+    timeout: float = 3.0,
+) -> dict[str, Any]:
+    """Try provider endpoints in order with normalized, body-free results."""
+    base = str(base_url or '').rstrip('/')
+    limit = max(0.5, float(timeout))
+    last_error = 'No response'
+    for path, json_key in checks:
+        endpoint = path or '/'
+        try:
+            response = requests.get(base + path, timeout=limit, headers=headers or {}, allow_redirects=True)
+            if response.status_code >= 500:
+                last_error = f'HTTP {response.status_code} on {endpoint}'
+                continue
+            if response.status_code >= 400:
+                last_error = f'HTTP {response.status_code} on {endpoint}'
+                continue
+            payload = None
+            if json_key:
+                try:
+                    payload = response.json()
+                except Exception:
+                    last_error = f'{endpoint} returned non-JSON (HTTP {response.status_code})'
+                    continue
+                if not isinstance(payload, dict) or json_key not in payload:
+                    last_error = f"Unexpected response from {endpoint} (missing '{json_key}')"
+                    continue
+            return {'ok': True, 'status': response.status_code, 'endpoint': endpoint, 'payload': payload}
+        except requests.exceptions.ConnectionError:
+            last_error = 'Connection refused'
+        except requests.exceptions.Timeout:
+            last_error = f'Timeout ({limit:g} s)'
+        except Exception as exc:
+            last_error = str(exc)
+    return {'ok': False, 'error': last_error}

routes/docker.py

@@ -12,6 +12,7 @@ from fastapi import APIRouter, Header, HTTPException
 from core.docker_client import _docker_available, _docker_get_json, _docker_post
 from core.config import _load_settings
 from core.validation import _normalize_service_url
+from core.provider_health import probe_provider
 
 router = APIRouter()
 
@@ -157,32 +158,8 @@ async def probe_url(url: str, type: str = "", provider_api_key: str = Header("",
     else:
         checks = [("", None)]
 
-    last_err = "No response"
+    result = probe_provider(base, checks, headers=hdrs, timeout=5)
-    for path, json_key in checks:
+    return {key: value for key, value in result.items() if key != 'payload'}
-        try:
-            r = requests.get(base + path, timeout=5, headers=hdrs, allow_redirects=True)
-            if r.status_code >= 500:
-                last_err = f"HTTP {r.status_code} on {path or '/'}"
-                continue
-            if r.status_code >= 400 and path:
-                continue
-            if json_key:
-                try:
-                    data = r.json()
-                    if json_key not in data:
-                        last_err = f"Unexpected response from {path} (missing '{json_key}')"
-                        continue
-                except Exception:
-                    last_err = f"{path} returned non-JSON (HTTP {r.status_code})"
-                    continue
-            return {"ok": True, "status": r.status_code, "endpoint": path or "/"}
-        except requests.exceptions.ConnectionError:
-            return {"ok": False, "error": "Connection refused"}
-        except requests.exceptions.Timeout:
-            return {"ok": False, "error": "Timeout (5 s)"}
-        except Exception as e:
-            last_err = str(e)
-    return {"ok": False, "error": last_err}
 
 @router.get("/api/engine-models")
 async def engine_models(url: str, type: str = "", provider_api_key: str = Header("", alias="X-Provider-API-Key")):

routes/stt.py

@@ -20,6 +20,7 @@ from core.constants import (
 )
 from core.registry import _registry_get, TEMP_DIR
 from core.validation import _validate_http_url, _copy_limited, _validated_upload_suffix
+from core.provider_health import probe_provider
 from core.audio import _to_wav_16k
 from core.database import background_job_create, background_job_get, background_job_update
 from core.voice import _AUDIO_EXTS, _UPLOAD_EXTS
@@ -234,25 +235,19 @@ def _stt_backend_health(url: str) -> tuple[bool, list[str]]:
     # `/v1/v1/models`, incorrectly marking a healthy WhisperX service down.
     api_base = base[:-3].rstrip("/") if base.endswith("/v1") else base
     models: list[str] = []
-    ok = False
+    health = probe_provider(api_base, [("/health", None)], timeout=2)
-    try:
+    models_result = probe_provider(
-        r = requests.get(f"{api_base}/health", timeout=2)
+        api_base, [("/v1/models", "data")],
-        ok = r.status_code == 200
+        headers={"Authorization": "Bearer sk-dummy-key"}, timeout=3,
-    except Exception:
+    )
-        pass
+    ok = bool(health['ok'] or models_result['ok'])
-    try:
+    payload = models_result.get('payload')
-        r = requests.get(f"{api_base}/v1/models", timeout=3, headers={"Authorization": "Bearer sk-dummy-key"})
+    data = payload.get("data", []) if isinstance(payload, dict) else []
-        if r.status_code == 200:
+    for item in data:
-            ok = True
+        if isinstance(item, dict) and item.get("id"):
-            payload = r.json()
+            models.append(str(item["id"]))
-            data = payload.get("data", []) if isinstance(payload, dict) else []
+        elif isinstance(item, str):
-            for item in data:
+            models.append(item)
-                if isinstance(item, dict) and item.get("id"):
-                    models.append(str(item["id"]))
-                elif isinstance(item, str):
-                    models.append(item)
-    except Exception:
-        pass
     if not ok:
         return False, models
     # Do not send a real audio transcription as a health probe. WhisperX's

routes/tts.py

@@ -38,6 +38,7 @@ from core.voice import (
 )
 from core.voice_index import indexed_voices, refresh_voice_index_background
 from core.validation import _validate_http_url
+from core.provider_health import probe_provider
 from core.registry import _registry_put, TEMP_DIR
 from core.tts_helpers import (
     _tts_request_audio, _open_tts_stream_response, _read_tts_stream_response,
@@ -321,15 +322,7 @@ def _backend_capabilities(backend: str) -> dict:
 
 
 def _backend_health(url: str) -> bool:
-    base = url.rstrip("/")
+    return bool(probe_provider(url, [("/health", None), ("/v1/health", None), ("/v1/audio/list_voices", None)], timeout=2)['ok'])
-    for ep in ("/health", "/v1/health", "/v1/audio/list_voices"):
-        try:
-            r = requests.get(f"{base}{ep}", timeout=2)
-            if r.status_code == 200:
-                return True
-        except Exception:
-            continue
-    return False
 
 
 def _backend_available(backend: str, voices: list, health: bool) -> bool:

tests/test_provider_health.py

@@ -0,0 +1,46 @@
+import unittest
+from unittest.mock import Mock, patch
+
+from core.provider_health import probe_provider
+
+
+class ProviderHealthTests(unittest.TestCase):
+    @patch('core.provider_health.requests.get')
+    def test_falls_back_to_next_endpoint_and_validates_json_shape(self, get):
+        first = Mock(status_code=503)
+        second = Mock(status_code=200)
+        second.json.return_value = {'data': [{'id': 'model-a'}]}
+        get.side_effect = [first, second]
+
+        result = probe_provider('http://engine', [('/health', None), ('/v1/models', 'data')])
+
+        self.assertTrue(result['ok'])
+        self.assertEqual(result['endpoint'], '/v1/models')
+        self.assertEqual(result['payload']['data'][0]['id'], 'model-a')
+        self.assertEqual(get.call_args_list[1].kwargs['timeout'], 3.0)
+
+    @patch('core.provider_health.requests.get')
+    def test_normalizes_connection_and_timeout_errors(self, get):
+        import requests
+        get.side_effect = requests.exceptions.ConnectionError()
+        result = probe_provider('http://engine', [('/health', None)])
+        self.assertFalse(result['ok'])
+        self.assertEqual(result['error'], 'Connection refused')
+
+        get.side_effect = requests.exceptions.Timeout()
+        result = probe_provider('http://engine', [('/health', None)], timeout=2)
+        self.assertFalse(result['ok'])
+        self.assertEqual(result['error'], 'Timeout (2 s)')
+
+    @patch('core.provider_health.requests.get')
+    def test_does_not_accept_missing_json_key(self, get):
+        response = Mock(status_code=200)
+        response.json.return_value = {'models': []}
+        get.return_value = response
+        result = probe_provider('http://engine', [('/v1/models', 'data')])
+        self.assertFalse(result['ok'])
+        self.assertIn("missing 'data'", result['error'])
+
+
+if __name__ == '__main__':
+    unittest.main()