feat: unify transient provider retries

CHANGELOG.md

@@ -50,6 +50,8 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
   and avoids unnecessary duplicate assignments when distinct catalogue voices are available.
 - **Attribution and Timeout Safety Coverage.** Added regressions preventing phantom narrator speakers, preserving source
   prose after malformed LLM JSON, clamping caller timeouts, and bounding upstream error text.
+- **Unified Provider Retries.** TTS and LLM provider requests now retry bounded connection, timeout, and transient HTTP
+  failures with exponential backoff while closing discarded responses.
 - **4 GiB Container Validation.** Verified the current image serves the application under a hard 4 GiB cgroup limit
   with no OOM kill; the probe recorded approximately 50.6 MiB idle memory and 0.09% CPU.
 - **Profile Completeness Coverage.** Added browser regression coverage for empty, partial, and fully populated

TODO.md

@@ -66,7 +66,7 @@ the first block before investing in architecture or polish.
 - [ ] **L — Add background jobs and bounded queues.** Image generation already has the full interactive job path,
   and transcription, voice design, and casting now have optional ID/status/cancel/timeout paths while preserving
   synchronous callers. Job metadata now persists in SQLite and marks interrupted work after restart; durable binary
-  result storage and a unified retry policy still need to be added.
+  result storage remains; outbound TTS and LLM calls now share a bounded transient-failure retry policy.
 - [ ] **L — Run the Book 2 generalization regression.** Added a neutral multi-speaker fixture and parser regression;
   a real Book 2 auto-cast/attribution output still needs to be captured and reviewed to confirm rules are not overfit
   to *Der Sturm*.

VERSION

@@ -1 +1 @@
-1.23.6
+1.24.0

core/config.py

@@ -18,6 +18,7 @@ from core.constants import (
     _KOKORO_DEFAULT, _VIBEVOICE_DEFAULT, _XTTS_DEFAULT, _FISHSPEECH_DEFAULT,
     _TTS_STREAM_DEFAULT,
 )
+from core.retry import request_with_retries
 from core.database import state_get, state_put, state_updated
 
 # ── Settings key whitelist ────────────────────────────────────────────────────
@@ -211,7 +212,7 @@ def _strip_tts_extra_params(payload: dict) -> dict:
 
 
 def _post_tts_with_fallback(endpoint: str, payload: dict, headers: dict, **kwargs) -> requests.Response:
-    resp = requests.post(endpoint, json=payload, headers=headers, **kwargs)
+    resp = request_with_retries("POST", endpoint, json=payload, headers=headers, **kwargs)
     fallback = _strip_tts_extra_params(payload)
     if resp.status_code in {400, 404, 415, 422} and fallback != payload:
         try:
@@ -220,7 +221,7 @@ def _post_tts_with_fallback(endpoint: str, payload: dict, headers: dict, **kwarg
             pass
         import logging
         logging.getLogger("uvicorn.error").warning("TTS backend rejected extra generation params; retrying without them")
-        return requests.post(endpoint, json=fallback, headers=headers, **kwargs)
+        return request_with_retries("POST", endpoint, json=fallback, headers=headers, **kwargs)
     return resp
 
 

core/retry.py

@@ -0,0 +1,40 @@
+"""Small, bounded retry primitives shared by outbound provider requests."""
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+
+import requests
+
+_RETRYABLE_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504})
+
+
+def retry_delay(attempt: int, *, base: float = 0.5, cap: float = 8.0) -> float:
+    """Return bounded exponential backoff for the zero-based retry attempt."""
+    return min(float(cap), max(0.0, float(base)) * (2 ** max(0, int(attempt))))
+
+
+def request_with_retries(method: str, url: str, *, attempts: int = 3,
+                         sleep: Callable[[float], None] | None = None, **kwargs):
+    """Make a request, retrying only transient transport/upstream failures.
+
+    The final response or exception is returned to the caller unchanged. Any
+    response discarded before a retry is closed so streaming sockets cannot
+    accumulate while a provider is cold or rate-limiting.
+    """
+    attempts = max(1, min(int(attempts), 5))
+    sleep = sleep or time.sleep
+    last_response = None
+    for attempt in range(attempts):
+        try:
+            response = requests.request(method, url, **kwargs)
+            last_response = response
+            if response.status_code not in _RETRYABLE_STATUS_CODES or attempt >= attempts - 1:
+                return response
+            response.close()
+        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
+            if attempt >= attempts - 1:
+                raise
+        if attempt < attempts - 1:
+            sleep(retry_delay(attempt))
+    return last_response

routes/conversation.py

@@ -34,6 +34,7 @@ from core.voice import (
 )
 from core.voice_index import rebuild_voice_index
 from core.database import background_job_create, background_job_get, background_job_update
+from core.retry import request_with_retries
 from core.tts_helpers import _preview_request_audio
 from routes.stt import _transcribe_audio, _clean_stt_backend
 
@@ -340,7 +341,7 @@ def _post_llm_chat_completion(llm_url: str, payload: dict, headers: dict, timeou
     last_exc = None
     for url in _llm_chat_completion_urls(llm_url):
         try:
-            resp = requests.post(url, json=payload, headers=headers, timeout=timeout)
+            resp = request_with_retries("POST", url, json=payload, headers=headers, timeout=timeout, attempts=2)
             last_resp = resp
             if resp.status_code in (404, 405) and url != _llm_chat_completion_urls(llm_url)[-1]:
                 continue

tests/test_security.py

@@ -59,6 +59,25 @@ class SecurityValidationTests(unittest.TestCase):
         self.assertEqual(_transcription_text_from_response(ResponseStub({"segments": [{"text": " one "}, {"text": "two"}]})), "one two")
         self.assertEqual(_transcription_text_from_response(ResponseStub(" plain text ")), "plain text")
 
+    def test_provider_retry_policy_is_bounded_and_closes_discarded_responses(self):
+        from unittest.mock import patch
+        from core.retry import request_with_retries, retry_delay
+        class ResponseStub:
+            def __init__(self, status):
+                self.status_code, self.closed = status, False
+            def close(self):
+                self.closed = True
+        first, second, final = ResponseStub(503), ResponseStub(429), ResponseStub(200)
+        with patch("core.retry.requests.request", side_effect=[first, second, final]) as request, \
+             patch("core.retry.time.sleep") as sleep:
+            result = request_with_retries("POST", "http://provider", attempts=3)
+        self.assertIs(result, final)
+        self.assertTrue(first.closed)
+        self.assertTrue(second.closed)
+        self.assertEqual(request.call_count, 3)
+        self.assertEqual(sleep.call_count, 2)
+        self.assertLessEqual(retry_delay(20), 8.0)
+
     def test_request_size_middleware_rejects_oversized_json(self):
         sent = []