From 5219bc45884384a7ad19062c4d3d07081bb7de6f Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Fri, 4 Sep 2026 09:36:48 +0200 Subject: [PATCH] feat: unify transient provider retries --- CHANGELOG.md | 2 ++ TODO.md | 2 +- VERSION | 2 +- core/config.py | 5 +++-- core/retry.py | 40 ++++++++++++++++++++++++++++++++++++++++ routes/conversation.py | 3 ++- tests/test_security.py | 19 +++++++++++++++++++ 7 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 core/retry.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7635957..21cb913 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi and avoids unnecessary duplicate assignments when distinct catalogue voices are available. - **Attribution and Timeout Safety Coverage.** Added regressions preventing phantom narrator speakers, preserving source prose after malformed LLM JSON, clamping caller timeouts, and bounding upstream error text. +- **Unified Provider Retries.** TTS and LLM provider requests now retry bounded connection, timeout, and transient HTTP + failures with exponential backoff while closing discarded responses. - **4 GiB Container Validation.** Verified the current image serves the application under a hard 4 GiB cgroup limit with no OOM kill; the probe recorded approximately 50.6 MiB idle memory and 0.09% CPU. - **Profile Completeness Coverage.** Added browser regression coverage for empty, partial, and fully populated diff --git a/TODO.md b/TODO.md index 983ccff..e50307f 100644 --- a/TODO.md +++ b/TODO.md @@ -66,7 +66,7 @@ the first block before investing in architecture or polish. - [ ] **L — Add background jobs and bounded queues.** Image generation already has the full interactive job path, and transcription, voice design, and casting now have optional ID/status/cancel/timeout paths while preserving synchronous callers. Job metadata now persists in SQLite and marks interrupted work after restart; durable binary - result storage and a unified retry policy still need to be added. + result storage remains; outbound TTS and LLM calls now share a bounded transient-failure retry policy. - [ ] **L — Run the Book 2 generalization regression.** Added a neutral multi-speaker fixture and parser regression; a real Book 2 auto-cast/attribution output still needs to be captured and reviewed to confirm rules are not overfit to *Der Sturm*. diff --git a/VERSION b/VERSION index d8c40e5..53cc1a6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.23.6 +1.24.0 diff --git a/core/config.py b/core/config.py index 616dbee..dfbe94d 100644 --- a/core/config.py +++ b/core/config.py @@ -18,6 +18,7 @@ from core.constants import ( _KOKORO_DEFAULT, _VIBEVOICE_DEFAULT, _XTTS_DEFAULT, _FISHSPEECH_DEFAULT, _TTS_STREAM_DEFAULT, ) +from core.retry import request_with_retries from core.database import state_get, state_put, state_updated # ── Settings key whitelist ──────────────────────────────────────────────────── @@ -211,7 +212,7 @@ def _strip_tts_extra_params(payload: dict) -> dict: def _post_tts_with_fallback(endpoint: str, payload: dict, headers: dict, **kwargs) -> requests.Response: - resp = requests.post(endpoint, json=payload, headers=headers, **kwargs) + resp = request_with_retries("POST", endpoint, json=payload, headers=headers, **kwargs) fallback = _strip_tts_extra_params(payload) if resp.status_code in {400, 404, 415, 422} and fallback != payload: try: @@ -220,7 +221,7 @@ def _post_tts_with_fallback(endpoint: str, payload: dict, headers: dict, **kwarg pass import logging logging.getLogger("uvicorn.error").warning("TTS backend rejected extra generation params; retrying without them") - return requests.post(endpoint, json=fallback, headers=headers, **kwargs) + return request_with_retries("POST", endpoint, json=fallback, headers=headers, **kwargs) return resp diff --git a/core/retry.py b/core/retry.py new file mode 100644 index 0000000..b643e3b --- /dev/null +++ b/core/retry.py @@ -0,0 +1,40 @@ +"""Small, bounded retry primitives shared by outbound provider requests.""" +from __future__ import annotations + +import time +from collections.abc import Callable + +import requests + +_RETRYABLE_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504}) + + +def retry_delay(attempt: int, *, base: float = 0.5, cap: float = 8.0) -> float: + """Return bounded exponential backoff for the zero-based retry attempt.""" + return min(float(cap), max(0.0, float(base)) * (2 ** max(0, int(attempt)))) + + +def request_with_retries(method: str, url: str, *, attempts: int = 3, + sleep: Callable[[float], None] | None = None, **kwargs): + """Make a request, retrying only transient transport/upstream failures. + + The final response or exception is returned to the caller unchanged. Any + response discarded before a retry is closed so streaming sockets cannot + accumulate while a provider is cold or rate-limiting. + """ + attempts = max(1, min(int(attempts), 5)) + sleep = sleep or time.sleep + last_response = None + for attempt in range(attempts): + try: + response = requests.request(method, url, **kwargs) + last_response = response + if response.status_code not in _RETRYABLE_STATUS_CODES or attempt >= attempts - 1: + return response + response.close() + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): + if attempt >= attempts - 1: + raise + if attempt < attempts - 1: + sleep(retry_delay(attempt)) + return last_response diff --git a/routes/conversation.py b/routes/conversation.py index eb73692..efa3dbe 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -34,6 +34,7 @@ from core.voice import ( ) from core.voice_index import rebuild_voice_index from core.database import background_job_create, background_job_get, background_job_update +from core.retry import request_with_retries from core.tts_helpers import _preview_request_audio from routes.stt import _transcribe_audio, _clean_stt_backend @@ -340,7 +341,7 @@ def _post_llm_chat_completion(llm_url: str, payload: dict, headers: dict, timeou last_exc = None for url in _llm_chat_completion_urls(llm_url): try: - resp = requests.post(url, json=payload, headers=headers, timeout=timeout) + resp = request_with_retries("POST", url, json=payload, headers=headers, timeout=timeout, attempts=2) last_resp = resp if resp.status_code in (404, 405) and url != _llm_chat_completion_urls(llm_url)[-1]: continue diff --git a/tests/test_security.py b/tests/test_security.py index 8f0a8f4..3e22fa8 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -59,6 +59,25 @@ class SecurityValidationTests(unittest.TestCase): self.assertEqual(_transcription_text_from_response(ResponseStub({"segments": [{"text": " one "}, {"text": "two"}]})), "one two") self.assertEqual(_transcription_text_from_response(ResponseStub(" plain text ")), "plain text") + def test_provider_retry_policy_is_bounded_and_closes_discarded_responses(self): + from unittest.mock import patch + from core.retry import request_with_retries, retry_delay + class ResponseStub: + def __init__(self, status): + self.status_code, self.closed = status, False + def close(self): + self.closed = True + first, second, final = ResponseStub(503), ResponseStub(429), ResponseStub(200) + with patch("core.retry.requests.request", side_effect=[first, second, final]) as request, \ + patch("core.retry.time.sleep") as sleep: + result = request_with_retries("POST", "http://provider", attempts=3) + self.assertIs(result, final) + self.assertTrue(first.closed) + self.assertTrue(second.closed) + self.assertEqual(request.call_count, 3) + self.assertEqual(sleep.call_count, 2) + self.assertLessEqual(retry_delay(20), 8.0) + def test_request_size_middleware_rejects_oversized_json(self): sent = []