diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f6b8fc..b5aabd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi --- +## [1.14.2] — 2026-07-06 + +### Fixed +- **Live streaming ("watch the LLM think") could get permanently stuck, silently breaking casting/character-sheets for the rest of the session** — confirmed in production: a stream held the shared attribution lock for 11+ minutes, causing every subsequent passage to be silently rejected as "busy" and fall back to the plain non-streaming view. This made it *look* like live-thinking had stopped working entirely, when actually one earlier passage never finished. Root cause: `requests`' `timeout=` on a `stream=True` call only guards the connection and first byte, not gaps between later body reads — if the LLM backend goes silent mid-stream, the blocked socket read can hang indefinitely, and since that's a native blocking call (not a Python-level `yield` point), neither a wall-clock check in the read loop nor the client disconnecting can interrupt it. Added a watchdog thread that force-closes the connection if a stream runs past its configured timeout, guaranteeing the shared lock always releases on schedule regardless of backend behavior. + +--- + ## [1.14.1] — 2026-07-06 ### Fixed diff --git a/VERSION b/VERSION index 63e799c..a4cc557 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.1 +1.14.2 diff --git a/routes/conversation.py b/routes/conversation.py index f26f01a..c49fa4c 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -49,6 +49,42 @@ router = APIRouter() # scenario this lock exists to prevent. _attribution_llm_lock = threading.Lock() + +@contextlib.contextmanager +def _watchdog_close(resp, timeout_seconds): + """Force-close `resp`'s connection if the wrapped block hasn't finished + within timeout_seconds. + + requests' own `timeout=` on a stream=True call only guards the connect + + first byte — NOT the gaps between subsequent body reads. If the LLM + backend goes silent mid-stream (no more chunks, connection left open), + the blocking socket recv() inside iter_lines() can hang forever, and + since that's a native blocking call rather than a Python-level `yield` + point, neither a wall-clock check in the loop body nor GeneratorExit from + a disconnected client can interrupt it — both only take effect at the + next bytecode boundary, which never arrives. Observed in production: a + stuck stream held the shared attribution lock for 11+ minutes, silently + starving every other passage. Closing the connection from a separate + watchdog thread forces the blocked recv() to raise, unblocking the + generator so the normal except/finally cleanup (incl. releasing the + lock) actually runs. + """ + done = threading.Event() + + def _kill(): + if not done.wait(timeout_seconds): + try: + resp.close() + except Exception: + pass + + t = threading.Thread(target=_kill, daemon=True) + t.start() + try: + yield + finally: + done.set() + # ── STT hallucination filter ────────────────────────────────────────────────── # Whisper commonly hallucinates these phrases on silence/noise. # Treat them as "no speech detected" rather than passing them to the LLM. @@ -816,21 +852,32 @@ async def character_sheets_stream(request: Request): return raw_parts = [] upstream.encoding = "utf-8" - for line in upstream.iter_lines(decode_unicode=True): - if not line or not line.startswith("data:"): - continue - chunk = line[5:].strip() - if chunk == "[DONE]": - break - try: - delta = json.loads(chunk)["choices"][0]["delta"] - except Exception: - continue - t = delta.get("reasoning_content") or delta.get("content") or "" - if delta.get("content"): - raw_parts.append(delta["content"]) - if t: - yield f'data: {json.dumps({"t": t})}\n\n' + # Two layers against a stuck stream: the wall-clock check catches + # a model that keeps trickling chunks past its budget, and + # _watchdog_close catches the connection going fully silent (which + # the in-loop check can't see, since a blocked socket read never + # reaches it — see _watchdog_close's docstring for why). + deadline = time.monotonic() + prep["timeout_seconds"] + with _watchdog_close(upstream, prep["timeout_seconds"]): + for line in upstream.iter_lines(decode_unicode=True): + if time.monotonic() > deadline: + print(f"[character-sheets/stream] wall-clock deadline hit after {prep['timeout_seconds']}s, aborting stream") + yield 'data: {"error": "LLM stream exceeded timeout"}\n\n' + return + if not line or not line.startswith("data:"): + continue + chunk = line[5:].strip() + if chunk == "[DONE]": + break + try: + delta = json.loads(chunk)["choices"][0]["delta"] + except Exception: + continue + t = delta.get("reasoning_content") or delta.get("content") or "" + if delta.get("content"): + raw_parts.append(delta["content"]) + if t: + yield f'data: {json.dumps({"t": t})}\n\n' result = _charsheets_parse("".join(raw_parts)) yield f'data: {json.dumps({"done": True, "result": result})}\n\n' except GeneratorExit: @@ -1372,24 +1419,38 @@ async def attribute_dialogue_stream(request: Request): # UTF-8 when decode_unicode=True guesses the encoding — every umlaut # then comes out mojibake'd ("Häfen" -> "Häfen"). Force UTF-8. upstream.encoding = "utf-8" - for line in upstream.iter_lines(decode_unicode=True): - if not line or not line.startswith("data:"): - continue - chunk = line[5:].strip() - if chunk == "[DONE]": - break - try: - delta = json.loads(chunk)["choices"][0]["delta"] - except Exception: - continue - # Reasoning models emit thoughts via reasoning_content; the - # answer JSON arrives via content. Forward both to the viewer, - # but only content counts toward the parseable answer. - t = delta.get("reasoning_content") or delta.get("content") or "" - if delta.get("content"): - raw_parts.append(delta["content"]) - if t: - yield f'data: {json.dumps({"t": t})}\n\n' + # Two layers against a stuck stream: the wall-clock check catches + # a model that keeps trickling chunks past its budget, and + # _watchdog_close catches the connection going fully silent (which + # the in-loop check can't see, since a blocked socket read never + # reaches it — see _watchdog_close's docstring for why). Observed + # in production without the watchdog: a stuck stream held the + # shared lock for 11+ minutes, silently starving every other + # passage/request. + deadline = time.monotonic() + prep["timeout_seconds"] + with _watchdog_close(upstream, prep["timeout_seconds"]): + for line in upstream.iter_lines(decode_unicode=True): + if time.monotonic() > deadline: + print(f"[attribute-dialogue/stream] wall-clock deadline hit after {prep['timeout_seconds']}s, aborting stream") + yield 'data: {"error": "LLM stream exceeded timeout"}\n\n' + return + if not line or not line.startswith("data:"): + continue + chunk = line[5:].strip() + if chunk == "[DONE]": + break + try: + delta = json.loads(chunk)["choices"][0]["delta"] + except Exception: + continue + # Reasoning models emit thoughts via reasoning_content; the + # answer JSON arrives via content. Forward both to the viewer, + # but only content counts toward the parseable answer. + t = delta.get("reasoning_content") or delta.get("content") or "" + if delta.get("content"): + raw_parts.append(delta["content"]) + if t: + yield f'data: {json.dumps({"t": t})}\n\n' result = _attribution_parse("".join(raw_parts), prep["text"]) yield f'data: {json.dumps({"done": True, "result": result})}\n\n' except GeneratorExit: diff --git a/static/index.html b/static/index.html index a31fe8c..2d4a10d 100644 --- a/static/index.html +++ b/static/index.html @@ -10,7 +10,7 @@ - + @@ -27,7 +27,7 @@ - + @@ -365,7 +365,7 @@ window.toggleNavTree = function(treeId, chevronId) { - +