diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f345e..f53e094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,10 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi object URLs, reducing retained browser memory during repeated auditioning. - **Character Library Smoke Coverage.** Added deterministic browser coverage for opening Characters / Cast, profile sorting, column visibility, and assigned-voice rating persistence. +- **Attribution Quote Repair.** Mixed German guillemet pairs are now handled by the deterministic fallback, and + normalized LLM segments are repaired when an epigraph is split before its closing quote. +- **Attribution Regression Coverage.** Added focused tests for mixed guillemets and the Schwester Larissa epigraph + split regression. - **Security Regression Coverage.** Added a Python test gate for character metadata validation, private-download URL rejection and oversized JSON request rejection. - **Navigation Reliability.** Rehearser phase requests now survive the feature-module initialization boundary. diff --git a/TODO.md b/TODO.md index 4e61dc4..7fe168e 100644 --- a/TODO.md +++ b/TODO.md @@ -46,8 +46,8 @@ the first block before investing in architecture or polish. The remaining segmentation work is upstream splitting/fusion, not another merge pass. - [ ] **S — Finish the remaining Playwright smoke checks.** Characters/Cast, profile sorting, column visibility, voice ratings, and starting/cancelling a generation job. -- [ ] **M — Fix the Schwester Larissa unclosed-guillemet split.** Add a regression test - for the epigraph and verify the parser does not split mid-sentence. +- [x] **M — Fix the Schwester Larissa unclosed-guillemet split.** Attribution normalization now repairs dialogue + split before mixed German guillemet closes (`»…‹` / `«…›`), with an epigraph regression test. - [ ] **M — Make profile portraits and character sheets use one canonical identity.** Shared structured prompts now version consistent pairs as v3 and fingerprint canonical appearance fields; add cross-record validation so mismatched profile/reference files are detected even when both are current-version. @@ -139,9 +139,9 @@ the first block before investing in architecture or polish. ## P0 — Data integrity and recovery -- [ ] Validate character sheets and voice metadata with schemas. -- [ ] Add database migrations instead of silently changing record shapes. -- [ ] Use atomic writes for `.meta.json` files. +- [x] Validate character sheets and voice metadata with schemas. +- [x] Add database migrations instead of silently changing record shapes. +- [x] Use atomic writes for `.meta.json` files. - [x] Create automatic backups before bulk deletes, voice moves, and recasts. - [x] Store prompt versions and generation history so bad regenerations can be audited and follow-up recovery can be built. @@ -152,7 +152,7 @@ the first block before investing in architecture or polish. - [ ] Character parsing and phantom-speaker prevention. - [ ] Voice-matching constraints. - [ ] Profile-completeness counting. -- [ ] Rating persistence. +- [x] Rating persistence. - [ ] STT endpoint normalization. - [ ] ComfyUI workflow submission. - [ ] API failure and timeout behavior. @@ -223,7 +223,8 @@ OCR/wording differences. Attribution work cannot fix these. separate is contributing. - The merge was predicted (on run 2 data) to cut unmatched 727 → 588. On run 3 it went to 750. **That prediction did not hold** and needs re-deriving on run 3. -- Known unfixed: the Schwester Larissa epigraph still splits mid-sentence +- Fixed: attribution normalization repairs the Schwester Larissa epigraph when the LLM splits it before a mixed + guillemet close; the regression is covered by `tests/test_conversation.py`. (unclosed-guillemet handling). ### Next steps, highest value first diff --git a/routes/conversation.py b/routes/conversation.py index e760235..71c978c 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -261,7 +261,7 @@ def _post_llm_chat_completion(llm_url: str, payload: dict, headers: dict, timeou def _fallback_attribute_response(text: str) -> dict: """Deterministic fallback for attribution when an upstream LLM fails.""" quote_re = re.compile( - r"»([^«]+)«|«([^»]+)»|„([^“”]+)[“”]|“([^”]+)”|\"([^\"]+)\"|「([^」]+)」|『([^』]+)』", + r"»([^«‹]+)[«‹]|«([^»›]+)[»›]|„([^“”]+)[“”]|“([^”]+)”|\"([^\"]+)\"|「([^」]+)」|『([^』]+)』", re.DOTALL, ) segments = [] @@ -288,6 +288,54 @@ def _fallback_attribute_response(text: str) -> dict: return {"segments": segments, "characters": chars, "fallback": True} +def _repair_unclosed_dialogue_segments(segments: list) -> list: + """Join dialogue with a following narration fragment split before its quote close. + + OCR and LLM output occasionally mix German guillemet pairs (``»…‹`` and + ``«…›``). When that happens, a long epigraph can be split at an arbitrary + chunk boundary. Only repair a dialogue segment whose opening quote is + still unmatched, and only consume the following narration through the + corresponding close marker; unrelated narration remains untouched. + """ + pairs = (("»", ("«", "‹")), ("«", ("»", "›")), ("„", ("“", "”")), ("“", ("”",)), ("「", ("」",)), ("『", ("』",))) + repaired = [dict(seg) if isinstance(seg, dict) else seg for seg in segments] + index = 0 + while index < len(repaired): + current = repaired[index] + if not isinstance(current, dict) or str(current.get("type") or "").lower() != "dialogue": + index += 1 + continue + text = str(current.get("text") or "") + pair = next(((open_mark, closes) for open_mark, closes in pairs if open_mark in text), None) + if not pair: + index += 1 + continue + open_mark, close_marks = pair + opening = text.find(open_mark) + if any(mark in text[opening + 1:] for mark in close_marks): + index += 1 + continue + for following in range(index + 1, len(repaired)): + candidate = repaired[following] + if not isinstance(candidate, dict): + continue + if str(candidate.get("type") or "").lower() != "narration": + break + candidate_text = str(candidate.get("text") or "") + close_at = min((candidate_text.find(mark) for mark in close_marks if candidate_text.find(mark) >= 0), default=-1) + if close_at < 0: + continue + current["text"] = text + " " + candidate_text[:close_at + 1] + remainder = candidate_text[close_at + 1:].strip() + if remainder: + candidate["text"] = remainder + else: + repaired.pop(following) + break + index += 1 + return repaired + + def _resolve_speak_voice(settings: dict, client_id: str, explicit_voice: str) -> str: if explicit_voice: return explicit_voice @@ -3695,6 +3743,8 @@ def _attribution_parse(raw: str, text: str) -> dict: # Do NOT try to parse its reasoning as a script. Abort and preserve the book text. segments = [{"speaker": "Narrator", "type": "narration", "text": text, "emotion": ""}] + segments = _repair_unclosed_dialogue_segments(segments) + # Normalise + collect speaker roster clean, chars = [], [] for seg in segments: diff --git a/tests/test_conversation.py b/tests/test_conversation.py new file mode 100644 index 0000000..33be133 --- /dev/null +++ b/tests/test_conversation.py @@ -0,0 +1,34 @@ +"""Focused attribution parser regressions.""" +from __future__ import annotations + +import os +import tempfile +import unittest + +os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-conversation-test-config-")) + +from routes.conversation import _attribution_parse, _fallback_attribute_response + + +class AttributionParserTests(unittest.TestCase): + def test_fallback_accepts_mixed_guillemets(self): + result = _fallback_attribute_response("Intro. »Ein langer Satz endet hier.‹ Danach.") + self.assertEqual([segment["type"] for segment in result["segments"]], ["narration", "dialogue", "narration"]) + self.assertIn("Ein langer Satz endet hier.", result["segments"][1]["text"]) + + def test_repairs_epigraph_split_before_mixed_guillemet_close(self): + raw = '{"segments":[' \ + '{"speaker":"Schwester Larissa","type":"dialogue","text":"»Von dem Windhager Felsgesteyn bis zur Kueste von Mendena ziehet es sich hin, von Ifirns Eisgebirg bis zu dem dampfenden Waldt der Mohas, und"},' \ + '{"speaker":"Narrator","type":"narration","text":"fürwahr, es ist ein seltsam Land. Lieblich bisweilen, mild und voll der guten Menschen. ‹ Zitiert aus dem Buch."}' \ + ']}' + result = _attribution_parse(raw, "unused") + self.assertEqual(len(result["segments"]), 2) + dialogue, narration = result["segments"] + self.assertEqual(dialogue["speaker"], "Schwester Larissa") + self.assertEqual(dialogue["type"], "dialogue") + self.assertTrue(dialogue["text"].endswith("Land. Lieblich bisweilen, mild und voll der guten Menschen. ‹")) + self.assertEqual(narration["text"], "Zitiert aus dem Buch.") + + +if __name__ == "__main__": + unittest.main()