"""Character extraction / merge hygiene (real-library audit, 2026-10-03). Fixtures are shaped like the broken real records (see architecture/character-extraction.md): * Kolon (book 02): race_species with the model's reasoning ("Warte: ...", "Nein, Kolon ist ...") and aliases/first/last naming Sharraz (a separate cast record). * Arthag (book 03): a lower-panel quote about Nyrilla picked over Arthag's own. * Mann / Fremder (book 03): one sheet stored in two records. * Marrad (book 02): a boy whose hedged age estimate rendered as an adult. """ from __future__ import annotations import hashlib import json import os import re import sqlite3 import tempfile import unittest from pathlib import Path os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-hygiene-test-config-")) import routes.conversation as conv # noqa: E402 from core import character_records as cr # noqa: E402 from scripts import audit_character_records as audit_script # noqa: E402 from scripts import clean_character_records as clean_script # noqa: E402 ROOT = Path(__file__).resolve().parent.parent KOLON_SPECIES = ( "Mensch (oder Ork-Verbündeter, Kontext unklar, aber 'Der Zwerg' wird separat erwähnt, Kolon scheint " "der strategische Kopf zu sein - Text sagt 'Der Zwerg lächelte grimmig' im selben Absatz wie Kolon. " "Warte: 'Der Zwerg lächelte grimmig ... Es wäre allein sein Verdienst ...'. Es ist mehrdeutig. " "Nein, Kolon ist in der Liste der bekannten Charaktere.)" ) KOLON_RELATIONSHIPS = "Führt die Orks; dient oder steht im Dienst von Sharraz Garthai; arbeitet mit Drasdech." class GenericTokenTests(unittest.TestCase): def test_labels_are_generic_names_are_not(self): for token in ("Zwerg", "der Zwerg", "Mann", "Fremder", "Fremde", "Alter Soldat", "Ork-General", "Bürgerwehroffizier", "der Tunneltreiber", "Kriegshäuptling", "Erster Offizier", "Von"): self.assertTrue(cr.is_generic_identity_token(token), token) for token in ("Arthag", "Kolon", "Sharraz Garthai", "Admiral Sanin", "Schlange Gamba", "Gutfolg"): self.assertFalse(cr.is_generic_identity_token(token), token) def test_shared_generic_alias_is_no_identity_evidence(self): arthag = cr.identity_tokens("Arthag", {"aliases": "der Zwerg, Zwerg, Schreiber"}, include_generic=True) kolon = cr.identity_tokens("Kolon", {"aliases": "Zwerg, der Zwerg"}, include_generic=True) self.assertTrue(arthag & kolon) self.assertEqual(cr.shared_identity_evidence(arthag, kolon), set()) def test_js_word_list_matches_python(self): js = (ROOT / "static" / "js" / "character-identity.js").read_text(encoding="utf-8") block = re.search(r"const GENERIC_WORDS = new Set\(\((.*?)\)\.split", js, re.S).group(1) words = set(" ".join(re.findall(r"'([^']*)'", block)).split()) self.assertEqual(words, set(cr.GENERIC_IDENTITY_WORDS)) arts = re.search(r"const ARTICLES = new Set\(\('([^']*)'\)", js).group(1).split() self.assertEqual(set(arts), set(cr.ARTICLES)) class ThinkingLeakageTests(unittest.TestCase): def test_real_leak_shapes_are_cut(self): self.assertEqual(cr.strip_thinking_leakage(KOLON_SPECIES), "Mensch") self.assertEqual(cr.strip_thinking_leakage( "Prinz des Reiches, begleitet von Alrik (oder ist Alrik selbst der Prinz? Der Text sagt 'Alrik " "hatte ihr bis zuletzt nicht glauben wollen'. Es gibt Verwechslungsgefahr.)"), "Prinz des Reiches, begleitet von Alrik") self.assertEqual(cr.strip_thinking_leakage( "Schwarzhaarig. Groß. Wait, 'Die Frau bückte die Bannerträgerin' is likely a typo."), "Schwarzhaarig. Groß.") self.assertEqual(cr.strip_thinking_leakage("Kolon or Arthag?Zwerg"), "Zwerg") def test_clean_text_is_untouched(self): clean = ("Sie hat ihren Auftraggeber (Inquisition) verraten oder ist sich ihrer eigenen Gefühle " "für Zerwas bewusst.") self.assertFalse(cr.has_thinking_leakage(clean)) self.assertEqual(cr.strip_thinking_leakage(clean), clean) self.assertEqual(cr.strip_thinking_leakage(None), None) class ParseIsolationTests(unittest.TestCase): RAW = json.dumps({"sheets": [{ "name": "Kolon", "aliases": "Sharraz Garthai, General, Ork-General, Sharraz, Zwerg, Kriegshäuptling", "first_name": "Sharraz", "last_name": "Garthai", "race_species": KOLON_SPECIES, "relationships": KOLON_RELATIONSHIPS, "physical": "Kleiner Mann mit einem weißen Bart.", "sources": [{"page": 142, "quote": "Kolon kratzte sich mit dem Griffel am Hinterkopf.", "line_hint": "mannerisms"}], }, { "name": "Marina", "aliases": "Tochter des Promos, das Mädchen", "race_species": "Mensch", }]}) ROSTER = ["Kolon", "Arthag", "Sharraz", "Zwerg", "Kriegshäuptling", "Marina", "Promos"] def test_roster_names_never_become_aliases_and_leakage_is_stripped(self): out = conv._charsheets_parse(self.RAW, self.ROSTER) kolon = next(s for s in out["sheets"] if s["name"] == "Kolon") aliases = [a.strip() for a in kolon["aliases"].split(",")] self.assertNotIn("Sharraz", aliases) self.assertNotIn("Sharraz Garthai", aliases) self.assertIn("Ork-General", aliases) # a generic title is data, not a claim self.assertEqual((kolon["first_name"], kolon["last_name"]), ("", "")) self.assertEqual(kolon["race_species"], "Mensch") self.assertEqual(kolon["relationships"], KOLON_RELATIONSHIPS) marina = next(s for s in out["sheets"] if s["name"] == "Marina") self.assertIn("Tochter des Promos", marina["aliases"]) # a relation, not a name def test_without_roster_aliases_are_kept_but_leakage_still_stripped(self): out = conv._charsheets_parse(self.RAW) kolon = out["sheets"][0] self.assertIn("Sharraz", kolon["aliases"]) self.assertEqual(kolon["race_species"], "Mensch") def test_roster_only_passed_in_cast_target_mode(self): base = {"text": "Kolon sprach.", "known_characters": ["Kolon", "Sharraz"], "llm_url": "http://x/v1"} self.assertEqual(conv._charsheets_prepare(dict(base, target_mode=True))["roster"], ["Kolon", "Sharraz"]) self.assertEqual(conv._charsheets_prepare(dict(base, target_mode=False))["roster"], []) class SceneQuoteTests(unittest.TestCase): ARTHAG = { "name": "Arthag", "aliases": "der Zwerg, Zwerg, Schreiber", "mannerisms": "Hockt in Ecken und rechnet auf Wachstafeln.", "sources": [ {"quote": "»Was soll die Frage?« Nyrilla drehte sich zum Zwerg um und rief laut.", "line_hint": "action"}, {"quote": "Arthag wog prüfend seine Streitaxt.", "line_hint": "capabilities"}, ], } def test_quote_naming_the_character_beats_a_higher_scored_foreign_one(self): self.assertEqual(conv._character_scene_evidence(dict(self.ARTHAG)), "Arthag wog prüfend seine Streitaxt.") def test_only_foreign_quotes_fall_back_to_own_text(self): sheet = dict(self.ARTHAG, sources=[self.ARTHAG["sources"][0], {"quote": "Marcian: »Und wer meldet sich?«", "line_hint": "action"}]) self.assertEqual(conv._character_scene_evidence(sheet), sheet["mannerisms"]) def test_real_alias_and_genitive_count(self): sheet = {"name": "Zerwas", "aliases": "Murlok von Mengbilla, der Vampir", "sources": [{"quote": "Murlok von Mengbilla lächelte und hob den Becher.", "line_hint": "action"}]} self.assertIn("Murlok", conv._character_scene_evidence(sheet)) self.assertTrue(cr.quote_mentions_identity("Marrads Augen funkelten.", "Marrad", {})) self.assertFalse(cr.quote_mentions_identity("Der Vampir lächelte.", "Zerwas", sheet)) def test_name_argument_used_when_sheet_has_no_name(self): sheet = {k: v for k, v in self.ARTHAG.items() if k != "name"} self.assertEqual(conv._character_scene_evidence(sheet, "Arthag"), "Arthag wog prüfend seine Streitaxt.") class AgeCategoryTests(unittest.TestCase): MARRAD = "Junges Mädchen / Junger Mann (im Text als Junge beschrieben)" def test_marrad_german_english_and_quote(self): self.assertEqual(conv._character_age_category({"age_estimate": self.MARRAD}), "teen") self.assertEqual(conv._character_age_category( {"age_estimate": "Young girl / young man (described in the text as a boy)"}), "teen") self.assertEqual(conv._character_age_category({ "age_estimate": self.MARRAD, "sources": [{"quote": "Fast acht Sommer, Kommandant.", "line_hint": "age_estimate"}]}), "child") self.assertEqual(conv._age_body_lock({"age_estimate": self.MARRAD})[0], "EXACTLY ONE TEENAGE PERSON.") def test_adults_stay_adult(self): for value in ("Junger Mann", "young man", "Junge Frau", "Erwachsen", "jung", "Mittleres Alter (mindestens 15 Jahre älter als in seiner Akademiezeit)", "jung bis mittelalterlich (ca. 25-40 Jahre)"): self.assertEqual(conv._character_age_category({"age_estimate": value}), "adult", value) # A boy noun in prose may be someone else; only label fields count. self.assertEqual(conv._character_age_category({"physical": "trägt einen Jungen auf den Schultern"}), "adult") # A non-age source hint never feeds the age. self.assertEqual(conv._character_age_category({ "age_estimate": "Erwachsen", "sources": [{"quote": "Acht Sommer diente er.", "line_hint": "image"}]}), "adult") def test_number_words_and_seasons(self): self.assertEqual(conv._character_age_category({"age_estimate": "zwölfjährig"}), "child") self.assertEqual(conv._character_age_category({"age_estimate": "etwa fünfzehn Winter"}), "teen") self.assertEqual(conv._character_age_category({"age_estimate": "about eight summers"}), "child") self.assertEqual(conv._character_age_category({"age_estimate": "Mädchen"}), "teen") self.assertEqual(conv._character_age_category({"age_estimate": "girl"}), "teen") def _make_db(path: Path) -> None: con = sqlite3.connect(path) con.execute("CREATE TABLE characters (id TEXT PRIMARY KEY, book TEXT, name TEXT, tags TEXT, voice TEXT, " "image TEXT, sheet TEXT, analysis TEXT, created TEXT, updated TEXT)") book = "Buch 02" copied = {"race_species": "Ork (vermutlich, da im Lager und als Druide)", "relationships": "Uigar Kai (dient ihm, hält den Steigbügel)", "physical": "rothaarig, erfrorenes Gesicht", "sources": [{"page": 188, "quote": "Langsam richtete sich der Fremde aus dem Wald auf.", "line_hint": "physical"}]} rows = [ ("Kolon", {"aliases": "Sharraz Garthai, Sharraz, Zwerg", "first_name": "Sharraz", "last_name": "Garthai", "race_species": KOLON_SPECIES, "relationships": KOLON_RELATIONSHIPS, "sources": [{"quote": "Kolon brummte wütend vor sich hin.", "line_hint": "mannerisms"}]}), ("Arthag", {"aliases": "der Zwerg, Zwerg", "race_species": KOLON_SPECIES, "relationships": KOLON_RELATIONSHIPS, "mannerisms": "Spuckt angewidert aus.", "sources": [{"quote": "Arthag stand ratlos vor der Versammlung und rief laut.", "line_hint": "action"}]}), ("Marrad", {"sources": [{"quote": "Arthag stand ratlos vor der Versammlung und rief laut.", "line_hint": "action"}, {"quote": "Statt einer Antwort salutierte er.", "line_hint": "mannerisms"}]}), ("Sharraz", {"race_species": "Ork"}), ("Mann", dict(copied, aliases="Fremde, Fremder, Gutfolg")), ("Fremder", dict(copied, aliases="Mann, Fremde, Gutfolg")), ("Promos", {"aliases": "Alchimist", "physical": "alter Mann mit zitternden Händen und Brille"}), ("Alchimist", {"aliases": "Promos", "physical": "kleiner Alter im fleckigen Kittel"}), ] for name, sheet in rows: con.execute("INSERT INTO characters (id, book, name, sheet, created, updated) VALUES (?,?,?,?,?,?)", (f"{book}::{name}".lower(), book, name, json.dumps(dict(sheet, name=name)), "2026-07-17", "2026-07-17")) con.commit() con.close() def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() class AuditAndCleanupScriptTests(unittest.TestCase): def setUp(self): self.tmp = Path(tempfile.mkdtemp(prefix="ttvc-hygiene-db-")) self.db = self.tmp / "chars.db" _make_db(self.db) def test_audit_reports_and_never_writes(self): before = _sha(self.db) result = audit_script.audit(audit_script.load_records(str(self.db))) self.assertEqual(_sha(self.db), before) summary, issues = result["summary"], result["issues"] pairs = {frozenset((d["a"], d["b"])) for d in issues["duplicated_sheets"]} self.assertIn(frozenset(("Kolon", "Arthag")), pairs) self.assertIn(frozenset(("Mann", "Fremder")), pairs) self.assertIn(("Kolon", "Sharraz"), {(c["name"], c["value"]) for c in issues["alias_collisions"]}) self.assertEqual({x["name"] for x in issues["thinking_leakage"]}, {"Kolon", "Arthag"}) self.assertIn("Marrad", {x["name"] for x in issues["foreign_sources"]}) self.assertEqual(summary["scene_quote_about_other"], 0) # the fixed picker skips them self.assertGreaterEqual(summary["generic_alias_links"], 1) def test_cleanup_dry_run_writes_report_only(self): before = _sha(self.db) out = self.tmp / "dry" self.assertEqual(clean_script.main([str(self.db), "--out", str(out)]), 0) self.assertEqual(_sha(self.db), before) report = json.loads((out / "report.json").read_text(encoding="utf-8")) self.assertFalse(report["applied"]) self.assertFalse((out / "backup.json").exists()) kolon = next(p for p in report["plans"] if p["name"] == "Kolon") kinds = {(c["kind"], c["field"]) for c in kolon["changes"]} self.assertIn(("thinking_leakage", "race_species"), kinds) self.assertIn(("foreign_identity", "aliases"), kinds) promos = [p for p in report["plans"] if p["name"] == "Promos"] self.assertTrue(all(c["kind"] == "review_identity" for p in promos for c in p["changes"])) def test_apply_needs_out_and_writes_backup_first(self): self.assertEqual(clean_script.main([str(self.db), "--apply"]), 2) out = self.tmp / "applied" self.assertEqual(clean_script.main([str(self.db), "--apply", "--out", str(out)]), 0) backup = json.loads((out / "backup.json").read_text(encoding="utf-8")) self.assertIn("Kolon", {r["name"] for r in backup}) con = sqlite3.connect(self.db) sheets = {n: json.loads(s) for n, s in con.execute("SELECT name, sheet FROM characters")} con.close() self.assertEqual(sheets["Kolon"]["race_species"], "Mensch") self.assertNotIn("Sharraz", sheets["Kolon"]["aliases"]) self.assertEqual(sheets["Promos"]["aliases"], "Alchimist") # review-only, untouched self.assertEqual(sheets["Mann"], json.loads(json.dumps(sheets["Mann"]))) self.assertEqual(len(sheets["Marrad"]["sources"]), 2) # sources only with the opt-in flag if __name__ == "__main__": unittest.main()