tts-voice-creator-clone-and-design-2/tests/test_character_record_hygiene.py

"""Character extraction / merge hygiene (real-library audit, 2026-10-03).

Fixtures are shaped like the broken real records (see
architecture/character-extraction.md):
* Kolon (book 02): race_species with the model's reasoning ("Warte: ...",
  "Nein, Kolon ist ...") and aliases/first/last naming Sharraz (a separate
  cast record).
* Arthag (book 03): a lower-panel quote about Nyrilla picked over Arthag's own.
* Mann / Fremder (book 03): one sheet stored in two records.
* Marrad (book 02): a boy whose hedged age estimate rendered as an adult.
"""
from __future__ import annotations

import hashlib
import json
import os
import re
import sqlite3
import tempfile
import unittest
from pathlib import Path

os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-hygiene-test-config-"))

import routes.conversation as conv  # noqa: E402
from core import character_records as cr  # noqa: E402
from scripts import audit_character_records as audit_script  # noqa: E402
from scripts import clean_character_records as clean_script  # noqa: E402

ROOT = Path(__file__).resolve().parent.parent

KOLON_SPECIES = (
    "Mensch (oder Ork-Verbündeter, Kontext unklar, aber 'Der Zwerg' wird separat erwähnt, Kolon scheint "
    "der strategische Kopf zu sein - Text sagt 'Der Zwerg lächelte grimmig' im selben Absatz wie Kolon. "
    "Warte: 'Der Zwerg lächelte grimmig ... Es wäre allein sein Verdienst ...'. Es ist mehrdeutig. "
    "Nein, Kolon ist in der Liste der bekannten Charaktere.)"
)
KOLON_RELATIONSHIPS = "Führt die Orks; dient oder steht im Dienst von Sharraz Garthai; arbeitet mit Drasdech."


class GenericTokenTests(unittest.TestCase):
    def test_labels_are_generic_names_are_not(self):
        for token in ("Zwerg", "der Zwerg", "Mann", "Fremder", "Fremde", "Alter Soldat", "Ork-General",
                      "Bürgerwehroffizier", "der Tunneltreiber", "Kriegshäuptling", "Erster Offizier", "Von"):
            self.assertTrue(cr.is_generic_identity_token(token), token)
        for token in ("Arthag", "Kolon", "Sharraz Garthai", "Admiral Sanin", "Schlange Gamba", "Gutfolg"):
            self.assertFalse(cr.is_generic_identity_token(token), token)

    def test_shared_generic_alias_is_no_identity_evidence(self):
        arthag = cr.identity_tokens("Arthag", {"aliases": "der Zwerg, Zwerg, Schreiber"}, include_generic=True)
        kolon = cr.identity_tokens("Kolon", {"aliases": "Zwerg, der Zwerg"}, include_generic=True)
        self.assertTrue(arthag & kolon)
        self.assertEqual(cr.shared_identity_evidence(arthag, kolon), set())

    def test_js_word_list_matches_python(self):
        js = (ROOT / "static" / "js" / "character-identity.js").read_text(encoding="utf-8")
        block = re.search(r"const GENERIC_WORDS = new Set\(\((.*?)\)\.split", js, re.S).group(1)
        words = set(" ".join(re.findall(r"'([^']*)'", block)).split())
        self.assertEqual(words, set(cr.GENERIC_IDENTITY_WORDS))
        arts = re.search(r"const ARTICLES = new Set\(\('([^']*)'\)", js).group(1).split()
        self.assertEqual(set(arts), set(cr.ARTICLES))


class ThinkingLeakageTests(unittest.TestCase):
    def test_real_leak_shapes_are_cut(self):
        self.assertEqual(cr.strip_thinking_leakage(KOLON_SPECIES), "Mensch")
        self.assertEqual(cr.strip_thinking_leakage(
            "Prinz des Reiches, begleitet von Alrik (oder ist Alrik selbst der Prinz? Der Text sagt 'Alrik "
            "hatte ihr bis zuletzt nicht glauben wollen'. Es gibt Verwechslungsgefahr.)"),
            "Prinz des Reiches, begleitet von Alrik")
        self.assertEqual(cr.strip_thinking_leakage(
            "Schwarzhaarig. Groß. Wait, 'Die Frau bückte die Bannerträgerin' is likely a typo."),
            "Schwarzhaarig. Groß.")
        self.assertEqual(cr.strip_thinking_leakage("<think>Kolon or Arthag?</think>Zwerg"), "Zwerg")

    def test_clean_text_is_untouched(self):
        clean = ("Sie hat ihren Auftraggeber (Inquisition) verraten oder ist sich ihrer eigenen Gefühle "
                 "für Zerwas bewusst.")
        self.assertFalse(cr.has_thinking_leakage(clean))
        self.assertEqual(cr.strip_thinking_leakage(clean), clean)
        self.assertEqual(cr.strip_thinking_leakage(None), None)


class ParseIsolationTests(unittest.TestCase):
    RAW = json.dumps({"sheets": [{
        "name": "Kolon", "aliases": "Sharraz Garthai, General, Ork-General, Sharraz, Zwerg, Kriegshäuptling",
        "first_name": "Sharraz", "last_name": "Garthai", "race_species": KOLON_SPECIES,
        "relationships": KOLON_RELATIONSHIPS, "physical": "Kleiner Mann mit einem weißen Bart.",
        "sources": [{"page": 142, "quote": "Kolon kratzte sich mit dem Griffel am Hinterkopf.", "line_hint": "mannerisms"}],
    }, {
        "name": "Marina", "aliases": "Tochter des Promos, das Mädchen", "race_species": "Mensch",
    }]})
    ROSTER = ["Kolon", "Arthag", "Sharraz", "Zwerg", "Kriegshäuptling", "Marina", "Promos"]

    def test_roster_names_never_become_aliases_and_leakage_is_stripped(self):
        out = conv._charsheets_parse(self.RAW, self.ROSTER)
        kolon = next(s for s in out["sheets"] if s["name"] == "Kolon")
        aliases = [a.strip() for a in kolon["aliases"].split(",")]
        self.assertNotIn("Sharraz", aliases)
        self.assertNotIn("Sharraz Garthai", aliases)
        self.assertIn("Ork-General", aliases)          # a generic title is data, not a claim
        self.assertEqual((kolon["first_name"], kolon["last_name"]), ("", ""))
        self.assertEqual(kolon["race_species"], "Mensch")
        self.assertEqual(kolon["relationships"], KOLON_RELATIONSHIPS)
        marina = next(s for s in out["sheets"] if s["name"] == "Marina")
        self.assertIn("Tochter des Promos", marina["aliases"])   # a relation, not a name

    def test_without_roster_aliases_are_kept_but_leakage_still_stripped(self):
        out = conv._charsheets_parse(self.RAW)
        kolon = out["sheets"][0]
        self.assertIn("Sharraz", kolon["aliases"])
        self.assertEqual(kolon["race_species"], "Mensch")

    def test_roster_only_passed_in_cast_target_mode(self):
        base = {"text": "Kolon sprach.", "known_characters": ["Kolon", "Sharraz"], "llm_url": "http://x/v1"}
        self.assertEqual(conv._charsheets_prepare(dict(base, target_mode=True))["roster"], ["Kolon", "Sharraz"])
        self.assertEqual(conv._charsheets_prepare(dict(base, target_mode=False))["roster"], [])


class SceneQuoteTests(unittest.TestCase):
    ARTHAG = {
        "name": "Arthag", "aliases": "der Zwerg, Zwerg, Schreiber",
        "mannerisms": "Hockt in Ecken und rechnet auf Wachstafeln.",
        "sources": [
            {"quote": "»Was soll die Frage?« Nyrilla drehte sich zum Zwerg um und rief laut.", "line_hint": "action"},
            {"quote": "Arthag wog prüfend seine Streitaxt.", "line_hint": "capabilities"},
        ],
    }

    def test_quote_naming_the_character_beats_a_higher_scored_foreign_one(self):
        self.assertEqual(conv._character_scene_evidence(dict(self.ARTHAG)), "Arthag wog prüfend seine Streitaxt.")

    def test_only_foreign_quotes_fall_back_to_own_text(self):
        sheet = dict(self.ARTHAG, sources=[self.ARTHAG["sources"][0],
                                           {"quote": "Marcian: »Und wer meldet sich?«", "line_hint": "action"}])
        self.assertEqual(conv._character_scene_evidence(sheet), sheet["mannerisms"])

    def test_real_alias_and_genitive_count(self):
        sheet = {"name": "Zerwas", "aliases": "Murlok von Mengbilla, der Vampir",
                 "sources": [{"quote": "Murlok von Mengbilla lächelte und hob den Becher.", "line_hint": "action"}]}
        self.assertIn("Murlok", conv._character_scene_evidence(sheet))
        self.assertTrue(cr.quote_mentions_identity("Marrads Augen funkelten.", "Marrad", {}))
        self.assertFalse(cr.quote_mentions_identity("Der Vampir lächelte.", "Zerwas", sheet))

    def test_name_argument_used_when_sheet_has_no_name(self):
        sheet = {k: v for k, v in self.ARTHAG.items() if k != "name"}
        self.assertEqual(conv._character_scene_evidence(sheet, "Arthag"), "Arthag wog prüfend seine Streitaxt.")


class AgeCategoryTests(unittest.TestCase):
    MARRAD = "Junges Mädchen / Junger Mann (im Text als Junge beschrieben)"

    def test_marrad_german_english_and_quote(self):
        self.assertEqual(conv._character_age_category({"age_estimate": self.MARRAD}), "teen")
        self.assertEqual(conv._character_age_category(
            {"age_estimate": "Young girl / young man (described in the text as a boy)"}), "teen")
        self.assertEqual(conv._character_age_category({
            "age_estimate": self.MARRAD,
            "sources": [{"quote": "Fast acht Sommer, Kommandant.", "line_hint": "age_estimate"}]}), "child")
        self.assertEqual(conv._age_body_lock({"age_estimate": self.MARRAD})[0], "EXACTLY ONE TEENAGE PERSON.")

    def test_adults_stay_adult(self):
        for value in ("Junger Mann", "young man", "Junge Frau", "Erwachsen", "jung",
                      "Mittleres Alter (mindestens 15 Jahre älter als in seiner Akademiezeit)",
                      "jung bis mittelalterlich (ca. 25-40 Jahre)"):
            self.assertEqual(conv._character_age_category({"age_estimate": value}), "adult", value)
        # A boy noun in prose may be someone else; only label fields count.
        self.assertEqual(conv._character_age_category({"physical": "trägt einen Jungen auf den Schultern"}), "adult")
        # A non-age source hint never feeds the age.
        self.assertEqual(conv._character_age_category({
            "age_estimate": "Erwachsen", "sources": [{"quote": "Acht Sommer diente er.", "line_hint": "image"}]}),
            "adult")

    def test_number_words_and_seasons(self):
        self.assertEqual(conv._character_age_category({"age_estimate": "zwölfjährig"}), "child")
        self.assertEqual(conv._character_age_category({"age_estimate": "etwa fünfzehn Winter"}), "teen")
        self.assertEqual(conv._character_age_category({"age_estimate": "about eight summers"}), "child")
        self.assertEqual(conv._character_age_category({"age_estimate": "Mädchen"}), "teen")
        self.assertEqual(conv._character_age_category({"age_estimate": "girl"}), "teen")


def _make_db(path: Path) -> None:
    con = sqlite3.connect(path)
    con.execute("CREATE TABLE characters (id TEXT PRIMARY KEY, book TEXT, name TEXT, tags TEXT, voice TEXT, "
                "image TEXT, sheet TEXT, analysis TEXT, created TEXT, updated TEXT)")
    book = "Buch 02"
    copied = {"race_species": "Ork (vermutlich, da im Lager und als Druide)",
              "relationships": "Uigar Kai (dient ihm, hält den Steigbügel)", "physical": "rothaarig, erfrorenes Gesicht",
              "sources": [{"page": 188, "quote": "Langsam richtete sich der Fremde aus dem Wald auf.", "line_hint": "physical"}]}
    rows = [
        ("Kolon", {"aliases": "Sharraz Garthai, Sharraz, Zwerg", "first_name": "Sharraz", "last_name": "Garthai",
                   "race_species": KOLON_SPECIES, "relationships": KOLON_RELATIONSHIPS,
                   "sources": [{"quote": "Kolon brummte wütend vor sich hin.", "line_hint": "mannerisms"}]}),
        ("Arthag", {"aliases": "der Zwerg, Zwerg", "race_species": KOLON_SPECIES, "relationships": KOLON_RELATIONSHIPS,
                    "mannerisms": "Spuckt angewidert aus.",
                    "sources": [{"quote": "Arthag stand ratlos vor der Versammlung und rief laut.", "line_hint": "action"}]}),
        ("Marrad", {"sources": [{"quote": "Arthag stand ratlos vor der Versammlung und rief laut.", "line_hint": "action"},
                                {"quote": "Statt einer Antwort salutierte er.", "line_hint": "mannerisms"}]}),
        ("Sharraz", {"race_species": "Ork"}),
        ("Mann", dict(copied, aliases="Fremde, Fremder, Gutfolg")),
        ("Fremder", dict(copied, aliases="Mann, Fremde, Gutfolg")),
        ("Promos", {"aliases": "Alchimist", "physical": "alter Mann mit zitternden Händen und Brille"}),
        ("Alchimist", {"aliases": "Promos", "physical": "kleiner Alter im fleckigen Kittel"}),
    ]
    for name, sheet in rows:
        con.execute("INSERT INTO characters (id, book, name, sheet, created, updated) VALUES (?,?,?,?,?,?)",
                    (f"{book}::{name}".lower(), book, name, json.dumps(dict(sheet, name=name)), "2026-07-17", "2026-07-17"))
    con.commit()
    con.close()


def _sha(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


class AuditAndCleanupScriptTests(unittest.TestCase):
    def setUp(self):
        self.tmp = Path(tempfile.mkdtemp(prefix="ttvc-hygiene-db-"))
        self.db = self.tmp / "chars.db"
        _make_db(self.db)

    def test_audit_reports_and_never_writes(self):
        before = _sha(self.db)
        result = audit_script.audit(audit_script.load_records(str(self.db)))
        self.assertEqual(_sha(self.db), before)
        summary, issues = result["summary"], result["issues"]
        pairs = {frozenset((d["a"], d["b"])) for d in issues["duplicated_sheets"]}
        self.assertIn(frozenset(("Kolon", "Arthag")), pairs)
        self.assertIn(frozenset(("Mann", "Fremder")), pairs)
        self.assertIn(("Kolon", "Sharraz"), {(c["name"], c["value"]) for c in issues["alias_collisions"]})
        self.assertEqual({x["name"] for x in issues["thinking_leakage"]}, {"Kolon", "Arthag"})
        self.assertIn("Marrad", {x["name"] for x in issues["foreign_sources"]})
        self.assertEqual(summary["scene_quote_about_other"], 0)   # the fixed picker skips them
        self.assertGreaterEqual(summary["generic_alias_links"], 1)

    def test_cleanup_dry_run_writes_report_only(self):
        before = _sha(self.db)
        out = self.tmp / "dry"
        self.assertEqual(clean_script.main([str(self.db), "--out", str(out)]), 0)
        self.assertEqual(_sha(self.db), before)
        report = json.loads((out / "report.json").read_text(encoding="utf-8"))
        self.assertFalse(report["applied"])
        self.assertFalse((out / "backup.json").exists())
        kolon = next(p for p in report["plans"] if p["name"] == "Kolon")
        kinds = {(c["kind"], c["field"]) for c in kolon["changes"]}
        self.assertIn(("thinking_leakage", "race_species"), kinds)
        self.assertIn(("foreign_identity", "aliases"), kinds)
        promos = [p for p in report["plans"] if p["name"] == "Promos"]
        self.assertTrue(all(c["kind"] == "review_identity" for p in promos for c in p["changes"]))

    def test_apply_needs_out_and_writes_backup_first(self):
        self.assertEqual(clean_script.main([str(self.db), "--apply"]), 2)
        out = self.tmp / "applied"
        self.assertEqual(clean_script.main([str(self.db), "--apply", "--out", str(out)]), 0)
        backup = json.loads((out / "backup.json").read_text(encoding="utf-8"))
        self.assertIn("Kolon", {r["name"] for r in backup})
        con = sqlite3.connect(self.db)
        sheets = {n: json.loads(s) for n, s in con.execute("SELECT name, sheet FROM characters")}
        con.close()
        self.assertEqual(sheets["Kolon"]["race_species"], "Mensch")
        self.assertNotIn("Sharraz", sheets["Kolon"]["aliases"])
        self.assertEqual(sheets["Promos"]["aliases"], "Alchimist")      # review-only, untouched
        self.assertEqual(sheets["Mann"], json.loads(json.dumps(sheets["Mann"])))
        self.assertEqual(len(sheets["Marrad"]["sources"]), 2)            # sources only with the opt-in flag


if __name__ == "__main__":
    unittest.main()