tts-voice-creator-clone-and-design-2/core/character_records.py

"""Identity and hygiene rules for extracted character records.

Pure functions (no I/O) shared by the character-sheet extraction parser
(routes/conversation.py), the scene-quote picker for image prompts, and the
read-only audit / dry-run cleanup scripts in scripts/.

Why these exist (all found on the real library, 2026-10-03, see
architecture/character-extraction.md):

* Two records were merged, or one record's sheet was copied into another,
  because they shared ONE identity token such as "Zwerg", "der Zwerg", "Mann"
  or "Fremder". Those are common nouns, roles and titles, not names: the book
  calls many people "der Zwerg". Only a non-generic token is evidence that two
  sheets describe the same person.
* The LLM put other cast members' names into a character's aliases (Kolon in
  book 02 carried "Sharraz Garthai", "Sharraz" and "Kriegshäuptling", all
  separate cast records).
* Reasoning text leaked into descriptive fields ("Warte: ...", "Nein, Kolon
  ist ...", "Der Text sagt ...").
* The story-moment quote was picked by line_hint/verb score only, so a quote
  about another cast member could become this character's lower panel.

The generic-word list is mirrored in static/js/character-identity.js; a unit
test keeps the two in sync.
"""
from __future__ import annotations

import re
from typing import Iterable

# Articles / determiners that make an alias a description rather than a name
# ("der Zwerg", "der Tunneltreiber", "the Executioner").
ARTICLES = frozenset((
    "der die das den dem des ein eine einer eines einem einen "
    "the a an"
).split())

# Common nouns, roles, ranks, kinship words, species and age/look adjectives.
# A token made only of these words (after articles) is generic: it can label
# many different people and is never evidence of shared identity.
GENERIC_IDENTITY_WORDS = frozenset((
    # people, age, kinship (de)
    "mann frau junge mädchen kind knabe bub greis greisin alte alter altes alten "
    "jung junger junges jungen menge männer frauen leute person gestalt wesen gast "
    "vater mutter sohn tochter bruder schwester onkel tante witwe witwer "
    # species / kinds (de)
    "mensch menschen zwerg zwergin zwerge elf elfe elfen auelfe ork orks orkin "
    "vampir vampirin dämon monster kreatur "
    # roles, ranks, professions (de)
    "krieger kriegerin wache wachen wächter reiter schreiber adept adepten "
    "zauberer zauberin magier magierin hexe inquisitor schmied oberst obrist "
    "general offizier offizierin hauptmann kommandant kommandantin kapitän "
    "admiral großadmiral marschall fremde fremder fremden stimme stimmen sklave "
    "sklavin sklaven nachbar nachbarin bote botin soldat soldatin soldaten "
    "ritter ritterin prinz prinzessin könig königin kaiser kaiserin graf gräfin "
    "baron baronin herzog herzogin junker herr herrin dame diener dienerin magd "
    "knecht händler kaufmann wirt wirtin bauer bäuerin fischer jäger priester "
    "priesterin geweihter mönch nonne schamane druide amazone henker veteran "
    "navigator begleiter begleiterin gefährte gefährtin anführer anführerin "
    "häuptling kriegshäuptling führer militärführer meister lehrer schüler "
    "bürger bürgerin bürgersohn weber kutscher bäckerstochter erzähler "
    "torwächter feldherr heerführer befehlshaber hauptfrau "
    # descriptive adjectives that often head a role label (de)
    "erster erste zweiter zweite dritter dritte von van zu vom zum zur de del "
    "vermummt vermummte vermummter vermummten groß große großer kleine kleiner "
    "dunkle dunkler blonde blonder rote roter tote toter schwarzhaarige "
    "unbekannt unbekannte unbekannter irre unsicher "
    # English
    "man woman boy girl child kid stranger old young person figure creature "
    "father mother son daughter brother sister "
    "human dwarf elf orc vampire demon monster "
    "guard soldier warrior knight officer general colonel captain commander "
    "admiral marshal prince princess king queen lord lady sir master servant "
    "merchant smith blacksmith priest mage wizard witch shaman druid "
    "executioner veteran navigator companion leader chief voice crowd "
    "messenger scribe inquisitor narrator unknown"
).split())

# A compound ending in one of these (length >= 5) is still a role label:
# "Bürgerwehroffizier", "Kriegshäuptling", "Ork-General" (split on '-').
_GENERIC_SUFFIXES = tuple(sorted((w for w in GENERIC_IDENTITY_WORDS if len(w) >= 5), key=len, reverse=True))

_ALIAS_SPLIT_RE = re.compile(
    r"[,;/|]|\baka\b|\baka\.\b|\balias(?:es)?\b|\bgenannt\b|\bnamens\b|\bcalled\b|\bknown as\b",
    re.IGNORECASE,
)
_WORD_RE = re.compile(r"[\w'’]+", re.UNICODE)


def normalize_identity_token(token: str) -> str:
    """Lower-case, drop parentheticals and surrounding punctuation."""
    text = re.sub(r"\([^)]*\)", " ", str(token or ""))
    text = re.sub(r"\s+", " ", text).strip(" \t\n.,;:!?\"'»«„“”‚‘’").lower()
    return text


def split_identity_tokens(value) -> list[str]:
    """Split an aliases/name value into individual name strings."""
    if value is None:
        return []
    if isinstance(value, (list, tuple)):
        value = ", ".join(str(v) for v in value if v)
    parts = [p.strip() for p in _ALIAS_SPLIT_RE.split(str(value))]
    return [p for p in parts if p and len(p) <= 80 and not re.match(r"^(needs?:|complete$)", p, re.I)]


def is_generic_identity_token(token: str) -> bool:
    """True when ``token`` is a common noun / role / title, not a name.

    "Zwerg", "der Zwerg", "Alter Soldat", "Ork-General", "Bürgerwehroffizier",
    "der Tunneltreiber" (article + description) are generic; "Arthag",
    "Sharraz Garthai", "Admiral Sanin" (contains a name) are not.
    """
    norm = normalize_identity_token(token)
    if not norm:
        return True
    words = [w for w in re.split(r"[\s\-]+", norm) if w]
    if not words:
        return True
    if words[0] in ARTICLES:
        # "der Henker", "der Mann mit dem roten Umhang": a description.
        return True
    for word in words:
        bare = word.strip("'’.")
        if len(bare) <= 1 or bare in ARTICLES or bare in GENERIC_IDENTITY_WORDS:
            continue
        if any(bare.endswith(suffix) and len(bare) > len(suffix) for suffix in _GENERIC_SUFFIXES):
            continue
        return False
    return True


def identity_tokens(name: str, sheet: dict | None, include_generic: bool = False) -> set[str]:
    """Normalized identity tokens of one character: name, real names, aliases."""
    sheet = sheet or {}
    out: set[str] = set()
    values = [name, sheet.get("first_name"), sheet.get("last_name"), sheet.get("full_name")]
    first, last = str(sheet.get("first_name") or "").strip(), str(sheet.get("last_name") or "").strip()
    if first and last:
        values.append(f"{first} {last}")
    for value in values:
        for part in split_identity_tokens(value):
            norm = normalize_identity_token(part)
            if norm and (include_generic or not is_generic_identity_token(part)):
                out.add(norm)
    for part in split_identity_tokens(sheet.get("aliases")):
        norm = normalize_identity_token(part)
        if norm and (include_generic or not is_generic_identity_token(part)):
            out.add(norm)
    own = normalize_identity_token(name)
    if own:
        out.add(own)   # the record's own name always identifies it, even "Fremder"
    return out


def shared_identity_evidence(a_tokens: Iterable[str], b_tokens: Iterable[str]) -> set[str]:
    """Shared tokens that actually prove identity (generic ones never do)."""
    a = {normalize_identity_token(t) for t in a_tokens}
    b = {normalize_identity_token(t) for t in b_tokens}
    return {t for t in a & b if t and not is_generic_identity_token(t)}


def text_mentions_token(text: str, token: str) -> bool:
    """Whole-word, case-insensitive mention; a German genitive -s counts."""
    token = normalize_identity_token(token)
    if len(token) < 2:
        return False
    pattern = r"(?<![\w])" + re.escape(token) + r"(?:s|'s|’s)?(?![\w])"
    return bool(re.search(pattern, str(text or ""), re.IGNORECASE))


def quote_mentions_identity(quote: str, name: str, sheet: dict | None) -> bool:
    """True when the quote names this character by name or a non-generic alias."""
    return any(text_mentions_token(quote, token) for token in identity_tokens(name, sheet))


def quote_mentions_other_cast(quote: str, own_name: str, own_sheet: dict | None,
                              cast: Iterable[tuple[str, dict]]) -> list[str]:
    """Names of OTHER cast members (non-generic names only) the quote mentions."""
    own = identity_tokens(own_name, own_sheet)
    hits = []
    for other_name, _other_sheet in cast:
        if normalize_identity_token(other_name) in own or is_generic_identity_token(other_name):
            continue
        if text_mentions_token(quote, other_name):
            hits.append(other_name)
    return hits


# ── Reasoning ("thinking") leakage ─────────────────────────────────────────

# Phrases that only appear when the model narrates its own deliberation inside
# a field value. Calibrated on the real library: every hit there was leakage,
# and none of these occur in legitimate descriptive text of the 270 records.
_THINKING_MARKER_RE = re.compile(
    r"<think>|</think>"
    r"|\b(?:aber\s+)?(?:warte|wait)\b\s*[:,.!]"
    r"|\bhmm+\b"
    r"|\bmoment mal\b"
    r"|\blass mich\b|\blet me\b"
    r"|\bes ist mehrdeutig\b|\bit(?:'s| is) ambiguous\b"
    r"|\b(?:der\s+)?text sagt\b|\bthe text says\b"
    r"|\bnein,\s+[A-ZÄÖÜ][\w-]*\s+ist\b|\bno,\s+[A-Z][\w-]*\s+is\b"
    r"|\bich werde (?:mich|fokussieren|annehmen)\b|\bfokussiere ich\b"
    r"|\b(?:korrektur|correction)\s*:",
    re.IGNORECASE,
)
# Field values the model writes as prose about the character. Identity/name
# fields are included: leakage there is just as wrong.
TEXT_FIELDS = (
    "aliases", "first_name", "last_name", "full_name", "title", "profession",
    "age_estimate", "race_species", "languages", "nationality_background",
    "social_class", "archetype", "physical", "clothing", "alignment", "arc_note",
    "attribute_high", "attribute_low", "skills", "capabilities", "backstory",
    "relationships", "motivation", "fears", "mannerisms", "communication_style",
    "voice_pattern", "secret", "conflict_style", "win_condition", "reputation",
    "religious_beliefs", "notes",
)


def has_thinking_leakage(value) -> bool:
    return isinstance(value, str) and bool(_THINKING_MARKER_RE.search(value))


def strip_thinking_leakage(value: str) -> str:
    """Cut a field value before the first deliberation marker.

    The cut backs up to the start of the clause that holds the marker (the
    last sentence end, ';' or unclosed '(' before it), so "Mensch (oder
    Ork-Verbündeter, ... Text sagt ...)" keeps only "Mensch". Text without a
    marker is returned unchanged.
    """
    if not isinstance(value, str):
        return value
    text = re.sub(r"<think>.*?</think>", "", value, flags=re.DOTALL | re.IGNORECASE)
    match = _THINKING_MARKER_RE.search(text)
    if not match:
        return text.strip() if text != value else value
    head = text[:match.start()]
    # Back up to the start of the clause holding the marker.
    depth, cut = 0, 0
    for i, ch in enumerate(head):
        if ch == "(":
            depth += 1
            if depth == 1:
                open_at = i
        elif ch == ")":
            depth = max(0, depth - 1)
    if depth > 0:
        cut = open_at          # the marker sits inside an unclosed parenthetical
        head = head[:cut]
    else:
        ends = [m.end() for m in re.finditer(r"[.;!?](?:\s|$)", head)]
        head = head[:ends[-1]] if ends else ""
    return head.strip().rstrip(",;:-–— ").strip()


def sanitize_sheet_text_fields(sheet: dict) -> dict[str, tuple[str, str]]:
    """Strip reasoning leakage from ``sheet`` in place; return {field: (old, new)}."""
    changes: dict[str, tuple[str, str]] = {}
    for field in TEXT_FIELDS:
        value = sheet.get(field)
        if not has_thinking_leakage(value):
            continue
        clean = strip_thinking_leakage(value)
        if clean != value:
            sheet[field] = clean
            changes[field] = (value, clean)
    return changes


def drop_foreign_identity(name: str, sheet: dict, cast_names: Iterable[str]) -> list[str]:
    """Remove aliases / real-name fields that are ANOTHER cast member's name.

    An alias that equals a different, separately cast character's name is the
    model mixing two people up in one passage (Kolon -> "Sharraz Garthai"),
    never a proven alternate name of this one. Mutates ``sheet``; returns the
    removed values (as "field=value").
    """
    own = normalize_identity_token(name)
    # Generic cast labels ("Zwerg", "Mann", "Fremder") are no identity claim:
    # Kolon may well be "der Zwerg". Only real names of others are removed.
    others = {normalize_identity_token(n) for n in cast_names if n and not is_generic_identity_token(n)}
    others.discard(own)
    others.discard("")
    if not others:
        return []
    # Real names of other cast members embedded in a longer alias
    # ("Sharraz Garthai" when "Sharraz" is cast, "Schlange Gamba"). Exact word
    # match only: "Darrags Sohn" (a kinship description) stays.
    named_others = [o for o in others if o not in own.split()]

    def foreign(value: str) -> bool:
        norm = normalize_identity_token(value)
        if not norm or norm == own:
            return False
        if norm in others:
            return True
        for other in named_others:
            match = re.search(r"(?<![\w])" + re.escape(other) + r"(?![\w])", norm)
            if not match:
                continue
            # Only a name-like alias ("Sharraz Garthai", "Schlange Gamba") is
            # another person's name. "Tochter des Promos", "Vater von Marrad",
            # "junger Gernot Brohm" describe a relation or a look: kept.
            original = re.sub(r"\([^)]*\)", " ", str(value))
            rest = re.sub(r"(?<![\w])" + re.escape(other) + r"(?![\w])", " ", original, flags=re.IGNORECASE)
            words = [w for w in re.split(r"[\s\-]+", rest) if w]
            if all(w[:1].isupper() and w.lower() not in GENERIC_IDENTITY_WORDS for w in words):
                return True
        return False

    removed: list[str] = []
    aliases = split_identity_tokens(sheet.get("aliases"))
    kept = []
    for alias in aliases:
        if foreign(alias):
            removed.append(f"aliases={alias}")
        else:
            kept.append(alias)
    if len(kept) != len(aliases):
        sheet["aliases"] = ", ".join(kept)
    first, last = str(sheet.get("first_name") or "").strip(), str(sheet.get("last_name") or "").strip()
    if first and last and foreign(f"{first} {last}"):
        removed += [f"first_name={first}", f"last_name={last}"]
        sheet["first_name"] = sheet["last_name"] = ""
    for field in ("first_name", "last_name", "full_name"):
        value = str(sheet.get(field) or "").strip()
        if value and foreign(value):
            removed.append(f"{field}={value}")
            sheet[field] = ""
    return removed


def scene_sources_about_character(name: str, sheet: dict) -> list[dict]:
    """Source entries whose quote names this character (name or real alias)."""
    sources = sheet.get("sources") if isinstance(sheet.get("sources"), list) else []
    return [s for s in sources if isinstance(s, dict)
            and quote_mentions_identity(str(s.get("quote") or ""), name, sheet)]