From 737687789fde0bcc9e983130d9e28070d46aa1a2 Mon Sep 17 00:00:00 2001
From: mARTin-B78 <mail@martin-bierschenk.de>
Date: Wed, 12 Aug 2026 11:09:38 +0200
Subject: [PATCH] Add cast comparison tool for validating automated casting
 against a manual one

Matches segments by normalised text rather than index (a recast splits and
merges segments, so indices never align) and reports where the automation
disagrees with a hand-corrected cast: lines left Unknown that the human
resolved, lines given a different speaker, and narration/dialogue type
disagreements. Intended for using a manually-optimised book as ground truth to
drive further attribution-rule work.

Known limitation: repeated identical lines match the first occurrence, so
duplicates can mispair.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 scripts/compare_casts.py | 86 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100755 scripts/compare_casts.py

diff --git a/scripts/compare_casts.py b/scripts/compare_casts.py
new file mode 100755
index 0000000..5707db5
--- /dev/null
+++ b/scripts/compare_casts.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Compare an automated cast against a manually-corrected one (ground truth).
+
+  python3 scripts/compare_casts.py <manual.json> <auto.json>
+
+Matches segments by normalised text rather than index (a recast splits and
+merges segments, so indices never line up), then reports where the automation
+disagrees with the human — which is the input for improving the rules.
+"""
+import json, re, sys, collections
+
+def norm(t):
+    t = re.sub(r'[»«„“”"\'`]', '', str(t or ''))
+    return re.sub(r'\s+', ' ', t).strip().lower()
+
+def load(p):
+    d = json.load(open(p, encoding='utf-8'))
+    return d.get('segments') or []
+
+def is_unknown(s):
+    sp = str(s.get('speaker') or '')
+    return s.get('type') == 'dialogue' and (not sp or sp.lower().startswith(('unknown', 'unbekannt')))
+
+def summarise(segs, label):
+    dia = [s for s in segs if s.get('type') == 'dialogue']
+    fused = [s for s in dia if re.search(r'\n\s*\n', (s.get('text') or '').strip())]
+    print(f"{label:10s} segments={len(segs):5d} dialogue={len(dia):5d} "
+          f"unknown={sum(1 for s in segs if is_unknown(s)):4d} fused={len(fused):3d} "
+          f"speakers={len({s.get('speaker') for s in dia if s.get('speaker')}):3d}")
+
+def main(man_path, auto_path):
+    man, auto = load(man_path), load(auto_path)
+    print("=" * 78)
+    summarise(man, "MANUAL"); summarise(auto, "AUTO")
+    print("=" * 78)
+
+    # index the manual cast by normalised text
+    mi = collections.defaultdict(list)
+    for i, s in enumerate(man):
+        k = norm(s.get('text'))
+        if k: mi[k].append(i)
+
+    agree = disagree = only_auto = 0
+    unknown_in_auto_known_in_man = []
+    wrong_speaker = []
+    type_diff = []
+    for s in auto:
+        k = norm(s.get('text'))
+        if not k: continue
+        hits = mi.get(k)
+        if not hits:
+            only_auto += 1
+            continue
+        m = man[hits[0]]
+        if m.get('type') != s.get('type'):
+            type_diff.append((s.get('text', '')[:60], m.get('type'), s.get('type')))
+        ms, as_ = str(m.get('speaker') or ''), str(s.get('speaker') or '')
+        if ms == as_:
+            agree += 1
+        else:
+            disagree += 1
+            if is_unknown(s) and not is_unknown(m):
+                unknown_in_auto_known_in_man.append((s.get('text', '')[:60], ms))
+            elif not is_unknown(s) and not is_unknown(m) and m.get('type') == 'dialogue':
+                wrong_speaker.append((s.get('text', '')[:60], ms, as_))
+
+    tot = agree + disagree
+    print(f"\nmatched segments: {tot}  agree={agree} ({100*agree//max(tot,1)}%)  disagree={disagree}")
+    print(f"segments only in AUTO (split/reworded): {only_auto}")
+
+    print(f"\n--- AUTO left Unknown where the human knew the speaker ({len(unknown_in_auto_known_in_man)}) ---")
+    for t, who in unknown_in_auto_known_in_man[:25]:
+        print(f"  should be {who:18s} | {t!r}")
+
+    print(f"\n--- AUTO picked a DIFFERENT speaker than the human ({len(wrong_speaker)}) ---")
+    for t, who, got in wrong_speaker[:25]:
+        print(f"  human={who:16s} auto={got:16s} | {t!r}")
+
+    print(f"\n--- type disagreements narration vs dialogue ({len(type_diff)}) ---")
+    for t, mt, at in type_diff[:20]:
+        print(f"  human={mt:9s} auto={at:9s} | {t!r}")
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print(__doc__); sys.exit(1)
+    main(sys.argv[1], sys.argv[2])