mARTin-B78
737687789f
Matches segments by normalised text rather than index (a recast splits and merges segments, so indices never align) and reports where the automation disagrees with a hand-corrected cast: lines left Unknown that the human resolved, lines given a different speaker, and narration/dialogue type disagreements. Intended for using a manually-optimised book as ground truth to drive further attribution-rule work. Known limitation: repeated identical lines match the first occurrence, so duplicates can mispair. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
87 lines
3.3 KiB
Python
Executable File
87 lines
3.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Compare an automated cast against a manually-corrected one (ground truth).
|
|
|
|
python3 scripts/compare_casts.py <manual.json> <auto.json>
|
|
|
|
Matches segments by normalised text rather than index (a recast splits and
|
|
merges segments, so indices never line up), then reports where the automation
|
|
disagrees with the human — which is the input for improving the rules.
|
|
"""
|
|
import json, re, sys, collections
|
|
|
|
def norm(t):
|
|
t = re.sub(r'[»«„“”"\'`]', '', str(t or ''))
|
|
return re.sub(r'\s+', ' ', t).strip().lower()
|
|
|
|
def load(p):
|
|
d = json.load(open(p, encoding='utf-8'))
|
|
return d.get('segments') or []
|
|
|
|
def is_unknown(s):
|
|
sp = str(s.get('speaker') or '')
|
|
return s.get('type') == 'dialogue' and (not sp or sp.lower().startswith(('unknown', 'unbekannt')))
|
|
|
|
def summarise(segs, label):
|
|
dia = [s for s in segs if s.get('type') == 'dialogue']
|
|
fused = [s for s in dia if re.search(r'\n\s*\n', (s.get('text') or '').strip())]
|
|
print(f"{label:10s} segments={len(segs):5d} dialogue={len(dia):5d} "
|
|
f"unknown={sum(1 for s in segs if is_unknown(s)):4d} fused={len(fused):3d} "
|
|
f"speakers={len({s.get('speaker') for s in dia if s.get('speaker')}):3d}")
|
|
|
|
def main(man_path, auto_path):
|
|
man, auto = load(man_path), load(auto_path)
|
|
print("=" * 78)
|
|
summarise(man, "MANUAL"); summarise(auto, "AUTO")
|
|
print("=" * 78)
|
|
|
|
# index the manual cast by normalised text
|
|
mi = collections.defaultdict(list)
|
|
for i, s in enumerate(man):
|
|
k = norm(s.get('text'))
|
|
if k: mi[k].append(i)
|
|
|
|
agree = disagree = only_auto = 0
|
|
unknown_in_auto_known_in_man = []
|
|
wrong_speaker = []
|
|
type_diff = []
|
|
for s in auto:
|
|
k = norm(s.get('text'))
|
|
if not k: continue
|
|
hits = mi.get(k)
|
|
if not hits:
|
|
only_auto += 1
|
|
continue
|
|
m = man[hits[0]]
|
|
if m.get('type') != s.get('type'):
|
|
type_diff.append((s.get('text', '')[:60], m.get('type'), s.get('type')))
|
|
ms, as_ = str(m.get('speaker') or ''), str(s.get('speaker') or '')
|
|
if ms == as_:
|
|
agree += 1
|
|
else:
|
|
disagree += 1
|
|
if is_unknown(s) and not is_unknown(m):
|
|
unknown_in_auto_known_in_man.append((s.get('text', '')[:60], ms))
|
|
elif not is_unknown(s) and not is_unknown(m) and m.get('type') == 'dialogue':
|
|
wrong_speaker.append((s.get('text', '')[:60], ms, as_))
|
|
|
|
tot = agree + disagree
|
|
print(f"\nmatched segments: {tot} agree={agree} ({100*agree//max(tot,1)}%) disagree={disagree}")
|
|
print(f"segments only in AUTO (split/reworded): {only_auto}")
|
|
|
|
print(f"\n--- AUTO left Unknown where the human knew the speaker ({len(unknown_in_auto_known_in_man)}) ---")
|
|
for t, who in unknown_in_auto_known_in_man[:25]:
|
|
print(f" should be {who:18s} | {t!r}")
|
|
|
|
print(f"\n--- AUTO picked a DIFFERENT speaker than the human ({len(wrong_speaker)}) ---")
|
|
for t, who, got in wrong_speaker[:25]:
|
|
print(f" human={who:16s} auto={got:16s} | {t!r}")
|
|
|
|
print(f"\n--- type disagreements narration vs dialogue ({len(type_diff)}) ---")
|
|
for t, mt, at in type_diff[:20]:
|
|
print(f" human={mt:9s} auto={at:9s} | {t!r}")
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) != 3:
|
|
print(__doc__); sys.exit(1)
|
|
main(sys.argv[1], sys.argv[2])
|