From d06f990379c6d9be84fd12e2297ab3d69f549b59 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Mon, 28 Sep 2026 20:09:42 +0200 Subject: [PATCH] feat(magazine): fast LLM calls for spelling agent, run-together word repair, magazine comparison tool Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 7 + core/spelling.py | 8 + routes/magazine.py | 59 +++--- scripts/compare_magazines.py | 311 ++++++++++++++++++++++++++++++++ tests/test_compare_magazines.py | 108 +++++++++++ tests/test_magazine_routes.py | 21 +++ tests/test_spelling.py | 11 ++ 7 files changed, 497 insertions(+), 28 deletions(-) create mode 100644 scripts/compare_magazines.py create mode 100644 tests/test_compare_magazines.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ef96c20..c9ebe66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi --- ## [Unreleased] +### Added +- `scripts/compare_magazines.py`: read-only comparison of two magazine folders (words, boxes, reading order, articles, quality numbers) for re-import checks. + +### Changed +- Spelling agent and acronym resolver call the language model with thinking off (about 4 s instead of about 27 s per call) and retry without the option if the server rejects it. +- The spelling agent treats run-together words (`dasich`) as errors and may split them at a space when both halves are real words and the letters are unchanged. + ### Added - **Paragraphs are fused across columns and pages for narration.** A paragraph that continues into the next column or onto a later page is now one block in the reader and the exported MP3, so a sentence is no longer cut in two at the seam. Words split at the seam are closed up (`jähr-` + `lich`); real compound hyphens are kept; headings, interview turns and finished paragraphs are never glued together. Highlighting still follows the region each sentence began in. diff --git a/core/spelling.py b/core/spelling.py index a884c1f..f7e888f 100644 --- a/core/spelling.py +++ b/core/spelling.py @@ -303,6 +303,14 @@ def accept_agent_fix(issue: dict, replacement: str) -> bool: """ replacement = (replacement or "").strip() token = issue["token"] + # Two words run together (dasich -> "das ich"): the same letters, split at a space, both halves + # real words. Nothing is invented, so this is provable without a suggestion list. + checker_ = _checker(issue.get("language") or "de") + halves = replacement.split(" ") + if (checker_ is not None and len(halves) == 2 and all(re.fullmatch(r"[A-Za-zÄÖÜäöüß]+", h) for h in halves) + and "".join(halves).casefold() == token.casefold() + and all(len(h) >= 2 and _known_form(h, checker_, issue.get("language") or "de") for h in halves)): + return True if (not replacement or replacement == token or not re.fullmatch(r"[A-Za-zÄÖÜäöüß]+(?:- (?:und|oder))?", replacement)): return False diff --git a/routes/magazine.py b/routes/magazine.py index ca319a4..5b8a5d2 100644 --- a/routes/magazine.py +++ b/routes/magazine.py @@ -2343,7 +2343,10 @@ _SPELLING_SYSTEM = ( "{{\"token\": \"Nvidia\", \"action\": \"keep\"}}]}}. " "Use replace only when the printed word is clearly an OCR or typing error of exactly one word, judged from its " "sentence; the replacement is that single word. Use keep for proper names, brands, technical terms, foreign " - "words and valid compounds, and whenever you are unsure. Never rephrase, add or remove any other words." + "words and valid compounds, and whenever you are unsure. " + "Two words run together (dasich = das ich) or a word with a missing or wrong letter is an error, never a " + "term to keep: for run-together words replace with the two words separated by one space. " + "Never rephrase, add or remove any other words." ) @@ -2414,21 +2417,12 @@ def _spelling_findings(d: Path, layout: dict[str, Any], article_id: str | None = def _agent_check_block(settings: dict[str, Any], language: str, text: str, issues: list[dict[str, Any]]) -> list[dict[str, Any]]: """Ask the LLM about one block's findings; returns its decisions (unvalidated).""" - from routes.conversation import _post_llm_chat_completion - model = str(settings.get("llm_model") or "").strip() - payload = { - "model": model, "temperature": 0.0, "stream": False, - "messages": [ - {"role": "system", "content": _SPELLING_SYSTEM.format( - language="German" if language == "de" else "English")}, - {"role": "user", "content": json.dumps( - [{"token": i["token"], "suggestions": i["suggestions"], "context": i["context"]} - for i in issues], ensure_ascii=False)}, - ], - } - headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"} - resp = _post_llm_chat_completion(settings.get("llm_url") or "http://localhost:11434/v1", - payload, headers, 120.0) + resp = _llm_chat_fast(settings, [ + {"role": "system", "content": _SPELLING_SYSTEM.format(language="German" if language == "de" else "English")}, + {"role": "user", "content": json.dumps( + [{"token": i["token"], "suggestions": i["suggestions"], "context": i["context"]} for i in issues], + ensure_ascii=False)}, + ]) resp.raise_for_status() decisions = _llm_json(resp.json()["choices"][0]["message"]["content"]).get("decisions") return decisions if isinstance(decisions, list) else [] @@ -2594,6 +2588,21 @@ _ACRONYM_SYSTEM = ( ) +def _llm_chat_fast(settings: dict[str, Any], messages: list[dict[str, str]], timeout: float = 120.0, max_tokens: int = 800): + """One chat completion with the model's thinking switched off (a reasoning model otherwise spends + ~30 s and hundreds of tokens per call on these small structured questions). A backend that rejects + the Qwen-style switch is asked again without it.""" + from routes.conversation import _post_llm_chat_completion + payload = {"model": str(settings.get("llm_model") or "").strip(), "temperature": 0.0, "stream": False, + "max_tokens": max_tokens, "messages": messages, "chat_template_kwargs": {"enable_thinking": False}} + headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"} + url = settings.get("llm_url") or "http://localhost:11434/v1" + resp = _post_llm_chat_completion(url, payload, headers, timeout) + if resp.status_code in (400, 422): + resp = _post_llm_chat_completion(url, {k: v for k, v in payload.items() if k != "chat_template_kwargs"}, headers, timeout) + return resp + + def _llm_json(raw: str) -> Any: """The JSON object in a model answer, tolerating code fences, think-tags and chatter around it.""" raw = re.sub(r".*?", "", raw or "", flags=re.S).strip() @@ -2661,19 +2670,13 @@ async def magazine_acronyms_resolve(doc_id: str, article_id: str): if not model: raise HTTPException(400, "No LLM model configured") language = "German" if state["language"] == "de" else "English" - payload = { - "model": model, "temperature": 0.1, "stream": False, - "messages": [ - {"role": "system", "content": _ACRONYM_SYSTEM.format(language=language)}, - {"role": "user", "content": json.dumps( - [{"acronym": e["acronym"], "context": e["context"]} for e in unknown], ensure_ascii=False)}, - ], - } - headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"} + messages = [ + {"role": "system", "content": _ACRONYM_SYSTEM.format(language=language)}, + {"role": "user", "content": json.dumps( + [{"acronym": e["acronym"], "context": e["context"]} for e in unknown], ensure_ascii=False)}, + ] try: - resp = await asyncio.to_thread( - _post_llm_chat_completion, settings.get("llm_url") or "http://localhost:11434/v1", - payload, headers, 150.0) + resp = await asyncio.to_thread(_llm_chat_fast, settings, messages, 150.0) resp.raise_for_status() decided = parse_acronym_decisions( _llm_json(resp.json()["choices"][0]["message"]["content"]).get("acronyms"), unknown) diff --git a/scripts/compare_magazines.py b/scripts/compare_magazines.py new file mode 100644 index 0000000..44b89bd --- /dev/null +++ b/scripts/compare_magazines.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""Compare two processed magazine folders (a baseline and a re-import of the same PDF). + +Read-only. Nothing is guessed by eye: every number comes from the two folders' +``layout.json`` / ``articles.json`` (and, when present, ``quality_report.json``). + + python scripts/compare_magazines.py BASELINE_DIR NEW_DIR [--out report.json] [--md report.md] + +What is compared, per page and overall: + +* text - the narratable words of each page, aligned word by word (difflib), so a + re-ordered or re-segmented but identical page still scores 1.0 +* boxes - regions matched by overlap (IoU >= 0.5): recall/precision, type agreement +* order - of the matched boxes, how many pairs are read in the same order +* articles - matched by shared text (word 3-grams); title, author, page span, status +* quality - pages needing a decision, unboxed areas, clipped edges, overlaps + +The JSON report lists every difference so a person (or a review screen) can decide, per +difference, which side is better. +""" +from __future__ import annotations + +import argparse +import difflib +import json +import re +import sys +from pathlib import Path +from typing import Any + +TEXT_TYPES = {"title", "heading", "deck", "byline", "body", "quote", "caption", "sidebar", "pullquote"} +NARRATED_TYPES = {"title", "heading", "deck", "byline", "body", "quote"} +_WORD = re.compile(r"[\wÄÖÜäöüß'’-]+", re.U) +_IOU_MATCH = 0.5 +_SIMILAR_PAGE = 0.98 + + +def load_magazine(folder: Path) -> dict[str, Any]: + folder = Path(folder) + layout = json.loads((folder / "layout.json").read_text("utf-8")) + articles = json.loads((folder / "articles.json").read_text("utf-8")).get("articles", []) + quality: dict[str, Any] = {} + if (folder / "quality_report.json").is_file(): + quality = json.loads((folder / "quality_report.json").read_text("utf-8")) + return {"folder": str(folder), "layout": layout, "articles": articles, "quality": quality} + + +def words(text: str) -> list[str]: + return [w.casefold().strip("-'’") for w in _WORD.findall(text or "") if w.strip("-'’")] + + +def page_words(layout: dict[str, Any], page: int) -> list[str]: + blocks = sorted((b for b in layout.get("blocks", []) if b["page"] == page and b["type"] in NARRATED_TYPES), + key=lambda b: b.get("reading_order", 0)) + out: list[str] = [] + for block in blocks: + out.extend(words(block.get("text", ""))) + return out + + +def align(a: list[str], b: list[str]) -> dict[str, Any]: + """Word-level alignment. Order-insensitive where it should be: each side is compared as read.""" + matcher = difflib.SequenceMatcher(None, a, b, autojunk=False) + only_a: list[str] = [] + only_b: list[str] = [] + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag in ("delete", "replace"): + only_a.extend(a[i1:i2]) + if tag in ("insert", "replace"): + only_b.extend(b[j1:j2]) + total = max(1, len(a) + len(b)) + same = total - len(only_a) - len(only_b) + return {"similarity": 1.0 if not (a or b) else round(same / total, 4), + "words_a": len(a), "words_b": len(b), "only_in_a": only_a, "only_in_b": only_b} + + +def iou(a: list[float], b: list[float]) -> float: + ax, ay, aw, ah = a + bx, by, bw, bh = b + w = max(0.0, min(ax + aw, bx + bw) - max(ax, bx)) + h = max(0.0, min(ay + ah, by + bh) - max(ay, by)) + inter = w * h + union = aw * ah + bw * bh - inter + return inter / union if union > 0 else 0.0 + + +def match_boxes(blocks_a: list[dict[str, Any]], blocks_b: list[dict[str, Any]]) -> list[tuple[dict[str, Any], dict[str, Any], float]]: + pairs = sorted(((iou(x["bbox"], y["bbox"]), i, j) for i, x in enumerate(blocks_a) for j, y in enumerate(blocks_b)), + reverse=True) + used_a: set[int] = set() + used_b: set[int] = set() + matched = [] + for score, i, j in pairs: + if score < _IOU_MATCH: + break + if i in used_a or j in used_b: + continue + used_a.add(i) + used_b.add(j) + matched.append((blocks_a[i], blocks_b[j], score)) + return matched + + +def order_agreement(matched: list[tuple[dict[str, Any], dict[str, Any], float]]) -> float | None: + """Share of matched box pairs that are read in the same relative order in both.""" + if len(matched) < 2: + return None + concordant = total = 0 + for i in range(len(matched)): + for j in range(i + 1, len(matched)): + da = matched[i][0].get("reading_order", 0) - matched[j][0].get("reading_order", 0) + db = matched[i][1].get("reading_order", 0) - matched[j][1].get("reading_order", 0) + if da == 0 or db == 0: + continue + total += 1 + concordant += (da > 0) == (db > 0) + return round(concordant / total, 4) if total else None + + +def compare_pages(a: dict[str, Any], b: dict[str, Any]) -> list[dict[str, Any]]: + pages_a = {p["page"] for p in a["layout"].get("pages", [])} + pages_b = {p["page"] for p in b["layout"].get("pages", [])} + rows = [] + for page in sorted(pages_a | pages_b): + blocks_a = [x for x in a["layout"].get("blocks", []) if x["page"] == page and x["type"] in TEXT_TYPES] + blocks_b = [x for x in b["layout"].get("blocks", []) if x["page"] == page and x["type"] in TEXT_TYPES] + text = align(page_words(a["layout"], page), page_words(b["layout"], page)) + matched = match_boxes(blocks_a, blocks_b) + images_a = [i for i in a["layout"].get("images", []) if i["page"] == page] + images_b = [i for i in b["layout"].get("images", []) if i["page"] == page] + figures = match_boxes([{"bbox": i["bbox"]} for i in images_a], [{"bbox": i["bbox"]} for i in images_b]) + row = { + "page": page, + "text_similarity": text["similarity"], "words_a": text["words_a"], "words_b": text["words_b"], + "only_in_a": " ".join(text["only_in_a"][:40]), "only_in_b": " ".join(text["only_in_b"][:40]), + "boxes_a": len(blocks_a), "boxes_b": len(blocks_b), "boxes_matched": len(matched), + "box_recall": round(len(matched) / len(blocks_a), 4) if blocks_a else None, + "box_precision": round(len(matched) / len(blocks_b), 4) if blocks_b else None, + "type_agreement": round(sum(x["type"] == y["type"] for x, y, _ in matched) / len(matched), 4) if matched else None, + "type_changes": sorted({f"{x['type']}->{y['type']}" for x, y, _ in matched if x["type"] != y["type"]}), + "order_agreement": order_agreement(matched), + "hand_ordered_in_a": bool(next((p for p in a["layout"].get("pages", []) if p["page"] == page), {}).get("order_locked")), + "figures_a": len(images_a), "figures_b": len(images_b), "figures_matched": len(figures), + "page_only_in": "a" if page not in pages_b else ("b" if page not in pages_a else ""), + } + flags = [] + if row["text_similarity"] < _SIMILAR_PAGE: + flags.append("text") + if row["box_recall"] is not None and row["box_recall"] < 0.9: + flags.append("boxes_missing_in_new") + if row["box_precision"] is not None and row["box_precision"] < 0.9: + flags.append("extra_boxes_in_new") + if row["type_agreement"] is not None and row["type_agreement"] < 0.95: + flags.append("types") + if row["order_agreement"] is not None and row["order_agreement"] < 0.95: + flags.append("order") + if row["figures_a"] != row["figures_b"]: + flags.append("figures") + row["flags"] = flags + rows.append(row) + return rows + + +def _article_text(magazine: dict[str, Any], article: dict[str, Any]) -> list[str]: + ids = set(article.get("blocks") or []) + blocks = sorted((b for b in magazine["layout"].get("blocks", []) if b["id"] in ids and b["type"] in NARRATED_TYPES), + key=lambda b: (b["page"], b.get("reading_order", 0))) + out: list[str] = [] + for block in blocks: + out.extend(words(block.get("text", ""))) + return out + + +def _shingles(ws: list[str], n: int = 3) -> set[tuple[str, ...]]: + return {tuple(ws[i:i + n]) for i in range(len(ws) - n + 1)} if len(ws) >= n else {tuple(ws)} if ws else set() + + +def compare_articles(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]: + texts_a = {x["id"]: _article_text(a, x) for x in a["articles"]} + texts_b = {x["id"]: _article_text(b, x) for x in b["articles"]} + sh_a = {k: _shingles(v) for k, v in texts_a.items()} + sh_b = {k: _shingles(v) for k, v in texts_b.items()} + scores = [] + for ia, sa in sh_a.items(): + for ib, sb in sh_b.items(): + if not sa or not sb: + continue + inter = len(sa & sb) + if inter: + scores.append((inter / len(sa | sb), ia, ib)) + scores.sort(reverse=True) + used_a: set[str] = set() + used_b: set[str] = set() + by_a = {x["id"]: x for x in a["articles"]} + by_b = {x["id"]: x for x in b["articles"]} + matched = [] + for score, ia, ib in scores: + if score < 0.3 or ia in used_a or ib in used_b: + continue + used_a.add(ia) + used_b.add(ib) + xa, xb = by_a[ia], by_b[ib] + diff = align(texts_a[ia], texts_b[ib]) + matched.append({ + "a": ia, "b": ib, "overlap": round(score, 4), "text_similarity": diff["similarity"], + "title_a": xa.get("title", ""), "title_b": xb.get("title", ""), + "author_a": xa.get("author", ""), "author_b": xb.get("author", ""), + "pages_a": xa.get("pages", []), "pages_b": xb.get("pages", []), + "status_a": xa.get("status"), "status_b": xb.get("status"), + "same_pages": xa.get("pages", []) == xb.get("pages", []), + }) + def brief(x: dict[str, Any], texts: dict[str, list[str]]) -> dict[str, Any]: + return {"id": x["id"], "title": x.get("title", ""), "pages": x.get("pages", []), "words": len(texts[x["id"]])} + return { + "matched": matched, + "only_in_a": [brief(x, texts_a) for x in a["articles"] if x["id"] not in used_a and texts_a[x["id"]]], + "only_in_b": [brief(x, texts_b) for x in b["articles"] if x["id"] not in used_b and texts_b[x["id"]]], + } + + +def quality_numbers(m: dict[str, Any]) -> dict[str, Any]: + rows = m["quality"].get("pages", []) + return { + "pages_needing_a_decision": sum(1 for r in rows if r.get("needs_review")), + "unboxed_areas": sum(len(r.get("uncovered", [])) for r in rows), + "clipped_edges": sum(len(r.get("clipped", [])) for r in rows), + "overlapping_pairs": sum(len(r.get("overlaps", [])) for r in rows), + "words_to_check": sum(len(r.get("spelling", [])) for r in rows), + } + + +def mean(values: list[float]) -> float | None: + values = [v for v in values if v is not None] + return round(sum(values) / len(values), 4) if values else None + + +def compare(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]: + pages = compare_pages(a, b) + articles = compare_articles(a, b) + words_a = sum(p["words_a"] for p in pages) + words_b = sum(p["words_b"] for p in pages) + summary = { + "baseline": a["folder"], "new": b["folder"], + "pages": len(pages), + "words": {"baseline": words_a, "new": words_b}, + "text_similarity_mean": mean([p["text_similarity"] for p in pages]), + "pages_with_text_differences": sum("text" in p["flags"] for p in pages), + "box_recall_mean": mean([p["box_recall"] for p in pages]), + "box_precision_mean": mean([p["box_precision"] for p in pages]), + "type_agreement_mean": mean([p["type_agreement"] for p in pages]), + "order_agreement_mean": mean([p["order_agreement"] for p in pages]), + "pages_flagged": sum(bool(p["flags"]) for p in pages), + "figures": {"baseline": sum(p["figures_a"] for p in pages), "new": sum(p["figures_b"] for p in pages)}, + "articles": {"baseline": len(a["articles"]), "new": len(b["articles"]), + "matched": len(articles["matched"]), "only_in_baseline": len(articles["only_in_a"]), + "only_in_new": len(articles["only_in_b"]), + "same_page_span": sum(x["same_pages"] for x in articles["matched"]), + "same_title": sum(x["title_a"].strip().casefold() == x["title_b"].strip().casefold() for x in articles["matched"])}, + "quality": {"baseline": quality_numbers(a), "new": quality_numbers(b)}, + } + return {"summary": summary, "pages": pages, "articles": articles} + + +def markdown(report: dict[str, Any], worst: int = 15) -> str: + s = report["summary"] + lines = ["# Magazine comparison", "", f"* baseline: `{s['baseline']}`", f"* new: `{s['new']}`", "", + "| measure | value |", "|---|---|", + f"| pages | {s['pages']} |", + f"| narratable words (baseline / new) | {s['words']['baseline']} / {s['words']['new']} |", + f"| text similarity (mean per page) | {s['text_similarity_mean']} |", + f"| pages with text differences | {s['pages_with_text_differences']} |", + f"| box recall / precision | {s['box_recall_mean']} / {s['box_precision_mean']} |", + f"| type agreement | {s['type_agreement_mean']} |", + f"| reading-order agreement | {s['order_agreement_mean']} |", + f"| figures (baseline / new) | {s['figures']['baseline']} / {s['figures']['new']} |", + f"| articles (baseline / new / matched) | {s['articles']['baseline']} / {s['articles']['new']} / {s['articles']['matched']} |", + f"| matched articles with the same page span | {s['articles']['same_page_span']} |", + f"| pages flagged | {s['pages_flagged']} |", "", "## Quality report", "", + "| | baseline | new |", "|---|---|---|"] + for key in s["quality"]["baseline"]: + lines.append(f"| {key.replace('_', ' ')} | {s['quality']['baseline'][key]} | {s['quality']['new'][key]} |") + flagged = sorted((p for p in report["pages"] if p["flags"]), key=lambda p: (p["text_similarity"], p["page"]))[:worst] + lines += ["", f"## The {len(flagged)} pages that differ most", "", "| page | flags | text | boxes b/n | order |", "|---|---|---|---|---|"] + for p in flagged: + lines.append(f"| {p['page']} | {', '.join(p['flags'])} | {p['text_similarity']} | {p['boxes_a']}/{p['boxes_b']} | {p['order_agreement']} |") + for title, key in (("Only in the baseline", "only_in_a"), ("Only in the new import", "only_in_b")): + rows = report["articles"][key] + lines += ["", f"## Articles {title.lower()} ({len(rows)})", ""] + lines += [f"* {r['title'][:60]!r} pages {r['pages']} ({r['words']} words)" for r in rows[:25]] + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("baseline", type=Path) + parser.add_argument("new", type=Path) + parser.add_argument("--out", type=Path, help="write the full JSON report here") + parser.add_argument("--md", type=Path, help="write a short Markdown summary here") + args = parser.parse_args(argv) + report = compare(load_magazine(args.baseline), load_magazine(args.new)) + if args.out: + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=1), "utf-8") + text = markdown(report) + if args.md: + args.md.write_text(text, "utf-8") + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_compare_magazines.py b/tests/test_compare_magazines.py new file mode 100644 index 0000000..ce2e9d4 --- /dev/null +++ b/tests/test_compare_magazines.py @@ -0,0 +1,108 @@ +"""scripts/compare_magazines.py: two folders with known differences give the expected numbers.""" +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import compare_magazines as cm # noqa: E402 + + +def blk(i, page, ty, text, bbox, order): + return {"id": i, "page": page, "type": ty, "text": text, "bbox": bbox, "reading_order": order} + + +def art(i, title, pages, blocks, status="approved", author=""): + return {"id": i, "title": title, "author": author, "pages": pages, "blocks": blocks, "images": [], "status": status} + + +class CompareTests(unittest.TestCase): + def folder(self, blocks, articles, images=(), quality=None): + d = Path(tempfile.mkdtemp(prefix="ttvc-cmp-")) + self.addCleanup(shutil.rmtree, d, True) + pages = sorted({b["page"] for b in blocks}) + (d / "layout.json").write_text(json.dumps({"pages": [{"page": p} for p in pages], "blocks": blocks, "images": list(images)})) + (d / "articles.json").write_text(json.dumps({"articles": articles})) + if quality: + (d / "quality_report.json").write_text(json.dumps(quality)) + return d + + def base(self): + return self.folder( + [blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1), + blk("b", 1, "body", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)], + [art("article-001", "Der Butler", [1, 2], ["t", "a", "b", "c"], author="Jo")]) + + def test_identical_folders_score_perfectly(self): + report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(self.base())) + s = report["summary"] + self.assertEqual((s["text_similarity_mean"], s["box_recall_mean"], s["box_precision_mean"], s["type_agreement_mean"], s["order_agreement_mean"]), + (1.0, 1.0, 1.0, 1.0, 1.0)) + self.assertEqual((s["pages_flagged"], s["articles"]["matched"], s["articles"]["only_in_baseline"], s["articles"]["only_in_new"]), (0, 1, 0, 0)) + self.assertEqual(s["articles"]["same_page_span"], 1) + + def test_a_misread_word_a_retyped_box_and_a_changed_reading_order_are_each_reported(self): + new = self.folder( + [blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 2), # order swapped + blk("b", 1, "caption", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 1), # retyped: no longer narrated + blk("c", 2, "body", "Und die zwelte Seite geht weiter.", [10, 10, 90, 100], 0)], # misread word + [art("article-001", "Der Butler", [1, 2], ["t", "a", "b", "c"], author="Jo")]) + report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new)) + page1 = next(p for p in report["pages"] if p["page"] == 1) + self.assertLess(page1["text_similarity"], 1.0) + self.assertIn("zweite", page1["only_in_a"]) # the retyped box's words are no longer read + self.assertEqual(page1["only_in_b"], "") + self.assertEqual(page1["type_changes"], ["body->caption"]) + self.assertLess(page1["order_agreement"], 1.0) + self.assertEqual(set(page1["flags"]), {"text", "types", "order"}) + page2 = next(p for p in report["pages"] if p["page"] == 2) + self.assertEqual((page2["only_in_a"], page2["only_in_b"], page2["flags"]), ("zweite", "zwelte", ["text"])) + self.assertEqual(report["summary"]["pages_flagged"], 2) + + def test_missing_and_extra_boxes_and_figures_show_up_as_recall_and_precision(self): + new = self.folder( + [blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1), + blk("x", 1, "body", "Ein ganz anderer Kasten.", [300, 300, 50, 50], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)], + [art("article-001", "Der Butler", [1, 2], ["t", "a", "x", "c"], author="Jo")], + images=[{"id": "i", "page": 1, "bbox": [0, 200, 100, 100]}]) + page1 = next(p for p in cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))["pages"] if p["page"] == 1) + self.assertEqual((page1["boxes_a"], page1["boxes_b"], page1["boxes_matched"]), (3, 3, 2)) + self.assertAlmostEqual(page1["box_recall"], 2 / 3, places=3) + self.assertAlmostEqual(page1["box_precision"], 2 / 3, places=3) + self.assertIn("boxes_missing_in_new", page1["flags"]) + self.assertIn("figures", page1["flags"]) + + def test_articles_are_matched_by_text_even_when_split_differently(self): + one = self.base() + split = self.folder( + [blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1), + blk("b", 1, "body", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)], + [art("article-001", "Der Butler", [1], ["t", "a", "b"], status="detected"), art("article-002", "Page 2", [2], ["c"])]) + report = cm.compare(cm.load_magazine(one), cm.load_magazine(split))["articles"] + self.assertEqual(len(report["matched"]), 1) # the bigger piece matches + self.assertFalse(report["matched"][0]["same_pages"]) + self.assertEqual([a["title"] for a in report["only_in_b"]], ["Page 2"]) + + def test_quality_numbers_and_the_markdown_summary(self): + q = {"pages": [{"page": 1, "needs_review": True, "uncovered": [[0, 0, 1, 1]], "clipped": [{}, {}], "overlaps": [], "spelling": ["a"]}]} + new = self.folder(json.loads((self.base() / "layout.json").read_text())["blocks"], [], quality=q) + report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new)) + self.assertEqual(report["summary"]["quality"]["new"], {"pages_needing_a_decision": 1, "unboxed_areas": 1, "clipped_edges": 2, + "overlapping_pairs": 0, "words_to_check": 1}) + text = cm.markdown(report) + self.assertIn("# Magazine comparison", text) + self.assertIn("| clipped edges | 0 | 2 |", text) + + def test_the_command_line_writes_json_and_markdown(self): + out = Path(tempfile.mkdtemp(prefix="ttvc-cmp-out-")) + self.addCleanup(shutil.rmtree, out, True) + code = cm.main([str(self.base()), str(self.base()), "--out", str(out / "r.json"), "--md", str(out / "r.md")]) + self.assertEqual(code, 0) + self.assertEqual(json.loads((out / "r.json").read_text())["summary"]["pages"], 2) + self.assertTrue((out / "r.md").read_text().startswith("# Magazine comparison")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_magazine_routes.py b/tests/test_magazine_routes.py index 5be4360..b5fed9a 100644 --- a/tests/test_magazine_routes.py +++ b/tests/test_magazine_routes.py @@ -1898,3 +1898,24 @@ class BoxFixesReorderThePageTests(unittest.TestCase): "blocks": [blk("r1", 110, 10, 0), blk("l1", 10, 10, 1), blk("r2", 110, 80, 2), blk("l2", 10, 80, 3)]} fixed, _, _ = _fix_page_geometry(d, layout, [], 1, {}) self.assertEqual([b["id"] for b in sorted(fixed["blocks"], key=lambda b: b["reading_order"])], ["l1", "l2", "r1", "r2"]) + + +class FastLlmCallTests(unittest.TestCase): + def test_thinking_is_switched_off_and_a_rejecting_backend_is_asked_again_without_the_switch(self): + from unittest.mock import MagicMock, patch + from routes.magazine import _llm_chat_fast + bad, good = MagicMock(status_code=422), MagicMock(status_code=200) + with patch('routes.conversation._post_llm_chat_completion', side_effect=[bad, good]) as post: + out = _llm_chat_fast({"llm_model": "m", "llm_url": "http://x"}, [{"role": "user", "content": "hi"}]) + self.assertIs(out, good) + first, second = post.call_args_list[0].args[1], post.call_args_list[1].args[1] + self.assertEqual(first["chat_template_kwargs"], {"enable_thinking": False}) + self.assertEqual((first["temperature"], first["max_tokens"]), (0.0, 800)) + self.assertNotIn("chat_template_kwargs", second) + + def test_a_backend_that_accepts_it_is_called_once(self): + from unittest.mock import MagicMock, patch + from routes.magazine import _llm_chat_fast + with patch('routes.conversation._post_llm_chat_completion', return_value=MagicMock(status_code=200)) as post: + _llm_chat_fast({"llm_model": "m"}, []) + self.assertEqual(post.call_count, 1) diff --git a/tests/test_spelling.py b/tests/test_spelling.py index df78624..902dc61 100644 --- a/tests/test_spelling.py +++ b/tests/test_spelling.py @@ -132,6 +132,17 @@ class NamesBrandsAndVocabularyTests(unittest.TestCase): self.assertEqual([i["token"] for i in found], ["wachst"]) +class RunTogetherWordTests(unittest.TestCase): + @patched + def test_run_together_words_are_repaired_by_splitting_them_at_a_space(self): + issue = {"token": "dasich", "language": "de", "suggestions": []} + with mock.patch("core.pronunciations._german_spellchecker", return_value=Checker(DE | {"das", "ich", "dasic"})): + self.assertTrue(spelling.accept_agent_fix(issue, "das ich")) + self.assertFalse(spelling.accept_agent_fix(issue, "das ihn")) # letters changed + self.assertFalse(spelling.accept_agent_fix(issue, "d asich")) # a half is not a word + self.assertFalse(spelling.accept_agent_fix({**issue, "token": "dasichx"}, "das ich")) + + class AgentGuardTests(unittest.TestCase): ISSUE = {"token": "Tesst", "language": "de", "suggestions": ["Tests", "Test"]}