feat(magazine): fast LLM calls for spelling agent, run-together word repair, magazine comparison tool
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
29b1f63e60
commit
d06f990379
@ -6,6 +6,13 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
|
||||
---
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
- `scripts/compare_magazines.py`: read-only comparison of two magazine folders (words, boxes, reading order, articles, quality numbers) for re-import checks.
|
||||
|
||||
### Changed
|
||||
- Spelling agent and acronym resolver call the language model with thinking off (about 4 s instead of about 27 s per call) and retry without the option if the server rejects it.
|
||||
- The spelling agent treats run-together words (`dasich`) as errors and may split them at a space when both halves are real words and the letters are unchanged.
|
||||
|
||||
|
||||
### Added
|
||||
- **Paragraphs are fused across columns and pages for narration.** A paragraph that continues into the next column or onto a later page is now one block in the reader and the exported MP3, so a sentence is no longer cut in two at the seam. Words split at the seam are closed up (`jähr-` + `lich`); real compound hyphens are kept; headings, interview turns and finished paragraphs are never glued together. Highlighting still follows the region each sentence began in.
|
||||
|
||||
@ -303,6 +303,14 @@ def accept_agent_fix(issue: dict, replacement: str) -> bool:
|
||||
"""
|
||||
replacement = (replacement or "").strip()
|
||||
token = issue["token"]
|
||||
# Two words run together (dasich -> "das ich"): the same letters, split at a space, both halves
|
||||
# real words. Nothing is invented, so this is provable without a suggestion list.
|
||||
checker_ = _checker(issue.get("language") or "de")
|
||||
halves = replacement.split(" ")
|
||||
if (checker_ is not None and len(halves) == 2 and all(re.fullmatch(r"[A-Za-zÄÖÜäöüß]+", h) for h in halves)
|
||||
and "".join(halves).casefold() == token.casefold()
|
||||
and all(len(h) >= 2 and _known_form(h, checker_, issue.get("language") or "de") for h in halves)):
|
||||
return True
|
||||
if (not replacement or replacement == token
|
||||
or not re.fullmatch(r"[A-Za-zÄÖÜäöüß]+(?:- (?:und|oder))?", replacement)):
|
||||
return False
|
||||
|
||||
@ -2343,7 +2343,10 @@ _SPELLING_SYSTEM = (
|
||||
"{{\"token\": \"Nvidia\", \"action\": \"keep\"}}]}}. "
|
||||
"Use replace only when the printed word is clearly an OCR or typing error of exactly one word, judged from its "
|
||||
"sentence; the replacement is that single word. Use keep for proper names, brands, technical terms, foreign "
|
||||
"words and valid compounds, and whenever you are unsure. Never rephrase, add or remove any other words."
|
||||
"words and valid compounds, and whenever you are unsure. "
|
||||
"Two words run together (dasich = das ich) or a word with a missing or wrong letter is an error, never a "
|
||||
"term to keep: for run-together words replace with the two words separated by one space. "
|
||||
"Never rephrase, add or remove any other words."
|
||||
)
|
||||
|
||||
|
||||
@ -2414,21 +2417,12 @@ def _spelling_findings(d: Path, layout: dict[str, Any], article_id: str | None =
|
||||
def _agent_check_block(settings: dict[str, Any], language: str, text: str,
|
||||
issues: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Ask the LLM about one block's findings; returns its decisions (unvalidated)."""
|
||||
from routes.conversation import _post_llm_chat_completion
|
||||
model = str(settings.get("llm_model") or "").strip()
|
||||
payload = {
|
||||
"model": model, "temperature": 0.0, "stream": False,
|
||||
"messages": [
|
||||
{"role": "system", "content": _SPELLING_SYSTEM.format(
|
||||
language="German" if language == "de" else "English")},
|
||||
{"role": "user", "content": json.dumps(
|
||||
[{"token": i["token"], "suggestions": i["suggestions"], "context": i["context"]}
|
||||
for i in issues], ensure_ascii=False)},
|
||||
],
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
|
||||
resp = _post_llm_chat_completion(settings.get("llm_url") or "http://localhost:11434/v1",
|
||||
payload, headers, 120.0)
|
||||
resp = _llm_chat_fast(settings, [
|
||||
{"role": "system", "content": _SPELLING_SYSTEM.format(language="German" if language == "de" else "English")},
|
||||
{"role": "user", "content": json.dumps(
|
||||
[{"token": i["token"], "suggestions": i["suggestions"], "context": i["context"]} for i in issues],
|
||||
ensure_ascii=False)},
|
||||
])
|
||||
resp.raise_for_status()
|
||||
decisions = _llm_json(resp.json()["choices"][0]["message"]["content"]).get("decisions")
|
||||
return decisions if isinstance(decisions, list) else []
|
||||
@ -2594,6 +2588,21 @@ _ACRONYM_SYSTEM = (
|
||||
)
|
||||
|
||||
|
||||
def _llm_chat_fast(settings: dict[str, Any], messages: list[dict[str, str]], timeout: float = 120.0, max_tokens: int = 800):
|
||||
"""One chat completion with the model's thinking switched off (a reasoning model otherwise spends
|
||||
~30 s and hundreds of tokens per call on these small structured questions). A backend that rejects
|
||||
the Qwen-style switch is asked again without it."""
|
||||
from routes.conversation import _post_llm_chat_completion
|
||||
payload = {"model": str(settings.get("llm_model") or "").strip(), "temperature": 0.0, "stream": False,
|
||||
"max_tokens": max_tokens, "messages": messages, "chat_template_kwargs": {"enable_thinking": False}}
|
||||
headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
|
||||
url = settings.get("llm_url") or "http://localhost:11434/v1"
|
||||
resp = _post_llm_chat_completion(url, payload, headers, timeout)
|
||||
if resp.status_code in (400, 422):
|
||||
resp = _post_llm_chat_completion(url, {k: v for k, v in payload.items() if k != "chat_template_kwargs"}, headers, timeout)
|
||||
return resp
|
||||
|
||||
|
||||
def _llm_json(raw: str) -> Any:
|
||||
"""The JSON object in a model answer, tolerating code fences, think-tags and chatter around it."""
|
||||
raw = re.sub(r"<think>.*?</think>", "", raw or "", flags=re.S).strip()
|
||||
@ -2661,19 +2670,13 @@ async def magazine_acronyms_resolve(doc_id: str, article_id: str):
|
||||
if not model:
|
||||
raise HTTPException(400, "No LLM model configured")
|
||||
language = "German" if state["language"] == "de" else "English"
|
||||
payload = {
|
||||
"model": model, "temperature": 0.1, "stream": False,
|
||||
"messages": [
|
||||
{"role": "system", "content": _ACRONYM_SYSTEM.format(language=language)},
|
||||
{"role": "user", "content": json.dumps(
|
||||
[{"acronym": e["acronym"], "context": e["context"]} for e in unknown], ensure_ascii=False)},
|
||||
],
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
|
||||
messages = [
|
||||
{"role": "system", "content": _ACRONYM_SYSTEM.format(language=language)},
|
||||
{"role": "user", "content": json.dumps(
|
||||
[{"acronym": e["acronym"], "context": e["context"]} for e in unknown], ensure_ascii=False)},
|
||||
]
|
||||
try:
|
||||
resp = await asyncio.to_thread(
|
||||
_post_llm_chat_completion, settings.get("llm_url") or "http://localhost:11434/v1",
|
||||
payload, headers, 150.0)
|
||||
resp = await asyncio.to_thread(_llm_chat_fast, settings, messages, 150.0)
|
||||
resp.raise_for_status()
|
||||
decided = parse_acronym_decisions(
|
||||
_llm_json(resp.json()["choices"][0]["message"]["content"]).get("acronyms"), unknown)
|
||||
|
||||
311
scripts/compare_magazines.py
Normal file
311
scripts/compare_magazines.py
Normal file
@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare two processed magazine folders (a baseline and a re-import of the same PDF).
|
||||
|
||||
Read-only. Nothing is guessed by eye: every number comes from the two folders'
|
||||
``layout.json`` / ``articles.json`` (and, when present, ``quality_report.json``).
|
||||
|
||||
python scripts/compare_magazines.py BASELINE_DIR NEW_DIR [--out report.json] [--md report.md]
|
||||
|
||||
What is compared, per page and overall:
|
||||
|
||||
* text - the narratable words of each page, aligned word by word (difflib), so a
|
||||
re-ordered or re-segmented but identical page still scores 1.0
|
||||
* boxes - regions matched by overlap (IoU >= 0.5): recall/precision, type agreement
|
||||
* order - of the matched boxes, how many pairs are read in the same order
|
||||
* articles - matched by shared text (word 3-grams); title, author, page span, status
|
||||
* quality - pages needing a decision, unboxed areas, clipped edges, overlaps
|
||||
|
||||
The JSON report lists every difference so a person (or a review screen) can decide, per
|
||||
difference, which side is better.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
TEXT_TYPES = {"title", "heading", "deck", "byline", "body", "quote", "caption", "sidebar", "pullquote"}
|
||||
NARRATED_TYPES = {"title", "heading", "deck", "byline", "body", "quote"}
|
||||
_WORD = re.compile(r"[\wÄÖÜäöüß'’-]+", re.U)
|
||||
_IOU_MATCH = 0.5
|
||||
_SIMILAR_PAGE = 0.98
|
||||
|
||||
|
||||
def load_magazine(folder: Path) -> dict[str, Any]:
|
||||
folder = Path(folder)
|
||||
layout = json.loads((folder / "layout.json").read_text("utf-8"))
|
||||
articles = json.loads((folder / "articles.json").read_text("utf-8")).get("articles", [])
|
||||
quality: dict[str, Any] = {}
|
||||
if (folder / "quality_report.json").is_file():
|
||||
quality = json.loads((folder / "quality_report.json").read_text("utf-8"))
|
||||
return {"folder": str(folder), "layout": layout, "articles": articles, "quality": quality}
|
||||
|
||||
|
||||
def words(text: str) -> list[str]:
|
||||
return [w.casefold().strip("-'’") for w in _WORD.findall(text or "") if w.strip("-'’")]
|
||||
|
||||
|
||||
def page_words(layout: dict[str, Any], page: int) -> list[str]:
|
||||
blocks = sorted((b for b in layout.get("blocks", []) if b["page"] == page and b["type"] in NARRATED_TYPES),
|
||||
key=lambda b: b.get("reading_order", 0))
|
||||
out: list[str] = []
|
||||
for block in blocks:
|
||||
out.extend(words(block.get("text", "")))
|
||||
return out
|
||||
|
||||
|
||||
def align(a: list[str], b: list[str]) -> dict[str, Any]:
|
||||
"""Word-level alignment. Order-insensitive where it should be: each side is compared as read."""
|
||||
matcher = difflib.SequenceMatcher(None, a, b, autojunk=False)
|
||||
only_a: list[str] = []
|
||||
only_b: list[str] = []
|
||||
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
||||
if tag in ("delete", "replace"):
|
||||
only_a.extend(a[i1:i2])
|
||||
if tag in ("insert", "replace"):
|
||||
only_b.extend(b[j1:j2])
|
||||
total = max(1, len(a) + len(b))
|
||||
same = total - len(only_a) - len(only_b)
|
||||
return {"similarity": 1.0 if not (a or b) else round(same / total, 4),
|
||||
"words_a": len(a), "words_b": len(b), "only_in_a": only_a, "only_in_b": only_b}
|
||||
|
||||
|
||||
def iou(a: list[float], b: list[float]) -> float:
|
||||
ax, ay, aw, ah = a
|
||||
bx, by, bw, bh = b
|
||||
w = max(0.0, min(ax + aw, bx + bw) - max(ax, bx))
|
||||
h = max(0.0, min(ay + ah, by + bh) - max(ay, by))
|
||||
inter = w * h
|
||||
union = aw * ah + bw * bh - inter
|
||||
return inter / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def match_boxes(blocks_a: list[dict[str, Any]], blocks_b: list[dict[str, Any]]) -> list[tuple[dict[str, Any], dict[str, Any], float]]:
|
||||
pairs = sorted(((iou(x["bbox"], y["bbox"]), i, j) for i, x in enumerate(blocks_a) for j, y in enumerate(blocks_b)),
|
||||
reverse=True)
|
||||
used_a: set[int] = set()
|
||||
used_b: set[int] = set()
|
||||
matched = []
|
||||
for score, i, j in pairs:
|
||||
if score < _IOU_MATCH:
|
||||
break
|
||||
if i in used_a or j in used_b:
|
||||
continue
|
||||
used_a.add(i)
|
||||
used_b.add(j)
|
||||
matched.append((blocks_a[i], blocks_b[j], score))
|
||||
return matched
|
||||
|
||||
|
||||
def order_agreement(matched: list[tuple[dict[str, Any], dict[str, Any], float]]) -> float | None:
|
||||
"""Share of matched box pairs that are read in the same relative order in both."""
|
||||
if len(matched) < 2:
|
||||
return None
|
||||
concordant = total = 0
|
||||
for i in range(len(matched)):
|
||||
for j in range(i + 1, len(matched)):
|
||||
da = matched[i][0].get("reading_order", 0) - matched[j][0].get("reading_order", 0)
|
||||
db = matched[i][1].get("reading_order", 0) - matched[j][1].get("reading_order", 0)
|
||||
if da == 0 or db == 0:
|
||||
continue
|
||||
total += 1
|
||||
concordant += (da > 0) == (db > 0)
|
||||
return round(concordant / total, 4) if total else None
|
||||
|
||||
|
||||
def compare_pages(a: dict[str, Any], b: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
pages_a = {p["page"] for p in a["layout"].get("pages", [])}
|
||||
pages_b = {p["page"] for p in b["layout"].get("pages", [])}
|
||||
rows = []
|
||||
for page in sorted(pages_a | pages_b):
|
||||
blocks_a = [x for x in a["layout"].get("blocks", []) if x["page"] == page and x["type"] in TEXT_TYPES]
|
||||
blocks_b = [x for x in b["layout"].get("blocks", []) if x["page"] == page and x["type"] in TEXT_TYPES]
|
||||
text = align(page_words(a["layout"], page), page_words(b["layout"], page))
|
||||
matched = match_boxes(blocks_a, blocks_b)
|
||||
images_a = [i for i in a["layout"].get("images", []) if i["page"] == page]
|
||||
images_b = [i for i in b["layout"].get("images", []) if i["page"] == page]
|
||||
figures = match_boxes([{"bbox": i["bbox"]} for i in images_a], [{"bbox": i["bbox"]} for i in images_b])
|
||||
row = {
|
||||
"page": page,
|
||||
"text_similarity": text["similarity"], "words_a": text["words_a"], "words_b": text["words_b"],
|
||||
"only_in_a": " ".join(text["only_in_a"][:40]), "only_in_b": " ".join(text["only_in_b"][:40]),
|
||||
"boxes_a": len(blocks_a), "boxes_b": len(blocks_b), "boxes_matched": len(matched),
|
||||
"box_recall": round(len(matched) / len(blocks_a), 4) if blocks_a else None,
|
||||
"box_precision": round(len(matched) / len(blocks_b), 4) if blocks_b else None,
|
||||
"type_agreement": round(sum(x["type"] == y["type"] for x, y, _ in matched) / len(matched), 4) if matched else None,
|
||||
"type_changes": sorted({f"{x['type']}->{y['type']}" for x, y, _ in matched if x["type"] != y["type"]}),
|
||||
"order_agreement": order_agreement(matched),
|
||||
"hand_ordered_in_a": bool(next((p for p in a["layout"].get("pages", []) if p["page"] == page), {}).get("order_locked")),
|
||||
"figures_a": len(images_a), "figures_b": len(images_b), "figures_matched": len(figures),
|
||||
"page_only_in": "a" if page not in pages_b else ("b" if page not in pages_a else ""),
|
||||
}
|
||||
flags = []
|
||||
if row["text_similarity"] < _SIMILAR_PAGE:
|
||||
flags.append("text")
|
||||
if row["box_recall"] is not None and row["box_recall"] < 0.9:
|
||||
flags.append("boxes_missing_in_new")
|
||||
if row["box_precision"] is not None and row["box_precision"] < 0.9:
|
||||
flags.append("extra_boxes_in_new")
|
||||
if row["type_agreement"] is not None and row["type_agreement"] < 0.95:
|
||||
flags.append("types")
|
||||
if row["order_agreement"] is not None and row["order_agreement"] < 0.95:
|
||||
flags.append("order")
|
||||
if row["figures_a"] != row["figures_b"]:
|
||||
flags.append("figures")
|
||||
row["flags"] = flags
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _article_text(magazine: dict[str, Any], article: dict[str, Any]) -> list[str]:
|
||||
ids = set(article.get("blocks") or [])
|
||||
blocks = sorted((b for b in magazine["layout"].get("blocks", []) if b["id"] in ids and b["type"] in NARRATED_TYPES),
|
||||
key=lambda b: (b["page"], b.get("reading_order", 0)))
|
||||
out: list[str] = []
|
||||
for block in blocks:
|
||||
out.extend(words(block.get("text", "")))
|
||||
return out
|
||||
|
||||
|
||||
def _shingles(ws: list[str], n: int = 3) -> set[tuple[str, ...]]:
|
||||
return {tuple(ws[i:i + n]) for i in range(len(ws) - n + 1)} if len(ws) >= n else {tuple(ws)} if ws else set()
|
||||
|
||||
|
||||
def compare_articles(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]:
|
||||
texts_a = {x["id"]: _article_text(a, x) for x in a["articles"]}
|
||||
texts_b = {x["id"]: _article_text(b, x) for x in b["articles"]}
|
||||
sh_a = {k: _shingles(v) for k, v in texts_a.items()}
|
||||
sh_b = {k: _shingles(v) for k, v in texts_b.items()}
|
||||
scores = []
|
||||
for ia, sa in sh_a.items():
|
||||
for ib, sb in sh_b.items():
|
||||
if not sa or not sb:
|
||||
continue
|
||||
inter = len(sa & sb)
|
||||
if inter:
|
||||
scores.append((inter / len(sa | sb), ia, ib))
|
||||
scores.sort(reverse=True)
|
||||
used_a: set[str] = set()
|
||||
used_b: set[str] = set()
|
||||
by_a = {x["id"]: x for x in a["articles"]}
|
||||
by_b = {x["id"]: x for x in b["articles"]}
|
||||
matched = []
|
||||
for score, ia, ib in scores:
|
||||
if score < 0.3 or ia in used_a or ib in used_b:
|
||||
continue
|
||||
used_a.add(ia)
|
||||
used_b.add(ib)
|
||||
xa, xb = by_a[ia], by_b[ib]
|
||||
diff = align(texts_a[ia], texts_b[ib])
|
||||
matched.append({
|
||||
"a": ia, "b": ib, "overlap": round(score, 4), "text_similarity": diff["similarity"],
|
||||
"title_a": xa.get("title", ""), "title_b": xb.get("title", ""),
|
||||
"author_a": xa.get("author", ""), "author_b": xb.get("author", ""),
|
||||
"pages_a": xa.get("pages", []), "pages_b": xb.get("pages", []),
|
||||
"status_a": xa.get("status"), "status_b": xb.get("status"),
|
||||
"same_pages": xa.get("pages", []) == xb.get("pages", []),
|
||||
})
|
||||
def brief(x: dict[str, Any], texts: dict[str, list[str]]) -> dict[str, Any]:
|
||||
return {"id": x["id"], "title": x.get("title", ""), "pages": x.get("pages", []), "words": len(texts[x["id"]])}
|
||||
return {
|
||||
"matched": matched,
|
||||
"only_in_a": [brief(x, texts_a) for x in a["articles"] if x["id"] not in used_a and texts_a[x["id"]]],
|
||||
"only_in_b": [brief(x, texts_b) for x in b["articles"] if x["id"] not in used_b and texts_b[x["id"]]],
|
||||
}
|
||||
|
||||
|
||||
def quality_numbers(m: dict[str, Any]) -> dict[str, Any]:
|
||||
rows = m["quality"].get("pages", [])
|
||||
return {
|
||||
"pages_needing_a_decision": sum(1 for r in rows if r.get("needs_review")),
|
||||
"unboxed_areas": sum(len(r.get("uncovered", [])) for r in rows),
|
||||
"clipped_edges": sum(len(r.get("clipped", [])) for r in rows),
|
||||
"overlapping_pairs": sum(len(r.get("overlaps", [])) for r in rows),
|
||||
"words_to_check": sum(len(r.get("spelling", [])) for r in rows),
|
||||
}
|
||||
|
||||
|
||||
def mean(values: list[float]) -> float | None:
|
||||
values = [v for v in values if v is not None]
|
||||
return round(sum(values) / len(values), 4) if values else None
|
||||
|
||||
|
||||
def compare(a: dict[str, Any], b: dict[str, Any]) -> dict[str, Any]:
|
||||
pages = compare_pages(a, b)
|
||||
articles = compare_articles(a, b)
|
||||
words_a = sum(p["words_a"] for p in pages)
|
||||
words_b = sum(p["words_b"] for p in pages)
|
||||
summary = {
|
||||
"baseline": a["folder"], "new": b["folder"],
|
||||
"pages": len(pages),
|
||||
"words": {"baseline": words_a, "new": words_b},
|
||||
"text_similarity_mean": mean([p["text_similarity"] for p in pages]),
|
||||
"pages_with_text_differences": sum("text" in p["flags"] for p in pages),
|
||||
"box_recall_mean": mean([p["box_recall"] for p in pages]),
|
||||
"box_precision_mean": mean([p["box_precision"] for p in pages]),
|
||||
"type_agreement_mean": mean([p["type_agreement"] for p in pages]),
|
||||
"order_agreement_mean": mean([p["order_agreement"] for p in pages]),
|
||||
"pages_flagged": sum(bool(p["flags"]) for p in pages),
|
||||
"figures": {"baseline": sum(p["figures_a"] for p in pages), "new": sum(p["figures_b"] for p in pages)},
|
||||
"articles": {"baseline": len(a["articles"]), "new": len(b["articles"]),
|
||||
"matched": len(articles["matched"]), "only_in_baseline": len(articles["only_in_a"]),
|
||||
"only_in_new": len(articles["only_in_b"]),
|
||||
"same_page_span": sum(x["same_pages"] for x in articles["matched"]),
|
||||
"same_title": sum(x["title_a"].strip().casefold() == x["title_b"].strip().casefold() for x in articles["matched"])},
|
||||
"quality": {"baseline": quality_numbers(a), "new": quality_numbers(b)},
|
||||
}
|
||||
return {"summary": summary, "pages": pages, "articles": articles}
|
||||
|
||||
|
||||
def markdown(report: dict[str, Any], worst: int = 15) -> str:
|
||||
s = report["summary"]
|
||||
lines = ["# Magazine comparison", "", f"* baseline: `{s['baseline']}`", f"* new: `{s['new']}`", "",
|
||||
"| measure | value |", "|---|---|",
|
||||
f"| pages | {s['pages']} |",
|
||||
f"| narratable words (baseline / new) | {s['words']['baseline']} / {s['words']['new']} |",
|
||||
f"| text similarity (mean per page) | {s['text_similarity_mean']} |",
|
||||
f"| pages with text differences | {s['pages_with_text_differences']} |",
|
||||
f"| box recall / precision | {s['box_recall_mean']} / {s['box_precision_mean']} |",
|
||||
f"| type agreement | {s['type_agreement_mean']} |",
|
||||
f"| reading-order agreement | {s['order_agreement_mean']} |",
|
||||
f"| figures (baseline / new) | {s['figures']['baseline']} / {s['figures']['new']} |",
|
||||
f"| articles (baseline / new / matched) | {s['articles']['baseline']} / {s['articles']['new']} / {s['articles']['matched']} |",
|
||||
f"| matched articles with the same page span | {s['articles']['same_page_span']} |",
|
||||
f"| pages flagged | {s['pages_flagged']} |", "", "## Quality report", "",
|
||||
"| | baseline | new |", "|---|---|---|"]
|
||||
for key in s["quality"]["baseline"]:
|
||||
lines.append(f"| {key.replace('_', ' ')} | {s['quality']['baseline'][key]} | {s['quality']['new'][key]} |")
|
||||
flagged = sorted((p for p in report["pages"] if p["flags"]), key=lambda p: (p["text_similarity"], p["page"]))[:worst]
|
||||
lines += ["", f"## The {len(flagged)} pages that differ most", "", "| page | flags | text | boxes b/n | order |", "|---|---|---|---|---|"]
|
||||
for p in flagged:
|
||||
lines.append(f"| {p['page']} | {', '.join(p['flags'])} | {p['text_similarity']} | {p['boxes_a']}/{p['boxes_b']} | {p['order_agreement']} |")
|
||||
for title, key in (("Only in the baseline", "only_in_a"), ("Only in the new import", "only_in_b")):
|
||||
rows = report["articles"][key]
|
||||
lines += ["", f"## Articles {title.lower()} ({len(rows)})", ""]
|
||||
lines += [f"* {r['title'][:60]!r} pages {r['pages']} ({r['words']} words)" for r in rows[:25]]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("baseline", type=Path)
|
||||
parser.add_argument("new", type=Path)
|
||||
parser.add_argument("--out", type=Path, help="write the full JSON report here")
|
||||
parser.add_argument("--md", type=Path, help="write a short Markdown summary here")
|
||||
args = parser.parse_args(argv)
|
||||
report = compare(load_magazine(args.baseline), load_magazine(args.new))
|
||||
if args.out:
|
||||
args.out.write_text(json.dumps(report, ensure_ascii=False, indent=1), "utf-8")
|
||||
text = markdown(report)
|
||||
if args.md:
|
||||
args.md.write_text(text, "utf-8")
|
||||
sys.stdout.write(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
108
tests/test_compare_magazines.py
Normal file
108
tests/test_compare_magazines.py
Normal file
@ -0,0 +1,108 @@
|
||||
"""scripts/compare_magazines.py: two folders with known differences give the expected numbers."""
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
||||
import compare_magazines as cm # noqa: E402
|
||||
|
||||
|
||||
def blk(i, page, ty, text, bbox, order):
|
||||
return {"id": i, "page": page, "type": ty, "text": text, "bbox": bbox, "reading_order": order}
|
||||
|
||||
|
||||
def art(i, title, pages, blocks, status="approved", author=""):
|
||||
return {"id": i, "title": title, "author": author, "pages": pages, "blocks": blocks, "images": [], "status": status}
|
||||
|
||||
|
||||
class CompareTests(unittest.TestCase):
|
||||
def folder(self, blocks, articles, images=(), quality=None):
|
||||
d = Path(tempfile.mkdtemp(prefix="ttvc-cmp-"))
|
||||
self.addCleanup(shutil.rmtree, d, True)
|
||||
pages = sorted({b["page"] for b in blocks})
|
||||
(d / "layout.json").write_text(json.dumps({"pages": [{"page": p} for p in pages], "blocks": blocks, "images": list(images)}))
|
||||
(d / "articles.json").write_text(json.dumps({"articles": articles}))
|
||||
if quality:
|
||||
(d / "quality_report.json").write_text(json.dumps(quality))
|
||||
return d
|
||||
|
||||
def base(self):
|
||||
return self.folder(
|
||||
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1),
|
||||
blk("b", 1, "body", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)],
|
||||
[art("article-001", "Der Butler", [1, 2], ["t", "a", "b", "c"], author="Jo")])
|
||||
|
||||
def test_identical_folders_score_perfectly(self):
|
||||
report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(self.base()))
|
||||
s = report["summary"]
|
||||
self.assertEqual((s["text_similarity_mean"], s["box_recall_mean"], s["box_precision_mean"], s["type_agreement_mean"], s["order_agreement_mean"]),
|
||||
(1.0, 1.0, 1.0, 1.0, 1.0))
|
||||
self.assertEqual((s["pages_flagged"], s["articles"]["matched"], s["articles"]["only_in_baseline"], s["articles"]["only_in_new"]), (0, 1, 0, 0))
|
||||
self.assertEqual(s["articles"]["same_page_span"], 1)
|
||||
|
||||
def test_a_misread_word_a_retyped_box_and_a_changed_reading_order_are_each_reported(self):
|
||||
new = self.folder(
|
||||
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 2), # order swapped
|
||||
blk("b", 1, "caption", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 1), # retyped: no longer narrated
|
||||
blk("c", 2, "body", "Und die zwelte Seite geht weiter.", [10, 10, 90, 100], 0)], # misread word
|
||||
[art("article-001", "Der Butler", [1, 2], ["t", "a", "b", "c"], author="Jo")])
|
||||
report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))
|
||||
page1 = next(p for p in report["pages"] if p["page"] == 1)
|
||||
self.assertLess(page1["text_similarity"], 1.0)
|
||||
self.assertIn("zweite", page1["only_in_a"]) # the retyped box's words are no longer read
|
||||
self.assertEqual(page1["only_in_b"], "")
|
||||
self.assertEqual(page1["type_changes"], ["body->caption"])
|
||||
self.assertLess(page1["order_agreement"], 1.0)
|
||||
self.assertEqual(set(page1["flags"]), {"text", "types", "order"})
|
||||
page2 = next(p for p in report["pages"] if p["page"] == 2)
|
||||
self.assertEqual((page2["only_in_a"], page2["only_in_b"], page2["flags"]), ("zweite", "zwelte", ["text"]))
|
||||
self.assertEqual(report["summary"]["pages_flagged"], 2)
|
||||
|
||||
def test_missing_and_extra_boxes_and_figures_show_up_as_recall_and_precision(self):
|
||||
new = self.folder(
|
||||
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1),
|
||||
blk("x", 1, "body", "Ein ganz anderer Kasten.", [300, 300, 50, 50], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)],
|
||||
[art("article-001", "Der Butler", [1, 2], ["t", "a", "x", "c"], author="Jo")],
|
||||
images=[{"id": "i", "page": 1, "bbox": [0, 200, 100, 100]}])
|
||||
page1 = next(p for p in cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))["pages"] if p["page"] == 1)
|
||||
self.assertEqual((page1["boxes_a"], page1["boxes_b"], page1["boxes_matched"]), (3, 3, 2))
|
||||
self.assertAlmostEqual(page1["box_recall"], 2 / 3, places=3)
|
||||
self.assertAlmostEqual(page1["box_precision"], 2 / 3, places=3)
|
||||
self.assertIn("boxes_missing_in_new", page1["flags"])
|
||||
self.assertIn("figures", page1["flags"])
|
||||
|
||||
def test_articles_are_matched_by_text_even_when_split_differently(self):
|
||||
one = self.base()
|
||||
split = self.folder(
|
||||
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1),
|
||||
blk("b", 1, "body", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)],
|
||||
[art("article-001", "Der Butler", [1], ["t", "a", "b"], status="detected"), art("article-002", "Page 2", [2], ["c"])])
|
||||
report = cm.compare(cm.load_magazine(one), cm.load_magazine(split))["articles"]
|
||||
self.assertEqual(len(report["matched"]), 1) # the bigger piece matches
|
||||
self.assertFalse(report["matched"][0]["same_pages"])
|
||||
self.assertEqual([a["title"] for a in report["only_in_b"]], ["Page 2"])
|
||||
|
||||
def test_quality_numbers_and_the_markdown_summary(self):
|
||||
q = {"pages": [{"page": 1, "needs_review": True, "uncovered": [[0, 0, 1, 1]], "clipped": [{}, {}], "overlaps": [], "spelling": ["a"]}]}
|
||||
new = self.folder(json.loads((self.base() / "layout.json").read_text())["blocks"], [], quality=q)
|
||||
report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))
|
||||
self.assertEqual(report["summary"]["quality"]["new"], {"pages_needing_a_decision": 1, "unboxed_areas": 1, "clipped_edges": 2,
|
||||
"overlapping_pairs": 0, "words_to_check": 1})
|
||||
text = cm.markdown(report)
|
||||
self.assertIn("# Magazine comparison", text)
|
||||
self.assertIn("| clipped edges | 0 | 2 |", text)
|
||||
|
||||
def test_the_command_line_writes_json_and_markdown(self):
|
||||
out = Path(tempfile.mkdtemp(prefix="ttvc-cmp-out-"))
|
||||
self.addCleanup(shutil.rmtree, out, True)
|
||||
code = cm.main([str(self.base()), str(self.base()), "--out", str(out / "r.json"), "--md", str(out / "r.md")])
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(json.loads((out / "r.json").read_text())["summary"]["pages"], 2)
|
||||
self.assertTrue((out / "r.md").read_text().startswith("# Magazine comparison"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@ -1898,3 +1898,24 @@ class BoxFixesReorderThePageTests(unittest.TestCase):
|
||||
"blocks": [blk("r1", 110, 10, 0), blk("l1", 10, 10, 1), blk("r2", 110, 80, 2), blk("l2", 10, 80, 3)]}
|
||||
fixed, _, _ = _fix_page_geometry(d, layout, [], 1, {})
|
||||
self.assertEqual([b["id"] for b in sorted(fixed["blocks"], key=lambda b: b["reading_order"])], ["l1", "l2", "r1", "r2"])
|
||||
|
||||
|
||||
class FastLlmCallTests(unittest.TestCase):
|
||||
def test_thinking_is_switched_off_and_a_rejecting_backend_is_asked_again_without_the_switch(self):
|
||||
from unittest.mock import MagicMock, patch
|
||||
from routes.magazine import _llm_chat_fast
|
||||
bad, good = MagicMock(status_code=422), MagicMock(status_code=200)
|
||||
with patch('routes.conversation._post_llm_chat_completion', side_effect=[bad, good]) as post:
|
||||
out = _llm_chat_fast({"llm_model": "m", "llm_url": "http://x"}, [{"role": "user", "content": "hi"}])
|
||||
self.assertIs(out, good)
|
||||
first, second = post.call_args_list[0].args[1], post.call_args_list[1].args[1]
|
||||
self.assertEqual(first["chat_template_kwargs"], {"enable_thinking": False})
|
||||
self.assertEqual((first["temperature"], first["max_tokens"]), (0.0, 800))
|
||||
self.assertNotIn("chat_template_kwargs", second)
|
||||
|
||||
def test_a_backend_that_accepts_it_is_called_once(self):
|
||||
from unittest.mock import MagicMock, patch
|
||||
from routes.magazine import _llm_chat_fast
|
||||
with patch('routes.conversation._post_llm_chat_completion', return_value=MagicMock(status_code=200)) as post:
|
||||
_llm_chat_fast({"llm_model": "m"}, [])
|
||||
self.assertEqual(post.call_count, 1)
|
||||
|
||||
@ -132,6 +132,17 @@ class NamesBrandsAndVocabularyTests(unittest.TestCase):
|
||||
self.assertEqual([i["token"] for i in found], ["wachst"])
|
||||
|
||||
|
||||
class RunTogetherWordTests(unittest.TestCase):
|
||||
@patched
|
||||
def test_run_together_words_are_repaired_by_splitting_them_at_a_space(self):
|
||||
issue = {"token": "dasich", "language": "de", "suggestions": []}
|
||||
with mock.patch("core.pronunciations._german_spellchecker", return_value=Checker(DE | {"das", "ich", "dasic"})):
|
||||
self.assertTrue(spelling.accept_agent_fix(issue, "das ich"))
|
||||
self.assertFalse(spelling.accept_agent_fix(issue, "das ihn")) # letters changed
|
||||
self.assertFalse(spelling.accept_agent_fix(issue, "d asich")) # a half is not a word
|
||||
self.assertFalse(spelling.accept_agent_fix({**issue, "token": "dasichx"}, "das ich"))
|
||||
|
||||
|
||||
class AgentGuardTests(unittest.TestCase):
|
||||
ISSUE = {"token": "Tesst", "language": "de", "suggestions": ["Tests", "Test"]}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user