109 lines
6.9 KiB
Python
109 lines
6.9 KiB
Python
"""scripts/compare_magazines.py: two folders with known differences give the expected numbers."""
|
|
import json
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
|
import compare_magazines as cm # noqa: E402
|
|
|
|
|
|
def blk(i, page, ty, text, bbox, order):
|
|
return {"id": i, "page": page, "type": ty, "text": text, "bbox": bbox, "reading_order": order}
|
|
|
|
|
|
def art(i, title, pages, blocks, status="approved", author=""):
|
|
return {"id": i, "title": title, "author": author, "pages": pages, "blocks": blocks, "images": [], "status": status}
|
|
|
|
|
|
class CompareTests(unittest.TestCase):
|
|
def folder(self, blocks, articles, images=(), quality=None):
|
|
d = Path(tempfile.mkdtemp(prefix="ttvc-cmp-"))
|
|
self.addCleanup(shutil.rmtree, d, True)
|
|
pages = sorted({b["page"] for b in blocks})
|
|
(d / "layout.json").write_text(json.dumps({"pages": [{"page": p} for p in pages], "blocks": blocks, "images": list(images)}))
|
|
(d / "articles.json").write_text(json.dumps({"articles": articles}))
|
|
if quality:
|
|
(d / "quality_report.json").write_text(json.dumps(quality))
|
|
return d
|
|
|
|
def base(self):
|
|
return self.folder(
|
|
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1),
|
|
blk("b", 1, "body", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)],
|
|
[art("article-001", "Der Butler", [1, 2], ["t", "a", "b", "c"], author="Jo")])
|
|
|
|
def test_identical_folders_score_perfectly(self):
|
|
report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(self.base()))
|
|
s = report["summary"]
|
|
self.assertEqual((s["text_similarity_mean"], s["box_recall_mean"], s["box_precision_mean"], s["type_agreement_mean"], s["order_agreement_mean"]),
|
|
(1.0, 1.0, 1.0, 1.0, 1.0))
|
|
self.assertEqual((s["pages_flagged"], s["articles"]["matched"], s["articles"]["only_in_baseline"], s["articles"]["only_in_new"]), (0, 1, 0, 0))
|
|
self.assertEqual(s["articles"]["same_page_span"], 1)
|
|
|
|
def test_a_misread_word_a_retyped_box_and_a_changed_reading_order_are_each_reported(self):
|
|
new = self.folder(
|
|
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 2), # order swapped
|
|
blk("b", 1, "caption", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 1), # retyped: no longer narrated
|
|
blk("c", 2, "body", "Und die zwelte Seite geht weiter.", [10, 10, 90, 100], 0)], # misread word
|
|
[art("article-001", "Der Butler", [1, 2], ["t", "a", "b", "c"], author="Jo")])
|
|
report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))
|
|
page1 = next(p for p in report["pages"] if p["page"] == 1)
|
|
self.assertLess(page1["text_similarity"], 1.0)
|
|
self.assertIn("zweite", page1["only_in_a"]) # the retyped box's words are no longer read
|
|
self.assertEqual(page1["only_in_b"], "")
|
|
self.assertEqual(page1["type_changes"], ["body->caption"])
|
|
self.assertLess(page1["order_agreement"], 1.0)
|
|
self.assertEqual(set(page1["flags"]), {"text", "types", "order"})
|
|
page2 = next(p for p in report["pages"] if p["page"] == 2)
|
|
self.assertEqual((page2["only_in_a"], page2["only_in_b"], page2["flags"]), ("zweite", "zwelte", ["text"]))
|
|
self.assertEqual(report["summary"]["pages_flagged"], 2)
|
|
|
|
def test_missing_and_extra_boxes_and_figures_show_up_as_recall_and_precision(self):
|
|
new = self.folder(
|
|
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1),
|
|
blk("x", 1, "body", "Ein ganz anderer Kasten.", [300, 300, 50, 50], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)],
|
|
[art("article-001", "Der Butler", [1, 2], ["t", "a", "x", "c"], author="Jo")],
|
|
images=[{"id": "i", "page": 1, "bbox": [0, 200, 100, 100]}])
|
|
page1 = next(p for p in cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))["pages"] if p["page"] == 1)
|
|
self.assertEqual((page1["boxes_a"], page1["boxes_b"], page1["boxes_matched"]), (3, 3, 2))
|
|
self.assertAlmostEqual(page1["box_recall"], 2 / 3, places=3)
|
|
self.assertAlmostEqual(page1["box_precision"], 2 / 3, places=3)
|
|
self.assertIn("boxes_missing_in_new", page1["flags"])
|
|
self.assertIn("figures", page1["flags"])
|
|
|
|
def test_articles_are_matched_by_text_even_when_split_differently(self):
|
|
one = self.base()
|
|
split = self.folder(
|
|
[blk("t", 1, "title", "Der Butler", [10, 10, 200, 30], 0), blk("a", 1, "body", "Der erste Absatz des Textes.", [10, 50, 90, 100], 1),
|
|
blk("b", 1, "body", "Der zweite Absatz folgt hier.", [120, 50, 90, 100], 2), blk("c", 2, "body", "Und die zweite Seite geht weiter.", [10, 10, 90, 100], 0)],
|
|
[art("article-001", "Der Butler", [1], ["t", "a", "b"], status="detected"), art("article-002", "Page 2", [2], ["c"])])
|
|
report = cm.compare(cm.load_magazine(one), cm.load_magazine(split))["articles"]
|
|
self.assertEqual(len(report["matched"]), 1) # the bigger piece matches
|
|
self.assertFalse(report["matched"][0]["same_pages"])
|
|
self.assertEqual([a["title"] for a in report["only_in_b"]], ["Page 2"])
|
|
|
|
def test_quality_numbers_and_the_markdown_summary(self):
|
|
q = {"pages": [{"page": 1, "needs_review": True, "uncovered": [[0, 0, 1, 1]], "clipped": [{}, {}], "overlaps": [], "spelling": ["a"]}]}
|
|
new = self.folder(json.loads((self.base() / "layout.json").read_text())["blocks"], [], quality=q)
|
|
report = cm.compare(cm.load_magazine(self.base()), cm.load_magazine(new))
|
|
self.assertEqual(report["summary"]["quality"]["new"], {"pages_needing_a_decision": 1, "unboxed_areas": 1, "clipped_edges": 2,
|
|
"overlapping_pairs": 0, "words_to_check": 1})
|
|
text = cm.markdown(report)
|
|
self.assertIn("# Magazine comparison", text)
|
|
self.assertIn("| clipped edges | 0 | 2 |", text)
|
|
|
|
def test_the_command_line_writes_json_and_markdown(self):
|
|
out = Path(tempfile.mkdtemp(prefix="ttvc-cmp-out-"))
|
|
self.addCleanup(shutil.rmtree, out, True)
|
|
code = cm.main([str(self.base()), str(self.base()), "--out", str(out / "r.json"), "--md", str(out / "r.md")])
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(json.loads((out / "r.json").read_text())["summary"]["pages"], 2)
|
|
self.assertTrue((out / "r.md").read_text().startswith("# Magazine comparison"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|