"""Magazine layout analysis regressions. These cover the pure geometry/classification logic rather than PyMuPDF I/O: reading order across columns and article boundary detection are what decide whether an article is narrated from its true beginning to its true end, so they are the parts worth locking down. """ from __future__ import annotations import os import tempfile import unittest os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-magazine-test-config-")) from core.magazine import ( BLOCK_AD, BLOCK_BODY, BLOCK_BYLINE, BLOCK_CAPTION, BLOCK_DECK, BLOCK_FURNITURE, BLOCK_PULLQUOTE, BLOCK_TITLE, Block, article_text, assign_reading_order, body_font_size, build_articles, classify_blocks, detect_columns, estimate_listen_seconds, find_continuation, find_repeated_lines, group_articles, looks_like_byline, mark_advert_pages, mark_pullquotes, parse_contents, score_contents_page, ) PAGE_W, PAGE_H = 595.0, 842.0 # A4 in points def block(bid, page, x, y, w, h, text, size=10.0, btype=BLOCK_BODY): return Block(id=bid, page=page, bbox=[x, y, w, h], type=btype, text=text, font_size=size) class ColumnGeometryTests(unittest.TestCase): def test_two_columns_detected_from_the_gutter(self): blocks = [ block("a", 1, 50, 100, 200, 300, "left copy"), block("b", 1, 320, 100, 200, 300, "right copy"), ] cols = detect_columns(blocks, PAGE_W) self.assertEqual(len(cols), 2) self.assertLess(cols[0][1], cols[1][0]) def test_single_column_when_no_gutter(self): blocks = [ block("a", 1, 50, 100, 480, 100, "wide copy"), block("b", 1, 50, 220, 480, 100, "more wide copy"), ] self.assertEqual(len(detect_columns(blocks, PAGE_W)), 1) def test_reading_order_finishes_a_column_before_starting_the_next(self): """The failure this prevents: interleaving columns by y, which splices the left column's sentence into the right column's.""" blocks = [ block("l1", 1, 50, 100, 200, 80, "left top"), block("r1", 1, 320, 100, 200, 80, "right top"), block("l2", 1, 50, 200, 200, 80, "left bottom"), block("r2", 1, 320, 200, 200, 80, "right bottom"), ] order = [b.id for b in assign_reading_order(blocks, PAGE_W)] self.assertEqual(order, ["l1", "l2", "r1", "r2"]) def test_full_width_headline_precedes_both_columns(self): blocks = [ block("l1", 1, 50, 200, 200, 80, "left copy"), block("r1", 1, 320, 200, 200, 80, "right copy"), block("head", 1, 50, 80, 470, 60, "The Future of Batteries", size=24.0), ] order = [b.id for b in assign_reading_order(blocks, PAGE_W)] self.assertEqual(order[0], "head") class ClassificationTests(unittest.TestCase): def test_body_size_is_not_hijacked_by_display_type(self): blocks = [ block("t", 1, 50, 50, 400, 60, "HUGE HEADLINE", size=40.0), block("b1", 1, 50, 150, 200, 200, "x" * 400, size=10.0), block("b2", 1, 320, 150, 200, 200, "y" * 400, size=10.0), ] self.assertAlmostEqual(body_font_size(blocks), 10.0, places=1) def test_headline_byline_and_body_are_separated(self): blocks = [ block("t", 1, 50, 60, 470, 50, "The Future of Batteries", size=26.0), block("by", 1, 50, 120, 200, 14, "By Jane Smith", size=10.0), block("b", 1, 50, 150, 200, 300, "Battery technology " * 40, size=10.0), ] classify_blocks(blocks, PAGE_H, PAGE_W) kinds = {b.id: b.type for b in blocks} self.assertEqual(kinds["t"], BLOCK_TITLE) self.assertEqual(kinds["by"], BLOCK_BYLINE) self.assertEqual(kinds["b"], BLOCK_BODY) def test_running_head_and_folio_become_furniture(self): blocks = [ block("head", 1, 50, 20, 200, 12, "SCIENTIFIC AMERICAN", size=8.0), block("folio", 1, 520, 820, 20, 12, "18", size=8.0), block("b", 1, 50, 150, 200, 300, "Body copy " * 40, size=10.0), ] classify_blocks(blocks, PAGE_H, PAGE_W, repeated={"SCIENTIFIC AMERICAN"}) kinds = {b.id: b.type for b in blocks} self.assertEqual(kinds["head"], BLOCK_FURNITURE) self.assertEqual(kinds["folio"], BLOCK_FURNITURE) self.assertEqual(kinds["b"], BLOCK_BODY) def test_small_text_next_to_a_figure_is_a_caption(self): """A photo page has no body copy of its own, so the baseline must come from the document -- otherwise the caption becomes its own reference.""" blocks = [block("cap", 1, 50, 420, 200, 20, "A lithium cell.", size=7.5)] classify_blocks(blocks, PAGE_H, PAGE_W, image_boxes=[[50, 200, 200, 210]], base_size=10.0) self.assertEqual(blocks[0].type, BLOCK_CAPTION) def test_small_text_away_from_any_figure_is_not_a_caption(self): blocks = [block("fn", 1, 50, 420, 200, 20, "See note 4.", size=7.5)] classify_blocks(blocks, PAGE_H, PAGE_W, image_boxes=[], base_size=10.0) self.assertNotEqual(blocks[0].type, BLOCK_CAPTION) def test_photo_page_caption_would_misclassify_on_a_per_page_baseline(self): """Guards the regression directly: no base_size means the lone caption measures itself and reads as body text.""" blocks = [block("cap", 1, 50, 420, 200, 20, "A lithium cell.", size=7.5)] classify_blocks(blocks, PAGE_H, PAGE_W, image_boxes=[[50, 200, 200, 210]]) self.assertEqual(blocks[0].type, BLOCK_BODY) def test_advertisement_marker_is_excluded(self): blocks = [block("ad", 1, 50, 50, 200, 20, "ADVERTISEMENT", size=9.0)] classify_blocks(blocks, PAGE_H, PAGE_W) self.assertEqual(blocks[0].type, BLOCK_AD) def test_an_advertisement_label_carries_its_whole_page(self): """The ad's own display type would otherwise read as a headline.""" blocks = [ block("m", 4, 50, 50, 200, 20, "ADVERTISEMENT", size=9.0), block("big", 4, 60, 400, 400, 60, "BUY OUR THING", size=40.0), block("folio", 4, 520, 820, 20, 12, "4", size=8.0), ] classify_blocks(blocks, PAGE_H, PAGE_W, repeated=set(), base_size=10.0) mark_advert_pages(blocks) kinds = {b.id: b.type for b in blocks} self.assertEqual(kinds["big"], BLOCK_AD) self.assertEqual(kinds["folio"], BLOCK_FURNITURE) def test_a_page_without_the_label_is_untouched(self): blocks = [ block("t", 3, 50, 60, 470, 50, "The Future of Batteries", size=26.0), block("b", 3, 50, 150, 200, 300, "Battery copy. " * 40, size=10.0), ] classify_blocks(blocks, PAGE_H, PAGE_W, base_size=10.0) mark_advert_pages(blocks) self.assertEqual(blocks[0].type, BLOCK_TITLE) def test_pullquote_repeating_body_copy_is_demoted(self): """Narrating a pull quote reads the same sentence twice.""" sentence = "Battery chemistry is entering an entirely new phase" blocks = [ block("b", 1, 50, 150, 200, 300, f"Some lead in. {sentence}. And more copy follows.", size=10.0), block("pq", 1, 320, 300, 200, 60, f"“{sentence}”", size=14.0), ] classify_blocks(blocks, PAGE_H, PAGE_W) mark_pullquotes(blocks) self.assertEqual(blocks[1].type, BLOCK_PULLQUOTE) def test_deck_that_does_not_repeat_body_stays_a_deck(self): blocks = [ block("b", 1, 50, 300, 200, 300, "Entirely different copy. " * 20, size=10.0), block("dk", 1, 50, 120, 470, 40, "Why the next decade belongs to solid state", size=14.0), ] classify_blocks(blocks, PAGE_H, PAGE_W) mark_pullquotes(blocks) self.assertEqual(blocks[1].type, BLOCK_DECK) def test_byline_rejects_a_sentence(self): self.assertEqual(looks_like_byline("By the time the cell charges, it is hot.")[0], False) self.assertEqual(looks_like_byline("By Jane Smith"), (True, "Jane Smith")) self.assertEqual(looks_like_byline("Von Jan Müller"), (True, "Jan Müller")) class FurnitureDetectionTests(unittest.TestCase): def test_lines_repeating_across_pages_are_furniture(self): pages = [["SCIENTIFIC AMERICAN", f"unique body {i}"] for i in range(10)] repeated = find_repeated_lines(pages) self.assertIn("SCIENTIFIC AMERICAN", repeated) self.assertNotIn("unique body 3", repeated) def test_no_false_positives_on_a_short_document(self): self.assertEqual(find_repeated_lines([["a"], ["b"]]), set()) class ContentsPageTests(unittest.TestCase): def test_contents_page_scores_above_a_feature_page(self): toc = [ "Contents", "Features", "The future of batteries ......... 18", "Inside the deep ocean ........... 30", "The new space race .............. 44", "Departments", "Books ........................... 60", "Reviews ......................... 64", ] feature = ["The Future of Batteries", "By Jane Smith", "Battery technology is entering a new phase. " * 20] self.assertGreater(score_contents_page(toc), 0.45) self.assertLess(score_contents_page(feature), 0.45) def test_entries_parse_with_section_and_page(self): entries = parse_contents([ "Contents", "Features", "The future of batteries ......... 18", "Inside the deep ocean ........... 30", "Departments", "Reviews ......................... 64", ]) self.assertEqual(len(entries), 3) self.assertEqual(entries[0], {"title": "The future of batteries", "page": 18, "section": "Features"}) self.assertEqual(entries[2]["section"], "Departments") def test_garbage_entries_are_rejected(self): self.assertEqual(parse_contents(["ab .... 99999", "x . 3"]), []) class ContinuationTests(unittest.TestCase): def test_continued_on_and_from_are_found(self): self.assertEqual(find_continuation("(continued on page 88)")[0], 88) self.assertEqual(find_continuation("Fortsetzung auf Seite 90")[0], 90) self.assertEqual(find_continuation("continued from page 27")[1], 27) def test_plain_text_has_no_continuation(self): self.assertEqual(find_continuation("The page turned slowly."), (None, None)) class ArticleGroupingTests(unittest.TestCase): def _issue_blocks(self): blocks = [ block("b0018-000", 18, 50, 60, 470, 50, "The Future of Batteries", size=26.0, btype=BLOCK_TITLE), block("b0018-001", 18, 50, 120, 200, 14, "By Jane Smith", size=10.0, btype=BLOCK_BYLINE), block("b0018-002", 18, 50, 150, 200, 300, "Battery copy. " * 40, size=10.0), block("b0030-000", 30, 50, 60, 470, 50, "Inside the Deep Ocean", size=26.0, btype=BLOCK_TITLE), block("b0030-001", 30, 50, 150, 200, 300, "Ocean copy. " * 40, size=10.0), ] for i, b in enumerate(blocks): b.reading_order = i return blocks def test_contents_entries_anchor_article_boundaries(self): contents = [ {"title": "The future of batteries", "page": 18, "section": "Features"}, {"title": "Inside the deep ocean", "page": 30, "section": "Features"}, ] arts = group_articles(self._issue_blocks(), contents, page_count=40) self.assertEqual(len(arts), 2) self.assertEqual(arts[0].pages[0], 18) self.assertEqual(arts[0].author, "Jane Smith") self.assertEqual(arts[1].pages[0], 30) # The first article must stop before the second begins. self.assertNotIn(30, arts[0].pages) def test_headline_starts_an_article_without_a_contents_entry(self): arts = group_articles(self._issue_blocks(), [], page_count=40) self.assertEqual(len(arts), 2) self.assertLess(arts[0].confidence, 0.9) # weaker signal, flagged for review def test_furniture_and_ads_never_enter_an_article(self): blocks = self._issue_blocks() ad = block("b0019-000", 19, 50, 50, 400, 400, "ADVERTISEMENT", size=9.0, btype=BLOCK_AD) folio = block("b0019-001", 19, 520, 820, 20, 12, "19", size=8.0, btype=BLOCK_FURNITURE) blocks.extend([ad, folio]) arts = group_articles(blocks, [{"title": "The future of batteries", "page": 18, "section": ""}], page_count=40) self.assertNotIn("b0019-000", arts[0].blocks) self.assertNotIn("b0019-001", arts[0].blocks) def test_an_advert_page_between_features_is_not_claimed(self): """The ad page starts no article, so it must not be swallowed by the preceding one and highlighted as part of that feature.""" blocks = self._issue_blocks() ad = block("b0019-000", 19, 50, 50, 400, 400, "ADVERTISEMENT", size=9.0, btype=BLOCK_AD) ad.reading_order = 50 blocks.append(ad) arts = group_articles(blocks, [ {"title": "The future of batteries", "page": 18, "section": ""}, {"title": "Inside the deep ocean", "page": 30, "section": ""}, ], page_count=40) self.assertNotIn(19, arts[0].pages) self.assertEqual(arts[0].pages, [18]) def test_a_jump_tail_is_absorbed_into_its_originating_article(self): blocks = self._issue_blocks() blocks[2].text += " (continued on page 88)" tail_title = block("b0088-000", 88, 50, 60, 470, 30, "Batteries, continued", size=18.0, btype=BLOCK_TITLE) tail_body = block("b0088-001", 88, 50, 110, 200, 300, "The rest of the battery copy. " * 20, size=10.0) tail_title.reading_order, tail_body.reading_order = 90, 91 blocks.extend([tail_title, tail_body]) arts = group_articles(blocks, [{"title": "The future of batteries", "page": 18, "section": ""}], page_count=90) first = arts[0] self.assertIn(88, first.pages) self.assertIn("b0088-001", first.blocks) # and the tail is no longer a standalone stub self.assertTrue(all("continued" not in a.title.lower() for a in arts[1:])) def test_a_page_with_only_a_headline_starts_no_article(self): """Covers and full-page adverts carry display type but no copy; each would otherwise become an entry the reviewer rejects on every issue.""" blocks = [block("b0005-000", 5, 50, 60, 470, 50, "Special Offer", size=30.0, btype=BLOCK_TITLE)] blocks[0].reading_order = 0 self.assertEqual(group_articles(blocks, [], page_count=10), []) def test_a_listed_article_with_no_body_copy_is_low_confidence(self): """A contents entry still anchors an article, but an empty one is flagged rather than silently trusted.""" blocks = [block("b0005-000", 5, 50, 60, 470, 50, "Special Offer", size=30.0, btype=BLOCK_TITLE)] blocks[0].reading_order = 0 arts = group_articles(blocks, [{"title": "Special Offer", "page": 5, "section": ""}], page_count=10) self.assertEqual(len(arts), 1) self.assertLess(arts[0].confidence, 0.4) def test_the_contents_page_is_not_itself_an_article(self): blocks = self._issue_blocks() toc = block("b0002-000", 2, 50, 90, 300, 40, "Contents", size=28.0, btype=BLOCK_TITLE) toc_entry = block("b0002-001", 2, 60, 175, 300, 20, "The future of batteries ......... 18", size=11.0) toc.reading_order, toc_entry.reading_order = 0, 1 blocks.extend([toc, toc_entry]) arts = group_articles(blocks, [], page_count=40, skip_pages={2}) self.assertTrue(all(a.pages[0] != 2 for a in arts)) class ArticleTextTests(unittest.TestCase): def test_only_narratable_blocks_in_reading_order(self): layout = { "page_count": 2, "blocks": [ {"id": "b2", "page": 1, "type": BLOCK_BODY, "reading_order": 2, "text": "second", "bbox": [0, 0, 1, 1]}, {"id": "b1", "page": 1, "type": BLOCK_TITLE, "reading_order": 1, "text": "first", "bbox": [0, 0, 1, 1]}, {"id": "cap", "page": 1, "type": BLOCK_CAPTION, "reading_order": 3, "text": "a caption", "bbox": [0, 0, 1, 1]}, {"id": "ad", "page": 1, "type": BLOCK_AD, "reading_order": 4, "text": "buy this", "bbox": [0, 0, 1, 1]}, ], } article = {"blocks": ["b1", "b2", "cap", "ad"]} got = [b["text"] for b in article_text(layout, article)] self.assertEqual(got, ["first", "second"]) with_caps = [b["text"] for b in article_text(layout, article, include_captions=True)] self.assertEqual(with_caps, ["first", "second", "a caption"]) def test_listening_estimate_counts_only_narratable_words(self): blocks = [ Block(id="b1", page=1, bbox=[0, 0, 1, 1], type=BLOCK_BODY, text=" ".join(["word"] * 155)), Block(id="ad", page=1, bbox=[0, 0, 1, 1], type=BLOCK_AD, text=" ".join(["spam"] * 500)), ] from core.magazine import Article art = Article(id="a1", title="t", blocks=["b1", "ad"]) self.assertEqual(estimate_listen_seconds(blocks, art), 60) class BuildArticlesTests(unittest.TestCase): def test_best_contents_candidate_drives_grouping(self): layout = { "page_count": 40, "blocks": [ {"id": "b0018-000", "page": 18, "type": BLOCK_TITLE, "reading_order": 0, "text": "The Future of Batteries", "bbox": [50, 60, 470, 50], "font_size": 26.0, "font_name": "", "bold": True, "column": 0, "confidence": 0.9, "source": "native"}, {"id": "b0018-001", "page": 18, "type": BLOCK_BODY, "reading_order": 1, "text": "Battery copy. " * 40, "bbox": [50, 150, 200, 300], "font_size": 10.0, "font_name": "", "bold": False, "column": 0, "confidence": 0.85, "source": "native"}, ], "contents_candidates": [ {"page": 2, "score": 0.5, "entries": []}, {"page": 3, "score": 0.9, "entries": [ {"title": "The future of batteries", "page": 18, "section": "Features"}]}, ], } arts = build_articles(layout) self.assertEqual(len(arts), 1) self.assertEqual(arts[0]["section"], "Features") self.assertGreater(arts[0]["estimated_seconds"], 0) if __name__ == "__main__": unittest.main()