mARTin-B78
55f147d7a7
Some checks are pending
Test and SonarQube / quality (push) Waiting to run
A vision OCR engine grounds a figure far less precisely than it
transcribes one. Over all 138 figure regions of a 204-page issue the
transcribed contents were right while the boxes were routinely offset,
undersized, or sitting on the neighbouring text column -- page 23's card
grid was reported at x 0.24-0.54 where the graphic runs 0.41-0.95.
snap_visual_bboxes() reduces the page render to an ink-coverage grid,
seeds a connected-component search with the engine's box, and re-joins
tiles a gutter apart so a card grid comes back as one figure. Text
blocks are excluded from the mask, which is what stops a figure
swallowing the column beside it. A candidate must be solidly inked or
framed on nearly all of its rows and columns (admitting a white-interior
screenshot), may not shrink below 30% of the seed, and reverts when two
figures land on the same graphic; anything else keeps the box it had.
Runs before the figure crop in both OCR paths, so the images shown
during playback are framed correctly too. Replaces refine_picture_
bounds(), which only recognised coloured panels and had no text fence.
Also adds "Fix figure boundaries" in review and POST
/api/magazine/{id}/figures/refine behind it, so an issue ingested before
this pass can be corrected without re-running OCR. Verified live against
c't KI Wissen 2026: 78 of 138 boxes corrected, the rest declined.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
221 lines
10 KiB
Python
221 lines
10 KiB
Python
"""Magazine route-layer regressions: path safety, review gating, script parsing."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
|
|
_CONFIG = tempfile.mkdtemp(prefix="ttvc-magazine-routes-test-")
|
|
os.environ.setdefault("CONFIG_DIR", _CONFIG)
|
|
|
|
from fastapi import HTTPException
|
|
|
|
from routes.magazine import _article_id, _doc_dir, _parse_script
|
|
|
|
|
|
class PathSafetyTests(unittest.TestCase):
|
|
def test_traversal_and_junk_ids_are_rejected(self):
|
|
for bad in ("../../etc", "a/b", "", "..", "x" * 41, "id-with-dash", "a.b"):
|
|
with self.assertRaises(HTTPException) as ctx:
|
|
_doc_dir(bad)
|
|
self.assertEqual(ctx.exception.status_code, 400, bad)
|
|
|
|
def test_wellformed_but_missing_id_is_a_404_not_a_400(self):
|
|
with self.assertRaises(HTTPException) as ctx:
|
|
_doc_dir("0123456789abcdef0123456789abcdef")
|
|
self.assertEqual(ctx.exception.status_code, 404)
|
|
|
|
def test_article_ids_reject_separators(self):
|
|
self.assertEqual(_article_id("article-001"), "article-001")
|
|
for bad in ("../x", "a/b", "", "a" * 65, "a b"):
|
|
with self.assertRaises(HTTPException):
|
|
_article_id(bad)
|
|
|
|
|
|
class PodcastScriptParsingTests(unittest.TestCase):
|
|
def test_plain_json_parses(self):
|
|
raw = json.dumps({
|
|
"title": "Batteries",
|
|
"turns": [{"host": "A", "text": "Hello."},
|
|
{"host": "B", "text": "Hi."}],
|
|
})
|
|
got = _parse_script(raw, ["A", "B"])
|
|
self.assertEqual(got["title"], "Batteries")
|
|
self.assertEqual([t["host"] for t in got["turns"]], ["A", "B"])
|
|
|
|
def test_markdown_fence_is_tolerated(self):
|
|
raw = '```json\n{"title":"T","turns":[{"host":"A","text":"Hi."}]}\n```'
|
|
self.assertEqual(_parse_script(raw, ["A", "B"])["turns"][0]["text"], "Hi.")
|
|
|
|
def test_prose_around_the_json_is_stripped(self):
|
|
raw = 'Sure! Here is the script:\n{"title":"T","turns":[{"host":"A","text":"Hi."}]}\nHope that helps.'
|
|
self.assertEqual(_parse_script(raw, ["A", "B"])["title"], "T")
|
|
|
|
def test_unknown_host_label_is_reassigned_not_dropped(self):
|
|
"""An unrecognised label would otherwise leave a turn with no voice."""
|
|
raw = json.dumps({"turns": [
|
|
{"host": "NARRATOR", "text": "One."},
|
|
{"host": "B", "text": "Two."},
|
|
]})
|
|
got = _parse_script(raw, ["A", "B"])
|
|
self.assertEqual(len(got["turns"]), 2)
|
|
self.assertTrue(all(t["host"] in ("A", "B") for t in got["turns"]))
|
|
|
|
def test_empty_turns_are_skipped(self):
|
|
raw = json.dumps({"turns": [
|
|
{"host": "A", "text": " "},
|
|
{"host": "B", "text": "Real line."},
|
|
]})
|
|
got = _parse_script(raw, ["A", "B"])
|
|
self.assertEqual(len(got["turns"]), 1)
|
|
|
|
def test_unusable_output_raises_502_not_500(self):
|
|
for raw in ("not json at all", "{}", json.dumps({"turns": []}),
|
|
json.dumps({"turns": [{"host": "A", "text": ""}]})):
|
|
with self.assertRaises(HTTPException) as ctx:
|
|
_parse_script(raw, ["A", "B"])
|
|
self.assertEqual(ctx.exception.status_code, 502, raw)
|
|
|
|
def test_overlong_turn_text_is_capped(self):
|
|
raw = json.dumps({"turns": [{"host": "A", "text": "x" * 5000}]})
|
|
self.assertEqual(len(_parse_script(raw, ["A"])["turns"][0]["text"]), 2000)
|
|
|
|
|
|
def tearDownModule():
|
|
shutil.rmtree(_CONFIG, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
class ReviewImageTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_image_selection_is_saved_and_unknown_ids_rejected(self):
|
|
from unittest.mock import AsyncMock, patch
|
|
from routes.magazine import magazine_update_article
|
|
article = {"id": "article-001", "images": []}
|
|
with patch('routes.magazine._require_magazine', return_value=(None, {})), \
|
|
patch('routes.magazine.mag.load_articles', return_value=[article]), \
|
|
patch('routes.magazine.mag.load_layout', return_value={"images": [{"id": "chart"}]}), \
|
|
patch('routes.magazine.mag.save_articles') as save:
|
|
request = AsyncMock()
|
|
request.json.return_value = {"images": ["chart"]}
|
|
result = await magazine_update_article('a' * 32, 'article-001', request)
|
|
self.assertEqual(result['images'], ['chart'])
|
|
save.assert_called_once()
|
|
save.reset_mock()
|
|
request.json.return_value = {"images": ["missing"]}
|
|
with self.assertRaises(HTTPException) as error:
|
|
await magazine_update_article('a' * 32, 'article-001', request)
|
|
self.assertEqual(error.exception.status_code, 400)
|
|
save.assert_not_called()
|
|
|
|
|
|
class AdvertisementReviewTests(unittest.TestCase):
|
|
def test_llm_marks_only_known_confident_advertising_regions(self):
|
|
from unittest.mock import Mock, patch
|
|
from routes.magazine import _recognise_advertisements
|
|
layout = {'blocks': [
|
|
{'id': 'editorial', 'page': 1, 'type': 'body', 'bbox': [0,0,100,100], 'text': 'Solar technology news'},
|
|
{'id': 'advert', 'page': 1, 'type': 'title', 'bbox': [0,100,100,100], 'text': 'Buy webinar tickets'},
|
|
]}
|
|
response = Mock()
|
|
response.json.return_value = {'choices': [{'message': {'content': json.dumps({'advertisements': [
|
|
{'id': 'advert', 'confidence': 0.99}, {'id': 'editorial', 'confidence': 0.4}, {'id': 'invented', 'confidence': 1}
|
|
]})}}]}
|
|
with patch('routes.conversation._post_llm_chat_completion', return_value=response):
|
|
_recognise_advertisements(layout, {'llm_model': 'test'})
|
|
self.assertEqual(layout['blocks'][0]['type'], 'body')
|
|
self.assertEqual(layout['blocks'][1]['type'], 'ad')
|
|
self.assertEqual(layout['advertisement_review'], 'done')
|
|
|
|
def test_failed_llm_keeps_existing_regions(self):
|
|
from unittest.mock import patch
|
|
from routes.magazine import _recognise_advertisements
|
|
layout = {'blocks': [{'id': 'b', 'page': 1, 'type': 'body', 'bbox': [0,0,1,1], 'text': 'Article'}]}
|
|
with patch('routes.conversation._post_llm_chat_completion', side_effect=RuntimeError('offline')):
|
|
_recognise_advertisements(layout, {'llm_model': 'test'})
|
|
self.assertEqual(layout['blocks'][0]['type'], 'body')
|
|
self.assertEqual(layout['advertisement_review'], 'partial_failure')
|
|
|
|
def test_semantic_review_collapses_a_full_page_advertisement(self):
|
|
from unittest.mock import Mock, patch
|
|
from routes.magazine import _recognise_semantic_layout
|
|
layout = {
|
|
'pages': [{'page': 33, 'width': 600, 'height': 800}],
|
|
'blocks': [
|
|
{'id': 'offer', 'page': 33, 'type': 'ad', 'bbox': [30,40,500,80], 'text': 'Subscriber discount'},
|
|
{'id': 'order', 'page': 33, 'type': 'ad', 'bbox': [30,500,500,80], 'text': 'Order now'},
|
|
],
|
|
}
|
|
response = Mock()
|
|
response.json.return_value = {'choices': [{'message': {'content': json.dumps({
|
|
'page_kind': 'advertisement', 'confidence': .99,
|
|
'regions': [{'type': 'advertisement', 'block_ids': ['offer', 'order']}],
|
|
})}}]}
|
|
with patch('routes.conversation._post_llm_chat_completion', return_value=response):
|
|
_recognise_semantic_layout(layout, {'llm_model': 'test'}, page=33)
|
|
self.assertEqual(layout['semantic_review'], 'done')
|
|
self.assertEqual(len(layout['blocks']), 1)
|
|
self.assertEqual(layout['blocks'][0]['type'], 'ad')
|
|
self.assertEqual(layout['blocks'][0]['bbox'], [0, 0, 600, 800])
|
|
|
|
|
|
class RefineFiguresJobTests(unittest.TestCase):
|
|
"""Document-wide figure re-snap: reports what moved, keeps text untouched."""
|
|
|
|
def _layout(self):
|
|
return {
|
|
"pages": [{"page": 1, "width": 600, "height": 800},
|
|
{"page": 2, "width": 600, "height": 800}],
|
|
"blocks": [
|
|
{"id": "fig", "page": 1, "type": "figure", "bbox": [10, 10, 50, 50], "text": ""},
|
|
{"id": "copy", "page": 2, "type": "body", "bbox": [0, 0, 100, 100], "text": "Body"},
|
|
],
|
|
}
|
|
|
|
def test_moved_boxes_are_counted_and_saved(self):
|
|
from unittest.mock import patch
|
|
from routes.magazine import _refine_figures_sync
|
|
layout = self._layout()
|
|
|
|
def refresh(_dir, doc, page=None):
|
|
if page == 1:
|
|
doc["blocks"][0]["bbox"] = [20, 20, 90, 90]
|
|
|
|
with patch('routes.magazine.mag.load_layout', return_value=layout), \
|
|
patch('routes.magazine.mag.refresh_visual_regions', side_effect=refresh), \
|
|
patch('routes.magazine.mag.save_layout') as save, \
|
|
patch('routes.magazine.background_job_update') as update:
|
|
_refine_figures_sync('a' * 32, 'job')
|
|
save.assert_called_once()
|
|
done = [c.kwargs for c in update.call_args_list if c.kwargs.get('done')]
|
|
self.assertEqual(done[-1]['stage'], 'done')
|
|
self.assertEqual(done[-1]['result'], {'moved': 1})
|
|
self.assertEqual(layout['blocks'][1]['bbox'], [0, 0, 100, 100])
|
|
|
|
def test_nothing_to_correct_still_finishes_cleanly(self):
|
|
from unittest.mock import patch
|
|
from routes.magazine import _refine_figures_sync
|
|
with patch('routes.magazine.mag.load_layout', return_value=self._layout()), \
|
|
patch('routes.magazine.mag.refresh_visual_regions'), \
|
|
patch('routes.magazine.mag.save_layout'), \
|
|
patch('routes.magazine.background_job_update') as update:
|
|
_refine_figures_sync('a' * 32, 'job')
|
|
done = [c.kwargs for c in update.call_args_list if c.kwargs.get('done')]
|
|
self.assertEqual(done[-1]['result'], {'moved': 0})
|
|
self.assertNotIn('error', done[-1])
|
|
|
|
def test_a_document_with_no_layout_reports_failure(self):
|
|
from unittest.mock import patch
|
|
from routes.magazine import _refine_figures_sync
|
|
with patch('routes.magazine.mag.load_layout', return_value=None), \
|
|
patch('routes.magazine.mag.save_layout') as save, \
|
|
patch('routes.magazine.background_job_update') as update:
|
|
_refine_figures_sync('a' * 32, 'job')
|
|
save.assert_not_called()
|
|
done = [c.kwargs for c in update.call_args_list if c.kwargs.get('done')]
|
|
self.assertEqual(done[-1]['stage'], 'failed')
|
|
self.assertIn('No layout', done[-1]['error'])
|