fix(magazine): unbreakable text can no longer collapse the review; endless repeated tokens are OCR noise
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
81e2c6b4ef
commit
28d6439604
@ -6,6 +6,9 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
|
||||
---
|
||||
|
||||
## [Unreleased]
|
||||
### Fixed
|
||||
- **A page's picture could vanish from the review (seen on c't p. 176).** One region held a 3,408-character token with no spaces (`Giro + 2.1.1.1.1…`, the OCR engine stuck in a loop). Its width was taken as the minimum width of the list column, which widened to 20,305 px and squeezed the page-picture column to 2 px. The review grid's tracks can now shrink (`minmax(0, …)`), and text in the review list, reading view, figure diffs and page cards wraps anywhere, so no content can collapse the layout again; a long URL or a loop now simply wraps. The cause is also handled: *Clean up boxes* (and every import and re-read, which run it) re-type a region that is mostly one endless repeated letter/digit token as OCR noise (kept, visible, reversible in the type picker, not read aloud). Leader dots, rules, section numbers, phone numbers, prices and URLs are not affected; run over all three of your magazines (about 6,000 regions) the rule flags exactly that one region on page 176 in each and nothing else. To clear it on an existing magazine press *Clean up boxes* or *Fix this page automatically*.
|
||||
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -477,6 +477,16 @@ Verified end-to-end against a synthetic 8-page issue (contents page, two-column
|
||||
feature with a byline and a page-8 jump, full-page advert, second feature,
|
||||
reviews) and against an image-only rendering of the same file to exercise OCR.
|
||||
|
||||
## Endless-token regions and unbreakable text
|
||||
|
||||
A recogniser stuck in a loop can emit one token with no spaces at all (c't p. 176: `Giro + 2.1.1.1.1…`, 3,408 characters). `_looks_like_repetition_loop`
|
||||
ignores a repeated unit without a real word on purpose (leader dots, rules), so it did not see it. `glued_repetition_share(text)` counts the share of a text
|
||||
covered by glued runs of a short unit (<= 6 characters) containing a letter or digit, repeated 30+ times; `clean_up_regions` re-types a region of at least 60
|
||||
characters where that share is >= 0.5 as `ocr_noise` (furniture, reversible). Measured on the three c't magazines (~6,000 regions): exactly one region each, none
|
||||
other. Independently of the data, the review must survive such text: a grid track is min-content unless told otherwise, so one unbreakable word widened the
|
||||
list column to 20,305 px and left the page picture 2 px. `.mag-review-split` uses `minmax(0, ...)` tracks, the list containers have `min-width: 0` and the text
|
||||
classes `overflow-wrap: anywhere`.
|
||||
|
||||
## Turning a page scanned sideways
|
||||
|
||||
`POST /api/magazine/{id}/pages/{page}/rotate {degrees: 90|180|270|-90}` starts the same background job as "OCR again"
|
||||
|
||||
@ -1251,6 +1251,24 @@ def _looks_like_repetition_loop(text: str) -> bool:
|
||||
for m in _REPETITION_RUN.finditer(text or ""))
|
||||
|
||||
|
||||
# A recogniser stuck in a loop can also emit one endless token with no spaces at all ("2.1.1.1.1.1 ..." thousands of
|
||||
# characters long, seen on a real shopping-advert page). _looks_like_repetition_loop deliberately ignores a repeated unit
|
||||
# without a real word, so leader dots and rules survive -- but a short unit that contains a letter or digit, repeated
|
||||
# this many times back to back, is never real text: a phone number, a price, a section number or a URL do not come
|
||||
# close to 30 identical repeats.
|
||||
_GLUED_RUN = re.compile(r"(\S{1,6}?)\1{29,}")
|
||||
_GLUED_UNIT_CONTENT = re.compile(r"[A-Za-z0-9À-ÖØ-öø-ÿ]")
|
||||
|
||||
|
||||
def glued_repetition_share(text: str) -> float:
|
||||
"""The share of `text` taken up by glued runs of a short letter/digit unit repeated 30+ times (0.0 - 1.0)."""
|
||||
text = text or ""
|
||||
if not text:
|
||||
return 0.0
|
||||
covered = sum(m.end() - m.start() for m in _GLUED_RUN.finditer(text) if _GLUED_UNIT_CONTENT.search(m.group(1)))
|
||||
return covered / len(text)
|
||||
|
||||
|
||||
def vision_ocr_page(png_path: Path, base_url: str, model: str,
|
||||
timeout: float = 150.0) -> list[dict[str, Any]] | None:
|
||||
"""OCR one rendered page with a connected vision-capable OCR engine.
|
||||
@ -4755,8 +4773,11 @@ def clean_up_regions(layout: dict[str, Any], pages: Iterable[int] | None = None)
|
||||
if text.startswith("(no text recognised"):
|
||||
continue
|
||||
if (len(re.sub(r"[\W_]", "", text)) <= 2 or text.lower().startswith("[non-text")
|
||||
or b["bbox"][3] < _MIN_TEXT_BOX_HEIGHT):
|
||||
mark(b, "ocr_noise") # includes the engine's own "[Non-Text]" and hairline boxes
|
||||
or b["bbox"][3] < _MIN_TEXT_BOX_HEIGHT
|
||||
or (len(text) >= 60 and glued_repetition_share(text) >= 0.5)):
|
||||
# includes the engine's own "[Non-Text]", hairline boxes, and a region that is mostly one endless
|
||||
# repeated token (a recogniser stuck in a loop)
|
||||
mark(b, "ocr_noise")
|
||||
elif len(text) <= 80 and any(_area_inside(b["bbox"], p) >= 0.9 for p in pictures.get(page_no, [])):
|
||||
mark(b, "in_picture")
|
||||
for small in text_boxes:
|
||||
|
||||
@ -5083,7 +5083,7 @@ input[list="lang-codes"] {
|
||||
.mag-review-pagenav-spacer { flex: 1; min-width: 0; }
|
||||
|
||||
/* ── Split screen: rendered page (left) + extracted blocks (right) ───────── */
|
||||
.mag-review-split { display: grid; grid-template-columns: 1.1fr 1fr; gap: 18px; align-items: start; }
|
||||
.mag-review-split { display: grid; grid-template-columns: minmax(0, 1.1fr) minmax(0, 1fr); gap: 18px; align-items: start; } /* minmax(0,..): unbreakable content must not widen a column */
|
||||
.mag-review-page { position: sticky; top: 12px; }
|
||||
.mag-review-page-wrap { position: relative; border: 1px solid var(--border); border-radius: var(--radius);
|
||||
overflow: hidden; background: var(--panel); }
|
||||
@ -5649,3 +5649,8 @@ details.mag-report-group summary { cursor: pointer; margin-bottom: 6px; }
|
||||
.mag-turn-group { display: inline-flex; gap: 4px; padding: 2px; border: 1px dashed var(--border); border-radius: var(--radius); }
|
||||
.mag-type-picker-list.opens-up { top: auto; bottom: 100%; margin: 0 0 2px; }
|
||||
.mag-type-picker-list { overflow-y: auto; }
|
||||
|
||||
/* Content that cannot break (an OCR loop, a long URL) must never widen the review or push the page picture out:
|
||||
grid tracks are min-content by default, so every track that holds text is allowed to shrink and text wraps anywhere. */
|
||||
.mag-review-split > *, .mag-blocks, .mag-block { min-width: 0; }
|
||||
.mag-block-text, .mag-diff, .mag-article, .mag-driving-lyrics, .mag-page-article-title, .mag-toc-title { overflow-wrap: anywhere; }
|
||||
|
||||
@ -2189,3 +2189,56 @@ test.describe('Draw region panel layering', () => {
|
||||
expect(box.x + box.width).toBeLessThanOrEqual(vp.width);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Unbreakable text cannot collapse the review', () => {
|
||||
const HUGE = 'Giro + 2.' + '1.'.repeat(2500);
|
||||
async function openWith(page, width = 1600) {
|
||||
await page.setViewportSize({ width, height: 900 });
|
||||
await mockBackend(page);
|
||||
const layout = { ...LAYOUT, blocks: [...LAYOUT.blocks,
|
||||
{ id: 'b9', page: 3, bbox: [50, 500, 200, 40], type: 'body', text: HUGE, reading_order: 3, confidence: 0.5, font_size: 10, font_name: '', bold: false, column: 0, source: 'vision_ocr' }] };
|
||||
await page.route(`**/api/magazine/${DOC_ID}/layout*`, route => route.fulfill({ json: layout }));
|
||||
await page.goto('/', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#app-skeleton', { state: 'detached', timeout: 15_000 });
|
||||
await navTo(page, 's-magazine');
|
||||
await page.locator('#mag-library-list .mag-card').first().click();
|
||||
await page.locator('#mag-contents-view-toggle').click();
|
||||
await page.locator('.mag-card-wrap').filter({ hasText: 'Page 3' }).getByRole('button', { name: /^Review/ }).click();
|
||||
await page.waitForSelector('#mag-undo-btn');
|
||||
await expect(page.locator('[data-block-row="b9"]')).toBeVisible();
|
||||
}
|
||||
|
||||
test('the page picture keeps its column and the list stays inside the window', async ({ page }) => {
|
||||
await openWith(page);
|
||||
const m = await page.evaluate(() => {
|
||||
const wrap = document.getElementById('mag-review-page-wrap').getBoundingClientRect();
|
||||
const split = document.querySelector('.mag-review-split');
|
||||
const cols = getComputedStyle(split).gridTemplateColumns.split(' ').map(parseFloat);
|
||||
const list = document.querySelector('.mag-blocks').getBoundingClientRect();
|
||||
return { wrapW: wrap.width, cols, listRight: list.right, vw: window.innerWidth, docW: document.documentElement.scrollWidth };
|
||||
});
|
||||
expect(m.wrapW).toBeGreaterThan(250); // the page picture is not squeezed to nothing
|
||||
expect(Math.min(...m.cols)).toBeGreaterThan(250); // both columns have real width
|
||||
expect(m.listRight).toBeLessThanOrEqual(m.vw + 1); // the list does not run off the window
|
||||
expect(m.docW).toBeLessThanOrEqual(m.vw + 1); // and the page gets no sideways scroll
|
||||
});
|
||||
|
||||
test('the same holds on a phone, where the columns stack', async ({ page }) => {
|
||||
await openWith(page, 390);
|
||||
const m = await page.evaluate(() => ({
|
||||
wrapW: document.getElementById('mag-review-page-wrap').getBoundingClientRect().width,
|
||||
docW: document.documentElement.scrollWidth, vw: window.innerWidth }));
|
||||
expect(m.wrapW).toBeGreaterThan(200);
|
||||
expect(m.docW).toBeLessThanOrEqual(m.vw + 1);
|
||||
});
|
||||
|
||||
test('the huge token is still shown, wrapped, so it can be fixed or deleted', async ({ page }) => {
|
||||
await openWith(page);
|
||||
const row = page.locator('[data-block-row="b9"] .mag-block-text');
|
||||
await expect(row).toContainText('Giro + 2.1.1.1');
|
||||
const box = await row.boundingBox();
|
||||
expect(box.width).toBeLessThan(1600);
|
||||
expect(box.height).toBeGreaterThan(100); // wrapped onto many lines instead of one 20,000 px line
|
||||
await expect(page.locator('[data-block-row="b9"]').getByRole('button', { name: /Delete region/ })).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
@ -946,6 +946,43 @@ class CleanUpBoxesTests(unittest.TestCase):
|
||||
self.assertEqual(types["long"][0], "body") # real text caught in a loose frame stays
|
||||
self.assertEqual(types["outside"][0], "body")
|
||||
|
||||
def test_a_region_that_is_one_endless_repeated_token_is_noise_but_real_text_with_repeats_is_not(self):
|
||||
"""Seen on a real advert page: 'Giro + 2.1.1.1.1 ...' thousands of characters long, no spaces."""
|
||||
from core.magazine import clean_up_regions
|
||||
junk = "Giro + " + "2." + "1." * 1700
|
||||
layout = self.base([
|
||||
self.blk("loop", 10, 10, 300, 40, junk),
|
||||
self.blk("digits", 10, 60, 300, 40, "0" * 120),
|
||||
self.blk("dots", 10, 110, 300, 12, "Kapitel 1 " + "." * 200 + " 17", "caption"),
|
||||
self.blk("toc", 10, 150, 300, 12, "2.1.1 Einleitung 2.1.2 Methode 2.1.3 Ergebnisse 2.1.4 Diskussion 2.2 Fazit 3.1.1.1 Anhang"),
|
||||
self.blk("url", 10, 170, 300, 12, "https://example.org/artikel/2026/09/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0?ref=newsletter&utm_source=heise"),
|
||||
self.blk("prose", 10, 190, 300, 40, "Ein ganz normaler Absatz mit genug Woertern, der nichts Verdaechtiges enthaelt. " * 3),
|
||||
self.blk("short", 10, 240, 300, 12, "1." * 10),
|
||||
])
|
||||
types = self.types(clean_up_regions(layout)[0])
|
||||
self.assertEqual(types["loop"], ("furniture", "ocr_noise"))
|
||||
self.assertEqual(types["digits"], ("furniture", "ocr_noise"))
|
||||
for kept in ("dots", "toc", "url", "prose", "short"):
|
||||
self.assertNotEqual(types[kept][1], "ocr_noise", kept)
|
||||
|
||||
def test_the_repetition_share_counts_only_letter_or_digit_units_repeated_thirty_times(self):
|
||||
from core.magazine import glued_repetition_share
|
||||
self.assertGreater(glued_repetition_share("1." * 100), 0.9)
|
||||
self.assertGreater(glued_repetition_share("ab" * 40), 0.9)
|
||||
self.assertEqual(glued_repetition_share("." * 500), 0.0) # leader dots and rules are not text, so not counted
|
||||
self.assertEqual(glued_repetition_share("- " * 100), 0.0)
|
||||
self.assertEqual(glued_repetition_share("1." * 29), 0.0) # one short of the threshold
|
||||
self.assertEqual(glued_repetition_share(""), 0.0)
|
||||
self.assertAlmostEqual(glued_repetition_share("Hallo " + "xy" * 50 + " Welt"), 100 / 111, places=2)
|
||||
|
||||
def test_clean_up_is_idempotent_for_the_new_rule(self):
|
||||
from core.magazine import clean_up_regions
|
||||
layout = self.base([self.blk("loop", 10, 10, 300, 40, "2." + "1." * 500)])
|
||||
once = clean_up_regions(layout)[0]
|
||||
twice, report = clean_up_regions(once)
|
||||
self.assertEqual(self.types(twice)["loop"], ("furniture", "ocr_noise"))
|
||||
self.assertEqual(report["ocr_noise"], []) # nothing left to re-type the second time
|
||||
|
||||
def test_one_or_two_characters_are_noise_but_a_phone_number_is_not(self):
|
||||
from core.magazine import clean_up_regions
|
||||
layout = self.base([
|
||||
|
||||
Loading…
Reference in New Issue
Block a user