diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f63563..ac1fbb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi - **Clean up boxes now also fits text boxes to their text.** A side that cuts through letters grows (at most a letter or two, never into a neighbouring box) and empty margins are trimmed. Boxes you drew or resized by hand are never touched. It also runs automatically after **OCR again (this page)**; on page 61 of the measured issue it widened the left-column paragraphs ~10 pt to cover their text, cutting clipped edges from 7 to 0. A box never grows into a neighbouring box; when it gains whole lines, its text is re-read from the enlarged crop and replaced only by a longer reading that still starts with the old text. ### Fixed +- **The review list shows each region's whole text**, not the first 220 characters, so every misspelling gets its red wavy underline (like a word processor). The engine's `[Non-Text]` marker and hairline boxes (under 3 pt tall) are now treated as OCR noise by *Clean up boxes* and after a page re-OCR, so they no longer show up as near-invisible regions. Fitted boxes keep a little air on grown sides. - **The quality check no longer flags correct text as "suspect OCR".** The word check guessed the language per fragment (`iPhones statt Token` read as English) and flagged valid German; it now uses the page's language, the same lenient German rules as the spellchecker, and treats words that recur on the page (names, brands) as vocabulary. Page 61 of the measured issue went from 11 suspect regions to 3 (one a genuinely truncated word), and the issue from 758 to 610 before the language fix's full effect. - Suspended compounds (`Programmier- und Hausaufgaben`) no longer lose their hyphen when regions are merged or layouts are grouped; already-ingested `Programmierund` shows in the spelling list with a one-click repair. diff --git a/core/magazine.py b/core/magazine.py index adac510..c6aae10 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -2905,6 +2905,7 @@ def _clipped_edges(pixels, page_meta: dict[str, Any], text_blocks: list[dict[str return found +_MIN_TEXT_BOX_HEIGHT = 3.0 # points: a text box thinner than this holds no line of text _SNAP_TEXT_STEP_PX = 2 # grow/shrink granularity _SNAP_TEXT_MAX_GROW_PX = 16 # never grow a side sideways further than about a letter or two _SNAP_TEXT_MAX_GROW_DOWN_PX = 90 # a paragraph cut short at the top/bottom can be a few lines @@ -3000,7 +3001,21 @@ def snap_text_boxes(doc_dir: Path, layout: dict[str, Any], bottom += step; budget["bottom"] -= step; grew = True if not grew: break - if (left, top, right, bottom) == original: + if (left, top, right, bottom) != original: + # a little air past the last inked line, so descenders are not touching the edge + pad = _SNAP_TEXT_PAD_PX + if left - pad >= 0 and not blocked(top, bottom, left - pad, left): + left -= pad + if right + pad <= width and not blocked(top, bottom, right, right + pad): + right += pad + if top - pad >= 0 and not blocked(top - pad, top, left, right): + top -= pad + if bottom + pad <= height and not blocked(bottom, bottom + pad, left, right): + bottom += pad + original_grown = True + else: + original_grown = False + if not original_grown: # Not clipped: trim the empty margins instead. A line is empty margin when it # holds (almost) no ink of its own. def line_empty(r0: int, r1: int, c0: int, c1: int) -> bool: @@ -3819,8 +3834,9 @@ def clean_up_regions(layout: dict[str, Any], pages: Iterable[int] | None = None) text = " ".join((b.get("text") or "").split()) if text.startswith("(no text recognised"): continue - if len(re.sub(r"[\W_]", "", text)) <= 2: - mark(b, "ocr_noise") + if (len(re.sub(r"[\W_]", "", text)) <= 2 or text.lower().startswith("[non-text") + or b["bbox"][3] < _MIN_TEXT_BOX_HEIGHT): + mark(b, "ocr_noise") # includes the engine's own "[Non-Text]" and hairline boxes elif len(text) <= 80 and any(_area_inside(b["bbox"], p) >= 0.9 for p in pictures.get(page_no, [])): mark(b, "in_picture") for small in text_boxes: diff --git a/static/js/magazine.js b/static/js/magazine.js index 10273e9..feda77e 100644 --- a/static/js/magazine.js +++ b/static/js/magazine.js @@ -1545,8 +1545,7 @@ function magReviewPageColumn(page, wrapId, interactive) { (interactive ? ' onclick="event.preventDefault();event.stopPropagation();magRowEdit(\'' + b.id + '\')" title="Click to edit"' : '') + '>' + (owner ? 'Article ' + owner.index + ': ' + esc(owner.title) + ' ' : '') + (b.continues_from_previous_page ? 'Continued from previous page ' : '') + - '' + magSpellMarked(b.id, b.text.slice(0, 220)) + '' + - (b.text.length > 220 ? '…' : '') + '' + + '' + magSpellMarked(b.id, b.text) + '' + (interactive ? magRowActions(b.id, i + 1, i + 1 < regions.length) : '') + ''; }); if (!regions.length) listHtml += '

Nothing detected on this page.

'; @@ -3241,7 +3240,7 @@ async function magSpellLoadPage() { const blocks = new Map((state.layout?.blocks || []).map((b) => [b.id, b])); document.querySelectorAll('#mag-review [data-text-of]').forEach((el) => { const b = blocks.get(el.dataset.textOf); - if (b) el.innerHTML = magSpellMarked(b.id, b.text.slice(0, 220)); + if (b) el.innerHTML = magSpellMarked(b.id, b.text); }); const count = $('mag-spell-count'); if (count) count.textContent = state.spell.issues.length ? String(state.spell.issues.length) : ''; diff --git a/static/style.css b/static/style.css index 6e66174..06e0ce4 100644 --- a/static/style.css +++ b/static/style.css @@ -5459,7 +5459,7 @@ input[list="lang-codes"] { } /* Magazine: spelling marks and the pronunciation/spelling panel */ -.mag-spell { background: transparent; color: inherit; text-decoration: underline wavy var(--danger, #c0392b); text-underline-offset: 3px; } +.mag-spell { background: transparent; color: inherit; text-decoration: underline wavy #d93025; text-decoration-thickness: 1.5px; text-underline-offset: 3px; cursor: help; } .mag-spell-panel, .mag-checks { margin-top: 12px; } .mag-spell-list { list-style: none; margin: 8px 0 0; padding: 0; display: grid; gap: 8px; } .mag-spell-list li { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; } diff --git a/tests/test_magazine.py b/tests/test_magazine.py index f3197ed..307b230 100644 --- a/tests/test_magazine.py +++ b/tests/test_magazine.py @@ -2800,3 +2800,16 @@ class RereadGrownRegionsTests(unittest.TestCase): from core.magazine import reread_grown_regions reread_grown_regions(pathlib.Path('/tmp/x'), self._layout("t"), [{"id": "a", "lines_added": False}], 'u', 'm') read.assert_not_called() + + +class NonTextNoiseTests(unittest.TestCase): + def test_the_engines_non_text_marker_and_hairline_boxes_become_noise(self): + from core.magazine import clean_up_regions + layout = {"pages": [{"page": 1, "width": 100, "height": 100}], "images": [], "blocks": [ + {"id": "n", "page": 1, "type": "body", "bbox": [10, 10, 80, 0.7], "text": "[Non-Text]"}, + {"id": "h", "page": 1, "type": "body", "bbox": [10, 20, 80, 1.0], "text": "A real looking sentence here."}, + {"id": "ok", "page": 1, "type": "body", "bbox": [10, 30, 80, 12], "text": "A real looking sentence here."}]} + out, report = clean_up_regions(layout) + types = {b["id"]: b["type"] for b in out["blocks"]} + self.assertEqual((types["n"], types["h"], types["ok"]), ("furniture", "furniture", "body")) + self.assertEqual(len(report["ocr_noise"]), 2)