fix(magazine): full text with spelling underlines in review rows; drop [Non-Text]/hairline boxes; pad grown boxes

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

CHANGELOG.md

@@ -15,6 +15,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
 - **Clean up boxes now also fits text boxes to their text.** A side that cuts through letters grows (at most a letter or two, never into a neighbouring box) and empty margins are trimmed. Boxes you drew or resized by hand are never touched. It also runs automatically after **OCR again (this page)**; on page 61 of the measured issue it widened the left-column paragraphs ~10 pt to cover their text, cutting clipped edges from 7 to 0. A box never grows into a neighbouring box; when it gains whole lines, its text is re-read from the enlarged crop and replaced only by a longer reading that still starts with the old text.
 
 ### Fixed
+- **The review list shows each region's whole text**, not the first 220 characters, so every misspelling gets its red wavy underline (like a word processor). The engine's `[Non-Text]` marker and hairline boxes (under 3 pt tall) are now treated as OCR noise by *Clean up boxes* and after a page re-OCR, so they no longer show up as near-invisible regions. Fitted boxes keep a little air on grown sides.
 - **The quality check no longer flags correct text as "suspect OCR".** The word check guessed the language per fragment (`iPhones statt Token` read as English) and flagged valid German; it now uses the page's language, the same lenient German rules as the spellchecker, and treats words that recur on the page (names, brands) as vocabulary. Page 61 of the measured issue went from 11 suspect regions to 3 (one a genuinely truncated word), and the issue from 758 to 610 before the language fix's full effect.
 - Suspended compounds (`Programmier- und Hausaufgaben`) no longer lose their hyphen when regions are merged or layouts are grouped; already-ingested `Programmierund` shows in the spelling list with a one-click repair.
 

core/magazine.py

@@ -2905,6 +2905,7 @@ def _clipped_edges(pixels, page_meta: dict[str, Any], text_blocks: list[dict[str
     return found
 
 
+_MIN_TEXT_BOX_HEIGHT = 3.0    # points: a text box thinner than this holds no line of text
 _SNAP_TEXT_STEP_PX = 2        # grow/shrink granularity
 _SNAP_TEXT_MAX_GROW_PX = 16   # never grow a side sideways further than about a letter or two
 _SNAP_TEXT_MAX_GROW_DOWN_PX = 90   # a paragraph cut short at the top/bottom can be a few lines
@@ -3000,7 +3001,21 @@ def snap_text_boxes(doc_dir: Path, layout: dict[str, Any],
                     bottom += step; budget["bottom"] -= step; grew = True
                 if not grew:
                     break
-            if (left, top, right, bottom) == original:
+            if (left, top, right, bottom) != original:
+                # a little air past the last inked line, so descenders are not touching the edge
+                pad = _SNAP_TEXT_PAD_PX
+                if left - pad >= 0 and not blocked(top, bottom, left - pad, left):
+                    left -= pad
+                if right + pad <= width and not blocked(top, bottom, right, right + pad):
+                    right += pad
+                if top - pad >= 0 and not blocked(top - pad, top, left, right):
+                    top -= pad
+                if bottom + pad <= height and not blocked(bottom, bottom + pad, left, right):
+                    bottom += pad
+                original_grown = True
+            else:
+                original_grown = False
+            if not original_grown:
                 # Not clipped: trim the empty margins instead. A line is empty margin when it
                 # holds (almost) no ink of its own.
                 def line_empty(r0: int, r1: int, c0: int, c1: int) -> bool:
@@ -3819,8 +3834,9 @@ def clean_up_regions(layout: dict[str, Any], pages: Iterable[int] | None = None)
             text = " ".join((b.get("text") or "").split())
             if text.startswith("(no text recognised"):
                 continue
-            if len(re.sub(r"[\W_]", "", text)) <= 2:
-                mark(b, "ocr_noise")
+            if (len(re.sub(r"[\W_]", "", text)) <= 2 or text.lower().startswith("[non-text")
+                    or b["bbox"][3] < _MIN_TEXT_BOX_HEIGHT):
+                mark(b, "ocr_noise")     # includes the engine's own "[Non-Text]" and hairline boxes
             elif len(text) <= 80 and any(_area_inside(b["bbox"], p) >= 0.9 for p in pictures.get(page_no, [])):
                 mark(b, "in_picture")
         for small in text_boxes:

static/js/magazine.js

@@ -1545,8 +1545,7 @@ function magReviewPageColumn(page, wrapId, interactive) {
         (interactive ? ' onclick="event.preventDefault();event.stopPropagation();magRowEdit(\'' + b.id + '\')" title="Click to edit"' : '') + '>' +
         (owner ? '<span class="mag-block-owner">Article ' + owner.index + ': ' + esc(owner.title) + '</span> ' : '') +
         (b.continues_from_previous_page ? '<span class="mag-block-owner">Continued from previous page</span> ' : '') +
-        '<span data-text-of="' + esc(b.id) + '">' + magSpellMarked(b.id, b.text.slice(0, 220)) + '</span>' +
-        (b.text.length > 220 ? '…' : '') + '</span>' +
+        '<span data-text-of="' + esc(b.id) + '">' + magSpellMarked(b.id, b.text) + '</span></span>' +
       (interactive ? magRowActions(b.id, i + 1, i + 1 < regions.length) : '') + '</label>';
   });
   if (!regions.length) listHtml += '<p class="muted">Nothing detected on this page.</p>';
@@ -3241,7 +3240,7 @@ async function magSpellLoadPage() {
   const blocks = new Map((state.layout?.blocks || []).map((b) => [b.id, b]));
   document.querySelectorAll('#mag-review [data-text-of]').forEach((el) => {
     const b = blocks.get(el.dataset.textOf);
-    if (b) el.innerHTML = magSpellMarked(b.id, b.text.slice(0, 220));
+    if (b) el.innerHTML = magSpellMarked(b.id, b.text);
   });
   const count = $('mag-spell-count');
   if (count) count.textContent = state.spell.issues.length ? String(state.spell.issues.length) : '';

static/style.css

@@ -5459,7 +5459,7 @@ input[list="lang-codes"] {
 }
 
 /* Magazine: spelling marks and the pronunciation/spelling panel */
-.mag-spell { background: transparent; color: inherit; text-decoration: underline wavy var(--danger, #c0392b); text-underline-offset: 3px; }
+.mag-spell { background: transparent; color: inherit; text-decoration: underline wavy #d93025; text-decoration-thickness: 1.5px; text-underline-offset: 3px; cursor: help; }
 .mag-spell-panel, .mag-checks { margin-top: 12px; }
 .mag-spell-list { list-style: none; margin: 8px 0 0; padding: 0; display: grid; gap: 8px; }
 .mag-spell-list li { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; }

tests/test_magazine.py

@@ -2800,3 +2800,16 @@ class RereadGrownRegionsTests(unittest.TestCase):
             from core.magazine import reread_grown_regions
             reread_grown_regions(pathlib.Path('/tmp/x'), self._layout("t"), [{"id": "a", "lines_added": False}], 'u', 'm')
         read.assert_not_called()
+
+
+class NonTextNoiseTests(unittest.TestCase):
+    def test_the_engines_non_text_marker_and_hairline_boxes_become_noise(self):
+        from core.magazine import clean_up_regions
+        layout = {"pages": [{"page": 1, "width": 100, "height": 100}], "images": [], "blocks": [
+            {"id": "n", "page": 1, "type": "body", "bbox": [10, 10, 80, 0.7], "text": "[Non-Text]"},
+            {"id": "h", "page": 1, "type": "body", "bbox": [10, 20, 80, 1.0], "text": "A real looking sentence here."},
+            {"id": "ok", "page": 1, "type": "body", "bbox": [10, 30, 80, 12], "text": "A real looking sentence here."}]}
+        out, report = clean_up_regions(layout)
+        types = {b["id"]: b["type"] for b in out["blocks"]}
+        self.assertEqual((types["n"], types["h"], types["ok"]), ("furniture", "furniture", "body"))
+        self.assertEqual(len(report["ocr_noise"]), 2)