diff --git a/CHANGELOG.md b/CHANGELOG.md index 174afa6..68a797c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,32 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi --- +## [1.26.4] — 2026-09-09 + +### Added +- **Native Baidu Unlimited-OCR Layout Parsing.** Magazine ingestion now uses Unlimited-OCR's documented + `document parsing.` mode and consumes its `<|det|>` bounding-box regions. Images, non-text graphics, charts and + tables are retained as visual regions instead of being mixed into narratable article prose; captions remain captions. + +### Changed +- **Magazine OCR Preview Uses Its Space.** The source page fills the available preview height while preserving its + aspect ratio. A draggable divider lets readers allocate space between the page and recognised text, and remembers + the choice. +- **Character Sheets Read Like Profiles.** Profile fields are now one-column entries with a bold heading and normal + prose beneath it, rather than compact label/value columns. The live passage/output drawer stays bottom-anchored and + its top grip resizes from the expected edge. + +### Fixed +- **Unlimited-OCR Repetition Guard Was Not Activated.** Requests now include the model's required per-request + n-gram settings and `gundam` image mode. Repeated fabricated LaTex and instruction-echo responses are rejected so + fallback OCR can be used instead of saving poisoned page text. +- **Character Line Totals Missed Aliases.** A profile now totals dialogue credited to its canonical name and recorded + identity aliases (for example, `Larissa` and `Schwester Larissa`) and refreshes the total during casting. +- **Page Furniture Does Not Become Article Text.** Native OCR regions now identify headers, footers and page numbers + as page furniture, keeping them out of narration while preserving their position for review. + +--- + ## [1.26.3] — 2026-09-09 ### Fixed diff --git a/VERSION b/VERSION index f8f7381..ea0928c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.26.3 +1.26.4 diff --git a/core/magazine.py b/core/magazine.py index 451109e..38ee4f5 100644 --- a/core/magazine.py +++ b/core/magazine.py @@ -742,6 +742,8 @@ _VISION_LABEL_TYPES = { "title": BLOCK_TITLE, "heading": BLOCK_HEADING, "caption": BLOCK_CAPTION, + "image_caption": BLOCK_CAPTION, + "table_caption": BLOCK_CAPTION, "headline": BLOCK_TITLE, "article_headline": BLOCK_TITLE, "paragraph": BLOCK_BODY, @@ -758,6 +760,8 @@ _VISION_LABEL_TYPES = { "graphic": BLOCK_CHART, "diagram": BLOCK_CHART, "table": BLOCK_TABLE, + "formula": BLOCK_CODE, + "list": BLOCK_BODY, "code": BLOCK_CODE, "code_block": BLOCK_CODE, "advertisement": BLOCK_AD, @@ -769,6 +773,15 @@ _VISION_LABEL_TYPES = { # the next such marker. Confirmed live against Baidu Unlimited-OCR's actual # output shape: "text [241, 90, 588, 249]Battery technology is entering...". _VISION_REGION = re.compile(r"(?m)^(\w+)\s*\[\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\]") +# Native Baidu Unlimited-OCR document-parsing output. The model card emits +# one marker per detected region, such as +# ``<|det|>table [0,0,999,999]<|/det|>...``. Its categories are precisely +# what we need to retain charts/tables/images as visual blocks instead of +# treating their axis labels as article prose. +_UNLIMITED_OCR_REGION = re.compile( + r"(?m)^<\|det\|>\s*(.*?)\s*(?:\[\s*(-?\d+(?:\.\d+)?)\s*,\s*" + r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*" + r"(-?\d+(?:\.\d+)?)\s*\])?\s*<\|/det\|>") def ocr_quality_score(text: str) -> float: @@ -848,6 +861,42 @@ def parse_vision_ocr_regions(raw: str) -> list[dict[str, Any]]: return regions +def parse_unlimited_ocr_regions(raw: str) -> list[dict[str, Any]]: + """Parse Unlimited-OCR's native ``<|det|>`` document-layout markers. + + Bounding boxes are in its 0–999-ish image coordinate space; callers mark + them for conversion to the rendered page's pixels before building blocks. + A `[Non-Text]` region is a real visual object even when it has no text. + """ + matches = list(_UNLIMITED_OCR_REGION.finditer(raw or "")) + regions = [] + for i, match in enumerate(matches): + raw_label = (match.group(1) or "").strip().strip("[]") + label = raw_label.lower().replace("-", "_").replace(" ", "_") + coords = match.groups()[1:] + if len(coords) != 4 or any(v is None for v in coords): + continue + x0, y0, x1, y1 = (float(v) for v in coords) + if x1 <= x0 or y1 <= y0: + continue + start = match.end() + end = matches[i + 1].start() if i + 1 < len(matches) else len(raw or "") + text = (raw or "")[start:end].strip() + # The model's generic Non-Text label is an image/figure. It must be + # retained even empty, so crop_visual_regions can preserve it. + if label in {"non_text", "nontext"}: + label = "image" + if not text and _VISION_LABEL_TYPES.get(label) not in VISUAL_TYPES: + continue + regions.append({ + "label": label, + "bbox": [x0, y0, x1 - x0, y1 - y0], + "text": text, + "native_normalized_bbox": True, + }) + return regions + + def _whole_page_region(content: str, png_path: Path) -> list[dict[str, Any]] | None: """Fallback for an engine that answers with plain text and no coordinate markers at all (confirmed live: KDL Frontier Parser nano -- fast, but no @@ -871,10 +920,18 @@ def _whole_page_region(content: str, png_path: Path) -> list[dict[str, Any]] | N escaped_newlines = len(re.findall(r"\\n", text)) equation_tokens = len(re.findall(r"`?\s*\\?equation\b", text, re.IGNORECASE)) prompt_echo = "do not use special symbols" in text.lower() + # Another observed loop starts with genuine page text and then repeats a + # fabricated LaTex fragment (``\\therefore m = \\frac{3}{11}``) until the + # response limit. It has no instruction echo or `equation` token, so it + # needs its own narrow signature. Requiring both uncommon controls many + # times avoids rejecting ordinary prose that merely mentions one formula. + therefore_tokens = len(re.findall(r"\\therefore\b", text, re.IGNORECASE)) + fraction_tokens = len(re.findall(r"\\frac\s*\{", text, re.IGNORECASE)) + latex_loop = therefore_tokens >= 4 and fraction_tokens >= 4 if (escaped_newlines >= 4 and equation_tokens >= 4) or ( prompt_echo and escaped_newlines >= 2 and equation_tokens >= 2 - ): - logging.getLogger(__name__).warning("Vision OCR returned instruction-echo artifact; discarding response") + ) or latex_loop: + logging.getLogger(__name__).warning("Vision OCR returned a repeated formatting artifact; discarding response") return None if len(text) < 20 or ocr_quality_score(text) < _VISION_QUALITY_THRESHOLD: return None @@ -911,22 +968,24 @@ def vision_ocr_page(png_path: Path, base_url: str, model: str, if not urlsplit(base_url).path: base_url += "/v1" + unlimited_ocr = "unlimited-ocr" in model.lower() + prompt = "document parsing." if unlimited_ocr else ( + "Recognise the page layout and transcribe visible text verbatim. " + "Return one region per headline, paragraph, image, chart, table, " + "code box, caption, header, footer or page number. Use the format " + "label [x0, y0, x1, y1]transcription with pixel coordinates in the " + "supplied image, one region starting on each new line. Labels: " + "title, paragraph, image, chart, table, code, caption, header, " + "footer, page_number, aside_text. Enclose a whole chart or table " + "in one box, not separate axis labels or cells. Preserve table " + "rows and columns as Markdown and code indentation and line breaks. " + "Images may have empty transcription; do not invent descriptions. " + "Do not merge neighbouring articles or include commentary." + ) payload = { "model": model, "messages": [{"role": "user", "content": [ - {"type": "text", "text": ( - "Recognise the page layout and transcribe visible text verbatim. " - "Return one region per headline, paragraph, image, chart, table, " - "code box, caption, header, footer or page number. Use the format " - "label [x0, y0, x1, y1]transcription with pixel coordinates in the " - "supplied image, one region starting on each new line. Labels: " - "title, paragraph, image, chart, table, code, caption, header, " - "footer, page_number, aside_text. Enclose a whole chart or table " - "in one box, not separate axis labels or cells. Preserve table " - "rows and columns as Markdown and code indentation and line breaks. " - "Images may have empty transcription; do not invent descriptions. " - "Do not merge neighbouring articles or include commentary." - )}, + {"type": "text", "text": prompt}, {"type": "image_url", "image_url": { "url": f"data:image/png;base64,{image_b64}", }}, @@ -934,6 +993,18 @@ def vision_ocr_page(png_path: Path, base_url: str, model: str, "max_tokens": 4000, "temperature": 0.0, } + # Unlimited-OCR's official vLLM recipe requires its per-request + # NGramPerReqLogitsProcessor settings. The container already exposes the + # processor, but without these fields it does nothing and the model can + # run into a repeated chart/formula loop. Keep the vendor-specific fields + # off generic OpenAI-compatible vision engines. + if unlimited_ocr: + payload.update({ + "skip_special_tokens": False, + "ngram_size": 35, + "window_size": 128, + "images_config": {"image_mode": "gundam"}, + }) try: resp = request_with_retries( "POST", f"{base_url.rstrip('/')}/chat/completions", @@ -952,7 +1023,21 @@ def vision_ocr_page(png_path: Path, base_url: str, model: str, if not isinstance(content, str): return None - regions = parse_vision_ocr_regions(content) + regions = parse_unlimited_ocr_regions(content) if unlimited_ocr else parse_vision_ocr_regions(content) + if regions and unlimited_ocr: + # Native markers use the model input's 0–999-ish coordinate space; + # blocks downstream expect coordinates of the rendered PNG. + try: + from PIL import Image + with Image.open(png_path) as im: + width, height = im.size + for region in regions: + if region.pop("native_normalized_bbox", False): + x, y, w, h = region["bbox"] + region["bbox"] = [x / 1000 * width, y / 1000 * height, + w / 1000 * width, h / 1000 * height] + except Exception: + return None return regions or _whole_page_region(content, png_path) diff --git a/static/css/audiobook.css b/static/css/audiobook.css index 604e6bb..797e122 100644 --- a/static/css/audiobook.css +++ b/static/css/audiobook.css @@ -1409,7 +1409,11 @@ .cs-progress-output { display: flex; flex-direction: column; - flex: 1 1 auto; + /* Keep the live-output sheet immediately above the progress footer. Its + top grip then behaves like a bottom-anchored drawer: changing height + moves the top edge only, never creates/removes a gap below the panel. */ + flex: 0 0 auto; + margin-top: auto; gap: 8px; min-height: 120px; max-height: 42vh; @@ -1552,10 +1556,8 @@ .cs-progress-layout), flex:1 here forced THIS panel to stretch to match, leaving a large empty gap below a short/not-yet-filled sheet (confirmed live with a short card next to a 72-character sidebar). - Shrink-to-content instead; .cs-progress-output below absorbs any - leftover column height since it's the piece meant to grow (it already - scrolls its own passage/live-output text, unlike this preview which - shows one bounded character sheet). */ + Shrink-to-content instead; the bottom-anchored .cs-progress-output + keeps its own scrollable, user-resizable height below it. */ margin: 10px 0 0; padding: 0; min-height: 220px; diff --git a/static/js/character-sheets.js b/static/js/character-sheets.js index 309f2e7..3474fa7 100644 --- a/static/js/character-sheets.js +++ b/static/js/character-sheets.js @@ -761,8 +761,6 @@ async function csGenerate(text, cacheKey, initialRoster, opts = {}) { const seed = csSeedSheet(seedRaw); const name = String(seed.name || '').trim(); if (!name) return; - const lc = lineCounts.get(name.toLowerCase()); - if (lc != null) seed.line_count = lc; map.set(name.toLowerCase(), seed); establishedKeys.add(name.toLowerCase()); const created = seedRaw?.created ? new Date(seedRaw.created).getTime() : NaN; @@ -777,11 +775,13 @@ async function csGenerate(text, cacheKey, initialRoster, opts = {}) { const key = name.toLowerCase(); if (!map.has(key)) { const blank = csBlankSheet(name); - const lc = lineCounts.get(key); - if (lc != null) blank.line_count = lc; map.set(key, blank); } }); + // Saved sheets can already have aliases (for example, "Schwester Larissa") + // while the dialogue cast keeps the shorter canonical name ("Larissa"), + // or vice versa. Apply the identity-aware tally after all seeds exist. + csAttachLineCounts([...map.values()], lineCounts); try { for (let i = 0; i < chunks.length; i++) { if (_cs.cancel) break; @@ -876,6 +876,10 @@ async function csGenerate(text, cacheKey, initialRoster, opts = {}) { }); }); const changed = csMerge(map, incomingSheets, { strictRoster: targetMode }); + // A response can reveal an alias only in this passage. Recalculate + // immediately so the live header/sidebar does not keep showing the + // old exact-name-only count until the full run has finished. + csAttachLineCounts([...map.values()], lineCounts); if (changed.length) prog.focus(changed[0].name); if (!targetMode) { (data.characters || []).forEach(n => { if (!roster.some(r => r.toLowerCase() === String(n || '').toLowerCase())) roster.push(n); }); @@ -1769,16 +1773,12 @@ function csOverviewCardHtml(rec) { `; } -// Field/section helpers matching the richer 2-column "lcd-*" layout (same -// classes the audiobook casting Profil panel and the full Library character -// editor use), instead of csCardHtml's old flat
of rows. +// Field/section helpers for the richer character profile layout. The profile +// itself uses stacked field headings and prose; the shared casting/editor +// components retain their denser layouts. function csLcdField(label, value, sheet, field) { if (!value) return ''; const badge = sheet && field ? csSourceBadgeHtml(sheet, field) : ''; - // A compact list row (label left, value right, thin separator) instead of - // the stacked label-above-value block — reads as a scannable list, closer - // to the old flat field list, while still living inside the grouped - // 2-column sections. if (!label) return `
${escHtml(String(value))}${badge ? ` ${badge}` : ''}
`; return `
${escHtml(label)}
${escHtml(String(value))}${badge ? ` ${badge}` : ''}
`; } @@ -1829,7 +1829,7 @@ function csCardHtml(s, meta = {}) { const voiceId = meta.voice || s.voice || ''; return `
${csProfileHeaderHtml(s, meta, csDetailActionsHtml(s))} -
+
Stimme
${voiceId ? escHtml(voiceId) : 'Noch keine Stimme'} @@ -2417,12 +2417,32 @@ function csGoToLibrary() { // Best-effort dialogue-line tally so the Library's table view can show a // "lines" column — only meaningful right after a cast/rehearsal run is in // memory, since line attribution itself isn't part of character-sheet -// generation. Matches case-insensitively since sheet names and roster/script -// speaker names aren't guaranteed identical casing. +// generation. A dialogue caster may use an explicit alias ("Schwester +// Larissa") while the profile is headed by the canonical name ("Larissa"). +// Count every distinct identity token recorded on the sheet, but deliberately +// do not infer from roles/titles: two different people can share those. +function csLineCountForSheet(sheet, counts) { + if (!counts || !counts.size) return null; + let total = 0; + let matched = false; + const seen = new Set(); + csNameTokens(sheet).forEach(function (token) { + const key = String(token || '').trim().toLowerCase(); + if (!key || seen.has(key)) return; + seen.add(key); + const count = Number(counts.get(key)); + if (Number.isFinite(count)) { + total += count; + matched = true; + } + }); + return matched ? total : null; +} + function csAttachLineCounts(sheets, counts) { if (!counts || !counts.size) return; sheets.forEach(function (s) { - const n = counts.get(String(s.name || '').trim().toLowerCase()); + const n = csLineCountForSheet(s, counts); if (n != null) s.line_count = n; }); } diff --git a/static/js/magazine.js b/static/js/magazine.js index 1569b93..7fe0e16 100644 --- a/static/js/magazine.js +++ b/static/js/magazine.js @@ -78,6 +78,46 @@ function fmtPages(pages) { return 'pp. ' + runs.join(', '); } +// The live OCR view is intentionally split: a portrait source page needs more +// horizontal room than a dense recognised-text pane on some screens, while on +// others the text is the thing being inspected. Keep a per-browser choice so +// the user does not have to re-adjust it for every magazine/page. +function setupMagIngestPreviewResize() { + const preview = $('mag-ingest-preview'); + const grip = $('mag-ingest-preview-resize'); + if (!preview || !grip || preview.dataset.resizeWired) return; + preview.dataset.resizeWired = '1'; + const storageKey = 'ttsvc_mag_ingest_page_pane_pct'; + const apply = (value) => { + const pct = Math.max(30, Math.min(70, Number(value) || 50)); + preview.style.gridTemplateColumns = `${pct}% 8px calc(${100 - pct}% - 8px)`; + return pct; + }; + try { + const saved = localStorage.getItem(storageKey); + if (saved != null) apply(saved); + } catch (_) {} + grip.addEventListener('pointerdown', (event) => { + if (window.matchMedia('(max-width: 860px)').matches) return; + event.preventDefault(); + preview.classList.add('is-resizing'); + const move = (e) => { + const rect = preview.getBoundingClientRect(); + apply(((e.clientX - rect.left) / rect.width) * 100); + }; + const stop = (e) => { + window.removeEventListener('pointermove', move); + window.removeEventListener('pointerup', stop); + preview.classList.remove('is-resizing'); + const rect = preview.getBoundingClientRect(); + const pct = apply(((e.clientX - rect.left) / rect.width) * 100); + try { localStorage.setItem(storageKey, String(pct)); } catch (_) {} + }; + window.addEventListener('pointermove', move); + window.addEventListener('pointerup', stop, { once: true }); + }); +} + // ── Library ────────────────────────────────────────────────────────────────── async function magLoadLibrary() { @@ -1469,6 +1509,7 @@ async function magInit() { magLoadLibrary(); magLoadVoices(); setupMagDropzone(); + setupMagIngestPreviewResize(); let lastId = null; try { lastId = localStorage.getItem(MAG_LAST_DOC_KEY); } catch (_) {} if (lastId) { diff --git a/static/sections/s-magazine.html b/static/sections/s-magazine.html index 02f919c..5baf5cc 100644 --- a/static/sections/s-magazine.html +++ b/static/sections/s-magazine.html @@ -63,6 +63,7 @@ Page Page currently being read
+
Recognised text
diff --git a/static/style.css b/static/style.css index 4cdc6c7..42cd2af 100644 --- a/static/style.css +++ b/static/style.css @@ -1237,6 +1237,25 @@ tr.lib-char-card.lib-chars-tbl-row:hover { transform: none; box-shadow: none; ba } .lcd-row dd { margin: 0; color: var(--text); line-height: 1.5; } .lcd-row dd .cs-source-mark { margin-left: 6px; vertical-align: middle; } +/* Character sheets are read as a profile, not a compact comparison table: + each field has a bold heading followed by normal prose in one column. + Scope this to the sheet so the denser casting/library editor remains intact. */ +.cs-profile-layout .lcd-sections { grid-template-columns: 1fr; } +.cs-profile-layout .lcd-section:nth-child(odd) { border-right: none; } +.cs-profile-layout .lcd-section:nth-child(odd) { background: var(--surface); } +.cs-profile-layout .lcd-section:nth-child(even) { background: var(--panel); } +.cs-profile-layout .lcd-row { + display: block; + padding: 10px 0; +} +.cs-profile-layout .lcd-row dt { + color: var(--text); + font-size: 12px; + font-weight: 800; + letter-spacing: .02em; + text-transform: none; +} +.cs-profile-layout .lcd-row dd { margin-top: 4px; font-size: 14px; line-height: 1.55; } .lcd-field-label { font-size:11px; font-weight:700; color:var(--subtext); text-transform:uppercase; letter-spacing:.5px; margin-bottom:5px; @@ -4797,15 +4816,22 @@ input[list="lang-codes"] { page on one side, recognised text filled in on the other as each page finishes. Dark, console-like treatment on purpose: distinct from the rest of the page so it clearly reads as a live process, not settled content. */ -.mag-ingest-preview { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; - background: var(--border); border-radius: var(--radius); overflow: hidden; margin-bottom: 14px; } -.mag-ingest-preview-side { background: #14161a; padding: 10px; display: flex; flex-direction: column; gap: 8px; min-height: 220px; } +.mag-ingest-preview { display: grid; grid-template-columns: minmax(280px, 1fr) 8px minmax(300px, 1fr); gap: 0; + background: var(--border); border-radius: var(--radius); overflow: hidden; margin-bottom: 14px; min-height: min(56vh, 540px); } +.mag-ingest-preview-side { background: #14161a; padding: 10px; display: flex; flex-direction: column; gap: 8px; min-height: 0; } .mag-ingest-preview-label { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: #8a8f98; } -.mag-ingest-preview-side img { max-width: 100%; max-height: 340px; object-fit: contain; margin: 0 auto; border-radius: 4px; } +.mag-ingest-preview-side img { width: 100%; min-width: 0; flex: 1 1 auto; min-height: 0; object-fit: contain; margin: 0 auto; border-radius: 4px; } .mag-ingest-preview-text { color: #d7dae0; font-size: 13px; line-height: 1.6; white-space: pre-wrap; - overflow-y: auto; max-height: 340px; font-family: ui-monospace, 'SF Mono', Consolas, monospace; } + overflow-y: auto; min-height: 0; flex: 1 1 auto; font-family: ui-monospace, 'SF Mono', Consolas, monospace; } +.mag-ingest-preview-resize { background: var(--border); cursor: col-resize; position: relative; touch-action: none; } +.mag-ingest-preview-resize::after { content: ''; position: absolute; inset: 0 2px; background: transparent; } +.mag-ingest-preview.is-resizing { user-select: none; cursor: col-resize; } +.mag-ingest-preview.is-resizing .mag-ingest-preview-resize, +.mag-ingest-preview-resize:hover { background: var(--accent); } @media (max-width: 860px) { - .mag-ingest-preview { grid-template-columns: 1fr; } + .mag-ingest-preview { grid-template-columns: 1fr; min-height: 0; } + .mag-ingest-preview-side { min-height: 300px; } + .mag-ingest-preview-resize { display: none; } } .mag-pill { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 12px; diff --git a/tests/test_magazine.py b/tests/test_magazine.py index 1c52f17..5360c75 100644 --- a/tests/test_magazine.py +++ b/tests/test_magazine.py @@ -54,6 +54,7 @@ from core.magazine import ( mark_sidebars, ocr_quality_score, parse_contents, + parse_unlimited_ocr_regions, parse_vision_ocr_regions, score_contents_page, vision_ocr_page, @@ -819,6 +820,55 @@ class VisionOcrPageCallTests(unittest.TestCase): self.assertIsNotNone(out) self.assertEqual(out[0]["label"], "text") + def test_unlimited_ocr_request_enables_its_required_repeat_guard(self): + body = {"choices": [{"message": {"content": + "<|det|>chart [10,10,110,60]<|/det|>"}}]} + with mock.patch("core.magazine.request_with_retries", + return_value=self._resp(json_body=body)) as request, \ + mock.patch("PIL.Image.open") as mopen: + mopen.return_value.__enter__.return_value.size = (1000, 1400) + self.assertIsNotNone(vision_ocr_page(pathlib.Path(__file__), "http://engine/v1", "baidu/Unlimited-OCR")) + payload = request.call_args.kwargs["json"] + self.assertFalse(payload["skip_special_tokens"]) + self.assertEqual(payload["ngram_size"], 35) + self.assertEqual(payload["window_size"], 128) + self.assertEqual(payload["images_config"], {"image_mode": "gundam"}) + self.assertEqual(payload["messages"][0]["content"][0]["text"], "document parsing.") + + def test_other_vision_engines_do_not_receive_unlimited_ocr_options(self): + body = {"choices": [{"message": {"content": + "text [10,10,110,60]Hello there, this is a real sentence."}}]} + with mock.patch("core.magazine.request_with_retries", + return_value=self._resp(json_body=body)) as request: + self.assertIsNotNone(vision_ocr_page(pathlib.Path(__file__), "http://engine/v1", "other-vision")) + payload = request.call_args.kwargs["json"] + self.assertNotIn("ngram_size", payload) + self.assertNotIn("window_size", payload) + + +class UnlimitedOcrRegionTests(unittest.TestCase): + # The URL/error-path tests below remain deliberately adjacent to the + # native-marker tests; share the same mocked OpenAI response helper. + def _resp(self, status=200, json_body=None): + import requests + resp = mock.Mock(status_code=status) + resp.raise_for_status = mock.Mock( + side_effect=None if status < 400 else requests.exceptions.HTTPError(f"HTTP {status}")) + resp.json = mock.Mock(return_value=json_body or {}) + return resp + + def test_native_detection_markers_preserve_visuals_and_page_furniture(self): + raw = "\n".join([ + "<|det|>text [10,20,900,180]<|/det|>Article body.", + "<|det|>chart [100,210,900,650]<|/det|>", + "<|det|>footer [10,930,700,960]<|/det|>Sources: OECD.", + "<|det|>page_number [940,930,980,960]<|/det|>18", + ]) + regions = parse_unlimited_ocr_regions(raw) + self.assertEqual([r["label"] for r in regions], ["text", "chart", "footer", "page_number"]) + self.assertEqual(regions[1]["text"], "") + self.assertTrue(all(r["native_normalized_bbox"] for r in regions)) + def test_bare_server_url_gets_v1_and_explicit_paths_are_preserved(self): body = {"choices": [{"message": {"content": "text [10,10,110,60]Hello there, this is a real sentence."}}]} @@ -1094,6 +1144,14 @@ class WholePageVisionFallbackTests(unittest.TestCase): mopen.return_value.__enter__.return_value.size = (897, 1049) self.assertIsNone(_whole_page_region(poisoned, pathlib.Path("/fake.png"))) + def test_repeated_fabricated_latex_loop_is_rejected(self): + """Keep a vision model's repeated formula loop out of OCR text.""" + from core.magazine import _whole_page_region + poisoned = "Studien widerlegen. " + (r"\( \therefore m = \frac{3}{11} \) " * 6) + with mock.patch("PIL.Image.open") as mopen: + mopen.return_value.__enter__.return_value.size = (897, 1049) + self.assertIsNone(_whole_page_region(poisoned, pathlib.Path("/fake.png"))) + def test_too_short_text_is_rejected(self): from core.magazine import _whole_page_region self.assertIsNone(_whole_page_region("Hi.", pathlib.Path("/fake.png")))