fix: use native Unlimited-OCR layout regions
Some checks are pending
Test and SonarQube / quality (push) Waiting to run

This commit is contained in:
mARTin-B78 2026-09-09 13:27:14 +02:00
parent b43bacc275
commit 272d4872bd
9 changed files with 302 additions and 43 deletions

View File

@ -9,6 +9,32 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
--- ---
## [1.26.4] — 2026-09-09
### Added
- **Native Baidu Unlimited-OCR Layout Parsing.** Magazine ingestion now uses Unlimited-OCR's documented
`document parsing.` mode and consumes its `<|det|>` bounding-box regions. Images, non-text graphics, charts and
tables are retained as visual regions instead of being mixed into narratable article prose; captions remain captions.
### Changed
- **Magazine OCR Preview Uses Its Space.** The source page fills the available preview height while preserving its
aspect ratio. A draggable divider lets readers allocate space between the page and recognised text, and remembers
the choice.
- **Character Sheets Read Like Profiles.** Profile fields are now one-column entries with a bold heading and normal
prose beneath it, rather than compact label/value columns. The live passage/output drawer stays bottom-anchored and
its top grip resizes from the expected edge.
### Fixed
- **Unlimited-OCR Repetition Guard Was Not Activated.** Requests now include the model's required per-request
n-gram settings and `gundam` image mode. Repeated fabricated LaTex and instruction-echo responses are rejected so
fallback OCR can be used instead of saving poisoned page text.
- **Character Line Totals Missed Aliases.** A profile now totals dialogue credited to its canonical name and recorded
identity aliases (for example, `Larissa` and `Schwester Larissa`) and refreshes the total during casting.
- **Page Furniture Does Not Become Article Text.** Native OCR regions now identify headers, footers and page numbers
as page furniture, keeping them out of narration while preserving their position for review.
---
## [1.26.3] — 2026-09-09 ## [1.26.3] — 2026-09-09
### Fixed ### Fixed

View File

@ -1 +1 @@
1.26.3 1.26.4

View File

@ -742,6 +742,8 @@ _VISION_LABEL_TYPES = {
"title": BLOCK_TITLE, "title": BLOCK_TITLE,
"heading": BLOCK_HEADING, "heading": BLOCK_HEADING,
"caption": BLOCK_CAPTION, "caption": BLOCK_CAPTION,
"image_caption": BLOCK_CAPTION,
"table_caption": BLOCK_CAPTION,
"headline": BLOCK_TITLE, "headline": BLOCK_TITLE,
"article_headline": BLOCK_TITLE, "article_headline": BLOCK_TITLE,
"paragraph": BLOCK_BODY, "paragraph": BLOCK_BODY,
@ -758,6 +760,8 @@ _VISION_LABEL_TYPES = {
"graphic": BLOCK_CHART, "graphic": BLOCK_CHART,
"diagram": BLOCK_CHART, "diagram": BLOCK_CHART,
"table": BLOCK_TABLE, "table": BLOCK_TABLE,
"formula": BLOCK_CODE,
"list": BLOCK_BODY,
"code": BLOCK_CODE, "code": BLOCK_CODE,
"code_block": BLOCK_CODE, "code_block": BLOCK_CODE,
"advertisement": BLOCK_AD, "advertisement": BLOCK_AD,
@ -769,6 +773,15 @@ _VISION_LABEL_TYPES = {
# the next such marker. Confirmed live against Baidu Unlimited-OCR's actual # the next such marker. Confirmed live against Baidu Unlimited-OCR's actual
# output shape: "text [241, 90, 588, 249]Battery technology is entering...". # output shape: "text [241, 90, 588, 249]Battery technology is entering...".
_VISION_REGION = re.compile(r"(?m)^(\w+)\s*\[\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\]") _VISION_REGION = re.compile(r"(?m)^(\w+)\s*\[\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\]")
# Native Baidu Unlimited-OCR document-parsing output. The model card emits
# one marker per detected region, such as
# ``<|det|>table [0,0,999,999]<|/det|>...``. Its categories are precisely
# what we need to retain charts/tables/images as visual blocks instead of
# treating their axis labels as article prose.
_UNLIMITED_OCR_REGION = re.compile(
r"(?m)^<\|det\|>\s*(.*?)\s*(?:\[\s*(-?\d+(?:\.\d+)?)\s*,\s*"
r"(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*"
r"(-?\d+(?:\.\d+)?)\s*\])?\s*<\|/det\|>")
def ocr_quality_score(text: str) -> float: def ocr_quality_score(text: str) -> float:
@ -848,6 +861,42 @@ def parse_vision_ocr_regions(raw: str) -> list[dict[str, Any]]:
return regions return regions
def parse_unlimited_ocr_regions(raw: str) -> list[dict[str, Any]]:
"""Parse Unlimited-OCR's native ``<|det|>`` document-layout markers.
Bounding boxes are in its 0–999-ish image coordinate space; callers mark
them for conversion to the rendered page's pixels before building blocks.
A `[Non-Text]` region is a real visual object even when it has no text.
"""
matches = list(_UNLIMITED_OCR_REGION.finditer(raw or ""))
regions = []
for i, match in enumerate(matches):
raw_label = (match.group(1) or "").strip().strip("[]")
label = raw_label.lower().replace("-", "_").replace(" ", "_")
coords = match.groups()[1:]
if len(coords) != 4 or any(v is None for v in coords):
continue
x0, y0, x1, y1 = (float(v) for v in coords)
if x1 <= x0 or y1 <= y0:
continue
start = match.end()
end = matches[i + 1].start() if i + 1 < len(matches) else len(raw or "")
text = (raw or "")[start:end].strip()
# The model's generic Non-Text label is an image/figure. It must be
# retained even empty, so crop_visual_regions can preserve it.
if label in {"non_text", "nontext"}:
label = "image"
if not text and _VISION_LABEL_TYPES.get(label) not in VISUAL_TYPES:
continue
regions.append({
"label": label,
"bbox": [x0, y0, x1 - x0, y1 - y0],
"text": text,
"native_normalized_bbox": True,
})
return regions
def _whole_page_region(content: str, png_path: Path) -> list[dict[str, Any]] | None: def _whole_page_region(content: str, png_path: Path) -> list[dict[str, Any]] | None:
"""Fallback for an engine that answers with plain text and no coordinate """Fallback for an engine that answers with plain text and no coordinate
markers at all (confirmed live: KDL Frontier Parser nano -- fast, but no markers at all (confirmed live: KDL Frontier Parser nano -- fast, but no
@ -871,10 +920,18 @@ def _whole_page_region(content: str, png_path: Path) -> list[dict[str, Any]] | N
escaped_newlines = len(re.findall(r"\\n", text)) escaped_newlines = len(re.findall(r"\\n", text))
equation_tokens = len(re.findall(r"`?\s*\\?equation\b", text, re.IGNORECASE)) equation_tokens = len(re.findall(r"`?\s*\\?equation\b", text, re.IGNORECASE))
prompt_echo = "do not use special symbols" in text.lower() prompt_echo = "do not use special symbols" in text.lower()
# Another observed loop starts with genuine page text and then repeats a
# fabricated LaTex fragment (``\\therefore m = \\frac{3}{11}``) until the
# response limit. It has no instruction echo or `equation` token, so it
# needs its own narrow signature. Requiring both uncommon controls many
# times avoids rejecting ordinary prose that merely mentions one formula.
therefore_tokens = len(re.findall(r"\\therefore\b", text, re.IGNORECASE))
fraction_tokens = len(re.findall(r"\\frac\s*\{", text, re.IGNORECASE))
latex_loop = therefore_tokens >= 4 and fraction_tokens >= 4
if (escaped_newlines >= 4 and equation_tokens >= 4) or ( if (escaped_newlines >= 4 and equation_tokens >= 4) or (
prompt_echo and escaped_newlines >= 2 and equation_tokens >= 2 prompt_echo and escaped_newlines >= 2 and equation_tokens >= 2
): ) or latex_loop:
logging.getLogger(__name__).warning("Vision OCR returned instruction-echo artifact; discarding response") logging.getLogger(__name__).warning("Vision OCR returned a repeated formatting artifact; discarding response")
return None return None
if len(text) < 20 or ocr_quality_score(text) < _VISION_QUALITY_THRESHOLD: if len(text) < 20 or ocr_quality_score(text) < _VISION_QUALITY_THRESHOLD:
return None return None
@ -911,22 +968,24 @@ def vision_ocr_page(png_path: Path, base_url: str, model: str,
if not urlsplit(base_url).path: if not urlsplit(base_url).path:
base_url += "/v1" base_url += "/v1"
unlimited_ocr = "unlimited-ocr" in model.lower()
prompt = "document parsing." if unlimited_ocr else (
"Recognise the page layout and transcribe visible text verbatim. "
"Return one region per headline, paragraph, image, chart, table, "
"code box, caption, header, footer or page number. Use the format "
"label [x0, y0, x1, y1]transcription with pixel coordinates in the "
"supplied image, one region starting on each new line. Labels: "
"title, paragraph, image, chart, table, code, caption, header, "
"footer, page_number, aside_text. Enclose a whole chart or table "
"in one box, not separate axis labels or cells. Preserve table "
"rows and columns as Markdown and code indentation and line breaks. "
"Images may have empty transcription; do not invent descriptions. "
"Do not merge neighbouring articles or include commentary."
)
payload = { payload = {
"model": model, "model": model,
"messages": [{"role": "user", "content": [ "messages": [{"role": "user", "content": [
{"type": "text", "text": ( {"type": "text", "text": prompt},
"Recognise the page layout and transcribe visible text verbatim. "
"Return one region per headline, paragraph, image, chart, table, "
"code box, caption, header, footer or page number. Use the format "
"label [x0, y0, x1, y1]transcription with pixel coordinates in the "
"supplied image, one region starting on each new line. Labels: "
"title, paragraph, image, chart, table, code, caption, header, "
"footer, page_number, aside_text. Enclose a whole chart or table "
"in one box, not separate axis labels or cells. Preserve table "
"rows and columns as Markdown and code indentation and line breaks. "
"Images may have empty transcription; do not invent descriptions. "
"Do not merge neighbouring articles or include commentary."
)},
{"type": "image_url", "image_url": { {"type": "image_url", "image_url": {
"url": f"data:image/png;base64,{image_b64}", "url": f"data:image/png;base64,{image_b64}",
}}, }},
@ -934,6 +993,18 @@ def vision_ocr_page(png_path: Path, base_url: str, model: str,
"max_tokens": 4000, "max_tokens": 4000,
"temperature": 0.0, "temperature": 0.0,
} }
# Unlimited-OCR's official vLLM recipe requires its per-request
# NGramPerReqLogitsProcessor settings. The container already exposes the
# processor, but without these fields it does nothing and the model can
# run into a repeated chart/formula loop. Keep the vendor-specific fields
# off generic OpenAI-compatible vision engines.
if unlimited_ocr:
payload.update({
"skip_special_tokens": False,
"ngram_size": 35,
"window_size": 128,
"images_config": {"image_mode": "gundam"},
})
try: try:
resp = request_with_retries( resp = request_with_retries(
"POST", f"{base_url.rstrip('/')}/chat/completions", "POST", f"{base_url.rstrip('/')}/chat/completions",
@ -952,7 +1023,21 @@ def vision_ocr_page(png_path: Path, base_url: str, model: str,
if not isinstance(content, str): if not isinstance(content, str):
return None return None
regions = parse_vision_ocr_regions(content) regions = parse_unlimited_ocr_regions(content) if unlimited_ocr else parse_vision_ocr_regions(content)
if regions and unlimited_ocr:
# Native markers use the model input's 0–999-ish coordinate space;
# blocks downstream expect coordinates of the rendered PNG.
try:
from PIL import Image
with Image.open(png_path) as im:
width, height = im.size
for region in regions:
if region.pop("native_normalized_bbox", False):
x, y, w, h = region["bbox"]
region["bbox"] = [x / 1000 * width, y / 1000 * height,
w / 1000 * width, h / 1000 * height]
except Exception:
return None
return regions or _whole_page_region(content, png_path) return regions or _whole_page_region(content, png_path)

View File

@ -1409,7 +1409,11 @@
.cs-progress-output { .cs-progress-output {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
flex: 1 1 auto; /* Keep the live-output sheet immediately above the progress footer. Its
top grip then behaves like a bottom-anchored drawer: changing height
moves the top edge only, never creates/removes a gap below the panel. */
flex: 0 0 auto;
margin-top: auto;
gap: 8px; gap: 8px;
min-height: 120px; min-height: 120px;
max-height: 42vh; max-height: 42vh;
@ -1552,10 +1556,8 @@
.cs-progress-layout), flex:1 here forced THIS panel to stretch to .cs-progress-layout), flex:1 here forced THIS panel to stretch to
match, leaving a large empty gap below a short/not-yet-filled sheet match, leaving a large empty gap below a short/not-yet-filled sheet
(confirmed live with a short card next to a 72-character sidebar). (confirmed live with a short card next to a 72-character sidebar).
Shrink-to-content instead; .cs-progress-output below absorbs any Shrink-to-content instead; the bottom-anchored .cs-progress-output
leftover column height since it's the piece meant to grow (it already keeps its own scrollable, user-resizable height below it. */
scrolls its own passage/live-output text, unlike this preview which
shows one bounded character sheet). */
margin: 10px 0 0; margin: 10px 0 0;
padding: 0; padding: 0;
min-height: 220px; min-height: 220px;

View File

@ -761,8 +761,6 @@ async function csGenerate(text, cacheKey, initialRoster, opts = {}) {
const seed = csSeedSheet(seedRaw); const seed = csSeedSheet(seedRaw);
const name = String(seed.name || '').trim(); const name = String(seed.name || '').trim();
if (!name) return; if (!name) return;
const lc = lineCounts.get(name.toLowerCase());
if (lc != null) seed.line_count = lc;
map.set(name.toLowerCase(), seed); map.set(name.toLowerCase(), seed);
establishedKeys.add(name.toLowerCase()); establishedKeys.add(name.toLowerCase());
const created = seedRaw?.created ? new Date(seedRaw.created).getTime() : NaN; const created = seedRaw?.created ? new Date(seedRaw.created).getTime() : NaN;
@ -777,11 +775,13 @@ async function csGenerate(text, cacheKey, initialRoster, opts = {}) {
const key = name.toLowerCase(); const key = name.toLowerCase();
if (!map.has(key)) { if (!map.has(key)) {
const blank = csBlankSheet(name); const blank = csBlankSheet(name);
const lc = lineCounts.get(key);
if (lc != null) blank.line_count = lc;
map.set(key, blank); map.set(key, blank);
} }
}); });
// Saved sheets can already have aliases (for example, "Schwester Larissa")
// while the dialogue cast keeps the shorter canonical name ("Larissa"),
// or vice versa. Apply the identity-aware tally after all seeds exist.
csAttachLineCounts([...map.values()], lineCounts);
try { try {
for (let i = 0; i < chunks.length; i++) { for (let i = 0; i < chunks.length; i++) {
if (_cs.cancel) break; if (_cs.cancel) break;
@ -876,6 +876,10 @@ async function csGenerate(text, cacheKey, initialRoster, opts = {}) {
}); });
}); });
const changed = csMerge(map, incomingSheets, { strictRoster: targetMode }); const changed = csMerge(map, incomingSheets, { strictRoster: targetMode });
// A response can reveal an alias only in this passage. Recalculate
// immediately so the live header/sidebar does not keep showing the
// old exact-name-only count until the full run has finished.
csAttachLineCounts([...map.values()], lineCounts);
if (changed.length) prog.focus(changed[0].name); if (changed.length) prog.focus(changed[0].name);
if (!targetMode) { if (!targetMode) {
(data.characters || []).forEach(n => { if (!roster.some(r => r.toLowerCase() === String(n || '').toLowerCase())) roster.push(n); }); (data.characters || []).forEach(n => { if (!roster.some(r => r.toLowerCase() === String(n || '').toLowerCase())) roster.push(n); });
@ -1769,16 +1773,12 @@ function csOverviewCardHtml(rec) {
</div>`; </div>`;
} }
// Field/section helpers matching the richer 2-column "lcd-*" layout (same // Field/section helpers for the richer character profile layout. The profile
// classes the audiobook casting Profil panel and the full Library character // itself uses stacked field headings and prose; the shared casting/editor
// editor use), instead of csCardHtml's old flat <dl> of rows. // components retain their denser layouts.
function csLcdField(label, value, sheet, field) { function csLcdField(label, value, sheet, field) {
if (!value) return ''; if (!value) return '';
const badge = sheet && field ? csSourceBadgeHtml(sheet, field) : ''; const badge = sheet && field ? csSourceBadgeHtml(sheet, field) : '';
// A compact list row (label left, value right, thin separator) instead of
// the stacked label-above-value block — reads as a scannable list, closer
// to the old flat field list, while still living inside the grouped
// 2-column sections.
if (!label) return `<div class="lcd-row lcd-row-full"><dd>${escHtml(String(value))}${badge ? ` ${badge}` : ''}</dd></div>`; if (!label) return `<div class="lcd-row lcd-row-full"><dd>${escHtml(String(value))}${badge ? ` ${badge}` : ''}</dd></div>`;
return `<div class="lcd-row"><dt>${escHtml(label)}</dt><dd>${escHtml(String(value))}${badge ? ` ${badge}` : ''}</dd></div>`; return `<div class="lcd-row"><dt>${escHtml(label)}</dt><dd>${escHtml(String(value))}${badge ? ` ${badge}` : ''}</dd></div>`;
} }
@ -1829,7 +1829,7 @@ function csCardHtml(s, meta = {}) {
const voiceId = meta.voice || s.voice || ''; const voiceId = meta.voice || s.voice || '';
return `<div class="cs-card cs-${s.tier === 'main' ? 'main' : 'supp'}" data-name="${escHtml(s.name)}"> return `<div class="cs-card cs-${s.tier === 'main' ? 'main' : 'supp'}" data-name="${escHtml(s.name)}">
${csProfileHeaderHtml(s, meta, csDetailActionsHtml(s))} ${csProfileHeaderHtml(s, meta, csDetailActionsHtml(s))}
<div class="lcd-body ab-cd-body"> <div class="lcd-body ab-cd-body cs-profile-layout">
<div class="lcd-voice-top"> <div class="lcd-voice-top">
<div class="lcd-section-label"><span class="mdi mdi-account-voice"></span> Stimme</div> <div class="lcd-section-label"><span class="mdi mdi-account-voice"></span> Stimme</div>
<span class="lcd-voice-label">${voiceId ? escHtml(voiceId) : '<span style="opacity:.5">Noch keine Stimme</span>'}</span> <span class="lcd-voice-label">${voiceId ? escHtml(voiceId) : '<span style="opacity:.5">Noch keine Stimme</span>'}</span>
@ -2417,12 +2417,32 @@ function csGoToLibrary() {
// Best-effort dialogue-line tally so the Library's table view can show a // Best-effort dialogue-line tally so the Library's table view can show a
// "lines" column — only meaningful right after a cast/rehearsal run is in // "lines" column — only meaningful right after a cast/rehearsal run is in
// memory, since line attribution itself isn't part of character-sheet // memory, since line attribution itself isn't part of character-sheet
// generation. Matches case-insensitively since sheet names and roster/script // generation. A dialogue caster may use an explicit alias ("Schwester
// speaker names aren't guaranteed identical casing. // Larissa") while the profile is headed by the canonical name ("Larissa").
// Count every distinct identity token recorded on the sheet, but deliberately
// do not infer from roles/titles: two different people can share those.
function csLineCountForSheet(sheet, counts) {
if (!counts || !counts.size) return null;
let total = 0;
let matched = false;
const seen = new Set();
csNameTokens(sheet).forEach(function (token) {
const key = String(token || '').trim().toLowerCase();
if (!key || seen.has(key)) return;
seen.add(key);
const count = Number(counts.get(key));
if (Number.isFinite(count)) {
total += count;
matched = true;
}
});
return matched ? total : null;
}
function csAttachLineCounts(sheets, counts) { function csAttachLineCounts(sheets, counts) {
if (!counts || !counts.size) return; if (!counts || !counts.size) return;
sheets.forEach(function (s) { sheets.forEach(function (s) {
const n = counts.get(String(s.name || '').trim().toLowerCase()); const n = csLineCountForSheet(s, counts);
if (n != null) s.line_count = n; if (n != null) s.line_count = n;
}); });
} }

View File

@ -78,6 +78,46 @@ function fmtPages(pages) {
return 'pp. ' + runs.join(', '); return 'pp. ' + runs.join(', ');
} }
// The live OCR view is intentionally split: a portrait source page needs more
// horizontal room than a dense recognised-text pane on some screens, while on
// others the text is the thing being inspected. Keep a per-browser choice so
// the user does not have to re-adjust it for every magazine/page.
function setupMagIngestPreviewResize() {
const preview = $('mag-ingest-preview');
const grip = $('mag-ingest-preview-resize');
if (!preview || !grip || preview.dataset.resizeWired) return;
preview.dataset.resizeWired = '1';
const storageKey = 'ttsvc_mag_ingest_page_pane_pct';
const apply = (value) => {
const pct = Math.max(30, Math.min(70, Number(value) || 50));
preview.style.gridTemplateColumns = `${pct}% 8px calc(${100 - pct}% - 8px)`;
return pct;
};
try {
const saved = localStorage.getItem(storageKey);
if (saved != null) apply(saved);
} catch (_) {}
grip.addEventListener('pointerdown', (event) => {
if (window.matchMedia('(max-width: 860px)').matches) return;
event.preventDefault();
preview.classList.add('is-resizing');
const move = (e) => {
const rect = preview.getBoundingClientRect();
apply(((e.clientX - rect.left) / rect.width) * 100);
};
const stop = (e) => {
window.removeEventListener('pointermove', move);
window.removeEventListener('pointerup', stop);
preview.classList.remove('is-resizing');
const rect = preview.getBoundingClientRect();
const pct = apply(((e.clientX - rect.left) / rect.width) * 100);
try { localStorage.setItem(storageKey, String(pct)); } catch (_) {}
};
window.addEventListener('pointermove', move);
window.addEventListener('pointerup', stop, { once: true });
});
}
// ── Library ────────────────────────────────────────────────────────────────── // ── Library ──────────────────────────────────────────────────────────────────
async function magLoadLibrary() { async function magLoadLibrary() {
@ -1469,6 +1509,7 @@ async function magInit() {
magLoadLibrary(); magLoadLibrary();
magLoadVoices(); magLoadVoices();
setupMagDropzone(); setupMagDropzone();
setupMagIngestPreviewResize();
let lastId = null; let lastId = null;
try { lastId = localStorage.getItem(MAG_LAST_DOC_KEY); } catch (_) {} try { lastId = localStorage.getItem(MAG_LAST_DOC_KEY); } catch (_) {}
if (lastId) { if (lastId) {

View File

@ -63,6 +63,7 @@
<span class="mag-ingest-preview-label">Page</span> <span class="mag-ingest-preview-label">Page</span>
<img id="mag-ingest-preview-img" alt="Page currently being read"> <img id="mag-ingest-preview-img" alt="Page currently being read">
</div> </div>
<div class="mag-ingest-preview-resize" id="mag-ingest-preview-resize" role="separator" aria-orientation="vertical" aria-label="Resize page and recognised text panes" title="Drag to resize page and recognised text panes"></div>
<div class="mag-ingest-preview-side"> <div class="mag-ingest-preview-side">
<span class="mag-ingest-preview-label">Recognised text</span> <span class="mag-ingest-preview-label">Recognised text</span>
<div id="mag-ingest-preview-text" class="mag-ingest-preview-text"></div> <div id="mag-ingest-preview-text" class="mag-ingest-preview-text"></div>

View File

@ -1237,6 +1237,25 @@ tr.lib-char-card.lib-chars-tbl-row:hover { transform: none; box-shadow: none; ba
} }
.lcd-row dd { margin: 0; color: var(--text); line-height: 1.5; } .lcd-row dd { margin: 0; color: var(--text); line-height: 1.5; }
.lcd-row dd .cs-source-mark { margin-left: 6px; vertical-align: middle; } .lcd-row dd .cs-source-mark { margin-left: 6px; vertical-align: middle; }
/* Character sheets are read as a profile, not a compact comparison table:
each field has a bold heading followed by normal prose in one column.
Scope this to the sheet so the denser casting/library editor remains intact. */
.cs-profile-layout .lcd-sections { grid-template-columns: 1fr; }
.cs-profile-layout .lcd-section:nth-child(odd) { border-right: none; }
.cs-profile-layout .lcd-section:nth-child(odd) { background: var(--surface); }
.cs-profile-layout .lcd-section:nth-child(even) { background: var(--panel); }
.cs-profile-layout .lcd-row {
display: block;
padding: 10px 0;
}
.cs-profile-layout .lcd-row dt {
color: var(--text);
font-size: 12px;
font-weight: 800;
letter-spacing: .02em;
text-transform: none;
}
.cs-profile-layout .lcd-row dd { margin-top: 4px; font-size: 14px; line-height: 1.55; }
.lcd-field-label { .lcd-field-label {
font-size:11px; font-weight:700; color:var(--subtext); text-transform:uppercase; font-size:11px; font-weight:700; color:var(--subtext); text-transform:uppercase;
letter-spacing:.5px; margin-bottom:5px; letter-spacing:.5px; margin-bottom:5px;
@ -4797,15 +4816,22 @@ input[list="lang-codes"] {
page on one side, recognised text filled in on the other as each page page on one side, recognised text filled in on the other as each page
finishes. Dark, console-like treatment on purpose: distinct from the rest finishes. Dark, console-like treatment on purpose: distinct from the rest
of the page so it clearly reads as a live process, not settled content. */ of the page so it clearly reads as a live process, not settled content. */
.mag-ingest-preview { display: grid; grid-template-columns: 1fr 1fr; gap: 1px; .mag-ingest-preview { display: grid; grid-template-columns: minmax(280px, 1fr) 8px minmax(300px, 1fr); gap: 0;
background: var(--border); border-radius: var(--radius); overflow: hidden; margin-bottom: 14px; } background: var(--border); border-radius: var(--radius); overflow: hidden; margin-bottom: 14px; min-height: min(56vh, 540px); }
.mag-ingest-preview-side { background: #14161a; padding: 10px; display: flex; flex-direction: column; gap: 8px; min-height: 220px; } .mag-ingest-preview-side { background: #14161a; padding: 10px; display: flex; flex-direction: column; gap: 8px; min-height: 0; }
.mag-ingest-preview-label { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: #8a8f98; } .mag-ingest-preview-label { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; color: #8a8f98; }
.mag-ingest-preview-side img { max-width: 100%; max-height: 340px; object-fit: contain; margin: 0 auto; border-radius: 4px; } .mag-ingest-preview-side img { width: 100%; min-width: 0; flex: 1 1 auto; min-height: 0; object-fit: contain; margin: 0 auto; border-radius: 4px; }
.mag-ingest-preview-text { color: #d7dae0; font-size: 13px; line-height: 1.6; white-space: pre-wrap; .mag-ingest-preview-text { color: #d7dae0; font-size: 13px; line-height: 1.6; white-space: pre-wrap;
overflow-y: auto; max-height: 340px; font-family: ui-monospace, 'SF Mono', Consolas, monospace; } overflow-y: auto; min-height: 0; flex: 1 1 auto; font-family: ui-monospace, 'SF Mono', Consolas, monospace; }
.mag-ingest-preview-resize { background: var(--border); cursor: col-resize; position: relative; touch-action: none; }
.mag-ingest-preview-resize::after { content: ''; position: absolute; inset: 0 2px; background: transparent; }
.mag-ingest-preview.is-resizing { user-select: none; cursor: col-resize; }
.mag-ingest-preview.is-resizing .mag-ingest-preview-resize,
.mag-ingest-preview-resize:hover { background: var(--accent); }
@media (max-width: 860px) { @media (max-width: 860px) {
.mag-ingest-preview { grid-template-columns: 1fr; } .mag-ingest-preview { grid-template-columns: 1fr; min-height: 0; }
.mag-ingest-preview-side { min-height: 300px; }
.mag-ingest-preview-resize { display: none; }
} }
.mag-pill { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 12px; .mag-pill { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 12px;

View File

@ -54,6 +54,7 @@ from core.magazine import (
mark_sidebars, mark_sidebars,
ocr_quality_score, ocr_quality_score,
parse_contents, parse_contents,
parse_unlimited_ocr_regions,
parse_vision_ocr_regions, parse_vision_ocr_regions,
score_contents_page, score_contents_page,
vision_ocr_page, vision_ocr_page,
@ -819,6 +820,55 @@ class VisionOcrPageCallTests(unittest.TestCase):
self.assertIsNotNone(out) self.assertIsNotNone(out)
self.assertEqual(out[0]["label"], "text") self.assertEqual(out[0]["label"], "text")
def test_unlimited_ocr_request_enables_its_required_repeat_guard(self):
body = {"choices": [{"message": {"content":
"<|det|>chart [10,10,110,60]<|/det|>"}}]}
with mock.patch("core.magazine.request_with_retries",
return_value=self._resp(json_body=body)) as request, \
mock.patch("PIL.Image.open") as mopen:
mopen.return_value.__enter__.return_value.size = (1000, 1400)
self.assertIsNotNone(vision_ocr_page(pathlib.Path(__file__), "http://engine/v1", "baidu/Unlimited-OCR"))
payload = request.call_args.kwargs["json"]
self.assertFalse(payload["skip_special_tokens"])
self.assertEqual(payload["ngram_size"], 35)
self.assertEqual(payload["window_size"], 128)
self.assertEqual(payload["images_config"], {"image_mode": "gundam"})
self.assertEqual(payload["messages"][0]["content"][0]["text"], "document parsing.")
def test_other_vision_engines_do_not_receive_unlimited_ocr_options(self):
body = {"choices": [{"message": {"content":
"text [10,10,110,60]Hello there, this is a real sentence."}}]}
with mock.patch("core.magazine.request_with_retries",
return_value=self._resp(json_body=body)) as request:
self.assertIsNotNone(vision_ocr_page(pathlib.Path(__file__), "http://engine/v1", "other-vision"))
payload = request.call_args.kwargs["json"]
self.assertNotIn("ngram_size", payload)
self.assertNotIn("window_size", payload)
class UnlimitedOcrRegionTests(unittest.TestCase):
# The URL/error-path tests below remain deliberately adjacent to the
# native-marker tests; share the same mocked OpenAI response helper.
def _resp(self, status=200, json_body=None):
import requests
resp = mock.Mock(status_code=status)
resp.raise_for_status = mock.Mock(
side_effect=None if status < 400 else requests.exceptions.HTTPError(f"HTTP {status}"))
resp.json = mock.Mock(return_value=json_body or {})
return resp
def test_native_detection_markers_preserve_visuals_and_page_furniture(self):
raw = "\n".join([
"<|det|>text [10,20,900,180]<|/det|>Article body.",
"<|det|>chart [100,210,900,650]<|/det|>",
"<|det|>footer [10,930,700,960]<|/det|>Sources: OECD.",
"<|det|>page_number [940,930,980,960]<|/det|>18",
])
regions = parse_unlimited_ocr_regions(raw)
self.assertEqual([r["label"] for r in regions], ["text", "chart", "footer", "page_number"])
self.assertEqual(regions[1]["text"], "")
self.assertTrue(all(r["native_normalized_bbox"] for r in regions))
def test_bare_server_url_gets_v1_and_explicit_paths_are_preserved(self): def test_bare_server_url_gets_v1_and_explicit_paths_are_preserved(self):
body = {"choices": [{"message": {"content": body = {"choices": [{"message": {"content":
"text [10,10,110,60]Hello there, this is a real sentence."}}]} "text [10,10,110,60]Hello there, this is a real sentence."}}]}
@ -1094,6 +1144,14 @@ class WholePageVisionFallbackTests(unittest.TestCase):
mopen.return_value.__enter__.return_value.size = (897, 1049) mopen.return_value.__enter__.return_value.size = (897, 1049)
self.assertIsNone(_whole_page_region(poisoned, pathlib.Path("/fake.png"))) self.assertIsNone(_whole_page_region(poisoned, pathlib.Path("/fake.png")))
def test_repeated_fabricated_latex_loop_is_rejected(self):
"""Keep a vision model's repeated formula loop out of OCR text."""
from core.magazine import _whole_page_region
poisoned = "Studien widerlegen. " + (r"\( \therefore m = \frac{3}{11} \) " * 6)
with mock.patch("PIL.Image.open") as mopen:
mopen.return_value.__enter__.return_value.size = (897, 1049)
self.assertIsNone(_whole_page_region(poisoned, pathlib.Path("/fake.png")))
def test_too_short_text_is_rejected(self): def test_too_short_text_is_rejected(self):
from core.magazine import _whole_page_region from core.magazine import _whole_page_region
self.assertIsNone(_whole_page_region("Hi.", pathlib.Path("/fake.png"))) self.assertIsNone(_whole_page_region("Hi.", pathlib.Path("/fake.png")))