feat(magazine): import a magazine from PNG/JPEG page images
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
0d3178f733
commit
0005ae0ebd
@ -7,6 +7,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
- **Magazine import from PNG/JPEG page images.** Choose or drop several scans or phone photos; they are checked by content (not by name), ordered by filename with natural numbering, turned into one scan-style PDF (page size from the image's own resolution, EXIF rotation applied, 72 dpi treated as unknown) and ingested like any scanned issue. Limits: 300 images, 25 MB and 60 megapixels each; a bad image rejects the whole upload and leaves nothing behind; an existing source is never replaced. Verified on real pages: OCR through the image path scored like the original PDF (F1 0.85 / 0.78).
|
||||
- `architecture/ocr-porting-prompt-general.md`: the OCR porting brief rewritten for general documents (letters, papers, forms, invoices). Adds privacy, pre-processing of skewed/photographed scans, document-type handling, field-level accuracy and handwriting, and tags every claim as measured on magazines, a general principle, or untested for these document types.
|
||||
|
||||
---
|
||||
|
||||
@ -77,7 +77,7 @@ turned into a podcast discussion between two or three voices you pick.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- The magazine as a PDF (native text or scanned — both work).
|
||||
- The magazine as a PDF (native text or scanned — both work), **or** its pages as PNG/JPEG images (scans or phone photos): choose or drop several at once and they become one magazine, pages in filename order (`page2` before `page10`). Up to 300 images of at most 25 MB each; a photo's EXIF rotation is applied. Images carry no text, so every page is read with OCR. Do not mix a PDF with images.
|
||||
- At least one voice in the library for narration.
|
||||
- For podcast mode: an LLM configured in Settings (`llm_url`, `llm_model`), and
|
||||
two or three voices chosen as hosts.
|
||||
|
||||
2
TODO.md
2
TODO.md
@ -656,7 +656,7 @@ casting. The original PDF remains the visual source of truth while approved arti
|
||||
### P0 — Magazine foundation
|
||||
|
||||
- [x] Add a Magazine project type beside Books and Theater Plays.
|
||||
- [~] Add PDF/JPG/PNG import with page previews and safe local storage. *(PDF + page renders done; JPG/PNG not done)*
|
||||
- [x] Add PDF/JPG/PNG import with page previews and safe local storage. *(2026-09-29: JPG/PNG page images become one scan-style PDF, see architecture/magazine-ingestion.md)*
|
||||
- [x] Add OCR-engine selection from Engines > OCR & Document Parsing, with page-image input and structured output.
|
||||
- [x] Detect contents/index pages and propose article titles, authors, and page ranges.
|
||||
- [x] Build a review screen where users approve or correct article boundaries before TTS.
|
||||
|
||||
@ -448,6 +448,17 @@ Verified end-to-end against a synthetic 8-page issue (contents page, two-column
|
||||
feature with a byline and a page-8 jump, full-page advert, second feature,
|
||||
reviews) and against an image-only rendering of the same file to exercise OCR.
|
||||
|
||||
## Import from page images
|
||||
|
||||
`POST /api/magazine/{id}/source-images` (multipart `files`, in the order sent) builds `source.pdf` with
|
||||
`images_to_pdf`, so nothing downstream changes: the PDF has no text layer and every page takes the OCR path.
|
||||
Content is sniffed from the header (PNG/JPEG), never from the name or declared type; limits are 300 files, 25 MB and
|
||||
60 megapixels per image (checked before decoding); EXIF orientation is applied (an unrotated JPEG is embedded
|
||||
byte-for-byte, not re-encoded). Page size comes from the image's dpi; 72 and anything below 100 is the placeholder most
|
||||
tools write, so it is treated as unknown (150 assumed) rather than producing a 33-inch page. The PDF is written through a
|
||||
temporary file, one bad image rejects the whole upload, and a magazine that already has a source returns 409 so pages that
|
||||
were reviewed can never be replaced. The browser orders files by name with numeric collation.
|
||||
|
||||
## Vision OCR escalation
|
||||
|
||||
Engines → OCR & Document Parsing lists layout-aware parsers (KDL Frontier Parser
|
||||
|
||||
@ -2367,6 +2367,81 @@ def ocr_page(png_path: Path, lang: str = "deu+eng",
|
||||
return out
|
||||
|
||||
|
||||
# ── Image import ─────────────────────────────────────────────────────────────
|
||||
|
||||
IMAGE_IMPORT_MAX_FILES = 300
|
||||
IMAGE_IMPORT_MAX_BYTES = 25 * 1024 * 1024 # per image
|
||||
IMAGE_IMPORT_MAX_PIXELS = 60_000_000 # refuses decompression bombs before decoding
|
||||
_ASSUMED_SCAN_DPI = 150 # when an image carries no (or an absurd) resolution
|
||||
|
||||
|
||||
class ImageImportError(ValueError):
|
||||
"""An uploaded page image was rejected; the message is safe to show the user."""
|
||||
|
||||
|
||||
def _sniff_image(data: bytes) -> str | None:
|
||||
"""'png' or 'jpeg' judged by the file's own header, never by its name or declared type."""
|
||||
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
return "png"
|
||||
if data[:3] == b"\xff\xd8\xff":
|
||||
return "jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def images_to_pdf(images: Sequence[tuple[str, bytes]], dest: Path) -> int:
|
||||
"""Combine page images (PNG/JPEG, in the given order) into one PDF at `dest`; returns the page count.
|
||||
|
||||
Each image becomes one page sized from its own resolution (so the app's fixed-dpi render of the
|
||||
page comes out at the scan's real proportions), with EXIF rotation applied. The result has no text
|
||||
layer, so ingestion OCRs every page exactly as for a scanned PDF. Nothing is written on failure.
|
||||
"""
|
||||
if not images:
|
||||
raise ImageImportError("Choose at least one image")
|
||||
if len(images) > IMAGE_IMPORT_MAX_FILES:
|
||||
raise ImageImportError(f"At most {IMAGE_IMPORT_MAX_FILES} images per magazine")
|
||||
import io
|
||||
import pymupdf
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
doc = pymupdf.open()
|
||||
try:
|
||||
for name, data in images:
|
||||
label = Path(name or "image").name[:80]
|
||||
if len(data) > IMAGE_IMPORT_MAX_BYTES:
|
||||
raise ImageImportError(f"{label} is larger than {IMAGE_IMPORT_MAX_BYTES // (1024 * 1024)} MB")
|
||||
kind = _sniff_image(data)
|
||||
if kind is None:
|
||||
raise ImageImportError(f"{label} is not a PNG or JPEG image")
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as im:
|
||||
width, height = im.size
|
||||
if width * height > IMAGE_IMPORT_MAX_PIXELS:
|
||||
raise ImageImportError(f"{label} has too many pixels")
|
||||
dpi = im.info.get("dpi", (0, 0))[0] or 0
|
||||
if im.getexif().get(0x0112, 1) not in (0, 1): # EXIF says the pixels are rotated/flipped
|
||||
upright = ImageOps.exif_transpose(im)
|
||||
buf = io.BytesIO()
|
||||
upright.convert("RGB" if kind == "jpeg" else upright.mode).save(
|
||||
buf, format="JPEG" if kind == "jpeg" else "PNG", **({"quality": 95} if kind == "jpeg" else {}))
|
||||
data = buf.getvalue()
|
||||
width, height = upright.size
|
||||
except ImageImportError:
|
||||
raise
|
||||
except Exception as e: # truncated / corrupt file
|
||||
raise ImageImportError(f"{label} could not be read as an image") from e
|
||||
# 72 (and anything below 100) is the placeholder most tools write, not a real scan resolution.
|
||||
dpi = float(dpi) if 100 <= float(dpi or 0) <= 1200 else _ASSUMED_SCAN_DPI
|
||||
w_pt, h_pt = width * 72.0 / dpi, height * 72.0 / dpi
|
||||
page = doc.new_page(width=w_pt, height=h_pt)
|
||||
page.insert_image(pymupdf.Rect(0, 0, w_pt, h_pt), stream=data)
|
||||
tmp = dest.with_suffix(".part")
|
||||
doc.save(str(tmp))
|
||||
tmp.replace(dest)
|
||||
return len(images)
|
||||
finally:
|
||||
doc.close()
|
||||
|
||||
|
||||
_VISION_QUALITY_THRESHOLD = 0.5 # below this, existing native/OCR text is escalated
|
||||
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi import APIRouter, File, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, Response
|
||||
|
||||
from core import magazine as mag
|
||||
@ -121,6 +121,29 @@ async def magazine_create(request: Request):
|
||||
return {"id": doc_id}
|
||||
|
||||
|
||||
@router.post("/api/magazine/{doc_id}/source-images")
|
||||
async def magazine_source_images(doc_id: str, files: list[UploadFile] = File(...)):
|
||||
"""Build the magazine's source.pdf from uploaded page images (PNG/JPEG, in the order sent).
|
||||
|
||||
Ingestion then treats it as any scanned PDF. Refused when a source already exists, so an upload can
|
||||
never silently replace pages that were already reviewed.
|
||||
"""
|
||||
d, _ = _require_magazine(doc_id)
|
||||
if (d / "source.pdf").exists():
|
||||
raise HTTPException(409, "This magazine already has a source file")
|
||||
if len(files) > mag.IMAGE_IMPORT_MAX_FILES:
|
||||
raise HTTPException(400, f"At most {mag.IMAGE_IMPORT_MAX_FILES} images per magazine")
|
||||
images: list[tuple[str, bytes]] = []
|
||||
for f in files:
|
||||
data = await f.read(mag.IMAGE_IMPORT_MAX_BYTES + 1)
|
||||
images.append((f.filename or "image", data))
|
||||
try:
|
||||
pages = await asyncio.to_thread(mag.images_to_pdf, images, d / "source.pdf")
|
||||
except mag.ImageImportError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
return {"ok": True, "pages": pages}
|
||||
|
||||
|
||||
@router.patch("/api/magazine/{doc_id}")
|
||||
async def magazine_update_meta(doc_id: str, request: Request):
|
||||
"""Edit title/issue/publisher after creation -- there was previously no
|
||||
|
||||
@ -230,7 +230,7 @@ async function magDelete(docId) {
|
||||
}
|
||||
|
||||
// ── Add-magazine dropzone (mirrors reader.js's setupReaderDropzone) ─────────
|
||||
const MAG_DROPZONE_DEFAULT_NOTE = 'Scanned issues work too — pages without selectable text are read with OCR.';
|
||||
const MAG_DROPZONE_DEFAULT_NOTE = 'Scanned issues and phone photos work too — pages without selectable text are read with OCR. Several images are read in filename order.';
|
||||
|
||||
function magFormatFilenameAsTitle(filename) {
|
||||
if (!filename) return '';
|
||||
@ -241,6 +241,25 @@ function magFormatFilenameAsTitle(filename) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
// What was chosen or dropped: exactly one PDF, or several page images (PNG/JPEG) read in filename order.
|
||||
// Returns { kind, files, label: {name, size} } or { error }. The server re-checks content by its bytes.
|
||||
function magClassifyFiles(list) {
|
||||
const files = Array.from(list || []).filter(Boolean);
|
||||
if (!files.length) return { error: 'Choose or drop a PDF or page images first' };
|
||||
const isPdf = (f) => f.type === 'application/pdf' || /\.pdf$/i.test(f.name);
|
||||
const isImg = (f) => /^image\/(png|jpeg)$/.test(f.type) || /\.(png|jpe?g)$/i.test(f.name);
|
||||
if (files.every(isPdf)) {
|
||||
if (files.length > 1) return { error: 'Choose one PDF, or several page images — not several PDFs' };
|
||||
return { kind: 'pdf', files, label: files[0] };
|
||||
}
|
||||
if (files.every(isImg)) {
|
||||
files.sort((a, b) => a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' }));
|
||||
const size = files.reduce((n, f) => n + f.size, 0);
|
||||
return { kind: 'images', files, label: { name: files.length === 1 ? files[0].name : files.length + ' page images', size } };
|
||||
}
|
||||
return { error: 'Only a PDF, or PNG/JPEG page images, can be imported' };
|
||||
}
|
||||
|
||||
function magShowDroppedFile(file) {
|
||||
const noteEl = $('mag-dropzone-note');
|
||||
const titleEl = $('mag-dropzone-title');
|
||||
@ -252,7 +271,7 @@ function magShowDroppedFile(file) {
|
||||
const sizeMb = (file.size / (1024 * 1024)).toFixed(1);
|
||||
if (dropzone) dropzone.classList.add('is-selected');
|
||||
if (noteEl) noteEl.textContent = `Selected: ${file.name} (${sizeMb} MB)`;
|
||||
if (titleEl) titleEl.textContent = 'PDF file selected — uploading...';
|
||||
if (titleEl) titleEl.textContent = 'File selected — uploading...';
|
||||
if (iconEl) iconEl.className = 'mdi mdi-file-check-outline reader-dropzone-icon';
|
||||
|
||||
// Auto-fill title from filename if title input is empty
|
||||
@ -262,7 +281,7 @@ function magShowDroppedFile(file) {
|
||||
} else {
|
||||
if (dropzone) dropzone.classList.remove('is-selected', 'is-uploading');
|
||||
if (noteEl) noteEl.textContent = MAG_DROPZONE_DEFAULT_NOTE;
|
||||
if (titleEl) titleEl.textContent = 'Drop a PDF document';
|
||||
if (titleEl) titleEl.textContent = 'Drop a PDF or page images';
|
||||
if (iconEl) iconEl.className = 'mdi mdi-cloud-upload-outline reader-dropzone-icon';
|
||||
}
|
||||
}
|
||||
@ -330,30 +349,29 @@ function setupMagDropzone() {
|
||||
dragCounter = 0;
|
||||
zone.classList.remove('dragover');
|
||||
|
||||
let file = e.dataTransfer?.files?.[0];
|
||||
if (!file && e.dataTransfer?.items?.[0]) {
|
||||
file = e.dataTransfer.items[0].getAsFile();
|
||||
let dropped = Array.from(e.dataTransfer?.files || []);
|
||||
if (!dropped.length && e.dataTransfer?.items?.length) {
|
||||
dropped = Array.from(e.dataTransfer.items).map((i) => i.getAsFile()).filter(Boolean);
|
||||
}
|
||||
if (!file) return;
|
||||
if (file.type !== 'application/pdf' && !/\.pdf$/i.test(file.name)) {
|
||||
return say('Only PDF files are supported', 'err');
|
||||
}
|
||||
magShowDroppedFile(file);
|
||||
magCreate(file);
|
||||
const picked = magClassifyFiles(dropped);
|
||||
if (picked.error) return say(picked.error, 'err');
|
||||
magShowDroppedFile(picked.label);
|
||||
magCreate(picked.files);
|
||||
});
|
||||
|
||||
input.addEventListener('change', () => {
|
||||
const file = input.files?.[0];
|
||||
if (file) {
|
||||
magShowDroppedFile(file);
|
||||
magCreate(file);
|
||||
}
|
||||
if (!input.files?.length) return;
|
||||
const picked = magClassifyFiles(input.files);
|
||||
if (picked.error) return say(picked.error, 'err');
|
||||
magShowDroppedFile(picked.label);
|
||||
magCreate(picked.files);
|
||||
});
|
||||
}
|
||||
|
||||
async function magCreate(droppedFile = null) {
|
||||
const file = droppedFile || $('mag-new-file')?.files?.[0];
|
||||
if (!file) return say('Choose or drop a PDF file first', 'err');
|
||||
async function magCreate(dropped = null) {
|
||||
const picked = magClassifyFiles(dropped ? (Array.isArray(dropped) ? dropped : [dropped]) : $('mag-new-file')?.files);
|
||||
if (picked.error) return say(picked.error, 'err');
|
||||
const file = picked.label;
|
||||
|
||||
let title = ($('mag-new-title')?.value || '').trim();
|
||||
if (!title) {
|
||||
@ -418,7 +436,13 @@ async function magCreate(droppedFile = null) {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ title, issue }),
|
||||
});
|
||||
await api('/api/reader/docs/' + id + '/source?ext=pdf', { method: 'PUT', body: file });
|
||||
if (picked.kind === 'pdf') {
|
||||
await api('/api/reader/docs/' + id + '/source?ext=pdf', { method: 'PUT', body: picked.files[0] });
|
||||
} else {
|
||||
const form = new FormData();
|
||||
picked.files.forEach((f) => form.append('files', f, f.name));
|
||||
await api('/api/magazine/' + id + '/source-images', { method: 'POST', body: form });
|
||||
}
|
||||
say('Magazine uploaded — starting ingestion', 'ok');
|
||||
|
||||
if ($('mag-new-title')) $('mag-new-title').value = '';
|
||||
|
||||
@ -25,22 +25,22 @@
|
||||
</label>
|
||||
</div>
|
||||
<div class="reader-paste-actions" style="margin-top: auto; padding-top: 6px;">
|
||||
<label class="btn-primary btn-sm reader-file-pick" id="mag-browse-btn" title="Choose a PDF file from your computer">
|
||||
<label class="btn-primary btn-sm reader-file-pick" id="mag-browse-btn" title="Choose a PDF file or page images (PNG/JPEG) from your computer">
|
||||
<span class="mdi mdi-folder-open-outline"></span> Durchsuchen
|
||||
<input type="file" id="mag-new-file" accept="application/pdf" style="display:none">
|
||||
<input type="file" id="mag-new-file" accept="application/pdf,image/png,image/jpeg" multiple style="display:none">
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="reader-dropzone" id="mag-dropzone" style="cursor: pointer;">
|
||||
<span class="mdi mdi-cloud-upload-outline reader-dropzone-icon" id="mag-dropzone-icon"></span>
|
||||
<strong id="mag-dropzone-title">Drop a PDF document</strong>
|
||||
<strong id="mag-dropzone-title">Drop a PDF or page images</strong>
|
||||
<span class="note" id="mag-dropzone-note">Scanned issues work too — pages without selectable text are read with OCR.</span>
|
||||
<div class="mag-dropzone-progress" id="mag-dropzone-progress" hidden>
|
||||
<div class="mag-dropzone-progress-bar"></div>
|
||||
</div>
|
||||
<label class="btn-secondary btn-sm reader-file-pick" id="mag-dropzone-pick-btn" title="Choose a PDF file">
|
||||
<span class="mdi mdi-file-upload-outline"></span> Import PDF
|
||||
<label class="btn-secondary btn-sm reader-file-pick" id="mag-dropzone-pick-btn" title="Choose a PDF file or page images (PNG/JPEG)">
|
||||
<span class="mdi mdi-file-upload-outline"></span> Import PDF or images
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -1400,3 +1400,46 @@ test('finding an issue online: search, licence shown, import starts ingestion',
|
||||
await expect.poll(() => imported.length).toBe(1);
|
||||
expect(imported[0]).toMatchObject({ source: 'internet-archive', item_id: 'sci-am-1923' });
|
||||
});
|
||||
|
||||
const TINY_PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', 'base64');
|
||||
|
||||
async function openMagazineLibrary(page) {
|
||||
await mockBackend(page);
|
||||
await page.goto('/', { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForSelector('#app-skeleton', { state: 'detached' });
|
||||
await navTo(page, 's-magazine');
|
||||
await expect(page.locator('#mag-new-file')).toBeAttached();
|
||||
}
|
||||
|
||||
test('several page images are uploaded together, in filename order', async ({ page }) => {
|
||||
await openMagazineLibrary(page);
|
||||
const posted = [];
|
||||
await page.route('**/api/magazine/docs', route => route.fulfill({ json: { id: DOC_ID } }));
|
||||
await page.route(`**/api/magazine/${DOC_ID}/source-images`, route => {
|
||||
posted.push(route.request().postDataBuffer().toString('latin1'));
|
||||
route.fulfill({ json: { ok: true, pages: 3 } });
|
||||
});
|
||||
await page.route(`**/api/magazine/${DOC_ID}/ingest`, route => route.fulfill({ json: { job_id: 'ing1' } }));
|
||||
await page.locator('#mag-new-file').setInputFiles([
|
||||
{ name: 'page10.png', mimeType: 'image/png', buffer: TINY_PNG },
|
||||
{ name: 'page2.png', mimeType: 'image/png', buffer: TINY_PNG },
|
||||
{ name: 'page1.png', mimeType: 'image/png', buffer: TINY_PNG },
|
||||
]);
|
||||
await expect.poll(() => posted.length).toBe(1);
|
||||
const names = [...posted[0].matchAll(/filename="([^"]+)"/g)].map(m => m[1]);
|
||||
expect(names).toEqual(['page1.png', 'page2.png', 'page10.png']);
|
||||
});
|
||||
|
||||
test('mixing a PDF with images, or choosing another type, is refused before anything is created', async ({ page }) => {
|
||||
await openMagazineLibrary(page);
|
||||
let created = 0;
|
||||
await page.route('**/api/magazine/docs', route => { created++; route.fulfill({ json: { id: DOC_ID } }); });
|
||||
await page.locator('#mag-new-file').setInputFiles([
|
||||
{ name: 'a.pdf', mimeType: 'application/pdf', buffer: Buffer.from('%PDF-1.4') },
|
||||
{ name: 'b.png', mimeType: 'image/png', buffer: TINY_PNG },
|
||||
]);
|
||||
await expect(page.locator('.toast, #toast, [role="status"]').filter({ hasText: /PDF|images/ }).first()).toBeVisible();
|
||||
await page.locator('#mag-new-file').setInputFiles([{ name: 'c.gif', mimeType: 'image/gif', buffer: Buffer.from('GIF89a') }]);
|
||||
await page.waitForTimeout(300);
|
||||
expect(created).toBe(0);
|
||||
});
|
||||
|
||||
139
tests/test_magazine_image_import.py
Normal file
139
tests/test_magazine_image_import.py
Normal file
@ -0,0 +1,139 @@
|
||||
"""Importing a magazine from page images: conversion to a scan-style PDF and its refusals."""
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-test-config-"))
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from core import magazine as mag
|
||||
|
||||
|
||||
def png(w=300, h=400, dpi=None, color=(250, 250, 250)):
|
||||
buf = io.BytesIO()
|
||||
kw = {"dpi": (dpi, dpi)} if dpi else {}
|
||||
Image.new("RGB", (w, h), color).save(buf, format="PNG", **kw)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def jpeg(w=300, h=400, orientation=None):
|
||||
buf = io.BytesIO()
|
||||
im = Image.new("RGB", (w, h), (200, 200, 200))
|
||||
exif = Image.Exif()
|
||||
if orientation:
|
||||
exif[0x0112] = orientation
|
||||
im.save(buf, format="JPEG", exif=exif)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class ImagesToPdfTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = Path(tempfile.mkdtemp(prefix="ttvc-img-"))
|
||||
self.dest = self.dir / "source.pdf"
|
||||
|
||||
def pages(self):
|
||||
import pymupdf
|
||||
with pymupdf.open(str(self.dest)) as d:
|
||||
return [(round(p.rect.width), round(p.rect.height), len(p.get_text().strip())) for p in d]
|
||||
|
||||
def test_pages_in_order_sized_from_resolution_with_no_text_layer(self):
|
||||
n = mag.images_to_pdf([("a.png", png(600, 900, dpi=300)), ("b.jpg", jpeg(150, 300))], self.dest)
|
||||
self.assertEqual(n, 2)
|
||||
# 600x900 px at 300 dpi = 144x216 pt; the jpeg has no dpi -> assumed 150 dpi = 72x144 pt
|
||||
self.assertEqual(self.pages(), [(144, 216, 0), (72, 144, 0)])
|
||||
|
||||
def test_exif_rotation_is_applied(self):
|
||||
mag.images_to_pdf([("r.jpg", jpeg(300, 200, orientation=6))], self.dest) # rotated 90°: becomes portrait
|
||||
w, h, _ = self.pages()[0]
|
||||
self.assertLess(w, h)
|
||||
|
||||
def test_unrotated_jpeg_is_not_reencoded(self):
|
||||
data = jpeg(300, 200)
|
||||
mag.images_to_pdf([("r.jpg", data)], self.dest)
|
||||
import pymupdf
|
||||
with pymupdf.open(str(self.dest)) as d:
|
||||
extracted = d.extract_image(d[0].get_images()[0][0])["image"]
|
||||
self.assertEqual(extracted, data)
|
||||
|
||||
def test_rejects_a_renamed_non_image_by_content(self):
|
||||
with self.assertRaisesRegex(mag.ImageImportError, "not a PNG or JPEG"):
|
||||
mag.images_to_pdf([("page.png", b"<html>not an image</html>")], self.dest)
|
||||
self.assertFalse(self.dest.exists())
|
||||
|
||||
def test_rejects_a_truncated_image(self):
|
||||
with self.assertRaisesRegex(mag.ImageImportError, "could not be read"):
|
||||
mag.images_to_pdf([("t.png", png()[:40])], self.dest)
|
||||
|
||||
def test_rejects_too_many_pixels_before_decoding(self):
|
||||
old = mag.IMAGE_IMPORT_MAX_PIXELS
|
||||
mag.IMAGE_IMPORT_MAX_PIXELS = 1000
|
||||
try:
|
||||
with self.assertRaisesRegex(mag.ImageImportError, "too many pixels"):
|
||||
mag.images_to_pdf([("big.png", png(300, 400))], self.dest)
|
||||
finally:
|
||||
mag.IMAGE_IMPORT_MAX_PIXELS = old
|
||||
|
||||
def test_rejects_oversize_file_and_empty_and_too_many(self):
|
||||
with self.assertRaisesRegex(mag.ImageImportError, "larger than"):
|
||||
mag.images_to_pdf([("x.png", b"\x89PNG\r\n\x1a\n" + b"0" * (mag.IMAGE_IMPORT_MAX_BYTES + 1))], self.dest)
|
||||
with self.assertRaises(mag.ImageImportError):
|
||||
mag.images_to_pdf([], self.dest)
|
||||
with self.assertRaisesRegex(mag.ImageImportError, "At most"):
|
||||
mag.images_to_pdf([("a.png", png(10, 10))] * (mag.IMAGE_IMPORT_MAX_FILES + 1), self.dest)
|
||||
|
||||
def test_one_bad_image_leaves_no_partial_pdf(self):
|
||||
with self.assertRaises(mag.ImageImportError):
|
||||
mag.images_to_pdf([("a.png", png()), ("b.png", b"nope")], self.dest)
|
||||
self.assertEqual(list(self.dir.iterdir()), [])
|
||||
|
||||
def test_name_is_only_used_as_a_short_label(self):
|
||||
with self.assertRaises(mag.ImageImportError) as ctx:
|
||||
mag.images_to_pdf([("../../etc/passwd" + "x" * 200, b"nope")], self.dest)
|
||||
self.assertNotIn("..", str(ctx.exception))
|
||||
|
||||
|
||||
class EndpointTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from unittest import mock
|
||||
import routes.magazine as routes
|
||||
self.routes = routes
|
||||
self.lib = Path(tempfile.mkdtemp(prefix="ttvc-lib-"))
|
||||
self.patch = mock.patch.object(routes, "_LIB", self.lib)
|
||||
self.patch.start()
|
||||
self.addCleanup(self.patch.stop)
|
||||
app = FastAPI()
|
||||
app.include_router(routes.router)
|
||||
self.client = TestClient(app)
|
||||
self.doc = self.client.post("/api/magazine/docs", json={"title": "T"}).json()["id"]
|
||||
|
||||
def test_upload_builds_the_source_pdf(self):
|
||||
r = self.client.post(f"/api/magazine/{self.doc}/source-images",
|
||||
files=[("files", ("1.png", png(), "image/png")), ("files", ("2.jpg", jpeg(), "image/jpeg"))])
|
||||
self.assertEqual(r.status_code, 200, r.text)
|
||||
self.assertEqual(r.json()["pages"], 2)
|
||||
self.assertTrue((self.lib / self.doc / "source.pdf").is_file())
|
||||
|
||||
def test_a_second_upload_cannot_replace_existing_source(self):
|
||||
files = [("files", ("1.png", png(), "image/png"))]
|
||||
self.assertEqual(self.client.post(f"/api/magazine/{self.doc}/source-images", files=files).status_code, 200)
|
||||
self.assertEqual(self.client.post(f"/api/magazine/{self.doc}/source-images", files=files).status_code, 409)
|
||||
|
||||
def test_bad_content_is_a_400_with_a_message(self):
|
||||
r = self.client.post(f"/api/magazine/{self.doc}/source-images",
|
||||
files=[("files", ("1.png", b"GIF89a....", "image/png"))])
|
||||
self.assertEqual(r.status_code, 400)
|
||||
self.assertIn("not a PNG or JPEG", r.json()["detail"])
|
||||
self.assertFalse((self.lib / self.doc / "source.pdf").exists())
|
||||
|
||||
def test_unknown_document_is_rejected(self):
|
||||
r = self.client.post("/api/magazine/nope/source-images", files=[("files", ("1.png", png(), "image/png"))])
|
||||
self.assertIn(r.status_code, (400, 404))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue
Block a user