mARTin-B78
c3c5c55f17
Also reuses the existing magFmtClock for the contents list (the duplicate tripped the frontend collision gate). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
111 lines
4.4 KiB
Python
111 lines
4.4 KiB
Python
"""Find a magazine issue in a legitimate source and import it as a new magazine project.
|
|
|
|
Search and download go through core.magazine_sources (Internet Archive + configured OPDS catalogues).
|
|
An import only creates the project and stores source.pdf; ingestion stays the existing, separate step so
|
|
the user still reviews before anything is narrated.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import re
|
|
import time
|
|
import uuid
|
|
|
|
from fastapi import APIRouter, HTTPException, Request
|
|
|
|
from core import magazine_sources as ms
|
|
from core.config import _load_settings
|
|
from core.constants import CONFIG_DIR
|
|
from core.database import background_job_create, background_job_update
|
|
from routes.magazine import _write_meta
|
|
|
|
router = APIRouter()
|
|
|
|
_LIB = CONFIG_DIR / "reader_library"
|
|
_import_lock = asyncio.Lock() # one download at a time: polite to the source and to the disk
|
|
_KEY = re.compile(r"[a-z0-9-]{1,40}")
|
|
|
|
|
|
def _providers() -> dict:
|
|
return ms.build_providers(_load_settings().get("magazine_opds_catalogs") or [])
|
|
|
|
|
|
def _provider(key: str):
|
|
provider = _providers().get(key if _KEY.fullmatch(key or "") else "")
|
|
if provider is None:
|
|
raise HTTPException(404, "Unknown source")
|
|
return provider
|
|
|
|
|
|
@router.get("/api/magazine-sources")
|
|
async def magazine_sources_list():
|
|
return {"sources": [{"key": p.key, "label": p.label, "kind": "opds" if p.key.startswith("opds-") else "archive"}
|
|
for p in _providers().values()]}
|
|
|
|
|
|
@router.get("/api/magazine-sources/{key}/search")
|
|
async def magazine_sources_search(key: str, q: str = "", page: int = 1):
|
|
provider = _provider(key)
|
|
try:
|
|
items = await asyncio.to_thread(provider.search, q, min(max(page, 1), 50))
|
|
except ms.SourceError as e:
|
|
raise HTTPException(502, str(e))
|
|
return {"source": key, "items": [i.as_dict() for i in items]}
|
|
|
|
|
|
def _download_sync(doc_id: str, job_id: str, key: str, item_id: str) -> None:
|
|
dest = _LIB / doc_id / "source.pdf"
|
|
try:
|
|
provider = _provider(key)
|
|
|
|
def progress(done: int, total: int) -> None:
|
|
mb = done / 1e6
|
|
detail = f"Downloaded {mb:.0f} of {total / 1e6:.0f} MB" if total else f"Downloaded {mb:.0f} MB"
|
|
background_job_update(job_id, stage="download", detail=detail)
|
|
|
|
background_job_update(job_id, stage="download", detail="Contacting the source")
|
|
size = ms.download_pdf(provider, item_id, dest, progress=progress)
|
|
background_job_update(job_id, stage="done", detail="Downloaded", done=True,
|
|
result={"doc_id": doc_id, "bytes": size})
|
|
except ms.SourceError as e:
|
|
background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e)[:300])
|
|
except HTTPException as e:
|
|
background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e.detail)[:300])
|
|
except Exception:
|
|
logging.exception("Magazine source download failed")
|
|
background_job_update(job_id, stage="error", detail="Download failed", done=True,
|
|
error="The download failed unexpectedly")
|
|
|
|
|
|
@router.post("/api/magazine-sources/import")
|
|
async def magazine_sources_import(request: Request):
|
|
body = await request.json()
|
|
key = str(body.get("source") or "")
|
|
item_id = str(body.get("item_id") or "")
|
|
provider = _provider(key)
|
|
if not item_id or len(item_id) > 2000:
|
|
raise HTTPException(400, "item_id is required")
|
|
if _import_lock.locked():
|
|
raise HTTPException(409, "Another import is still downloading")
|
|
title = re.sub(r"\s+", " ", str(body.get("title") or "")).strip()[:300] or "Untitled magazine"
|
|
doc_id = uuid.uuid4().hex
|
|
d = _LIB / doc_id
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
_write_meta(d, {
|
|
"kind": "magazine", "title": title, "issue": str(body.get("issue") or "").strip()[:120],
|
|
"publisher": "", "created": now, "updated": now, "ingest": {"status": "empty"},
|
|
"default_voice": "", "podcast": {"hosts": []}, "progress": {},
|
|
"source": {"provider": provider.label, "item": item_id[:500]},
|
|
})
|
|
job_id = uuid.uuid4().hex
|
|
background_job_create(job_id, "magazine-source-import", fingerprint=doc_id)
|
|
|
|
async def run():
|
|
async with _import_lock:
|
|
await asyncio.to_thread(_download_sync, doc_id, job_id, key, item_id)
|
|
|
|
asyncio.create_task(run())
|
|
return {"id": doc_id, "job_id": job_id}
|