"""Find a magazine issue in a legitimate source and import it as a new magazine project. Search and download go through core.magazine_sources (Internet Archive + configured OPDS catalogues). An import only creates the project and stores source.pdf; ingestion stays the existing, separate step so the user still reviews before anything is narrated. """ from __future__ import annotations import asyncio import logging import re import time import uuid from fastapi import APIRouter, HTTPException, Request from core import magazine_sources as ms from core.config import _load_settings from core.constants import CONFIG_DIR from core.database import background_job_create, background_job_update from routes.magazine import _write_meta router = APIRouter() _LIB = CONFIG_DIR / "reader_library" _import_lock = asyncio.Lock() # one download at a time: polite to the source and to the disk _KEY = re.compile(r"[a-z0-9-]{1,40}") def _providers() -> dict: return ms.build_providers(_load_settings().get("magazine_opds_catalogs") or []) def _provider(key: str): provider = _providers().get(key if _KEY.fullmatch(key or "") else "") if provider is None: raise HTTPException(404, "Unknown source") return provider @router.get("/api/magazine-sources") async def magazine_sources_list(): return {"sources": [{"key": p.key, "label": p.label, "kind": "opds" if p.key.startswith("opds-") else "archive"} for p in _providers().values()]} @router.get("/api/magazine-sources/{key}/search") async def magazine_sources_search(key: str, q: str = "", page: int = 1): provider = _provider(key) try: items = await asyncio.to_thread(provider.search, q, min(max(page, 1), 50)) except ms.SourceError as e: raise HTTPException(502, str(e)) return {"source": key, "items": [i.as_dict() for i in items]} def _download_sync(doc_id: str, job_id: str, key: str, item_id: str) -> None: dest = _LIB / doc_id / "source.pdf" try: provider = _provider(key) def progress(done: int, total: int) -> None: mb = done / 1e6 detail = f"Downloaded {mb:.0f} of {total / 1e6:.0f} MB" if total else f"Downloaded {mb:.0f} MB" background_job_update(job_id, stage="download", detail=detail) background_job_update(job_id, stage="download", detail="Contacting the source") size = ms.download_pdf(provider, item_id, dest, progress=progress) background_job_update(job_id, stage="done", detail="Downloaded", done=True, result={"doc_id": doc_id, "bytes": size}) except ms.SourceError as e: background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e)[:300]) except HTTPException as e: background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e.detail)[:300]) except Exception: logging.exception("Magazine source download failed") background_job_update(job_id, stage="error", detail="Download failed", done=True, error="The download failed unexpectedly") @router.post("/api/magazine-sources/import") async def magazine_sources_import(request: Request): body = await request.json() key = str(body.get("source") or "") item_id = str(body.get("item_id") or "") provider = _provider(key) if not item_id or len(item_id) > 2000: raise HTTPException(400, "item_id is required") if _import_lock.locked(): raise HTTPException(409, "Another import is still downloading") title = re.sub(r"\s+", " ", str(body.get("title") or "")).strip()[:300] or "Untitled magazine" doc_id = uuid.uuid4().hex d = _LIB / doc_id d.mkdir(parents=True, exist_ok=True) now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) _write_meta(d, { "kind": "magazine", "title": title, "issue": str(body.get("issue") or "").strip()[:120], "publisher": "", "created": now, "updated": now, "ingest": {"status": "empty"}, "default_voice": "", "podcast": {"hosts": []}, "progress": {}, "source": {"provider": provider.label, "item": item_id[:500]}, }) job_id = uuid.uuid4().hex background_job_create(job_id, "magazine-source-import", fingerprint=doc_id) async def run(): async with _import_lock: await asyncio.to_thread(_download_sync, doc_id, job_id, key, item_id) asyncio.create_task(run()) return {"id": doc_id, "job_id": job_id}