"""Find a magazine issue in a legitimate source and import it as a new magazine project. Search and download go through core.magazine_sources (Internet Archive + configured OPDS catalogues). An import only creates the project and stores source.pdf; ingestion stays the existing, separate step so the user still reviews before anything is narrated. """ from __future__ import annotations import asyncio import logging import re import time import uuid from fastapi import APIRouter, HTTPException, Request from core import magazine_sources as ms from core.config import _load_settings, _save_settings from core.constants import CONFIG_DIR from core.database import background_job_create, background_job_update from routes.magazine import _write_meta router = APIRouter() _LIB = CONFIG_DIR / "reader_library" _import_lock = asyncio.Lock() # one download at a time: polite to the source and to the disk _KEY = re.compile(r"[a-z0-9-]{1,40}") def _providers() -> dict: return ms.build_providers(_load_settings().get("magazine_opds_catalogs") or []) def _provider(key: str): provider = _providers().get(key if _KEY.fullmatch(key or "") else "") if provider is None: raise HTTPException(404, "Unknown source") return provider @router.get("/api/magazine-sources") async def magazine_sources_list(): return {"sources": [{"key": p.key, "label": p.label, "kind": "opds" if p.key.startswith("opds-") else "archive"} for p in _providers().values()]} _ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]{0,63}") MAX_CATALOGUES = 10 def _clean_catalogues(raw) -> list[dict]: """Validate a catalogue list from the browser. Anything doubtful is refused, never silently repaired.""" from urllib.parse import urlsplit if not isinstance(raw, list) or len(raw) > MAX_CATALOGUES: raise HTTPException(400, f"Send a list of at most {MAX_CATALOGUES} catalogues") out, seen = [], set() for item in raw: if not isinstance(item, dict): raise HTTPException(400, "Each catalogue must be an object") name = re.sub(r"\s+", " ", str(item.get("name") or "")).strip()[:80] url = str(item.get("url") or "").strip() parts = urlsplit(url) if parts.scheme not in ("http", "https") or not parts.hostname: raise HTTPException(400, f"\u201c{name or url[:40]}\u201d: the address must start with http:// or https://") if parts.username or parts.password: raise HTTPException(400, "Put the login in the username field, not in the address") if len(url) > 500: raise HTTPException(400, "The address is too long") username = str(item.get("username") or "").strip()[:120] env = str(item.get("password_env") or "").strip() if env and not _ENV_NAME.fullmatch(env): raise HTTPException(400, "The password variable name may only use capital letters, digits and underscores") if bool(username) != bool(env): raise HTTPException(400, "A login needs both a username and the name of the password variable") if url in seen: raise HTTPException(400, "That catalogue is already listed") seen.add(url) out.append({"name": name or parts.hostname, "url": url, "username": username, "password_env": env}) return out @router.get("/api/magazine-sources/catalogues") async def magazine_catalogues_get(): import os cats = _load_settings().get("magazine_opds_catalogs") or [] return {"catalogues": [ {"name": str(c.get("name") or ""), "url": str(c.get("url") or ""), "username": str(c.get("username") or ""), "password_env": str(c.get("password_env") or ""), "password_set": bool(c.get("password_env") and os.environ.get(str(c.get("password_env")))) } for c in cats if isinstance(c, dict)]} @router.put("/api/magazine-sources/catalogues") async def magazine_catalogues_put(request: Request): body = await request.json() cats = _clean_catalogues(body.get("catalogues") if isinstance(body, dict) else None) settings = _load_settings() settings["magazine_opds_catalogs"] = cats _save_settings(settings) return {"ok": True, "count": len(cats)} @router.get("/api/magazine-sources/{key}/search") async def magazine_sources_search(key: str, q: str = "", page: int = 1): provider = _provider(key) try: items = await asyncio.to_thread(provider.search, q, min(max(page, 1), 50)) except ms.SourceError as e: raise HTTPException(502, str(e)) return {"source": key, "items": [i.as_dict() for i in items]} def _download_sync(doc_id: str, job_id: str, key: str, item_id: str) -> None: dest = _LIB / doc_id / "source.pdf" try: provider = _provider(key) def progress(done: int, total: int) -> None: mb = done / 1e6 detail = f"Downloaded {mb:.0f} of {total / 1e6:.0f} MB" if total else f"Downloaded {mb:.0f} MB" background_job_update(job_id, stage="download", detail=detail) background_job_update(job_id, stage="download", detail="Contacting the source") size = ms.download_pdf(provider, item_id, dest, progress=progress) background_job_update(job_id, stage="done", detail="Downloaded", done=True, result={"doc_id": doc_id, "bytes": size}) except ms.SourceError as e: background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e)[:300]) except HTTPException as e: background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e.detail)[:300]) except Exception: logging.exception("Magazine source download failed") background_job_update(job_id, stage="error", detail="Download failed", done=True, error="The download failed unexpectedly") @router.post("/api/magazine-sources/import") async def magazine_sources_import(request: Request): body = await request.json() key = str(body.get("source") or "") item_id = str(body.get("item_id") or "") provider = _provider(key) if not item_id or len(item_id) > 2000: raise HTTPException(400, "item_id is required") if _import_lock.locked(): raise HTTPException(409, "Another import is still downloading") title = re.sub(r"\s+", " ", str(body.get("title") or "")).strip()[:300] or "Untitled magazine" doc_id = uuid.uuid4().hex d = _LIB / doc_id d.mkdir(parents=True, exist_ok=True) now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) _write_meta(d, { "kind": "magazine", "title": title, "issue": str(body.get("issue") or "").strip()[:120], "publisher": "", "created": now, "updated": now, "ingest": {"status": "empty"}, "default_voice": "", "podcast": {"hosts": []}, "progress": {}, "source": {"provider": provider.label, "item": item_id[:500]}, }) job_id = uuid.uuid4().hex background_job_create(job_id, "magazine-source-import", fingerprint=doc_id) async def run(): async with _import_lock: await asyncio.to_thread(_download_sync, doc_id, job_id, key, item_id) asyncio.create_task(run()) return {"id": doc_id, "job_id": job_id}