168 lines
7.1 KiB
Python
168 lines
7.1 KiB
Python
"""Find a magazine issue in a legitimate source and import it as a new magazine project.
|
|
|
|
Search and download go through core.magazine_sources (Internet Archive + configured OPDS catalogues).
|
|
An import only creates the project and stores source.pdf; ingestion stays the existing, separate step so
|
|
the user still reviews before anything is narrated.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import re
|
|
import time
|
|
import uuid
|
|
|
|
from fastapi import APIRouter, HTTPException, Request
|
|
|
|
from core import magazine_sources as ms
|
|
from core.tasks import spawn
|
|
from core.config import _load_settings, _save_settings
|
|
from core.constants import CONFIG_DIR
|
|
from core.database import background_job_create, background_job_update
|
|
from routes.magazine import _write_meta
|
|
|
|
router = APIRouter()
|
|
|
|
_LIB = CONFIG_DIR / "reader_library"
|
|
_import_lock = asyncio.Lock() # one download at a time: polite to the source and to the disk
|
|
_KEY = re.compile(r"[a-z0-9-]{1,40}")
|
|
|
|
|
|
def _providers() -> dict:
|
|
return ms.build_providers(_load_settings().get("magazine_opds_catalogs") or [])
|
|
|
|
|
|
def _provider(key: str):
|
|
provider = _providers().get(key if _KEY.fullmatch(key or "") else "")
|
|
if provider is None:
|
|
raise HTTPException(404, "Unknown source")
|
|
return provider
|
|
|
|
|
|
@router.get("/api/magazine-sources")
|
|
async def magazine_sources_list():
|
|
return {"sources": [{"key": p.key, "label": p.label, "kind": "opds" if p.key.startswith("opds-") else "archive"}
|
|
for p in _providers().values()]}
|
|
|
|
|
|
_ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]{0,63}")
|
|
MAX_CATALOGUES = 10
|
|
|
|
|
|
def _clean_catalogues(raw) -> list[dict]:
|
|
"""Validate a catalogue list from the browser. Anything doubtful is refused, never silently repaired."""
|
|
from urllib.parse import urlsplit
|
|
if not isinstance(raw, list) or len(raw) > MAX_CATALOGUES:
|
|
raise HTTPException(400, f"Send a list of at most {MAX_CATALOGUES} catalogues")
|
|
out, seen = [], set()
|
|
for item in raw:
|
|
if not isinstance(item, dict):
|
|
raise HTTPException(400, "Each catalogue must be an object")
|
|
name = re.sub(r"\s+", " ", str(item.get("name") or "")).strip()[:80]
|
|
url = str(item.get("url") or "").strip()
|
|
parts = urlsplit(url)
|
|
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
raise HTTPException(400, f"\u201c{name or url[:40]}\u201d: the address must start with http:// or https://")
|
|
if parts.username or parts.password:
|
|
raise HTTPException(400, "Put the login in the username field, not in the address")
|
|
if len(url) > 500:
|
|
raise HTTPException(400, "The address is too long")
|
|
username = str(item.get("username") or "").strip()[:120]
|
|
env = str(item.get("password_env") or "").strip()
|
|
if env and not _ENV_NAME.fullmatch(env):
|
|
raise HTTPException(400, "The password variable name may only use capital letters, digits and underscores")
|
|
if bool(username) != bool(env):
|
|
raise HTTPException(400, "A login needs both a username and the name of the password variable")
|
|
if url in seen:
|
|
raise HTTPException(400, "That catalogue is already listed")
|
|
seen.add(url)
|
|
out.append({"name": name or parts.hostname, "url": url, "username": username, "password_env": env})
|
|
return out
|
|
|
|
|
|
@router.get("/api/magazine-sources/catalogues")
|
|
async def magazine_catalogues_get():
|
|
import os
|
|
cats = _load_settings().get("magazine_opds_catalogs") or []
|
|
return {"catalogues": [
|
|
{"name": str(c.get("name") or ""), "url": str(c.get("url") or ""), "username": str(c.get("username") or ""),
|
|
"password_env": str(c.get("password_env") or ""),
|
|
"password_set": bool(c.get("password_env") and os.environ.get(str(c.get("password_env")))) }
|
|
for c in cats if isinstance(c, dict)]}
|
|
|
|
|
|
@router.put("/api/magazine-sources/catalogues")
|
|
async def magazine_catalogues_put(request: Request):
|
|
body = await request.json()
|
|
cats = _clean_catalogues(body.get("catalogues") if isinstance(body, dict) else None)
|
|
settings = _load_settings()
|
|
settings["magazine_opds_catalogs"] = cats
|
|
_save_settings(settings)
|
|
return {"ok": True, "count": len(cats)}
|
|
|
|
|
|
@router.get("/api/magazine-sources/{key}/search")
|
|
async def magazine_sources_search(key: str, q: str = "", page: int = 1):
|
|
provider = _provider(key)
|
|
try:
|
|
items = await asyncio.to_thread(provider.search, q, min(max(page, 1), 50))
|
|
except ms.SourceError as e:
|
|
raise HTTPException(502, str(e))
|
|
return {"source": key, "items": [i.as_dict() for i in items]}
|
|
|
|
|
|
def _download_sync(doc_id: str, job_id: str, key: str, item_id: str) -> None:
|
|
dest = _LIB / doc_id / "source.pdf"
|
|
try:
|
|
provider = _provider(key)
|
|
|
|
def progress(done: int, total: int) -> None:
|
|
mb = done / 1e6
|
|
detail = f"Downloaded {mb:.0f} of {total / 1e6:.0f} MB" if total else f"Downloaded {mb:.0f} MB"
|
|
background_job_update(job_id, stage="download", detail=detail)
|
|
|
|
background_job_update(job_id, stage="download", detail="Contacting the source")
|
|
size = ms.download_pdf(provider, item_id, dest, progress=progress)
|
|
background_job_update(job_id, stage="done", detail="Downloaded", done=True,
|
|
result={"doc_id": doc_id, "bytes": size})
|
|
except ms.SourceError as e:
|
|
background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e)[:300])
|
|
except HTTPException as e:
|
|
background_job_update(job_id, stage="error", detail="Download failed", done=True, error=str(e.detail)[:300])
|
|
except Exception:
|
|
logging.exception("Magazine source download failed")
|
|
background_job_update(job_id, stage="error", detail="Download failed", done=True,
|
|
error="The download failed unexpectedly")
|
|
|
|
|
|
@router.post("/api/magazine-sources/import")
|
|
async def magazine_sources_import(request: Request):
|
|
body = await request.json()
|
|
key = str(body.get("source") or "")
|
|
item_id = str(body.get("item_id") or "")
|
|
provider = _provider(key)
|
|
if not item_id or len(item_id) > 2000:
|
|
raise HTTPException(400, "item_id is required")
|
|
if _import_lock.locked():
|
|
raise HTTPException(409, "Another import is still downloading")
|
|
title = re.sub(r"\s+", " ", str(body.get("title") or "")).strip()[:300] or "Untitled magazine"
|
|
doc_id = uuid.uuid4().hex
|
|
d = _LIB / doc_id
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
_write_meta(d, {
|
|
"kind": "magazine", "title": title, "issue": str(body.get("issue") or "").strip()[:120],
|
|
"publisher": "", "created": now, "updated": now, "ingest": {"status": "empty"},
|
|
"default_voice": "", "podcast": {"hosts": []}, "progress": {},
|
|
"source": {"provider": provider.label, "item": item_id[:500]},
|
|
})
|
|
job_id = uuid.uuid4().hex
|
|
background_job_create(job_id, "magazine-source-import", fingerprint=doc_id)
|
|
|
|
async def run():
|
|
async with _import_lock:
|
|
await asyncio.to_thread(_download_sync, doc_id, job_id, key, item_id)
|
|
|
|
spawn(run())
|
|
return {"id": doc_id, "job_id": job_id}
|