tts-voice-creator-clone-and-design-2/routes/magazine.py

"""Magazine projects: layout-aware ingestion, article review, podcast scripts.

Magazines live in the same store as Read Aloud documents
(<CONFIG_DIR>/reader_library/<id>/) and are distinguished by meta.json's
``kind: "magazine"``, so the existing library listing, cover, progress and
audio endpoints keep working unchanged.  Alongside those, a magazine adds:

    layout.json        every classified block on every page, with bboxes
    articles.json      detected articles; nothing is narrated until approved
    pages/NNNN.png     rendered pages for the PDF view and OCR
    images/*.png       extracted figures
    podcast/<id>.json  generated two/three-host scripts

Ingestion is a restart-safe background job because an 84-page issue takes
minutes, and the browser must not have to stay open for it.
"""
from __future__ import annotations

import asyncio
import json
import re
import time
import uuid
from pathlib import Path
from typing import Any

from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import FileResponse

from core import magazine as mag
from core.config import _load_settings
from core.constants import CONFIG_DIR
from core.database import (
    background_job_create,
    background_job_get,
    background_job_update,
)

router = APIRouter()

_LIB = CONFIG_DIR / "reader_library"
_MAX_HOSTS = 3
_MIN_HOSTS = 2


# ── Path safety ──────────────────────────────────────────────────────────────

def _doc_dir(doc_id: str) -> Path:
    """Resolve a document directory, rejecting anything but a generated id."""
    if not doc_id or not doc_id.isalnum() or len(doc_id) > 40:
        raise HTTPException(400, "Bad document id")
    d = _LIB / doc_id
    if not d.is_dir():
        raise HTTPException(404, "Document not found")
    return d


def _article_id(article_id: str) -> str:
    if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", article_id or ""):
        raise HTTPException(400, "Bad article id")
    return article_id


def _read_meta(d: Path) -> dict[str, Any]:
    try:
        return json.loads((d / "meta.json").read_text("utf-8"))
    except (OSError, ValueError):
        return {}


def _write_meta(d: Path, meta: dict[str, Any]) -> None:
    (d / "meta.json").write_text(json.dumps(meta, ensure_ascii=False), "utf-8")


def _require_magazine(doc_id: str) -> tuple[Path, dict[str, Any]]:
    d = _doc_dir(doc_id)
    meta = _read_meta(d)
    if meta.get("kind") != "magazine":
        raise HTTPException(400, "Not a magazine project")
    return d, meta


# ── Project creation ─────────────────────────────────────────────────────────

@router.post("/api/magazine/docs")
async def magazine_create(request: Request):
    """Create an empty magazine project; the PDF is uploaded separately."""
    body = await request.json()
    title = str(body.get("title") or "").strip()[:300] or "Untitled magazine"
    doc_id = uuid.uuid4().hex
    d = _LIB / doc_id
    d.mkdir(parents=True, exist_ok=True)
    now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    _write_meta(d, {
        "kind": "magazine",
        "title": title,
        "issue": str(body.get("issue") or "").strip()[:120],
        "publisher": str(body.get("publisher") or "").strip()[:120],
        "created": now,
        "updated": now,
        "ingest": {"status": "empty"},
        "default_voice": str(body.get("default_voice") or "").strip()[:200],
        "podcast": {"hosts": []},
        "progress": {},
    })
    return {"id": doc_id}


# ── Stage A–C: ingestion ─────────────────────────────────────────────────────

def _ingest_sync(doc_id: str, job_id: str, ocr_lang: str, render_dpi: int,
                 vision_ocr_url: str | None, vision_ocr_model: str | None,
                 vision_ocr_fallback_url: str | None = None,
                 vision_ocr_fallback_model: str | None = None) -> None:
    """Run the extraction pipeline; recorded as a restart-safe job."""
    d = _LIB / doc_id
    pdf = d / "source.pdf"
    try:
        if not pdf.is_file():
            raise FileNotFoundError("No source.pdf uploaded for this magazine")

        def progress(page: int, total: int) -> None:
            background_job_update(
                job_id, stage="extract",
                detail=f"Analysing page {page} of {total}",
            )

        background_job_update(job_id, stage="extract", detail="Opening PDF")
        layout = mag.extract_pages(pdf, d, render_dpi=render_dpi, ocr_lang=ocr_lang,
                                   vision_ocr_url=vision_ocr_url,
                                   vision_ocr_model=vision_ocr_model,
                                   vision_ocr_fallback_url=vision_ocr_fallback_url,
                                   vision_ocr_fallback_model=vision_ocr_fallback_model,
                                   progress=progress)
        mag.save_layout(d, layout)

        background_job_update(job_id, stage="articles", detail="Detecting articles")
        articles = mag.build_articles(layout)
        mag.save_articles(d, articles)

        meta = _read_meta(d)
        meta["pageCount"] = layout.get("page_count", 0)
        meta["ingest"] = {
            "status": "done",
            "finished": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            "articles": len(articles),
            "ocr_pages": sum(
                1 for b in layout.get("blocks", []) if b.get("source") == "ocr"
            ),
            "vision_ocr_pages": layout.get("vision_ocr_pages", 0),
        }
        _write_meta(d, meta)
        background_job_update(job_id, stage="done", done=True,
                              detail=f"{len(articles)} articles detected",
                              result={"articles": len(articles)})
    except Exception as exc:                       # noqa: BLE001 - surfaced to UI
        meta = _read_meta(d)
        meta["ingest"] = {"status": "failed", "error": str(exc)[:500]}
        _write_meta(d, meta)
        background_job_update(job_id, stage="failed", done=True, error=str(exc)[:1000])


@router.post("/api/magazine/{doc_id}/ingest")
async def magazine_ingest(doc_id: str, request: Request):
    d, meta = _require_magazine(doc_id)
    if not (d / "source.pdf").is_file():
        raise HTTPException(400, "Upload the PDF first")

    body = {}
    try:
        body = await request.json()
    except Exception:
        pass
    ocr_lang = str(body.get("ocr_lang") or "deu+eng")[:40]
    if not re.fullmatch(r"[a-z]{3}(?:\+[a-z]{3})*", ocr_lang):
        raise HTTPException(400, "Bad OCR language")
    try:
        render_dpi = max(72, min(200, int(body.get("dpi") or 110)))
    except (TypeError, ValueError):
        render_dpi = 110

    # A page needing OCR (or whose existing text scores as mostly garbage --
    # see ocr_quality_score) is escalated to this engine before falling back
    # to tesseract. Off by default per-request unless the deployment enabled
    # it in Settings; a request can still force it off with vision_ocr:false
    # even when enabled, e.g. for a fast re-run once boundaries are already
    # approved and only re-detection is wanted.
    settings = _load_settings()
    vision_ocr_url = vision_ocr_model = None
    vision_ocr_fallback_url = vision_ocr_fallback_model = None
    want_vision = body.get("vision_ocr")
    if want_vision is None:
        want_vision = bool(settings.get("vision_ocr_enabled"))
    if want_vision:
        vision_ocr_url = str(body.get("vision_ocr_url") or settings.get("vision_ocr_url") or "").strip()
        vision_ocr_model = str(body.get("vision_ocr_model") or settings.get("vision_ocr_model") or "").strip()
        if not (vision_ocr_url and vision_ocr_model):
            vision_ocr_url = vision_ocr_model = None
        vision_ocr_fallback_url = str(
            body.get("vision_ocr_fallback_url") or settings.get("vision_ocr_fallback_url") or "").strip()
        vision_ocr_fallback_model = str(
            body.get("vision_ocr_fallback_model") or settings.get("vision_ocr_fallback_model") or "").strip()
        if not (vision_ocr_fallback_url and vision_ocr_fallback_model):
            vision_ocr_fallback_url = vision_ocr_fallback_model = None

    job_id = uuid.uuid4().hex
    background_job_create(job_id, "magazine-ingest", fingerprint=doc_id)
    meta["ingest"] = {"status": "running", "job_id": job_id}
    _write_meta(d, meta)

    asyncio.create_task(
        asyncio.to_thread(_ingest_sync, doc_id, job_id, ocr_lang, render_dpi,
                          vision_ocr_url, vision_ocr_model,
                          vision_ocr_fallback_url, vision_ocr_fallback_model)
    )
    return {"job_id": job_id}


@router.get("/api/magazine/{doc_id}/ingest/status")
async def magazine_ingest_status(doc_id: str):
    _, meta = _require_magazine(doc_id)
    job_id = (meta.get("ingest") or {}).get("job_id")
    job = background_job_get(job_id) if job_id else None
    return {"ingest": meta.get("ingest") or {"status": "empty"}, "job": job}


# ── Layout and articles ──────────────────────────────────────────────────────

@router.get("/api/magazine/{doc_id}/layout")
async def magazine_layout(doc_id: str, page: int | None = None):
    """The layout document, optionally narrowed to one page."""
    d, _ = _require_magazine(doc_id)
    layout = mag.load_layout(d)
    if not layout:
        raise HTTPException(404, "Not ingested yet")
    if page is not None:
        layout = dict(layout)
        layout["blocks"] = [b for b in layout.get("blocks", []) if b.get("page") == page]
        layout["pages"] = [p for p in layout.get("pages", []) if p.get("page") == page]
    return layout


@router.get("/api/magazine/{doc_id}/articles")
async def magazine_articles(doc_id: str):
    d, meta = _require_magazine(doc_id)
    articles = mag.load_articles(d)
    # Lets the frontend skip the per-article podcast GET when nothing was ever
    # generated, instead of a 404 on every article open.
    for a in articles:
        a["has_podcast"] = (d / "podcast" / f"{a['id']}.json").is_file()
    return {
        "articles": articles,
        "progress": meta.get("progress") or {},
        "default_voice": meta.get("default_voice") or "",
        "podcast": meta.get("podcast") or {"hosts": []},
    }


@router.patch("/api/magazine/{doc_id}/articles/{article_id}")
async def magazine_update_article(doc_id: str, article_id: str, request: Request):
    """Stage D: the human review that gates narration.

    Only fields a reviewer can legitimately change are writable; block ids are
    validated against the layout so a bad edit cannot point an article at
    blocks that do not exist.
    """
    d, _ = _require_magazine(doc_id)
    article_id = _article_id(article_id)
    articles = mag.load_articles(d)
    target = next((a for a in articles if a.get("id") == article_id), None)
    if target is None:
        raise HTTPException(404, "Article not found")

    body = await request.json()

    if "status" in body:
        status = str(body["status"])
        if status not in ("detected", "approved", "rejected"):
            raise HTTPException(400, "Bad status")
        target["status"] = status
    if "title" in body:
        target["title"] = str(body["title"]).strip()[:300]
    if "author" in body:
        target["author"] = str(body["author"]).strip()[:120]
    if "section" in body:
        target["section"] = str(body["section"]).strip()[:120]

    if "blocks" in body:
        known = {b["id"] for b in mag.load_layout(d).get("blocks", [])}
        requested = body["blocks"]
        if not isinstance(requested, list):
            raise HTTPException(400, "blocks must be a list")
        unknown = [b for b in requested if b not in known]
        if unknown:
            raise HTTPException(400, f"Unknown block ids: {unknown[:5]}")
        target["blocks"] = requested
        layout_blocks = {b["id"]: b for b in mag.load_layout(d).get("blocks", [])}
        target["pages"] = sorted({
            layout_blocks[b]["page"] for b in requested if b in layout_blocks
        })

    if "pages" in body:
        pages = body["pages"]
        if not isinstance(pages, list) or not all(isinstance(p, int) for p in pages):
            raise HTTPException(400, "pages must be a list of integers")
        target["pages"] = sorted(set(pages))

    mag.save_articles(d, articles)
    return target


@router.get("/api/magazine/{doc_id}/articles/{article_id}/text")
async def magazine_article_text(doc_id: str, article_id: str,
                                captions: int = 0):
    """The narratable text of one article, per block, in reading order."""
    d, _ = _require_magazine(doc_id)
    article_id = _article_id(article_id)
    articles = mag.load_articles(d)
    target = next((a for a in articles if a.get("id") == article_id), None)
    if target is None:
        raise HTTPException(404, "Article not found")
    blocks = mag.article_text(mag.load_layout(d), target,
                             include_captions=bool(captions))
    return {"article": target, "blocks": blocks}


@router.patch("/api/magazine/{doc_id}/progress")
async def magazine_progress(doc_id: str, request: Request):
    """Per-article resume position, so each article remembers its own place."""
    d, meta = _require_magazine(doc_id)
    body = await request.json()
    article_id = _article_id(str(body.get("article") or ""))
    progress = dict(meta.get("progress") or {})
    entry = dict(progress.get(article_id) or {})
    if "sentence" in body:
        entry["sentence"] = max(0, int(body["sentence"]))
    if "seconds" in body:
        entry["seconds"] = max(0, float(body["seconds"]))
    progress[article_id] = entry
    meta["progress"] = progress
    meta["updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    _write_meta(d, meta)
    return {"ok": True}


# ── Page renders and figures ─────────────────────────────────────────────────

@router.get("/api/magazine/{doc_id}/pages/{page}")
async def magazine_page_image(doc_id: str, page: int):
    d, _ = _require_magazine(doc_id)
    if page < 1 or page > 10000:
        raise HTTPException(400, "Bad page")
    png = d / "pages" / f"{page:04d}.png"
    if not png.is_file():
        raise HTTPException(404, "Page not rendered")
    return FileResponse(png, media_type="image/png")


@router.get("/api/magazine/{doc_id}/images/{name}")
async def magazine_image(doc_id: str, name: str):
    d, _ = _require_magazine(doc_id)
    if not re.fullmatch(r"[A-Za-z0-9_-]{1,64}\.png", name or ""):
        raise HTTPException(400, "Bad image name")
    path = d / "images" / name
    if not path.is_file():
        raise HTTPException(404, "Image not found")
    return FileResponse(path, media_type="image/png")


# ── Podcast mode ─────────────────────────────────────────────────────────────

_PODCAST_SYSTEM = (
    "You write podcast scripts in which {n} hosts discuss a magazine article "
    "for listeners who have not read it. Cover the article's actual content "
    "and conclusions faithfully -- never invent facts, statistics, quotes or "
    "study results that are not in the source text. If the article is "
    "inconclusive, let the hosts say so rather than manufacturing a verdict.\n"
    "\n"
    "Return STRICT JSON, no markdown fence, in this shape:\n"
    '{{"title": "...", "turns": [{{"host": "A", "text": "..."}}]}}\n'
    "\n"
    "Rules:\n"
    "- Use only host labels {labels}.\n"
    "- 12-30 turns. Each turn is one or two spoken sentences.\n"
    "- Open by naming the article and its author, close with a short takeaway.\n"
    "- Write speech, not prose: contractions, plain words, no bullet points, "
    "no stage directions, no sound-effect cues.\n"
    "- Write in {language}.\n"
)


def _podcast_dir(d: Path) -> Path:
    p = d / "podcast"
    p.mkdir(parents=True, exist_ok=True)
    return p


def _parse_script(raw: str, labels: list[str]) -> dict[str, Any]:
    """Parse the model's JSON, tolerating a stray markdown fence."""
    text = (raw or "").strip()
    if text.startswith("```"):
        text = re.sub(r"^```[a-zA-Z]*\s*|\s*```$", "", text).strip()
    start, end = text.find("{"), text.rfind("}")
    if start != -1 and end > start:
        text = text[start:end + 1]
    try:
        data = json.loads(text)
    except ValueError as exc:
        raise HTTPException(502, f"LLM did not return usable JSON: {exc}") from exc

    turns_in = data.get("turns")
    if not isinstance(turns_in, list) or not turns_in:
        raise HTTPException(502, "LLM returned no turns")

    allowed = set(labels)
    turns: list[dict[str, str]] = []
    for turn in turns_in:
        if not isinstance(turn, dict):
            continue
        host = str(turn.get("host") or "").strip().upper()[:8]
        line = str(turn.get("text") or "").strip()
        if not line:
            continue
        # An unknown label would otherwise silently drop to no voice at all.
        if host not in allowed:
            host = labels[len(turns) % len(labels)]
        turns.append({"host": host, "text": line[:2000]})
    if not turns:
        raise HTTPException(502, "LLM returned no usable turns")
    return {"title": str(data.get("title") or "").strip()[:300], "turns": turns}


@router.post("/api/magazine/{doc_id}/articles/{article_id}/podcast")
async def magazine_podcast_script(doc_id: str, article_id: str, request: Request):
    """Turn one approved article into a 2-3 host podcast script.

    The script is generated and stored here; synthesis reuses the existing TTS
    pipeline, one voice per host, exactly as multi-speaker audiobook segments
    are rendered today.
    """
    from routes.conversation import (          # local import avoids a cycle
        _post_llm_chat_completion,
        _request_timeout_seconds,
    )

    d, meta = _require_magazine(doc_id)
    article_id = _article_id(article_id)
    articles = mag.load_articles(d)
    target = next((a for a in articles if a.get("id") == article_id), None)
    if target is None:
        raise HTTPException(404, "Article not found")
    if target.get("status") != "approved":
        # Generating from unreviewed boundaries wastes an LLM call on text that
        # may contain a neighbouring article or an advert.
        raise HTTPException(400, "Approve the article before generating a podcast")

    body = await request.json()
    hosts = body.get("hosts")
    if not isinstance(hosts, list) or not (_MIN_HOSTS <= len(hosts) <= _MAX_HOSTS):
        raise HTTPException(400, f"Pick {_MIN_HOSTS} or {_MAX_HOSTS} host voices")
    parsed_hosts: list[dict[str, str]] = []
    for i, host in enumerate(hosts):
        if not isinstance(host, dict):
            raise HTTPException(400, "Bad host entry")
        voice = str(host.get("voice") or "").strip()[:200]
        if not voice:
            raise HTTPException(400, "Every host needs a voice")
        parsed_hosts.append({
            "label": chr(ord("A") + i),
            "voice": voice,
            "name": str(host.get("name") or "").strip()[:60] or f"Host {chr(ord('A') + i)}",
        })
    labels = [h["label"] for h in parsed_hosts]

    blocks = mag.article_text(mag.load_layout(d), target,
                              include_captions=bool(body.get("captions")))
    source = "\n\n".join(b["text"] for b in blocks).strip()
    if len(source) < 200:
        raise HTTPException(400, "Article has too little text for a podcast")
    # Keep the prompt inside a sane context window; the tail of a very long
    # feature contributes little to a discussion of its argument.
    source = source[:24000]

    settings = _load_settings()
    llm_url = (body.get("llm_url") or settings.get("llm_url")
               or "http://localhost:11434/v1")
    model = (body.get("model") or settings.get("llm_model") or "").strip()
    if not model:
        raise HTTPException(400, "No LLM model configured")
    language = str(body.get("language") or "the language of the article")[:60]

    system = _PODCAST_SYSTEM.format(
        n=len(parsed_hosts),
        labels=", ".join(labels),
        language=language,
    )
    roster = "\n".join(f"{h['label']} = {h['name']}" for h in parsed_hosts)
    user = (
        f"Hosts:\n{roster}\n\n"
        f"Article title: {target.get('title') or 'Untitled'}\n"
        f"Author: {target.get('author') or 'unknown'}\n\n"
        f"Article text:\n{source}"
    )

    payload = {
        "model": model,
        "messages": [
            {"role": "system", "content": system},
            {"role": "user", "content": user},
        ],
        "temperature": float(body.get("temperature") or 0.7),
        "stream": False,
    }
    headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"}
    timeout = _request_timeout_seconds(body.get("timeout"), default=300.0)

    resp = await asyncio.to_thread(
        _post_llm_chat_completion, llm_url, payload, headers, timeout
    )
    if resp.status_code >= 400:
        raise HTTPException(502, f"LLM error {resp.status_code}: {resp.text[:300]}")
    try:
        content = resp.json()["choices"][0]["message"]["content"]
    except (ValueError, KeyError, IndexError) as exc:
        raise HTTPException(502, "Unexpected LLM response shape") from exc

    script = _parse_script(content, labels)
    script.update({
        "article": article_id,
        "hosts": parsed_hosts,
        "model": model,
        "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
    })
    (_podcast_dir(d) / f"{article_id}.json").write_text(
        json.dumps(script, ensure_ascii=False), "utf-8")

    # Remember the cast so the next article defaults to the same hosts.
    meta["podcast"] = {"hosts": parsed_hosts}
    _write_meta(d, meta)
    return script


@router.get("/api/magazine/{doc_id}/articles/{article_id}/podcast")
async def magazine_podcast_get(doc_id: str, article_id: str):
    d, _ = _require_magazine(doc_id)
    article_id = _article_id(article_id)
    path = d / "podcast" / f"{article_id}.json"
    if not path.is_file():
        raise HTTPException(404, "No podcast script generated yet")
    try:
        return json.loads(path.read_text("utf-8"))
    except (OSError, ValueError) as exc:
        raise HTTPException(500, "Podcast script unreadable") from exc


@router.delete("/api/magazine/{doc_id}/articles/{article_id}/podcast")
async def magazine_podcast_delete(doc_id: str, article_id: str):
    d, _ = _require_magazine(doc_id)
    article_id = _article_id(article_id)
    (d / "podcast" / f"{article_id}.json").unlink(missing_ok=True)
    return {"ok": True}