#!/usr/bin/env python3 """Untitled magazine entries ("Page 33"): propose merges into the previous article and titles. Dry run by default: prints what would change and writes nothing. ``--apply`` first writes ``articles.backup-untitled-.json`` next to ``articles.json``, then merges every suggestion at or above ``--min-confidence`` (page order, so chains of continuation pages collapse into their article), marks merged articles ``audio_stale`` (audio files are kept), and writes suggested titles into placeholder titles only. Titles someone set are never touched. python3 scripts/magazine_untitled.py [--apply] [--no-llm] [--json out.json] See architecture/magazine-ingestion.md, "Untitled entries". """ from __future__ import annotations import argparse import datetime as _dt import json import os import re import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) os.environ.setdefault("CONFIG_DIR", str(ROOT / "config")) from core import magazine as mag # noqa: E402 from core import magazine_untitled as mu # noqa: E402 def resolve_doc_dir(arg: str) -> Path: path = Path(arg) if path.is_dir(): return path if not re.fullmatch(r"[0-9a-f]{32}", arg): raise SystemExit(f"Not a directory or document id: {arg}") from core.constants import CONFIG_DIR path = Path(CONFIG_DIR) / "reader_library" / arg if not path.is_dir(): raise SystemExit(f"No magazine at {path}") return path def make_chat(timeout: float = 30.0): """A chat(messages) -> str callable on the app's configured LLM, or None when switched off.""" from core.config import _load_settings settings = _load_settings() if settings.get("magazine_llm_titles") is False: return None from routes.conversation import _post_llm_chat_completion url = settings.get("llm_url") or "http://localhost:11434/v1" headers = {"Authorization": f"Bearer {settings.get('llm_api_key') or 'sk-dummy-key'}"} state = {"down": ""} def chat(messages): if state["down"]: # one failure is enough: fall back for the run raise RuntimeError(state["down"]) try: return _ask(messages) except Exception as exc: # noqa: BLE001 - reported once, then deterministic state["down"] = f"LLM unavailable ({str(exc)[:120]}); using deterministic titles and heuristics" print(state["down"], file=sys.stderr) raise def _ask(messages): payload = {"model": str(settings.get("llm_model") or "").strip(), "temperature": 0.0, "stream": False, "max_tokens": 60, "messages": messages, "chat_template_kwargs": {"enable_thinking": False}} resp = _post_llm_chat_completion(url, payload, headers, timeout) if resp.status_code in (400, 422): payload.pop("chat_template_kwargs") resp = _post_llm_chat_completion(url, payload, headers, timeout) resp.raise_for_status() return resp.json()["choices"][0]["message"]["content"] or "" return chat def run(doc_dir: Path, *, apply: bool, min_confidence: float, chat=None) -> dict: layout = mag.load_layout(doc_dir) articles = mag.load_articles(doc_dir) if not layout or not articles: raise SystemExit(f"No layout/articles in {doc_dir}") cache = mu.load_title_cache(doc_dir) language = mu.doc_language(layout) llm_check = mu.llm_continuation_check(chat) if chat else None new_cache = {"entries": dict(cache.get("entries") or {})} def titler(article_id: str, text: str) -> str | None: if chat is None: return None return mu.llm_title(chat, text, language) result = mu.plan(layout, articles, min_confidence=min_confidence, cache=cache, llm_check=llm_check, titler=titler if chat else None) index = mu._Index(layout, result["articles"]) for t in result["titles"]: if t["source"] == "llm": a = next(x for x in result["articles"] if x["id"] == t["id"]) new_cache["entries"][t["id"]] = {"title": t["after"], "hash": mu.text_hash(index, a), "prompt": mu.PROMPT_VERSION} summary = { "doc_dir": str(doc_dir), "mode": "apply" if apply else "dry-run", "before": {"entries": len(articles), "untitled": result["before"]["count"], "kinds": result["before"]["kinds"], "merge_suggestions": result["before"]["merge_suggestions"]}, "merges": result["merges"], "titles": result["titles"], "after": {"entries": len(result["articles"])}, "pending": result["pending"], "audio_kept_orphaned": [m["source"] for m in result["merges"] if (doc_dir / "audio" / f"{m['source']}.mp3").is_file()], "audio_stale": sorted({m["target"] for m in result["merges"]}), } if apply: stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%S") backup = doc_dir / f"articles.backup-untitled-{stamp}.json" backup.write_text((doc_dir / "articles.json").read_text("utf-8"), "utf-8") final = result["articles"] summary["titles_applied"] = mu.apply_titles(final, result["titles"]) mag.save_articles(doc_dir, final) mu.save_title_cache(doc_dir, new_cache) summary["backup"] = str(backup) return summary def print_summary(s: dict) -> None: b = s["before"] print(f"{s['mode'].upper()} {s['doc_dir']}") print(f"entries {b['entries']} -> {s['after']['entries']}; untitled/duplicate {b['untitled']}: {b['kinds']}") print(f"merges: {len(s['merges'])} (audio marked stale on {len(s['audio_stale'])}, " f"{len(s['audio_kept_orphaned'])} MP3s of removed entries kept on disk)") for m in s["merges"]: print(f" {m['source']} {m['title_before'][:28]!r:30} -> {m['target']} ({m['confidence']:.2f}) " f"| {m['first_text'][:50]!r}") print(f"suggestions left for review (below --min-confidence): {len(s['pending'])}") for m in s["pending"]: print(f" {m['id']} {m['title'][:28]!r:30} -> {m['into']} ({m['confidence']:.2f}) {m['page_label']}") print(f"titles: {len(s['titles'])}") for t in s["titles"]: print(f" {t['id']} {t['before'][:12]!r:14} -> {t['after'][:60]!r} [{t['source']}/{t['kind']}] {t['page_label']}") if s.get("backup"): print(f"backup: {s['backup']}; titles applied: {s.get('titles_applied')}") def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("doc", help="magazine document id or its directory") mode = p.add_mutually_exclusive_group() mode.add_argument("--dry-run", action="store_true", default=True, help="report only (default)") mode.add_argument("--apply", action="store_true", help="write changes (backup first)") p.add_argument("--min-confidence", type=float, default=mu.HIGH_AT, help=f"merge threshold (default {mu.HIGH_AT})") p.add_argument("--no-llm", action="store_true", help="heuristics and deterministic titles only") p.add_argument("--json", help="also write the full result as JSON to this path") args = p.parse_args(argv) doc_dir = resolve_doc_dir(args.doc) chat = None if args.no_llm else make_chat() summary = run(doc_dir, apply=args.apply, min_confidence=args.min_confidence, chat=chat) print_summary(summary) if args.json: Path(args.json).write_text(json.dumps(summary, ensure_ascii=False, indent=1), "utf-8") return 0 if __name__ == "__main__": raise SystemExit(main())