feat(magazine): exclude entries from the untitled-page cleanup
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a60a5a92ca
commit
e1aca0ba34
@ -552,9 +552,13 @@ def merge_entries(layout: dict[str, Any], articles: list[dict[str, Any]], source
|
|||||||
def plan(layout: dict[str, Any], articles: Sequence[dict[str, Any]], *, min_confidence: float = HIGH_AT,
|
def plan(layout: dict[str, Any], articles: Sequence[dict[str, Any]], *, min_confidence: float = HIGH_AT,
|
||||||
cache: dict[str, Any] | None = None,
|
cache: dict[str, Any] | None = None,
|
||||||
llm_check: Callable[[str, str], bool | None] | None = None,
|
llm_check: Callable[[str, str], bool | None] | None = None,
|
||||||
titler: Callable[[str, str], str | None] | None = None) -> dict[str, Any]:
|
titler: Callable[[str, str], str | None] | None = None,
|
||||||
|
exclude: Sequence[str] = ()) -> dict[str, Any]:
|
||||||
"""Simulate merges (page order, chains resolve) and titles on a copy. Returns
|
"""Simulate merges (page order, chains resolve) and titles on a copy. Returns
|
||||||
{articles, merges, titles, before}. Nothing is written."""
|
{articles, merges, titles, before}. Nothing is written. Entries in ``exclude`` are
|
||||||
|
left as they are: never merged away into another entry and never retitled (they may still
|
||||||
|
receive merge-ins from the pages after them)."""
|
||||||
|
skip = {str(x) for x in exclude}
|
||||||
work = json.loads(json.dumps(list(articles)))
|
work = json.loads(json.dumps(list(articles)))
|
||||||
language = doc_language(layout)
|
language = doc_language(layout)
|
||||||
before = report(layout, articles, cache)
|
before = report(layout, articles, cache)
|
||||||
@ -564,7 +568,7 @@ def plan(layout: dict[str, Any], articles: Sequence[dict[str, Any]], *, min_conf
|
|||||||
order = [r["id"] for r in report(layout, work, cache)["entries"]]
|
order = [r["id"] for r in report(layout, work, cache)["entries"]]
|
||||||
for aid in order:
|
for aid in order:
|
||||||
a = next((x for x in work if x["id"] == aid), None)
|
a = next((x for x in work if x["id"] == aid), None)
|
||||||
if a is None:
|
if a is None or aid in skip:
|
||||||
continue
|
continue
|
||||||
s = suggestion_for(_Index(layout, work), a, llm_check)
|
s = suggestion_for(_Index(layout, work), a, llm_check)
|
||||||
if not s or s["confidence"] < min_confidence:
|
if not s or s["confidence"] < min_confidence:
|
||||||
@ -581,7 +585,7 @@ def plan(layout: dict[str, Any], articles: Sequence[dict[str, Any]], *, min_conf
|
|||||||
ann = annotate(layout, work, cache, language)
|
ann = annotate(layout, work, cache, language)
|
||||||
for a in work:
|
for a in work:
|
||||||
info = ann[a["id"]]
|
info = ann[a["id"]]
|
||||||
if info.get("title_source") != "suggested":
|
if a["id"] in skip or info.get("title_source") != "suggested":
|
||||||
continue
|
continue
|
||||||
title, source = info["suggested_title"], info["suggested_title_source"]
|
title, source = info["suggested_title"], info["suggested_title_source"]
|
||||||
if titler is not None and source == "sentence":
|
if titler is not None and source == "sentence":
|
||||||
|
|||||||
@ -7,7 +7,7 @@ suggestion at or above ``--min-confidence`` (page order, so chains of continuati
|
|||||||
into their article), marks merged articles ``audio_stale`` (audio files are kept), and writes
|
into their article), marks merged articles ``audio_stale`` (audio files are kept), and writes
|
||||||
suggested titles into placeholder titles only. Titles someone set are never touched.
|
suggested titles into placeholder titles only. Titles someone set are never touched.
|
||||||
|
|
||||||
python3 scripts/magazine_untitled.py <doc-id | path/to/doc_dir> [--apply] [--no-llm] [--json out.json]
|
python3 scripts/magazine_untitled.py <doc-id | path/to/doc_dir> [--apply] [--no-llm] [--exclude ID,ID] [--json out.json]
|
||||||
|
|
||||||
See architecture/magazine-ingestion.md, "Untitled entries".
|
See architecture/magazine-ingestion.md, "Untitled entries".
|
||||||
"""
|
"""
|
||||||
@ -76,7 +76,7 @@ def make_chat(timeout: float = 30.0):
|
|||||||
return chat
|
return chat
|
||||||
|
|
||||||
|
|
||||||
def run(doc_dir: Path, *, apply: bool, min_confidence: float, chat=None) -> dict:
|
def run(doc_dir: Path, *, apply: bool, min_confidence: float, chat=None, exclude=()) -> dict:
|
||||||
layout = mag.load_layout(doc_dir)
|
layout = mag.load_layout(doc_dir)
|
||||||
articles = mag.load_articles(doc_dir)
|
articles = mag.load_articles(doc_dir)
|
||||||
if not layout or not articles:
|
if not layout or not articles:
|
||||||
@ -92,7 +92,7 @@ def run(doc_dir: Path, *, apply: bool, min_confidence: float, chat=None) -> dict
|
|||||||
return mu.llm_title(chat, text, language)
|
return mu.llm_title(chat, text, language)
|
||||||
|
|
||||||
result = mu.plan(layout, articles, min_confidence=min_confidence, cache=cache,
|
result = mu.plan(layout, articles, min_confidence=min_confidence, cache=cache,
|
||||||
llm_check=llm_check, titler=titler if chat else None)
|
llm_check=llm_check, titler=titler if chat else None, exclude=exclude)
|
||||||
index = mu._Index(layout, result["articles"])
|
index = mu._Index(layout, result["articles"])
|
||||||
for t in result["titles"]:
|
for t in result["titles"]:
|
||||||
if t["source"] == "llm":
|
if t["source"] == "llm":
|
||||||
@ -151,12 +151,15 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
mode.add_argument("--apply", action="store_true", help="write changes (backup first)")
|
mode.add_argument("--apply", action="store_true", help="write changes (backup first)")
|
||||||
p.add_argument("--min-confidence", type=float, default=mu.HIGH_AT,
|
p.add_argument("--min-confidence", type=float, default=mu.HIGH_AT,
|
||||||
help=f"merge threshold (default {mu.HIGH_AT})")
|
help=f"merge threshold (default {mu.HIGH_AT})")
|
||||||
|
p.add_argument("--exclude", action="append", default=[], metavar="ID[,ID...]",
|
||||||
|
help="leave these article ids as they are (not merged into another entry, no new title); repeatable")
|
||||||
p.add_argument("--no-llm", action="store_true", help="heuristics and deterministic titles only")
|
p.add_argument("--no-llm", action="store_true", help="heuristics and deterministic titles only")
|
||||||
p.add_argument("--json", help="also write the full result as JSON to this path")
|
p.add_argument("--json", help="also write the full result as JSON to this path")
|
||||||
args = p.parse_args(argv)
|
args = p.parse_args(argv)
|
||||||
doc_dir = resolve_doc_dir(args.doc)
|
doc_dir = resolve_doc_dir(args.doc)
|
||||||
chat = None if args.no_llm else make_chat()
|
chat = None if args.no_llm else make_chat()
|
||||||
summary = run(doc_dir, apply=args.apply, min_confidence=args.min_confidence, chat=chat)
|
summary = run(doc_dir, apply=args.apply, min_confidence=args.min_confidence, chat=chat,
|
||||||
|
exclude=[i.strip() for v in args.exclude for i in v.split(",") if i.strip()])
|
||||||
print_summary(summary)
|
print_summary(summary)
|
||||||
if args.json:
|
if args.json:
|
||||||
Path(args.json).write_text(json.dumps(summary, ensure_ascii=False, indent=1), "utf-8")
|
Path(args.json).write_text(json.dumps(summary, ensure_ascii=False, indent=1), "utf-8")
|
||||||
|
|||||||
@ -316,6 +316,18 @@ class PlanTests(unittest.TestCase):
|
|||||||
layout, articles = sample()
|
layout, articles = sample()
|
||||||
self.assertEqual(mu.plan(layout, articles, min_confidence=1.01)["merges"], [])
|
self.assertEqual(mu.plan(layout, articles, min_confidence=1.01)["merges"], [])
|
||||||
|
|
||||||
|
def test_excluded_entries_are_left_untouched(self):
|
||||||
|
layout, articles = sample()
|
||||||
|
base = mu.plan(layout, articles)
|
||||||
|
self.assertTrue(base["merges"])
|
||||||
|
src = base["merges"][0]["source"]
|
||||||
|
result = mu.plan(layout, articles, exclude=[src])
|
||||||
|
self.assertTrue(all(m["source"] != src for m in result["merges"]))
|
||||||
|
self.assertNotIn(src, {t["id"] for t in result["titles"]})
|
||||||
|
kept = next(a for a in result["articles"] if a["id"] == src)
|
||||||
|
orig = next(a for a in articles if a["id"] == src)
|
||||||
|
self.assertEqual(kept["title"], orig["title"])
|
||||||
|
|
||||||
def test_apply_titles_only_touches_placeholders(self):
|
def test_apply_titles_only_touches_placeholders(self):
|
||||||
_, articles = sample()
|
_, articles = sample()
|
||||||
n = mu.apply_titles(articles, [{"id": "article-001", "after": "X"}, {"id": "article-004", "after": "Anzeige",
|
n = mu.apply_titles(articles, [{"id": "article-001", "after": "X"}, {"id": "article-004", "after": "Anzeige",
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user