From 5a7c73ea425dedc0302864c383dac4edfb4c5e22 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Sun, 4 Oct 2026 17:02:41 +0200 Subject: [PATCH] feat(export): provenance tags and optional disclosure for exported audio ID3 provenance tags (synthetic speech, generator, version, date, opt-in voice name) on magazine MP3/ZIP/EPUB exports via a stream-copied temp copy; verify endpoint and CLI; AudioSeal tier is a documented stub. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + Manual.md | 9 +++++ architecture/watermarking.md | 26 ++++++++++++ core/config.py | 2 +- core/export_provenance.py | 71 +++++++++++++++++++++++++++++++++ routes/magazine.py | 68 ++++++++++++++++++++++++++----- static/js/magazine.js | 29 +++++++++++++- static/sections/s-magazine.html | 6 +++ tests/magazine.spec.js | 8 +++- tests/test_watermark_tags.py | 51 +++++++++++++++++++++++ 10 files changed, 258 insertions(+), 13 deletions(-) create mode 100644 architecture/watermarking.md create mode 100644 core/export_provenance.py create mode 100644 tests/test_watermark_tags.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0902350..c0a2892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi ## [Unreleased] ### Added +- **Provenance tags on exported audio.** Magazine MP3 downloads, `audio.zip` and the MP3s inside the EPUB get ID3 tags (`synthetic_speech=true`, `generator`, `generator_version`, `generated_date`, comment; voice name only on opt-in) via one ffmpeg stream copy of a temp copy; stored audio is untouched. Per-export checkboxes on the magazine export row ("Mark as synthetic speech", "include voice name"), default from new setting `export_provenance_default` (on). `POST /api/export/provenance` and `python -m core.export_provenance file.mp3` read the tags back. Inaudible AudioSeal watermarking is researched (MIT code and weights, torch needed, weak message recovery after MP3/AAC) but only a stub `watermark(path)` that reports unavailable. Spoken disclosure is not implemented yet. Tests: `tests/test_watermark_tags.py`, `tests/magazine.spec.js`. See `architecture/watermarking.md`. - **Inline text tags for TTS (setting `inline_tags_enabled`, default OFF).** `[pause 500ms]`/`[pause 1.2s]` (max 10 s each, 60 s per text), `[spell]USA[/spell]` (German letter names for German text), `[slow]..[/slow]` / `[fast]..[/fast]` (0.85 / 1.15 via ffmpeg atempo, innermost wins, unclosed ends at paragraph). Known control tags (e.g. `[laugh]`) are stripped, never spoken; `[1]` and `[sic]` stay. Only `POST /api/tts-preview` renders them; other paths are listed in `architecture/inline-tags.md`. Tests: `tests/test_ssml_lite.py`. - **Optional vocal isolation for URL/YouTube imports (default OFF).** `POST /api/isolate-vocals {id}` runs the Demucs CLI (`htdemucs`, two stems) on a registered temp audio and registers the vocals as a new id; the original stays valid. Needs `VOCAL_ISOLATION=1`, a `demucs` binary (or `VOCAL_ISOLATION_CMD`) and the weights already on disk; the app never downloads anything and nothing is added to the main image. Limits: 15 min input, 15 min runtime (`VOCAL_ISOLATION_MAX_SECONDS`/`_TIMEOUT`), one job at a time, cancelled on client disconnect, temp files always removed. `GET /api/version` reports `features.vocal_isolation`; the "Remove background music" button (clone page URL card and voice library add-URL) appears only then, with an aria-live status and a play original / play without music toggle. Tests: `tests/test_vocal_isolation.py`, `tests/vocal-isolation.spec.js`. See `architecture/vocal-isolation.md`. - **Voice export/import bundles.** `GET /api/voices/{id}/export` streams a `.voice.zip` (audio, `reference.txt`, `meta.json` with seed, design prompt, rating/tags, `provenance`, `audio_problems`; optional picture and consent recording; `manifest.json` with format version, app version and sha256 per file; no temp files). `POST /api/voices/import` validates at the boundary: 150 MB upload cap, fixed flat member names only (zip-slip/path traversal rejected), per-file, total and compression-ratio caps, hashes verified, audio decoded, meta and provenance re-validated; a taken name gets a `_2` suffix, an identical voice is a no-op, nothing is overwritten. Voice library toolbar: "Export selected" and "Import bundle" (44px). Tests: `tests/test_voice_bundle.py`, `tests/voice-bundle.spec.js`. See `architecture/voice-bundle.md`. diff --git a/Manual.md b/Manual.md index 5d3eaae..2697c77 100644 --- a/Manual.md +++ b/Manual.md @@ -750,6 +750,15 @@ sent. Websites that redistribute commercial magazines without a licence are deli **Mobile / accessibility:** data only (API fields). The index shows `display_title` with `page_label` in smaller text, and the full title stays available to screen readers. **Offline / privacy:** the heuristics run locally. LLM titles use only the LLM configured in Settings and send it at most 1,500 characters of the entry's text. +### Marking exported audio as synthetic speech + +- **Purpose:** exported MP3s (single download, "Download all (ZIP)", "EPUB with audio") carry tags saying the speech is AI-generated. +- **Prerequisites:** an article with generated audio. No extra software. +- **Steps:** in the export row tick or untick "Mark as synthetic speech (tags)" (default on, remembered as a setting) and optionally "include voice name". Then download as usual. +- **Result:** the copy you download has tags `synthetic_speech`, `generator`, `generator_version`, `generated_date` (and `voice` if chosen). The stored audio is unchanged. To check a file: `python -m core.export_provenance file.mp3` or `POST /api/export/provenance` with the file as body. +- **Errors:** if tagging fails the download answers "Could not add provenance tags"; untick the box to export untagged. +- **Mobile / accessibility:** plain labelled checkboxes. **Offline:** works offline. **Privacy:** no user names or paths; the voice name only if you tick it. Tags can be removed by anyone, so they are an honest label, not protection. An inaudible watermark and a spoken disclosure are not available yet. + ### Errors | What you see | What happened | What to do | diff --git a/architecture/watermarking.md b/architecture/watermarking.md new file mode 100644 index 0000000..4411801 --- /dev/null +++ b/architecture/watermarking.md @@ -0,0 +1,26 @@ +# Provenance tags and watermarking for exported audio + +Status: tier 1 implemented (`core/export_provenance.py`); tier 2 (AudioSeal) stubbed, not installed. + +## Research (checked 2026-10-04) + +**AudioSeal (facebookresearch/audioseal)** +- Licence: README states code AND model weights are MIT ("full MIT license, including the license for the model weights", since April 2024). Commercial use allowed. Verdict: licence acceptable. +- Requirements: Python >= 3.8, PyTorch >= 1.13, OmegaConf, NumPy (einops for streaming). Weights are fetched from the Hugging Face hub on first use (network download, so must be baked in or mounted for an offline box). +- ARM64/CPU: the host has torch 2.10 (aarch64, cu130), but the app image has no torch (nothing in Dockerfile/requirements). The generator is a small conv/LSTM net at 16 kHz and runs on CPU; no CPU benchmark was found in the docs, so cost per hour of audio is unverified (measure before promising). Docs flag torchaudio backend quirks (needs `soundfile` or torchaudio 2.1). +- Robustness (paper "Proactive Detection of Voice Cloning with Localized Watermarking", arXiv 2401.17264, and third-party comparisons): detection (is-watermarked) stays high under MP3/AAC (mean AUC about 0.97; 128 kbps AAC detection 1.0), but exact recovery of the 16-bit message under AAC 128k falls to about 0 and newer comparisons report < 60% message accuracy under MP3/AAC. So: only "watermarked yes/no" is dependable after re-encode, not a voice/version id. +- What is detectable: per-sample (1/16000 s) localized probability that audio was watermarked, plus optional 16-bit message. Needs the AudioSeal detector (python+torch); no public third party checks it. Trained at 16 kHz, resampling needed for 22-44 kHz audio. Adversarial removal is possible. + +**Tier 1 alternatives (no dependencies)** +- Provenance tags in the container (ID3v2.3 TXXX/COMM via ffmpeg `-metadata`): `synthetic_speech=true`, `generator`, `generator_version`, `generated_date`, optional `voice` (name only), `comment`. Honest, trivially strippable, survives only while the container is kept. +- Spoken disclosure ("Dieses Hörbuch wurde synthetisch erzeugt"): not strippable by metadata removal, but needs a TTS call plus a re-encoding concat; NOT implemented yet (see Open decisions). + +## Design + +1. Tier 1 (implemented): on export, one ffmpeg stream copy (`-c copy -map_metadata 0 -id3v2_version 3`) of the MP3 into a temp copy with the tags. The stored article audio is never changed. Applies to: single article download, `audio.zip`, and the MP3s inside the EPUB. Query `?provenance=1|0` and `&voice=1` per export; absent -> setting `export_provenance_default` (default true). No user names, paths or secrets are written; voice only when opted in, basename only; ffmpeg errors are not forwarded (they can contain paths). +2. Verify: `POST /api/export/provenance` (raw audio body, 100 MB cap) returns the tags and `synthetic`; CLI `python -m core.export_provenance file.mp3` (exit 0 if tagged). +3. Tier 2: `watermark(path)` raises `NotImplementedError`; `watermark_available()` is False. Enabling it later means: optional extra image/venv with torch + audioseal and pre-mounted weights, a feature flag, lazy import inside `watermark()`, and a decision on re-encode (watermark must be applied to PCM before the MP3 encode, i.e. at synthesis time, not on the finished MP3 copy). + +## Open decisions +- Spoken disclosure: render once per language via the normal TTS path and concat at export (re-encode) - needs owner go-ahead. +- Tier 2: needs torch in a sidecar container (not the main image); recommended only if an inaudible mark is really required, given weak message recovery after AAC/MP3. diff --git a/core/config.py b/core/config.py index 84d4154..327627a 100644 --- a/core/config.py +++ b/core/config.py @@ -38,7 +38,7 @@ _SETTINGS_KEYS = { "stt_language", "stt_preferred_backend", "auto_refine", "refine_model", "refine_fillers", "refine_repetitions", "refine_corrections", "refine_punctuation", - "captures_default_voice", + "captures_default_voice", "export_provenance_default", "client_voice_bindings", "llm_url", "llm_model", "llm_api_key", # Magazine layout is a vision task. Kept separate from the general text diff --git a/core/export_provenance.py b/core/export_provenance.py new file mode 100644 index 0000000..9b62a65 --- /dev/null +++ b/core/export_provenance.py @@ -0,0 +1,71 @@ +"""Provenance tags for exported generated audio (tier 1) and the AudioSeal stub (tier 2). + +Tags are written with one ffmpeg stream copy (no re-encode) into a COPY of the MP3; the stored +article audio is never modified. Tags never contain paths, user names or secrets; the voice is +included only when the caller opts in. See architecture/watermarking.md. +""" +from __future__ import annotations + +import json +import re +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +GENERATOR = "tts-voice-creator-clone-and-design-2" +_KEYS = ("synthetic_speech", "generator", "generator_version", "generated_date", "voice") +_CTRL = re.compile(r"[\x00-\x1f\x7f]") + + +def build_tags(app_version: str, voice: str = "", include_voice: bool = False, today: str = "") -> dict[str, str]: + tags = { + "synthetic_speech": "true", + "generator": GENERATOR, + "generator_version": _CTRL.sub("", str(app_version))[:32], + "generated_date": today or datetime.now(timezone.utc).strftime("%Y-%m-%d"), + } + if include_voice and voice: + tags["voice"] = _CTRL.sub("", Path(str(voice)).name)[:80] # name only, never a path + tags["comment"] = f"Synthetic speech (AI-generated) by {GENERATOR} {tags['generator_version']}" + return tags + + +def apply_tags(src: Path, dst: Path, tags: dict[str, str]) -> Path: + """Copy src to dst (stream copy) with the tags; raises RuntimeError on ffmpeg failure.""" + cmd = ["ffmpeg", "-y", "-v", "error", "-i", str(src), "-map", "0:a", "-map_metadata", "0", + "-c", "copy", "-id3v2_version", "3"] + for k, v in tags.items(): + cmd += ["-metadata", f"{k}={v}"] + cmd.append(str(dst)) + r = subprocess.run(cmd, capture_output=True, timeout=120) + if r.returncode != 0: + raise RuntimeError("tagging failed") # ffmpeg stderr may contain paths: not forwarded + return dst + + +def read_tags(path: Path) -> dict[str, str]: + """Provenance tags found in an audio file ({} if none). Never raises.""" + try: + r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format_tags", "-of", "json", str(path)], + capture_output=True, timeout=30) + found = (json.loads(r.stdout or b"{}").get("format") or {}).get("tags") or {} + except (OSError, ValueError, subprocess.SubprocessError): + return {} + low = {str(k).lower(): str(v) for k, v in found.items()} + return {k: low[k] for k in _KEYS if k in low} + + +def watermark(path: Path) -> Path: + """Tier 2 (AudioSeal) is not installed in this image: always reports unavailable.""" + raise NotImplementedError("Inaudible watermarking (AudioSeal) is not available in this build") + + +def watermark_available() -> bool: + return False + + +if __name__ == "__main__": # verify CLI: python -m core.export_provenance file.mp3 + import sys + found = read_tags(Path(sys.argv[1])) if len(sys.argv) == 2 else {} + print(json.dumps(found, indent=2)) + sys.exit(0 if found.get("synthetic_speech") == "true" else 1) diff --git a/routes/magazine.py b/routes/magazine.py index 40cf73e..2d2dff4 100644 --- a/routes/magazine.py +++ b/routes/magazine.py @@ -24,6 +24,7 @@ import io import json import logging import re +import shutil import statistics import time import uuid @@ -3296,8 +3297,41 @@ def _article_download_filename(meta: dict, article: dict) -> str: return (name[:180] or article.get("id") or "article") + ".mp3" +def _export_mp3(mp3: Path, tmpdir: Path, provenance: str, voice: str) -> Path: + """The MP3 to export: a tagged copy when provenance tags are on (query param, else the setting).""" + from core import export_provenance as ep + from core.constants import __version__ + on = bool(_load_settings().get("export_provenance_default", True)) if provenance == "" else provenance == "1" + if not on: + return mp3 + name = "" + if voice == "1": + segs = (mag.read_audio_timings(mp3) or {}).get("segments") or [] + name = next((x.get("voice") for x in segs if isinstance(x, dict) and x.get("voice")), "") or "" + try: + return ep.apply_tags(mp3, tmpdir / mp3.name, ep.build_tags(__version__, name, voice == "1")) + except (RuntimeError, OSError, __import__("subprocess").SubprocessError): + raise HTTPException(500, "Could not add provenance tags to the export") + + +@router.post("/api/export/provenance") +async def export_provenance_verify(request: Request): + """Verify: read the provenance tags back from an uploaded audio file (raw body, 100 MB cap).""" + import tempfile + from core import export_provenance as ep + body = await request.body() + if not body or len(body) > 100 * 1024 * 1024: + raise HTTPException(400, "Send an audio file of up to 100 MB as the request body") + with tempfile.TemporaryDirectory(prefix="prov-") as t: + f = Path(t) / "in.bin" + f.write_bytes(body) + tags = await asyncio.to_thread(ep.read_tags, f) + return {"tags": tags, "synthetic": tags.get("synthetic_speech") == "true", + "watermark_available": ep.watermark_available()} + + @router.get("/api/magazine/{doc_id}/articles/{article_id}/audio") -async def magazine_article_audio_file(doc_id: str, article_id: str): +async def magazine_article_audio_file(doc_id: str, article_id: str, provenance: str = "", voice: str = ""): d, meta = _require_magazine(doc_id) article_id = _article_id(article_id) path = d / "audio" / f"{article_id}.mp3" @@ -3305,7 +3339,16 @@ async def magazine_article_audio_file(doc_id: str, article_id: str): raise HTTPException(404, "No audio generated yet") articles = mag.load_articles(d) article = next((a for a in articles if a.get("id") == article_id), {"id": article_id}) - return FileResponse(path, media_type="audio/mpeg", filename=_article_download_filename(meta, article)) + from starlette.background import BackgroundTask + import tempfile + tmp = Path(tempfile.mkdtemp(prefix="mag-prov-")) + try: + out = await asyncio.to_thread(_export_mp3, path, tmp, provenance, voice) + except BaseException: + shutil.rmtree(tmp, ignore_errors=True) + raise + return FileResponse(out, media_type="audio/mpeg", filename=_article_download_filename(meta, article), + background=BackgroundTask(shutil.rmtree, tmp, ignore_errors=True)) def _audio_generate_batch_sync(doc_id: str, job_id: str, article_ids: list[str], voice: str, @@ -3359,7 +3402,7 @@ async def magazine_generate_batch_audio(doc_id: str, request: Request): @router.get("/api/magazine/{doc_id}/epub") -async def magazine_epub(doc_id: str, article: str = ""): +async def magazine_epub(doc_id: str, article: str = "", provenance: str = "", voice: str = ""): """The issue (or one article) as an EPUB 3 with Media Overlays: text that highlights as the MP3 plays. Only approved articles that already have audio are included, in listing order. Sentence timings are @@ -3378,6 +3421,8 @@ async def magazine_epub(doc_id: str, article: str = ""): if not articles: raise HTTPException(404, "Article not found") parts = [] + import tempfile + tagdir = Path(tempfile.mkdtemp(prefix="mag-prov-")) for a in articles: mp3 = d / "audio" / f"{a['id']}.mp3" if a.get("status") != "approved" or not mp3.is_file(): @@ -3391,12 +3436,13 @@ async def magazine_epub(doc_id: str, article: str = ""): continue # an unreadable MP3 cannot be given timings at all duration_ms = int(length * 1000) parts.append({"title": a.get("title") or "", "author": a.get("author") or "", "blocks": blocks, - "mp3": mp3, "duration_ms": duration_ms, "timings": timings, + "mp3": await asyncio.to_thread(_export_mp3, mp3, tagdir, provenance, voice), + "duration_ms": duration_ms, "timings": timings, "spoken_chars": mag.spoken_chars_for(blocks, timings)}) if not parts: + shutil.rmtree(tagdir, ignore_errors=True) raise HTTPException(404, "No approved article has audio yet") language = mag.detect_article_language(b["text"] for p in parts for b in p["blocks"]) - import tempfile tmp = tempfile.NamedTemporaryFile(prefix="mag-epub-", suffix=".epub", delete=False) tmp.close() out = Path(tmp.name) @@ -3406,17 +3452,19 @@ async def magazine_epub(doc_id: str, article: str = ""): articles=parts, identifier=f"urn:uuid:{doc_id[:8]}-{doc_id[8:12]}-{doc_id[12:16]}-{doc_id[16:20]}-{doc_id[20:32]}") except ValueError as exc: out.unlink(missing_ok=True) + shutil.rmtree(tagdir, ignore_errors=True) raise HTTPException(404, str(exc)) except BaseException: out.unlink(missing_ok=True) + shutil.rmtree(tagdir, ignore_errors=True) raise filename = re.sub(r"[^A-Za-z0-9._ -]+", "_", str(meta.get("title") or "magazine")).strip(" ._")[:80] or "magazine" return FileResponse(out, media_type="application/epub+zip", filename=f"{filename}.epub", - background=BackgroundTask(lambda: out.unlink(missing_ok=True))) + background=BackgroundTask(lambda: (out.unlink(missing_ok=True), shutil.rmtree(tagdir, ignore_errors=True)))) @router.get("/api/magazine/{doc_id}/audio.zip") -async def magazine_audio_zip(doc_id: str): +async def magazine_audio_zip(doc_id: str, provenance: str = "", voice: str = ""): d, meta = _require_magazine(doc_id) audio_dir = d / "audio" if not audio_dir.is_dir(): @@ -3431,7 +3479,8 @@ async def magazine_audio_zip(doc_id: str): if not ordered: raise HTTPException(404, "No audio generated yet") buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf: + import tempfile + with tempfile.TemporaryDirectory(prefix="mag-prov-") as tagdir, zipfile.ZipFile(buf, "w", zipfile.ZIP_STORED) as zf: for i, a in enumerate(ordered, start=1): page = (a.get("pages") or [0])[0] name = _article_download_filename(meta, a) @@ -3439,7 +3488,8 @@ async def magazine_audio_zip(doc_id: str): # by filename gives the reading order even where two articles # start on the same page. arcname = f"{i:03d} - " + name - zf.write(audio_dir / f"{a['id']}.mp3", arcname=arcname) + zf.write(await asyncio.to_thread(_export_mp3, audio_dir / f"{a['id']}.mp3", Path(tagdir), provenance, voice), + arcname=arcname) buf.seek(0) safe_title = re.sub(r"[^\w \-]+", "", meta.get("title") or "magazine").strip() or "magazine" return Response(content=buf.getvalue(), media_type="application/zip", diff --git a/static/js/magazine.js b/static/js/magazine.js index 829afa8..f5db6cc 100644 --- a/static/js/magazine.js +++ b/static/js/magazine.js @@ -1543,18 +1543,42 @@ async function magGenerateAllAudio(force = false) { if (!$('mag-playlist')?.hidden) renderPlaylistTracks(); } +// Provenance tags on exported audio: per-export checkboxes, default from the saved setting. +let _magProvInit = false; +function magProvQuery() { + const on = $('mag-prov-tags')?.checked !== false; + return '?provenance=' + (on ? '1' : '0') + ($('mag-prov-voice')?.checked ? '&voice=1' : ''); +} +function magProvChanged() { + magUpdateAudioZipLink(); + fetch('/api/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ export_provenance_default: $('mag-prov-tags')?.checked !== false }) }).catch(() => {}); +} +async function magProvInit() { + if (_magProvInit) return; + _magProvInit = true; + try { + const s = await fetch('/api/settings').then((r) => r.json()); + const box = $('mag-prov-tags'); + if (box) box.checked = s.export_provenance_default !== false; + } catch (e) { /* default stays on */ } + magUpdateAudioZipLink(); +} + function magUpdateAudioZipLink() { + magProvInit(); const link = $('mag-audio-zip-link'); if (!link) return; + const q = magProvQuery(); const any = state.articles.some((a) => a.has_audio); link.hidden = !any; - if (any) link.href = '/api/magazine/' + state.doc.id + '/audio.zip'; + if (any) link.href = '/api/magazine/' + state.doc.id + '/audio.zip' + q; // The EPUB carries only approved articles that have audio. const epub = $('mag-epub-link'); if (epub) { const ready = state.articles.some((a) => a.has_audio && a.status === 'approved'); epub.hidden = !ready; - if (ready) epub.href = '/api/magazine/' + state.doc.id + '/epub'; + if (ready) epub.href = '/api/magazine/' + state.doc.id + '/epub' + q; } } @@ -6965,6 +6989,7 @@ root.magChecksToggled = magChecksToggled; root.magAcrSave = magAcrSave; root.magAcrResolve = magAcrResolve; root.magInit = magInit; +root.magProvChanged = magProvChanged; MAG.magazineApi = { load: magLoadLibrary, refresh: magRefreshArticles, splitSentences, fmtPages }; if (document.readyState === 'loading') { diff --git a/static/sections/s-magazine.html b/static/sections/s-magazine.html index 3516c65..ab24db3 100644 --- a/static/sections/s-magazine.html +++ b/static/sections/s-magazine.html @@ -159,6 +159,12 @@ title="An EPUB 3 book whose text highlights while the audio plays (Media Overlays). Approved articles that already have audio."> EPUB with audio + +