From 690673625db9da2362b48d6b29cdfeeaa75e2de5 Mon Sep 17 00:00:00 2001 From: mARTin-B78 Date: Fri, 4 Sep 2026 09:48:37 +0200 Subject: [PATCH] feat: persist background job result artifacts --- CHANGELOG.md | 3 +- TODO.md | 3 +- VERSION | 2 +- core/database.py | 64 +++++++++++++++++++++++++++++++++++++---- routes/conversation.py | 26 ++++++++++++++++- routes/jobs.py | 29 ++++++++++++++++++- routes/tts.py | 15 +++++++++- static/dist/main.min.js | 2 +- static/js/jobs.js | 13 +++++++++ static/style.css | 1 + tests/test_security.py | 25 ++++++++++++++++ 11 files changed, 171 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7fe75..65054c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) · versioned wi ### Added - **Background Job History Panel.** The Benchmark section now shows sanitized, restart-safe background-job status, progress, elapsed time, errors, and cancellation controls. Replay is deliberately limited to reopening the source - workflow because prompts, text, audio, and image inputs are never retained for implicit retries. + workflow because prompts, text, audio, and image inputs are never retained for implicit retries. Completed voice-design + audio and consistent-image outputs are now stored as bounded atomic artifacts with authenticated download links. - **Protected Backup Recovery.** Added an authenticated backup inventory and restore API for character and rehearsal snapshots, with path-safe file selection and atomic pre-mutation backups. - **Voice Trash Recovery.** Voice deletes and group deletes now expose a path-safe trash inventory and restore API, diff --git a/TODO.md b/TODO.md index 415dbd1..c07ae9f 100644 --- a/TODO.md +++ b/TODO.md @@ -67,7 +67,8 @@ the first block before investing in architecture or polish. and transcription, voice design, and casting now have optional ID/status/cancel/timeout paths while preserving synchronous callers. Job metadata now persists in SQLite and marks interrupted work after restart; durable binary result storage remains; outbound TTS and LLM calls now share a bounded transient-failure retry policy, and `/api/jobs` - exposes sanitized recent job history. + exposes sanitized recent job history. Completed voice-design audio and consistent-image outputs now use bounded, + atomic artifact files with authenticated download URLs; transcription text remains deliberately non-persistent. - [ ] **L — Run the Book 2 generalization regression.** Added a neutral multi-speaker fixture and parser regression; a real Book 2 auto-cast/attribution output still needs to be captured and reviewed to confirm rules are not overfit to *Der Sturm*. diff --git a/VERSION b/VERSION index ae96cc7..2f4320f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.24.3 +1.24.4 diff --git a/core/database.py b/core/database.py index 0417cc5..d777e0a 100644 --- a/core/database.py +++ b/core/database.py @@ -10,6 +10,8 @@ from __future__ import annotations import json import os +import re +import shutil import sqlite3 import tempfile import time @@ -22,6 +24,8 @@ from core.constants import CONFIG_DIR from core.validation import _validate_character_record _DB_PATH = CONFIG_DIR / "tts_creator.db" +_BACKGROUND_ARTIFACT_DIR = CONFIG_DIR / "background-artifacts" +_BACKGROUND_ARTIFACT_MAX_BYTES = 50 * 1024 * 1024 _DDL = """ CREATE TABLE IF NOT EXISTS characters ( @@ -246,6 +250,55 @@ def _safe_background_result(value: Any) -> str | None: return _j(clean(value))[:12000] +def background_job_store_artifact(job_id: str, name: str, payload: bytes, media_type: str = "application/octet-stream") -> dict: + """Persist one bounded binary result outside SQLite and return safe metadata.""" + safe_job = str(job_id or "") + safe_name = str(name or "") + if not re.fullmatch(r"[A-Za-z0-9_-]{1,80}", safe_job) or not re.fullmatch(r"[A-Za-z0-9_-]+\.[A-Za-z0-9]{1,8}", safe_name): + raise ValueError("invalid background artifact name") + data = bytes(payload or b"") + if not data: + raise ValueError("background artifact is empty") + if len(data) > _BACKGROUND_ARTIFACT_MAX_BYTES: + raise ValueError("background artifact exceeds 50 MB") + target_dir = _BACKGROUND_ARTIFACT_DIR / safe_job + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / safe_name + fd, tmp_name = tempfile.mkstemp(prefix=f".{safe_name}.", suffix=".tmp", dir=target_dir) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_name, target) + except Exception: + try: os.unlink(tmp_name) + except OSError: pass + raise + return {"name": safe_name, "media_type": str(media_type or "application/octet-stream")[:100], "size": len(data)} + + +def background_job_artifact_path(job_id: str, name: str) -> Path | None: + """Resolve an artifact only when it belongs to an existing job.""" + if not re.fullmatch(r"[A-Za-z0-9_-]{1,80}", str(job_id or "")) or not re.fullmatch(r"[A-Za-z0-9_-]+\.[A-Za-z0-9]{1,8}", str(name or "")): + return None + with _open() as conn: + exists = conn.execute("SELECT 1 FROM background_jobs WHERE job_id=?", (str(job_id),)).fetchone() + if exists is None: + return None + target = (_BACKGROUND_ARTIFACT_DIR / str(job_id) / str(name)).resolve() + try: target.relative_to(_BACKGROUND_ARTIFACT_DIR.resolve()) + except ValueError: return None + return target if target.is_file() else None + + +def _remove_background_artifacts(job_id: str) -> None: + """Remove artifacts when the corresponding bounded history row expires.""" + safe_job = str(job_id or "") + if re.fullmatch(r"[A-Za-z0-9_-]{1,80}", safe_job): + shutil.rmtree(_BACKGROUND_ARTIFACT_DIR / safe_job, ignore_errors=True) + + def background_job_create(job_id: str, kind: str, *, fingerprint: str = "", created: float | None = None) -> None: now = float(created if created is not None else time.time()) with _open() as conn: @@ -254,11 +307,12 @@ def background_job_create(job_id: str, kind: str, *, fingerprint: str = "", crea "VALUES (?,?,?,?,?,?,?,?,?,?,?)", (str(job_id), str(kind)[:80], "queued", "Queued", 0, 0, None, None, str(fingerprint)[:128], now, now), ) - conn.execute( - "DELETE FROM background_jobs WHERE job_id IN (" - "SELECT job_id FROM background_jobs WHERE done=1 ORDER BY updated ASC " - "LIMIT -1 OFFSET 255)" - ) + expired = conn.execute( + "SELECT job_id FROM background_jobs WHERE done=1 ORDER BY updated ASC LIMIT -1 OFFSET 255" + ).fetchall() + for row in expired: + conn.execute("DELETE FROM background_jobs WHERE job_id=?", (row["job_id"],)) + _remove_background_artifacts(row["job_id"]) conn.commit() diff --git a/routes/conversation.py b/routes/conversation.py index efa3dbe..ba379be 100644 --- a/routes/conversation.py +++ b/routes/conversation.py @@ -33,7 +33,7 @@ from core.voice import ( _voice_audio_files, _is_internal_voice_file, ) from core.voice_index import rebuild_voice_index -from core.database import background_job_create, background_job_get, background_job_update +from core.database import background_job_create, background_job_get, background_job_store_artifact, background_job_update from core.retry import request_with_retries from core.tts_helpers import _preview_request_audio from routes.stt import _transcribe_audio, _clean_stt_backend @@ -192,6 +192,29 @@ def _cij_set(job_id: str, **fields) -> None: error=fields.get("error"), result=fields.get("result")) +def _persist_consistent_image_artifacts(job_id: str, result: dict) -> dict: + """Move generated data-URI images to bounded durable files before status storage.""" + persisted = dict(result or {}) + artifacts = [] + for key, filename in (("profile", "profile.png"), ("reference", "reference.png")): + value = persisted.get(key) + if not isinstance(value, str) or not value.startswith("data:") or ";base64," not in value: + continue + header, encoded = value.split(",", 1) + media_type = header[5:].split(";", 1)[0] or "image/png" + try: + payload = base64.b64decode(encoded, validate=True) + artifacts.append({"kind": key, **background_job_store_artifact(job_id, filename, payload, media_type)}) + persisted.pop(key, None) + except Exception as exc: + persisted.pop(key, None) + persisted.setdefault("artifact_error", "One or more generated images could not be persisted") + print(f"[background-artifact] {key} persistence failed: {exc}") + if artifacts: + persisted["artifacts"] = artifacts + return persisted + + class _ConsistentImageCancelled(Exception): """Internal cooperative cancellation signal for a consistent-image worker.""" @@ -3403,6 +3426,7 @@ async def character_generate_consistent_images(request: Request): result["prompt_version"] = _CHARACTER_VISUAL_PROMPT_VERSION result["identity_consistent"] = True result["identity_fingerprint"] = _character_identity_fingerprint(sheet) + result = _persist_consistent_image_artifacts(job_id, result) _cij_set(job_id, stage="done", detail="Done", done=True, result=result) except HTTPException as exc: _cij_set(job_id, stage="error", detail="Failed", done=True, error=str(exc.detail)) diff --git a/routes/jobs.py b/routes/jobs.py index df1146c..606665b 100644 --- a/routes/jobs.py +++ b/routes/jobs.py @@ -4,8 +4,9 @@ from __future__ import annotations import time from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse -from core.database import background_job_get, background_job_list +from core.database import background_job_artifact_path, background_job_get, background_job_list router = APIRouter() @@ -15,6 +16,18 @@ def _public_job(job: dict) -> dict: payload.pop("fingerprint", None) payload.pop("created", None) payload.pop("updated", None) + result = payload.get("result") + if isinstance(result, dict): + result = dict(result) + if isinstance(result.get("artifact"), dict) and result["artifact"].get("name"): + result["artifact"] = {**result["artifact"], "url": f"/api/jobs/{job.get('job_id')}/artifacts/{result['artifact']['name']}"} + if isinstance(result.get("artifacts"), list): + result["artifacts"] = [ + {**item, "url": f"/api/jobs/{job.get('job_id')}/artifacts/{item['name']}"} + if isinstance(item, dict) and item.get("name") else item + for item in result["artifacts"] + ] + payload["result"] = result payload["elapsed"] = round(time.time() - float(job.get("created") or time.time()), 1) return payload @@ -30,3 +43,17 @@ async def jobs_get(job_id: str): if job is None: raise HTTPException(404, "Unknown or expired job") return _public_job(job) + + +@router.get("/api/jobs/{job_id}/artifacts/{name}") +async def jobs_artifact(job_id: str, name: str): + path = background_job_artifact_path(job_id, name) + if path is None: + raise HTTPException(404, "Unknown or expired job artifact") + media_type = "application/octet-stream" + suffix = path.suffix.lower() + if suffix == ".wav": media_type = "audio/wav" + elif suffix == ".mp3": media_type = "audio/mpeg" + elif suffix == ".png": media_type = "image/png" + elif suffix in {".jpg", ".jpeg"}: media_type = "image/jpeg" + return FileResponse(path, media_type=media_type, headers={"Cache-Control": "private, no-store"}) diff --git a/routes/tts.py b/routes/tts.py index 92e302f..a351489 100644 --- a/routes/tts.py +++ b/routes/tts.py @@ -49,7 +49,7 @@ from core.tts_helpers import ( _fishspeech_request_audio, ) from core.audio import _duration, _export_normalized_wav -from core.database import background_job_create, background_job_get, background_job_update +from core.database import background_job_create, background_job_get, background_job_store_artifact, background_job_update from core.docker_client import _docker_available, _docker_post logger = logging.getLogger("uvicorn.error") @@ -151,6 +151,19 @@ async def _run_voice_design_job(job_id: str, data: dict, idempotency_key: str, r _voice_design_job_set(job_id, stage="cancelled", detail="Cancelled", done=True, cancelled=True, error="Voice design cancelled") else: + # The registry is process-local and its temp file is not restart-safe. + # Keep a bounded durable copy while leaving prompts and source text out + # of SQLite and out of the public job status payload. + try: + from core.registry import _registry_get + source = _registry_get(str((result or {}).get("id") or "")) + if source and source.is_file(): + result = dict(result or {}) + result["artifact"] = background_job_store_artifact(job_id, "voice.wav", source.read_bytes(), "audio/wav") + except Exception as exc: + result = dict(result or {}) + result["artifact_error"] = "Voice audio could not be persisted" + logger.warning("background voice artifact persistence failed: %s", exc) _voice_design_job_set(job_id, stage="done", detail="Done", done=True, result=result) except asyncio.TimeoutError: call.add_done_callback(lambda task: task.exception() if not task.cancelled() else None) diff --git a/static/dist/main.min.js b/static/dist/main.min.js index 9d0dc0e..997cf2d 100644 --- a/static/dist/main.min.js +++ b/static/dist/main.min.js @@ -197,7 +197,7 @@ var _a,_b,_c,_d,_e,_f,_g,_h,_i,_j,_k,_l,_m,_n,_o,_p,_q,_r,_s,_t,_u,_v,_w,_x,_y,_ ${can?'':""} - `}function render(data){const items=data.items||[];items.length?(grid.innerHTML=items.map(card).join(""),grid.querySelectorAll(".fa-card").forEach((el,i)=>{var _a3,_b3;const v=items[i];(_a3=el.querySelector(".fa-play"))==null||_a3.addEventListener("click",e=>{e.stopPropagation(),preview(v,el)}),(_b3=el.querySelector(".fa-import"))==null||_b3.addEventListener("click",e=>{e.stopPropagation(),importVoice(v,el)})})):grid.innerHTML='
No voices found \u2014 try a different search, language or filter.
';const pager=$2("fa-pager");if(pager){pager.hidden=!1,$2("fa-prev").disabled=_page<=1;const pages=Math.max(1,Math.ceil((data.total||0)/(data.page_size||24)));$2("fa-next").disabled=_page>=pages,$2("fa-pager-info").textContent=`Page ${_page} \xB7 ${(data.total||0).toLocaleString()} voices`}}async function browse(){status2("Loading voices from fish.audio\u2026"),grid.innerHTML='
Loading\u2026
';try{const d=await fetch("/api/fishaudio/voices?"+params().toString()).then(r=>{if(!r.ok)throw new Error("HTTP "+r.status);return r.json()});render(d),status2(d.offline?"\u26A0 fish.audio unreachable \u2014 showing cached results":`${(d.total||0).toLocaleString()} matching voices`,!!d.offline)}catch(e){grid.innerHTML=`
Failed to load: ${esc(e.message)}
`,status2("")}}function stopPreview(){_audio&&(_audio.pause(),_audio=null),_playingCard==null||_playingCard.classList.remove("fa-card-playing"),_playingCard=null}function preview(v,el){if(_playingCard===el){stopPreview();return}stopPreview(),v.sample_audio&&(_audio=new Audio(v.sample_audio),_playingCard=el,el.classList.add("fa-card-playing"),_audio.addEventListener("ended",stopPreview),_audio.play().catch(()=>{stopPreview(),typeof toast=="function"&&toast("Could not play preview","error")}))}async function importVoice(v,el){const btn=el.querySelector(".fa-import"),orig=btn.innerHTML;btn.disabled=!0,btn.innerHTML='';const lang=(v.language||"EN").slice(0,2).toUpperCase(),base=(typeof _umlautSafe=="function"?_umlautSafe(v.title||"fishaudio"):v.title||"fishaudio").replace(/[^A-Za-z0-9]+/g,"_").replace(/^_+|_+$/g,"").slice(0,40)||"Voice",voiceId=`${lang}_${base}`;try{const r=await fetch("/api/quick-import-voice",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({voice_id:voiceId,audio_url:v.sample_audio,transcript:v.sample_text||v.default_text||""})});if(!r.ok){const e=await r.json().catch(()=>({}));throw new Error(e.detail||r.statusText)}const d=await r.json();typeof saveMeta=="function"&&await saveMeta(d.voice_id,{name:v.title||base,tag:"fish-audio",group:"fish-audio",origin:"cloned",gender:(v.gender||"").charAt(0).toUpperCase(),note:(v.description||"").slice(0,180)}).catch(()=>{}),v.image&&await fetch("/api/voice/picture-url",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({voice_id:d.voice_id,image_url:v.image})}).catch(()=>{}),btn.innerHTML=' Imported',el.classList.add("fa-card-imported"),typeof toast=="function"&&toast(`Imported "${v.title}" as ${d.voice_id}`,"success"),typeof loadVoiceLibrary=="function"&&loadVoiceLibrary().catch(()=>{})}catch(e){btn.disabled=!1,btn.innerHTML=orig,typeof toast=="function"&&toast("Import failed: "+e.message,"error")}}(_a2=$2("fa-fetch"))==null||_a2.addEventListener("click",()=>{_page=1,browse()}),(_b2=$2("fa-search"))==null||_b2.addEventListener("keydown",e=>{e.key==="Enter"&&(_page=1,browse())}),(_c2=$2("fa-tag"))==null||_c2.addEventListener("keydown",e=>{e.key==="Enter"&&(_page=1,browse(),applyFilters())}),(_d2=$2("fa-lang"))==null||_d2.addEventListener("change",()=>{_page=1,browse()}),(_e2=$2("fa-sort"))==null||_e2.addEventListener("change",()=>{_page=1,browse()}),(_f2=$2("fa-prev"))==null||_f2.addEventListener("click",()=>{_page>1&&(_page--,browse())}),(_g2=$2("fa-next"))==null||_g2.addEventListener("click",()=>{_page++,browse()});const pop=$2("fa-filter-pop");function positionPop(){const btn=$2("fa-filter-btn");if(!pop||!btn||pop.hidden)return;const r=btn.getBoundingClientRect(),w=pop.offsetWidth||320,margin=8;let left=Math.min(r.right-w,window.innerWidth-w-margin);left{e.stopPropagation(),openPop()}),(_i2=$2("fa-filter-close"))==null||_i2.addEventListener("click",e=>{e.stopPropagation(),openPop(!1)}),pop==null||pop.addEventListener("click",e=>e.stopPropagation()),document.addEventListener("click",()=>openPop(!1)),document.addEventListener("keydown",e=>{e.key==="Escape"&&openPop(!1)}),window.addEventListener("resize",positionPop),window.addEventListener("scroll",positionPop,!0),document.querySelectorAll("#fa-filter-pop .fa-pchip").forEach(chip=>{chip.addEventListener("click",()=>{const wrap=chip.closest(".fa-pchips");wrap.classList.contains("fa-pchips-multi")?chip.classList.toggle("active"):(wrap.querySelectorAll(".fa-pchip").forEach(c=>c.classList.remove("active")),chip.classList.add("active")),updateFilterBadge()})}),(_j2=$2("fa-tag"))==null||_j2.addEventListener("input",updateFilterBadge);function applyFilters(){_page=1,openPop(!1),browse()}(_k2=$2("fa-filter-apply"))==null||_k2.addEventListener("click",applyFilters),(_l2=$2("fa-filter-reset"))==null||_l2.addEventListener("click",()=>{document.querySelectorAll('#fa-filter-pop .fa-pchips[data-group="tag"] .fa-pchip').forEach(c=>c.classList.remove("active")),["gender","age"].forEach(g=>{const wrap=document.querySelector(`#fa-filter-pop .fa-pchips[data-group="${g}"]`);wrap==null||wrap.querySelectorAll(".fa-pchip").forEach((c,i)=>c.classList.toggle("active",i===0))}),$2("fa-tag")&&($2("fa-tag").value=""),updateFilterBadge(),applyFilters()})}(),function(){const tabs=document.getElementById("gvo-tabs");if(!tabs)return;const map={direct:"tab-getvoices",fish:"fa-browser-card",eleven:"el-browser-card"};function show(src){map[src]||(src="direct"),Object.entries(map).forEach(([k,id])=>{const el=document.getElementById(id);el&&(k===src?el.style.removeProperty("display"):el.style.setProperty("display","none","important"))}),tabs.querySelectorAll(".gvo-tab").forEach(t=>t.classList.toggle("active",t.dataset.src===src));try{localStorage.setItem("gvo-src",src)}catch{}}tabs.querySelectorAll(".gvo-tab").forEach(t=>t.addEventListener("click",()=>show(t.dataset.src)));let saved="direct";try{saved=localStorage.getItem("gvo-src")||"direct"}catch{}show(saved)}(),function(){const list=document.getElementById("background-jobs-list"),refresh=document.getElementById("background-jobs-refresh");if(!list||!refresh)return;const labels={transcription:"Transcription",voice_design:"Voice design",casting:"Voice casting",consistent_images:"Consistent images"};let loading=!1;function node(value,fallback="\u2014"){const item=document.createElement("span");return item.textContent=value==null||value===""?fallback:String(value),item}function statusClass(job){return job.cancelled||job.stage==="cancelled"?"is-cancelled":job.error||job.stage==="error"||job.stage==="failed"?"is-error":job.done?"is-done":"is-active"}function statusLabel(job){return job.cancelled||job.stage==="cancelled"?"Cancelled":job.error||job.stage==="error"||job.stage==="failed"?"Failed":job.done?"Completed":job.stage==="queued"?"Queued":"Running"}function cancelEndpoint(kind,id){const paths={transcription:"/api/transcribe/cancel/",voice_design:"/api/voice-design/cancel/",casting:"/api/match-characters-voices/cancel/",consistent_images:"/api/character-generate-consistent-images/cancel/"};return paths[kind]?paths[kind]+encodeURIComponent(id):""}function render(jobs){if(list.replaceChildren(),!jobs.length){const empty=node("No background jobs yet. Start a transcription, voice design, cast, or image workflow to see it here.");empty.className="perf-history-empty",list.appendChild(empty);return}jobs.forEach(job=>{const row=document.createElement("div");row.className="background-job-row "+statusClass(job);const main=document.createElement("div");main.className="background-job-main";const title=document.createElement("strong");title.textContent=labels[job.kind]||job.kind||"Background job",main.appendChild(title);const id=document.createElement("code");id.textContent=String(job.job_id||"").slice(0,12),main.appendChild(id);const detail=node(job.error||job.detail||job.stage);detail.className="background-job-detail",main.appendChild(detail);const meta=node(`${statusLabel(job)} \xB7 ${Number(job.elapsed||0).toFixed(1)}s`);meta.className="background-job-meta",main.appendChild(meta),row.appendChild(main);const actions=document.createElement("div");if(actions.className="background-job-actions",!job.done&&!job.cancelled&&job.job_id){const endpoint=cancelEndpoint(job.kind,job.job_id);if(endpoint){const button=document.createElement("button");button.type="button",button.className="btn-secondary btn-sm",button.textContent="Cancel",button.addEventListener("click",async()=>{button.disabled=!0;try{const response=await fetch(endpoint,{method:"POST"});if(!response.ok)throw new Error(`HTTP ${response.status}`);await load()}catch(error){button.disabled=!1,typeof toast=="function"&&toast("Could not cancel job: "+error.message,"error")}}),actions.appendChild(button)}}const note=node("Reopen source to retry");note.className="background-job-retry-note",note.title="Inputs are not retained for replay, by design.",actions.appendChild(note),row.appendChild(actions),list.appendChild(row)})}async function load(){if(!loading){loading=!0,refresh.disabled=!0;try{const response=await fetch("/api/jobs?limit=50",{cache:"no-store"});if(!response.ok)throw new Error(`HTTP ${response.status}`);const payload=await response.json();render(Array.isArray(payload.jobs)?payload.jobs:[])}catch(error){list.replaceChildren();const empty=node("Background job history unavailable: "+error.message);empty.className="perf-history-empty",list.appendChild(empty)}finally{loading=!1,refresh.disabled=!1}}}refresh.addEventListener("click",load),load(),setInterval(()=>{var _a2;(_a2=document.getElementById("s-performance"))!=null&&_a2.classList.contains("is-active")&&load()},1e4)}();function cleanBaseUrl(url){return String(url||"").trim().replace(/\/+$/,"")}function getTtsBaseUrl(){var _a2;return cleanBaseUrl((_a2=$("s-tts-url"))==null?void 0:_a2.value)||"http://localhost:8020"}function getTtsV1Url(){const base=getTtsBaseUrl();return base.endsWith("/v1")?base:base+"/v1"}function getTtsStreamBaseUrl(){var _a2;return cleanBaseUrl(((_a2=$("s-tts-stream-url"))==null?void 0:_a2.value)||_appSettings.tts_stream_url)||"http://localhost:8023"}function getTtsStreamV1Url(){const base=getTtsStreamBaseUrl();return base.endsWith("/v1")?base:base+"/v1"}function getCreatorV1Url(){const loc=window.location,protocol=loc.protocol||"http:",port=loc.port?":"+loc.port:"",host=loc.hostname==="0.0.0.0"?"localhost":loc.hostname;return`${protocol}//${host}${port}/v1`}function updateCreatorUrlHints(){const warning=$("routing-url-warning"),badBindHost=window.location.hostname==="0.0.0.0";warning&&warning.classList.toggle("show",badBindHost)}function activeVoiceIds(){return(_voices||[]).filter(v=>v.enabled!==!1).slice().sort((a,b)=>a.id.localeCompare(b.id)).map(v=>v.id)}function integrationVoiceExample(){return activeVoiceIds()[0]||"EN_F_ExampleVoice"}function integrationVoiceList(){const ids=activeVoiceIds();return ids.length?ids.join(", "):"EN_F_ExampleVoice, DE_M_ExampleVoice"}function virtualDesignVoiceIds(){return Object.keys(loadDesignPresets?loadDesignPresets():{}).sort((a,b)=>a.localeCompare(b)).map(name=>"vd_"+(typeof _umlautSafe=="function"?_umlautSafe(name):name).replace(/[^A-Za-z0-9_.-]+/g,"_").replace(/^_+|_+$/g,""))}function renderIntegrationSnippets(){if(!$("snippet-sillytavern"))return;const base=getTtsBaseUrl(),v1=getTtsV1Url(),streamV1=getTtsStreamV1Url(),proxyV1=getCreatorV1Url(),proxyBase=proxyV1.replace(/\/v1$/,"");updateCreatorUrlHints();const voice=integrationVoiceExample(),voices=integrationVoiceList(),vdVoices=virtualDesignVoiceIds(),vdVoice=vdVoices[0]||"vd_EN_F_Warm_Narrator";$("integration-url-label").textContent="TTS backend: "+base,$("snippet-sillytavern").textContent=`Provider: OpenAI compatible TTS + `}function render(data){const items=data.items||[];items.length?(grid.innerHTML=items.map(card).join(""),grid.querySelectorAll(".fa-card").forEach((el,i)=>{var _a3,_b3;const v=items[i];(_a3=el.querySelector(".fa-play"))==null||_a3.addEventListener("click",e=>{e.stopPropagation(),preview(v,el)}),(_b3=el.querySelector(".fa-import"))==null||_b3.addEventListener("click",e=>{e.stopPropagation(),importVoice(v,el)})})):grid.innerHTML='
No voices found \u2014 try a different search, language or filter.
';const pager=$2("fa-pager");if(pager){pager.hidden=!1,$2("fa-prev").disabled=_page<=1;const pages=Math.max(1,Math.ceil((data.total||0)/(data.page_size||24)));$2("fa-next").disabled=_page>=pages,$2("fa-pager-info").textContent=`Page ${_page} \xB7 ${(data.total||0).toLocaleString()} voices`}}async function browse(){status2("Loading voices from fish.audio\u2026"),grid.innerHTML='
Loading\u2026
';try{const d=await fetch("/api/fishaudio/voices?"+params().toString()).then(r=>{if(!r.ok)throw new Error("HTTP "+r.status);return r.json()});render(d),status2(d.offline?"\u26A0 fish.audio unreachable \u2014 showing cached results":`${(d.total||0).toLocaleString()} matching voices`,!!d.offline)}catch(e){grid.innerHTML=`
Failed to load: ${esc(e.message)}
`,status2("")}}function stopPreview(){_audio&&(_audio.pause(),_audio=null),_playingCard==null||_playingCard.classList.remove("fa-card-playing"),_playingCard=null}function preview(v,el){if(_playingCard===el){stopPreview();return}stopPreview(),v.sample_audio&&(_audio=new Audio(v.sample_audio),_playingCard=el,el.classList.add("fa-card-playing"),_audio.addEventListener("ended",stopPreview),_audio.play().catch(()=>{stopPreview(),typeof toast=="function"&&toast("Could not play preview","error")}))}async function importVoice(v,el){const btn=el.querySelector(".fa-import"),orig=btn.innerHTML;btn.disabled=!0,btn.innerHTML='';const lang=(v.language||"EN").slice(0,2).toUpperCase(),base=(typeof _umlautSafe=="function"?_umlautSafe(v.title||"fishaudio"):v.title||"fishaudio").replace(/[^A-Za-z0-9]+/g,"_").replace(/^_+|_+$/g,"").slice(0,40)||"Voice",voiceId=`${lang}_${base}`;try{const r=await fetch("/api/quick-import-voice",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({voice_id:voiceId,audio_url:v.sample_audio,transcript:v.sample_text||v.default_text||""})});if(!r.ok){const e=await r.json().catch(()=>({}));throw new Error(e.detail||r.statusText)}const d=await r.json();typeof saveMeta=="function"&&await saveMeta(d.voice_id,{name:v.title||base,tag:"fish-audio",group:"fish-audio",origin:"cloned",gender:(v.gender||"").charAt(0).toUpperCase(),note:(v.description||"").slice(0,180)}).catch(()=>{}),v.image&&await fetch("/api/voice/picture-url",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({voice_id:d.voice_id,image_url:v.image})}).catch(()=>{}),btn.innerHTML=' Imported',el.classList.add("fa-card-imported"),typeof toast=="function"&&toast(`Imported "${v.title}" as ${d.voice_id}`,"success"),typeof loadVoiceLibrary=="function"&&loadVoiceLibrary().catch(()=>{})}catch(e){btn.disabled=!1,btn.innerHTML=orig,typeof toast=="function"&&toast("Import failed: "+e.message,"error")}}(_a2=$2("fa-fetch"))==null||_a2.addEventListener("click",()=>{_page=1,browse()}),(_b2=$2("fa-search"))==null||_b2.addEventListener("keydown",e=>{e.key==="Enter"&&(_page=1,browse())}),(_c2=$2("fa-tag"))==null||_c2.addEventListener("keydown",e=>{e.key==="Enter"&&(_page=1,browse(),applyFilters())}),(_d2=$2("fa-lang"))==null||_d2.addEventListener("change",()=>{_page=1,browse()}),(_e2=$2("fa-sort"))==null||_e2.addEventListener("change",()=>{_page=1,browse()}),(_f2=$2("fa-prev"))==null||_f2.addEventListener("click",()=>{_page>1&&(_page--,browse())}),(_g2=$2("fa-next"))==null||_g2.addEventListener("click",()=>{_page++,browse()});const pop=$2("fa-filter-pop");function positionPop(){const btn=$2("fa-filter-btn");if(!pop||!btn||pop.hidden)return;const r=btn.getBoundingClientRect(),w=pop.offsetWidth||320,margin=8;let left=Math.min(r.right-w,window.innerWidth-w-margin);left{e.stopPropagation(),openPop()}),(_i2=$2("fa-filter-close"))==null||_i2.addEventListener("click",e=>{e.stopPropagation(),openPop(!1)}),pop==null||pop.addEventListener("click",e=>e.stopPropagation()),document.addEventListener("click",()=>openPop(!1)),document.addEventListener("keydown",e=>{e.key==="Escape"&&openPop(!1)}),window.addEventListener("resize",positionPop),window.addEventListener("scroll",positionPop,!0),document.querySelectorAll("#fa-filter-pop .fa-pchip").forEach(chip=>{chip.addEventListener("click",()=>{const wrap=chip.closest(".fa-pchips");wrap.classList.contains("fa-pchips-multi")?chip.classList.toggle("active"):(wrap.querySelectorAll(".fa-pchip").forEach(c=>c.classList.remove("active")),chip.classList.add("active")),updateFilterBadge()})}),(_j2=$2("fa-tag"))==null||_j2.addEventListener("input",updateFilterBadge);function applyFilters(){_page=1,openPop(!1),browse()}(_k2=$2("fa-filter-apply"))==null||_k2.addEventListener("click",applyFilters),(_l2=$2("fa-filter-reset"))==null||_l2.addEventListener("click",()=>{document.querySelectorAll('#fa-filter-pop .fa-pchips[data-group="tag"] .fa-pchip').forEach(c=>c.classList.remove("active")),["gender","age"].forEach(g=>{const wrap=document.querySelector(`#fa-filter-pop .fa-pchips[data-group="${g}"]`);wrap==null||wrap.querySelectorAll(".fa-pchip").forEach((c,i)=>c.classList.toggle("active",i===0))}),$2("fa-tag")&&($2("fa-tag").value=""),updateFilterBadge(),applyFilters()})}(),function(){const tabs=document.getElementById("gvo-tabs");if(!tabs)return;const map={direct:"tab-getvoices",fish:"fa-browser-card",eleven:"el-browser-card"};function show(src){map[src]||(src="direct"),Object.entries(map).forEach(([k,id])=>{const el=document.getElementById(id);el&&(k===src?el.style.removeProperty("display"):el.style.setProperty("display","none","important"))}),tabs.querySelectorAll(".gvo-tab").forEach(t=>t.classList.toggle("active",t.dataset.src===src));try{localStorage.setItem("gvo-src",src)}catch{}}tabs.querySelectorAll(".gvo-tab").forEach(t=>t.addEventListener("click",()=>show(t.dataset.src)));let saved="direct";try{saved=localStorage.getItem("gvo-src")||"direct"}catch{}show(saved)}(),function(){const list=document.getElementById("background-jobs-list"),refresh=document.getElementById("background-jobs-refresh");if(!list||!refresh)return;const labels={transcription:"Transcription",voice_design:"Voice design",casting:"Voice casting",consistent_images:"Consistent images"};let loading=!1;function node(value,fallback="\u2014"){const item=document.createElement("span");return item.textContent=value==null||value===""?fallback:String(value),item}function statusClass(job){return job.cancelled||job.stage==="cancelled"?"is-cancelled":job.error||job.stage==="error"||job.stage==="failed"?"is-error":job.done?"is-done":"is-active"}function statusLabel(job){return job.cancelled||job.stage==="cancelled"?"Cancelled":job.error||job.stage==="error"||job.stage==="failed"?"Failed":job.done?"Completed":job.stage==="queued"?"Queued":"Running"}function cancelEndpoint(kind,id){const paths={transcription:"/api/transcribe/cancel/",voice_design:"/api/voice-design/cancel/",casting:"/api/match-characters-voices/cancel/",consistent_images:"/api/character-generate-consistent-images/cancel/"};return paths[kind]?paths[kind]+encodeURIComponent(id):""}function render(jobs){if(list.replaceChildren(),!jobs.length){const empty=node("No background jobs yet. Start a transcription, voice design, cast, or image workflow to see it here.");empty.className="perf-history-empty",list.appendChild(empty);return}jobs.forEach(job=>{const row=document.createElement("div");row.className="background-job-row "+statusClass(job);const main=document.createElement("div");main.className="background-job-main";const title=document.createElement("strong");title.textContent=labels[job.kind]||job.kind||"Background job",main.appendChild(title);const id=document.createElement("code");id.textContent=String(job.job_id||"").slice(0,12),main.appendChild(id);const detail=node(job.error||job.detail||job.stage);detail.className="background-job-detail",main.appendChild(detail);const meta=node(`${statusLabel(job)} \xB7 ${Number(job.elapsed||0).toFixed(1)}s`);meta.className="background-job-meta",main.appendChild(meta);const storedResult=job.result&&typeof job.result=="object"?job.result:{},artifactList=[];storedResult.artifact&&artifactList.push(storedResult.artifact),Array.isArray(storedResult.artifacts)&&artifactList.push(...storedResult.artifacts),artifactList.filter(item=>item&&typeof item.url=="string"&&item.url.startsWith("/api/jobs/")).forEach(item=>{const link=document.createElement("a");link.className="background-job-artifact",link.href=item.url,link.target="_blank",link.rel="noopener",link.textContent=`Download ${item.kind||"result"}`,main.appendChild(link)}),row.appendChild(main);const actions=document.createElement("div");if(actions.className="background-job-actions",!job.done&&!job.cancelled&&job.job_id){const endpoint=cancelEndpoint(job.kind,job.job_id);if(endpoint){const button=document.createElement("button");button.type="button",button.className="btn-secondary btn-sm",button.textContent="Cancel",button.addEventListener("click",async()=>{button.disabled=!0;try{const response=await fetch(endpoint,{method:"POST"});if(!response.ok)throw new Error(`HTTP ${response.status}`);await load()}catch(error){button.disabled=!1,typeof toast=="function"&&toast("Could not cancel job: "+error.message,"error")}}),actions.appendChild(button)}}const note=node("Reopen source to retry");note.className="background-job-retry-note",note.title="Inputs are not retained for replay, by design.",actions.appendChild(note),row.appendChild(actions),list.appendChild(row)})}async function load(){if(!loading){loading=!0,refresh.disabled=!0;try{const response=await fetch("/api/jobs?limit=50",{cache:"no-store"});if(!response.ok)throw new Error(`HTTP ${response.status}`);const payload=await response.json();render(Array.isArray(payload.jobs)?payload.jobs:[])}catch(error){list.replaceChildren();const empty=node("Background job history unavailable: "+error.message);empty.className="perf-history-empty",list.appendChild(empty)}finally{loading=!1,refresh.disabled=!1}}}refresh.addEventListener("click",load),load(),setInterval(()=>{var _a2;(_a2=document.getElementById("s-performance"))!=null&&_a2.classList.contains("is-active")&&load()},1e4)}();function cleanBaseUrl(url){return String(url||"").trim().replace(/\/+$/,"")}function getTtsBaseUrl(){var _a2;return cleanBaseUrl((_a2=$("s-tts-url"))==null?void 0:_a2.value)||"http://localhost:8020"}function getTtsV1Url(){const base=getTtsBaseUrl();return base.endsWith("/v1")?base:base+"/v1"}function getTtsStreamBaseUrl(){var _a2;return cleanBaseUrl(((_a2=$("s-tts-stream-url"))==null?void 0:_a2.value)||_appSettings.tts_stream_url)||"http://localhost:8023"}function getTtsStreamV1Url(){const base=getTtsStreamBaseUrl();return base.endsWith("/v1")?base:base+"/v1"}function getCreatorV1Url(){const loc=window.location,protocol=loc.protocol||"http:",port=loc.port?":"+loc.port:"",host=loc.hostname==="0.0.0.0"?"localhost":loc.hostname;return`${protocol}//${host}${port}/v1`}function updateCreatorUrlHints(){const warning=$("routing-url-warning"),badBindHost=window.location.hostname==="0.0.0.0";warning&&warning.classList.toggle("show",badBindHost)}function activeVoiceIds(){return(_voices||[]).filter(v=>v.enabled!==!1).slice().sort((a,b)=>a.id.localeCompare(b.id)).map(v=>v.id)}function integrationVoiceExample(){return activeVoiceIds()[0]||"EN_F_ExampleVoice"}function integrationVoiceList(){const ids=activeVoiceIds();return ids.length?ids.join(", "):"EN_F_ExampleVoice, DE_M_ExampleVoice"}function virtualDesignVoiceIds(){return Object.keys(loadDesignPresets?loadDesignPresets():{}).sort((a,b)=>a.localeCompare(b)).map(name=>"vd_"+(typeof _umlautSafe=="function"?_umlautSafe(name):name).replace(/[^A-Za-z0-9_.-]+/g,"_").replace(/^_+|_+$/g,""))}function renderIntegrationSnippets(){if(!$("snippet-sillytavern"))return;const base=getTtsBaseUrl(),v1=getTtsV1Url(),streamV1=getTtsStreamV1Url(),proxyV1=getCreatorV1Url(),proxyBase=proxyV1.replace(/\/v1$/,"");updateCreatorUrlHints();const voice=integrationVoiceExample(),voices=integrationVoiceList(),vdVoices=virtualDesignVoiceIds(),vdVoice=vdVoices[0]||"vd_EN_F_Warm_Narrator";$("integration-url-label").textContent="TTS backend: "+base,$("snippet-sillytavern").textContent=`Provider: OpenAI compatible TTS API base URL: ${v1} API key: dummy Model: qwen3-tts diff --git a/static/js/jobs.js b/static/js/jobs.js index ea49175..468e09f 100644 --- a/static/js/jobs.js +++ b/static/js/jobs.js @@ -54,6 +54,19 @@ const meta = node(`${statusLabel(job)} · ${Number(job.elapsed || 0).toFixed(1)}s`); meta.className = 'background-job-meta'; main.appendChild(meta); + const storedResult = job.result && typeof job.result === 'object' ? job.result : {}; + const artifactList = []; + if (storedResult.artifact) artifactList.push(storedResult.artifact); + if (Array.isArray(storedResult.artifacts)) artifactList.push(...storedResult.artifacts); + artifactList.filter(item => item && typeof item.url === 'string' && item.url.startsWith('/api/jobs/')).forEach(item => { + const link = document.createElement('a'); + link.className = 'background-job-artifact'; + link.href = item.url; + link.target = '_blank'; + link.rel = 'noopener'; + link.textContent = `Download ${item.kind || 'result'}`; + main.appendChild(link); + }); row.appendChild(main); const actions = document.createElement('div'); actions.className = 'background-job-actions'; diff --git a/static/style.css b/static/style.css index afecbf6..111e40b 100644 --- a/static/style.css +++ b/static/style.css @@ -3741,6 +3741,7 @@ code { background: var(--panel); border-radius: 4px; padding: 1px 5px; font-fami .background-job-main strong { font-size: 13px; } .background-job-main code { color: var(--subtext); font-size: 10px; } .background-job-detail, .background-job-meta { grid-column: 1 / -1; color: var(--subtext); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.background-job-artifact { grid-column: 1 / -1; color: var(--accent); font-size: 11px; } .background-job-row.is-error .background-job-detail { color: var(--red); } .background-job-row.is-done .background-job-meta { color: var(--green, #22c55e); } .background-job-actions { display: flex; align-items: center; gap: 9px; flex-shrink: 0; } diff --git a/tests/test_security.py b/tests/test_security.py index 976d109..d9496a1 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -225,6 +225,31 @@ class SecurityValidationTests(unittest.TestCase): conn.execute("DELETE FROM background_jobs WHERE job_id=?", (job_id,)) conn.commit() + def test_background_job_artifact_is_atomic_bounded_and_job_scoped(self): + from core.database import background_job_artifact_path, background_job_create, background_job_store_artifact, background_job_update + job_id = "artifact-job-test" + background_job_create(job_id, "test") + try: + meta = background_job_store_artifact(job_id, "result.wav", b"RIFF-test", "audio/wav") + self.assertEqual(meta["name"], "result.wav") + path = background_job_artifact_path(job_id, "result.wav") + self.assertIsNotNone(path) + self.assertEqual(path.read_bytes(), b"RIFF-test") + background_job_update(job_id, stage="done", done=True, result={"artifact": meta}) + from routes.jobs import jobs_artifact, jobs_get + self.assertEqual(asyncio.run(jobs_get(job_id))["result"]["artifact"]["url"], f"/api/jobs/{job_id}/artifacts/result.wav") + self.assertEqual(asyncio.run(jobs_artifact(job_id, "result.wav")).path, path) + self.assertIsNone(background_job_artifact_path(job_id, "../result.wav")) + with self.assertRaises(ValueError): + background_job_store_artifact(job_id, "bad.bin", b"x" * (50 * 1024 * 1024 + 1)) + finally: + from core.database import _open, _BACKGROUND_ARTIFACT_DIR + with _open() as conn: + conn.execute("DELETE FROM background_jobs WHERE job_id=?", (job_id,)) + conn.commit() + import shutil + shutil.rmtree(_BACKGROUND_ARTIFACT_DIR / job_id, ignore_errors=True) + def test_background_job_persistence_prunes_old_completed_rows(self): from core.database import _open, background_job_create, background_job_update prefix = "prune-job-"