A page needing OCR (or whose already-extracted text -- native or
tesseract alike -- scores as mostly garbage via ocr_quality_score) is
now escalated to a connected vision OCR engine before falling back to
tesseract. Falls back automatically on any failure (unreachable engine,
timeout, unparseable output); never fatal to ingestion.
Verified live against ocr-baidu-unlimited-ocr on a real 28-page scanned
c't magazine excerpt (Epson ES-580W, no scanner OCR): every garbled
article title tesseract produced ("K | soll Arbeit abnehmen", ". |",
"@)X", "Owns mee") was replaced by the real headline ("1. Irrtum: KI
ist ein allgemeiner Jobkiller", etc.), and furniture detection jumped
from 44 to 151 blocks because the engine labels running headers/
footers/page numbers directly instead of needing fuzzy post-hoc
matching. 26 of 28 pages used the vision path; the remaining 2 fell
back to tesseract automatically.
Also fixes two real defects found investigating the same real-world
data, both regression-tested against the actual garbled strings
observed:
- Exact-match repeated-line detection (running header/footer) silently
failed once tesseract OCR'd the same line differently each time --
cluster_running_furniture fuzzy-matches near-edge blocks instead.
- ocr_page() built one Block per OCR text-LINE rather than per
paragraph, so a single anomalous line (a chart number, scan noise, a
wrapped headline continuation) could independently trigger title
classification -- now grouped by paragraph like the native PyMuPDF
path, using the paragraph's median line height rather than its total
bbox height for font size.
A vision-engine label (footer/title/page_number/caption) is trusted
outright and skips classify_blocks' font-size heuristics entirely,
since the engine gives no reliable font metric; an ambiguous "text"
region gets a synthesized estimate from its paragraph area and length
(estimate_font_size_from_area) and still goes through normal
classification.
Off unless vision_ocr_url/vision_ocr_model are configured
(vision_ocr_enabled setting, default on if reachable); per-request
override via the ingest endpoint body. Default engine URL uses
host.docker.internal, confirmed live -- the OCR container runs on its
own Docker network, not dgx_net, so a container-name default would not
resolve from the app container.
28 new tests (136 total in the module). ocr_quality_score is
deliberately coarse and documented as such: it catches text that is
mostly symbol noise, not a plausible single-word misread inside an
otherwise normal sentence (confirmed live: a misread "KI" -> "K |"
still scored 0.81) -- that class of error needs a dictionary or
language-model check, out of scope here.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
152 lines
8.1 KiB
Python
152 lines
8.1 KiB
Python
"""Boot-time defaults, path constants, in-memory log buffer, and routing log."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
# ── Version ───────────────────────────────────────────────────────────────────
|
|
|
|
def _read_version() -> str:
|
|
v_file = Path(__file__).parent.parent / "VERSION"
|
|
try:
|
|
return v_file.read_text().strip()
|
|
except Exception:
|
|
return "0.0.0"
|
|
|
|
__version__: str = _read_version()
|
|
|
|
# ── Boot-time defaults ────────────────────────────────────────────────────────
|
|
|
|
_VOICES_DIR_DEFAULT = os.environ.get("VOICES_DIR", "/voices")
|
|
_OUTPUT_DIR_DEFAULT = os.environ.get("OUTPUT_DIR", "/voices/active_voices")
|
|
_WHISPER_DEFAULT = os.environ.get("WHISPER_URL", "http://192.168.178.8:8010")
|
|
_TTS_DEFAULT = os.environ.get("TTS_URL", "http://192.168.178.8:8020")
|
|
_TTS_STREAM_DEFAULT = os.environ.get("TTS_STREAM_URL", "http://192.168.178.8:8023")
|
|
_CUSTOMVOICE_DEFAULT = os.environ.get("CUSTOMVOICE_URL", "http://192.168.178.8:8022")
|
|
_VOICE_DESIGN_DEFAULT = os.environ.get("VOICE_DESIGN_URL", "http://192.168.178.8:8021")
|
|
_NVIDIA_ROUTER_DEFAULT = os.environ.get("NVIDIA_SPEECH_ROUTER_URL", "http://192.168.178.8:8090")
|
|
_NVIDIA_TTS_DEFAULT = os.environ.get("NVIDIA_MAGPIE_TTS_URL", "http://192.168.178.8:8091")
|
|
_NVIDIA_ASR_DEFAULT = os.environ.get("NVIDIA_PARAKEET_ASR_URL", "http://192.168.178.8:8092")
|
|
_NVIDIA_CLONE_DEFAULT = os.environ.get("NVIDIA_TTS_CLONE_URL", "http://192.168.178.8:8093")
|
|
_NVIDIA_ZEROSHOT_DEFAULT = os.environ.get("NVIDIA_ZEROSHOT_TTS_URL", _NVIDIA_CLONE_DEFAULT)
|
|
_NVIDIA_FLOW_DEFAULT = os.environ.get("NVIDIA_FLOW_TTS_URL", "http://192.168.178.8:8094")
|
|
_FASTER_WHISPER_DEFAULT = os.environ.get("FASTER_WHISPER_URL", "http://192.168.178.8:8010")
|
|
_WHISPER_CPP_DEFAULT = os.environ.get("WHISPER_CPP_URL", "http://192.168.178.8:8085")
|
|
_GROQ_STT_ENDPOINT = "https://api.groq.com/openai/v1"
|
|
_KOKORO_DEFAULT = os.environ.get("KOKORO_URL", "http://192.168.178.8:8880/v1")
|
|
# Layout-aware OCR engine for magazine ingestion (see routes/magazine.py). Not
|
|
# a hardware URL like the others above -- points at whichever "ocr-*" engine
|
|
# card (Engines > OCR & Document Parsing) is configured. Defaults to the
|
|
# Baidu Unlimited-OCR container's HOST-published port via host.docker.internal
|
|
# (wired in docker-compose.yml's extra_hosts) rather than its container name --
|
|
# confirmed live that it runs on its own Docker network, not dgx_net, so a
|
|
# container-name default would not resolve from the app container.
|
|
_VISION_OCR_DEFAULT = os.environ.get("VISION_OCR_URL", "http://host.docker.internal:8098/v1")
|
|
_VISION_OCR_MODEL_DEFAULT = os.environ.get("VISION_OCR_MODEL", "baidu/Unlimited-OCR")
|
|
_VIBEVOICE_DEFAULT = os.environ.get("VIBEVOICE_URL", "http://192.168.178.8:8027")
|
|
_XTTS_DEFAULT = os.environ.get("XTTS_URL", "http://192.168.178.8:8005")
|
|
_FISHSPEECH_DEFAULT = os.environ.get("FISHSPEECH_URL", "http://192.168.178.8:38080")
|
|
_TTS_CONTAINER = os.environ.get("TTS_CONTAINER_NAME", "faster-qwen3-tts")
|
|
_TTS_CONTAINERS_RAW = os.environ.get("TTS_CONTAINER_NAMES", "") # comma-separated override
|
|
_VOICE_DESIGN_MODEL = os.environ.get("VOICE_DESIGN_MODEL", "Qwen3-TTS-12Hz-1.7B-VoiceDesign")
|
|
_VOICE_TARGET_DBFS = float(os.environ.get("VOICE_TARGET_DBFS", "-20.0"))
|
|
_VOICE_PEAK_DBFS = float(os.environ.get("VOICE_PEAK_DBFS", "-1.0"))
|
|
# Keep uploads bounded by default. Individual endpoints apply smaller limits for
|
|
# pictures and sound assets; MAX_UPLOAD_MB remains configurable for unusually
|
|
# large source audio or PDFs without making an accidental 1 GB request the
|
|
# default for every multipart endpoint.
|
|
_MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_MB", "256")) * 1024 * 1024
|
|
_MAX_JSON_BYTES = int(os.environ.get("MAX_JSON_MB", "32")) * 1024 * 1024
|
|
_MAX_RAW_REQUEST_BYTES = int(os.environ.get("MAX_RAW_REQUEST_MB", "256")) * 1024 * 1024
|
|
_MAX_PICTURE_BYTES = int(os.environ.get("MAX_PICTURE_MB", "10")) * 1024 * 1024
|
|
_MAX_SOUND_BYTES = int(os.environ.get("MAX_SOUND_MB", "25")) * 1024 * 1024
|
|
_MAX_TTS_OUTPUT_SECONDS = float(os.environ.get("MAX_TTS_OUTPUT_SECONDS", "60"))
|
|
_STT_REQUEST_TIMEOUT = float(os.environ.get("STT_REQUEST_TIMEOUT", os.environ.get("REQUEST_TIMEOUT", "900")))
|
|
_BENCHMARK_TEXT = os.environ.get("VOICE_BENCHMARK_TEXT", "This is a short realtime voice benchmark.")
|
|
_BENCHMARK_SENTENCES = [
|
|
("short", _BENCHMARK_TEXT),
|
|
("medium", "The quick brown fox jumps over the lazy dog near the river bank."),
|
|
(
|
|
"long",
|
|
"Artificial intelligence is transforming the way we interact with technology. "
|
|
"From voice assistants to autonomous vehicles, machine learning models are becoming "
|
|
"an integral part of everyday life.",
|
|
),
|
|
]
|
|
_ALLOW_PRIVATE_DOWNLOADS = os.environ.get("ALLOW_PRIVATE_DOWNLOADS", "").lower() in {"1", "true", "yes"}
|
|
# External callers must present the app's inbound key by default. Set this only
|
|
# for a deliberately isolated local deployment that has another auth layer.
|
|
_REQUIRE_EXTERNAL_API_KEY = os.environ.get("REQUIRE_EXTERNAL_API_KEY", "1").lower() not in {"0", "false", "no", "off"}
|
|
|
|
# ── Config dir calculation ────────────────────────────────────────────────────
|
|
|
|
logger = logging.getLogger("uvicorn.error")
|
|
|
|
|
|
def _default_config_dir() -> Path:
|
|
env_dir = os.environ.get("CONFIG_DIR")
|
|
if env_dir:
|
|
return Path(env_dir)
|
|
current = Path("/home/app/.config/tts-voice-creator")
|
|
legacy = Path("/home/app/.config/voice-clone-factory")
|
|
if legacy.exists():
|
|
legacy.mkdir(parents=True, exist_ok=True)
|
|
if current.exists():
|
|
for name in ("settings.json", "voice_design_presets.json", "tts_routes.json"):
|
|
src, dst = current / name, legacy / name
|
|
if src.exists() and not dst.exists():
|
|
try:
|
|
shutil.copy2(src, dst)
|
|
except Exception as exc:
|
|
logger.warning("Could not migrate config %s to mounted config dir: %s", name, exc)
|
|
return legacy
|
|
return current
|
|
|
|
|
|
CONFIG_DIR = _default_config_dir()
|
|
CONFIG_FILE = CONFIG_DIR / "settings.json"
|
|
DESIGN_PRESETS_FILE = CONFIG_DIR / "voice_design_presets.json"
|
|
TTS_ROUTES_FILE = CONFIG_DIR / "tts_routes.json"
|
|
|
|
# ── Static dir ────────────────────────────────────────────────────────────────
|
|
|
|
STATIC_DIR = Path(__file__).parent.parent / "static"
|
|
STATIC_DIR.mkdir(exist_ok=True)
|
|
|
|
# ── In-memory log buffer ───────────────────────────────────────────────────────
|
|
|
|
_LOG_BUFFER_MAX = 400
|
|
_log_buffer: list[dict] = []
|
|
|
|
|
|
class _BufferHandler(logging.Handler):
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
try:
|
|
_log_buffer.insert(0, {
|
|
"ts": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
|
|
"level": record.levelname,
|
|
"name": record.name,
|
|
"msg": record.getMessage(),
|
|
})
|
|
del _log_buffer[_LOG_BUFFER_MAX:]
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ── Routing log ───────────────────────────────────────────────────────────────
|
|
|
|
_ROUTING_LOG_MAX = int(os.environ.get("TTS_ROUTING_LOG_MAX", "120"))
|
|
_routing_log: list[dict] = []
|
|
|
|
|
|
def _routing_log_add(**entry) -> None:
|
|
item = {
|
|
"ts": datetime.now(timezone.utc).isoformat(),
|
|
**entry,
|
|
}
|
|
_routing_log.insert(0, item)
|
|
del _routing_log[_ROUTING_LOG_MAX:]
|