"""Voice sources scraping, proxy-audio, quick-import, yt-dlp download, ElevenLabs."""
from __future__ import annotations
import asyncio
import glob
import json
import re
import uuid
from datetime import datetime, timezone
from html import unescape
from pathlib import Path
from urllib.parse import quote, urljoin, urlparse
import requests
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import StreamingResponse
from core.config import _load_settings
from core.constants import _VOICES_DIR_DEFAULT, _MAX_UPLOAD_BYTES, _ALLOW_PRIVATE_DOWNLOADS
from core.registry import _registry_put, TEMP_DIR
from core.validation import _validate_http_url
from core.audio import _to_wav_24k, _duration
from core.voice import (
_AUDIO_EXTS, _active_voices_dir,
_remove_audio_variants, _export_normalized_wav, _load_meta, _save_meta,
)
router = APIRouter()
# ── Voice source helpers ──────────────────────────────────────────────────────
_VOICE_SOURCE_HEADERS = {
"User-Agent": "TTS Voice Creator voice-source-scraper/1.0 (+local)",
"Accept": "text/html,application/json,text/plain;q=0.9,*/*;q=0.8",
}
DEFAULT_VOICE_SOURCE_URLS = [
"https://aiartes.com/voiceai",
"https://sample-files.com/downloads/audio/wav/voice-sample.wav",
"https://freesound.org/people/Scott%20Simpson/",
"https://lanceblairvo.com/raw-voiceover-samples/",
"https://github.com/yaph/tts-samples/tree/main/mp3",
"https://github.com/jim-schwoebel/voice_datasets",
]
_DIRECT_AUDIO_RE = re.compile(r"\.(?:mp3|wav|ogg|flac|m4a|aac)(?:$|[?#])", re.I)
_LANG_HINTS = {
"english": "English", "german": "German", "deutsch": "German", "french": "French",
"spanish": "Spanish", "italian": "Italian", "portuguese": "Portuguese",
"dutch": "Dutch", "polish": "Polish", "russian": "Russian", "japanese": "Japanese",
"korean": "Korean", "chinese": "Chinese", "arabic": "Arabic", "swedish": "Swedish",
"turkish": "Turkish", "hindi": "Hindi",
}
_PROXY_AUDIO_DOMAINS: set[str] = {
"drive.usercontent.google.com",
"drive.google.com",
"aiartes.com",
"freesound.org",
"lanceblairvo.com",
"raw.githubusercontent.com",
"sample-files.com",
}
_GDRIVE_MIME_EXT = {
"audio/mpeg": "mp3", "audio/mp3": "mp3",
"audio/wav": "wav", "audio/x-wav": "wav",
"audio/ogg": "ogg", "audio/vorbis": "ogg",
"audio/flac": "flac", "audio/x-flac": "flac",
"audio/aac": "aac", "audio/x-aac": "aac",
"audio/x-m4a": "m4a", "audio/m4a": "m4a", "audio/mp4": "m4a",
}
def _source_import_url(value: str) -> str:
raw = str(value or "").strip()
if not re.match(r"^https?://", raw, re.I):
raise HTTPException(400, "Expected an http(s) URL")
return raw
def _source_download_limit(name: str, fallback_mb: int) -> int:
import os
value = globals().get(name)
if isinstance(value, int):
return value
return int(os.environ.get(name.lstrip("_").replace("_BYTES", "_MB"), str(fallback_mb))) * 1024 * 1024
def _source_get(url: str, timeout: int = 12) -> requests.Response:
r = requests.get(url, headers=_VOICE_SOURCE_HEADERS, timeout=timeout)
r.raise_for_status()
return r
def _plain_text(value: str) -> str:
value = re.sub(r"<[^>]+>", " ", str(value or ""))
value = unescape(value)
return re.sub(r"\s+", " ", value).strip()
def _voice_source_id(url: str) -> str:
parsed = urlparse(url)
base = (parsed.netloc + parsed.path).strip("/").lower()
base = re.sub(r"[^a-z0-9]+", "-", base).strip("-")
return base[:80] or "source"
def _source_name_from_url(url: str) -> str:
parsed = urlparse(url)
host = parsed.netloc.replace("www.", "")
tail = Path(parsed.path.rstrip("/")).stem.replace("-", " ").replace("_", " ").strip()
return f"{host} / {tail}" if tail else host or url
def _guess_language(*values: str) -> str:
text = " ".join(str(v or "") for v in values).lower()
for key, label in _LANG_HINTS.items():
if re.search(rf"\b{re.escape(key)}\b", text):
return label
code = re.search(r"(?:^|[^a-z])(en|de|fr|es|it|pt|nl|pl|ru|ja|ko|zh|ar|sv|tr|hi)(?:[^a-z]|$)", text)
return {"en": "English", "de": "German", "fr": "French", "es": "Spanish", "it": "Italian",
"pt": "Portuguese", "nl": "Dutch", "pl": "Polish", "ru": "Russian", "ja": "Japanese",
"ko": "Korean", "zh": "Chinese", "ar": "Arabic", "sv": "Swedish", "tr": "Turkish",
"hi": "Hindi"}.get(code.group(1), "Unknown") if code else "Unknown"
def _guess_gender(*values: str) -> str:
text = " ".join(str(v or "") for v in values).lower()
if re.search(r"\b(female|woman|girl|fem|_f_|-f-)\b", text):
return "Female"
if re.search(r"\b(male|man|boy|masc|_m_|-m-)\b", text):
return "Male"
return "Unknown"
def _source_item(source_id: str, source_name: str, name: str, kind: str,
page_url: str, audio_url: str = "", image_url: str = "",
category: str = "", description: str = "", file_type: str = "",
language: str = "", gender: str = "") -> dict:
if not file_type and audio_url:
match = re.search(r"\.([A-Za-z0-9]+)(?:$|[?#])", audio_url)
file_type = match.group(1).lower() if match else "audio"
language = language or _guess_language(name, kind, category, description, page_url, audio_url)
gender = gender or _guess_gender(name, kind, category, description, page_url, audio_url)
return {
"id": f"{source_id}:{uuid.uuid5(uuid.NAMESPACE_URL, page_url + audio_url + name + kind)}",
"source_id": source_id,
"source": source_name,
"name": _plain_text(name)[:160],
"kind": _plain_text(kind)[:80],
"category": _plain_text(category)[:80],
"description": _plain_text(description)[:420],
"page_url": page_url,
"audio_url": audio_url,
"image_url": image_url,
"file_type": file_type,
"language": language,
"gender": gender,
"direct_audio": bool(audio_url),
}
def _source_result(source_id: str, name: str, homepage: str, description: str, items: list[dict]) -> dict:
return {
"id": source_id,
"name": name,
"homepage": homepage,
"description": description,
"items": items,
"count": len(items),
"direct_audio": sum(1 for item in items if item.get("direct_audio")),
}
def _direct_audio_source(url: str) -> dict:
source_id = _voice_source_id(url)
name = _source_name_from_url(url)
item = _source_item(
source_id, name,
Path(urlparse(url).path).stem.replace("-", " ").replace("_", " ") or "Voice sample",
"Direct audio file", url, audio_url=url,
category="Direct audio",
description="Direct audio URL from the editable source list.",
)
return _source_result(source_id, name, url, "Single direct audio URL.", [item])
def _generic_audio_page_source(url: str) -> dict:
source_id = _voice_source_id(url)
source_name = _source_name_from_url(url)
body = _source_get(url).text
title = re.search(r"
]*>(.*?)", body, re.I | re.S)
page_title = _plain_text(title.group(1)) if title else source_name
seen = set()
items = []
for match in re.finditer(r"""(?:src|href)=["']([^"']+\.(?:mp3|wav|ogg|flac|m4a|aac)(?:[^"']*)?)["']""", body, re.I):
audio_url = urljoin(url, unescape(match.group(1)))
if audio_url in seen:
continue
seen.add(audio_url)
stem = Path(urlparse(audio_url).path).stem.replace("-", " ").replace("_", " ").strip() or page_title
items.append(_source_item(
source_id, source_name, stem, "Audio sample", url,
audio_url=audio_url, category="Page audio",
description=f"Audio link found on {page_title}.",
))
if not items:
items.append(_source_item(
source_id, source_name, page_title, "Source page", url,
category="Source page",
description="No direct audio file links were found. Open the source page for previews, licensing, and download details.",
))
return _source_result(source_id, source_name, url, f"Audio links scraped from {page_title}.", items)
def _scrape_aiartes_voiceai() -> dict:
homepage = "https://aiartes.com/voiceai"
body = _source_get(homepage).text
items = []
for card in body.split('
')[1:]:
title = re.search(r'
\s*([^<]+?)\s*
', card, re.S)
if not title:
continue
image = re.search(r']+src="([^"]+)"', card, re.S)
image_url = urljoin(homepage, image.group(1)) if image else ""
pairs = re.findall(
r'