192 lines
9.4 KiB
Python
192 lines
9.4 KiB
Python
"""LibriVox / Wikimedia Commons voice sources: validation, caps, host checks, mapping, provenance, cache/throttle (HTTP mocked)."""
|
|
import asyncio
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-vs-test-config-"))
|
|
from core import open_voice_sources as ovs
|
|
import routes.sources as src
|
|
|
|
|
|
def book(i=1, lang="English", url="https://www.archive.org/download/x/a.mp3", sections=False):
|
|
b = {"id": str(i), "title": f"Book {i}", "language": lang, "url_librivox": f"https://librivox.org/b{i}/",
|
|
"totaltimesecs": 600, "authors": [{"first_name": "Jane", "last_name": "Austen"}]}
|
|
if sections:
|
|
b["sections"] = [{"title": "Ch 1", "listen_url": url, "playtime": "600", "readers": [{"display_name": "Ann"}]}]
|
|
return b
|
|
|
|
|
|
def commons(**over):
|
|
info = {"url": "https://upload.wikimedia.org/a.opus", "descriptionurl": "https://commons.wikimedia.org/wiki/File:A.opus",
|
|
"mediatype": "AUDIO", "duration": 12.4,
|
|
"extmetadata": {"LicenseShortName": {"value": "CC BY-SA 4.0"}, "Artist": {"value": "<a href='x'>Bob & Co</a>"},
|
|
"LicenseUrl": {"value": "https://creativecommons.org/licenses/by-sa/4.0"}}}
|
|
info.update(over)
|
|
return {"query": {"pages": {"1": {"title": "File:A.opus", "index": 1, "imageinfo": [info]}}}}
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self):
|
|
ovs._cache.clear()
|
|
ovs._last_call.clear()
|
|
p = mock.patch.object(ovs.time, "sleep")
|
|
p.start()
|
|
self.addCleanup(p.stop)
|
|
|
|
|
|
class ValidationTests(Base):
|
|
def test_query_and_lang(self):
|
|
for bad in ("", " ", "x" * 81):
|
|
with self.assertRaises(ovs.SourceError):
|
|
ovs.clean_query(bad)
|
|
self.assertEqual(ovs.clean_query(" a\x00 \n b "), "a b")
|
|
with self.assertRaises(ovs.SourceError):
|
|
ovs.clean_lang("en;rm")
|
|
with self.assertRaises(ovs.SourceError):
|
|
ovs.search("evil", "x")
|
|
|
|
def test_host_check(self):
|
|
self.assertTrue(ovs.host_ok("https://ia80.us.archive.org/x.mp3", ovs.AUDIO_HOSTS))
|
|
for bad in ("http://archive.org/x", "https://evilarchive.org/x", "https://archive.org.evil.com/x",
|
|
"https://u:p@archive.org/x", "https://127.0.0.1/x", "file:///etc/passwd"):
|
|
self.assertFalse(ovs.host_ok(bad, ovs.AUDIO_HOSTS), bad)
|
|
|
|
|
|
class MappingTests(Base):
|
|
def test_librivox_list_is_light_filtered_and_capped(self):
|
|
books = [book(i) for i in range(15)] + [book(98, lang="German"), dict(book(97), url_librivox="https://evil.example/x/"), dict(book(96), id="x;1")]
|
|
with mock.patch.object(ovs, "_get_json", return_value={"books": books}) as g:
|
|
items = ovs.search("librivox", "pride", "en")
|
|
self.assertEqual(len(items), ovs.MAX_RESULTS)
|
|
self.assertNotIn("extended", g.call_args[0][1])
|
|
it = items[0]
|
|
self.assertEqual((it["author"], it["language"], it["duration"], it["audio_url"], it["book_id"]), ("Jane Austen", "English", 600, "", "0"))
|
|
self.assertTrue(it["licence_url"].startswith("https://librivox.org/"))
|
|
self.assertTrue(all(i["language"] == "English" and "evil" not in i["source_url"] for i in items))
|
|
|
|
def test_librivox_first_section(self):
|
|
bad = book(2, url="https://evil.example/a.mp3", sections=True)
|
|
bad["sections"].append({"title": "Ch 2", "listen_url": "https://ia1.us.archive.org/b.mp3", "playtime": "5", "readers": []})
|
|
with mock.patch.object(ovs, "_get_json", return_value={"books": [bad]}) as g:
|
|
r = ovs.librivox_first_section("2")
|
|
self.assertEqual(g.call_args[0][1]["extended"], 1)
|
|
self.assertEqual((r["audio_url"], r["section"], r["duration"]), ("https://ia1.us.archive.org/b.mp3", "Ch 2", 5))
|
|
for badid in ("", "abc", "1;2", "1" * 10):
|
|
with self.assertRaises(ovs.SourceError):
|
|
ovs.librivox_first_section(badid)
|
|
with mock.patch.object(ovs, "_get_json", return_value={"books": [book(3, url="https://evil.example/a")]}):
|
|
with self.assertRaises(ovs.SourceError) as cm:
|
|
ovs.librivox_first_section("3")
|
|
self.assertEqual(cm.exception.status, 404)
|
|
|
|
def test_wikimedia_mapping(self):
|
|
with mock.patch.object(ovs, "_get_json", return_value=commons()):
|
|
it = ovs.search("wikimedia", "speech")[0]
|
|
self.assertEqual((it["title"], it["reader"], it["licence"], it["duration"]), ("A", "Bob & Co", "CC BY-SA 4.0", 12))
|
|
self.assertIn("Bob & Co / A, CC BY-SA 4.0", it["attribution"])
|
|
|
|
def test_wikimedia_drops_unsafe_or_unlicensed(self):
|
|
for over in ({"url": "https://evil.example/a.opus"}, {"mediatype": "VIDEO"}, {"extmetadata": {}}):
|
|
ovs._cache.clear()
|
|
with mock.patch.object(ovs, "_get_json", return_value=commons(**over)):
|
|
self.assertEqual(ovs.search("wikimedia", "speech"), [])
|
|
|
|
|
|
class ProvenanceTests(Base):
|
|
def test_librivox(self):
|
|
p = ovs.import_provenance_for("librivox", "https://librivox.org/b/", "", "")
|
|
self.assertEqual(p["basis"], "public_domain")
|
|
self.assertIn("reader's declared licence", p["note"])
|
|
self.assertEqual(p["source"], "https://librivox.org/b/")
|
|
|
|
def test_wikimedia(self):
|
|
p = ovs.import_provenance_for("wikimedia", "https://commons.wikimedia.org/wiki/File:A.opus", "CC BY 4.0", "Bob / A")
|
|
self.assertEqual((p["basis"], p["licence"]), ("licensed", "CC BY 4.0"))
|
|
self.assertIn("Bob / A", p["note"])
|
|
self.assertEqual(ovs.import_provenance_for("wikimedia", "https://c.org/", "Public domain", "")["basis"], "public_domain")
|
|
with self.assertRaises(ValueError):
|
|
ovs.import_provenance_for("other", "", "", "")
|
|
|
|
|
|
class CacheThrottleTests(Base):
|
|
def test_cache_hit_skips_upstream(self):
|
|
with mock.patch.object(ovs, "_get_json", return_value={"books": [book()]}) as g:
|
|
ovs.search("librivox", "Pride"); ovs.search("librivox", "pride")
|
|
self.assertEqual(g.call_count, 1)
|
|
|
|
def test_throttle_sleeps_and_errors_are_generic(self):
|
|
resp = mock.Mock(status_code=200)
|
|
resp.raise_for_status.return_value = None
|
|
resp.iter_content.return_value = [b'{"books": []}']
|
|
with mock.patch.object(ovs.requests, "get", return_value=resp) as g:
|
|
ovs._get_json(ovs.LIBRIVOX_API, {}); ovs._get_json(ovs.LIBRIVOX_API, {})
|
|
self.assertTrue(ovs.time.sleep.called)
|
|
self.assertFalse(g.call_args.kwargs["allow_redirects"])
|
|
self.assertIn("User-Agent", g.call_args.kwargs["headers"])
|
|
with mock.patch.object(ovs.requests, "get", side_effect=OSError("secret host")):
|
|
with self.assertRaises(ovs.SourceError) as cm:
|
|
ovs._get_json(ovs.COMMONS_API, {})
|
|
self.assertNotIn("secret", str(cm.exception))
|
|
|
|
|
|
class RetryErrorTests(Base):
|
|
def setUp(self):
|
|
super().setUp()
|
|
ovs._neg.clear()
|
|
|
|
def test_timeout_retried_once_then_clear_message_and_negative_cache(self):
|
|
with mock.patch.object(ovs.requests, "get", side_effect=ovs.requests.Timeout("secret")) as g:
|
|
with self.assertRaises(ovs.SourceError) as cm:
|
|
ovs.search("librivox", "Faust")
|
|
self.assertEqual(g.call_count, 2)
|
|
self.assertIn("in time", str(cm.exception))
|
|
self.assertEqual(cm.exception.status, 502)
|
|
with self.assertRaises(ovs.SourceError): # remembered: no further upstream call
|
|
ovs.search("librivox", "Faust")
|
|
self.assertEqual(g.call_count, 2)
|
|
|
|
def test_retry_recovers_and_messages_differ(self):
|
|
resp = mock.Mock(status_code=200)
|
|
resp.raise_for_status.return_value = None
|
|
resp.iter_content.return_value = [b'{"books": []}']
|
|
with mock.patch.object(ovs.requests, "get", side_effect=[ovs.requests.ConnectionError("x"), resp]):
|
|
self.assertEqual(ovs._get_json(ovs.LIBRIVOX_API, {}), {"books": []})
|
|
for exc, text in ((ovs.requests.ConnectionError("x"), "unreachable"), (ValueError("x"), "unreadable")):
|
|
with mock.patch.object(ovs.requests, "get", side_effect=exc):
|
|
with self.assertRaises(ovs.SourceError) as cm:
|
|
ovs._get_json(ovs.LIBRIVOX_API, {})
|
|
self.assertIn(text, str(cm.exception))
|
|
|
|
|
|
class SettingsTests(unittest.TestCase):
|
|
def test_export_provenance_default_exposed(self):
|
|
from core import config
|
|
self.assertIs(config._load_settings().get("export_provenance_default"), True)
|
|
|
|
|
|
class RouteTests(Base):
|
|
def test_search_route_errors_and_allowlist(self):
|
|
with self.assertRaises(src.HTTPException) as cm:
|
|
asyncio.run(src.open_voice_source_search("librivox", ""))
|
|
self.assertEqual(cm.exception.status_code, 400)
|
|
self.assertIn("archive.org", src._PROXY_AUDIO_DOMAINS)
|
|
self.assertIn("upload.wikimedia.org", src._PROXY_AUDIO_DOMAINS)
|
|
|
|
def test_quick_import_rejects_foreign_host(self):
|
|
req = mock.Mock()
|
|
async def js(): return {"audio_url": "https://example.com/a.mp3", "voice_id": "x", "source_kind": "librivox"}
|
|
req.json = js
|
|
with mock.patch.object(src, "_source_import_url", side_effect=lambda u: u):
|
|
with self.assertRaises(src.HTTPException) as cm:
|
|
asyncio.run(src.quick_import_voice(req))
|
|
self.assertEqual(cm.exception.status_code, 400)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|