"""LibriVox / Wikimedia Commons voice sources: validation, caps, host checks, mapping, provenance, cache/throttle (HTTP mocked).""" import asyncio import os import sys import tempfile import unittest from pathlib import Path from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parents[1])) os.environ.setdefault("CONFIG_DIR", tempfile.mkdtemp(prefix="ttvc-vs-test-config-")) from core import open_voice_sources as ovs import routes.sources as src def book(i=1, lang="English", url="https://www.archive.org/download/x/a.mp3", sections=False): b = {"id": str(i), "title": f"Book {i}", "language": lang, "url_librivox": f"https://librivox.org/b{i}/", "totaltimesecs": 600, "authors": [{"first_name": "Jane", "last_name": "Austen"}]} if sections: b["sections"] = [{"title": "Ch 1", "listen_url": url, "playtime": "600", "readers": [{"display_name": "Ann"}]}] return b def commons(**over): info = {"url": "https://upload.wikimedia.org/a.opus", "descriptionurl": "https://commons.wikimedia.org/wiki/File:A.opus", "mediatype": "AUDIO", "duration": 12.4, "extmetadata": {"LicenseShortName": {"value": "CC BY-SA 4.0"}, "Artist": {"value": "Bob & Co"}, "LicenseUrl": {"value": "https://creativecommons.org/licenses/by-sa/4.0"}}} info.update(over) return {"query": {"pages": {"1": {"title": "File:A.opus", "index": 1, "imageinfo": [info]}}}} class Base(unittest.TestCase): def setUp(self): ovs._cache.clear() ovs._last_call.clear() p = mock.patch.object(ovs.time, "sleep") p.start() self.addCleanup(p.stop) class ValidationTests(Base): def test_query_and_lang(self): for bad in ("", " ", "x" * 81): with self.assertRaises(ovs.SourceError): ovs.clean_query(bad) self.assertEqual(ovs.clean_query(" a\x00 \n b "), "a b") with self.assertRaises(ovs.SourceError): ovs.clean_lang("en;rm") with self.assertRaises(ovs.SourceError): ovs.search("evil", "x") def test_host_check(self): self.assertTrue(ovs.host_ok("https://ia80.us.archive.org/x.mp3", ovs.AUDIO_HOSTS)) for bad in ("http://archive.org/x", "https://evilarchive.org/x", "https://archive.org.evil.com/x", "https://u:p@archive.org/x", "https://127.0.0.1/x", "file:///etc/passwd"): self.assertFalse(ovs.host_ok(bad, ovs.AUDIO_HOSTS), bad) class MappingTests(Base): def test_librivox_list_is_light_filtered_and_capped(self): books = [book(i) for i in range(15)] + [book(98, lang="German"), dict(book(97), url_librivox="https://evil.example/x/"), dict(book(96), id="x;1")] with mock.patch.object(ovs, "_get_json", return_value={"books": books}) as g: items = ovs.search("librivox", "pride", "en") self.assertEqual(len(items), ovs.MAX_RESULTS) self.assertNotIn("extended", g.call_args[0][1]) it = items[0] self.assertEqual((it["author"], it["language"], it["duration"], it["audio_url"], it["book_id"]), ("Jane Austen", "English", 600, "", "0")) self.assertTrue(it["licence_url"].startswith("https://librivox.org/")) self.assertTrue(all(i["language"] == "English" and "evil" not in i["source_url"] for i in items)) def test_librivox_first_section(self): bad = book(2, url="https://evil.example/a.mp3", sections=True) bad["sections"].append({"title": "Ch 2", "listen_url": "https://ia1.us.archive.org/b.mp3", "playtime": "5", "readers": []}) with mock.patch.object(ovs, "_get_json", return_value={"books": [bad]}) as g: r = ovs.librivox_first_section("2") self.assertEqual(g.call_args[0][1]["extended"], 1) self.assertEqual((r["audio_url"], r["section"], r["duration"]), ("https://ia1.us.archive.org/b.mp3", "Ch 2", 5)) for badid in ("", "abc", "1;2", "1" * 10): with self.assertRaises(ovs.SourceError): ovs.librivox_first_section(badid) with mock.patch.object(ovs, "_get_json", return_value={"books": [book(3, url="https://evil.example/a")]}): with self.assertRaises(ovs.SourceError) as cm: ovs.librivox_first_section("3") self.assertEqual(cm.exception.status, 404) def test_wikimedia_mapping(self): with mock.patch.object(ovs, "_get_json", return_value=commons()): it = ovs.search("wikimedia", "speech")[0] self.assertEqual((it["title"], it["reader"], it["licence"], it["duration"]), ("A", "Bob & Co", "CC BY-SA 4.0", 12)) self.assertIn("Bob & Co / A, CC BY-SA 4.0", it["attribution"]) def test_wikimedia_drops_unsafe_or_unlicensed(self): for over in ({"url": "https://evil.example/a.opus"}, {"mediatype": "VIDEO"}, {"extmetadata": {}}): ovs._cache.clear() with mock.patch.object(ovs, "_get_json", return_value=commons(**over)): self.assertEqual(ovs.search("wikimedia", "speech"), []) class ProvenanceTests(Base): def test_librivox(self): p = ovs.import_provenance_for("librivox", "https://librivox.org/b/", "", "") self.assertEqual(p["basis"], "public_domain") self.assertIn("reader's declared licence", p["note"]) self.assertEqual(p["source"], "https://librivox.org/b/") def test_wikimedia(self): p = ovs.import_provenance_for("wikimedia", "https://commons.wikimedia.org/wiki/File:A.opus", "CC BY 4.0", "Bob / A") self.assertEqual((p["basis"], p["licence"]), ("licensed", "CC BY 4.0")) self.assertIn("Bob / A", p["note"]) self.assertEqual(ovs.import_provenance_for("wikimedia", "https://c.org/", "Public domain", "")["basis"], "public_domain") with self.assertRaises(ValueError): ovs.import_provenance_for("other", "", "", "") class CacheThrottleTests(Base): def test_cache_hit_skips_upstream(self): with mock.patch.object(ovs, "_get_json", return_value={"books": [book()]}) as g: ovs.search("librivox", "Pride"); ovs.search("librivox", "pride") self.assertEqual(g.call_count, 1) def test_throttle_sleeps_and_errors_are_generic(self): resp = mock.Mock(status_code=200) resp.raise_for_status.return_value = None resp.iter_content.return_value = [b'{"books": []}'] with mock.patch.object(ovs.requests, "get", return_value=resp) as g: ovs._get_json(ovs.LIBRIVOX_API, {}); ovs._get_json(ovs.LIBRIVOX_API, {}) self.assertTrue(ovs.time.sleep.called) self.assertFalse(g.call_args.kwargs["allow_redirects"]) self.assertIn("User-Agent", g.call_args.kwargs["headers"]) with mock.patch.object(ovs.requests, "get", side_effect=OSError("secret host")): with self.assertRaises(ovs.SourceError) as cm: ovs._get_json(ovs.COMMONS_API, {}) self.assertNotIn("secret", str(cm.exception)) class RetryErrorTests(Base): def setUp(self): super().setUp() ovs._neg.clear() def test_timeout_retried_once_then_clear_message_and_negative_cache(self): with mock.patch.object(ovs.requests, "get", side_effect=ovs.requests.Timeout("secret")) as g: with self.assertRaises(ovs.SourceError) as cm: ovs.search("librivox", "Faust") self.assertEqual(g.call_count, 2) self.assertIn("in time", str(cm.exception)) self.assertEqual(cm.exception.status, 502) with self.assertRaises(ovs.SourceError): # remembered: no further upstream call ovs.search("librivox", "Faust") self.assertEqual(g.call_count, 2) def test_retry_recovers_and_messages_differ(self): resp = mock.Mock(status_code=200) resp.raise_for_status.return_value = None resp.iter_content.return_value = [b'{"books": []}'] with mock.patch.object(ovs.requests, "get", side_effect=[ovs.requests.ConnectionError("x"), resp]): self.assertEqual(ovs._get_json(ovs.LIBRIVOX_API, {}), {"books": []}) for exc, text in ((ovs.requests.ConnectionError("x"), "unreachable"), (ValueError("x"), "unreadable")): with mock.patch.object(ovs.requests, "get", side_effect=exc): with self.assertRaises(ovs.SourceError) as cm: ovs._get_json(ovs.LIBRIVOX_API, {}) self.assertIn(text, str(cm.exception)) class SettingsTests(unittest.TestCase): def test_export_provenance_default_exposed(self): from core import config self.assertIs(config._load_settings().get("export_provenance_default"), True) class RouteTests(Base): def test_search_route_errors_and_allowlist(self): with self.assertRaises(src.HTTPException) as cm: asyncio.run(src.open_voice_source_search("librivox", "")) self.assertEqual(cm.exception.status_code, 400) self.assertIn("archive.org", src._PROXY_AUDIO_DOMAINS) self.assertIn("upload.wikimedia.org", src._PROXY_AUDIO_DOMAINS) def test_quick_import_rejects_foreign_host(self): req = mock.Mock() async def js(): return {"audio_url": "https://example.com/a.mp3", "voice_id": "x", "source_kind": "librivox"} req.json = js with mock.patch.object(src, "_source_import_url", side_effect=lambda u: u): with self.assertRaises(src.HTTPException) as cm: asyncio.run(src.quick_import_voice(req)) self.assertEqual(cm.exception.status_code, 400) if __name__ == "__main__": unittest.main()