mARTin-B78
91caa6cf90
- fuse_paragraph_blocks joins a paragraph split by a column or page break into one narration block (reader, podcast source and MP3), keeping source regions for highlighting - language decided once per article; per-article acronym glossary (LLM classifies unknown acronyms from context), letter-spelled acronym lists, extended DE/EN database - core/spelling: dictionary findings (compounds, inflections, line-break hyphens, other-language words), marked in review and listed per page/article; guarded LLM agent fixes; quality run now includes the spelling pass - keep suspended compound hyphens (Programmier- und) when merging regions / building layouts Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
91 lines
4.2 KiB
Python
91 lines
4.2 KiB
Python
import unittest
|
|
|
|
from core.pronunciations import (
|
|
detect_article_language,
|
|
expand_abbreviations,
|
|
find_unknown_acronyms,
|
|
language_for_block,
|
|
normalize_ocr_text,
|
|
prepare_text_for_tts,
|
|
remove_linebreak_hyphens,
|
|
suspect_ocr_tokens,
|
|
)
|
|
|
|
|
|
class PronunciationTests(unittest.TestCase):
|
|
def test_fuses_printed_line_break_but_keeps_real_compound(self):
|
|
self.assertEqual(remove_linebreak_hyphens("program-\nmieren mit einem KI-Modell"),
|
|
"programmieren mit einem KI-Modell")
|
|
self.assertEqual(remove_linebreak_hyphens("Hardware- und Software"),
|
|
"Hardware- und Software")
|
|
self.assertEqual(remove_linebreak_hyphens("einge\u00adtrichtert"), "eingetrichtert")
|
|
|
|
def test_existing_flattened_ocr_layouts_are_repaired_too(self):
|
|
self.assertEqual(remove_linebreak_hyphens("eingetrich- tert"), "eingetrichtert")
|
|
|
|
def test_german_umlaut_and_acronyms_are_prepared_for_speech(self):
|
|
spoken = prepare_text_for_tts(
|
|
"KI wird jahrlich wichtiger und APIs sind naturlich verfügbar.", "de")
|
|
self.assertIn("Künstliche Intelligenz", spoken)
|
|
self.assertIn("jährlich", spoken)
|
|
self.assertIn("Programmierschnittstellen", spoken)
|
|
self.assertIn("natürlich", spoken)
|
|
|
|
def test_ki_expansion_is_token_bounded(self):
|
|
self.assertEqual(prepare_text_for_tts("KI hilft der KI-Branche.", "de"),
|
|
"Künstliche Intelligenz hilft der Künstliche Intelligenz-Branche.")
|
|
self.assertEqual(prepare_text_for_tts("Kiel bleibt Kiel.", "de"),
|
|
"Kiel bleibt Kiel.")
|
|
|
|
def test_english_acronyms_use_english_expansions(self):
|
|
self.assertEqual(expand_abbreviations("AI uses an API.", "en"),
|
|
"artificial intelligence uses an application programming interface.")
|
|
|
|
def test_unknown_technical_words_and_names_stay_unchanged(self):
|
|
text = "OpenAI und PyMuPDF bleiben unverändert."
|
|
self.assertEqual(normalize_ocr_text(text, "de"), text)
|
|
|
|
def test_spellcheck_only_flags_suspects_and_never_rewrites_text(self):
|
|
class Checker:
|
|
def __contains__(self, word):
|
|
return word != 'softwarentwicklng'
|
|
|
|
from unittest import mock
|
|
with mock.patch('core.pronunciations._german_spellchecker', return_value=Checker()):
|
|
self.assertEqual(
|
|
suspect_ocr_tokens('Die Softwarentwicklng hilft OpenAI.', 'de'),
|
|
['Softwarentwicklng'],
|
|
)
|
|
|
|
def test_article_language_beats_a_short_fragment(self):
|
|
german = ["Die Firma hat das Produkt nicht mit dem Modell verglichen und ist damit zufrieden.", "AI"]
|
|
self.assertEqual(detect_article_language(german), "de")
|
|
self.assertEqual(language_for_block("AI", "de"), "de")
|
|
|
|
def test_a_decisively_english_block_in_a_german_article_is_spoken_in_english(self):
|
|
quote = "The model was trained with the data that they have from the users and will not be shared."
|
|
self.assertEqual(language_for_block(quote, "de"), "en")
|
|
self.assertEqual(language_for_block("Das ist ein kurzer Satz mit AI.", "en"), "en")
|
|
|
|
def test_glossary_from_the_article_context_overrides_and_extends_the_database(self):
|
|
text = "Die SSD und der NAS laufen; ML bleibt."
|
|
self.assertEqual(
|
|
expand_abbreviations(text, "de", {"NAS": "Netzwerkspeicher", "ML": "M L"}),
|
|
"Die S S D und der Netzwerkspeicher laufen; M L bleibt.")
|
|
|
|
def test_letter_spelled_acronyms_work_in_both_languages(self):
|
|
self.assertEqual(expand_abbreviations("Ein USB-Stick per USB.", "en"), "Ein U S B-Stick per U S B.")
|
|
self.assertEqual(expand_abbreviations("Per HDMI.", "de"), "Per H D M I.")
|
|
|
|
def test_unknown_acronyms_are_found_with_context_and_headlines_are_skipped(self):
|
|
text = "Der NAS speichert alles. Der NAS ist schnell. DIE GROSSE ÜBERSCHRIFT HIER. Mit KI und SSD."
|
|
found = find_unknown_acronyms(text, "de")
|
|
self.assertEqual([e["acronym"] for e in found], ["NAS"])
|
|
self.assertEqual(found[0]["count"], 2)
|
|
self.assertIn("NAS", found[0]["context"])
|
|
self.assertEqual(find_unknown_acronyms(text, "de", {"NAS": "x"}), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|