158 lines
7.6 KiB
Python
158 lines
7.6 KiB
Python
"""Spelling findings: what is flagged, what is deliberately not, and the agent-fix guards."""
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
from core import spelling
|
|
|
|
DE = {"die", "software", "entwicklung", "braucht", "grafik", "karte", "jährlich", "modell", "ist", "gut",
|
|
"das", "und", "test", "tests", "der", "hilft", "nicht", "datenbank", "daten", "bank", "wächst", "jahrlich", "sowie", "laufen", "modell", "sind", "sagt", "heute"}
|
|
EN = {"the", "software", "engineers", "use", "graphics", "card", "message", "receive", "this", "database", "then", "and", "help"}
|
|
|
|
|
|
class Checker:
|
|
def __init__(self, words):
|
|
self.words = set(words)
|
|
|
|
def __contains__(self, word):
|
|
return word.casefold() in self.words
|
|
|
|
def candidates(self, word):
|
|
return {w for w in self.words if spelling._edit_distance(word, w, 1) <= 1 and w != word}
|
|
|
|
word_usage_frequency = None
|
|
|
|
|
|
def patched(fn):
|
|
def wrapper(*args, **kwargs):
|
|
with mock.patch("core.pronunciations._german_spellchecker", return_value=Checker(DE)), \
|
|
mock.patch("core.pronunciations._english_spellchecker", return_value=Checker(EN)):
|
|
return fn(*args, **kwargs)
|
|
return wrapper
|
|
|
|
|
|
class FindingTests(unittest.TestCase):
|
|
@patched
|
|
def test_a_german_typo_is_flagged_with_a_suggestion_and_position(self):
|
|
text = "Die Datenbank wächst jährlich, der Tesst ist gut."
|
|
[issue] = spelling.find_spelling_issues(text, "de")
|
|
self.assertEqual((issue["token"], text[issue["start"]:issue["end"]]), ("Tesst", "Tesst"))
|
|
self.assertIn("Test", issue["suggestions"])
|
|
|
|
@patched
|
|
def test_german_compounds_of_known_words_are_not_typos(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Die Softwareentwicklung braucht Grafikkarte.", "de"), [])
|
|
self.assertEqual(spelling.find_spelling_issues("Die Datenbankx", "de")[0]["token"], "Datenbankx")
|
|
self.assertEqual(spelling.find_spelling_issues("Die Datenbanks", "de"), [])
|
|
|
|
@patched
|
|
def test_names_acronyms_short_words_and_camel_case_brands_are_skipped(self):
|
|
text = "Die OpenAI und PyMuPDF sowie NASA und Xyz laufen."
|
|
self.assertEqual([i["token"] for i in spelling.find_spelling_issues(text, "de")], [])
|
|
|
|
@patched
|
|
def test_english_words_inside_a_german_text_are_not_typos(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Die message ist gut.", "de"), [])
|
|
|
|
@patched
|
|
def test_english_finds_typos_and_sentence_initial_capitals_but_not_midsentence_names(self):
|
|
issues = spelling.find_spelling_issues("Recieve the messge. Then Nvidia and Zorblax help.", "en")
|
|
self.assertEqual([i["token"] for i in issues], ["Recieve", "messge"])
|
|
self.assertIn("message", issues[1]["suggestions"])
|
|
|
|
@patched
|
|
def test_the_reviewers_dictionary_silences_a_word(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Der Tesst ist gut.", "de", ignore={"tesst"}), [])
|
|
|
|
@patched
|
|
def test_hyphenated_parts_are_checked_separately(self):
|
|
[issue] = spelling.find_spelling_issues("Das KI-Modelll ist gut.", "de")
|
|
self.assertEqual(issue["token"], "Modelll")
|
|
self.assertEqual("Das KI-Modelll ist gut."[issue["start"]:issue["end"]], "Modelll")
|
|
|
|
@patched
|
|
def test_recurring_identical_unknown_words_are_treated_as_vocabulary(self):
|
|
blocks = [{"id": f"b{i}", "page": 1, "text": "Der Zorbex ist gut."} for i in range(3)]
|
|
blocks.append({"id": "x", "page": 1, "text": "Der Tesst ist gut."})
|
|
self.assertEqual([i["token"] for i in spelling.block_issues(blocks, "de")], ["Tesst"])
|
|
|
|
@patched
|
|
def test_block_issues_carry_block_id_page_and_context(self):
|
|
[issue] = spelling.block_issues([{"id": "b7", "page": 4, "text": "Der Tesst ist gut."}], "de")
|
|
self.assertEqual((issue["block_id"], issue["page"]), ("b7", 4))
|
|
self.assertIn("Tesst", issue["context"])
|
|
|
|
|
|
class LineBreakAndSuspendedCompoundTests(unittest.TestCase):
|
|
@patched
|
|
def test_a_printed_line_break_is_checked_as_one_word_and_edited_as_the_printed_span(self):
|
|
text = "Der Tes- st ist gut."
|
|
[issue] = spelling.find_spelling_issues(text, "de")
|
|
self.assertEqual((issue["token"], issue["raw"]), ("Tesst", "Tes- st"))
|
|
self.assertEqual(spelling.apply_fix(text, issue["start"], issue["end"], issue["raw"], "Test"), "Der Test ist gut.")
|
|
|
|
@patched
|
|
def test_a_correctly_broken_word_and_a_suspended_compound_are_not_findings(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Die Daten- bank ist gut.", "de"), [])
|
|
self.assertEqual(spelling.find_spelling_issues("Die Daten- und Datenbank.", "de"), [])
|
|
|
|
@patched
|
|
def test_the_bare_stem_before_a_hyphen_is_not_a_finding(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Die Daten- und Modell gut.", "de"), [])
|
|
self.assertEqual(spelling.find_spelling_issues("Das Daten-Modell ist gut.", "de"), [])
|
|
self.assertEqual([i["token"] for i in spelling.find_spelling_issues("Das Dattten-Modell ist gut.", "de")], ["Dattten"])
|
|
|
|
@patched
|
|
def test_a_glued_suspended_compound_gets_a_one_click_repair_that_passes_the_guard(self):
|
|
[issue] = spelling.find_spelling_issues("Die Datenbankund Tests sind gut.", "de")
|
|
self.assertEqual(issue["suggestions"], ["Datenbank- und"])
|
|
self.assertTrue(spelling.accept_agent_fix(issue, "Datenbank- und"))
|
|
self.assertFalse(spelling.accept_agent_fix(issue, "Datenbank- und mehr"))
|
|
|
|
|
|
class NamesBrandsAndVocabularyTests(unittest.TestCase):
|
|
@patched
|
|
def test_built_in_company_and_product_names_are_never_findings(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Die Nvidia und Anthropic sind gut.", "de"), [])
|
|
|
|
@patched
|
|
def test_two_unknown_capitalised_words_in_a_row_are_a_persons_name(self):
|
|
self.assertEqual(spelling.find_spelling_issues("Das sagt Juliane Jarke heute.", "de"), [])
|
|
self.assertEqual([i["token"] for i in spelling.find_spelling_issues("Das sagt Jarke heute.", "de")], ["Jarke"])
|
|
|
|
@patched
|
|
def test_a_word_printed_repeatedly_in_the_issue_is_vocabulary(self):
|
|
text = "Die Zorbex ist gut."
|
|
self.assertEqual([i["token"] for i in spelling.find_spelling_issues(text, "de")], ["Zorbex"])
|
|
self.assertEqual(spelling.find_spelling_issues(text, "de", vocabulary={"zorbex"}), [])
|
|
|
|
@patched
|
|
def test_a_missing_umlaut_stays_flagged_even_though_it_recurs(self):
|
|
# 'jahrlich' is in the fake German dictionary as its umlaut form 'jährlich'
|
|
found = spelling.find_spelling_issues("Das Modell ist gut und wachst nicht.", "de", vocabulary={"wachst"})
|
|
self.assertEqual([i["token"] for i in found], ["wachst"])
|
|
|
|
|
|
class AgentGuardTests(unittest.TestCase):
|
|
ISSUE = {"token": "Tesst", "language": "de", "suggestions": ["Tests", "Test"]}
|
|
|
|
@patched
|
|
def test_a_listed_suggestion_is_accepted(self):
|
|
self.assertTrue(spelling.accept_agent_fix(self.ISSUE, "Test"))
|
|
|
|
@patched
|
|
def test_a_known_word_within_two_edits_is_accepted(self):
|
|
self.assertTrue(spelling.accept_agent_fix({"token": "jahrlch", "language": "de", "suggestions": []}, "jährlich"))
|
|
|
|
@patched
|
|
def test_rewrites_phrases_unknown_words_and_far_words_are_refused(self):
|
|
for bad in ("", "Tesst", "Der Test", "Test!", "Datenbank", "Qxzv", "Test1"):
|
|
self.assertFalse(spelling.accept_agent_fix(self.ISSUE, bad), bad)
|
|
|
|
def test_apply_fix_refuses_stale_offsets(self):
|
|
self.assertEqual(spelling.apply_fix("ab Tesst cd", 3, 8, "Tesst", "Test"), "ab Test cd")
|
|
self.assertIsNone(spelling.apply_fix("ab Xesst cd", 3, 8, "Tesst", "Test"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|