test_fruchst_english_round.py
languages/tools/test_fruchst_english_round.py · 230 line(s) · py
import gzip
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path
MODULE_PATH = Path(__file__).with_name("fruchst_english_round.py")
SPEC = importlib.util.spec_from_file_location("fruchst_english_round", MODULE_PATH)
module = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(module)
import fruchst_lexemes as lexeme_module # noqa: E402 (importable once the module above set sys.path)
def review(tag, words, senses_by_word):
entries = []
for rank, word in enumerate(words, 1):
en = senses_by_word.get(word, [])
entries.append({
"rank": rank, "word": word, "candidate_pos": None, "frequency_count": 10 - rank,
"definition_status": "manual_lookup_required", "aligned_candidates": [],
"edition_sense_candidates": {"en": en, "fr": [], "es": []},
"edition_candidate_counts": {"en": len(en), "fr": 0, "es": 0},
})
return {"language_tag": tag, "target_language_mapping_note": None,
"definition_sources": {"en": {"url": "https://example.invalid/en.jsonl.gz"}}, "entries": entries}
def rsense(gloss, line, index=1, pos="noun", tags=(), code=None):
return {"wikidata_sense_qids": [], "pos": pos, "glosses": [gloss], "tags": list(tags), "senseid": [],
"source_line": line, "sense_index": index, "source_lang_code": code}
class CleanGlossTests(unittest.TestCase):
def test_wiki_link_markup_is_cleaned_for_display(self):
text, issues = module.clean_gloss(
"[[Appendix:Swahili_noun_classes#N class|n class_((IX))]] inflected form of -a")
self.assertEqual(text, "n class (IX) inflected form of -a")
self.assertIn("wiki_link_markup", issues)
def test_stray_bracket(self):
text, issues = module.clean_gloss("concave part of a [[butter] churn")
self.assertEqual(text, "concave part of a butter churn")
self.assertTrue(issues)
def test_pointer_heuristic(self):
self.assertTrue(module.is_pointer_text("plural of mtu"))
self.assertTrue(module.is_pointer_text("feminine singular of هٰذَا (hāḏā)"))
self.assertFalse(module.is_pointer_text("To form a mental picture of, to perceive"))
self.assertFalse(module.is_pointer_text("time"))
class EnglishRoundTests(unittest.TestCase):
def run_round(self, dump_entries, reviews, variants=None):
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "en.jsonl.gz"
with gzip.open(path, "wt", encoding="utf-8") as stream:
for entry in dump_entries:
stream.write(json.dumps(entry, ensure_ascii=False) + "\n")
scanner = module.Scanner(reviews, variants or {}, module.english_ranks(reviews))
scanner.first_pass(path)
wanted = module.pointer_requests(scanner)
requested = {c: set(w) for c, w in wanted.items()}
for _ in range(module.MAX_HOPS):
if not wanted:
break
scanner.lemma_pass(path, wanted)
wanted = module.next_hop_requests(scanner, requested)
for c, w in wanted.items():
requested.setdefault(c, set()).update(w)
return {tag: module.build_profile(Path(f"{tag}.review.json"), doc, scanner, None)
for tag, doc in reviews.items()}
def test_form_of_resolved_corroboration_and_translation_only(self):
dump = [
# line 1: lemma هٰذَا (page هذا)
{"word": "هذا", "lang_code": "ar", "pos": "pron",
"senses": [{"glosses": ["this (masculine singular)"]}]},
# line 2: هذه as a form of هٰذَا
{"word": "هذه", "lang_code": "ar", "pos": "pron",
"forms": [{"form": "hāḏihī", "tags": ["romanization"]}],
"senses": [{"glosses": ["feminine singular of هٰذَا (hāḏā)"], "tags": ["form-of"],
"form_of": [{"word": "هٰذَا"}]}]},
# line 3: من with a rare noun sense and the preposition
{"word": "من", "lang_code": "ar", "pos": "noun",
"senses": [{"glosses": ["favor, benefit"], "tags": ["rare"]}]},
{"word": "من", "lang_code": "ar", "pos": "prep",
"senses": [{"glosses": ["from, of"],
"examples": [{"text": "مِنَ البَيْت", "english": "from the house"}]}]},
# English entries listing the Arabic words in translation tables
{"word": "from", "lang_code": "en", "pos": "prep", "senses": [{"glosses": ["Away from"]}],
"translations": [{"lang_code": "ar", "word": "مِن", "sense": "with the source or provenance of"}]},
{"word": "time", "lang_code": "en", "pos": "noun",
"senses": [{"glosses": ["The inevitable progression into the future"],
"translations": [{"code": "ar", "word": "وَقْت", "roman": "waqt"}]}]},
]
ar = review("ar", ["هذه", "من", "وقت"], {
"هذه": [rsense("feminine singular of هٰذَا (hāḏā)", 2, pos="pronoun", tags=["form-of"], code="ar")],
"من": [rsense("favor, benefit", 3, tags=["rare"], code="ar"),
rsense("from, of", 4, pos="preposition", code="ar")],
})
en = review("en", ["from", "time"], {})
out = self.run_round(dump, {"ar": ar, "en": en})["ar"]
rows = {r["word"]: r for r in out["entries"]}
hadhihi = rows["هذه"]
self.assertEqual(hadhihi["english_status"], "lemma_of_form_found")
self.assertEqual(hadhihi["lead_candidate"]["text"], "this (masculine singular)")
self.assertEqual(hadhihi["definitions"][0]["lemma"], "هذا")
self.assertEqual(hadhihi["romanizations"], ["hāḏihī"])
min_ = rows["من"]
self.assertEqual(min_["lead_candidate"]["text"], "from, of")
self.assertEqual(min_["definitions"][0]["corroborated_by_translation_tables"], ["from"])
self.assertEqual(min_["definitions"][0]["examples"][0]["english"], "from the house")
self.assertEqual(min_["definitions"][1]["register"], ["rare"])
waqt = rows["وقت"]
self.assertEqual(waqt["english_status"], "translation_equivalent_found")
self.assertTrue(waqt["definitions"][0]["text"].startswith("time"))
self.assertEqual(waqt["definitions"][0]["evidence"]["match"], "optional_marks")
self.assertEqual(waqt["lead_candidate"]["review_status"], "required")
counts = out["counts"]
self.assertEqual(counts["words_with_any_english_definition"], 3)
self.assertEqual(counts["words_gaining_english_definition"], 1)
self.assertEqual(counts["pointer_senses_resolved"], 1)
self.assertEqual(counts["production_definitions_approved"], 0)
def test_two_hop_pointer_and_hanja(self):
dump = [
{"word": "go", "lang_code": "pl", "pos": "pron",
"senses": [{"glosses": ["genitive of on"], "tags": ["form-of"], "form_of": [{"word": "on"}]}]},
{"word": "on", "lang_code": "pl", "pos": "pron",
"senses": [{"glosses": ["alternative form of jego"], "tags": ["alt-of"], "alt_of": [{"word": "jego"}]}]},
{"word": "jego", "lang_code": "pl", "pos": "pron", "senses": [{"glosses": ["him"]}]},
{"word": "왜", "lang_code": "ko", "pos": "syllable", "senses": [{"glosses": ["娃"]}]},
{"word": "娃", "lang_code": "ko", "pos": "character", "senses": [{"glosses": ["pretty, beautiful"]}]},
]
pl = review("pl", ["go"], {"go": [rsense("genitive of on", 1, pos="pronoun", tags=["form-of"], code="pl")]})
ko = review("ko", ["왜"], {"왜": [rsense("娃", 4, pos="syllable", code="ko")]})
out = self.run_round(dump, {"pl": pl, "ko": ko})
go = out["pl"]["entries"][0]
self.assertEqual(go["lead_candidate"]["text"], "him")
self.assertEqual(go["definitions"][0]["hops"], 2)
wae = out["ko"]["entries"][0]
self.assertEqual(wae["lead_candidate"]["text"], "pretty, beautiful")
self.assertIn("hanja_reading_gloss", wae["gloss_notes"][0]["issues"])
def test_review_glosses_are_not_modified(self):
ar = review("ar", ["هذه"], {"هذه": [rsense("feminine singular of هٰذَا (hāḏā)", 1, tags=["form-of"])]})
before = json.dumps(ar, sort_keys=True)
self.run_round([], {"ar": ar})
self.assertEqual(json.dumps(ar, sort_keys=True), before)
def test_rule_based_keys_and_flags(self):
dump = [
{"word": "당신", "lang_code": "ko", "pos": "pron", "senses": [{"glosses": ["you (honorific)"]}]},
{"word": "그거", "lang_code": "ko", "pos": "pron", "senses": [{"glosses": ["that thing"]}]},
{"word": "ഞാൻ", "lang_code": "ml", "pos": "pron", "senses": [{"glosses": ["I"]}]},
{"word": "что", "lang_code": "ru", "pos": "pron", "senses": [{"glosses": ["what"]}]},
]
filler = [chr(0xAC00 + 28 * i) for i in range(25)]
ko = review("ko", ["당신이", "그건", "있는", "사람", "you"] + filler, {"사람": [rsense("person", 9, code="ko")]})
ml = review("ml", ["ഞാന്"], {})
uk = review("uk", ["что", "так"], {"так": [rsense("so", 9, pos="adverb", code="uk")]})
out = self.run_round(dump, {"ko": ko, "ml": ml, "uk": uk})
rows = {r["word"]: r for r in out["ko"]["entries"]}
self.assertEqual(rows["당신이"]["lead_candidate"]["text"], "you (honorific)")
self.assertEqual(rows["당신이"]["definitions"][0]["affix"], "이")
self.assertEqual(rows["그건"]["definitions"][0]["variant_method"], "contracted_particle")
self.assertEqual(rows["있는"]["definitions"], [])
self.assertEqual(rows["you"]["flags"].get("script_mismatch"), "LATIN")
self.assertEqual(out["ml"]["entries"][0]["lead_candidate"]["text"], "I")
self.assertEqual(out["ml"]["entries"][0]["definitions"][0]["variant_method"], "chillu_normalized")
chto = out["uk"]["entries"][0]
self.assertEqual(chto["definitions"], [])
self.assertEqual(chto["flags"]["found_only_in_other_languages"], ["ru"])
self.assertEqual(out["uk"]["counts"]["other_language_codes"], {"ru": 1})
def test_wikidata_lexemes(self):
lexemes = [
{"type": "lexeme", "id": "L1", "language": "Q9610", "lexicalCategory": "Q1084",
"lemmas": {"bn": {"language": "bn", "value": "বই"}},
"forms": [{"id": "L1-F2", "representations": {"bn": {"language": "bn", "value": "বইটি"}},
"grammaticalFeatures": ["Q110786"]}],
"senses": [{"id": "L1-S1", "glosses": {"en": {"language": "en", "value": "book"}},
"claims": {"P5137": [{"mainsnak": {"datavalue": {"value": {"id": "Q571"}}}}]}},
{"id": "L1-S2", "glosses": {"bn": {"language": "bn", "value": "x"}},
"claims": {"P5972": [{"mainsnak": {"datavalue": {"value": {"id": "L9-S1"}}}}]}}]},
{"type": "lexeme", "id": "L9", "language": "Q1860", "lexicalCategory": "Q1084",
"lemmas": {"en": {"language": "en", "value": "volume"}},
"senses": [{"id": "L9-S1", "glosses": {"en": {"language": "en", "value": "a single book"}}}]},
]
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "lexemes.json.gz"
with gzip.open(path, "wt", encoding="utf-8") as stream:
stream.write("[\n" + ",\n".join(json.dumps(x, ensure_ascii=False) for x in lexemes) + "\n]\n")
hits = lexeme_module.scan_lexemes(path, {"bn": {"বই", "বইটি"}})
book = hits[("bn", "বই")]
self.assertEqual(book[0]["gloss"], "book")
self.assertEqual(book[0]["items"], ["Q571"])
self.assertEqual(book[1]["english_translations"][0]["english_lemma"], "volume")
self.assertEqual(hits[("bn", "বইটি")][0]["match"], "form")
bn = review("bn", ["বই", "বইটি"], {})
scanner = module.Scanner({"bn": bn}, {}, {})
scanner.lexemes = hits
out = module.build_profile(Path("bn.review.json"), bn, scanner, None)
rows = {r["word"]: r for r in out["entries"]}
self.assertEqual(rows["বই"]["english_status"], "lexeme_sense_found")
self.assertEqual(rows["বই"]["lead_candidate"]["text"], "book")
self.assertEqual(rows["বইটি"]["definitions"][0]["display"], "form of বই: book")
def test_lexeme_lemma_uses_profile_script(self):
lexeme = {"type": "lexeme", "id": "L5", "language": "Q58635", "lexicalCategory": "Q1084",
"lemmas": {"pnb": {"language": "pnb", "value": "دِن"}, "pa": {"language": "pa", "value": "ਦਿਨ"}},
"forms": [{"id": "L5-F3", "representations": {"pa": {"language": "pa", "value": "ਦਿਨੀਂ"}}}],
"senses": [{"id": "L5-S1", "glosses": {"en": {"language": "en", "value": "day"}}}]}
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "lexemes.json"
path.write_text("[\n" + json.dumps(lexeme, ensure_ascii=False) + "\n]\n", encoding="utf-8")
hits = lexeme_module.scan_lexemes(path, {"pa-Guru": {"ਦਿਨੀਂ"}})
self.assertEqual(hits[("pa-Guru", "ਦਿਨੀਂ")][0]["lemma"], "ਦਿਨ")
if __name__ == "__main__":
unittest.main()