test_fruchst_definition_batch.py
languages/tools/test_fruchst_definition_batch.py · 210 line(s) · py
import gzip
import importlib.util
import json
import tempfile
import unittest
from pathlib import Path
MODULE_PATH = Path(__file__).with_name("fruchst_definition_batch.py")
SPEC = importlib.util.spec_from_file_location("fruchst_definition_batch", MODULE_PATH)
module = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(module)
def source_meta():
return {
edition: {
"url": f"https://example.invalid/{edition}.jsonl.gz",
"dump_revision": "2026-01-01",
"file": edition,
"sha256": "0" * 64,
"size_bytes": 1,
"retrieved_at_utc": "2026-01-01T00:00:00+00:00",
}
for edition in module.EDITIONS
}
def sense(gloss, line, sense_index=1, qids=None):
return {
"glosses": [gloss],
"qids": qids or [],
"tags": [],
"senseid": [],
"source_line": line,
"sense_index": sense_index,
}
class DefinitionBatchTests(unittest.TestCase):
def test_pos_aliases(self):
self.assertEqual(module.pos_key("NOUN"), "noun")
self.assertEqual(module.pos_key("ADP"), "preposition")
self.assertEqual(module.pos_key("AUX"), "auxiliary")
def test_qid_extraction(self):
self.assertEqual(
module.qids_for({"wikidata": ["Q42", "Q1", "bad", "Q42"]}),
["Q1", "Q42"],
)
def test_gzip_extraction(self):
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "fixture.jsonl.gz"
entry = {
"word": "casa",
"lang_code": "es",
"pos": "noun",
"senses": [{"glosses": ["house"], "wikidata": ["Q3947"]}],
}
with gzip.open(path, "wt", encoding="utf-8") as stream:
stream.write(json.dumps(entry, ensure_ascii=False) + "\n")
found = module.extract_edition(path, {"es": {"casa"}})
self.assertIn(("Q3947", "noun"), found["es"]["casa"])
def test_build_queue_filters_known_candidate_pos_and_keeps_review_gate(self):
doc = {"language_tag": "es", "source": {"id": "fixture"}}
words = {"casa": {"pos": "NOUN", "frequency_count": 5}}
noun = sense("house", 1, qids=["Q3947"])
verb = sense("to marry", 2, qids=["Q999"])
senses = {("Q3947", "noun"): [noun], ("Q999", "verb"): [verb]}
by_edition = {
edition: {"es": {"casa": senses}}
for edition in module.EDITIONS
}
queue = module.build_queue("es", doc, words, by_edition, source_meta())
row = queue["entries"][0]
self.assertEqual(len(row["aligned_candidates"]), 1)
self.assertEqual(
row["aligned_candidates"][0]["wikidata_sense_qid"], "Q3947"
)
self.assertEqual(row["edition_candidate_counts"], {
"en": 1, "fr": 1, "es": 1
})
self.assertEqual(queue["counts"]["production_definitions_approved"], 0)
self.assertEqual(
queue["definition_sources"]["en"]["url"],
"https://example.invalid/en.jsonl.gz",
)
def test_glosses_without_qids_are_preserved_for_manual_review(self):
doc = {"language_tag": "es", "source": {"id": "fixture"}}
words = {"casa": {"pos": "NOUN", "frequency_count": 5}}
by_edition = {
edition: {
"es": {"casa": {("", "noun"): [sense("house", 1)]}}
}
for edition in module.EDITIONS
}
queue = module.build_queue("es", doc, words, by_edition, source_meta())
row = queue["entries"][0]
self.assertEqual(row["definition_status"], "edition_gloss_candidates_found")
self.assertEqual(row["aligned_candidates"], [])
self.assertEqual(
row["edition_candidate_counts"], {"en": 1, "fr": 1, "es": 1}
)
self.assertEqual(
row["edition_sense_candidates"]["fr"][0]["glosses"], ["house"]
)
self.assertEqual(queue["counts"]["words_with_edition_gloss_candidates"], 1)
self.assertEqual(queue["counts"]["production_definitions_approved"], 0)
def test_all_common_senses_are_preserved(self):
doc = {"language_tag": "es", "source": {"id": "fixture"}}
words = {"casa": {"pos": "NOUN", "frequency_count": 5}}
senses = {
("Q3947", "noun"): [sense("house", 1, qids=["Q3947"])],
("Q41176", "noun"): [sense("home", 2, qids=["Q41176"])],
}
by_edition = {
edition: {"es": {"casa": senses}}
for edition in module.EDITIONS
}
queue = module.build_queue("es", doc, words, by_edition, source_meta())
self.assertEqual(len(queue["entries"][0]["aligned_candidates"]), 2)
self.assertEqual(queue["counts"]["production_definitions_approved"], 0)
def test_unknown_candidate_pos_preserves_all_pos(self):
doc = {"language_tag": "es", "source": {"id": "fixture"}}
words = {"casa": {"pos": None, "frequency_count": 5}}
senses = {
("Q3947", "noun"): [sense("house", 1, qids=["Q3947"])],
("Q999", "verb"): [sense("to marry", 2, qids=["Q999"])],
}
by_edition = {
edition: {"es": {"casa": senses}}
for edition in module.EDITIONS
}
queue = module.build_queue("es", doc, words, by_edition, source_meta())
row = queue["entries"][0]
self.assertFalse(row["candidate_pos_known"])
self.assertEqual(row["edition_candidate_counts"], {
"en": 2, "fr": 2, "es": 2
})
self.assertEqual(
{candidate["pos"] for candidate in row["edition_sense_candidates"]["en"]},
{"noun", "verb"},
)
self.assertEqual(len(row["aligned_candidates"]), 2)
def test_loader_accepts_frequencywords_candidate_filename(self):
original_targets = module.TARGETS
original_codes = module.WIKTIONARY_LANG_CODES
try:
module.TARGETS = ("zh-Hans",)
module.WIKTIONARY_LANG_CODES = {"zh-Hans": "zh"}
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
entries = [
{
"candidate_rank": rank,
"word": f"词{rank}",
"pos": None,
"frequency_count": 1001 - rank,
}
for rank in range(1, 1001)
]
(directory / "frequencywords-zh-Hans.json").write_text(
json.dumps({
"language_tag": "zh-Hans",
"candidate_count": 1000,
"source": {"dataset": "FrequencyWords"},
"entries": entries,
}, ensure_ascii=False),
encoding="utf-8",
)
docs, words, wanted, paths = module.load_candidates(directory)
self.assertEqual(len(words["zh-Hans"]), 1000)
self.assertEqual(len(wanted["zh-Hans"]), 1000)
self.assertEqual(paths["zh-Hans"].name, "frequencywords-zh-Hans.json")
self.assertEqual(docs["zh-Hans"]["language_tag"], "zh-Hans")
finally:
module.TARGETS = original_targets
module.WIKTIONARY_LANG_CODES = original_codes
def test_explicit_wiktionary_language_code_mapping(self):
original_targets = module.TARGETS
original_codes = module.WIKTIONARY_LANG_CODES
try:
module.TARGETS = ("zh-Hans",)
module.WIKTIONARY_LANG_CODES = {"zh-Hans": "zh"}
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "fixture.jsonl"
entry = {
"word": "词",
"lang_code": "zh",
"pos": "noun",
"senses": [{"glosses": ["word"]}],
}
path.write_text(json.dumps(entry, ensure_ascii=False) + "\n", encoding="utf-8")
found = module.extract_edition(path, {"zh-Hans": {"词"}})
self.assertIn("词", found["zh-Hans"])
self.assertIn(("", "noun"), found["zh-Hans"]["词"])
finally:
module.TARGETS = original_targets
module.WIKTIONARY_LANG_CODES = original_codes
if __name__ == "__main__":
unittest.main()