fruchst_definition_batch.py
languages/tools/fruchst_definition_batch.py · 502 line(s) · py
#!/usr/bin/env python3
"""Stage EN/FR/ES Wiktionary gloss candidates for every staged FRUCHST language.
Streams each Kaikki/Wiktextract JSONL(.gz) source once and retains all exact
NFC headword/POS gloss candidates. Wikidata QIDs are optional alignment hints;
this tool never approves a definition or chooses a sense.
"""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import re
import unicodedata
from datetime import datetime, timezone
from pathlib import Path
EDITIONS = ("en", "fr", "es")
SOURCE_CATALOG = {
"en": {
"url": "https://kaikki.org/dictionary/raw-wiktextract-data.jsonl.gz",
"dump_revision": "2026-10-01",
},
"fr": {
"url": "https://kaikki.org/frwiktionary/raw-wiktextract-data.jsonl.gz",
"dump_revision": "2026-10-01",
},
"es": {
"url": "https://kaikki.org/eswiktionary/raw-wiktextract-data.jsonl.gz",
"dump_revision": "2026-11-01",
},
}
# Candidate file tags map explicitly to the language codes used by Wiktionary.
# zh-Hans is sourced from FrequencyWords' zh_cn (Simplified Chinese) list, while
# Wiktionary uses the broader "zh" language code; this mapping is therefore
# disclosed in every resulting queue and requires script review.
TARGETS = (
"en", "zh-Hans", "hi", "es", "ar", "fr", "bn", "pt", "ru", "id",
"ur", "de", "ja", "vi", "te", "tr", "tl", "ta", "fa", "ko", "th",
"it", "pl", "uk", "ml", "ms", "nl", "ro", "kk", "si", "af",
# unilex batch
"pcm", "mr", "ha", "sw", "jv", "gu", "am", "kn", "yo", "my",
"or", "pa-Guru", "su", "prs", "uz", "so", "ceb", "mag", "km", "mai",
)
WIKTIONARY_LANG_CODES = {
"en": "en",
"zh-Hans": "zh",
"hi": "hi",
"es": "es",
"ar": "ar",
"fr": "fr",
"bn": "bn",
"pt": "pt",
"ru": "ru",
"id": "id",
"ur": "ur",
"de": "de",
"ja": "ja",
"vi": "vi",
"te": "te",
"tr": "tr",
"tl": "tl",
"ta": "ta",
"fa": "fa",
"ko": "ko",
"th": "th",
"it": "it",
"pl": "pl",
"uk": "uk",
"ml": "ml",
"ms": "ms",
"nl": "nl",
"ro": "ro",
"kk": "kk",
"si": "si",
"af": "af",
"pcm": "pcm",
"mr": "mr",
"ha": "ha",
"sw": "sw",
"jv": "jv",
"gu": "gu",
"am": "am",
"kn": "kn",
"yo": "yo",
"my": "my",
"or": "or",
"pa-Guru": "pa",
"su": "su",
# Dari: Wiktionary files most Dari vocabulary under Persian. Both codes are
# searched and every gloss records the code it came from.
"prs": ("prs", "fa"),
"uz": "uz",
"so": "so",
"ceb": "ceb",
"mag": "mag",
"km": "km",
"mai": "mai",
}
TARGET_MAPPING_NOTES = {
"zh-Hans": (
"Wiktionary entries use the broad 'zh' code; the candidate word list "
"is the Simplified Chinese zh_cn frequency set. Verify script/variety "
"for every selected sense before approval."
),
"pa-Guru": (
"Wiktionary 'pa' covers Gurmukhi and Shahmukhi Punjabi; candidates are "
"Gurmukhi (Eastern Punjabi), so only Gurmukhi headwords can match exactly."
),
"prs": (
"Dari candidates are matched against Wiktionary Dari ('prs') and Persian "
"('fa') entries. Each gloss records source_lang_code; a Persian gloss "
"must be confirmed as valid for Dari usage before approval."
),
"uz": (
"Candidates are Cyrillic-script Uzbek; Wiktionary Uzbek lemmas are mostly "
"Latin-script, so exact matches are expected to be sparse."
),
"sw": "Wiktionary 'sw' (Swahili); candidates are Standard Swahili web text.",
}
def wiktionary_codes(language_tag: str) -> tuple[str, ...]:
value = WIKTIONARY_LANG_CODES[language_tag]
return (value,) if isinstance(value, str) else tuple(value)
POS_ALIASES = {
"adj": "adjective",
"adv": "adverb",
"aux": "auxiliary",
"noun": "noun",
"verb": "verb",
"pron": "pronoun",
"det": "determiner",
"prep": "preposition",
"adp": "preposition",
"postp": "postposition",
"conj": "conjunction",
"cconj": "conjunction",
"sconj": "conjunction",
"intj": "interjection",
"num": "numeral",
"part": "particle",
"proper": "proper noun",
"propn": "proper noun",
}
def norm(value: str) -> str:
return unicodedata.normalize("NFC", value)
def pos_key(value: object) -> str:
raw = str(value or "").strip().casefold().replace("_", " ")
return POS_ALIASES.get(raw, raw)
def qids_for(sense: dict) -> list[str]:
values = sense.get("wikidata", [])
if isinstance(values, str):
values = [values]
if not isinstance(values, list):
return []
return sorted({
value for value in values
if isinstance(value, str) and re.fullmatch(r"Q[1-9][0-9]*", value)
})
def candidate_path_for(directory: Path, language_tag: str) -> Path:
"""Find exactly one known candidate file for a registered language tag."""
matches = [
path for prefix in ("wordhoard", "frequencywords", "unilex")
if (path := directory / f"{prefix}-{language_tag}.json").is_file()
]
if len(matches) != 1:
raise ValueError(
f"{directory}: expected exactly one wordhoard/frequencywords/unilex file "
f"for {language_tag}, found {len(matches)}"
)
return matches[0]
def load_candidates(directory: Path):
docs, words, wanted, candidate_paths = {}, {}, {}, {}
if set(WIKTIONARY_LANG_CODES) != set(TARGETS):
raise ValueError("TARGETS and WIKTIONARY_LANG_CODES must cover the same tags")
for language_tag in TARGETS:
path = candidate_path_for(directory, language_tag)
doc = json.loads(path.read_text(encoding="utf-8"))
entries = doc.get("entries", [])
if doc.get("language_tag") != language_tag:
raise ValueError(f"{path}: language_tag does not match {language_tag}")
if doc.get("candidate_count") != 1000 or len(entries) != 1000:
raise ValueError(f"{path}: expected exactly 1,000 candidates")
mapped = {}
for expected_rank, item in enumerate(entries, 1):
if not isinstance(item, dict):
raise ValueError(f"{path}: candidate {expected_rank} is not an object")
if item.get("candidate_rank") != expected_rank:
raise ValueError(f"{path}: candidate ranks must be contiguous 1..1000")
raw_word = item.get("word")
if not isinstance(raw_word, str) or not raw_word.strip():
raise ValueError(f"{path}: candidate {expected_rank} has an invalid word")
word = norm(raw_word)
if word != raw_word:
raise ValueError(f"{path}: candidate {expected_rank} is not NFC-normalized")
if word in mapped:
raise ValueError(f"{path}: duplicate NFC headword {word!r}")
mapped[word] = item
docs[language_tag] = doc
words[language_tag] = mapped
wanted[language_tag] = set(mapped)
candidate_paths[language_tag] = path
return docs, words, wanted, candidate_paths
def hash_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def extract_edition(path: Path, wanted: dict[str, set[str]]):
"""Extract target-language entries by explicit Wiktionary language code."""
tags_by_wiktionary_code = {}
for language_tag in WIKTIONARY_LANG_CODES:
for wiktionary_code in wiktionary_codes(language_tag):
tags_by_wiktionary_code.setdefault(wiktionary_code, []).append(language_tag)
found = {language_tag: {} for language_tag in TARGETS}
opener = gzip.open if path.suffix.casefold() == ".gz" else open
with opener(path, "rt", encoding="utf-8") as stream:
for line_no, line in enumerate(stream, 1):
if not line.strip():
continue
try:
entry = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
matching_tags = tags_by_wiktionary_code.get(entry.get("lang_code"), ())
if not matching_tags:
continue
raw_word = entry.get("word")
if not isinstance(raw_word, str):
continue
word = norm(raw_word)
tags = [tag for tag in matching_tags if word in wanted[tag]]
if not tags:
continue
pos = pos_key(entry.get("pos"))
senses = entry.get("senses", [])
if not isinstance(senses, list):
continue
for sense_index, sense in enumerate(senses, 1):
if not isinstance(sense, dict):
continue
glosses = sense.get("glosses") or sense.get("raw_glosses") or []
if isinstance(glosses, str):
glosses = [glosses]
glosses = [
norm(gloss.strip())
for gloss in glosses
if isinstance(gloss, str) and gloss.strip()
]
if not glosses:
continue
qids = qids_for(sense)
record = {
"glosses": glosses,
"qids": qids,
"tags": sorted(
tag for tag in (sense.get("tags") or [])
if isinstance(tag, str)
),
"senseid": sense.get("senseid", []),
"source_line": line_no,
"sense_index": sense_index,
"source_lang_code": entry.get("lang_code"),
}
for language_tag in tags:
by_word = found[language_tag].setdefault(word, {})
if qids:
for qid in qids:
by_word.setdefault((qid, pos), []).append(record)
else:
# QIDs are optional metadata; never discard an otherwise
# usable gloss just because Wiktextract has no QID.
by_word.setdefault(("", pos), []).append(record)
return found
def build_queue(
language_tag,
doc,
words,
by_edition,
source_meta,
candidate_file=None,
):
rows, words_with_matches, total_matches = [], 0, 0
codes = wiktionary_codes(language_tag)
target_wiktionary_code = codes[0] if len(codes) == 1 else list(codes)
if candidate_file is None:
candidate_file = Path(f"languages/datasets/frequency-candidates/wordhoard-{language_tag}.json")
for rank, (word, candidate) in enumerate(words.items(), 1):
wanted_pos = pos_key(candidate.get("pos"))
groups = {}
for edition in EDITIONS:
raw = by_edition[edition].get(language_tag, {}).get(word, {})
# A null POS from the frequency source means "unknown", not a
# license to discard every sense. Retain all POS values in that case.
groups[edition] = {
key: senses for key, senses in raw.items()
if not wanted_pos or key[1] == wanted_pos
}
common = (
set(groups["en"]) & set(groups["fr"]) & set(groups["es"])
)
common = {
(qid, pos) for qid, pos in common
if qid and (not wanted_pos or pos == wanted_pos)
}
aligned = []
for qid, pos in sorted(common):
total_matches += 1
aligned.append({
"wikidata_sense_qid": qid,
"pos": pos,
"definitions": {
edition: [
sense["glosses"]
for sense in groups[edition][(qid, pos)]
]
for edition in EDITIONS
},
"source_evidence": {
edition: [{
"edition": edition,
"source_line": sense["source_line"],
"sense_index": sense["sense_index"],
"senseid": sense["senseid"],
"tags": sense["tags"],
"source_lang_code": sense.get("source_lang_code"),
} for sense in groups[edition][(qid, pos)]]
for edition in EDITIONS
},
"review_status": "required",
})
edition_candidates = {}
for edition in EDITIONS:
candidates = []
for (qid, pos), senses in sorted(groups[edition].items()):
for sense in senses:
candidates.append({
"wikidata_sense_qids": [qid] if qid else [],
"pos": pos,
"glosses": sense["glosses"],
"tags": sense["tags"],
"senseid": sense["senseid"],
"source_line": sense["source_line"],
"sense_index": sense["sense_index"],
"source_lang_code": sense.get("source_lang_code"),
})
edition_candidates[edition] = candidates
if aligned:
words_with_matches += 1
rows.append({
"rank": rank,
"word": word,
"candidate_pos": candidate.get("pos"),
"candidate_pos_known": bool(wanted_pos),
"frequency_count": candidate.get("frequency_count"),
"definition_status": (
"qid_pos_candidates_found" if aligned else
"edition_gloss_candidates_found"
if all(edition_candidates[edition] for edition in EDITIONS) else
"manual_lookup_required"
),
"aligned_candidates": aligned,
"edition_sense_candidates": edition_candidates,
"edition_candidate_counts": {
edition: len(edition_candidates[edition])
for edition in EDITIONS
},
})
words_with_all_edition_glosses = sum(
1 for row in rows
if all(row["edition_sense_candidates"][edition] for edition in EDITIONS)
)
return {
"schema": "SLeeLa FRUCHST definition review queue v4",
"language_tag": doc["language_tag"],
"target_wiktionary_lang_code": target_wiktionary_code,
"target_language_mapping_note": TARGET_MAPPING_NOTES.get(language_tag),
"frequency_candidate_file": candidate_file.as_posix(),
"frequency_source": doc["source"],
"definition_sources": {
edition: {
"project": "Kaikki/Wiktextract extraction of Wiktionary",
"url": source_meta[edition]["url"],
"dump_revision": source_meta[edition]["dump_revision"],
"file": source_meta[edition]["file"],
"sha256": source_meta[edition]["sha256"],
"size_bytes": source_meta[edition]["size_bytes"],
"retrieved_at_utc": source_meta[edition]["retrieved_at_utc"],
"license": (
"Wiktionary dual license: CC BY-SA 4.0 and GFDL; "
"preserve attribution and verify edition notices"
),
} for edition in EDITIONS
},
"alignment_policy": (
"Exact NFC headword and explicit target-language-to-Wiktionary-code mapping. "
"Candidate POS is applied when supplied; if the frequency source has no POS, "
"all matching POS senses are retained. All edition glosses are preserved "
"independently, including senses without Wikidata QIDs. Shared sense-level "
"QIDs create optional cross-edition alignment candidates only. Every "
"candidate remains for human review; no sense is auto-selected or approved."
),
"counts": {
"candidate_words": len(rows),
"words_with_qid_pos_candidates": words_with_matches,
"words_with_edition_gloss_candidates": words_with_all_edition_glosses,
"words_requiring_manual_lookup": len(rows) - words_with_all_edition_glosses,
"qid_pos_candidate_triples": total_matches,
"production_definitions_approved": 0,
},
"production_status": "review_required",
"entries": rows,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--candidates", type=Path, required=True)
parser.add_argument("--en", type=Path, required=True)
parser.add_argument("--fr", type=Path, required=True)
parser.add_argument("--es", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
docs, words, wanted, candidate_paths = load_candidates(args.candidates)
source_paths = {"en": args.en, "fr": args.fr, "es": args.es}
retrieved = datetime.now(timezone.utc).isoformat()
source_meta = {
edition: {
**SOURCE_CATALOG[edition],
"file": str(path),
"sha256": hash_file(path),
"size_bytes": path.stat().st_size,
"retrieved_at_utc": retrieved,
}
for edition, path in source_paths.items()
}
by_edition = {
edition: extract_edition(path, wanted)
for edition, path in source_paths.items()
}
args.output.mkdir(parents=True, exist_ok=True)
for language_tag in TARGETS:
candidate_file = candidate_paths[language_tag]
queue = build_queue(
language_tag,
docs[language_tag],
words[language_tag],
by_edition,
source_meta,
candidate_file,
)
output_path = args.output / f"{candidate_file.stem}.review.json"
output_path.write_text(
json.dumps(queue, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(json.dumps({
"language": language_tag,
"candidate_file": candidate_file.name,
**queue["counts"],
}, sort_keys=True))
if __name__ == "__main__":
main()