fruchst_definition_queue.py
languages/tools/fruchst_definition_queue.py · 221 line(s) · py
#!/usr/bin/env python3
"""Build a conservative FRUCHST definition review queue from Kaikki/Wiktextract JSONL.
The tool never selects a sense automatically. It only groups English, French,
and Spanish gloss candidates when all three editions expose the same sense QID
and POS for the target-language headword. Every result remains review-required.
"""
from __future__ import annotations
import argparse
import gzip
import json
import re
import sys
import unicodedata
from pathlib import Path
from itertools import product
EDITIONS = ("en", "fr", "es")
POS_ALIASES = {
"adj": "adjective",
"adv": "adverb",
"aux": "auxiliary",
"noun": "noun",
"verb": "verb",
"pron": "pronoun",
"det": "determiner",
"prep": "preposition",
"postp": "postposition",
"conj": "conjunction",
"intj": "interjection",
"num": "numeral",
"part": "particle",
"proper": "proper noun",
"propn": "proper noun",
}
def norm(value: str) -> str:
return unicodedata.normalize("NFC", value)
def pos_key(value: object) -> str:
raw = str(value or "").strip().casefold().replace("_", " ")
return POS_ALIASES.get(raw, raw)
def qids_for(sense: dict) -> list[str]:
values = sense.get("wikidata", [])
if isinstance(values, str):
values = [values]
if not isinstance(values, list):
return []
return sorted({v for v in values if isinstance(v, str) and re.fullmatch(r"Q[1-9][0-9]*", v)})
def load_words(path: Path) -> tuple[dict, dict[str, dict]]:
doc = json.loads(path.read_text(encoding="utf-8"))
if doc.get("candidate_count") != 1000 or len(doc.get("entries", [])) != 1000:
raise ValueError(f"{path}: expected exactly 1,000 staged candidates")
words = {}
for item in doc["entries"]:
word = norm(item["word"])
if word in words:
raise ValueError(f"{path}: duplicate normalized headword {word!r}")
words[word] = item
return doc, words
def extract_edition(path: Path, target_lang_code: str, wanted: set[str]) -> dict:
found: dict[str, list[dict]] = {}
opener = gzip.open if path.suffix.casefold() == ".gz" else open
with opener(path, "rt", encoding="utf-8") as stream:
for line_no, line in enumerate(stream, 1):
if not line.strip():
continue
try:
entry = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
if entry.get("lang_code") != target_lang_code:
continue
word = entry.get("word")
if not isinstance(word, str):
continue
word = norm(word)
if word not in wanted:
continue
pos = pos_key(entry.get("pos"))
senses = entry.get("senses", [])
if not isinstance(senses, list):
continue
for sense_index, sense in enumerate(senses, 1):
if not isinstance(sense, dict):
continue
glosses = sense.get("glosses") or sense.get("raw_glosses") or []
if isinstance(glosses, str):
glosses = [glosses]
glosses = [norm(g.strip()) for g in glosses if isinstance(g, str) and g.strip()]
if not glosses:
continue
qids = qids_for(sense)
for qid in qids:
found.setdefault(word, []).append({
"qid": qid,
"pos": pos,
"glosses": glosses,
"tags": sorted(t for t in (sense.get("tags") or []) if isinstance(t, str)),
"senseid": sense.get("senseid", []) if isinstance(sense.get("senseid", []), list) else [sense.get("senseid")],
"source_word": word,
"source_pos": entry.get("pos", ""),
"source_line": line_no,
"sense_index": sense_index,
})
return found
def build_queue(candidate_path: Path, target_lang_code: str, edition_paths: dict[str, Path]) -> dict:
candidate_doc, candidates = load_words(candidate_path)
wanted = set(candidates)
by_edition = {
edition: extract_edition(path, target_lang_code, wanted)
for edition, path in edition_paths.items()
}
rows = []
aligned_total = 0
for rank, (word, candidate) in enumerate(candidates.items(), 1):
per_edition = {
edition: by_edition[edition].get(word, [])
for edition in EDITIONS
}
grouped = {edition: {} for edition in EDITIONS}
for edition in EDITIONS:
for sense in per_edition[edition]:
grouped[edition].setdefault((sense["qid"], sense["pos"]), []).append(sense)
common = set(grouped["en"]) & set(grouped["fr"]) & set(grouped["es"])
aligned = []
for qid, pos in sorted(common):
# Preserve every matching sense; never hide polysemy by overwriting.
for en_sense, fr_sense, es_sense in product(
grouped["en"][(qid, pos)], grouped["fr"][(qid, pos)], grouped["es"][(qid, pos)]
):
selected = {"en": en_sense, "fr": fr_sense, "es": es_sense}
aligned.append({
"wikidata_sense_qid": qid,
"pos": pos,
"definitions": {edition: selected[edition]["glosses"] for edition in EDITIONS},
"source_evidence": {
edition: {
"edition": edition,
"source_line": selected[edition]["source_line"],
"sense_index": selected[edition]["sense_index"],
"senseid": selected[edition]["senseid"],
"tags": selected[edition]["tags"],
} for edition in EDITIONS
},
"review_status": "required",
})
if aligned:
aligned_total += 1
rows.append({
"rank": rank,
"word": word,
"candidate_pos": candidate.get("pos"),
"frequency_count": candidate.get("frequency_count"),
"definition_status": "qid_pos_candidates_found" if aligned else "manual_lookup_required",
"aligned_candidates": aligned,
"edition_candidate_counts": {edition: len(per_edition[edition]) for edition in EDITIONS},
})
return {
"schema": "SLeeLa FRUCHST definition review queue v1",
"language_tag": candidate_doc["language_tag"],
"target_wiktionary_lang_code": target_lang_code,
"frequency_candidate_file": str(candidate_path),
"frequency_source": candidate_doc["source"],
"definition_sources": {
edition: {
"file": str(edition_paths[edition]),
"project": "Kaikki/Wiktextract extraction of Wiktionary",
"license": "Wiktionary dual license: CC BY-SA 4.0 and GFDL; verify edition-specific notices and attribution before redistribution",
}
for edition in EDITIONS
},
"alignment_policy": "A candidate triple is emitted only when all three editions contain the exact NFC headword, same sense-level Wikidata QID, and same normalized POS. QID/POS agreement is evidence for human review, not automatic semantic approval. No candidate is selected automatically.",
"counts": {
"candidate_words": len(rows),
"words_with_qid_pos_candidates": aligned_total,
"words_requiring_manual_lookup": len(rows) - aligned_total,
"production_definitions_approved": 0,
},
"production_status": "review_required",
"entries": rows,
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("candidate_file", type=Path)
parser.add_argument("target_lang_code")
parser.add_argument("--en", required=True, type=Path, help="English-edition Wiktextract JSONL")
parser.add_argument("--fr", required=True, type=Path, help="French-edition Wiktextract JSONL")
parser.add_argument("--es", required=True, type=Path, help="Spanish-edition Wiktextract JSONL")
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
try:
queue = build_queue(args.candidate_file, args.target_lang_code, {
"en": args.en, "fr": args.fr, "es": args.es
})
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(queue, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(queue["counts"], sort_keys=True))
except (OSError, ValueError, KeyError, TypeError) as exc:
print(f"fruchst-definition-queue: error: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())