fruchst_english_round.py
languages/tools/fruchst_english_round.py · 918 line(s) · py
#!/usr/bin/env python3
"""English round: add every English definition English Wiktionary can supply.
Add-only companion to the review queues. It never edits or removes a review
gloss; it writes one ``<stem>.english.json`` per profile that collects, ranks
and explains all English definition evidence for each headword:
tier 1 own_sense the headword's own English Wiktionary sense
tier 2 lemma_of_form a form-of / alt-of / synonym-of sense resolved
to the lemma's own definitions (up to 2 hops),
and Korean hanja-reading senses resolved to the
hanja character's definition
tier 3 lexeme_sense Wikidata Lexemes (CC0): English gloss or English
translation link of a lexeme whose lemma or
form is the headword (optional --lexemes)
tier 4 form_of_gloss a pointer gloss whose lemma could not be found,
kept verbatim (add-only)
tier 5 close_variant the close-variant pass (definition-variants/),
with its own form-of senses resolved as well,
plus rule-based keys for words still without
English (Malayalam chillu spelling, Korean
particles, Amharic prefixes)
tier 6 translation_equivalent English Wiktionary translation tables that list
the headword (exact or with optional marks)
Within tier 1, senses corroborated by a translation table (the English word
that lists the headword also appears in the gloss) rank first; obsolete,
archaic, rare and similar senses rank last; otherwise Wiktionary order holds.
The first ranked definition is recorded as ``lead_candidate``. It is a
reviewer's starting point only: nothing is selected or approved here.
Gloss-quality notes (wiki-markup remnants, duplicates, unresolved pointers,
non-English glosses) are recorded with a cleaned display text; the original
glosses stay exactly as staged in the review queue.
"""
from __future__ import annotations
import argparse
import gzip
import json
import re
import sys
import unicodedata
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from fruchst_definition_batch import TARGETS, norm, pos_key, wiktionary_codes # noqa: E402
from fruchst_close_variants import ALEF_CODES, ALEF_TAGS, strip_key, uz_latin # noqa: E402
from fruchst_english_segments import profile_scripts, script_mismatch, segment_keys # noqa: E402
import fruchst_lexemes # noqa: E402
SCHEMA = "SLeeLa FRUCHST English round v1"
TIERS = ("own_sense", "lemma_of_form", "lexeme_sense", "form_of_gloss", "close_variant", "translation_equivalent")
MAX_HOPS = 2
MAX_LEMMA_SENSES = 6
MAX_TRANSLATIONS = 40
MAX_EXAMPLES = 2
MAX_DEFINITIONS = 40 # added (non-review) definitions per word
# Translation tables use the ISO code of the variety listed; Mandarin rows are
# "cmn", while English Wiktionary entries file Chinese under "zh".
TRANSLATION_CODES = {"zh-Hans": ("zh", "cmn")}
# Korean senses whose gloss is only a hanja character are resolved through
# that character's Korean, then Chinese, entry.
HANJA_CODES = ("ko", "zh")
POINTER_TAGS = {"form-of", "alt-of", "synonym-of"}
DEMOTED_TAGS = {
"obsolete", "archaic", "rare", "dated", "historical", "nonstandard",
"misspelling", "proscribed", "uncommon", "nonce-word", "hapax", "error-misspelling",
}
CAUTION_TAGS = {"dialectal", "regional", "slang", "vulgar", "derogatory", "offensive", "colloquial", "informal"}
POINTER_WORDS = re.compile(
r"\b(?:plural|singular|feminine|masculine|neuter|dual|past|present|passive|alternative|"
r"alternate|obsolete|archaic|nonstandard|misspelling|romanization|transliteration|"
r"abbreviation|initialism|acronym|contraction|diminutive|augmentative|superlative|"
r"comparative|inflection|genitive|dative|accusative|nominative|vocative|locative|"
r"instrumental|ablative|imperative|participle|gerund|verbal noun|infinitive|"
r"conjugated|inflected|variant|spelling|form|clipping|ellipsis|synonym|stem|"
r"person|degree|tense|mood|aspect)\b",
re.I,
)
NON_POINTER_START = {
"to", "a", "an", "the", "one", "used", "someone", "something", "any", "that", "which",
"denotes", "denoting", "indicates", "indicating", "expresses", "expressing", "refers", "referring",
"attached", "marks", "marking", "forms", "forming", "makes",
}
HAN_RE = re.compile(r"^[\u2E80-\u2FFF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\U00020000-\U0003FFFF]+$")
# ---------------------------------------------------------------- gloss text
def clean_gloss(text: str) -> tuple[str, list[str]]:
"""Return a display text with wiki-markup remnants removed, and the issues found."""
issues = []
out = text
if re.search(r"\bTemplate:", out):
issues.append("unexpanded_template")
if "[[" in out or "]]" in out:
issues.append("wiki_link_markup")
out = re.sub(r"\[\[(?:[^\]|]*\|)?([^\]]*)\]\]", r"\1", out)
out = out.replace("[[", "").replace("]]", "")
if out.count("[") != out.count("]"):
out = out.replace("[", "").replace("]", "")
if re.search(r"_\(\(.*?\)\)", out):
issues.append("template_label_markup")
out = re.sub(r"_\(\((.*?)\)\)", r" (\1)", out)
if re.search(r"<[a-z/][^>]*>|&[a-z]+;", out):
issues.append("html_markup")
out = re.sub(r"<[^>]+>", "", out)
out = re.sub(r" ", " ", out)
squeezed = re.sub(r"\s+", " ", out).strip()
squeezed = re.sub(r"\s+([,;:.)])", r"\1", squeezed).replace("( ", "(")
if squeezed != out and "whitespace" not in issues and not issues:
issues.append("whitespace")
return squeezed, issues
def is_pointer_text(text: str) -> bool:
"""Heuristic fallback: does the gloss only point at another word ("plural of X")?
Used only when the dump gives no form_of/alt_of link. Parenthesised notes
and anything after a colon are ignored, the part before "of"/"for" must be
a short label naming a grammatical relation, and "form of address" is a
real sense.
"""
head = re.split(r":", re.sub(r"\([^()]*\)", " ", text), maxsplit=1)[0]
head = re.sub(r"\s+", " ", head).strip()
match = re.match(r"^(.{1,80}?)\s+(?:of|for)\s+\S", head)
if not match:
return False
prefix = match.group(1)
words = prefix.split()
if (not words or words[0].casefold() in NON_POINTER_START or len(words) > 7
or re.search(r"[,;]", prefix) or re.search(r"\bform of address\b", head, re.I)):
return False
return bool(POINTER_WORDS.search(prefix))
def text_targets(text: str) -> list[str]:
"""Pointer target named in the gloss text: the first word after "of"/"for"."""
head = re.split(r":", re.sub(r"\([^()]*\)", " ", text), maxsplit=1)[0]
match = re.search(r"\s(?:of|for)\s+([^\s,;(/+]+)", head)
if not match:
return []
word = match.group(1).strip(".,;:\"'“”‘’")
return [norm(word)] if word else []
LEADING_FILLER = re.compile(r"^(?:to|a|an|the|or|and|be|being|one's|oneself)\s+", re.I)
def equivalents(text: str) -> set[str]:
"""The English equivalents a gloss lists: its comma/semicolon segments,
without parenthesised notes and leading "to", "a", "the", "or"."""
body = re.sub(r"\([^()]*\)", " ", text)
body = body.split(":", 1)[1] if ":" in body and len(body.split(":", 1)[1].strip()) > 0 else body
out = set()
for seg in re.split(r"[,;/]| or ", body):
seg = re.sub(r"\s+", " ", seg).strip(" .!?\"'").casefold()
while seg:
stripped = LEADING_FILLER.sub("", seg)
if stripped == seg:
break
seg = stripped
if seg:
out.add(seg)
return out
def demotion(tags) -> int:
return 1 if set(tags or []) & DEMOTED_TAGS else 0
# ---------------------------------------------------------------- dump access
def opener(path: Path):
return gzip.open(path, "rt", encoding="utf-8") if path.suffix.casefold() == ".gz" else open(path, encoding="utf-8")
def gloss_list(sense: dict) -> list[str]:
glosses = sense.get("glosses") or sense.get("raw_glosses") or []
if isinstance(glosses, str):
glosses = [glosses]
return [norm(g.strip()) for g in glosses if isinstance(g, str) and g.strip()]
def pointer_targets(sense: dict) -> list[str]:
out = []
for key in ("form_of", "alt_of"):
for item in sense.get(key) or []:
if isinstance(item, dict) and isinstance(item.get("word"), str) and item["word"].strip():
word = norm(item["word"].strip())
if word not in out:
out.append(word)
return out
def examples_of(sense: dict) -> list[dict]:
out = []
for ex in sense.get("examples") or []:
if not isinstance(ex, dict):
continue
text = ex.get("text")
english = ex.get("english") or ex.get("translation")
if not isinstance(text, str) or not isinstance(english, str) or not english.strip():
continue
item = {"text": norm(text.strip()), "english": norm(english.strip())}
if isinstance(ex.get("roman"), str) and ex["roman"].strip():
item["roman"] = norm(ex["roman"].strip())
out.append(item)
if len(out) >= MAX_EXAMPLES:
break
return out
def romanizations_of(entry: dict) -> list[str]:
out = []
for form in entry.get("forms") or []:
if not isinstance(form, dict) or not isinstance(form.get("form"), str):
continue
tags = set(form.get("tags") or [])
if "romanization" in tags and not tags & {"inflection-template", "table-tags"}:
value = norm(form["form"].strip())
if value and value not in out:
out.append(value)
if len(out) >= 3:
break
return out
def lemma_senses(entry: dict) -> list[dict]:
out = []
for index, sense in enumerate(entry.get("senses") or [], 1):
if not isinstance(sense, dict):
continue
glosses = gloss_list(sense)
if not glosses:
continue
tags = sorted(t for t in sense.get("tags") or [] if isinstance(t, str))
out.append({"sense_index": index, "glosses": glosses, "tags": tags,
"pointer_targets": pointer_targets(sense), "examples": examples_of(sense)})
return out
def translation_items(entry: dict):
for item in entry.get("translations") or []:
yield item, None
for index, sense in enumerate(entry.get("senses") or [], 1):
if isinstance(sense, dict):
for item in sense.get("translations") or []:
yield item, (index, gloss_list(sense))
# ---------------------------------------------------------------- scanning
class Scanner:
"""Collects all dump evidence for every profile in at most 1 + MAX_HOPS passes."""
def __init__(self, reviews: dict, variants: dict, english_rank: dict[str, int]):
self.reviews = reviews
self.variants = variants
self.english_rank = english_rank
self.head_index: dict[tuple[str, str], list[str]] = {} # (code, word) -> tags
self.trans_exact: dict[tuple[str, str], list[tuple]] = {}
self.trans_strip: dict[tuple[str, str], list[tuple]] = {}
self.extras: dict[tuple[str, int, int], dict] = {} # (tag, line, sense) -> extras
self.romans: dict[tuple[str, str], list[str]] = {}
self.translations: dict[tuple[str, str], list[dict]] = {}
self.lemmas: dict[tuple[str, str], list[dict]] = {} # (code, key) -> entries
self.lexemes: dict[tuple[str, str], list[dict]] = {} # (tag, word) -> lexeme senses
self.lexeme_source = None
self.scripts = {tag: profile_scripts(e["word"] for e in doc["entries"]) for tag, doc in reviews.items()}
for tag, doc in reviews.items():
codes = wiktionary_codes(tag)
tcodes = TRANSLATION_CODES.get(tag, codes)
for entry in doc["entries"]:
word = entry["word"]
for code in codes:
self.head_index.setdefault((code, word), []).append(tag)
if tag == "en":
continue
keys = [word] + ([uz_latin(word)] if tag == "uz" else [])
for code in tcodes:
for key in keys:
self.trans_exact.setdefault((code, key), []).append((tag, word))
self.trans_strip.setdefault((code, strip_key(key, tag in ALEF_TAGS)), []).append((tag, word))
self.trans_codes = {code for code, _ in self.trans_exact}
# Words with no own English sense and no close variant: rule-based
# segment keys, and which other languages list the same spelling.
self.missing: dict[str, list[str]] = {}
self.seg_index: dict[tuple[str, str], list[tuple]] = {}
self.segments: dict[tuple[str, str], list[dict]] = {}
self.other_langs: dict[tuple[str, str], set[str]] = {}
for tag, doc in reviews.items():
vrows = {r["word"]: r for r in variants.get(tag, {}).get("entries", [])}
for entry in doc["entries"]:
word = entry["word"]
if entry["edition_sense_candidates"].get("en") or vrows.get(word, {}).get("close_matches"):
continue
self.missing.setdefault(word, []).append(tag)
for key in segment_keys(tag, word):
for code in wiktionary_codes(tag):
self.seg_index.setdefault((code, key["stem"]), []).append((tag, word, key))
# pass 1 ------------------------------------------------------------
def first_pass(self, path: Path) -> None:
with opener(path) as stream:
for line_no, line in enumerate(stream, 1):
if not line.strip():
continue
entry = json.loads(line)
code, word = entry.get("lang_code"), entry.get("word")
if not isinstance(word, str):
continue
word = norm(word)
for tag in self.head_index.get((code, word), ()):
self._record_head(tag, word, entry, line_no)
for tag in self.missing.get(word, ()):
if code not in wiktionary_codes(tag):
self.other_langs.setdefault((tag, word), set()).add(code)
for tag, head, key in self.seg_index.get((code, word), ()):
self._record_segment(tag, head, key, entry, line_no)
if code == "en" and ('"translations"' in line):
self._record_translations(word, entry, line_no)
def _record_segment(self, tag, head, key, entry, line_no):
pos = pos_key(entry.get("pos"))
if key["allowed_pos"] is not None and pos not in key["allowed_pos"]:
return
senses = [s for s in lemma_senses(entry) if not s["pointer_targets"] and not is_pointer_text(s["glosses"][-1])]
if not senses:
return
bucket = self.segments.setdefault((tag, head), [])
if len(bucket) < 12:
bucket.append({"stem": key["stem"], "method": key["method"], "affix": key["affix"],
"label": key["label"], "pos": pos, "source_line": line_no,
"lang_code": entry.get("lang_code"), "senses": senses[:MAX_LEMMA_SENSES]})
def _record_head(self, tag, word, entry, line_no):
romans = romanizations_of(entry)
bucket = self.romans.setdefault((tag, word), [])
for r in romans:
if r not in bucket:
bucket.append(r)
for index, sense in enumerate(entry.get("senses") or [], 1):
if not isinstance(sense, dict):
continue
item = {"pointer_targets": pointer_targets(sense), "examples": examples_of(sense)}
for key in ("qualifier", "head_nr"):
if isinstance(sense.get(key), str):
item[key] = sense[key]
raw = sense.get("raw_glosses")
if isinstance(raw, list) and raw and raw != sense.get("glosses"):
item["raw_glosses"] = [norm(g) for g in raw if isinstance(g, str)]
self.extras[(tag, line_no, index)] = item
def _record_translations(self, english, entry, line_no):
pos = pos_key(entry.get("pos"))
for item, sense_ctx in translation_items(entry):
if not isinstance(item, dict):
continue
code = item.get("lang_code") or item.get("code")
target = item.get("word")
if not isinstance(code, str) or not isinstance(target, str) or not target.strip():
continue
if code not in self.trans_codes:
continue
target = norm(target.strip())
exact = self.trans_exact.get((code, target), [])
hits = [(t, h, "exact") for t, h in exact]
seen = {(t, h) for t, h in exact}
for t, h in self.trans_strip.get((code, strip_key(target, code in ALEF_CODES)), []):
if (t, h) not in seen:
hits.append((t, h, "optional_marks"))
seen.add((t, h))
for tag, head, match in hits:
bucket = self.translations.setdefault((tag, head), [])
if len(bucket) >= MAX_TRANSLATIONS * 3:
continue
record = {
"english_word": english,
"english_pos": pos,
"translation_sense": item.get("sense") if isinstance(item.get("sense"), str) else None,
"listed_form": target,
"match": match,
"translation_lang_code": code,
"source_line": line_no,
}
if sense_ctx:
record["english_sense_index"] = sense_ctx[0]
record["english_sense_glosses"] = sense_ctx[1]
for key in ("roman", "english", "alt"):
if isinstance(item.get(key), str) and item[key].strip():
record["translation_" + key] = norm(item[key].strip())
tags = sorted(t for t in item.get("tags") or [] if isinstance(t, str))
if tags:
record["tags"] = tags
if record not in bucket:
bucket.append(record)
# lemma passes ------------------------------------------------------
def lemma_pass(self, path: Path, wanted: dict[str, set[str]]) -> None:
"""Load every entry whose (code, exact or loose key) is wanted."""
wanted = {code: words for code, words in wanted.items() if isinstance(code, str) and words}
loose = {code: {strip_key(w, code in ALEF_CODES, True) for w in words} for code, words in wanted.items()}
with opener(path) as stream:
for line_no, line in enumerate(stream, 1):
if not line.strip():
continue
entry = json.loads(line)
code, word = entry.get("lang_code"), entry.get("word")
if code not in wanted or not isinstance(word, str):
continue
word = norm(word)
key = strip_key(word, code in ALEF_CODES, True)
if word not in wanted[code] and key not in loose[code]:
continue
senses = lemma_senses(entry)
if not senses:
continue
record = {"word": word, "pos": pos_key(entry.get("pos")), "lang_code": code,
"source_line": line_no, "senses": senses}
for k in {(code, word), (code, "~" + key)}:
bucket = self.lemmas.setdefault(k, [])
if record not in bucket:
bucket.append(record)
def lemma_lookup(self, code: str, word: str) -> tuple[list[dict], str]:
exact = self.lemmas.get((code, word), [])
if exact:
return exact, "exact"
loose = self.lemmas.get((code, "~" + strip_key(word, code in ALEF_CODES, True)), [])
return loose, "optional_marks"
# ---------------------------------------------------------------- building
def pointer_requests(scanner: Scanner) -> dict[str, set[str]]:
wanted: dict[str, set[str]] = {}
for tag, doc in scanner.reviews.items():
for entry in doc["entries"]:
for sense in entry["edition_sense_candidates"].get("en", []):
code = sense.get("source_lang_code") or wiktionary_codes(tag)[0]
extra = scanner.extras.get((tag, sense["source_line"], sense["sense_index"]), {})
targets = extra.get("pointer_targets") or (
text_targets(clean_gloss(sense["glosses"][-1])[0])
if is_pointer_text(clean_gloss(sense["glosses"][-1])[0]) else [])
for target in targets:
wanted.setdefault(code, set()).add(target)
if tag == "ko":
for g in sense["glosses"]:
if HAN_RE.match(g.strip()):
for c in HANJA_CODES:
wanted.setdefault(c, set()).add(g.strip())
for row in scanner.variants.get(tag, {}).get("entries", []):
for match in row.get("close_matches", []):
for s in match.get("english_senses", []):
for target in s.get("form_of", []):
wanted.setdefault(match.get("lang_code"), set()).add(norm(target))
return wanted
def next_hop_requests(scanner: Scanner, already: dict[str, set[str]]) -> dict[str, set[str]]:
wanted: dict[str, set[str]] = {}
for (code, key), records in scanner.lemmas.items():
if key.startswith("~"):
continue
for record in records:
if any(not s["pointer_targets"] and not is_pointer_text(s["glosses"][-1]) for s in record["senses"]):
continue
for s in record["senses"]:
for target in s["pointer_targets"]:
if target not in already.get(code, set()):
wanted.setdefault(code, set()).add(target)
return wanted
def resolve(scanner: Scanner, code: str, target: str, want_pos: str | None, hops: int = 1, trail=()) -> list[dict]:
"""Substantive senses of a pointer target, following at most MAX_HOPS pointers."""
records, how = scanner.lemma_lookup(code, target)
same = [r for r in records if want_pos and r["pos"] == want_pos]
records = same or records
out = []
for record in records:
substantive = [s for s in record["senses"]
if not s["pointer_targets"] and not is_pointer_text(s["glosses"][-1])]
if substantive:
substantive.sort(key=lambda s: (demotion(s["tags"]), s["sense_index"]))
for s in substantive[:MAX_LEMMA_SENSES]:
out.append({"lemma": record["word"], "lemma_pos": record["pos"], "lemma_lang_code": code,
"lemma_match": how, "pos_matched": bool(want_pos) and record["pos"] == want_pos,
"hops": hops, "via": list(trail), "source_line": record["source_line"],
"sense_index": s["sense_index"], "glosses": s["glosses"], "tags": s["tags"],
"examples": s["examples"]})
elif hops < MAX_HOPS:
for s in record["senses"]:
for nxt in s["pointer_targets"]:
if nxt != target and nxt not in trail:
out += resolve(scanner, code, nxt, record["pos"], hops + 1, trail + (record["word"],))
return out
def definition(tier, text_glosses, pos, evidence, tags=(), examples=(), **extra) -> dict:
display, issues = clean_gloss(text_glosses[-1])
item = {"tier": TIERS.index(tier) + 1, "method": tier, "text": display, "pos": pos}
if len(text_glosses) > 1:
item["context"] = [clean_gloss(g)[0] for g in text_glosses[:-1]]
if issues:
item["cleaned_from"] = text_glosses[-1]
if tags:
item["tags"] = list(tags)
cautions = sorted(set(tags) & (DEMOTED_TAGS | CAUTION_TAGS))
if cautions:
item["register"] = cautions
if examples:
item["examples"] = list(examples)
item.update(extra)
item["evidence"] = evidence
return item
def build_entry(tag: str, entry: dict, scanner: Scanner, variant_row: dict | None, mapping_note) -> tuple[dict, dict]:
word = entry["word"]
senses = entry["edition_sense_candidates"].get("en", [])
trans = scanner.translations.get((tag, word), [])
trans.sort(key=lambda t: (t["match"] != "exact", scanner.english_rank.get(t["english_word"], 10**6),
t["source_line"]))
trans = trans[:MAX_TRANSLATIONS]
lexeme_senses = scanner.lexemes.get((tag, word), [])
lexeme_words = {link["english_lemma"] for r in lexeme_senses for link in r.get("english_translations", [])}
trans_words = sorted({t["english_word"] for t in trans} | lexeme_words,
key=lambda w: scanner.english_rank.get(w, 10**6))
defs, notes, stats = [], [], {"pointer_resolved": 0, "pointer_unresolved": 0, "cleaned": 0, "duplicates": 0}
seen_text: dict[tuple[str, str], int] = {}
def push(item):
key = (item["text"].casefold(), item.get("pos") or "")
if key in seen_text:
stats["duplicates"] += 1
prior = defs[seen_text[key]]
prior.setdefault("also_supported_by", []).append(item["evidence"])
return
seen_text[key] = len(defs)
defs.append(item)
# tier 1 and 2 -------------------------------------------------------
own = []
for order, sense in enumerate(senses):
code = sense.get("source_lang_code") or wiktionary_codes(tag)[0]
extra = scanner.extras.get((tag, sense["source_line"], sense["sense_index"]), {})
glosses = sense["glosses"]
last = glosses[-1]
display, issues = clean_gloss(last)
ev = {"source": "enwiktionary", "source_line": sense["source_line"], "sense_index": sense["sense_index"],
"lang_code": code}
if issues:
stats["cleaned"] += 1
notes.append({"source_line": sense["source_line"], "sense_index": sense["sense_index"],
"issues": issues, "original": last, "display": display})
targets = list(extra.get("pointer_targets", []))
pointer = bool(targets) or bool(set(sense.get("tags") or []) & POINTER_TAGS) or is_pointer_text(display)
relation = clean_gloss(" ".join(glosses))[0]
if pointer and not targets:
targets = text_targets(display) or text_targets(relation)
hanja = tag == "ko" and HAN_RE.match(last.strip())
if hanja:
notes.append({"source_line": sense["source_line"], "sense_index": sense["sense_index"],
"issues": ["hanja_reading_gloss"], "original": last})
resolved = []
for c in HANJA_CODES:
resolved = resolve(scanner, c, last.strip(), None)
if resolved:
break
for r in resolved:
push(definition("lemma_of_form", r["glosses"], r["lemma_pos"],
{**ev, "lemma_source_line": r["source_line"], "lemma_sense_index": r["sense_index"]},
r["tags"], r["examples"], relation=f"hanja {last.strip()}", lemma=r["lemma"],
lemma_lang_code=r["lemma_lang_code"], hops=r["hops"],
display=f"{display} (hanja): {clean_gloss(r['glosses'][-1])[0]}"))
stats["pointer_resolved" if resolved else "pointer_unresolved"] += 1
if not resolved:
push(definition("form_of_gloss", glosses, sense.get("pos"), ev, sense.get("tags") or [],
relation=f"hanja {last.strip()}"))
continue
if pointer:
resolved = []
for target in targets:
resolved += resolve(scanner, code, target, sense.get("pos"))
if resolved:
stats["pointer_resolved"] += 1
else:
stats["pointer_unresolved"] += 1
notes.append({"source_line": sense["source_line"], "sense_index": sense["sense_index"],
"issues": ["pointer_gloss_unresolved"], "original": last,
"pointer_targets": targets})
push(definition("form_of_gloss", glosses, sense.get("pos"), ev, sense.get("tags") or [],
relation=relation, pointer_targets=targets))
for r in resolved:
lemma_gloss = clean_gloss(r["glosses"][-1])[0]
push(definition("lemma_of_form", r["glosses"], r["lemma_pos"],
{**ev, "lemma_source_line": r["source_line"], "lemma_sense_index": r["sense_index"]},
r["tags"], r["examples"], relation=relation, lemma=r["lemma"],
lemma_lang_code=r["lemma_lang_code"], lemma_match=r["lemma_match"],
pos_matched=r["pos_matched"], hops=r["hops"],
**({"via": r["via"]} if r["via"] else {}),
display=f"{relation}: {lemma_gloss}"))
continue
tags = sense.get("tags") or []
extras = {}
for k in ("qualifier", "raw_glosses"):
if k in extra:
extras[k] = extra[k]
if code != wiktionary_codes(tag)[0] and mapping_note:
extras["caution"] = f"gloss is filed under Wiktionary '{code}'; confirm it is valid for '{tag}'"
own.append(definition("own_sense", glosses, sense.get("pos"), ev, tags, extra.get("examples", []), **extras))
# Rank own senses and resolved lemmas by translation-table corroboration:
# an English word whose translation table lists this headword and which is
# itself one of the equivalents a gloss lists. More frequent English
# corroborators rank higher; circular glosses (gloss == headword) and
# obsolete/archaic/rare senses rank last; Wiktionary order breaks ties.
def corroborate(item, exclude=frozenset()):
eq = equivalents(item["text"])
hits = [w for w in trans_words if w.casefold() in eq and w not in exclude]
if hits:
item["corroborated_by_translation_tables"] = hits[:8]
if item["text"].strip(" .").casefold() == word.casefold():
item["circular_gloss"] = True
return hits
def rank_key(item, position):
hits = item.get("corroborated_by_translation_tables") or []
best = min((scanner.english_rank.get(w, 10**6) for w in hits), default=10**7)
return (bool(item.get("circular_gloss")), not hits, demotion(item.get("tags")), best, position)
lemma_defs = defs[:]
defs.clear(); seen_text.clear()
for item in own + lemma_defs:
corroborate(item)
own = [d for _, d in sorted(enumerate(own), key=lambda p: rank_key(p[1], p[0]))]
lemma_defs.sort(key=lambda d: d["tier"])
tier2 = [d for d in lemma_defs if d["method"] == "lemma_of_form"]
rest = [d for d in lemma_defs if d["method"] != "lemma_of_form"]
tier2 = [d for _, d in sorted(enumerate(tier2), key=lambda p: rank_key(p[1], p[0]))]
for item in own + tier2 + rest:
push(item)
# tier 3: Wikidata Lexemes -------------------------------------------
for r in lexeme_senses:
links = r.get("english_translations", [])
if r.get("gloss"):
text = r["gloss"]
else:
text = ", ".join(dict.fromkeys(link["english_lemma"] for link in links))
ev = {"source": "wikidata-lexemes", "lexeme": r["lexeme"], "sense": r["sense"], "match": r["match"],
"match_how": r["match_how"]}
extra_l = {"lexeme_lemma": r["lemma"]}
if r.get("items"):
extra_l["wikidata_items"] = r["items"]
if links:
extra_l["english_translations"] = links
if r["match"] == "form":
extra_l["display"] = f"form of {r['lemma']}: {text}"
if r.get("grammatical_features"):
extra_l["grammatical_features"] = r["grammatical_features"]
if r["match_how"] != "exact":
extra_l["caution"] = "matched with optional marks removed; confirm it is the same word"
item = definition("lexeme_sense", [text], r["pos"], ev, **extra_l)
# A sense's own translation links are not independent corroboration.
corroborate(item, exclude={link["english_lemma"] for link in links})
push(item)
# tier 3 -------------------------------------------------------------
if variant_row:
for match in variant_row.get("close_matches", []):
mev = {"source": "definition-variants", "variant": match.get("variant"), "variant_method": match["method"],
"source_line": match.get("source_line"), "lang_code": match.get("lang_code")}
for s in match.get("english_senses", []):
ev = {**mev, "sense_index": s.get("sense_index")}
extra_v = {"variant": match.get("variant"), "variant_method": match["method"]}
if match.get("caution"):
extra_v["caution"] = match["caution"]
resolved = []
for target in s.get("form_of") or []:
resolved += resolve(scanner, match.get("lang_code"), norm(target), match.get("pos"))
for r in resolved:
push(definition("close_variant", r["glosses"], r["lemma_pos"],
{**ev, "lemma_source_line": r["source_line"]}, r["tags"], r["examples"],
relation=clean_gloss(s["glosses"][-1])[0], lemma=r["lemma"], **extra_v))
if resolved:
continue
if s.get("form_of") or is_pointer_text(clean_gloss(s["glosses"][-1])[0]):
extra_v["pointer_gloss"] = True
push(definition("close_variant", s["glosses"], match.get("pos"), ev, s.get("tags") or [], **extra_v))
# rule-based keys (Malayalam chillu, Korean particles, Amharic prefixes)
for seg in scanner.segments.get((tag, word), []):
for s in seg["senses"]:
ev = {"source": "enwiktionary", "source_line": seg["source_line"], "sense_index": s["sense_index"],
"lang_code": seg["lang_code"], "variant": seg["stem"], "variant_method": seg["method"]}
extra_s = {"variant": seg["stem"], "variant_method": seg["method"]}
gloss = clean_gloss(s["glosses"][-1])[0]
if seg["affix"]:
extra_s["affix"] = seg["affix"]
extra_s["affix_meaning"] = seg["label"]
extra_s["display"] = f"{seg['stem']} ({gloss}) + {seg['affix']} ({seg['label']})"
extra_s["caution"] = "segmented by rule; confirm the stem and affix"
push(definition("close_variant", s["glosses"], seg["pos"], ev, s["tags"], s["examples"], **extra_s))
# tier 6 -------------------------------------------------------------
for t in trans:
label = t.get("translation_sense") or (t.get("english_sense_glosses") or [None])[-1]
text = t["english_word"] if not label else f"{t['english_word']} ({label})"
ev = {"source": "enwiktionary-translations", "english_word": t["english_word"],
"english_pos": t["english_pos"], "source_line": t["source_line"], "listed_form": t["listed_form"],
"match": t["match"], "translation_lang_code": t["translation_lang_code"]}
if t.get("english_sense_index"):
ev["english_sense_index"] = t["english_sense_index"]
extra_t = {k: t[k] for k in ("translation_roman", "translation_english", "tags") if k in t}
if t["match"] != "exact":
extra_t["caution"] = "listed with optional marks (e.g. vowel points or accents); confirm it is the same word"
push(definition("translation_equivalent", [text], t["english_pos"], ev, **extra_t))
defs.sort(key=lambda d: (d["tier"], bool(d.get("pointer_gloss"))))
# Review-derived senses (own_sense, form_of_gloss) are never dropped; the
# cap applies only to the evidence this pass adds.
kept, added = [], 0
for d in defs:
if d["method"] in ("own_sense", "form_of_gloss"):
kept.append(d)
elif added < MAX_DEFINITIONS:
kept.append(d)
added += 1
defs = kept
tiers_present = sorted({d["tier"] for d in defs})
status = ("no_english_definition_found" if not defs else TIERS[tiers_present[0] - 1] + "_found")
lead = None
if defs:
pool = [(i, d) for i, d in enumerate(defs) if d["tier"] <= 3] or list(enumerate(defs))
_, best = min(pool, key=lambda p: rank_key(p[1], p[0])[:4] + (p[1]["tier"], p[0]))
lead = {k: best[k] for k in ("tier", "method", "text", "pos") if k in best}
if "display" in best:
lead["display"] = best["display"]
if best.get("corroborated_by_translation_tables"):
lead["corroborated_by"] = best["corroborated_by_translation_tables"][:3]
lead["review_status"] = "required"
flags = {}
odd_script = script_mismatch(word, scanner.scripts.get(tag, {}))
if odd_script:
flags["script_mismatch"] = odd_script
if not defs and scanner.other_langs.get((tag, word)):
flags["found_only_in_other_languages"] = sorted(scanner.other_langs[(tag, word)])
row = {
"rank": entry["rank"],
"word": word,
"frequency_count": entry.get("frequency_count"),
"candidate_pos": entry.get("candidate_pos"),
"english_status": status,
"lead_candidate": lead,
"romanizations": scanner.romans.get((tag, word), []),
"translation_table_english_words": trans_words[:15],
"definitions": defs,
"flags": flags,
"gloss_notes": notes,
"review_status": "required",
}
stats["tiers"] = tiers_present
return row, stats
def build_profile(path: Path, doc: dict, scanner: Scanner, variants_path: Path | None) -> dict:
tag = doc["language_tag"]
vdoc = scanner.variants.get(tag, {})
vrows = {r["word"]: r for r in vdoc.get("entries", [])}
rows = []
counts = {
"candidate_words": len(doc["entries"]),
"baseline_words_with_own_english_sense": 0,
"baseline_words_with_substantive_own_sense": 0,
"words_with_any_english_definition": 0,
"words_by_best_tier": {t: 0 for t in TIERS},
"words_without_english_definition": 0,
"words_gaining_english_definition": 0,
"words_gaining_substantive_definition": 0,
"pointer_senses_resolved": 0,
"pointer_senses_unresolved": 0,
"glosses_cleaned_for_display": 0,
"duplicate_definitions_merged": 0,
"definitions_total": 0,
"words_with_translation_corroborated_lead": 0,
"words_with_examples": 0,
"words_with_romanization": 0,
"words_with_wikidata_lexeme_sense": 0,
"words_with_rule_based_key_definition": 0,
"words_flagged_script_mismatch": 0,
"words_found_only_in_other_languages": 0,
"other_language_codes": {},
"production_definitions_approved": 0,
}
for entry in doc["entries"]:
row, stats = build_entry(tag, entry, scanner, vrows.get(entry["word"]), doc.get("target_language_mapping_note"))
rows.append(row)
had = bool(entry["edition_sense_candidates"].get("en"))
own_sub = 1 in stats["tiers"]
counts["baseline_words_with_own_english_sense"] += had
counts["baseline_words_with_substantive_own_sense"] += own_sub
if row["definitions"]:
counts["words_with_any_english_definition"] += 1
counts["words_by_best_tier"][TIERS[stats["tiers"][0] - 1]] += 1
if not had:
counts["words_gaining_english_definition"] += 1
if not own_sub:
counts["words_gaining_substantive_definition"] += 1
else:
counts["words_without_english_definition"] += 1
counts["pointer_senses_resolved"] += stats["pointer_resolved"]
counts["pointer_senses_unresolved"] += stats["pointer_unresolved"]
counts["glosses_cleaned_for_display"] += stats["cleaned"]
counts["duplicate_definitions_merged"] += stats["duplicates"]
counts["definitions_total"] += len(row["definitions"])
if row["lead_candidate"] and row["lead_candidate"].get("corroborated_by"):
counts["words_with_translation_corroborated_lead"] += 1
counts["words_with_examples"] += any(d.get("examples") for d in row["definitions"])
counts["words_with_romanization"] += bool(row["romanizations"])
counts["words_with_wikidata_lexeme_sense"] += any(d["method"] == "lexeme_sense" for d in row["definitions"])
counts["words_with_rule_based_key_definition"] += any(
d.get("variant_method") in ("chillu_normalized", "particle_stripped", "contracted_particle", "prefix_stripped")
for d in row["definitions"])
counts["words_flagged_script_mismatch"] += "script_mismatch" in row["flags"]
others = row["flags"].get("found_only_in_other_languages")
if others:
counts["words_found_only_in_other_languages"] += 1
for code in others:
counts["other_language_codes"][code] = counts["other_language_codes"].get(code, 0) + 1
top = sorted(counts["other_language_codes"].items(), key=lambda kv: -kv[1])[:10]
counts["other_language_codes"] = dict(top)
return {
"schema": SCHEMA,
"language_tag": tag,
"review_queue_file": path.as_posix(),
"variants_file": variants_path.as_posix() if variants_path else None,
"target_language_mapping_note": doc.get("target_language_mapping_note"),
"tier_order": list(TIERS),
"policy": (
"Add-only. Every review-queue gloss is kept unchanged; this file adds the English evidence "
"English Wiktionary and Wikidata Lexemes supply: resolved form-of lemmas, lexeme senses, close "
"variants and rule-based keys, translation-table equivalents, usage examples with English "
"translations, and romanizations. Words with no English are flagged when their script is rare "
"in the profile or when only other languages have an entry for the spelling. Definitions are "
"ranked (own sense corroborated by a translation table, then own sense in Wiktionary order "
"with obsolete/archaic/rare senses last, then resolved lemma, close variant, translation "
"equivalent). The lead candidate is a reviewer's starting point; nothing is selected or approved."
),
"source": {**doc["definition_sources"]["en"]},
"lexeme_source": scanner.lexeme_source,
"counts": counts,
"production_status": "review_required",
"entries": rows,
}
def english_ranks(reviews: dict) -> dict[str, int]:
doc = reviews.get("en")
if not doc:
return {}
return {e["word"]: e["rank"] for e in doc["entries"]}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--reviews", type=Path, required=True)
parser.add_argument("--variants", type=Path, required=True)
parser.add_argument("--en", type=Path, required=True)
parser.add_argument("--lexemes", type=Path, help="Wikidata lexeme JSON dump (.json.gz/.bz2), optional")
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
reviews, paths, vpaths, variants = {}, {}, {}, {}
for path in sorted(args.reviews.glob("*.review.json")):
doc = json.loads(path.read_text(encoding="utf-8"))
if doc["language_tag"] in TARGETS:
reviews[doc["language_tag"]] = doc
paths[doc["language_tag"]] = path
vpath = args.variants / path.name.replace(".review.json", ".variants.json")
if vpath.is_file():
variants[doc["language_tag"]] = json.loads(vpath.read_text(encoding="utf-8"))
vpaths[doc["language_tag"]] = vpath
scanner = Scanner(reviews, variants, english_ranks(reviews))
if args.lexemes:
scanner.lexeme_source = {**fruchst_lexemes.SOURCE, "file": args.lexemes.name,
"size_bytes": args.lexemes.stat().st_size}
scanner.lexemes = fruchst_lexemes.scan_lexemes(
args.lexemes, {tag: {e["word"] for e in doc["entries"]} for tag, doc in reviews.items()})
scanner.first_pass(args.en)
wanted = pointer_requests(scanner)
requested = {c: set(w) for c, w in wanted.items()}
for hop in range(MAX_HOPS):
if not wanted:
break
scanner.lemma_pass(args.en, wanted)
wanted = next_hop_requests(scanner, requested)
for c, w in wanted.items():
requested.setdefault(c, set()).update(w)
args.output.mkdir(parents=True, exist_ok=True)
for tag, doc in reviews.items():
out = build_profile(paths[tag], doc, scanner, vpaths.get(tag))
target = args.output / paths[tag].name.replace(".review.json", ".english.json")
target.write_text(json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8")
print(json.dumps({"language": tag, **out["counts"]}, ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())