Home /Modules/languages/fruchst_english_round.py

fruchst_english_round.py

languages/tools/fruchst_english_round.py · 918 line(s) · py

#!/usr/bin/env python3
"""English round: add every English definition English Wiktionary can supply.

Add-only companion to the review queues. It never edits or removes a review
gloss; it writes one ``<stem>.english.json`` per profile that collects, ranks
and explains all English definition evidence for each headword:

  tier 1  own_sense              the headword's own English Wiktionary sense
  tier 2  lemma_of_form          a form-of / alt-of / synonym-of sense resolved
                                 to the lemma's own definitions (up to 2 hops),
                                 and Korean hanja-reading senses resolved to the
                                 hanja character's definition
  tier 3  lexeme_sense           Wikidata Lexemes (CC0): English gloss or English
                                 translation link of a lexeme whose lemma or
                                 form is the headword (optional --lexemes)
  tier 4  form_of_gloss          a pointer gloss whose lemma could not be found,
                                 kept verbatim (add-only)
  tier 5  close_variant          the close-variant pass (definition-variants/),
                                 with its own form-of senses resolved as well,
                                 plus rule-based keys for words still without
                                 English (Malayalam chillu spelling, Korean
                                 particles, Amharic prefixes)
  tier 6  translation_equivalent English Wiktionary translation tables that list
                                 the headword (exact or with optional marks)

Within tier 1, senses corroborated by a translation table (the English word
that lists the headword also appears in the gloss) rank first; obsolete,
archaic, rare and similar senses rank last; otherwise Wiktionary order holds.
The first ranked definition is recorded as ``lead_candidate``. It is a
reviewer's starting point only: nothing is selected or approved here.

Gloss-quality notes (wiki-markup remnants, duplicates, unresolved pointers,
non-English glosses) are recorded with a cleaned display text; the original
glosses stay exactly as staged in the review queue.
"""
from __future__ import annotations

import argparse
import gzip
import json
import re
import sys
import unicodedata
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from fruchst_definition_batch import TARGETS, norm, pos_key, wiktionary_codes  # noqa: E402
from fruchst_close_variants import ALEF_CODES, ALEF_TAGS, strip_key, uz_latin  # noqa: E402
from fruchst_english_segments import profile_scripts, script_mismatch, segment_keys  # noqa: E402
import fruchst_lexemes  # noqa: E402

SCHEMA = "SLeeLa FRUCHST English round v1"
TIERS = ("own_sense", "lemma_of_form", "lexeme_sense", "form_of_gloss", "close_variant", "translation_equivalent")
MAX_HOPS = 2
MAX_LEMMA_SENSES = 6
MAX_TRANSLATIONS = 40
MAX_EXAMPLES = 2
MAX_DEFINITIONS = 40  # added (non-review) definitions per word

# Translation tables use the ISO code of the variety listed; Mandarin rows are
# "cmn", while English Wiktionary entries file Chinese under "zh".
TRANSLATION_CODES = {"zh-Hans": ("zh", "cmn")}
# Korean senses whose gloss is only a hanja character are resolved through
# that character's Korean, then Chinese, entry.
HANJA_CODES = ("ko", "zh")
POINTER_TAGS = {"form-of", "alt-of", "synonym-of"}
DEMOTED_TAGS = {
    "obsolete", "archaic", "rare", "dated", "historical", "nonstandard",
    "misspelling", "proscribed", "uncommon", "nonce-word", "hapax", "error-misspelling",
}
CAUTION_TAGS = {"dialectal", "regional", "slang", "vulgar", "derogatory", "offensive", "colloquial", "informal"}
POINTER_WORDS = re.compile(
    r"\b(?:plural|singular|feminine|masculine|neuter|dual|past|present|passive|alternative|"
    r"alternate|obsolete|archaic|nonstandard|misspelling|romanization|transliteration|"
    r"abbreviation|initialism|acronym|contraction|diminutive|augmentative|superlative|"
    r"comparative|inflection|genitive|dative|accusative|nominative|vocative|locative|"
    r"instrumental|ablative|imperative|participle|gerund|verbal noun|infinitive|"
    r"conjugated|inflected|variant|spelling|form|clipping|ellipsis|synonym|stem|"
    r"person|degree|tense|mood|aspect)\b",
    re.I,
)
NON_POINTER_START = {
    "to", "a", "an", "the", "one", "used", "someone", "something", "any", "that", "which",
    "denotes", "denoting", "indicates", "indicating", "expresses", "expressing", "refers", "referring",
    "attached", "marks", "marking", "forms", "forming", "makes",
}
HAN_RE = re.compile(r"^[\u2E80-\u2FFF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\U00020000-\U0003FFFF]+$")


# ---------------------------------------------------------------- gloss text

def clean_gloss(text: str) -> tuple[str, list[str]]:
    """Return a display text with wiki-markup remnants removed, and the issues found."""
    issues = []
    out = text
    if re.search(r"\bTemplate:", out):
        issues.append("unexpanded_template")
    if "[[" in out or "]]" in out:
        issues.append("wiki_link_markup")
        out = re.sub(r"\[\[(?:[^\]|]*\|)?([^\]]*)\]\]", r"\1", out)
        out = out.replace("[[", "").replace("]]", "")
        if out.count("[") != out.count("]"):
            out = out.replace("[", "").replace("]", "")
    if re.search(r"_\(\(.*?\)\)", out):
        issues.append("template_label_markup")
        out = re.sub(r"_\(\((.*?)\)\)", r" (\1)", out)
    if re.search(r"<[a-z/][^>]*>|&[a-z]+;", out):
        issues.append("html_markup")
        out = re.sub(r"<[^>]+>", "", out)
        out = re.sub(r"&nbsp;", " ", out)
    squeezed = re.sub(r"\s+", " ", out).strip()
    squeezed = re.sub(r"\s+([,;:.)])", r"\1", squeezed).replace("( ", "(")
    if squeezed != out and "whitespace" not in issues and not issues:
        issues.append("whitespace")
    return squeezed, issues


def is_pointer_text(text: str) -> bool:
    """Heuristic fallback: does the gloss only point at another word ("plural of X")?

    Used only when the dump gives no form_of/alt_of link. Parenthesised notes
    and anything after a colon are ignored, the part before "of"/"for" must be
    a short label naming a grammatical relation, and "form of address" is a
    real sense.
    """
    head = re.split(r":", re.sub(r"\([^()]*\)", " ", text), maxsplit=1)[0]
    head = re.sub(r"\s+", " ", head).strip()
    match = re.match(r"^(.{1,80}?)\s+(?:of|for)\s+\S", head)
    if not match:
        return False
    prefix = match.group(1)
    words = prefix.split()
    if (not words or words[0].casefold() in NON_POINTER_START or len(words) > 7
            or re.search(r"[,;]", prefix) or re.search(r"\bform of address\b", head, re.I)):
        return False
    return bool(POINTER_WORDS.search(prefix))


def text_targets(text: str) -> list[str]:
    """Pointer target named in the gloss text: the first word after "of"/"for"."""
    head = re.split(r":", re.sub(r"\([^()]*\)", " ", text), maxsplit=1)[0]
    match = re.search(r"\s(?:of|for)\s+([^\s,;(/+]+)", head)
    if not match:
        return []
    word = match.group(1).strip(".,;:\"'“”‘’")
    return [norm(word)] if word else []


LEADING_FILLER = re.compile(r"^(?:to|a|an|the|or|and|be|being|one's|oneself)\s+", re.I)


def equivalents(text: str) -> set[str]:
    """The English equivalents a gloss lists: its comma/semicolon segments,
    without parenthesised notes and leading "to", "a", "the", "or"."""
    body = re.sub(r"\([^()]*\)", " ", text)
    body = body.split(":", 1)[1] if ":" in body and len(body.split(":", 1)[1].strip()) > 0 else body
    out = set()
    for seg in re.split(r"[,;/]| or ", body):
        seg = re.sub(r"\s+", " ", seg).strip(" .!?\"'").casefold()
        while seg:
            stripped = LEADING_FILLER.sub("", seg)
            if stripped == seg:
                break
            seg = stripped
        if seg:
            out.add(seg)
    return out


def demotion(tags) -> int:
    return 1 if set(tags or []) & DEMOTED_TAGS else 0


# ---------------------------------------------------------------- dump access

def opener(path: Path):
    return gzip.open(path, "rt", encoding="utf-8") if path.suffix.casefold() == ".gz" else open(path, encoding="utf-8")


def gloss_list(sense: dict) -> list[str]:
    glosses = sense.get("glosses") or sense.get("raw_glosses") or []
    if isinstance(glosses, str):
        glosses = [glosses]
    return [norm(g.strip()) for g in glosses if isinstance(g, str) and g.strip()]


def pointer_targets(sense: dict) -> list[str]:
    out = []
    for key in ("form_of", "alt_of"):
        for item in sense.get(key) or []:
            if isinstance(item, dict) and isinstance(item.get("word"), str) and item["word"].strip():
                word = norm(item["word"].strip())
                if word not in out:
                    out.append(word)
    return out


def examples_of(sense: dict) -> list[dict]:
    out = []
    for ex in sense.get("examples") or []:
        if not isinstance(ex, dict):
            continue
        text = ex.get("text")
        english = ex.get("english") or ex.get("translation")
        if not isinstance(text, str) or not isinstance(english, str) or not english.strip():
            continue
        item = {"text": norm(text.strip()), "english": norm(english.strip())}
        if isinstance(ex.get("roman"), str) and ex["roman"].strip():
            item["roman"] = norm(ex["roman"].strip())
        out.append(item)
        if len(out) >= MAX_EXAMPLES:
            break
    return out


def romanizations_of(entry: dict) -> list[str]:
    out = []
    for form in entry.get("forms") or []:
        if not isinstance(form, dict) or not isinstance(form.get("form"), str):
            continue
        tags = set(form.get("tags") or [])
        if "romanization" in tags and not tags & {"inflection-template", "table-tags"}:
            value = norm(form["form"].strip())
            if value and value not in out:
                out.append(value)
        if len(out) >= 3:
            break
    return out


def lemma_senses(entry: dict) -> list[dict]:
    out = []
    for index, sense in enumerate(entry.get("senses") or [], 1):
        if not isinstance(sense, dict):
            continue
        glosses = gloss_list(sense)
        if not glosses:
            continue
        tags = sorted(t for t in sense.get("tags") or [] if isinstance(t, str))
        out.append({"sense_index": index, "glosses": glosses, "tags": tags,
                    "pointer_targets": pointer_targets(sense), "examples": examples_of(sense)})
    return out


def translation_items(entry: dict):
    for item in entry.get("translations") or []:
        yield item, None
    for index, sense in enumerate(entry.get("senses") or [], 1):
        if isinstance(sense, dict):
            for item in sense.get("translations") or []:
                yield item, (index, gloss_list(sense))


# ---------------------------------------------------------------- scanning

class Scanner:
    """Collects all dump evidence for every profile in at most 1 + MAX_HOPS passes."""

    def __init__(self, reviews: dict, variants: dict, english_rank: dict[str, int]):
        self.reviews = reviews
        self.variants = variants
        self.english_rank = english_rank
        self.head_index: dict[tuple[str, str], list[str]] = {}       # (code, word) -> tags
        self.trans_exact: dict[tuple[str, str], list[tuple]] = {}
        self.trans_strip: dict[tuple[str, str], list[tuple]] = {}
        self.extras: dict[tuple[str, int, int], dict] = {}           # (tag, line, sense) -> extras
        self.romans: dict[tuple[str, str], list[str]] = {}
        self.translations: dict[tuple[str, str], list[dict]] = {}
        self.lemmas: dict[tuple[str, str], list[dict]] = {}          # (code, key) -> entries
        self.lexemes: dict[tuple[str, str], list[dict]] = {}         # (tag, word) -> lexeme senses
        self.lexeme_source = None
        self.scripts = {tag: profile_scripts(e["word"] for e in doc["entries"]) for tag, doc in reviews.items()}
        for tag, doc in reviews.items():
            codes = wiktionary_codes(tag)
            tcodes = TRANSLATION_CODES.get(tag, codes)
            for entry in doc["entries"]:
                word = entry["word"]
                for code in codes:
                    self.head_index.setdefault((code, word), []).append(tag)
                if tag == "en":
                    continue
                keys = [word] + ([uz_latin(word)] if tag == "uz" else [])
                for code in tcodes:
                    for key in keys:
                        self.trans_exact.setdefault((code, key), []).append((tag, word))
                        self.trans_strip.setdefault((code, strip_key(key, tag in ALEF_TAGS)), []).append((tag, word))
        self.trans_codes = {code for code, _ in self.trans_exact}
        # Words with no own English sense and no close variant: rule-based
        # segment keys, and which other languages list the same spelling.
        self.missing: dict[str, list[str]] = {}
        self.seg_index: dict[tuple[str, str], list[tuple]] = {}
        self.segments: dict[tuple[str, str], list[dict]] = {}
        self.other_langs: dict[tuple[str, str], set[str]] = {}
        for tag, doc in reviews.items():
            vrows = {r["word"]: r for r in variants.get(tag, {}).get("entries", [])}
            for entry in doc["entries"]:
                word = entry["word"]
                if entry["edition_sense_candidates"].get("en") or vrows.get(word, {}).get("close_matches"):
                    continue
                self.missing.setdefault(word, []).append(tag)
                for key in segment_keys(tag, word):
                    for code in wiktionary_codes(tag):
                        self.seg_index.setdefault((code, key["stem"]), []).append((tag, word, key))

    # pass 1 ------------------------------------------------------------
    def first_pass(self, path: Path) -> None:
        with opener(path) as stream:
            for line_no, line in enumerate(stream, 1):
                if not line.strip():
                    continue
                entry = json.loads(line)
                code, word = entry.get("lang_code"), entry.get("word")
                if not isinstance(word, str):
                    continue
                word = norm(word)
                for tag in self.head_index.get((code, word), ()):
                    self._record_head(tag, word, entry, line_no)
                for tag in self.missing.get(word, ()):
                    if code not in wiktionary_codes(tag):
                        self.other_langs.setdefault((tag, word), set()).add(code)
                for tag, head, key in self.seg_index.get((code, word), ()):
                    self._record_segment(tag, head, key, entry, line_no)
                if code == "en" and ('"translations"' in line):
                    self._record_translations(word, entry, line_no)

    def _record_segment(self, tag, head, key, entry, line_no):
        pos = pos_key(entry.get("pos"))
        if key["allowed_pos"] is not None and pos not in key["allowed_pos"]:
            return
        senses = [s for s in lemma_senses(entry) if not s["pointer_targets"] and not is_pointer_text(s["glosses"][-1])]
        if not senses:
            return
        bucket = self.segments.setdefault((tag, head), [])
        if len(bucket) < 12:
            bucket.append({"stem": key["stem"], "method": key["method"], "affix": key["affix"],
                           "label": key["label"], "pos": pos, "source_line": line_no,
                           "lang_code": entry.get("lang_code"), "senses": senses[:MAX_LEMMA_SENSES]})

    def _record_head(self, tag, word, entry, line_no):
        romans = romanizations_of(entry)
        bucket = self.romans.setdefault((tag, word), [])
        for r in romans:
            if r not in bucket:
                bucket.append(r)
        for index, sense in enumerate(entry.get("senses") or [], 1):
            if not isinstance(sense, dict):
                continue
            item = {"pointer_targets": pointer_targets(sense), "examples": examples_of(sense)}
            for key in ("qualifier", "head_nr"):
                if isinstance(sense.get(key), str):
                    item[key] = sense[key]
            raw = sense.get("raw_glosses")
            if isinstance(raw, list) and raw and raw != sense.get("glosses"):
                item["raw_glosses"] = [norm(g) for g in raw if isinstance(g, str)]
            self.extras[(tag, line_no, index)] = item

    def _record_translations(self, english, entry, line_no):
        pos = pos_key(entry.get("pos"))
        for item, sense_ctx in translation_items(entry):
            if not isinstance(item, dict):
                continue
            code = item.get("lang_code") or item.get("code")
            target = item.get("word")
            if not isinstance(code, str) or not isinstance(target, str) or not target.strip():
                continue
            if code not in self.trans_codes:
                continue
            target = norm(target.strip())
            exact = self.trans_exact.get((code, target), [])
            hits = [(t, h, "exact") for t, h in exact]
            seen = {(t, h) for t, h in exact}
            for t, h in self.trans_strip.get((code, strip_key(target, code in ALEF_CODES)), []):
                if (t, h) not in seen:
                    hits.append((t, h, "optional_marks"))
                    seen.add((t, h))
            for tag, head, match in hits:
                bucket = self.translations.setdefault((tag, head), [])
                if len(bucket) >= MAX_TRANSLATIONS * 3:
                    continue
                record = {
                    "english_word": english,
                    "english_pos": pos,
                    "translation_sense": item.get("sense") if isinstance(item.get("sense"), str) else None,
                    "listed_form": target,
                    "match": match,
                    "translation_lang_code": code,
                    "source_line": line_no,
                }
                if sense_ctx:
                    record["english_sense_index"] = sense_ctx[0]
                    record["english_sense_glosses"] = sense_ctx[1]
                for key in ("roman", "english", "alt"):
                    if isinstance(item.get(key), str) and item[key].strip():
                        record["translation_" + key] = norm(item[key].strip())
                tags = sorted(t for t in item.get("tags") or [] if isinstance(t, str))
                if tags:
                    record["tags"] = tags
                if record not in bucket:
                    bucket.append(record)

    # lemma passes ------------------------------------------------------
    def lemma_pass(self, path: Path, wanted: dict[str, set[str]]) -> None:
        """Load every entry whose (code, exact or loose key) is wanted."""
        wanted = {code: words for code, words in wanted.items() if isinstance(code, str) and words}
        loose = {code: {strip_key(w, code in ALEF_CODES, True) for w in words} for code, words in wanted.items()}
        with opener(path) as stream:
            for line_no, line in enumerate(stream, 1):
                if not line.strip():
                    continue
                entry = json.loads(line)
                code, word = entry.get("lang_code"), entry.get("word")
                if code not in wanted or not isinstance(word, str):
                    continue
                word = norm(word)
                key = strip_key(word, code in ALEF_CODES, True)
                if word not in wanted[code] and key not in loose[code]:
                    continue
                senses = lemma_senses(entry)
                if not senses:
                    continue
                record = {"word": word, "pos": pos_key(entry.get("pos")), "lang_code": code,
                          "source_line": line_no, "senses": senses}
                for k in {(code, word), (code, "~" + key)}:
                    bucket = self.lemmas.setdefault(k, [])
                    if record not in bucket:
                        bucket.append(record)

    def lemma_lookup(self, code: str, word: str) -> tuple[list[dict], str]:
        exact = self.lemmas.get((code, word), [])
        if exact:
            return exact, "exact"
        loose = self.lemmas.get((code, "~" + strip_key(word, code in ALEF_CODES, True)), [])
        return loose, "optional_marks"


# ---------------------------------------------------------------- building

def pointer_requests(scanner: Scanner) -> dict[str, set[str]]:
    wanted: dict[str, set[str]] = {}
    for tag, doc in scanner.reviews.items():
        for entry in doc["entries"]:
            for sense in entry["edition_sense_candidates"].get("en", []):
                code = sense.get("source_lang_code") or wiktionary_codes(tag)[0]
                extra = scanner.extras.get((tag, sense["source_line"], sense["sense_index"]), {})
                targets = extra.get("pointer_targets") or (
                    text_targets(clean_gloss(sense["glosses"][-1])[0])
                    if is_pointer_text(clean_gloss(sense["glosses"][-1])[0]) else [])
                for target in targets:
                    wanted.setdefault(code, set()).add(target)
                if tag == "ko":
                    for g in sense["glosses"]:
                        if HAN_RE.match(g.strip()):
                            for c in HANJA_CODES:
                                wanted.setdefault(c, set()).add(g.strip())
        for row in scanner.variants.get(tag, {}).get("entries", []):
            for match in row.get("close_matches", []):
                for s in match.get("english_senses", []):
                    for target in s.get("form_of", []):
                        wanted.setdefault(match.get("lang_code"), set()).add(norm(target))
    return wanted


def next_hop_requests(scanner: Scanner, already: dict[str, set[str]]) -> dict[str, set[str]]:
    wanted: dict[str, set[str]] = {}
    for (code, key), records in scanner.lemmas.items():
        if key.startswith("~"):
            continue
        for record in records:
            if any(not s["pointer_targets"] and not is_pointer_text(s["glosses"][-1]) for s in record["senses"]):
                continue
            for s in record["senses"]:
                for target in s["pointer_targets"]:
                    if target not in already.get(code, set()):
                        wanted.setdefault(code, set()).add(target)
    return wanted


def resolve(scanner: Scanner, code: str, target: str, want_pos: str | None, hops: int = 1, trail=()) -> list[dict]:
    """Substantive senses of a pointer target, following at most MAX_HOPS pointers."""
    records, how = scanner.lemma_lookup(code, target)
    same = [r for r in records if want_pos and r["pos"] == want_pos]
    records = same or records
    out = []
    for record in records:
        substantive = [s for s in record["senses"]
                       if not s["pointer_targets"] and not is_pointer_text(s["glosses"][-1])]
        if substantive:
            substantive.sort(key=lambda s: (demotion(s["tags"]), s["sense_index"]))
            for s in substantive[:MAX_LEMMA_SENSES]:
                out.append({"lemma": record["word"], "lemma_pos": record["pos"], "lemma_lang_code": code,
                            "lemma_match": how, "pos_matched": bool(want_pos) and record["pos"] == want_pos,
                            "hops": hops, "via": list(trail), "source_line": record["source_line"],
                            "sense_index": s["sense_index"], "glosses": s["glosses"], "tags": s["tags"],
                            "examples": s["examples"]})
        elif hops < MAX_HOPS:
            for s in record["senses"]:
                for nxt in s["pointer_targets"]:
                    if nxt != target and nxt not in trail:
                        out += resolve(scanner, code, nxt, record["pos"], hops + 1, trail + (record["word"],))
    return out


def definition(tier, text_glosses, pos, evidence, tags=(), examples=(), **extra) -> dict:
    display, issues = clean_gloss(text_glosses[-1])
    item = {"tier": TIERS.index(tier) + 1, "method": tier, "text": display, "pos": pos}
    if len(text_glosses) > 1:
        item["context"] = [clean_gloss(g)[0] for g in text_glosses[:-1]]
    if issues:
        item["cleaned_from"] = text_glosses[-1]
    if tags:
        item["tags"] = list(tags)
        cautions = sorted(set(tags) & (DEMOTED_TAGS | CAUTION_TAGS))
        if cautions:
            item["register"] = cautions
    if examples:
        item["examples"] = list(examples)
    item.update(extra)
    item["evidence"] = evidence
    return item


def build_entry(tag: str, entry: dict, scanner: Scanner, variant_row: dict | None, mapping_note) -> tuple[dict, dict]:
    word = entry["word"]
    senses = entry["edition_sense_candidates"].get("en", [])
    trans = scanner.translations.get((tag, word), [])
    trans.sort(key=lambda t: (t["match"] != "exact", scanner.english_rank.get(t["english_word"], 10**6),
                              t["source_line"]))
    trans = trans[:MAX_TRANSLATIONS]
    lexeme_senses = scanner.lexemes.get((tag, word), [])
    lexeme_words = {link["english_lemma"] for r in lexeme_senses for link in r.get("english_translations", [])}
    trans_words = sorted({t["english_word"] for t in trans} | lexeme_words,
                         key=lambda w: scanner.english_rank.get(w, 10**6))

    defs, notes, stats = [], [], {"pointer_resolved": 0, "pointer_unresolved": 0, "cleaned": 0, "duplicates": 0}
    seen_text: dict[tuple[str, str], int] = {}

    def push(item):
        key = (item["text"].casefold(), item.get("pos") or "")
        if key in seen_text:
            stats["duplicates"] += 1
            prior = defs[seen_text[key]]
            prior.setdefault("also_supported_by", []).append(item["evidence"])
            return
        seen_text[key] = len(defs)
        defs.append(item)

    # tier 1 and 2 -------------------------------------------------------
    own = []
    for order, sense in enumerate(senses):
        code = sense.get("source_lang_code") or wiktionary_codes(tag)[0]
        extra = scanner.extras.get((tag, sense["source_line"], sense["sense_index"]), {})
        glosses = sense["glosses"]
        last = glosses[-1]
        display, issues = clean_gloss(last)
        ev = {"source": "enwiktionary", "source_line": sense["source_line"], "sense_index": sense["sense_index"],
              "lang_code": code}
        if issues:
            stats["cleaned"] += 1
            notes.append({"source_line": sense["source_line"], "sense_index": sense["sense_index"],
                          "issues": issues, "original": last, "display": display})
        targets = list(extra.get("pointer_targets", []))
        pointer = bool(targets) or bool(set(sense.get("tags") or []) & POINTER_TAGS) or is_pointer_text(display)
        relation = clean_gloss(" ".join(glosses))[0]
        if pointer and not targets:
            targets = text_targets(display) or text_targets(relation)
        hanja = tag == "ko" and HAN_RE.match(last.strip())
        if hanja:
            notes.append({"source_line": sense["source_line"], "sense_index": sense["sense_index"],
                          "issues": ["hanja_reading_gloss"], "original": last})
            resolved = []
            for c in HANJA_CODES:
                resolved = resolve(scanner, c, last.strip(), None)
                if resolved:
                    break
            for r in resolved:
                push(definition("lemma_of_form", r["glosses"], r["lemma_pos"],
                                {**ev, "lemma_source_line": r["source_line"], "lemma_sense_index": r["sense_index"]},
                                r["tags"], r["examples"], relation=f"hanja {last.strip()}", lemma=r["lemma"],
                                lemma_lang_code=r["lemma_lang_code"], hops=r["hops"],
                                display=f"{display} (hanja): {clean_gloss(r['glosses'][-1])[0]}"))
            stats["pointer_resolved" if resolved else "pointer_unresolved"] += 1
            if not resolved:
                push(definition("form_of_gloss", glosses, sense.get("pos"), ev, sense.get("tags") or [],
                                relation=f"hanja {last.strip()}"))
            continue
        if pointer:
            resolved = []
            for target in targets:
                resolved += resolve(scanner, code, target, sense.get("pos"))
            if resolved:
                stats["pointer_resolved"] += 1
            else:
                stats["pointer_unresolved"] += 1
                notes.append({"source_line": sense["source_line"], "sense_index": sense["sense_index"],
                              "issues": ["pointer_gloss_unresolved"], "original": last,
                              "pointer_targets": targets})
                push(definition("form_of_gloss", glosses, sense.get("pos"), ev, sense.get("tags") or [],
                                relation=relation, pointer_targets=targets))
            for r in resolved:
                lemma_gloss = clean_gloss(r["glosses"][-1])[0]
                push(definition("lemma_of_form", r["glosses"], r["lemma_pos"],
                                {**ev, "lemma_source_line": r["source_line"], "lemma_sense_index": r["sense_index"]},
                                r["tags"], r["examples"], relation=relation, lemma=r["lemma"],
                                lemma_lang_code=r["lemma_lang_code"], lemma_match=r["lemma_match"],
                                pos_matched=r["pos_matched"], hops=r["hops"],
                                **({"via": r["via"]} if r["via"] else {}),
                                display=f"{relation}: {lemma_gloss}"))
            continue
        tags = sense.get("tags") or []
        extras = {}
        for k in ("qualifier", "raw_glosses"):
            if k in extra:
                extras[k] = extra[k]
        if code != wiktionary_codes(tag)[0] and mapping_note:
            extras["caution"] = f"gloss is filed under Wiktionary '{code}'; confirm it is valid for '{tag}'"
        own.append(definition("own_sense", glosses, sense.get("pos"), ev, tags, extra.get("examples", []), **extras))

    # Rank own senses and resolved lemmas by translation-table corroboration:
    # an English word whose translation table lists this headword and which is
    # itself one of the equivalents a gloss lists. More frequent English
    # corroborators rank higher; circular glosses (gloss == headword) and
    # obsolete/archaic/rare senses rank last; Wiktionary order breaks ties.
    def corroborate(item, exclude=frozenset()):
        eq = equivalents(item["text"])
        hits = [w for w in trans_words if w.casefold() in eq and w not in exclude]
        if hits:
            item["corroborated_by_translation_tables"] = hits[:8]
        if item["text"].strip(" .").casefold() == word.casefold():
            item["circular_gloss"] = True
        return hits

    def rank_key(item, position):
        hits = item.get("corroborated_by_translation_tables") or []
        best = min((scanner.english_rank.get(w, 10**6) for w in hits), default=10**7)
        return (bool(item.get("circular_gloss")), not hits, demotion(item.get("tags")), best, position)

    lemma_defs = defs[:]
    defs.clear(); seen_text.clear()
    for item in own + lemma_defs:
        corroborate(item)
    own = [d for _, d in sorted(enumerate(own), key=lambda p: rank_key(p[1], p[0]))]
    lemma_defs.sort(key=lambda d: d["tier"])
    tier2 = [d for d in lemma_defs if d["method"] == "lemma_of_form"]
    rest = [d for d in lemma_defs if d["method"] != "lemma_of_form"]
    tier2 = [d for _, d in sorted(enumerate(tier2), key=lambda p: rank_key(p[1], p[0]))]
    for item in own + tier2 + rest:
        push(item)

    # tier 3: Wikidata Lexemes -------------------------------------------
    for r in lexeme_senses:
        links = r.get("english_translations", [])
        if r.get("gloss"):
            text = r["gloss"]
        else:
            text = ", ".join(dict.fromkeys(link["english_lemma"] for link in links))
        ev = {"source": "wikidata-lexemes", "lexeme": r["lexeme"], "sense": r["sense"], "match": r["match"],
              "match_how": r["match_how"]}
        extra_l = {"lexeme_lemma": r["lemma"]}
        if r.get("items"):
            extra_l["wikidata_items"] = r["items"]
        if links:
            extra_l["english_translations"] = links
        if r["match"] == "form":
            extra_l["display"] = f"form of {r['lemma']}: {text}"
            if r.get("grammatical_features"):
                extra_l["grammatical_features"] = r["grammatical_features"]
        if r["match_how"] != "exact":
            extra_l["caution"] = "matched with optional marks removed; confirm it is the same word"
        item = definition("lexeme_sense", [text], r["pos"], ev, **extra_l)
        # A sense's own translation links are not independent corroboration.
        corroborate(item, exclude={link["english_lemma"] for link in links})
        push(item)

    # tier 3 -------------------------------------------------------------
    if variant_row:
        for match in variant_row.get("close_matches", []):
            mev = {"source": "definition-variants", "variant": match.get("variant"), "variant_method": match["method"],
                   "source_line": match.get("source_line"), "lang_code": match.get("lang_code")}
            for s in match.get("english_senses", []):
                ev = {**mev, "sense_index": s.get("sense_index")}
                extra_v = {"variant": match.get("variant"), "variant_method": match["method"]}
                if match.get("caution"):
                    extra_v["caution"] = match["caution"]
                resolved = []
                for target in s.get("form_of") or []:
                    resolved += resolve(scanner, match.get("lang_code"), norm(target), match.get("pos"))
                for r in resolved:
                    push(definition("close_variant", r["glosses"], r["lemma_pos"],
                                    {**ev, "lemma_source_line": r["source_line"]}, r["tags"], r["examples"],
                                    relation=clean_gloss(s["glosses"][-1])[0], lemma=r["lemma"], **extra_v))
                if resolved:
                    continue
                if s.get("form_of") or is_pointer_text(clean_gloss(s["glosses"][-1])[0]):
                    extra_v["pointer_gloss"] = True
                push(definition("close_variant", s["glosses"], match.get("pos"), ev, s.get("tags") or [], **extra_v))

    # rule-based keys (Malayalam chillu, Korean particles, Amharic prefixes)
    for seg in scanner.segments.get((tag, word), []):
        for s in seg["senses"]:
            ev = {"source": "enwiktionary", "source_line": seg["source_line"], "sense_index": s["sense_index"],
                  "lang_code": seg["lang_code"], "variant": seg["stem"], "variant_method": seg["method"]}
            extra_s = {"variant": seg["stem"], "variant_method": seg["method"]}
            gloss = clean_gloss(s["glosses"][-1])[0]
            if seg["affix"]:
                extra_s["affix"] = seg["affix"]
                extra_s["affix_meaning"] = seg["label"]
                extra_s["display"] = f"{seg['stem']} ({gloss}) + {seg['affix']} ({seg['label']})"
                extra_s["caution"] = "segmented by rule; confirm the stem and affix"
            push(definition("close_variant", s["glosses"], seg["pos"], ev, s["tags"], s["examples"], **extra_s))

    # tier 6 -------------------------------------------------------------
    for t in trans:
        label = t.get("translation_sense") or (t.get("english_sense_glosses") or [None])[-1]
        text = t["english_word"] if not label else f"{t['english_word']} ({label})"
        ev = {"source": "enwiktionary-translations", "english_word": t["english_word"],
              "english_pos": t["english_pos"], "source_line": t["source_line"], "listed_form": t["listed_form"],
              "match": t["match"], "translation_lang_code": t["translation_lang_code"]}
        if t.get("english_sense_index"):
            ev["english_sense_index"] = t["english_sense_index"]
        extra_t = {k: t[k] for k in ("translation_roman", "translation_english", "tags") if k in t}
        if t["match"] != "exact":
            extra_t["caution"] = "listed with optional marks (e.g. vowel points or accents); confirm it is the same word"
        push(definition("translation_equivalent", [text], t["english_pos"], ev, **extra_t))

    defs.sort(key=lambda d: (d["tier"], bool(d.get("pointer_gloss"))))
    # Review-derived senses (own_sense, form_of_gloss) are never dropped; the
    # cap applies only to the evidence this pass adds.
    kept, added = [], 0
    for d in defs:
        if d["method"] in ("own_sense", "form_of_gloss"):
            kept.append(d)
        elif added < MAX_DEFINITIONS:
            kept.append(d)
            added += 1
    defs = kept
    tiers_present = sorted({d["tier"] for d in defs})
    status = ("no_english_definition_found" if not defs else TIERS[tiers_present[0] - 1] + "_found")
    lead = None
    if defs:
        pool = [(i, d) for i, d in enumerate(defs) if d["tier"] <= 3] or list(enumerate(defs))
        _, best = min(pool, key=lambda p: rank_key(p[1], p[0])[:4] + (p[1]["tier"], p[0]))
        lead = {k: best[k] for k in ("tier", "method", "text", "pos") if k in best}
        if "display" in best:
            lead["display"] = best["display"]
        if best.get("corroborated_by_translation_tables"):
            lead["corroborated_by"] = best["corroborated_by_translation_tables"][:3]
        lead["review_status"] = "required"
    flags = {}
    odd_script = script_mismatch(word, scanner.scripts.get(tag, {}))
    if odd_script:
        flags["script_mismatch"] = odd_script
    if not defs and scanner.other_langs.get((tag, word)):
        flags["found_only_in_other_languages"] = sorted(scanner.other_langs[(tag, word)])
    row = {
        "rank": entry["rank"],
        "word": word,
        "frequency_count": entry.get("frequency_count"),
        "candidate_pos": entry.get("candidate_pos"),
        "english_status": status,
        "lead_candidate": lead,
        "romanizations": scanner.romans.get((tag, word), []),
        "translation_table_english_words": trans_words[:15],
        "definitions": defs,
        "flags": flags,
        "gloss_notes": notes,
        "review_status": "required",
    }
    stats["tiers"] = tiers_present
    return row, stats


def build_profile(path: Path, doc: dict, scanner: Scanner, variants_path: Path | None) -> dict:
    tag = doc["language_tag"]
    vdoc = scanner.variants.get(tag, {})
    vrows = {r["word"]: r for r in vdoc.get("entries", [])}
    rows = []
    counts = {
        "candidate_words": len(doc["entries"]),
        "baseline_words_with_own_english_sense": 0,
        "baseline_words_with_substantive_own_sense": 0,
        "words_with_any_english_definition": 0,
        "words_by_best_tier": {t: 0 for t in TIERS},
        "words_without_english_definition": 0,
        "words_gaining_english_definition": 0,
        "words_gaining_substantive_definition": 0,
        "pointer_senses_resolved": 0,
        "pointer_senses_unresolved": 0,
        "glosses_cleaned_for_display": 0,
        "duplicate_definitions_merged": 0,
        "definitions_total": 0,
        "words_with_translation_corroborated_lead": 0,
        "words_with_examples": 0,
        "words_with_romanization": 0,
        "words_with_wikidata_lexeme_sense": 0,
        "words_with_rule_based_key_definition": 0,
        "words_flagged_script_mismatch": 0,
        "words_found_only_in_other_languages": 0,
        "other_language_codes": {},
        "production_definitions_approved": 0,
    }
    for entry in doc["entries"]:
        row, stats = build_entry(tag, entry, scanner, vrows.get(entry["word"]), doc.get("target_language_mapping_note"))
        rows.append(row)
        had = bool(entry["edition_sense_candidates"].get("en"))
        own_sub = 1 in stats["tiers"]
        counts["baseline_words_with_own_english_sense"] += had
        counts["baseline_words_with_substantive_own_sense"] += own_sub
        if row["definitions"]:
            counts["words_with_any_english_definition"] += 1
            counts["words_by_best_tier"][TIERS[stats["tiers"][0] - 1]] += 1
            if not had:
                counts["words_gaining_english_definition"] += 1
            if not own_sub:
                counts["words_gaining_substantive_definition"] += 1
        else:
            counts["words_without_english_definition"] += 1
        counts["pointer_senses_resolved"] += stats["pointer_resolved"]
        counts["pointer_senses_unresolved"] += stats["pointer_unresolved"]
        counts["glosses_cleaned_for_display"] += stats["cleaned"]
        counts["duplicate_definitions_merged"] += stats["duplicates"]
        counts["definitions_total"] += len(row["definitions"])
        if row["lead_candidate"] and row["lead_candidate"].get("corroborated_by"):
            counts["words_with_translation_corroborated_lead"] += 1
        counts["words_with_examples"] += any(d.get("examples") for d in row["definitions"])
        counts["words_with_romanization"] += bool(row["romanizations"])
        counts["words_with_wikidata_lexeme_sense"] += any(d["method"] == "lexeme_sense" for d in row["definitions"])
        counts["words_with_rule_based_key_definition"] += any(
            d.get("variant_method") in ("chillu_normalized", "particle_stripped", "contracted_particle", "prefix_stripped")
            for d in row["definitions"])
        counts["words_flagged_script_mismatch"] += "script_mismatch" in row["flags"]
        others = row["flags"].get("found_only_in_other_languages")
        if others:
            counts["words_found_only_in_other_languages"] += 1
            for code in others:
                counts["other_language_codes"][code] = counts["other_language_codes"].get(code, 0) + 1
    top = sorted(counts["other_language_codes"].items(), key=lambda kv: -kv[1])[:10]
    counts["other_language_codes"] = dict(top)
    return {
        "schema": SCHEMA,
        "language_tag": tag,
        "review_queue_file": path.as_posix(),
        "variants_file": variants_path.as_posix() if variants_path else None,
        "target_language_mapping_note": doc.get("target_language_mapping_note"),
        "tier_order": list(TIERS),
        "policy": (
            "Add-only. Every review-queue gloss is kept unchanged; this file adds the English evidence "
            "English Wiktionary and Wikidata Lexemes supply: resolved form-of lemmas, lexeme senses, close "
            "variants and rule-based keys, translation-table equivalents, usage examples with English "
            "translations, and romanizations. Words with no English are flagged when their script is rare "
            "in the profile or when only other languages have an entry for the spelling. Definitions are "
            "ranked (own sense corroborated by a translation table, then own sense in Wiktionary order "
            "with obsolete/archaic/rare senses last, then resolved lemma, close variant, translation "
            "equivalent). The lead candidate is a reviewer's starting point; nothing is selected or approved."
        ),
        "source": {**doc["definition_sources"]["en"]},
        "lexeme_source": scanner.lexeme_source,
        "counts": counts,
        "production_status": "review_required",
        "entries": rows,
    }


def english_ranks(reviews: dict) -> dict[str, int]:
    doc = reviews.get("en")
    if not doc:
        return {}
    return {e["word"]: e["rank"] for e in doc["entries"]}


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--reviews", type=Path, required=True)
    parser.add_argument("--variants", type=Path, required=True)
    parser.add_argument("--en", type=Path, required=True)
    parser.add_argument("--lexemes", type=Path, help="Wikidata lexeme JSON dump (.json.gz/.bz2), optional")
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()

    reviews, paths, vpaths, variants = {}, {}, {}, {}
    for path in sorted(args.reviews.glob("*.review.json")):
        doc = json.loads(path.read_text(encoding="utf-8"))
        if doc["language_tag"] in TARGETS:
            reviews[doc["language_tag"]] = doc
            paths[doc["language_tag"]] = path
            vpath = args.variants / path.name.replace(".review.json", ".variants.json")
            if vpath.is_file():
                variants[doc["language_tag"]] = json.loads(vpath.read_text(encoding="utf-8"))
                vpaths[doc["language_tag"]] = vpath

    scanner = Scanner(reviews, variants, english_ranks(reviews))
    if args.lexemes:
        scanner.lexeme_source = {**fruchst_lexemes.SOURCE, "file": args.lexemes.name,
                                 "size_bytes": args.lexemes.stat().st_size}
        scanner.lexemes = fruchst_lexemes.scan_lexemes(
            args.lexemes, {tag: {e["word"] for e in doc["entries"]} for tag, doc in reviews.items()})
    scanner.first_pass(args.en)
    wanted = pointer_requests(scanner)
    requested = {c: set(w) for c, w in wanted.items()}
    for hop in range(MAX_HOPS):
        if not wanted:
            break
        scanner.lemma_pass(args.en, wanted)
        wanted = next_hop_requests(scanner, requested)
        for c, w in wanted.items():
            requested.setdefault(c, set()).update(w)

    args.output.mkdir(parents=True, exist_ok=True)
    for tag, doc in reviews.items():
        out = build_profile(paths[tag], doc, scanner, vpaths.get(tag))
        target = args.output / paths[tag].name.replace(".review.json", ".english.json")
        target.write_text(json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8")
        print(json.dumps({"language": tag, **out["counts"]}, ensure_ascii=False, sort_keys=True))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())