Home /Modules/languages/fruchst_close_variants.py

fruchst_close_variants.py

languages/tools/fruchst_close_variants.py · 269 line(s) · py

#!/usr/bin/env python3
"""Find close English definitions for FRUCHST headwords that have none.

For each headword whose review queue has no English Wiktionary sense, look for
a close variant in English Wiktionary (same target language) and record that
variant's English glosses as a *close* definition. Methods, closest first:

  case_variant          same word, different letter case
  orthographic_variant  same word after removing optional marks: Latin/Cyrillic/
                        Greek diacritics, Arabic short vowels and tatweel, and
                        unifying Arabic-script letter variants (ي/ى/ی, ك/ک; for
                        Arabic only also أ/إ/آ/ٱ to ا). ة and ه are kept distinct.
  diacritic_variant     looser: all Latin/Cyrillic/Greek accent and tone marks
                        removed; flagged because marks can distinguish words
  inflected_form        the word is listed in another entry's inflection table
                        ("forms"); the close definition is that lemma's gloss
  transliteration       Uzbek only: official Cyrillic-to-Latin conversion
  clitic_stripped       Arabic only: a leading و/ف/ب/ل/ك and/or ال removed

Every match keeps the variant word, method, source line and tags. A close
definition is not the headword's own definition; all rows stay review-required.
"""
from __future__ import annotations

import argparse
import gzip
import json
import sys
import unicodedata
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
from fruchst_definition_batch import SOURCE_CATALOG, TARGETS, norm, pos_key, wiktionary_codes  # noqa: E402

METHOD_ORDER = ("case_variant", "orthographic_variant", "inflected_form", "transliteration", "diacritic_variant", "clitic_stripped")
SKIP_FORM_TAGS = {"romanization", "transliteration", "table-tags", "inflection-template", "class", "canonical"}
MAX_MATCHES = 20

# Short vowels, tanween, shadda, sukun, dagger alef and tatweel. Hamza and
# madda (U+0653-U+0655) are letters-in-use and are kept.
ARABIC_MARKS = {chr(c) for c in range(0x064B, 0x0653)} | {"\u0670", "\u0640"}
# Letter variants that are interchangeable spellings. Teh marbuta (ة) and heh
# (ه) are NOT merged: they distinguish words (كله "all of it" vs كلة "shell").
ARABIC_LETTERS = str.maketrans({"ى": "ي", "ی": "ي", "ې": "ي", "ك": "ک"})
# Hamza seat omission is a common informal Arabic spelling. In Urdu/Persian,
# alef madda is distinct (آتا "comes" vs اتا), so this applies to Arabic only.
ARABIC_ALEF = str.maketrans({"أ": "ا", "إ": "ا", "آ": "ا", "ٱ": "ا"})
ALEF_TAGS = {"ar"}
ALEF_CODES = {"ar"}
# Chinese inflection-table "forms" also list dialectal and alternative
# characters; only the plain simplified form of a traditional entry is used.
ZH_FORM_TAGS = {"Simplified-Chinese"}
STRIP_SCRIPTS = ("LATIN", "CYRILLIC", "GREEK")
# Conventionally optional spellings: Russian/Belarusian ё is usually written е;
# Romanian cedilla letters are legacy encodings of the comma-below letters.
OPTIONAL_LETTERS = str.maketrans({"ё": "е", "ş": "ș", "ţ": "ț"})
APOSTROPHES = str.maketrans({"'": "ʻ", "‘": "ʻ", "’": "ʼ", "`": "ʻ", "ʹ": "ʻ"})

UZ_CYR = {
    "а": "a", "б": "b", "в": "v", "г": "g", "ғ": "gʻ", "д": "d", "е": "e", "ё": "yo", "ж": "j",
    "з": "z", "и": "i", "й": "y", "к": "k", "қ": "q", "л": "l", "м": "m", "н": "n", "о": "o",
    "п": "p", "р": "r", "с": "s", "т": "t", "у": "u", "ў": "oʻ", "ф": "f", "х": "x", "ҳ": "h",
    "ц": "ts", "ч": "ch", "ш": "sh", "ъ": "ʼ", "ь": "", "э": "e", "ю": "yu", "я": "ya",
}
UZ_VOWELS = set("аеёиоуўэюя")
AR_PREFIXES = ("وال", "فال", "بال", "كال", "لل", "ال", "و", "ف", "ب", "ل", "ك")


def strip_key(word: str, alef: bool = False, diacritics: bool = False) -> str:
    """Case-fold and remove conventionally optional marks.

    With diacritics=True, every Latin/Cyrillic/Greek combining mark is also
    removed (looser: tone and accent marks can distinguish words). Abugida
    vowel signs are never touched.
    """
    out = []
    word = "".join(ch for ch in unicodedata.normalize("NFC", word) if ch not in ARABIC_MARKS)
    word = word.casefold().translate(OPTIONAL_LETTERS)
    for ch in unicodedata.normalize("NFD", word):
        if diacritics and unicodedata.category(ch) == "Mn" and out:
            base = unicodedata.name(out[-1], "")
            if base.startswith(STRIP_SCRIPTS):
                continue
        out.append(ch)
    key = unicodedata.normalize("NFC", "".join(out)).translate(ARABIC_LETTERS).translate(APOSTROPHES)
    return key.translate(ARABIC_ALEF) if alef else key


def uz_latin(word: str) -> str:
    """Uzbek Cyrillic to Latin (1995 official alphabet); 'е' is 'ye' word-initially and after vowels."""
    out = []
    for i, ch in enumerate(word.lower()):
        if ch == "е" and (i == 0 or word.lower()[i - 1] in UZ_VOWELS):
            out.append("ye")
        else:
            out.append(UZ_CYR.get(ch, ch))
    return "".join(out)


def ar_stems(word: str) -> list[str]:
    stems = []
    for prefix in AR_PREFIXES:
        if word.startswith(prefix) and len(word) - len(prefix) >= 2:
            stems.append(word[len(prefix):])
    return stems


def sense_summaries(entry: dict) -> list[dict]:
    out = []
    for index, sense in enumerate(entry.get("senses") or [], 1):
        if not isinstance(sense, dict):
            continue
        glosses = [norm(g.strip()) for g in sense.get("glosses") or [] if isinstance(g, str) and g.strip()]
        if not glosses:
            continue
        item = {"sense_index": index, "glosses": glosses}
        tags = sorted(t for t in sense.get("tags") or [] if isinstance(t, str))
        if tags:
            item["tags"] = tags
        form_of = [f.get("word") for f in sense.get("form_of") or [] if isinstance(f, dict) and f.get("word")]
        if form_of:
            item["form_of"] = form_of
        out.append(item)
    return out


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--reviews", type=Path, required=True)
    parser.add_argument("--en", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()

    reviews = {}
    for path in sorted(args.reviews.glob("*.review.json")):
        doc = json.loads(path.read_text(encoding="utf-8"))
        if doc["language_tag"] in TARGETS:
            reviews[path] = doc

    # Lookup keys: (wiktionary_code, key) -> [(tag, headword, method)]
    exact_keys: dict[tuple, list] = {}
    strip_keys: dict[tuple, list] = {}
    loose_keys: dict[tuple, list] = {}
    missing: dict[str, list[str]] = {}
    for doc in reviews.values():
        tag = doc["language_tag"]
        words = [e["word"] for e in doc["entries"] if not e["edition_sense_candidates"]["en"]]
        if tag == "en":
            words = []
        missing[tag] = words
        for code in wiktionary_codes(tag):
            for word in words:
                exact_keys.setdefault((code, word.casefold()), []).append((tag, word, "case_variant"))
                strip_keys.setdefault((code, strip_key(word, tag in ALEF_TAGS)), []).append((tag, word, "orthographic_variant"))
                loose_keys.setdefault((code, strip_key(word, tag in ALEF_TAGS, True)), []).append((tag, word, "diacritic_variant"))
                if tag == "uz":
                    strip_keys.setdefault((code, strip_key(uz_latin(word))), []).append((tag, word, "transliteration"))
                if tag == "ar":
                    for stem in ar_stems(word):
                        strip_keys.setdefault((code, strip_key(stem, True)), []).append((tag, word, "clitic_stripped"))
    codes = {code for code, _ in exact_keys}
    form_keys = {k: [(t, w, "inflected_form") for t, w, _ in v] for k, v in exact_keys.items()}

    matches: dict[tuple[str, str], list[dict]] = {}

    def add(tag, word, method, entry, line_no, extra=None):
        bucket = matches.setdefault((tag, word), [])
        if len(bucket) >= MAX_MATCHES:
            return
        senses = sense_summaries(entry)
        if not senses:
            return
        record = {"method": method, "variant": norm(entry["word"]), "pos": pos_key(entry.get("pos")),
                  "lang_code": entry.get("lang_code"), "source_line": line_no, "english_senses": senses}
        if extra:
            record.update(extra)
        if record not in bucket:
            bucket.append(record)

    opener = gzip.open if args.en.suffix.casefold() == ".gz" else open
    with opener(args.en, "rt", encoding="utf-8") as stream:
        for line_no, line in enumerate(stream, 1):
            entry = json.loads(line) if line.strip() else {}
            code = entry.get("lang_code")
            word = entry.get("word")
            if code not in codes or not isinstance(word, str):
                continue
            word = norm(word)
            for tag, head, method in exact_keys.get((code, word.casefold()), []):
                if head != word:
                    add(tag, head, method, entry, line_no)
            seen_exact = {(t, h) for t, h, _ in exact_keys.get((code, word.casefold()), [])}
            safe_hits = strip_keys.get((code, strip_key(word, code in ALEF_CODES)), [])
            for tag, head, method in safe_hits:
                if method == "orthographic_variant" and (tag, head) in seen_exact:
                    continue
                add(tag, head, method, entry, line_no)
            seen_safe = seen_exact | {(t, h) for t, h, _ in safe_hits}
            for tag, head, method in loose_keys.get((code, strip_key(word, code in ALEF_CODES, True)), []):
                if (tag, head) not in seen_safe:
                    add(tag, head, method, entry, line_no, {"caution": "accent or tone marks differ; may be a different word"})
            for form in entry.get("forms") or []:
                if not isinstance(form, dict) or not isinstance(form.get("form"), str):
                    continue
                tags = set(form.get("tags") or [])
                if tags & SKIP_FORM_TAGS:
                    continue
                if code == "zh" and tags != ZH_FORM_TAGS:
                    continue
                surface = norm(form["form"].strip())
                extra = {"form_tags": sorted(tags), "listed_form": surface}
                exact_hits = form_keys.get((code, surface.casefold()), [])
                for tag, head, method in exact_hits:
                    add(tag, head, method, entry, line_no, extra)
                exact_heads = {(t, h) for t, h, _ in exact_hits}
                for tag, head, via in strip_keys.get((code, strip_key(surface, code in ALEF_CODES)), []):
                    if (tag, head) not in exact_heads:
                        add(tag, head, "inflected_form", entry, line_no, {**extra, "via": via})

    args.output.mkdir(parents=True, exist_ok=True)
    rank = {m: i for i, m in enumerate(METHOD_ORDER)}
    for path, doc in reviews.items():
        tag = doc["language_tag"]
        rows, by_method = [], {m: 0 for m in METHOD_ORDER}
        missing_set = set(missing[tag])
        for entry in doc["entries"]:
            if entry["word"] not in missing_set:
                continue
            found = sorted(matches.get((tag, entry["word"]), []), key=lambda r: (rank[r["method"]], r["source_line"]))
            best = found[0]["method"] if found else None
            if best:
                by_method[best] += 1
            rows.append({"rank": entry["rank"], "word": entry["word"],
                         "status": "close_definition_found" if found else "no_close_variant",
                         "closest_method": best, "close_matches": found, "review_status": "required"})
        counts = {
            "candidate_words": len(doc["entries"]),
            "words_without_english": len(rows),
            "words_with_close_definition": sum(1 for r in rows if r["close_matches"]),
            "by_closest_method": by_method,
            "production_definitions_approved": 0,
        }
        out = {
            "schema": "SLeeLa FRUCHST close-variant English definitions v1",
            "language_tag": tag,
            "review_queue_file": path.as_posix(),
            "wiktionary_codes": list(wiktionary_codes(tag)),
            "method_order": list(METHOD_ORDER),
            "policy": (
                "Only headwords with no English Wiktionary sense are listed. A close definition is the "
                "English gloss of a variant (case, optional marks, inflection-table form, Uzbek "
                "transliteration, or Arabic clitic removal) in the same Wiktionary language. It is not "
                "the headword's own definition; nothing is selected or approved."
            ),
            "source": {**SOURCE_CATALOG["en"],
                       "license": "Wiktionary dual license: CC BY-SA 4.0 and GFDL; preserve attribution"},
            "counts": counts,
            "production_status": "review_required",
            "entries": rows,
        }
        target = args.output / path.name.replace(".review.json", ".variants.json")
        target.write_text(json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8")
        print(json.dumps({"language": tag, **counts}, ensure_ascii=False, sort_keys=True))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())