Home /Modules/languages/fruchst_lexemes.py

fruchst_lexemes.py

languages/tools/fruchst_lexemes.py · 157 line(s) · py

#!/usr/bin/env python3
"""Wikidata Lexemes (CC0) as an independent English-definition source.

Streams the Wikidata lexeme JSON dump (one entity per line inside a JSON
array) once. A FRUCHST headword matches a lexeme when it equals the lexeme's
lemma or one of its form representations in a mapped language (exactly, or
with optional marks removed). For each matching lexeme sense, the English
evidence is the sense's English gloss, and/or the English lexemes it is linked
to by "translation" (P5972) statements. The sense's "item for this sense"
(P5137) QID is kept for later cross-edition alignment.
"""
from __future__ import annotations

import bz2
import gzip
import json
from pathlib import Path

from fruchst_definition_batch import norm, wiktionary_codes
from fruchst_close_variants import ALEF_TAGS, strip_key
from fruchst_english_segments import script_of

SOURCE = {
    "project": "Wikidata Lexemes",
    "url": "https://dumps.wikimedia.org/wikidatawiki/entities/latest-lexemes.json.gz",
    "license": "CC0 1.0 (Wikidata structured data); attribution not required but recorded",
}
# Representation language keys beyond the Wiktionary codes.
EXTRA_KEYS = {
    "zh-Hans": ("zh", "zh-hans", "cmn", "cmn-hans"),
    "pa-Guru": ("pa", "pa-guru"),
    "uz": ("uz", "uz-cyrl"),
    "prs": ("prs", "fa", "fa-af"),
    "ms": ("ms", "zsm"),
    "tl": ("tl", "fil"),
}
LEXICAL_CATEGORIES = {
    "Q1084": "noun", "Q24905": "verb", "Q34698": "adjective", "Q380057": "adverb",
    "Q36224": "pronoun", "Q4833830": "preposition", "Q36484": "conjunction",
    "Q83034": "interjection", "Q63116": "numeral", "Q147276": "proper noun",
}
MAX_SENSES_PER_WORD = 20


def opener(path: Path):
    suffix = path.suffix.casefold()
    if suffix == ".gz":
        return gzip.open(path, "rt", encoding="utf-8")
    if suffix == ".bz2":
        return bz2.open(path, "rt", encoding="utf-8")
    return open(path, encoding="utf-8")


def entity_lines(path: Path):
    with opener(path) as stream:
        for line in stream:
            line = line.strip().rstrip(",")
            if not line or line in ("[", "]"):
                continue
            yield json.loads(line)


def statement_ids(claims: dict, prop: str) -> list[str]:
    out = []
    for claim in (claims or {}).get(prop, []) or []:
        value = (((claim or {}).get("mainsnak") or {}).get("datavalue") or {}).get("value")
        if isinstance(value, dict) and isinstance(value.get("id"), str):
            out.append(value["id"])
    return out


def scan_lexemes(path: Path, words_by_tag: dict[str, set[str]]) -> dict[tuple[str, str], list[dict]]:
    """Return (tag, word) -> English sense records from matching lexemes."""
    keymap: dict[str, list[str]] = {}
    for tag in words_by_tag:
        for key in set(wiktionary_codes(tag)) | set(EXTRA_KEYS.get(tag, ())):
            keymap.setdefault(key.casefold(), []).append(tag)
    exact: dict[tuple[str, str], list[str]] = {}
    loose: dict[tuple[str, str], list[str]] = {}
    for tag, words in words_by_tag.items():
        for word in words:
            exact.setdefault((tag, word), []).append(word)
            loose.setdefault((tag, strip_key(word, tag in ALEF_TAGS)), []).append(word)

    english: dict[str, tuple[str, str | None]] = {}       # English sense id -> (lemma, gloss)
    hits: dict[tuple[str, str], list[dict]] = {}
    for entity in entity_lines(path):
        if entity.get("type") != "lexeme":
            continue
        lemmas = entity.get("lemmas") or {}
        senses = entity.get("senses") or []
        if "en" in lemmas and entity.get("language") == "Q1860":
            for sense in senses:
                gloss = ((sense.get("glosses") or {}).get("en") or {}).get("value")
                english[sense.get("id")] = (norm(lemmas["en"]["value"]), norm(gloss) if gloss else None)
        if not senses:
            continue
        reps = [(k, v.get("value"), "lemma", []) for k, v in lemmas.items() if isinstance(v, dict)]
        for form in entity.get("forms") or []:
            for k, v in (form.get("representations") or {}).items():
                if isinstance(v, dict):
                    reps.append((k, v.get("value"), "form", form.get("grammaticalFeatures") or []))
        matched: dict[tuple[str, str], tuple[str, str, list]] = {}
        for key, value, kind, features in reps:
            if not isinstance(value, str):
                continue
            tags = keymap.get(key.casefold()) or keymap.get(key.split("-")[0].casefold()) or []
            value = norm(value)
            for tag in tags:
                for word in exact.get((tag, value), []):
                    prior = matched.get((tag, word))
                    if prior is None or (prior[0] != "lemma" and kind == "lemma") or prior[1] != "exact":
                        matched[(tag, word)] = (kind, "exact", features)
                for word in loose.get((tag, strip_key(value, tag in ALEF_TAGS)), []):
                    matched.setdefault((tag, word), (kind, "optional_marks", features))
        if not matched:
            continue
        lemma_reps = [(k, norm(v["value"])) for k, v in lemmas.items() if isinstance(v, dict) and v.get("value")]

        def lemma_for(tag: str, script_sample: str) -> str | None:
            """The lemma spelling in the matched profile's own language key and script."""
            own = [v for k, v in lemma_reps if tag in (keymap.get(k.casefold()) or [])]
            same_script = [v for v in own if script_of(v) == script_of(script_sample)]
            return (same_script or own or [v for _, v in lemma_reps] or [None])[0]
        pos = LEXICAL_CATEGORIES.get(entity.get("lexicalCategory"))
        sense_records = []
        for sense in senses:
            gloss = ((sense.get("glosses") or {}).get("en") or {}).get("value")
            claims = sense.get("claims") or {}
            record = {"lexeme": entity.get("id"), "sense": sense.get("id"), "pos": pos,
                      "lexical_category": entity.get("lexicalCategory"), "gloss": norm(gloss) if gloss else None,
                      "items": statement_ids(claims, "P5137"), "translation_senses": statement_ids(claims, "P5972")}
            if record["gloss"] or record["translation_senses"]:
                sense_records.append(record)
        for (tag, word), (kind, how, features) in matched.items():
            bucket = hits.setdefault((tag, word), [])
            for record in sense_records:
                if len(bucket) < MAX_SENSES_PER_WORD:
                    bucket.append({**record, "lemma": lemma_for(tag, word), "match": kind, "match_how": how,
                                   **({"grammatical_features": features} if kind == "form" else {})})

    # Resolve translation links to English lexeme senses.
    for records in hits.values():
        for record in records:
            linked = []
            for sid in record.pop("translation_senses"):
                if sid in english:
                    lemma, gloss = english[sid]
                    linked.append({"english_lemma": lemma, "english_gloss": gloss, "english_sense": sid})
            if linked:
                record["english_translations"] = linked
    for key in list(hits):
        hits[key] = [r for r in hits[key] if r.get("gloss") or r.get("english_translations")]
        if not hits[key]:
            del hits[key]
    return hits