fruchst_lexemes.py
languages/tools/fruchst_lexemes.py · 157 line(s) · py
#!/usr/bin/env python3
"""Wikidata Lexemes (CC0) as an independent English-definition source.
Streams the Wikidata lexeme JSON dump (one entity per line inside a JSON
array) once. A FRUCHST headword matches a lexeme when it equals the lexeme's
lemma or one of its form representations in a mapped language (exactly, or
with optional marks removed). For each matching lexeme sense, the English
evidence is the sense's English gloss, and/or the English lexemes it is linked
to by "translation" (P5972) statements. The sense's "item for this sense"
(P5137) QID is kept for later cross-edition alignment.
"""
from __future__ import annotations
import bz2
import gzip
import json
from pathlib import Path
from fruchst_definition_batch import norm, wiktionary_codes
from fruchst_close_variants import ALEF_TAGS, strip_key
from fruchst_english_segments import script_of
SOURCE = {
"project": "Wikidata Lexemes",
"url": "https://dumps.wikimedia.org/wikidatawiki/entities/latest-lexemes.json.gz",
"license": "CC0 1.0 (Wikidata structured data); attribution not required but recorded",
}
# Representation language keys beyond the Wiktionary codes.
EXTRA_KEYS = {
"zh-Hans": ("zh", "zh-hans", "cmn", "cmn-hans"),
"pa-Guru": ("pa", "pa-guru"),
"uz": ("uz", "uz-cyrl"),
"prs": ("prs", "fa", "fa-af"),
"ms": ("ms", "zsm"),
"tl": ("tl", "fil"),
}
LEXICAL_CATEGORIES = {
"Q1084": "noun", "Q24905": "verb", "Q34698": "adjective", "Q380057": "adverb",
"Q36224": "pronoun", "Q4833830": "preposition", "Q36484": "conjunction",
"Q83034": "interjection", "Q63116": "numeral", "Q147276": "proper noun",
}
MAX_SENSES_PER_WORD = 20
def opener(path: Path):
suffix = path.suffix.casefold()
if suffix == ".gz":
return gzip.open(path, "rt", encoding="utf-8")
if suffix == ".bz2":
return bz2.open(path, "rt", encoding="utf-8")
return open(path, encoding="utf-8")
def entity_lines(path: Path):
with opener(path) as stream:
for line in stream:
line = line.strip().rstrip(",")
if not line or line in ("[", "]"):
continue
yield json.loads(line)
def statement_ids(claims: dict, prop: str) -> list[str]:
out = []
for claim in (claims or {}).get(prop, []) or []:
value = (((claim or {}).get("mainsnak") or {}).get("datavalue") or {}).get("value")
if isinstance(value, dict) and isinstance(value.get("id"), str):
out.append(value["id"])
return out
def scan_lexemes(path: Path, words_by_tag: dict[str, set[str]]) -> dict[tuple[str, str], list[dict]]:
"""Return (tag, word) -> English sense records from matching lexemes."""
keymap: dict[str, list[str]] = {}
for tag in words_by_tag:
for key in set(wiktionary_codes(tag)) | set(EXTRA_KEYS.get(tag, ())):
keymap.setdefault(key.casefold(), []).append(tag)
exact: dict[tuple[str, str], list[str]] = {}
loose: dict[tuple[str, str], list[str]] = {}
for tag, words in words_by_tag.items():
for word in words:
exact.setdefault((tag, word), []).append(word)
loose.setdefault((tag, strip_key(word, tag in ALEF_TAGS)), []).append(word)
english: dict[str, tuple[str, str | None]] = {} # English sense id -> (lemma, gloss)
hits: dict[tuple[str, str], list[dict]] = {}
for entity in entity_lines(path):
if entity.get("type") != "lexeme":
continue
lemmas = entity.get("lemmas") or {}
senses = entity.get("senses") or []
if "en" in lemmas and entity.get("language") == "Q1860":
for sense in senses:
gloss = ((sense.get("glosses") or {}).get("en") or {}).get("value")
english[sense.get("id")] = (norm(lemmas["en"]["value"]), norm(gloss) if gloss else None)
if not senses:
continue
reps = [(k, v.get("value"), "lemma", []) for k, v in lemmas.items() if isinstance(v, dict)]
for form in entity.get("forms") or []:
for k, v in (form.get("representations") or {}).items():
if isinstance(v, dict):
reps.append((k, v.get("value"), "form", form.get("grammaticalFeatures") or []))
matched: dict[tuple[str, str], tuple[str, str, list]] = {}
for key, value, kind, features in reps:
if not isinstance(value, str):
continue
tags = keymap.get(key.casefold()) or keymap.get(key.split("-")[0].casefold()) or []
value = norm(value)
for tag in tags:
for word in exact.get((tag, value), []):
prior = matched.get((tag, word))
if prior is None or (prior[0] != "lemma" and kind == "lemma") or prior[1] != "exact":
matched[(tag, word)] = (kind, "exact", features)
for word in loose.get((tag, strip_key(value, tag in ALEF_TAGS)), []):
matched.setdefault((tag, word), (kind, "optional_marks", features))
if not matched:
continue
lemma_reps = [(k, norm(v["value"])) for k, v in lemmas.items() if isinstance(v, dict) and v.get("value")]
def lemma_for(tag: str, script_sample: str) -> str | None:
"""The lemma spelling in the matched profile's own language key and script."""
own = [v for k, v in lemma_reps if tag in (keymap.get(k.casefold()) or [])]
same_script = [v for v in own if script_of(v) == script_of(script_sample)]
return (same_script or own or [v for _, v in lemma_reps] or [None])[0]
pos = LEXICAL_CATEGORIES.get(entity.get("lexicalCategory"))
sense_records = []
for sense in senses:
gloss = ((sense.get("glosses") or {}).get("en") or {}).get("value")
claims = sense.get("claims") or {}
record = {"lexeme": entity.get("id"), "sense": sense.get("id"), "pos": pos,
"lexical_category": entity.get("lexicalCategory"), "gloss": norm(gloss) if gloss else None,
"items": statement_ids(claims, "P5137"), "translation_senses": statement_ids(claims, "P5972")}
if record["gloss"] or record["translation_senses"]:
sense_records.append(record)
for (tag, word), (kind, how, features) in matched.items():
bucket = hits.setdefault((tag, word), [])
for record in sense_records:
if len(bucket) < MAX_SENSES_PER_WORD:
bucket.append({**record, "lemma": lemma_for(tag, word), "match": kind, "match_how": how,
**({"grammatical_features": features} if kind == "form" else {})})
# Resolve translation links to English lexeme senses.
for records in hits.values():
for record in records:
linked = []
for sid in record.pop("translation_senses"):
if sid in english:
lemma, gloss = english[sid]
linked.append({"english_lemma": lemma, "english_gloss": gloss, "english_sense": sid})
if linked:
record["english_translations"] = linked
for key in list(hits):
hits[key] = [r for r in hits[key] if r.get("gloss") or r.get("english_translations")]
if not hits[key]:
del hits[key]
return hits