fruchst_english_segments.py
languages/tools/fruchst_english_segments.py · 118 line(s) · py
#!/usr/bin/env python3
"""Rule-based lookup keys and script checks for the FRUCHST English round.
These rules only propose a dictionary headword to look up; the definition
always comes from that headword's own English Wiktionary entry, and every
segmented match records the rule, the stem and the stripped affix.
chillu_normalized Malayalam: word-final consonant + virama written in the
pre-Unicode-5.1 way (ന്) is the atomic chillu (ൻ) that
Wiktionary uses. Not a segmentation; no caution.
particle_stripped Korean: a trailing case/topic/other particle is removed
(당신이 -> 당신 + 이); the stem must be a noun, pronoun,
proper noun or numeral entry.
contracted_particle Korean: a final ㄴ (topic) or ㄹ (object) contracted onto
the last syllable is removed (그건 -> 그거 + -ㄴ).
prefix_stripped Amharic: a leading preposition prefix (የ, በ, ለ, ከ, እንደ) is
removed; the stem must be a noun, proper noun, adjective
or pronoun entry.
"""
from __future__ import annotations
import unicodedata
NOMINAL_POS = {"noun", "pronoun", "proper noun", "numeral", "name"}
ML_CHILLU = {"ണ്": "ൺ", "ന്": "ൻ", "ര്": "ർ", "ല്": "ൽ", "ള്": "ൾ", "ക്": "ൿ"}
KO_PARTICLES = (
("에서는", "at/in (topic)"), ("에게서", "from (a person)"), ("한테서", "from (a person)"),
("으로는", "by/toward (topic)"), ("이라고", "quotative"), ("에서", "at/in/from (location)"),
("에게", "to (a person)"), ("한테", "to (a person)"), ("으로", "by/toward/as"), ("까지", "until/up to"),
("부터", "from (starting point)"), ("보다", "than"), ("처럼", "like"), ("마다", "every"),
("이나", "or/as many as"), ("이랑", "with/and"), ("라고", "quotative"), ("이야", "vocative/copula"),
("은", "topic marker"), ("는", "topic marker"), ("이", "subject marker"), ("가", "subject marker"),
("을", "object marker"), ("를", "object marker"), ("의", "possessive marker"), ("에", "at/to"),
("도", "also/too"), ("로", "by/toward/as"), ("와", "and/with"), ("과", "and/with"),
("만", "only"), ("야", "vocative/copula"), ("랑", "with/and"), ("요", "polite ending"),
)
# Hangul syllable = 0xAC00 + (initial*21 + medial)*28 + final; final 4 = ㄴ, 8 = ㄹ.
KO_CONTRACTED_FINALS = {4: ("-ㄴ", "contracted topic marker"), 8: ("-ㄹ", "contracted object marker")}
AM_PREFIXES = (("እንደ", "like/as"), ("የ", "of (genitive)"), ("በ", "in/by/with"), ("ለ", "for/to"), ("ከ", "from/than"))
AM_STEM_POS = NOMINAL_POS | {"adjective"}
def segment_keys(tag: str, word: str) -> list[dict]:
"""Candidate dictionary headwords for a word with no English entry."""
out = []
if tag == "ml":
for virama, chillu in ML_CHILLU.items():
if word.endswith(virama) and len(word) > len(virama):
out.append({"stem": word[: -len(virama)] + chillu, "method": "chillu_normalized",
"affix": None, "label": None, "allowed_pos": None})
break
elif tag == "ko":
for particle, label in KO_PARTICLES:
if word.endswith(particle) and len(word) > len(particle):
out.append({"stem": word[: -len(particle)], "method": "particle_stripped",
"affix": particle, "label": label, "allowed_pos": NOMINAL_POS})
last = word[-1]
code = ord(last) - 0xAC00
if 0 <= code < 11172 and code % 28 in KO_CONTRACTED_FINALS:
base = chr(0xAC00 + code - code % 28)
out.append({"stem": word[:-1] + base, "method": "contracted_particle",
"affix": KO_CONTRACTED_FINALS[code % 28][0],
"label": KO_CONTRACTED_FINALS[code % 28][1], "allowed_pos": NOMINAL_POS})
elif tag == "am":
for prefix, label in AM_PREFIXES:
if word.startswith(prefix) and len(word) - len(prefix) >= 2:
out.append({"stem": word[len(prefix):], "method": "prefix_stripped",
"affix": prefix, "label": label, "allowed_pos": AM_STEM_POS})
break
return [k for k in out if k["stem"] and k["stem"] != word]
def script_of(word: str) -> str | None:
"""Unicode script group of the first letter (Japanese and Korean kana/Han grouped)."""
for ch in word:
if not ch.isalpha():
continue
name = unicodedata.name(ch, "")
if not name:
return None
first = name.split()[0]
if first in {"CJK", "HIRAGANA", "KATAKANA"}:
return "HAN-KANA"
if first == "HANGUL":
return "HANGUL"
return first
return None
def profile_scripts(words) -> dict[str, float]:
counts: dict[str, int] = {}
total = 0
for word in words:
script = script_of(word)
if script:
counts[script] = counts.get(script, 0) + 1
total += 1
return {s: c / total for s, c in counts.items()} if total else {}
def script_mismatch(word: str, shares: dict[str, float]) -> str | None:
"""Return the word's script when it differs from the profile's dominant script.
Han characters inside a Hangul profile are expected (hanja) and not flagged.
"""
script = script_of(word)
if script is None or not shares:
return None
dominant = max(shares, key=shares.get)
if script == dominant or shares[dominant] < 0.5:
return None
if script == "HAN-KANA" and dominant == "HANGUL":
return None
return script