Home /Modules/languages/stage_wordhoard.py

stage_wordhoard.py

languages/tools/stage_wordhoard.py · 136 line(s) · py

#!/usr/bin/env python3
"""Stage reproducible FRUCHST frequency candidates from wordhoard samples."""

from __future__ import annotations

import argparse
import csv
import json
import sys
import unicodedata
import urllib.error
import urllib.request
from pathlib import Path

UPSTREAM = "https://github.com/natema/wordhoard"
REVISION = "1bc5730e8d6e682c416c03680b7cb8c6c7ca8cd0"
LANGUAGES = {
    "en": "english",
    "fr": "french",
    "es": "spanish",
    "de": "standard-german",
}
LICENSE = "CC-BY-SA-4.0"


def read_candidates(language: str, limit: int = 1000) -> list[dict]:
    if language not in LANGUAGES:
        raise ValueError(f"unsupported first-batch language: {language}")
    url = (
        f"https://raw.githubusercontent.com/natema/wordhoard/{REVISION}/"
        f"samples/{language}.csv"
    )
    request = urllib.request.Request(
        url, headers={"User-Agent": "SLeeLa-FRUCHST-stager/1.0"}
    )
    with urllib.request.urlopen(request, timeout=60) as response:
        text = response.read().decode("utf-8-sig")

    candidates: list[dict] = []
    seen: set[str] = set()
    for row in csv.DictReader(text.splitlines()):
        word = unicodedata.normalize("NFC", (row.get("lemma") or "").strip())
        if not word or word in seen:
            continue
        try:
            source_rank = int(row["frequency_rank"])
            frequency_count = int(row["frequency_count"])
        except (KeyError, TypeError, ValueError) as exc:
            raise ValueError("upstream row has invalid frequency fields") from exc
        seen.add(word)
        candidates.append(
            {
                "candidate_rank": len(candidates) + 1,
                "source_rank": source_rank,
                "word": word,
                "pos": row.get("pos") or None,
                "frequency_count": frequency_count,
                "cefr_estimate": row.get("cefr_estimate") or None,
            }
        )
        if len(candidates) == limit:
            break
    if len(candidates) != limit:
        raise ValueError(
            f"{language}: expected {limit} unique headwords, got {len(candidates)}"
        )
    return candidates


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("language", choices=sorted(LANGUAGES))
    parser.add_argument("output", type=Path)
    parser.add_argument("--limit", type=int, default=1000)
    args = parser.parse_args()
    if args.limit != 1000:
        parser.error("production staging requires exactly 1,000 candidates")

    try:
        entries = read_candidates(args.language, args.limit)
        document = {
            "schema": "SLeeLa FRUCHST frequency candidate staging v1",
            "language_tag": args.language,
            "language_directory": LANGUAGES[args.language],
            "source": {
                "dataset": "wordhoard",
                "upstream": UPSTREAM,
                "revision": REVISION,
                "sample": f"samples/{args.language}.csv",
                "data_license": LICENSE,
                "frequency_source": (
                    "OpenSubtitles-2018 via hermitdave/FrequencyWords"
                ),
                "selection": (
                    "First 1,000 unique NFC-normalized lemmas in source "
                    "frequency_rank order; duplicate lemmas across POS rows "
                    "collapse to their first/highest-frequency row."
                ),
                "attribution": [
                    "wordhoard contributors — https://github.com/natema/wordhoard",
                    "OpenSubtitles-2018 via hermitdave/FrequencyWords",
                    "spaCy — https://spacy.io",
                    "Wiktionary contributors via Kaikki/Wiktextract",
                ],
                "license_notice": (
                    "Dataset CC BY-SA 4.0; preserve attribution and "
                    "share-alike terms. Upstream code is MIT, not the data."
                ),
                "limitations": [
                    "Subtitle-domain frequency list, not a balanced corpus.",
                    "Raw occurrence counts are not comparable across languages.",
                    "This file stages headwords only; definitions remain pending.",
                ],
            },
            "candidate_count": len(entries),
            "definitions_status": "pending",
            "entries": entries,
        }
        args.output.parent.mkdir(parents=True, exist_ok=True)
        args.output.write_text(
            json.dumps(document, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        print(
            f"staged {len(entries)} unique {args.language} headwords "
            f"from wordhoard {REVISION}"
        )
    except (OSError, ValueError, urllib.error.URLError) as exc:
        print(f"stage_wordhoard: error: {exc}", file=sys.stderr)
        return 2
    return 0


if __name__ == "__main__":
    raise SystemExit(main())