Home /Modules/languages/fruchst.py

fruchst.py

languages/tools/fruchst.py · 287 line(s) · py

#!/usr/bin/env python3
"""Build and validate FRUCHST v1 binary lexical dictionaries."""

from __future__ import annotations

import argparse
import hashlib
import json
import math
import re
import struct
import os
import sys
import tempfile
import unicodedata
from pathlib import Path

MAGIC = b"FRUCHST\0"
VERSION_MAJOR = 1
VERSION_MINOR = 0
HEADER = struct.Struct(">8sHHHHIHHQQ32s")
ENTRY_PREFIX = struct.Struct(">HdI")
U32 = struct.Struct(">I")
EXPECTED_ENTRIES = 1000
LANG_TAG = re.compile(r"^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$")


class FruchstError(ValueError):
    pass


def nfc_text(value: object, field: str) -> bytes:
    if not isinstance(value, str):
        raise FruchstError(f"{field} must be a string")
    normalized = unicodedata.normalize("NFC", value)
    if not normalized or normalized != normalized.strip():
        raise FruchstError(f"{field} must be non-empty and have no outer whitespace")
    try:
        return normalized.encode("utf-8", errors="strict")
    except UnicodeEncodeError as exc:
        raise FruchstError(f"{field} is not valid Unicode") from exc


def pack_string(value: object, field: str) -> bytes:
    encoded = nfc_text(value, field)
    return U32.pack(len(encoded)) + encoded


def load_input(path: Path) -> dict:
    try:
        data = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as exc:
        raise FruchstError(f"cannot read input JSON: {exc}") from exc
    if not isinstance(data, dict):
        raise FruchstError("input root must be an object")
    return data


def build(data: dict) -> bytes:
    language_tag = data.get("language_tag")
    source_id = data.get("source_id")
    if not isinstance(language_tag, str) or not LANG_TAG.fullmatch(language_tag):
        raise FruchstError("language_tag must be a valid-looking BCP 47 tag")
    lang_bytes = nfc_text(language_tag, "language_tag")
    source_bytes = nfc_text(source_id, "source_id")
    if len(lang_bytes) > 65535 or len(source_bytes) > 65535:
        raise FruchstError("language_tag or source_id is too long")

    entries = data.get("entries")
    if not isinstance(entries, list) or len(entries) != EXPECTED_ENTRIES:
        raise FruchstError(f"production input must contain exactly {EXPECTED_ENTRIES} entries")

    payload = bytearray(lang_bytes + source_bytes)
    seen = set()
    for expected_rank, entry in enumerate(entries, 1):
        if not isinstance(entry, dict):
            raise FruchstError(f"entry {expected_rank}: entry must be an object")
        rank = entry.get("rank")
        if isinstance(rank, bool) or not isinstance(rank, int) or rank != expected_rank:
            raise FruchstError(f"entry {expected_rank}: ranks must be contiguous integers in order")
        word_value = entry.get("word")
        word_bytes = nfc_text(word_value, f"entry {expected_rank} word")
        word = word_bytes.decode("utf-8")
        if word in seen:
            raise FruchstError(f"entry {expected_rank}: duplicate normalized headword {word!r}")
        seen.add(word)
        frequency = entry.get("frequency", 0.0)
        if isinstance(frequency, bool) or not isinstance(frequency, (int, float)):
            raise FruchstError(f"entry {expected_rank}: frequency must be numeric")
        frequency = float(frequency)
        if not math.isfinite(frequency) or frequency < 0:
            raise FruchstError(f"entry {expected_rank}: frequency must be finite and non-negative")
        definitions = entry.get("definitions")
        if not isinstance(definitions, dict):
            raise FruchstError(f"entry {expected_rank}: definitions object is required")
        payload.extend(ENTRY_PREFIX.pack(expected_rank, frequency, len(word_bytes)))
        payload.extend(word_bytes)
        for code in ("en", "fr", "es"):
            payload.extend(pack_string(definitions.get(code), f"entry {expected_rank} definition {code}"))

    digest = hashlib.sha256(payload).digest()
    header = HEADER.pack(
        MAGIC, VERSION_MAJOR, VERSION_MINOR, HEADER.size, 1,
        EXPECTED_ENTRIES, len(lang_bytes), len(source_bytes),
        0, len(payload), digest
    )
    return header + payload


def read_u32(data: bytes, offset: int) -> tuple[int, int]:
    if offset + 4 > len(data):
        raise FruchstError("truncated string length")
    return U32.unpack_from(data, offset)[0], offset + 4


def read_string(data: bytes, offset: int, length: int, field: str) -> tuple[str, int]:
    end = offset + length
    if end > len(data):
        raise FruchstError(f"truncated {field}")
    try:
        value = data[offset:end].decode("utf-8", errors="strict")
    except UnicodeDecodeError as exc:
        raise FruchstError(f"{field} is not valid UTF-8") from exc
    if not value or unicodedata.normalize("NFC", value) != value:
        raise FruchstError(f"{field} is empty or not NFC-normalized")
    return value, end


def validate(blob: bytes) -> dict:
    if len(blob) < HEADER.size:
        raise FruchstError("file is shorter than the fixed header")
    (magic, major, minor, header_size, flags, count, lang_len, source_len,
     generated_epoch, payload_len, digest) = HEADER.unpack_from(blob)
    if magic != MAGIC:
        raise FruchstError("bad FRUCHST magic")
    if major != VERSION_MAJOR or minor > VERSION_MINOR:
        raise FruchstError(f"unsupported FRUCHST version {major}.{minor}")
    if header_size != HEADER.size or flags & ~1:
        raise FruchstError("invalid header size or unknown required flags")
    if count != EXPECTED_ENTRIES:
        raise FruchstError(f"expected {EXPECTED_ENTRIES} entries, found {count}")
    payload = blob[HEADER.size:]
    if len(payload) != payload_len:
        raise FruchstError("payload length mismatch")
    if hashlib.sha256(payload).digest() != digest:
        raise FruchstError("payload SHA-256 mismatch")

    offset = 0
    language_tag, offset = read_string(payload, offset, lang_len, "language tag")
    source_id, offset = read_string(payload, offset, source_len, "source id")
    if not LANG_TAG.fullmatch(language_tag):
        raise FruchstError("invalid BCP 47 language tag")

    seen = set()
    for rank in range(1, count + 1):
        if offset + ENTRY_PREFIX.size > len(payload):
            raise FruchstError("truncated entry prefix")
        actual_rank, frequency, word_len = ENTRY_PREFIX.unpack_from(payload, offset)
        offset += ENTRY_PREFIX.size
        if actual_rank != rank:
            raise FruchstError(f"rank sequence error at entry {rank}")
        if not math.isfinite(frequency) or frequency < 0:
            raise FruchstError(f"invalid frequency at rank {rank}")
        word, offset = read_string(payload, offset, word_len, f"headword {rank}")
        if word in seen:
            raise FruchstError(f"duplicate headword at rank {rank}")
        seen.add(word)
        for code in ("en", "fr", "es"):
            length, offset = read_u32(payload, offset)
            _, offset = read_string(payload, offset, length, f"{code} definition at rank {rank}")
    if offset != len(payload):
        raise FruchstError("unexpected trailing bytes")
    return {
        "format": "FRUCHST",
        "version": f"{major}.{minor}",
        "language_tag": language_tag,
        "source_id": source_id,
        "entry_count": count,
        "payload_sha256": digest.hex(),
        "file_sha256": hashlib.sha256(blob).hexdigest(),
        "generated_epoch": generated_epoch,
        "valid": True,
    }



def validate_production_input(data: dict) -> None:
    provenance = data.get("provenance")
    if not isinstance(provenance, dict):
        raise FruchstError("production provenance object is required")
    for field in ("frequency_source", "attribution", "license_review"):
        if not provenance.get(field):
            raise FruchstError(f"provenance.{field} is required")
    sources = provenance.get("definition_sources")
    if not isinstance(sources, dict):
        raise FruchstError("provenance.definition_sources is required")
    for code in ("en", "fr", "es"):
        source = sources.get(code)
        if not isinstance(source, dict):
            raise FruchstError(f"provenance.definition_sources.{code} is required")
        for field in ("url", "dump_revision", "sha256", "license"):
            if not isinstance(source.get(field), str) or not source[field].strip():
                raise FruchstError(f"definition source {code} requires {field}")
        if not re.fullmatch(r"[0-9a-f]{64}", source["sha256"]):
            raise FruchstError(f"definition source {code} has invalid SHA-256")
    entries = data.get("entries")
    if not isinstance(entries, list) or len(entries) != EXPECTED_ENTRIES:
        raise FruchstError(f"production input must contain exactly {EXPECTED_ENTRIES} entries")
    for rank, entry in enumerate(entries, 1):
        if not isinstance(entry, dict) or entry.get("review_status") != "approved":
            raise FruchstError(f"entry {rank}: human approval is required")
        selected = entry.get("selected_sense")
        if not isinstance(selected, dict):
            raise FruchstError(f"entry {rank}: selected_sense evidence is required")
        for field in ("sense_id", "evidence", "reviewer", "approved_at_utc"):
            if not isinstance(selected.get(field), str) or not selected[field].strip():
                raise FruchstError(f"entry {rank}: selected_sense.{field} is required")


def write_atomic(path: Path, content: bytes) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
    try:
        with os.fdopen(fd, "wb") as stream:
            stream.write(content)
            stream.flush()
            os.fsync(stream.fileno())
        os.replace(temporary, path)
    except Exception:
        try:
            os.unlink(temporary)
        except OSError:
            pass
        raise


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest="command", required=True)
    build_parser = sub.add_parser("build")
    build_parser.add_argument("input", type=Path)
    build_parser.add_argument("output", type=Path)
    build_parser.add_argument("--manifest", type=Path, required=True)
    validate_parser = sub.add_parser("validate")
    validate_parser.add_argument("binary", type=Path)
    validate_parser.add_argument("--manifest", type=Path)
    args = parser.parse_args()

    try:
        if args.command == "build":
            data = load_input(args.input)
            validate_production_input(data)
            blob = build(data)
            result = validate(blob)
            manifest_out = {
                "format": "FRUCHST",
                "format_version": "1.0",
                "language_tag": data["language_tag"],
                "source_id": data["source_id"],
                "frequency_unit": data.get("frequency_unit"),
                "entry_count": EXPECTED_ENTRIES,
                "binary_file": args.output.name,
                "binary_sha256": result["file_sha256"],
                "provenance": data["provenance"],
            }
            manifest_bytes = (json.dumps(manifest_out, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
            write_atomic(args.output, blob)
            write_atomic(args.manifest, manifest_bytes)
            print(json.dumps(result, ensure_ascii=False, indent=2))
        else:
            result = validate(args.binary.read_bytes())
            if args.manifest:
                manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
                if manifest.get("format") != "FRUCHST" or manifest.get("binary_sha256") != result["file_sha256"]:
                    raise FruchstError("manifest does not match the validated binary")
                if manifest.get("language_tag") != result["language_tag"] or manifest.get("source_id") != result["source_id"]:
                    raise FruchstError("manifest identity does not match the validated binary")
            print(json.dumps(result, ensure_ascii=False, indent=2))
    except (OSError, FruchstError, KeyError) as exc:
        print(f"fruchst: error: {exc}", file=sys.stderr)
        return 2
    return 0


if __name__ == "__main__":
    raise SystemExit(main())