fruchst.py
languages/tools/fruchst.py · 287 line(s) · py
#!/usr/bin/env python3
"""Build and validate FRUCHST v1 binary lexical dictionaries."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import re
import struct
import os
import sys
import tempfile
import unicodedata
from pathlib import Path
MAGIC = b"FRUCHST\0"
VERSION_MAJOR = 1
VERSION_MINOR = 0
HEADER = struct.Struct(">8sHHHHIHHQQ32s")
ENTRY_PREFIX = struct.Struct(">HdI")
U32 = struct.Struct(">I")
EXPECTED_ENTRIES = 1000
LANG_TAG = re.compile(r"^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$")
class FruchstError(ValueError):
pass
def nfc_text(value: object, field: str) -> bytes:
if not isinstance(value, str):
raise FruchstError(f"{field} must be a string")
normalized = unicodedata.normalize("NFC", value)
if not normalized or normalized != normalized.strip():
raise FruchstError(f"{field} must be non-empty and have no outer whitespace")
try:
return normalized.encode("utf-8", errors="strict")
except UnicodeEncodeError as exc:
raise FruchstError(f"{field} is not valid Unicode") from exc
def pack_string(value: object, field: str) -> bytes:
encoded = nfc_text(value, field)
return U32.pack(len(encoded)) + encoded
def load_input(path: Path) -> dict:
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise FruchstError(f"cannot read input JSON: {exc}") from exc
if not isinstance(data, dict):
raise FruchstError("input root must be an object")
return data
def build(data: dict) -> bytes:
language_tag = data.get("language_tag")
source_id = data.get("source_id")
if not isinstance(language_tag, str) or not LANG_TAG.fullmatch(language_tag):
raise FruchstError("language_tag must be a valid-looking BCP 47 tag")
lang_bytes = nfc_text(language_tag, "language_tag")
source_bytes = nfc_text(source_id, "source_id")
if len(lang_bytes) > 65535 or len(source_bytes) > 65535:
raise FruchstError("language_tag or source_id is too long")
entries = data.get("entries")
if not isinstance(entries, list) or len(entries) != EXPECTED_ENTRIES:
raise FruchstError(f"production input must contain exactly {EXPECTED_ENTRIES} entries")
payload = bytearray(lang_bytes + source_bytes)
seen = set()
for expected_rank, entry in enumerate(entries, 1):
if not isinstance(entry, dict):
raise FruchstError(f"entry {expected_rank}: entry must be an object")
rank = entry.get("rank")
if isinstance(rank, bool) or not isinstance(rank, int) or rank != expected_rank:
raise FruchstError(f"entry {expected_rank}: ranks must be contiguous integers in order")
word_value = entry.get("word")
word_bytes = nfc_text(word_value, f"entry {expected_rank} word")
word = word_bytes.decode("utf-8")
if word in seen:
raise FruchstError(f"entry {expected_rank}: duplicate normalized headword {word!r}")
seen.add(word)
frequency = entry.get("frequency", 0.0)
if isinstance(frequency, bool) or not isinstance(frequency, (int, float)):
raise FruchstError(f"entry {expected_rank}: frequency must be numeric")
frequency = float(frequency)
if not math.isfinite(frequency) or frequency < 0:
raise FruchstError(f"entry {expected_rank}: frequency must be finite and non-negative")
definitions = entry.get("definitions")
if not isinstance(definitions, dict):
raise FruchstError(f"entry {expected_rank}: definitions object is required")
payload.extend(ENTRY_PREFIX.pack(expected_rank, frequency, len(word_bytes)))
payload.extend(word_bytes)
for code in ("en", "fr", "es"):
payload.extend(pack_string(definitions.get(code), f"entry {expected_rank} definition {code}"))
digest = hashlib.sha256(payload).digest()
header = HEADER.pack(
MAGIC, VERSION_MAJOR, VERSION_MINOR, HEADER.size, 1,
EXPECTED_ENTRIES, len(lang_bytes), len(source_bytes),
0, len(payload), digest
)
return header + payload
def read_u32(data: bytes, offset: int) -> tuple[int, int]:
if offset + 4 > len(data):
raise FruchstError("truncated string length")
return U32.unpack_from(data, offset)[0], offset + 4
def read_string(data: bytes, offset: int, length: int, field: str) -> tuple[str, int]:
end = offset + length
if end > len(data):
raise FruchstError(f"truncated {field}")
try:
value = data[offset:end].decode("utf-8", errors="strict")
except UnicodeDecodeError as exc:
raise FruchstError(f"{field} is not valid UTF-8") from exc
if not value or unicodedata.normalize("NFC", value) != value:
raise FruchstError(f"{field} is empty or not NFC-normalized")
return value, end
def validate(blob: bytes) -> dict:
if len(blob) < HEADER.size:
raise FruchstError("file is shorter than the fixed header")
(magic, major, minor, header_size, flags, count, lang_len, source_len,
generated_epoch, payload_len, digest) = HEADER.unpack_from(blob)
if magic != MAGIC:
raise FruchstError("bad FRUCHST magic")
if major != VERSION_MAJOR or minor > VERSION_MINOR:
raise FruchstError(f"unsupported FRUCHST version {major}.{minor}")
if header_size != HEADER.size or flags & ~1:
raise FruchstError("invalid header size or unknown required flags")
if count != EXPECTED_ENTRIES:
raise FruchstError(f"expected {EXPECTED_ENTRIES} entries, found {count}")
payload = blob[HEADER.size:]
if len(payload) != payload_len:
raise FruchstError("payload length mismatch")
if hashlib.sha256(payload).digest() != digest:
raise FruchstError("payload SHA-256 mismatch")
offset = 0
language_tag, offset = read_string(payload, offset, lang_len, "language tag")
source_id, offset = read_string(payload, offset, source_len, "source id")
if not LANG_TAG.fullmatch(language_tag):
raise FruchstError("invalid BCP 47 language tag")
seen = set()
for rank in range(1, count + 1):
if offset + ENTRY_PREFIX.size > len(payload):
raise FruchstError("truncated entry prefix")
actual_rank, frequency, word_len = ENTRY_PREFIX.unpack_from(payload, offset)
offset += ENTRY_PREFIX.size
if actual_rank != rank:
raise FruchstError(f"rank sequence error at entry {rank}")
if not math.isfinite(frequency) or frequency < 0:
raise FruchstError(f"invalid frequency at rank {rank}")
word, offset = read_string(payload, offset, word_len, f"headword {rank}")
if word in seen:
raise FruchstError(f"duplicate headword at rank {rank}")
seen.add(word)
for code in ("en", "fr", "es"):
length, offset = read_u32(payload, offset)
_, offset = read_string(payload, offset, length, f"{code} definition at rank {rank}")
if offset != len(payload):
raise FruchstError("unexpected trailing bytes")
return {
"format": "FRUCHST",
"version": f"{major}.{minor}",
"language_tag": language_tag,
"source_id": source_id,
"entry_count": count,
"payload_sha256": digest.hex(),
"file_sha256": hashlib.sha256(blob).hexdigest(),
"generated_epoch": generated_epoch,
"valid": True,
}
def validate_production_input(data: dict) -> None:
provenance = data.get("provenance")
if not isinstance(provenance, dict):
raise FruchstError("production provenance object is required")
for field in ("frequency_source", "attribution", "license_review"):
if not provenance.get(field):
raise FruchstError(f"provenance.{field} is required")
sources = provenance.get("definition_sources")
if not isinstance(sources, dict):
raise FruchstError("provenance.definition_sources is required")
for code in ("en", "fr", "es"):
source = sources.get(code)
if not isinstance(source, dict):
raise FruchstError(f"provenance.definition_sources.{code} is required")
for field in ("url", "dump_revision", "sha256", "license"):
if not isinstance(source.get(field), str) or not source[field].strip():
raise FruchstError(f"definition source {code} requires {field}")
if not re.fullmatch(r"[0-9a-f]{64}", source["sha256"]):
raise FruchstError(f"definition source {code} has invalid SHA-256")
entries = data.get("entries")
if not isinstance(entries, list) or len(entries) != EXPECTED_ENTRIES:
raise FruchstError(f"production input must contain exactly {EXPECTED_ENTRIES} entries")
for rank, entry in enumerate(entries, 1):
if not isinstance(entry, dict) or entry.get("review_status") != "approved":
raise FruchstError(f"entry {rank}: human approval is required")
selected = entry.get("selected_sense")
if not isinstance(selected, dict):
raise FruchstError(f"entry {rank}: selected_sense evidence is required")
for field in ("sense_id", "evidence", "reviewer", "approved_at_utc"):
if not isinstance(selected.get(field), str) or not selected[field].strip():
raise FruchstError(f"entry {rank}: selected_sense.{field} is required")
def write_atomic(path: Path, content: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
try:
with os.fdopen(fd, "wb") as stream:
stream.write(content)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
except Exception:
try:
os.unlink(temporary)
except OSError:
pass
raise
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="command", required=True)
build_parser = sub.add_parser("build")
build_parser.add_argument("input", type=Path)
build_parser.add_argument("output", type=Path)
build_parser.add_argument("--manifest", type=Path, required=True)
validate_parser = sub.add_parser("validate")
validate_parser.add_argument("binary", type=Path)
validate_parser.add_argument("--manifest", type=Path)
args = parser.parse_args()
try:
if args.command == "build":
data = load_input(args.input)
validate_production_input(data)
blob = build(data)
result = validate(blob)
manifest_out = {
"format": "FRUCHST",
"format_version": "1.0",
"language_tag": data["language_tag"],
"source_id": data["source_id"],
"frequency_unit": data.get("frequency_unit"),
"entry_count": EXPECTED_ENTRIES,
"binary_file": args.output.name,
"binary_sha256": result["file_sha256"],
"provenance": data["provenance"],
}
manifest_bytes = (json.dumps(manifest_out, ensure_ascii=False, indent=2) + "\n").encode("utf-8")
write_atomic(args.output, blob)
write_atomic(args.manifest, manifest_bytes)
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
result = validate(args.binary.read_bytes())
if args.manifest:
manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
if manifest.get("format") != "FRUCHST" or manifest.get("binary_sha256") != result["file_sha256"]:
raise FruchstError("manifest does not match the validated binary")
if manifest.get("language_tag") != result["language_tag"] or manifest.get("source_id") != result["source_id"]:
raise FruchstError("manifest identity does not match the validated binary")
print(json.dumps(result, ensure_ascii=False, indent=2))
except (OSError, FruchstError, KeyError) as exc:
print(f"fruchst: error: {exc}", file=sys.stderr)
return 2
return 0
if __name__ == "__main__":
raise SystemExit(main())