build_vocabulary.py
artificial-intelligence/large-language-models/tools/build_vocabulary.py · 142 line(s) · py
#!/usr/bin/env python3
"""
build_vocabulary.py — corpus -> tokenizer bridge for the SLeeLa LLM subsystem.
Takes the seed base vocabulary (tokenizers/base-vocabulary.json) and a lexical
corpus (vocabulary/corpus/*.jsonl), and produces a built tokenizer vocabulary
plus a build report. It is the ONLY sanctioned path from lexical records to
token IDs, and it enforces the subsystem's standing rules:
* The base vocabulary is laid down FIRST, so special tokens keep their fixed
low IDs and base tokens keep their declared IDs. New corpus-derived tokens
are appended with the next free IDs; a token already present keeps its ID.
* A token ID is NEVER taken from a dictionary headword. Headwords (and their
synonyms/antonyms) are run through the tokenizer algorithm; the resulting
token strings are what get IDs. (For the 'whitespace' algorithm a headword
is lowercased, NFC-normalized, and split on whitespace.)
* A reference_only record is treated as metadata only. Its definition text is
never read into the vocabulary, and the builder fails if asked to import
one as content (--strict).
* Only token STRINGS cross the bridge. Definitions never become tokens.
Stdlib only. Deterministic output (stable ordering).
"""
import argparse, json, sys, unicodedata
from pathlib import Path
ADDABLE = {"definition_imported", "sleela_authored"}
def normalize_ws(text):
"""Whitespace-algorithm normalization: NFC + lowercase, split on whitespace."""
text = unicodedata.normalize("NFC", text).lower()
return [t for t in text.split() if t]
def load_base(path):
data = json.loads(Path(path).read_text(encoding="utf-8"))
order = [] # token strings in ID order
seen = {}
for s in data.get("specials", []):
tok = s["token"]
if tok in seen:
raise ValueError(f"duplicate special token '{tok}' in base set")
seen[tok] = len(order)
order.append(tok)
for t in data.get("tokens", []):
tok = t["token"]
if tok in seen:
raise ValueError(f"duplicate token '{tok}' in base set")
seen[tok] = len(order)
order.append(tok)
return data.get("vocabulary_id", "unknown"), order, seen
def iter_corpus(paths):
for p in paths:
with open(p, encoding="utf-8") as fh:
for lineno, line in enumerate(fh, 1):
line = line.strip()
if not line:
continue
try:
rec = json.loads(line)
except json.JSONDecodeError as e:
raise ValueError(f"{p}:{lineno}: invalid JSON: {e}") from None
yield p, lineno, rec
def main(argv=None):
ap = argparse.ArgumentParser(description="Build a tokenizer vocabulary from the base set + lexical corpus.")
ap.add_argument("--base", required=True, help="tokenizers/base-vocabulary.json")
ap.add_argument("--corpus", nargs="*", default=[], help="vocabulary/corpus/*.jsonl lexical records")
ap.add_argument("--out", help="write built vocabulary (token-per-line, ID = line number) here")
ap.add_argument("--report", help="write a JSON build report here")
ap.add_argument("--strict", action="store_true", help="fail if a reference_only record exposes sense content")
args = ap.parse_args(argv)
try:
vocab_id, order, index = load_base(args.base)
except (OSError, ValueError, json.JSONDecodeError) as e:
print(f"build: base load failed: {e}", file=sys.stderr)
return 2
base_count = len(order)
added, skipped_ref, provenance = [], 0, {}
try:
for p, lineno, rec in iter_corpus(args.corpus):
status = rec.get("record_status")
headword = rec.get("headword", "")
# Rule: reference-only records contribute NO content, and must not carry senses.
if status == "reference_only":
if rec.get("senses"):
msg = f"{p}:{lineno}: reference_only record '{rec.get('record_id')}' carries sense content"
if args.strict:
print(f"build: {msg}", file=sys.stderr)
return 3
skipped_ref += 1
continue
if status not in ADDABLE:
# unknown status: skip defensively (the gate validates status separately)
continue
# Collect candidate strings: the headword and any synonym/antonym surface forms.
surfaces = [headword]
for sense in rec.get("senses", []):
surfaces += sense.get("synonyms", [])
surfaces += sense.get("antonyms", [])
for surface in surfaces:
for tok in normalize_ws(surface):
if tok not in index:
index[tok] = len(order)
order.append(tok)
added.append(tok)
provenance.setdefault(tok, []).append(rec.get("record_id", "?"))
except (OSError, ValueError) as e:
print(f"build: corpus read failed: {e}", file=sys.stderr)
return 2
if args.out:
Path(args.out).write_text("\n".join(order) + "\n", encoding="utf-8")
report = {
"base_vocabulary_id": vocab_id,
"base_token_count": base_count,
"corpus_files": [str(x) for x in args.corpus],
"tokens_added": len(added),
"reference_only_skipped": skipped_ref,
"final_token_count": len(order),
"added_tokens": sorted(set(added)),
}
if args.report:
Path(args.report).write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(f"build: base '{vocab_id}' {base_count} tokens; "
f"+{len(added)} from corpus; {skipped_ref} reference-only skipped; "
f"{len(order)} total.")
return 0
if __name__ == "__main__":
raise SystemExit(main())