validate_vocabulary.py
artificial-intelligence/large-language-models/tools/validate_vocabulary.py · 424 line(s) · py
#!/usr/bin/env python3
"""
validate_vocabulary.py โ the careful-word-set gate for the SLeeLa LLM subsystem.
Validates, fail-closed, with exact locations:
BASE VOCABULARY (tokenizers/base-vocabulary.json)
* matches base-vocabulary.schema.json (jsonschema if installed, else a
built-in structural subset check);
* IDs are a gap-free 0..N-1 sequence, specials occupy the lowest IDs and
precede all ordinary tokens, each id equals its array position;
* token strings are unique and non-empty.
LEXICAL CORPUS (vocabulary/corpus/*.jsonl, vocabulary/samples/*.jsonl)
* every line is UTF-8 JSON matching lexical-corpus.schema.json (samples use
the older sample shape and are checked structurally);
* LICENSING DISCIPLINE:
- record_status 'reference_only' => senses MUST be empty;
- record_status 'definition_imported' => must carry source.license and
attribution, and the license must NOT look proprietary;
- record_status 'sleela_authored' => source.license must be an open
SLeeLa license (CC0/CC-BY) and must not claim a third-party source;
* record_id values are unique within a file.
Exit 0 = all good; non-zero = at least one failure (each printed with location).
Stdlib only; uses jsonschema automatically when present.
"""
import json, re, sys
from pathlib import Path
try:
import jsonschema # type: ignore
HAVE_JSONSCHEMA = True
except Exception:
HAVE_JSONSCHEMA = False
PROPRIETARY = re.compile(r"propriet|all rights reserved|commercial-use|redistribution .*require", re.I)
OPEN_SELF = re.compile(r"^(cc0|cc-?by|public domain)", re.I)
errors = []
def err(where, msg):
errors.append(f"{where}: {msg}")
def load_json(path):
return json.loads(Path(path).read_text(encoding="utf-8"))
def schema_check(instance, schema, where):
if HAVE_JSONSCHEMA:
validator = jsonschema.Draft7Validator(schema)
for e in sorted(validator.iter_errors(instance), key=lambda e: e.path):
loc = "/".join(str(p) for p in e.path) or "(root)"
err(where, f"schema: {loc}: {e.message}")
return
# Minimal built-in fallback: required keys + type of top level only.
req = schema.get("required", [])
if isinstance(instance, dict):
for key in req:
if key not in instance:
err(where, f"schema(subset): missing required key '{key}'")
def validate_base(base_path, schema_dir):
where = str(base_path)
try:
data = load_json(base_path)
except (OSError, json.JSONDecodeError) as e:
err(where, f"cannot read/parse: {e}")
return
schema_p = schema_dir / "base-vocabulary.schema.json"
if schema_p.exists():
schema_check(data, load_json(schema_p), where)
specials = data.get("specials", [])
tokens = data.get("tokens", [])
seen = {}
expected = 0
for i, s in enumerate(specials):
if s.get("id") != expected:
err(where, f"specials[{i}] token '{s.get('token')}' has id {s.get('id')}, expected {expected}")
tok = s.get("token", "")
if not tok:
err(where, f"specials[{i}] has empty token")
if tok in seen:
err(where, f"duplicate token '{tok}' (specials[{i}] and index {seen[tok]})")
seen[tok] = expected
expected += 1
for i, t in enumerate(tokens):
if t.get("id") != expected:
err(where, f"tokens[{i}] '{t.get('token')}' has id {t.get('id')}, expected {expected} (ids must be gap-free and specials must precede tokens)")
tok = t.get("token", "")
if not tok:
err(where, f"tokens[{i}] has empty token")
if tok in seen:
err(where, f"duplicate token '{tok}' (tokens[{i}] and index {seen[tok]})")
seen[tok] = expected
expected += 1
# abi_version, if present, is a positive integer
if "abi_version" in data and (not isinstance(data["abi_version"], int) or data["abi_version"] < 1):
err(where, "abi_version must be an integer >= 1")
def validate_corpus_record(where, rec, ids_seen, strict_schema=None, schema_dir=None):
rid = rec.get("record_id", "(no record_id)")
if strict_schema is not None:
schema_check(rec, strict_schema, f"{where} [{rid}]")
if rid in ids_seen:
err(where, f"duplicate record_id '{rid}'")
ids_seen.add(rid)
status = rec.get("record_status")
senses = rec.get("senses", []) or []
src = rec.get("source", {}) or {}
lic = src.get("license", "") or ""
if status == "reference_only":
if senses:
err(where, f"[{rid}] reference_only record must have EMPTY senses (found {len(senses)}); proprietary content must not be stored")
integ = rec.get("integration", {}) or {}
if integ.get("definition_stored") is True:
err(where, f"[{rid}] reference_only record declares definition_stored=true")
elif status == "definition_imported":
if not lic:
err(where, f"[{rid}] definition_imported record must declare source.license")
if PROPRIETARY.search(lic):
err(where, f"[{rid}] definition_imported but license looks proprietary: '{lic}' โ use reference_only")
if not rec.get("attribution"):
err(where, f"[{rid}] definition_imported record must carry attribution")
if not senses:
err(where, f"[{rid}] definition_imported record has no senses")
elif status == "sleela_authored":
if not OPEN_SELF.match(lic.strip()):
err(where, f"[{rid}] sleela_authored record must use an open license (CC0/CC-BY), got '{lic}'")
name = (src.get("name") or "").lower()
if any(k in name for k in ("oxford", "merriam", "wordnet", "princeton")):
err(where, f"[{rid}] sleela_authored record must not claim a third-party source ('{src.get('name')}')")
if not senses:
err(where, f"[{rid}] sleela_authored record has no senses")
else:
err(where, f"[{rid}] unknown or missing record_status: {status!r}")
def validate_jsonl(path, schema_path, schema_dir):
where = str(path)
schema = load_json(schema_path) if (schema_path and schema_path.exists()) else None
ids_seen = set()
try:
with open(path, encoding="utf-8") as fh:
lines = fh.readlines()
except OSError as e:
err(where, f"cannot read: {e}")
return
any_record = False
for lineno, line in enumerate(lines, 1):
line = line.strip()
if not line:
continue
any_record = True
try:
rec = json.loads(line)
except json.JSONDecodeError as e:
err(f"{where}:{lineno}", f"invalid JSON: {e}")
continue
# Only corpus records carry schema_version 1.0; sample fixtures use the
# older shape and are checked for the licensing invariants only.
use_schema = schema if rec.get("schema_version") == "1.0" else None
validate_corpus_record(f"{where}:{lineno}", rec, ids_seen, use_schema, schema_dir)
if not any_record:
err(where, "file has no records")
MIN_BYTES = 2 * 1024**3 # 2 GiB
MAX_BYTES = 16 * 1024**3 # 16 GiB
def tier_for_bytes(b):
if b == 0:
return "unknown"
if b < 3 * 1024**3:
return "compact"
if b < 6 * 1024**3:
return "standard"
if b < 12 * 1024**3:
return "capable"
return "large"
def validate_capacity(path, schema_path):
"""Schema + advisory-envelope + tier_hint agreement for user capacity profiles."""
where = str(path)
schema = load_json(schema_path) if (schema_path and schema_path.exists()) else None
try:
with open(path, encoding="utf-8") as fh:
lines = fh.readlines()
except OSError as e:
err(where, f"cannot read: {e}")
return
ids = set()
for lineno, line in enumerate(lines, 1):
line = line.strip()
if not line:
continue
loc = f"{where}:{lineno}"
try:
rec = json.loads(line)
except json.JSONDecodeError as e:
err(loc, f"invalid JSON: {e}")
continue
if schema is not None:
schema_check(rec, schema, loc)
mid = rec.get("model_id", "")
if mid in ids:
err(loc, f"duplicate model_id '{mid}'")
ids.add(mid)
wb = rec.get("weight_bytes", 0)
# The envelope is advisory: a value outside 2..16 GB is a WARNING, not a hard
# failure (the user owns the model choice). Record it as an issue only when
# strictly outside, so examples in-envelope stay clean.
if isinstance(wb, int) and wb > 0:
if wb < MIN_BYTES:
err(loc, f"[{mid}] weight_bytes {wb} below 2 GB envelope (advisory)")
elif wb > MAX_BYTES:
err(loc, f"[{mid}] weight_bytes {wb} above 16 GB envelope (advisory)")
hint = rec.get("tier_hint")
if hint and hint != tier_for_bytes(wb):
err(loc, f"[{mid}] tier_hint '{hint}' disagrees with derived tier '{tier_for_bytes(wb)}'")
CODE_LANGS = {"c", "fortran", "pascal", "python", "shell", "sleela", "other"}
CODE_TASKS = {"summarize", "dry-run", "refactor", "translate", "debug", "complete"}
CODE_CHECKS = {"none", "compile", "test", "trace"}
def validate_code_models(path, schema_path):
"""Schema + consistency checks for user-owned Code LLM profiles."""
where = str(path)
schema = load_json(schema_path) if (schema_path and schema_path.exists()) else None
try:
with open(path, encoding="utf-8") as fh:
lines = fh.readlines()
except OSError as e:
err(where, f"cannot read: {e}")
return
ids = set()
for lineno, line in enumerate(lines, 1):
line = line.strip()
if not line:
continue
loc = f"{where}:{lineno}"
try:
rec = json.loads(line)
except json.JSONDecodeError as e:
err(loc, f"invalid JSON: {e}")
continue
if schema is not None:
schema_check(rec, schema, loc)
mid = rec.get("model_id", "")
if mid in ids:
err(loc, f"duplicate model_id '{mid}'")
ids.add(mid)
lang = rec.get("primary_language")
if lang not in CODE_LANGS:
err(loc, f"[{mid}] invalid primary_language '{lang}'")
if lang == "other" and not rec.get("other_language"):
err(loc, f"[{mid}] primary_language 'other' requires other_language")
tasks = rec.get("tasks", [])
if not tasks:
err(loc, f"[{mid}] tasks must be non-empty")
for t in tasks:
if t not in CODE_TASKS:
err(loc, f"[{mid}] invalid task '{t}'")
cm = rec.get("check_mode")
if cm is not None and cm not in CODE_CHECKS:
err(loc, f"[{mid}] invalid check_mode '{cm}'")
# Honesty: dry-run/debug claim behavior; with check_mode 'none' that behavior
# is unverifiable. This is allowed, but the note should say so (advisory).
if cm == "none" and ("dry-run" in tasks or "debug" in tasks):
note = (rec.get("note") or "").lower()
if "unverif" not in note and "unverified" not in note:
err(loc, f"[{mid}] check_mode 'none' with dry-run/debug: note should flag that behavior claims are unverified")
def main(argv=None):
root = Path(__file__).resolve().parent.parent # large-language-models/
schema_dir_tok = root / "tokenizers"
schema_dir_vocab = root / "vocabulary"
# Base vocabulary
base = schema_dir_tok / "base-vocabulary.json"
if base.exists():
validate_base(base, schema_dir_tok)
else:
err(str(base), "missing base vocabulary")
corpus_schema = schema_dir_vocab / "lexical-corpus.schema.json"
# Corpus files
corpus_dir = schema_dir_vocab / "corpus"
corpus_files = sorted(corpus_dir.glob("*.jsonl")) if corpus_dir.exists() else []
for f in corpus_files:
validate_jsonl(f, corpus_schema, schema_dir_vocab)
# Sample fixtures (licensing invariants only)
samples_dir = schema_dir_vocab / "samples"
for f in sorted(samples_dir.glob("*.jsonl")) if samples_dir.exists() else []:
validate_jsonl(f, corpus_schema, schema_dir_vocab)
# Model-capacity profiles (user-owned; advisory 2..16 GB envelope)
models_dir = root / "models"
cap_schema = models_dir / "capacity-profile.schema.json"
cap_files = sorted(models_dir.glob("capacity-profile*.jsonl")) if models_dir.exists() else []
for f in cap_files:
validate_capacity(f, cap_schema)
# Code LLM profiles (user-owned code-specialized models)
code_schema = models_dir / "code-model.schema.json"
code_files = sorted(models_dir.glob("code-model*.jsonl")) if models_dir.exists() else []
for f in code_files:
validate_code_models(f, code_schema)
# SLeeLa-specialized Code LLM profile (single JSON object)
sleela_dir = root / "sleela"
sleela_profile = sleela_dir / "sleela-language-profile.json"
sleela_ok = 0
if sleela_profile.exists():
sleela_ok = 1
where = str(sleela_profile)
try:
prof = load_json(sleela_profile)
if prof.get("primary_language") != "sleela":
err(where, "SLeeLa profile must set primary_language 'sleela'")
lk = prof.get("language_knowledge", {})
rw = lk.get("reserved_words", [])
for must in ("record", "native", "enum", "class", "struct"):
if must not in rw:
err(where, f"reserved_words should list '{must}'")
if lk.get("limits", {}).get("max_class_or_struct_fields") != 64:
err(where, "max_class_or_struct_fields must be 64 (SL_MAX_STRUCT_FIELDS)")
tasks = prof.get("tasks", [])
for t in tasks:
if t not in CODE_TASKS:
err(where, f"invalid task '{t}'")
except (OSError, json.JSONDecodeError) as e:
err(where, f"cannot read/parse: {e}")
# Parametry: known-models registry + binding examples
parametry_dir = root / "parametry"
parametry_ok = 0
known_names = {}
if parametry_dir.exists():
km = parametry_dir / "known-models.json"
if km.exists():
parametry_ok = 1
where = str(km)
try:
reg = load_json(km)
for i, m in enumerate(reg.get("models", [])):
nm = m.get("name", "")
kind = m.get("kind")
if not nm:
err(where, f"models[{i}] missing name")
if kind not in ("llm", "neural-net"):
err(where, f"models[{i}] '{nm}' invalid kind '{kind}'")
known_names[nm] = kind
if kind == "llm":
wb = m.get("weight_bytes", 0)
if not (MIN_BYTES <= wb <= MAX_BYTES):
err(where, f"models[{i}] '{nm}' llm weight_bytes {wb} outside 2..16 GB envelope")
elif kind == "neural-net":
ls = m.get("layer_sizes", [])
if len(ls) < 2:
err(where, f"models[{i}] '{nm}' neural-net needs >=2 layer_sizes")
except (OSError, json.JSONDecodeError) as e:
err(where, f"cannot read/parse: {e}")
bind_schema = parametry_dir / "binding.schema.json"
for bf in sorted(parametry_dir.glob("bindings*.jsonl")):
where = str(bf)
schema = load_json(bind_schema) if bind_schema.exists() else None
try:
blines = open(bf, encoding="utf-8").read().splitlines()
except OSError as e:
err(where, f"cannot read: {e}"); continue
for lineno, line in enumerate(blines, 1):
line = line.strip()
if not line:
continue
loc = f"{where}:{lineno}"
try:
rec = json.loads(line)
except json.JSONDecodeError as e:
err(loc, f"invalid JSON: {e}"); continue
if schema is not None:
schema_check(rec, schema, loc)
nm = rec.get("name")
path = rec.get("local_path")
if not nm and not path:
err(loc, "binding needs a name or a local_path")
# Known name must exist in the registry and match kind.
if nm and known_names:
if nm not in known_names:
err(loc, f"name '{nm}' is not in known-models.json")
elif known_names[nm] != rec.get("kind"):
err(loc, f"name '{nm}' kind '{known_names[nm]}' != binding kind '{rec.get('kind')}'")
# A LOCAL neural net must carry a shape.
if rec.get("kind") == "neural-net" and not nm and not rec.get("layer_sizes"):
err(loc, "local neural-net binding requires layer_sizes")
if errors:
print(f"validate: FAILED with {len(errors)} issue(s):", file=sys.stderr)
for e in errors:
print(f" - {e}", file=sys.stderr)
return 1
scope = (f"base + {len(corpus_files)} corpus file(s) + {len(cap_files)} capacity + "
f"{len(code_files)} code-model + {sleela_ok} SLeeLa + {parametry_ok} parametry")
print(f"validate: OK โ {scope} pass schema, ID, licensing, capacity, code, SLeeLa, and parametry checks"
+ ("" if HAVE_JSONSCHEMA else " (built-in schema subset; install jsonschema for full checks)"))
return 0
if __name__ == "__main__":
raise SystemExit(main())