Home /Modules/artificial-intelligence/validate_vocabulary.py

validate_vocabulary.py

artificial-intelligence/large-language-models/tools/validate_vocabulary.py · 424 line(s) · py

#!/usr/bin/env python3
"""
validate_vocabulary.py โ€” the careful-word-set gate for the SLeeLa LLM subsystem.

Validates, fail-closed, with exact locations:

  BASE VOCABULARY (tokenizers/base-vocabulary.json)
    * matches base-vocabulary.schema.json (jsonschema if installed, else a
      built-in structural subset check);
    * IDs are a gap-free 0..N-1 sequence, specials occupy the lowest IDs and
      precede all ordinary tokens, each id equals its array position;
    * token strings are unique and non-empty.

  LEXICAL CORPUS (vocabulary/corpus/*.jsonl, vocabulary/samples/*.jsonl)
    * every line is UTF-8 JSON matching lexical-corpus.schema.json (samples use
      the older sample shape and are checked structurally);
    * LICENSING DISCIPLINE:
        - record_status 'reference_only'  => senses MUST be empty;
        - record_status 'definition_imported' => must carry source.license and
          attribution, and the license must NOT look proprietary;
        - record_status 'sleela_authored' => source.license must be an open
          SLeeLa license (CC0/CC-BY) and must not claim a third-party source;
    * record_id values are unique within a file.

Exit 0 = all good; non-zero = at least one failure (each printed with location).
Stdlib only; uses jsonschema automatically when present.
"""
import json, re, sys
from pathlib import Path

try:
    import jsonschema  # type: ignore
    HAVE_JSONSCHEMA = True
except Exception:
    HAVE_JSONSCHEMA = False

PROPRIETARY = re.compile(r"propriet|all rights reserved|commercial-use|redistribution .*require", re.I)
OPEN_SELF = re.compile(r"^(cc0|cc-?by|public domain)", re.I)

errors = []


def err(where, msg):
    errors.append(f"{where}: {msg}")


def load_json(path):
    return json.loads(Path(path).read_text(encoding="utf-8"))


def schema_check(instance, schema, where):
    if HAVE_JSONSCHEMA:
        validator = jsonschema.Draft7Validator(schema)
        for e in sorted(validator.iter_errors(instance), key=lambda e: e.path):
            loc = "/".join(str(p) for p in e.path) or "(root)"
            err(where, f"schema: {loc}: {e.message}")
        return
    # Minimal built-in fallback: required keys + type of top level only.
    req = schema.get("required", [])
    if isinstance(instance, dict):
        for key in req:
            if key not in instance:
                err(where, f"schema(subset): missing required key '{key}'")


def validate_base(base_path, schema_dir):
    where = str(base_path)
    try:
        data = load_json(base_path)
    except (OSError, json.JSONDecodeError) as e:
        err(where, f"cannot read/parse: {e}")
        return
    schema_p = schema_dir / "base-vocabulary.schema.json"
    if schema_p.exists():
        schema_check(data, load_json(schema_p), where)

    specials = data.get("specials", [])
    tokens = data.get("tokens", [])
    seen = {}
    expected = 0
    for i, s in enumerate(specials):
        if s.get("id") != expected:
            err(where, f"specials[{i}] token '{s.get('token')}' has id {s.get('id')}, expected {expected}")
        tok = s.get("token", "")
        if not tok:
            err(where, f"specials[{i}] has empty token")
        if tok in seen:
            err(where, f"duplicate token '{tok}' (specials[{i}] and index {seen[tok]})")
        seen[tok] = expected
        expected += 1
    for i, t in enumerate(tokens):
        if t.get("id") != expected:
            err(where, f"tokens[{i}] '{t.get('token')}' has id {t.get('id')}, expected {expected} (ids must be gap-free and specials must precede tokens)")
        tok = t.get("token", "")
        if not tok:
            err(where, f"tokens[{i}] has empty token")
        if tok in seen:
            err(where, f"duplicate token '{tok}' (tokens[{i}] and index {seen[tok]})")
        seen[tok] = expected
        expected += 1
    # abi_version, if present, is a positive integer
    if "abi_version" in data and (not isinstance(data["abi_version"], int) or data["abi_version"] < 1):
        err(where, "abi_version must be an integer >= 1")


def validate_corpus_record(where, rec, ids_seen, strict_schema=None, schema_dir=None):
    rid = rec.get("record_id", "(no record_id)")
    if strict_schema is not None:
        schema_check(rec, strict_schema, f"{where} [{rid}]")

    if rid in ids_seen:
        err(where, f"duplicate record_id '{rid}'")
    ids_seen.add(rid)

    status = rec.get("record_status")
    senses = rec.get("senses", []) or []
    src = rec.get("source", {}) or {}
    lic = src.get("license", "") or ""

    if status == "reference_only":
        if senses:
            err(where, f"[{rid}] reference_only record must have EMPTY senses (found {len(senses)}); proprietary content must not be stored")
        integ = rec.get("integration", {}) or {}
        if integ.get("definition_stored") is True:
            err(where, f"[{rid}] reference_only record declares definition_stored=true")
    elif status == "definition_imported":
        if not lic:
            err(where, f"[{rid}] definition_imported record must declare source.license")
        if PROPRIETARY.search(lic):
            err(where, f"[{rid}] definition_imported but license looks proprietary: '{lic}' โ€” use reference_only")
        if not rec.get("attribution"):
            err(where, f"[{rid}] definition_imported record must carry attribution")
        if not senses:
            err(where, f"[{rid}] definition_imported record has no senses")
    elif status == "sleela_authored":
        if not OPEN_SELF.match(lic.strip()):
            err(where, f"[{rid}] sleela_authored record must use an open license (CC0/CC-BY), got '{lic}'")
        name = (src.get("name") or "").lower()
        if any(k in name for k in ("oxford", "merriam", "wordnet", "princeton")):
            err(where, f"[{rid}] sleela_authored record must not claim a third-party source ('{src.get('name')}')")
        if not senses:
            err(where, f"[{rid}] sleela_authored record has no senses")
    else:
        err(where, f"[{rid}] unknown or missing record_status: {status!r}")


def validate_jsonl(path, schema_path, schema_dir):
    where = str(path)
    schema = load_json(schema_path) if (schema_path and schema_path.exists()) else None
    ids_seen = set()
    try:
        with open(path, encoding="utf-8") as fh:
            lines = fh.readlines()
    except OSError as e:
        err(where, f"cannot read: {e}")
        return
    any_record = False
    for lineno, line in enumerate(lines, 1):
        line = line.strip()
        if not line:
            continue
        any_record = True
        try:
            rec = json.loads(line)
        except json.JSONDecodeError as e:
            err(f"{where}:{lineno}", f"invalid JSON: {e}")
            continue
        # Only corpus records carry schema_version 1.0; sample fixtures use the
        # older shape and are checked for the licensing invariants only.
        use_schema = schema if rec.get("schema_version") == "1.0" else None
        validate_corpus_record(f"{where}:{lineno}", rec, ids_seen, use_schema, schema_dir)
    if not any_record:
        err(where, "file has no records")


MIN_BYTES = 2 * 1024**3   # 2 GiB
MAX_BYTES = 16 * 1024**3  # 16 GiB


def tier_for_bytes(b):
    if b == 0:
        return "unknown"
    if b < 3 * 1024**3:
        return "compact"
    if b < 6 * 1024**3:
        return "standard"
    if b < 12 * 1024**3:
        return "capable"
    return "large"


def validate_capacity(path, schema_path):
    """Schema + advisory-envelope + tier_hint agreement for user capacity profiles."""
    where = str(path)
    schema = load_json(schema_path) if (schema_path and schema_path.exists()) else None
    try:
        with open(path, encoding="utf-8") as fh:
            lines = fh.readlines()
    except OSError as e:
        err(where, f"cannot read: {e}")
        return
    ids = set()
    for lineno, line in enumerate(lines, 1):
        line = line.strip()
        if not line:
            continue
        loc = f"{where}:{lineno}"
        try:
            rec = json.loads(line)
        except json.JSONDecodeError as e:
            err(loc, f"invalid JSON: {e}")
            continue
        if schema is not None:
            schema_check(rec, schema, loc)
        mid = rec.get("model_id", "")
        if mid in ids:
            err(loc, f"duplicate model_id '{mid}'")
        ids.add(mid)
        wb = rec.get("weight_bytes", 0)
        # The envelope is advisory: a value outside 2..16 GB is a WARNING, not a hard
        # failure (the user owns the model choice). Record it as an issue only when
        # strictly outside, so examples in-envelope stay clean.
        if isinstance(wb, int) and wb > 0:
            if wb < MIN_BYTES:
                err(loc, f"[{mid}] weight_bytes {wb} below 2 GB envelope (advisory)")
            elif wb > MAX_BYTES:
                err(loc, f"[{mid}] weight_bytes {wb} above 16 GB envelope (advisory)")
            hint = rec.get("tier_hint")
            if hint and hint != tier_for_bytes(wb):
                err(loc, f"[{mid}] tier_hint '{hint}' disagrees with derived tier '{tier_for_bytes(wb)}'")


CODE_LANGS = {"c", "fortran", "pascal", "python", "shell", "sleela", "other"}
CODE_TASKS = {"summarize", "dry-run", "refactor", "translate", "debug", "complete"}
CODE_CHECKS = {"none", "compile", "test", "trace"}


def validate_code_models(path, schema_path):
    """Schema + consistency checks for user-owned Code LLM profiles."""
    where = str(path)
    schema = load_json(schema_path) if (schema_path and schema_path.exists()) else None
    try:
        with open(path, encoding="utf-8") as fh:
            lines = fh.readlines()
    except OSError as e:
        err(where, f"cannot read: {e}")
        return
    ids = set()
    for lineno, line in enumerate(lines, 1):
        line = line.strip()
        if not line:
            continue
        loc = f"{where}:{lineno}"
        try:
            rec = json.loads(line)
        except json.JSONDecodeError as e:
            err(loc, f"invalid JSON: {e}")
            continue
        if schema is not None:
            schema_check(rec, schema, loc)
        mid = rec.get("model_id", "")
        if mid in ids:
            err(loc, f"duplicate model_id '{mid}'")
        ids.add(mid)
        lang = rec.get("primary_language")
        if lang not in CODE_LANGS:
            err(loc, f"[{mid}] invalid primary_language '{lang}'")
        if lang == "other" and not rec.get("other_language"):
            err(loc, f"[{mid}] primary_language 'other' requires other_language")
        tasks = rec.get("tasks", [])
        if not tasks:
            err(loc, f"[{mid}] tasks must be non-empty")
        for t in tasks:
            if t not in CODE_TASKS:
                err(loc, f"[{mid}] invalid task '{t}'")
        cm = rec.get("check_mode")
        if cm is not None and cm not in CODE_CHECKS:
            err(loc, f"[{mid}] invalid check_mode '{cm}'")
        # Honesty: dry-run/debug claim behavior; with check_mode 'none' that behavior
        # is unverifiable. This is allowed, but the note should say so (advisory).
        if cm == "none" and ("dry-run" in tasks or "debug" in tasks):
            note = (rec.get("note") or "").lower()
            if "unverif" not in note and "unverified" not in note:
                err(loc, f"[{mid}] check_mode 'none' with dry-run/debug: note should flag that behavior claims are unverified")


def main(argv=None):
    root = Path(__file__).resolve().parent.parent  # large-language-models/
    schema_dir_tok = root / "tokenizers"
    schema_dir_vocab = root / "vocabulary"

    # Base vocabulary
    base = schema_dir_tok / "base-vocabulary.json"
    if base.exists():
        validate_base(base, schema_dir_tok)
    else:
        err(str(base), "missing base vocabulary")

    corpus_schema = schema_dir_vocab / "lexical-corpus.schema.json"
    # Corpus files
    corpus_dir = schema_dir_vocab / "corpus"
    corpus_files = sorted(corpus_dir.glob("*.jsonl")) if corpus_dir.exists() else []
    for f in corpus_files:
        validate_jsonl(f, corpus_schema, schema_dir_vocab)
    # Sample fixtures (licensing invariants only)
    samples_dir = schema_dir_vocab / "samples"
    for f in sorted(samples_dir.glob("*.jsonl")) if samples_dir.exists() else []:
        validate_jsonl(f, corpus_schema, schema_dir_vocab)

    # Model-capacity profiles (user-owned; advisory 2..16 GB envelope)
    models_dir = root / "models"
    cap_schema = models_dir / "capacity-profile.schema.json"
    cap_files = sorted(models_dir.glob("capacity-profile*.jsonl")) if models_dir.exists() else []
    for f in cap_files:
        validate_capacity(f, cap_schema)

    # Code LLM profiles (user-owned code-specialized models)
    code_schema = models_dir / "code-model.schema.json"
    code_files = sorted(models_dir.glob("code-model*.jsonl")) if models_dir.exists() else []
    for f in code_files:
        validate_code_models(f, code_schema)

    # SLeeLa-specialized Code LLM profile (single JSON object)
    sleela_dir = root / "sleela"
    sleela_profile = sleela_dir / "sleela-language-profile.json"
    sleela_ok = 0
    if sleela_profile.exists():
        sleela_ok = 1
        where = str(sleela_profile)
        try:
            prof = load_json(sleela_profile)
            if prof.get("primary_language") != "sleela":
                err(where, "SLeeLa profile must set primary_language 'sleela'")
            lk = prof.get("language_knowledge", {})
            rw = lk.get("reserved_words", [])
            for must in ("record", "native", "enum", "class", "struct"):
                if must not in rw:
                    err(where, f"reserved_words should list '{must}'")
            if lk.get("limits", {}).get("max_class_or_struct_fields") != 64:
                err(where, "max_class_or_struct_fields must be 64 (SL_MAX_STRUCT_FIELDS)")
            tasks = prof.get("tasks", [])
            for t in tasks:
                if t not in CODE_TASKS:
                    err(where, f"invalid task '{t}'")
        except (OSError, json.JSONDecodeError) as e:
            err(where, f"cannot read/parse: {e}")

    # Parametry: known-models registry + binding examples
    parametry_dir = root / "parametry"
    parametry_ok = 0
    known_names = {}
    if parametry_dir.exists():
        km = parametry_dir / "known-models.json"
        if km.exists():
            parametry_ok = 1
            where = str(km)
            try:
                reg = load_json(km)
                for i, m in enumerate(reg.get("models", [])):
                    nm = m.get("name", "")
                    kind = m.get("kind")
                    if not nm:
                        err(where, f"models[{i}] missing name")
                    if kind not in ("llm", "neural-net"):
                        err(where, f"models[{i}] '{nm}' invalid kind '{kind}'")
                    known_names[nm] = kind
                    if kind == "llm":
                        wb = m.get("weight_bytes", 0)
                        if not (MIN_BYTES <= wb <= MAX_BYTES):
                            err(where, f"models[{i}] '{nm}' llm weight_bytes {wb} outside 2..16 GB envelope")
                    elif kind == "neural-net":
                        ls = m.get("layer_sizes", [])
                        if len(ls) < 2:
                            err(where, f"models[{i}] '{nm}' neural-net needs >=2 layer_sizes")
            except (OSError, json.JSONDecodeError) as e:
                err(where, f"cannot read/parse: {e}")
        bind_schema = parametry_dir / "binding.schema.json"
        for bf in sorted(parametry_dir.glob("bindings*.jsonl")):
            where = str(bf)
            schema = load_json(bind_schema) if bind_schema.exists() else None
            try:
                blines = open(bf, encoding="utf-8").read().splitlines()
            except OSError as e:
                err(where, f"cannot read: {e}"); continue
            for lineno, line in enumerate(blines, 1):
                line = line.strip()
                if not line:
                    continue
                loc = f"{where}:{lineno}"
                try:
                    rec = json.loads(line)
                except json.JSONDecodeError as e:
                    err(loc, f"invalid JSON: {e}"); continue
                if schema is not None:
                    schema_check(rec, schema, loc)
                nm = rec.get("name")
                path = rec.get("local_path")
                if not nm and not path:
                    err(loc, "binding needs a name or a local_path")
                # Known name must exist in the registry and match kind.
                if nm and known_names:
                    if nm not in known_names:
                        err(loc, f"name '{nm}' is not in known-models.json")
                    elif known_names[nm] != rec.get("kind"):
                        err(loc, f"name '{nm}' kind '{known_names[nm]}' != binding kind '{rec.get('kind')}'")
                # A LOCAL neural net must carry a shape.
                if rec.get("kind") == "neural-net" and not nm and not rec.get("layer_sizes"):
                    err(loc, "local neural-net binding requires layer_sizes")

    if errors:
        print(f"validate: FAILED with {len(errors)} issue(s):", file=sys.stderr)
        for e in errors:
            print(f"  - {e}", file=sys.stderr)
        return 1
    scope = (f"base + {len(corpus_files)} corpus file(s) + {len(cap_files)} capacity + "
             f"{len(code_files)} code-model + {sleela_ok} SLeeLa + {parametry_ok} parametry")
    print(f"validate: OK โ€” {scope} pass schema, ID, licensing, capacity, code, SLeeLa, and parametry checks"
          + ("" if HAVE_JSONSCHEMA else " (built-in schema subset; install jsonschema for full checks)"))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())