Home /Modules/artificial-intelligence/base-vocabulary.schema.json

base-vocabulary.schema.json

artificial-intelligence/large-language-models/tokenizers/base-vocabulary.schema.json · 105 line(s) · json

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://sleela.dev/ai/llm/tokenizers/base-vocabulary-1.0.json",
  "title": "SLeeLa base tokenizer vocabulary (version 1.0)",
  "description": "Declarative, provenance-tracked seed vocabulary the LLM tokenizer is initialized with. Token IDs are explicit, stable, insertion-order indices (they must equal the entry's array position). Special tokens occupy the lowest, reserved IDs. This file is the token-ID table only; it is NOT a lexical dictionary and carries no definitions.",
  "type": "object",
  "additionalProperties": false,
  "required": ["schema_version", "vocabulary_id", "tokenizer", "provenance", "specials", "tokens"],
  "properties": {
    "schema_version": { "const": "1.0" },
    "vocabulary_id": {
      "type": "string",
      "pattern": "^[a-z0-9][a-z0-9._-]*$",
      "description": "Stable identifier for this base vocabulary, referenced by a model's tokenizer_id."
    },
    "abi_version": {
      "type": "integer",
      "minimum": 1,
      "description": "Mirrors SLLMV_BASE_ABI_VERSION of the native loader this file targets."
    },
    "tokenizer": {
      "type": "object",
      "additionalProperties": false,
      "required": ["algorithm", "normalization", "unicode"],
      "properties": {
        "algorithm": {
          "type": "string",
          "enum": ["whitespace", "byte-fallback", "bpe", "wordpiece", "unigram"],
          "description": "The tokenizer algorithm this base set is intended for. The bundled native store implements 'whitespace'; other values document intent for an application-supplied tokenizer."
        },
        "normalization": {
          "type": "string",
          "enum": ["none", "nfc", "nfkc", "lowercase", "nfc+lowercase", "nfkc+lowercase"]
        },
        "unicode": {
          "type": "string",
          "description": "Unicode version the token strings were curated against, e.g. '15.1', or 'ascii' for an ASCII-only set."
        }
      }
    },
    "provenance": {
      "type": "object",
      "additionalProperties": false,
      "required": ["author", "date", "origin", "license", "description"],
      "properties": {
        "author": { "type": "string", "minLength": 1 },
        "date": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" },
        "origin": { "type": "string", "minLength": 1 },
        "license": { "type": "string", "minLength": 1 },
        "description": { "type": "string", "minLength": 1 },
        "sources": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": ["name", "license"],
            "properties": {
              "name": { "type": "string", "minLength": 1 },
              "url": { "type": "string" },
              "license": { "type": "string", "minLength": 1 },
              "note": { "type": "string" }
            }
          }
        }
      }
    },
    "specials": {
      "type": "array",
      "description": "Reserved special tokens. Each occupies a fixed low ID equal to its array position; specials always precede ordinary tokens.",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": ["id", "token", "role"],
        "properties": {
          "id": { "type": "integer", "minimum": 0 },
          "token": { "type": "string", "minLength": 1 },
          "role": {
            "type": "string",
            "enum": ["pad", "unk", "bos", "eos", "sep", "mask", "system", "user", "assistant", "tool", "reserved"]
          },
          "note": { "type": "string" }
        }
      }
    },
    "tokens": {
      "type": "array",
      "description": "Ordinary tokens in ID order. Each is a distinct, non-empty UTF-8 string. The first ordinary token's ID is the count of specials.",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": ["id", "token"],
        "properties": {
          "id": { "type": "integer", "minimum": 0 },
          "token": { "type": "string", "minLength": 1 },
          "category": {
            "type": "string",
            "enum": ["whitespace", "digit", "punctuation", "symbol", "function-word", "core-word", "affix", "byte"],
            "description": "Informational grouping; does not affect the token ID."
          }
        }
      }
    }
  }
}