base-vocabulary.json
artificial-intelligence/large-language-models/tokenizers/base-vocabulary.json · 164 line(s) · json
{
"schema_version": "1.0",
"vocabulary_id": "sleela-base-en-1.0",
"abi_version": 1,
"tokenizer": {
"algorithm": "whitespace",
"normalization": "nfc+lowercase",
"unicode": "15.1"
},
"provenance": {
"author": "SLeeLa project",
"date": "2026-11-06",
"origin": "International",
"license": "CC0-1.0",
"description": "A small, SLeeLa-authored seed vocabulary for the whitespace tokenizer primitive: reserved special tokens at fixed low IDs, then digits, common punctuation and symbols, closed-class English function words, and a compact core of open-class words. It is a token-ID table only (no definitions), deliberately original so it carries no third-party licensing obligations, and is meant to be extended by the corpus->tokenizer builder. It is not a production BPE/WordPiece model.",
"sources": [
{
"name": "SLeeLa project (original authorship)",
"license": "CC0-1.0",
"note": "Token strings are ordinary English orthography and a hand-curated closed-class/core list; no third-party vocabulary file was copied."
}
]
},
"specials": [
{ "id": 0, "token": "<pad>", "role": "pad", "note": "padding; must be ID 0 so zero-fill is safe" },
{ "id": 1, "token": "<unk>", "role": "unk", "note": "unknown / out-of-vocabulary fallback" },
{ "id": 2, "token": "<bos>", "role": "bos", "note": "beginning of sequence" },
{ "id": 3, "token": "<eos>", "role": "eos", "note": "end of sequence" },
{ "id": 4, "token": "<sep>", "role": "sep", "note": "segment / turn separator" },
{ "id": 5, "token": "<mask>", "role": "mask", "note": "masked-token objective" },
{ "id": 6, "token": "<sys>", "role": "system", "note": "system-role marker" },
{ "id": 7, "token": "<user>", "role": "user", "note": "user-role marker" },
{ "id": 8, "token": "<asst>", "role": "assistant", "note": "assistant-role marker" },
{ "id": 9, "token": "<tool>", "role": "tool", "note": "tool/function-call marker" }
],
"tokens": [
{ "id": 10, "token": "0", "category": "digit" },
{ "id": 11, "token": "1", "category": "digit" },
{ "id": 12, "token": "2", "category": "digit" },
{ "id": 13, "token": "3", "category": "digit" },
{ "id": 14, "token": "4", "category": "digit" },
{ "id": 15, "token": "5", "category": "digit" },
{ "id": 16, "token": "6", "category": "digit" },
{ "id": 17, "token": "7", "category": "digit" },
{ "id": 18, "token": "8", "category": "digit" },
{ "id": 19, "token": "9", "category": "digit" },
{ "id": 20, "token": ".", "category": "punctuation" },
{ "id": 21, "token": ",", "category": "punctuation" },
{ "id": 22, "token": "?", "category": "punctuation" },
{ "id": 23, "token": "!", "category": "punctuation" },
{ "id": 24, "token": ";", "category": "punctuation" },
{ "id": 25, "token": ":", "category": "punctuation" },
{ "id": 26, "token": "-", "category": "punctuation" },
{ "id": 27, "token": "(", "category": "punctuation" },
{ "id": 28, "token": ")", "category": "punctuation" },
{ "id": 29, "token": "\"", "category": "punctuation" },
{ "id": 30, "token": "'", "category": "punctuation" },
{ "id": 31, "token": "+", "category": "symbol" },
{ "id": 32, "token": "=", "category": "symbol" },
{ "id": 33, "token": "/", "category": "symbol" },
{ "id": 34, "token": "%", "category": "symbol" },
{ "id": 35, "token": "the", "category": "function-word" },
{ "id": 36, "token": "a", "category": "function-word" },
{ "id": 37, "token": "an", "category": "function-word" },
{ "id": 38, "token": "and", "category": "function-word" },
{ "id": 39, "token": "or", "category": "function-word" },
{ "id": 40, "token": "but", "category": "function-word" },
{ "id": 41, "token": "if", "category": "function-word" },
{ "id": 42, "token": "then", "category": "function-word" },
{ "id": 43, "token": "else", "category": "function-word" },
{ "id": 44, "token": "not", "category": "function-word" },
{ "id": 45, "token": "no", "category": "function-word" },
{ "id": 46, "token": "yes", "category": "function-word" },
{ "id": 47, "token": "of", "category": "function-word" },
{ "id": 48, "token": "to", "category": "function-word" },
{ "id": 49, "token": "in", "category": "function-word" },
{ "id": 50, "token": "on", "category": "function-word" },
{ "id": 51, "token": "at", "category": "function-word" },
{ "id": 52, "token": "by", "category": "function-word" },
{ "id": 53, "token": "for", "category": "function-word" },
{ "id": 54, "token": "from", "category": "function-word" },
{ "id": 55, "token": "with", "category": "function-word" },
{ "id": 56, "token": "without", "category": "function-word" },
{ "id": 57, "token": "as", "category": "function-word" },
{ "id": 58, "token": "is", "category": "function-word" },
{ "id": 59, "token": "are", "category": "function-word" },
{ "id": 60, "token": "was", "category": "function-word" },
{ "id": 61, "token": "were", "category": "function-word" },
{ "id": 62, "token": "be", "category": "function-word" },
{ "id": 63, "token": "been", "category": "function-word" },
{ "id": 64, "token": "being", "category": "function-word" },
{ "id": 65, "token": "has", "category": "function-word" },
{ "id": 66, "token": "have", "category": "function-word" },
{ "id": 67, "token": "had", "category": "function-word" },
{ "id": 68, "token": "do", "category": "function-word" },
{ "id": 69, "token": "does", "category": "function-word" },
{ "id": 70, "token": "did", "category": "function-word" },
{ "id": 71, "token": "will", "category": "function-word" },
{ "id": 72, "token": "would", "category": "function-word" },
{ "id": 73, "token": "can", "category": "function-word" },
{ "id": 74, "token": "could", "category": "function-word" },
{ "id": 75, "token": "should", "category": "function-word" },
{ "id": 76, "token": "may", "category": "function-word" },
{ "id": 77, "token": "might", "category": "function-word" },
{ "id": 78, "token": "must", "category": "function-word" },
{ "id": 79, "token": "this", "category": "function-word" },
{ "id": 80, "token": "that", "category": "function-word" },
{ "id": 81, "token": "these", "category": "function-word" },
{ "id": 82, "token": "those", "category": "function-word" },
{ "id": 83, "token": "it", "category": "function-word" },
{ "id": 84, "token": "its", "category": "function-word" },
{ "id": 85, "token": "they", "category": "function-word" },
{ "id": 86, "token": "them", "category": "function-word" },
{ "id": 87, "token": "we", "category": "function-word" },
{ "id": 88, "token": "you", "category": "function-word" },
{ "id": 89, "token": "i", "category": "function-word" },
{ "id": 90, "token": "he", "category": "function-word" },
{ "id": 91, "token": "she", "category": "function-word" },
{ "id": 92, "token": "who", "category": "function-word" },
{ "id": 93, "token": "what", "category": "function-word" },
{ "id": 94, "token": "when", "category": "function-word" },
{ "id": 95, "token": "where", "category": "function-word" },
{ "id": 96, "token": "why", "category": "function-word" },
{ "id": 97, "token": "how", "category": "function-word" },
{ "id": 98, "token": "which", "category": "function-word" },
{ "id": 99, "token": "all", "category": "function-word" },
{ "id": 100, "token": "any", "category": "function-word" },
{ "id": 101, "token": "some", "category": "function-word" },
{ "id": 102, "token": "each", "category": "function-word" },
{ "id": 103, "token": "every", "category": "function-word" },
{ "id": 104, "token": "more", "category": "function-word" },
{ "id": 105, "token": "most", "category": "function-word" },
{ "id": 106, "token": "than", "category": "function-word" },
{ "id": 107, "token": "very", "category": "function-word" },
{ "id": 108, "token": "computer", "category": "core-word" },
{ "id": 109, "token": "machine", "category": "core-word" },
{ "id": 110, "token": "program", "category": "core-word" },
{ "id": 111, "token": "data", "category": "core-word" },
{ "id": 112, "token": "model", "category": "core-word" },
{ "id": 113, "token": "language", "category": "core-word" },
{ "id": 114, "token": "token", "category": "core-word" },
{ "id": 115, "token": "word", "category": "core-word" },
{ "id": 116, "token": "number", "category": "core-word" },
{ "id": 117, "token": "value", "category": "core-word" },
{ "id": 118, "token": "system", "category": "core-word" },
{ "id": 119, "token": "process", "category": "core-word" },
{ "id": 120, "token": "function", "category": "core-word" },
{ "id": 121, "token": "code", "category": "core-word" },
{ "id": 122, "token": "source", "category": "core-word" },
{ "id": 123, "token": "text", "category": "core-word" },
{ "id": 124, "token": "input", "category": "core-word" },
{ "id": 125, "token": "output", "category": "core-word" },
{ "id": 126, "token": "train", "category": "core-word" },
{ "id": 127, "token": "learn", "category": "core-word" },
{ "id": 128, "token": "answer", "category": "core-word" },
{ "id": 129, "token": "question", "category": "core-word" },
{ "id": 130, "token": "example", "category": "core-word" },
{ "id": 131, "token": "true", "category": "core-word" },
{ "id": 132, "token": "false", "category": "core-word" },
{ "id": 133, "token": "please", "category": "core-word" },
{ "id": 134, "token": "thank", "category": "core-word" }
]
}