SLLlmBaseVocabulary.sleela
lib/large-language-models/SLLlmBaseVocabulary.sleela · 133 line(s) · sleela
<< lib/large-language-models/SLLlmBaseVocabulary.sleela
SLeeLa Standard Library Definition.
Definition: SLeeLa mirror of the base/seed vocabulary loader (sllmbv_ in
large_language_models_base_vocabulary.h). It populates an empty vocabulary
store with the compiled-in "sleela-base-en-1.0" table: ten special tokens at
fixed low ids (pad=0 .. tool=9), then ordinary tokens from id 10. It builds
only the token-id table and never imports dictionary definitions, and it fails
closed if a token does not land at its declared id. >>
#sleela 3.0
material;
class SLLlmBaseVocabulary {
< ---- Descriptive metadata (version 1.11 nested-friendly accessors) ---- >
<< Eleven catalogued attributes with getter/setter pairs, so a GUI, loader,
or catalog can read and (where appropriate) override this record.
Values are faithful to the subject's real origin where publicly known. >>
String metaVersion = "1.0.0";
int metaId = 8205;
String metaAuthor = "Max Rupplin";
String metaDate = "2026-11-06";
String metaCopyright = "MEARVK LLC";
String metaTrademark = "none";
String metaCorporation = "MEARVK LLC";
String metaOrigin = "International";
String metaBirthDate = "2026-11-06";
int metaScienceRating = 6;
< metaNationalSchedule: the seed table is CC0-1.0 public-domain dedication. >
String metaNationalSchedule = "CC0-1.0";
String getMetaVersion() { return metaVersion; }
void setMetaVersion(String v) { metaVersion = v; }
int getMetaId() { return metaId; }
void setMetaId(int v) { metaId = v; }
String getAuthor() { return metaAuthor; }
void setAuthor(String v) { metaAuthor = v; }
String getDate() { return metaDate; }
void setDate(String v) { metaDate = v; }
String getCopyright() { return metaCopyright; }
void setCopyright(String v) { metaCopyright = v; }
String getTrademark() { return metaTrademark; }
void setTrademark(String v) { metaTrademark = v; }
String getCorporation() { return metaCorporation; }
void setCorporation(String v) { metaCorporation = v; }
String getOrigin() { return metaOrigin; }
void setOrigin(String v) { metaOrigin = v; }
String getBirthDate() { return metaBirthDate; }
void setBirthDate(String v) { metaBirthDate = v; }
int getScienceRating() { return metaScienceRating; }
void setScienceRating(int v) { metaScienceRating = v; }
String getNationalSchedule() { return metaNationalSchedule; }
void setNationalSchedule(String v) { metaNationalSchedule = v; }
< sllmbv_role mirror: the fixed low ids of the special tokens. >
static final int ROLE_PAD = 0;
static final int ROLE_UNK = 1;
static final int ROLE_BOS = 2;
static final int ROLE_EOS = 3;
static final int ROLE_SEP = 4;
static final int ROLE_MASK = 5;
static final int ROLE_SYSTEM = 6;
static final int ROLE_USER = 7;
static final int ROLE_ASSISTANT = 8;
static final int ROLE_TOOL = 9;
static final int ROLE_RESERVED = 10;
static final int ROLE_COUNT = 11;
< sllmbv_status mirror. >
static final int STATUS_OK = 0;
static final int STATUS_INVALID_ARGUMENT = 1;
static final int STATUS_OUT_OF_MEMORY = 2;
static final int STATUS_PARSE_ERROR = 3;
static final int STATUS_ID_MISMATCH = 4;
static final int STATUS_DUPLICATE_TOKEN = 5;
static final int STATUS_ROLE_UNSET = 6;
< SLLMBV_NO_ID sentinel for an undeclared role. >
static final int NO_ID = -1;
String vocabularyId = "sleela-base-en-1.0";
< The ten special tokens, declaration order == role id order. >
String specialTokenFor(int role) {
if (role == ROLE_PAD) { return "<pad>"; }
if (role == ROLE_UNK) { return "<unk>"; }
if (role == ROLE_BOS) { return "<bos>"; }
if (role == ROLE_EOS) { return "<eos>"; }
if (role == ROLE_SEP) { return "<sep>"; }
if (role == ROLE_MASK) { return "<mask>"; }
if (role == ROLE_SYSTEM) { return "<sys>"; }
if (role == ROLE_USER) { return "<user>"; }
if (role == ROLE_ASSISTANT) { return "<asst>"; }
if (role == ROLE_TOOL) { return "<tool>"; }
return null;
}
<< sllmbv_load_default: load the compiled-in seed table into an EMPTY vocab,
verifying each special lands at its declared id (fail closed otherwise). >>
int loadDefault(SLLlmVocabulary vocab) {
if (vocab == null) { return STATUS_INVALID_ARGUMENT; }
if (vocab.size() != 0) { return STATUS_INVALID_ARGUMENT; }
int role = ROLE_PAD;
while (role <= ROLE_TOOL) {
String tok = specialTokenFor(role);
int id = vocab.add(tok);
if (id != role) { return STATUS_ID_MISMATCH; }
role = role + 1;
}
< Ordinary tokens would follow from id 10 (digits, punctuation, core words). >
< The full table lives in tokenizers/base-vocabulary.json in the native tree. >
return STATUS_OK;
}
< sllmbv_special_id: resolve a role to its id, or NO_ID. >
int specialId(int role) {
if (role < ROLE_PAD || role > ROLE_TOOL) { return NO_ID; }
return role;
}
String defaultVocabularyId() { return vocabularyId; }
String statusString(int status) {
if (status == STATUS_OK) { return "ok"; }
if (status == STATUS_INVALID_ARGUMENT) { return "invalid argument"; }
if (status == STATUS_OUT_OF_MEMORY) { return "out of memory"; }
if (status == STATUS_PARSE_ERROR) { return "parse error"; }
if (status == STATUS_ID_MISMATCH) { return "id mismatch"; }
if (status == STATUS_DUPLICATE_TOKEN) { return "duplicate token"; }
if (status == STATUS_ROLE_UNSET) { return "role unset"; }
return "unknown";
}
boolean available() { return vocabularyId != null; }
}