SLLlmVocabulary.sleela
lib/large-language-models/SLLlmVocabulary.sleela · 110 line(s) · sleela
<< lib/large-language-models/SLLlmVocabulary.sleela
SLeeLa Standard Library Definition.
Definition: SLeeLa mirror of the token-ID vocabulary store (sllmv_ in
large_language_models_vocabulary.h). Token ids are stable insertion-order
indices; tokens are exact, non-empty UTF-8. encode() splits on whitespace and
preserves punctuation as part of each token. >>
#sleela 3.0
material;
class SLLlmVocabulary {
< ---- Descriptive metadata (version 1.11 nested-friendly accessors) ---- >
<< Eleven catalogued attributes with getter/setter pairs, so a GUI, loader,
or catalog can read and (where appropriate) override this record.
Values are faithful to the subject's real origin where publicly known. >>
String metaVersion = "1.0.0";
int metaId = 8204;
String metaAuthor = "Max Rupplin";
String metaDate = "2026-11-06";
String metaCopyright = "MEARVK LLC";
String metaTrademark = "none";
String metaCorporation = "MEARVK LLC";
String metaOrigin = "International";
String metaBirthDate = "2026-11-06";
int metaScienceRating = 6;
String metaNationalSchedule = "none";
String getMetaVersion() { return metaVersion; }
void setMetaVersion(String v) { metaVersion = v; }
int getMetaId() { return metaId; }
void setMetaId(int v) { metaId = v; }
String getAuthor() { return metaAuthor; }
void setAuthor(String v) { metaAuthor = v; }
String getDate() { return metaDate; }
void setDate(String v) { metaDate = v; }
String getCopyright() { return metaCopyright; }
void setCopyright(String v) { metaCopyright = v; }
String getTrademark() { return metaTrademark; }
void setTrademark(String v) { metaTrademark = v; }
String getCorporation() { return metaCorporation; }
void setCorporation(String v) { metaCorporation = v; }
String getOrigin() { return metaOrigin; }
void setOrigin(String v) { metaOrigin = v; }
String getBirthDate() { return metaBirthDate; }
void setBirthDate(String v) { metaBirthDate = v; }
int getScienceRating() { return metaScienceRating; }
void setScienceRating(int v) { metaScienceRating = v; }
String getNationalSchedule() { return metaNationalSchedule; }
void setNationalSchedule(String v) { metaNationalSchedule = v; }
< sllmv_status mirror. >
static final int STATUS_OK = 0;
static final int STATUS_INVALID_ARGUMENT = 1;
static final int STATUS_OUT_OF_MEMORY = 2;
static final int STATUS_NOT_FOUND = 3;
static final int STATUS_BUFFER_TOO_SMALL = 4;
< The token table. Parallel arrays keep this portable across SLeeLa targets. >
String[] tokens;
int count;
int capacity;
void configure(int cap) {
if (cap < 16) { cap = 16; }
tokens = new String[cap];
capacity = cap;
count = 0;
}
int size() { return count; }
< sllmv_vocabulary_add: append a token, return its stable id (= prior count). >
int add(String token) {
if (token == null || token == "") { return -1; }
if (count >= capacity) { grow(); }
tokens[count] = token;
int id = count;
count = count + 1;
return id;
}
void grow() {
int newCap = capacity * 2;
String[] bigger = new String[newCap];
int i = 0;
while (i < count) {
bigger[i] = tokens[i];
i = i + 1;
}
tokens = bigger;
capacity = newCap;
}
< sllmv_vocabulary_find: return the id for a token, or -1 if absent. >
int find(String token) {
int i = 0;
while (i < count) {
if (tokens[i] == token) { return i; }
i = i + 1;
}
return -1;
}
< sllmv_vocabulary_token: return the token for an id, or null if out of range. >
String tokenAt(int id) {
if (id < 0 || id >= count) { return null; }
return tokens[id];
}
boolean available() { return tokens != null; }
}