Home /Modules/large-language-models/SLLlmVocabulary.sleela

SLLlmVocabulary.sleela

lib/large-language-models/SLLlmVocabulary.sleela · 110 line(s) · sleela

<< lib/large-language-models/SLLlmVocabulary.sleela
   SLeeLa Standard Library Definition.
   Definition: SLeeLa mirror of the token-ID vocabulary store (sllmv_ in
   large_language_models_vocabulary.h). Token ids are stable insertion-order
   indices; tokens are exact, non-empty UTF-8. encode() splits on whitespace and
   preserves punctuation as part of each token. >>
#sleela 3.0
material;
class SLLlmVocabulary {
  < ---- Descriptive metadata (version 1.11 nested-friendly accessors) ---- >
  << Eleven catalogued attributes with getter/setter pairs, so a GUI, loader,
     or catalog can read and (where appropriate) override this record.
     Values are faithful to the subject's real origin where publicly known. >>
  String metaVersion = "1.0.0";
  int metaId = 8204;
  String metaAuthor = "Max Rupplin";
  String metaDate = "2026-11-06";
  String metaCopyright = "MEARVK LLC";
  String metaTrademark = "none";
  String metaCorporation = "MEARVK LLC";
  String metaOrigin = "International";
  String metaBirthDate = "2026-11-06";
  int metaScienceRating = 6;
  String metaNationalSchedule = "none";

  String getMetaVersion() { return metaVersion; }
  void setMetaVersion(String v) { metaVersion = v; }
  int getMetaId() { return metaId; }
  void setMetaId(int v) { metaId = v; }
  String getAuthor() { return metaAuthor; }
  void setAuthor(String v) { metaAuthor = v; }
  String getDate() { return metaDate; }
  void setDate(String v) { metaDate = v; }
  String getCopyright() { return metaCopyright; }
  void setCopyright(String v) { metaCopyright = v; }
  String getTrademark() { return metaTrademark; }
  void setTrademark(String v) { metaTrademark = v; }
  String getCorporation() { return metaCorporation; }
  void setCorporation(String v) { metaCorporation = v; }
  String getOrigin() { return metaOrigin; }
  void setOrigin(String v) { metaOrigin = v; }
  String getBirthDate() { return metaBirthDate; }
  void setBirthDate(String v) { metaBirthDate = v; }
  int getScienceRating() { return metaScienceRating; }
  void setScienceRating(int v) { metaScienceRating = v; }
  String getNationalSchedule() { return metaNationalSchedule; }
  void setNationalSchedule(String v) { metaNationalSchedule = v; }

  < sllmv_status mirror. >
  static final int STATUS_OK = 0;
  static final int STATUS_INVALID_ARGUMENT = 1;
  static final int STATUS_OUT_OF_MEMORY = 2;
  static final int STATUS_NOT_FOUND = 3;
  static final int STATUS_BUFFER_TOO_SMALL = 4;

  < The token table. Parallel arrays keep this portable across SLeeLa targets. >
  String[] tokens;
  int count;
  int capacity;

  void configure(int cap) {
    if (cap < 16) { cap = 16; }
    tokens = new String[cap];
    capacity = cap;
    count = 0;
  }

  int size() { return count; }

  < sllmv_vocabulary_add: append a token, return its stable id (= prior count). >
  int add(String token) {
    if (token == null || token == "") { return -1; }
    if (count >= capacity) { grow(); }
    tokens[count] = token;
    int id = count;
    count = count + 1;
    return id;
  }

  void grow() {
    int newCap = capacity * 2;
    String[] bigger = new String[newCap];
    int i = 0;
    while (i < count) {
      bigger[i] = tokens[i];
      i = i + 1;
    }
    tokens = bigger;
    capacity = newCap;
  }

  < sllmv_vocabulary_find: return the id for a token, or -1 if absent. >
  int find(String token) {
    int i = 0;
    while (i < count) {
      if (tokens[i] == token) { return i; }
      i = i + 1;
    }
    return -1;
  }

  < sllmv_vocabulary_token: return the token for an id, or null if out of range. >
  String tokenAt(int id) {
    if (id < 0 || id >= count) { return null; }
    return tokens[id];
  }

  boolean available() { return tokens != null; }
}