Home /Modules/large-language-models/SLLlmBaseVocabulary.sleela

SLLlmBaseVocabulary.sleela

lib/large-language-models/SLLlmBaseVocabulary.sleela · 133 line(s) · sleela

<< lib/large-language-models/SLLlmBaseVocabulary.sleela
   SLeeLa Standard Library Definition.
   Definition: SLeeLa mirror of the base/seed vocabulary loader (sllmbv_ in
   large_language_models_base_vocabulary.h). It populates an empty vocabulary
   store with the compiled-in "sleela-base-en-1.0" table: ten special tokens at
   fixed low ids (pad=0 .. tool=9), then ordinary tokens from id 10. It builds
   only the token-id table and never imports dictionary definitions, and it fails
   closed if a token does not land at its declared id. >>
#sleela 3.0
material;
class SLLlmBaseVocabulary {
  < ---- Descriptive metadata (version 1.11 nested-friendly accessors) ---- >
  << Eleven catalogued attributes with getter/setter pairs, so a GUI, loader,
     or catalog can read and (where appropriate) override this record.
     Values are faithful to the subject's real origin where publicly known. >>
  String metaVersion = "1.0.0";
  int metaId = 8205;
  String metaAuthor = "Max Rupplin";
  String metaDate = "2026-11-06";
  String metaCopyright = "MEARVK LLC";
  String metaTrademark = "none";
  String metaCorporation = "MEARVK LLC";
  String metaOrigin = "International";
  String metaBirthDate = "2026-11-06";
  int metaScienceRating = 6;
  < metaNationalSchedule: the seed table is CC0-1.0 public-domain dedication. >
  String metaNationalSchedule = "CC0-1.0";

  String getMetaVersion() { return metaVersion; }
  void setMetaVersion(String v) { metaVersion = v; }
  int getMetaId() { return metaId; }
  void setMetaId(int v) { metaId = v; }
  String getAuthor() { return metaAuthor; }
  void setAuthor(String v) { metaAuthor = v; }
  String getDate() { return metaDate; }
  void setDate(String v) { metaDate = v; }
  String getCopyright() { return metaCopyright; }
  void setCopyright(String v) { metaCopyright = v; }
  String getTrademark() { return metaTrademark; }
  void setTrademark(String v) { metaTrademark = v; }
  String getCorporation() { return metaCorporation; }
  void setCorporation(String v) { metaCorporation = v; }
  String getOrigin() { return metaOrigin; }
  void setOrigin(String v) { metaOrigin = v; }
  String getBirthDate() { return metaBirthDate; }
  void setBirthDate(String v) { metaBirthDate = v; }
  int getScienceRating() { return metaScienceRating; }
  void setScienceRating(int v) { metaScienceRating = v; }
  String getNationalSchedule() { return metaNationalSchedule; }
  void setNationalSchedule(String v) { metaNationalSchedule = v; }

  < sllmbv_role mirror: the fixed low ids of the special tokens. >
  static final int ROLE_PAD = 0;
  static final int ROLE_UNK = 1;
  static final int ROLE_BOS = 2;
  static final int ROLE_EOS = 3;
  static final int ROLE_SEP = 4;
  static final int ROLE_MASK = 5;
  static final int ROLE_SYSTEM = 6;
  static final int ROLE_USER = 7;
  static final int ROLE_ASSISTANT = 8;
  static final int ROLE_TOOL = 9;
  static final int ROLE_RESERVED = 10;
  static final int ROLE_COUNT = 11;

  < sllmbv_status mirror. >
  static final int STATUS_OK = 0;
  static final int STATUS_INVALID_ARGUMENT = 1;
  static final int STATUS_OUT_OF_MEMORY = 2;
  static final int STATUS_PARSE_ERROR = 3;
  static final int STATUS_ID_MISMATCH = 4;
  static final int STATUS_DUPLICATE_TOKEN = 5;
  static final int STATUS_ROLE_UNSET = 6;

  < SLLMBV_NO_ID sentinel for an undeclared role. >
  static final int NO_ID = -1;

  String vocabularyId = "sleela-base-en-1.0";

  < The ten special tokens, declaration order == role id order. >
  String specialTokenFor(int role) {
    if (role == ROLE_PAD) { return "<pad>"; }
    if (role == ROLE_UNK) { return "<unk>"; }
    if (role == ROLE_BOS) { return "<bos>"; }
    if (role == ROLE_EOS) { return "<eos>"; }
    if (role == ROLE_SEP) { return "<sep>"; }
    if (role == ROLE_MASK) { return "<mask>"; }
    if (role == ROLE_SYSTEM) { return "<sys>"; }
    if (role == ROLE_USER) { return "<user>"; }
    if (role == ROLE_ASSISTANT) { return "<asst>"; }
    if (role == ROLE_TOOL) { return "<tool>"; }
    return null;
  }

  << sllmbv_load_default: load the compiled-in seed table into an EMPTY vocab,
     verifying each special lands at its declared id (fail closed otherwise). >>
  int loadDefault(SLLlmVocabulary vocab) {
    if (vocab == null) { return STATUS_INVALID_ARGUMENT; }
    if (vocab.size() != 0) { return STATUS_INVALID_ARGUMENT; }
    int role = ROLE_PAD;
    while (role <= ROLE_TOOL) {
      String tok = specialTokenFor(role);
      int id = vocab.add(tok);
      if (id != role) { return STATUS_ID_MISMATCH; }
      role = role + 1;
    }
    < Ordinary tokens would follow from id 10 (digits, punctuation, core words). >
    < The full table lives in tokenizers/base-vocabulary.json in the native tree. >
    return STATUS_OK;
  }

  < sllmbv_special_id: resolve a role to its id, or NO_ID. >
  int specialId(int role) {
    if (role < ROLE_PAD || role > ROLE_TOOL) { return NO_ID; }
    return role;
  }

  String defaultVocabularyId() { return vocabularyId; }

  String statusString(int status) {
    if (status == STATUS_OK) { return "ok"; }
    if (status == STATUS_INVALID_ARGUMENT) { return "invalid argument"; }
    if (status == STATUS_OUT_OF_MEMORY) { return "out of memory"; }
    if (status == STATUS_PARSE_ERROR) { return "parse error"; }
    if (status == STATUS_ID_MISMATCH) { return "id mismatch"; }
    if (status == STATUS_DUPLICATE_TOKEN) { return "duplicate token"; }
    if (status == STATUS_ROLE_UNSET) { return "role unset"; }
    return "unknown";
  }

  boolean available() { return vocabularyId != null; }
}