Home /Modules/artificial-intelligence/large_language_models_base_vocabulary.c

large_language_models_base_vocabulary.c

artificial-intelligence/large-language-models/large_language_models_base_vocabulary.c · 116 line(s) · c

#include "large_language_models_base_vocabulary.h"
#include <string.h>

/* ---- Compiled-in default seed set: sleela-base-en-1.0 ----
   This mirrors tokenizers/base-vocabulary.json. The ordering here IS the
   contract: specials first (ascending from id 0), then ordinary tokens. The
   loader verifies, token by token, that the store hands back the same IDs. */

static const char *const kDefaultVocabularyId = "sleela-base-en-1.0";

static const char *const kSpecialTokens[] = {
 "<pad>", "<unk>", "<bos>", "<eos>", "<sep>", "<mask>", "<sys>", "<user>", "<asst>", "<tool>"
};
static const sllmbv_role kSpecialRoles[] = {
 SLLMBV_ROLE_PAD, SLLMBV_ROLE_UNK, SLLMBV_ROLE_BOS, SLLMBV_ROLE_EOS, SLLMBV_ROLE_SEP,
 SLLMBV_ROLE_MASK, SLLMBV_ROLE_SYSTEM, SLLMBV_ROLE_USER, SLLMBV_ROLE_ASSISTANT, SLLMBV_ROLE_TOOL
};
static const size_t kSpecialCount = sizeof(kSpecialTokens)/sizeof(kSpecialTokens[0]);

static const char *const kTokens[] = {
 "0","1","2","3","4","5","6","7","8","9",
 ".",",","?","!",";",":","-","(",")","\"","'",
 "+","=","/","%",
 "the","a","an","and","or","but","if","then","else","not","no","yes",
 "of","to","in","on","at","by","for","from","with","without","as",
 "is","are","was","were","be","been","being","has","have","had",
 "do","does","did","will","would","can","could","should","may","might","must",
 "this","that","these","those","it","its","they","them","we","you","i","he","she",
 "who","what","when","where","why","how","which",
 "all","any","some","each","every","more","most","than","very",
 "computer","machine","program","data","model","language","token","word","number","value",
 "system","process","function","code","source","text","input","output","train","learn",
 "answer","question","example","true","false","please","thank"
};
static const size_t kTokenCount = sizeof(kTokens)/sizeof(kTokens[0]);

const char *sllmbv_default_vocabulary_id(void){ return kDefaultVocabularyId; }

const char *sllmbv_status_string(sllmbv_status status){
 switch(status){
  case SLLMBV_OK: return "ok";
  case SLLMBV_INVALID_ARGUMENT: return "invalid argument";
  case SLLMBV_OUT_OF_MEMORY: return "out of memory";
  case SLLMBV_PARSE_ERROR: return "parse error";
  case SLLMBV_ID_MISMATCH: return "token did not land at its declared id";
  case SLLMBV_DUPLICATE_TOKEN: return "duplicate token in base set";
  case SLLMBV_ROLE_UNSET: return "special-token role not declared";
 }
 return "unknown status";
}

uint32_t sllmbv_special_id(const sllmbv_specials *specials, sllmbv_role role){
 if(!specials || role<0 || role>=SLLMBV_ROLE_COUNT) return SLLMBV_NO_ID;
 return specials->id[role];
}

static sllmbv_status map_vocab_status(sllmv_status st){
 switch(st){
  case SLLMV_OK: return SLLMBV_OK;
  case SLLMV_OUT_OF_MEMORY: return SLLMBV_OUT_OF_MEMORY;
  case SLLMV_INVALID_ARGUMENT: return SLLMBV_INVALID_ARGUMENT;
  default: return SLLMBV_PARSE_ERROR;
 }
}

sllmbv_status sllmbv_load_arrays(
 sllmv_vocabulary *vocabulary,
 const char *const *special_tokens, const sllmbv_role *special_roles, size_t special_count,
 const char *const *tokens, size_t token_count,
 sllmbv_specials *specials)
{
 if(!vocabulary) return SLLMBV_INVALID_ARGUMENT;
 if(special_count && (!special_tokens || !special_roles)) return SLLMBV_INVALID_ARGUMENT;
 if(token_count && !tokens) return SLLMBV_INVALID_ARGUMENT;
 /* The reserved-ID guarantee only holds if we seed into an empty store. */
 if(sllmv_vocabulary_size(vocabulary)!=0) return SLLMBV_ID_MISMATCH;

 sllmbv_specials resolved;
 for(size_t r=0;r<SLLMBV_ROLE_COUNT;r++) resolved.id[r]=SLLMBV_NO_ID;

 uint32_t expected=0;
 for(size_t i=0;i<special_count;i++,expected++){
  const char *tok=special_tokens[i];
  if(!tok || !*tok) return SLLMBV_INVALID_ARGUMENT;
  uint32_t id=0;
  /* A duplicate add returns the existing id instead of a fresh one. */
  uint32_t before=(uint32_t)sllmv_vocabulary_size(vocabulary);
  sllmv_status st=sllmv_vocabulary_add(vocabulary,tok,&id);
  if(st!=SLLMV_OK) return map_vocab_status(st);
  if(id!=expected) return SLLMBV_ID_MISMATCH;
  if((uint32_t)sllmv_vocabulary_size(vocabulary)!=before+1) return SLLMBV_DUPLICATE_TOKEN;
  sllmbv_role role=special_roles[i];
  if(role<0 || role>=SLLMBV_ROLE_COUNT) return SLLMBV_INVALID_ARGUMENT;
  resolved.id[role]=id;
 }
 for(size_t i=0;i<token_count;i++,expected++){
  const char *tok=tokens[i];
  if(!tok || !*tok) return SLLMBV_INVALID_ARGUMENT;
  uint32_t id=0;
  uint32_t before=(uint32_t)sllmv_vocabulary_size(vocabulary);
  sllmv_status st=sllmv_vocabulary_add(vocabulary,tok,&id);
  if(st!=SLLMV_OK) return map_vocab_status(st);
  if(id!=expected) return SLLMBV_ID_MISMATCH;
  if((uint32_t)sllmv_vocabulary_size(vocabulary)!=before+1) return SLLMBV_DUPLICATE_TOKEN;
 }
 if(specials) *specials=resolved;
 return SLLMBV_OK;
}

sllmbv_status sllmbv_load_default(sllmv_vocabulary *vocabulary, sllmbv_specials *specials){
 return sllmbv_load_arrays(vocabulary,
  kSpecialTokens, kSpecialRoles, kSpecialCount,
  kTokens, kTokenCount,
  specials);
}