#include "large_language_models_base_vocabulary.h" #include /* ---- Compiled-in default seed set: sleela-base-en-1.0 ---- This mirrors tokenizers/base-vocabulary.json. The ordering here IS the contract: specials first (ascending from id 0), then ordinary tokens. The loader verifies, token by token, that the store hands back the same IDs. */ static const char *const kDefaultVocabularyId = "sleela-base-en-1.0"; static const char *const kSpecialTokens[] = { "", "", "", "", "", "", "", "", "", "" }; static const sllmbv_role kSpecialRoles[] = { SLLMBV_ROLE_PAD, SLLMBV_ROLE_UNK, SLLMBV_ROLE_BOS, SLLMBV_ROLE_EOS, SLLMBV_ROLE_SEP, SLLMBV_ROLE_MASK, SLLMBV_ROLE_SYSTEM, SLLMBV_ROLE_USER, SLLMBV_ROLE_ASSISTANT, SLLMBV_ROLE_TOOL }; static const size_t kSpecialCount = sizeof(kSpecialTokens)/sizeof(kSpecialTokens[0]); static const char *const kTokens[] = { "0","1","2","3","4","5","6","7","8","9", ".",",","?","!",";",":","-","(",")","\"","'", "+","=","/","%", "the","a","an","and","or","but","if","then","else","not","no","yes", "of","to","in","on","at","by","for","from","with","without","as", "is","are","was","were","be","been","being","has","have","had", "do","does","did","will","would","can","could","should","may","might","must", "this","that","these","those","it","its","they","them","we","you","i","he","she", "who","what","when","where","why","how","which", "all","any","some","each","every","more","most","than","very", "computer","machine","program","data","model","language","token","word","number","value", "system","process","function","code","source","text","input","output","train","learn", "answer","question","example","true","false","please","thank" }; static const size_t kTokenCount = sizeof(kTokens)/sizeof(kTokens[0]); const char *sllmbv_default_vocabulary_id(void){ return kDefaultVocabularyId; } const char *sllmbv_status_string(sllmbv_status status){ switch(status){ case SLLMBV_OK: return "ok"; case SLLMBV_INVALID_ARGUMENT: return "invalid argument"; case SLLMBV_OUT_OF_MEMORY: return "out of memory"; case SLLMBV_PARSE_ERROR: return "parse error"; case SLLMBV_ID_MISMATCH: return "token did not land at its declared id"; case SLLMBV_DUPLICATE_TOKEN: return "duplicate token in base set"; case SLLMBV_ROLE_UNSET: return "special-token role not declared"; } return "unknown status"; } uint32_t sllmbv_special_id(const sllmbv_specials *specials, sllmbv_role role){ if(!specials || role<0 || role>=SLLMBV_ROLE_COUNT) return SLLMBV_NO_ID; return specials->id[role]; } static sllmbv_status map_vocab_status(sllmv_status st){ switch(st){ case SLLMV_OK: return SLLMBV_OK; case SLLMV_OUT_OF_MEMORY: return SLLMBV_OUT_OF_MEMORY; case SLLMV_INVALID_ARGUMENT: return SLLMBV_INVALID_ARGUMENT; default: return SLLMBV_PARSE_ERROR; } } sllmbv_status sllmbv_load_arrays( sllmv_vocabulary *vocabulary, const char *const *special_tokens, const sllmbv_role *special_roles, size_t special_count, const char *const *tokens, size_t token_count, sllmbv_specials *specials) { if(!vocabulary) return SLLMBV_INVALID_ARGUMENT; if(special_count && (!special_tokens || !special_roles)) return SLLMBV_INVALID_ARGUMENT; if(token_count && !tokens) return SLLMBV_INVALID_ARGUMENT; /* The reserved-ID guarantee only holds if we seed into an empty store. */ if(sllmv_vocabulary_size(vocabulary)!=0) return SLLMBV_ID_MISMATCH; sllmbv_specials resolved; for(size_t r=0;r=SLLMBV_ROLE_COUNT) return SLLMBV_INVALID_ARGUMENT; resolved.id[role]=id; } for(size_t i=0;i