base_vocabulary_selftest.c
artificial-intelligence/large-language-models/tools/base_vocabulary_selftest.c · 53 line(s) · c
/* Self-test for the base-vocabulary loader. Builds standalone:
* cc -std=c11 -O2 -Wall -Wextra -pedantic \
* ../large_language_models_vocabulary.c \
* ../large_language_models_base_vocabulary.c \
* base_vocabulary_selftest.c -o base_vocabulary_selftest
* Returns 0 on success; prints the first failing check otherwise. */
#include "../large_language_models_base_vocabulary.h"
#include <stdio.h>
#include <string.h>
static int fail(const char *msg){ fprintf(stderr, "selftest: FAIL: %s\n", msg); return 1; }
int main(void){
sllmv_vocabulary *v=NULL;
if(sllmv_vocabulary_create(&v)!=SLLMV_OK || !v) return fail("create");
sllmbv_specials sp;
sllmbv_status st = sllmbv_load_default(v, &sp);
if(st!=SLLMBV_OK){ fprintf(stderr,"load: %s\n", sllmbv_status_string(st)); return fail("load_default"); }
/* Reserved special IDs land at the fixed low positions. */
if(sllmbv_special_id(&sp, SLLMBV_ROLE_PAD) != 0) return fail("pad != 0");
if(sllmbv_special_id(&sp, SLLMBV_ROLE_UNK) != 1) return fail("unk != 1");
if(sllmbv_special_id(&sp, SLLMBV_ROLE_BOS) != 2) return fail("bos != 2");
if(sllmbv_special_id(&sp, SLLMBV_ROLE_EOS) != 3) return fail("eos != 3");
/* Round-trip: id -> token -> id. */
const char *tok=NULL;
if(sllmv_vocabulary_token(v, 0, &tok)!=SLLMV_OK || strcmp(tok,"<pad>")!=0) return fail("token(0) != <pad>");
uint32_t id=0;
if(sllmv_vocabulary_find(v, "the", &id)!=SLLMV_OK) return fail("find('the')");
const char *back=NULL;
if(sllmv_vocabulary_token(v, id, &back)!=SLLMV_OK || strcmp(back,"the")!=0) return fail("round-trip 'the'");
if(sllmv_vocabulary_find(v, "computer", &id)!=SLLMV_OK) return fail("find('computer')");
/* Encode a small line through the seeded whitespace tokenizer. */
uint32_t *ids=NULL; size_t n=0;
if(sllmv_vocabulary_encode(v, "the computer is a machine", &ids, &n)!=SLLMV_OK) return fail("encode");
if(n!=5){ sllmv_vocabulary_free_ids(ids); return fail("encode count != 5"); }
sllmv_vocabulary_free_ids(ids);
/* Loading into a non-empty vocabulary must fail closed. */
st = sllmbv_load_default(v, &sp);
if(st==SLLMBV_OK) return fail("second load should fail (non-empty)");
sllmv_vocabulary_destroy(v);
printf("selftest: OK — base '%s' seeded; specials fixed; round-trip + encode pass\n",
sllmbv_default_vocabulary_id());
return 0;
}