Home /Modules/artificial-intelligence/large_language_models_base_vocabulary.hpp

large_language_models_base_vocabulary.hpp

artificial-intelligence/large-language-models/large_language_models_base_vocabulary.hpp · 44 line(s) · c

#ifndef SLEELA_AI_LLM_BASE_VOCABULARY_HPP
#define SLEELA_AI_LLM_BASE_VOCABULARY_HPP
#include "large_language_models_base_vocabulary.h"
#include "large_language_models_vocabulary.hpp"
#include <array>
#include <cstdint>
#include <optional>
#include <stdexcept>
#include <string>
namespace sleela::ai {

// Resolved special-token IDs for a seeded base vocabulary. A role that was not
// declared reports std::nullopt rather than a sentinel.
class BaseSpecials {
public:
 explicit BaseSpecials(const sllmbv_specials& raw) : raw_(raw) {}
 std::optional<std::uint32_t> id(sllmbv_role role) const {
  std::uint32_t v = sllmbv_special_id(&raw_, role);
  if (v == SLLMBV_NO_ID) return std::nullopt;
  return v;
 }
 std::optional<std::uint32_t> pad()  const { return id(SLLMBV_ROLE_PAD); }
 std::optional<std::uint32_t> unk()  const { return id(SLLMBV_ROLE_UNK); }
 std::optional<std::uint32_t> bos()  const { return id(SLLMBV_ROLE_BOS); }
 std::optional<std::uint32_t> eos()  const { return id(SLLMBV_ROLE_EOS); }
 std::optional<std::uint32_t> sep()  const { return id(SLLMBV_ROLE_SEP); }
 std::optional<std::uint32_t> mask() const { return id(SLLMBV_ROLE_MASK); }
 const sllmbv_specials& raw() const noexcept { return raw_; }
private:
 sllmbv_specials raw_;
};

// Seed the compiled-in default base vocabulary (sleela-base-en-1.0) into an
// empty LanguageModelVocabulary. Throws std::runtime_error on any failure,
// including a non-empty target or an ID mismatch (fail closed).
BaseSpecials seed_default_base_vocabulary(LanguageModelVocabulary& vocab);

// Low-level: operate directly on the C handle. Kept minimal to preserve the
// opaque handle; ownership of the handle stays with the caller.
BaseSpecials seed_default_base_vocabulary_handle(sllmv_vocabulary* handle);

} // namespace sleela::ai
#endif