spb/forge Public MIT

Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.

C++ 61.2% C 23% Python 7.6% TeX 7.2% CMake 1.1%

1003 B · 34 lines c

Raw Blame History

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2#pragma once34#include <cstdint>5#include <map>6#include <string>7#include <vector>89namespace forge::tok {1011// Byte-level BPE (minbpe BasicTokenizer semantics: no regex pre-split —12// fine for small domain vocabs, see RESEARCH.md §7). The .model file is13// produced by tools/train_tokenizer.py, whose encoder this must match14// exactly:15//   forgebpe v1\n16//   <vocab_size>\n17//   <id> <left> <right>\n     (one line per merge, ids from 256 upward)18class BPETokenizer {19public:20    void load(const std::string& model_path);2122    std::vector<int32_t> encode(const std::string& text) const;23    std::string decode(const std::vector<int32_t>& ids) const;2425    int64_t vocab_size() const { return int64_t(vocab_.size()); }2627private:28    // merge ranks: (left, right) -> merged id; rank order == id order29    std::map<std::pair<int32_t, int32_t>, int32_t> merges_;30    std::vector<std::string> vocab_; // id -> bytes31};3233} // namespace forge::tok34