// Author: Simon-Pierre Boucher — contact@spboucher.ai
#pragma once

#include <cstdint>
#include <map>
#include <string>
#include <vector>

namespace forge::tok {

// Byte-level BPE (minbpe BasicTokenizer semantics: no regex pre-split —
// fine for small domain vocabs, see RESEARCH.md §7). The .model file is
// produced by tools/train_tokenizer.py, whose encoder this must match
// exactly:
//   forgebpe v1\n
//   <vocab_size>\n
//   <id> <left> <right>\n     (one line per merge, ids from 256 upward)
class BPETokenizer {
public:
    void load(const std::string& model_path);

    std::vector<int32_t> encode(const std::string& text) const;
    std::string decode(const std::vector<int32_t>& ids) const;

    int64_t vocab_size() const { return int64_t(vocab_.size()); }

private:
    // merge ranks: (left, right) -> merged id; rank order == id order
    std::map<std::pair<int32_t, int32_t>, int32_t> merges_;
    std::vector<std::string> vocab_; // id -> bytes
};

} // namespace forge::tok