1"""Self-contained character-level tokenizer for the Phase 4 learnability experiments (no corpus dependency)."""2from __future__ import annotations34import json5from pathlib import Path67PRINTABLE = [chr(i) for i in range(32, 127)]8910class CharTokenizer:11 def __init__(self):12 self.specials = ["<eot>", "<pad>"]13 self.itos = self.specials + PRINTABLE14 self.stoi = {s: i for i, s in enumerate(self.itos)}15 self.eot_id = 016 self.pad_id = 11718 @property19 def vocab_size(self) -> int:20 return len(self.itos)2122 def encode(self, text: str) -> list[int]:23 return [self.stoi.get(ch, self.stoi["?"]) for ch in text]2425 def decode(self, ids) -> str:26 return "".join(self.itos[i] if i >= len(self.specials) else "" for i in ids)2728 def save(self, path):29 Path(path).write_text(json.dumps({"itos": self.itos}))30