"""Self-contained character-level tokenizer for the Phase 4 learnability experiments (no corpus dependency)."""
from __future__ import annotations

import json
from pathlib import Path

PRINTABLE = [chr(i) for i in range(32, 127)]


class CharTokenizer:
    def __init__(self):
        self.specials = ["<eot>", "<pad>"]
        self.itos = self.specials + PRINTABLE
        self.stoi = {s: i for i, s in enumerate(self.itos)}
        self.eot_id = 0
        self.pad_id = 1

    @property
    def vocab_size(self) -> int:
        return len(self.itos)

    def encode(self, text: str) -> list[int]:
        return [self.stoi.get(ch, self.stoi["?"]) for ch in text]

    def decode(self, ids) -> str:
        return "".join(self.itos[i] if i >= len(self.specials) else "" for i in ids)

    def save(self, path):
        Path(path).write_text(json.dumps({"itos": self.itos}))