Repository Public Research preview
1"""n1_02_tokenizer_audit.py — WordPiece truncation audit of the 512-word chunks under all-MiniLM-L6-v22(AI_C-06 (a), EDIT-05). Tokenizer only, no inference.34Samples (deterministic, hash-based, seed 20261105):5 R : 10,000 random chunks;6 K : 10,000 random chunks containing >= 1 CORE AI keyword (regex, lower-cased);7 T : the 5,000 top-scoring chunks on score_ai_adoption and 5,000 on score_ai_infrastructure.8For every sampled chunk: WordPiece token count (with [CLS]/[SEP]), truncation flag at the configured9max_seq_length (read from the cached model's sentence_bert_config.json), share of tokens dropped,10and the token position of every AI keyword hit (fast-tokenizer offsets) -> hits inside / beyond the window.11Outputs: results/tokenizer_audit.json, tokenizer_sample.csv (one row per sampled chunk).12"""13import os, sys, re, json, glob14import numpy as np, pandas as pd15sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))16from n1_common import * # noqa17os.environ.setdefault("HF_HUB_OFFLINE", "1"); os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")18from transformers import AutoTokenizer1920# --- configured max_seq_length: cached model config + pipeline code (no override in src/embeddings/models.py)21snap = glob.glob(os.path.expanduser("~/.cache/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/snapshots/*/"))[0]22sbert_cfg = json.load(open(snap + "sentence_bert_config.json"))23tok_cfg = json.load(open(snap + "tokenizer_config.json"))24MAXLEN = int(sbert_cfg["max_seq_length"]) # 256 (includes [CLS] and [SEP])25CONTENT = MAXLEN - 226tok = AutoTokenizer.from_pretrained(snap)27print("max_seq_length (sentence_bert_config.json) =", MAXLEN, "| tokenizer model_max_length =", tok_cfg.get("model_max_length"), "| fast =", tok.is_fast)2829c = connect(); views(c)30SCORE_COLS = "score_ai_adoption, score_ai_infrastructure, score_ai_disruption_risk, score_automation_labor, score_cybersecurity"3132def pull(sql, frame):33 df = c.execute(sql).df(); df["frame"] = frame; return df3435with Timer("samples"):36 base = f"""SELECT ch.chunk_id, ch.accession_number, cs.cik, cs.ticker, cs.form_type, cs.fiscal_year, cs.section_name,37 ch.chunk_order, ch.word_count, ch.char_count, ch.chunk_text, {SCORE_COLS}38 FROM chunks ch JOIN chunk_scores cs USING (chunk_id)"""39 R = pull(base + f" WHERE hash(ch.chunk_id || '{SEED}') % 1072 = 0 LIMIT 10000", "R")40 K = pull(base + f" WHERE hash(ch.chunk_id || '{SEED}') % 4 = 0 AND regexp_matches(lower(ch.chunk_text), '{CORE_RE}') LIMIT 10000", "K")41 T1 = pull(base + " ORDER BY cs.score_ai_adoption DESC LIMIT 5000", "T_adopt")42 T2 = pull(base + " ORDER BY cs.score_ai_infrastructure DESC LIMIT 5000", "T_infra")43 df = pd.concat([R, K, T1, T2], ignore_index=True)44 print({k: len(v) for k, v in [("R", R), ("K", K), ("T_adopt", T1), ("T_infra", T2)]})4546core_re = re.compile(CORE_RE, re.I); acr_re = re.compile(ACRONYM_RE)4748def analyse(text):49 enc = tok(text, add_special_tokens=True, truncation=False, return_offsets_mapping=True)50 ids = enc["input_ids"]; offs = enc["offset_mapping"]51 n = len(ids)52 # char position of the last token kept (content tokens 1..CONTENT)53 if n > MAXLEN:54 cut_char = offs[CONTENT][0] # start of the first dropped content token55 else:56 cut_char = len(text)57 starts = np.array([o[0] for o in offs[1:-1]], dtype=np.int64) # content tokens58 hits = [m.start() for m in core_re.finditer(text)] + [m.start() for m in acr_re.finditer(text)]59 hits.sort()60 tok_pos = np.searchsorted(starts, np.array(hits, dtype=np.int64), side="right") if hits else np.array([], dtype=int)61 inside = int((tok_pos <= CONTENT).sum()); outside = int((tok_pos > CONTENT).sum())62 first_hit_tok = int(tok_pos[0]) if len(tok_pos) else -163 words_before_cut = len(text[:cut_char].split())64 return n, cut_char, inside, outside, first_hit_tok, words_before_cut6566with Timer(f"tokenizing {len(df):,} chunks"):67 res = [analyse(t) for t in df["chunk_text"].tolist()]68df[["n_tokens", "cut_char", "hits_inside", "hits_beyond", "first_hit_token", "words_in_window"]] = pd.DataFrame(res, index=df.index)69df["truncated"] = df["n_tokens"] > MAXLEN70df["share_tokens_dropped"] = np.where(df["truncated"], (df["n_tokens"] - MAXLEN) / df["n_tokens"], 0.0)71df["share_chars_dropped"] = 1 - df["cut_char"] / df["char_count"].clip(lower=1)72df["n_hits"] = df["hits_inside"] + df["hits_beyond"]73df["hit_class"] = np.select([df.n_hits == 0, (df.hits_inside > 0) & (df.hits_beyond == 0), (df.hits_inside == 0) & (df.hits_beyond > 0)],74 ["none", "inside_only", "beyond_only"], "both")75df["tokens_per_word"] = df["n_tokens"] / df["word_count"].clip(lower=1)7677def summarise(g):78 full = g[g.word_count >= 500]79 return {80 "n": int(len(g)), "n_full_512w": int(len(full)),81 "tokens_median": float(g.n_tokens.median()), "tokens_mean": float(g.n_tokens.mean()),82 "tokens_p05": float(g.n_tokens.quantile(.05)), "tokens_p95": float(g.n_tokens.quantile(.95)),83 "tokens_per_word_median": float(g.tokens_per_word.median()),84 "share_truncated": float(g.truncated.mean()), "share_truncated_full_512w": float(full.truncated.mean()) if len(full) else None,85 "share_of_token_mass_dropped": float((g.n_tokens - g.n_tokens.clip(upper=MAXLEN)).sum() / g.n_tokens.sum()),86 "mean_share_tokens_dropped_if_truncated": float(g.loc[g.truncated, "share_tokens_dropped"].mean()) if g.truncated.any() else 0.0,87 "mean_share_chars_dropped": float(g.share_chars_dropped.mean()),88 "words_in_window_median": float(g.loc[g.truncated, "words_in_window"].median()) if g.truncated.any() else None,89 "share_with_ai_hit": float((g.n_hits > 0).mean()),90 "hits_total": int(g.n_hits.sum()), "hits_beyond_window": int(g.hits_beyond.sum()),91 "share_hits_beyond_window": float(g.hits_beyond.sum() / max(g.n_hits.sum(), 1)),92 "share_hit_chunks_all_hits_beyond": float((g.hit_class == "beyond_only").sum() / max((g.n_hits > 0).sum(), 1)),93 "hit_class_counts": g.hit_class.value_counts().to_dict(),94 "mean_score_adopt_by_hit_class": g.groupby("hit_class").score_ai_adoption.mean().round(4).to_dict(),95 "mean_score_infra_by_hit_class": g.groupby("hit_class").score_ai_infrastructure.mean().round(4).to_dict(),96 "median_score_adopt_by_hit_class": g.groupby("hit_class").score_ai_adoption.median().round(4).to_dict(),97 "p90_score_adopt_by_hit_class": g.groupby("hit_class").score_ai_adoption.quantile(.9).round(4).to_dict(),98 }99100out = {"max_seq_length": MAXLEN, "content_tokens": CONTENT, "model_max_length_tokenizer": tok_cfg.get("model_max_length"),101 "pipeline_override": "none: src/embeddings/models.py calls SentenceTransformer(model_name, device) without setting max_seq_length; config/embeddings.yaml lists max_tokens: 256 for all-MiniLM-L6-v2 (informational only)",102 "chunker": "512 words, overlap 64, min 50 (src/parsing/chunker.py)", "seed": SEED,103 "frames": {fr: summarise(g) for fr, g in df.groupby("frame")}}104# by form / section for the random frame105Rdf = df[df.frame == "R"]106out["random_by_form"] = {k: {"n": int(len(g)), "tokens_median": float(g.n_tokens.median()), "share_truncated": float(g.truncated.mean()),107 "share_token_mass_dropped": float((g.n_tokens - g.n_tokens.clip(upper=MAXLEN)).sum() / g.n_tokens.sum())}108 for k, g in Rdf.groupby("form_type")}109out["random_by_section"] = {k: {"n": int(len(g)), "tokens_median": float(g.n_tokens.median()), "share_truncated": float(g.truncated.mean())}110 for k, g in Rdf.groupby("section_name")}111out["random_tokens_hist"] = np.histogram(Rdf.n_tokens, bins=[0, 64, 128, 192, 256, 320, 384, 448, 512, 640, 768, 1024, 100000])[0].tolist()112out["random_tokens_hist_bins"] = [0, 64, 128, 192, 256, 320, 384, 448, 512, 640, 768, 1024, "inf"]113# position of the first AI hit (K frame): share of hit chunks whose FIRST hit lies beyond the window114Kdf = df[(df.frame == "K") & (df.n_hits > 0)]115out["K_first_hit_beyond_window_share"] = float((Kdf.first_hit_token > CONTENT).mean())116out["K_first_hit_token_quantiles"] = Kdf.first_hit_token.quantile([.1, .25, .5, .75, .9]).round(0).to_dict()117# top-scoring chunks: how many are truncated, and does the score depend on hits inside the window118for fr in ["T_adopt", "T_infra"]:119 g = df[df.frame == fr]120 out["frames"][fr]["share_top_chunks_truncated"] = float(g.truncated.mean())121 out["frames"][fr]["share_top_chunks_short_lt_256_tokens"] = float((g.n_tokens <= MAXLEN).mean())122 out["frames"][fr]["words_median"] = float(g.word_count.median())123# correlation between score and hits inside vs beyond (random+K pooled)124P = df[df.frame.isin(["R", "K"])]125from scipy.stats import spearmanr126out["pooled_RK_spearman_adopt_vs_hits_inside"] = float(spearmanr(P.score_ai_adoption, P.hits_inside).correlation)127out["pooled_RK_spearman_adopt_vs_hits_beyond"] = float(spearmanr(P.score_ai_adoption, P.hits_beyond).correlation)128# regression-style contrast: among chunks with exactly the same number of hits, inside vs beyond129tmp = P[P.n_hits.between(1, 3)]130out["pooled_RK_1to3hits_mean_adopt_inside_only"] = float(tmp[tmp.hit_class == "inside_only"].score_ai_adoption.mean())131out["pooled_RK_1to3hits_mean_adopt_beyond_only"] = float(tmp[tmp.hit_class == "beyond_only"].score_ai_adoption.mean())132out["pooled_RK_nohit_mean_adopt"] = float(P[P.hit_class == "none"].score_ai_adoption.mean())133out["pooled_RK_1to3hits_n"] = tmp.hit_class.value_counts().to_dict()134dump_json(out, f"{RES}/tokenizer_audit.json")135cols = ["frame", "chunk_id", "accession_number", "cik", "ticker", "form_type", "fiscal_year", "section_name", "chunk_order", "word_count", "char_count",136 "n_tokens", "truncated", "share_tokens_dropped", "share_chars_dropped", "words_in_window", "n_hits", "hits_inside", "hits_beyond", "first_hit_token", "hit_class",137 "score_ai_adoption", "score_ai_infrastructure", "score_ai_disruption_risk", "score_automation_labor", "score_cybersecurity"]138df[cols].to_csv(f"{RES}/tokenizer_sample.csv", index=False)139print(json.dumps({k: out["frames"][k] for k in out["frames"]}, indent=1, default=str)[:4000])140