1"""Deterministic scorers. LongBench-style normalization for QA F1/EM; numeric with tolerance; yes/no; abstention."""2from __future__ import annotations3import re, string4from collections import Counter56ABSTAIN_PATTERNS = [r"\bunknown\b", r"\bnot (?:stated|mentioned|provided|specified|given|available)\b", r"\bdoes not (?:say|state|mention|specify)\b",7 r"\bcannot be determined\b", r"\bno information\b", r"\bnot in the document\b", r"\bi don't know\b", r"\bnot found\b", r"\binsufficient information\b"]89def normalize_answer(s: str) -> str:10 s = s.lower()11 s = "".join(ch for ch in s if ch not in set(string.punctuation))12 s = re.sub(r"\b(a|an|the)\b", " ", s)13 return " ".join(s.split())1415def f1_score(pred: str, gold: str) -> float:16 p, g = normalize_answer(pred).split(), normalize_answer(gold).split()17 common = Counter(p) & Counter(g)18 ns = sum(common.values())19 if ns == 0:20 return 0.021 prec, rec = ns / len(p), ns / len(g)22 return 2 * prec * rec / (prec + rec)2324def qa_f1(pred: str, golds: list[str]) -> float:25 return max(f1_score(pred, g) for g in golds) if golds else 0.02627def exact_match(pred: str, golds: list[str]) -> float:28 n = normalize_answer(pred)29 return float(any(n == normalize_answer(g) for g in golds))3031def contains_match(pred: str, golds: list[str]) -> float:32 n = normalize_answer(pred)33 return float(any(normalize_answer(g) in n for g in golds if normalize_answer(g)))3435_num_re = re.compile(r"[-+]?(?:\d{1,3}(?:[, ]\d{3})+|\d+)(?:\.\d+)?")3637def extract_numbers(s: str) -> list[float]:38 out = []39 for m in _num_re.finditer(s):40 t = m.group(0).replace(",", "").replace(" ", "").replace(" ", "")41 try:42 out.append(float(t))43 except ValueError:44 pass45 return out4647def numeric_match(pred: str, gold: str, tol: float = 0.0) -> float:48 g = extract_numbers(gold)49 if not g:50 return exact_match(pred, [gold])51 gv = g[0]52 nums = extract_numbers(pred)53 if not nums:54 return 0.055 # accept if the first number (or the only number) matches; penalize answers listing many numbers by requiring first or last56 cands = [nums[0], nums[-1]] if len(nums) > 1 else nums57 for v in cands:58 if gv == 0:59 if abs(v) <= tol: return 1.060 elif abs(v - gv) <= max(tol * abs(gv), 1e-9):61 return 1.062 return 0.06364def yesno_match(pred: str, gold: str) -> float:65 n = normalize_answer(pred)66 toks = n.split()67 has_yes = "yes" in toks[:3]; has_no = "no" in toks[:3] or n.startswith("not ")68 if has_yes and not has_no: return float(gold == "yes")69 if has_no and not has_yes: return float(gold == "no")70 return 0.07172def is_abstention(pred: str) -> bool:73 n = pred.lower()74 return any(re.search(p, n) for p in ABSTAIN_PATTERNS)7576def score_synthetic(pred: str, q: dict) -> dict:77 at = q["answer_type"]78 if at == "numeric":79 s = numeric_match(pred, q["answer"], q.get("tolerance", 0.0))80 elif at == "yesno":81 s = yesno_match(pred, q["answer"])82 elif at == "abstain":83 s = float(is_abstention(pred))84 else:85 golds = [q["answer"]] + list(q.get("aliases", []))86 s = exact_match(pred, golds)87 if s == 0.0:88 s = contains_match(pred, golds) # lenient containment counts as correct for exact-type answers89 return {"correct": s, "abstained": float(is_abstention(pred)), "f1": qa_f1(pred, [q["answer"]])}9091def score_public(pred: str, golds: list[str]) -> dict:92 return {"f1": qa_f1(pred, golds), "em": exact_match(pred, golds), "abstained": float(is_abstention(pred))}93