1"""Controlled private-fact task generator (deterministic, seeded).23Generates fictional 'project ledger' documents whose facts cannot be inferred from pretraining: invented entity names,4random identifiers, amounts, dates, units, status events (approved/rejected, cancelled vs postponed), timestamped5revisions with an explicit precedence rule, distractor projects with similar names and plausible wrong numbers,6boilerplate filler to reach a target token length. Each document ships with (a) the text, (b) questions by type,7(c) an evaluator-only answer key and fact ledger. Keys are written to a separate directory never exposed to agents.8"""9from __future__ import annotations10import random, json, math, datetime as dt11from dataclasses import dataclass, field, asdict12from typing import Any1314SYLL = ["var", "ko", "lith", "zen", "dra", "mel", "qui", "tor", "ban", "sel", "ur", "nak", "fel", "ost", "ri", "gam", "vol", "tis", "pra", "lun",15 "ker", "oby", "mun", "tal", "ix", "ard", "wen", "sol", "bri", "tek", "nov", "ald", "ymi", "ruk", "esh"]16FIRST = ["Teodora", "Milos", "Ingrid", "Rasheed", "Yevgenia", "Kwame", "Solveig", "Tarek", "Bohdan", "Nnamdi", "Lorcan", "Priyamvada", "Oksana", "Haruki",17 "Zéphyrine", "Casimir", "Ludivine", "Ewan", "Marisol", "Ragnvald", "Aurelio", "Dagny", "Idris", "Wilhelmina"]18LAST = ["Qvistrand", "Obuya", "Ferrante", "Mäkelä", "Dubrovnik", "Achterberg", "Nwachukwu", "Sørlie", "Baptiste-Roy", "Zelenko", "Haldorsen", "Petrakis",19 "Ouellet-Marchand", "Kirchhoff", "Abubakar", "Lindqvist", "Tremblay-Côté", "Varga", "Okonkwo", "Bergström", "Ishikawa", "Delacroix", "Mbeki", "Rousseau"]20TOWNS = ["Chibougamau", "Matagami", "Fermont", "Lebel-sur-Quévillon", "Sept-Îles", "Rouyn-Noranda", "Val-d'Or", "Schefferville", "Baie-Comeau", "Amos", "La Tuque",21 "Senneterre", "Gaspé", "Mont-Laurier", "Témiscaming"]22CURR = ["CAD", "EUR", "USD", "CHF", "NOK"]23UNITS = [("MW", "kW", 1000.0), ("km", "m", 1000.0), ("t", "kg", 1000.0), ("ha", "m²", 10000.0)]24FILLER = [25 "The committee reiterates that all figures are reported in nominal terms unless otherwise stated.",26 "Procurement follows the standard three-quote rule for purchases above the departmental threshold.",27 "Quarterly reviews are held on the second Tuesday of the month following the quarter end.",28 "Environmental monitoring reports are archived by the records office for a period of seven years.",29 "Travel reimbursements require original receipts and a signed itinerary.",30 "Site access badges must be renewed annually and returned on project completion.",31 "The internal audit function reports to the governance board and not to project leads.",32 "Vendor onboarding includes a conflict-of-interest declaration and a bank verification letter.",33 "All timestamps in this ledger are recorded in local time at the head office.",34 "Minutes are circulated within five business days and approved at the following meeting.",35 "Health and safety briefings are mandatory for all visitors before entering an active site.",36 "Capital expenditure is tracked separately from operating costs in the ledger system.",37 "Contingency reserves may only be released with written approval from the finance director.",38 "Printed copies of this memo are uncontrolled; the electronic record prevails.",39 "Project codes follow the pattern of a name, a hyphen and a sequence number.",40 "Where two entries conflict, the entry with the later timestamp supersedes the earlier one.",41]4243def _name(rng: random.Random) -> str:44 n = rng.choice([2, 3])45 s = "".join(rng.choice(SYLL) for _ in range(n))46 return s.capitalize()4748def _ident(rng: random.Random) -> str:49 L = "ABCDEFGHJKLMNPQRSTUVWXYZ"50 return f"{rng.choice(L)}{rng.choice(L)}-{rng.randint(10,99)}{rng.choice(L)}-{rng.randint(1000,9999)}"5152def _lot(rng: random.Random) -> str:53 return f"Lot {rng.randint(10,999)}-{rng.choice('ABCDEFGHKLMNPRSTVWXZ')}"5455def _date(rng: random.Random, y0=2029, y1=2034) -> dt.date:56 return dt.date(rng.randint(y0, y1), rng.randint(1, 12), rng.randint(1, 28))5758def fmt_date(d: dt.date) -> str:59 return d.strftime("%-d %B %Y")6061def fmt_amt(x: int) -> str:62 return f"{x:,}"6364@dataclass65class Fact:66 fact_id: str67 entity: str68 attribute: str69 value: str70 kind: str # identifier | amount | date | status | person | place | quantity71 sentence: str72 char_start: int = -173 char_end: int = -17475@dataclass76class Question:77 question_id: str78 qtype: str79 question: str80 answer: str81 answer_type: str # exact | numeric | yesno | abstain | choice82 tolerance: float83 supporting_fact_ids: list84 answer_position_frac: float # relative position of the main supporting sentence in the document (0..1)85 aliases: list = field(default_factory=list)8687@dataclass88class Project:89 name: str90 code: str91 proposer: str92 proposed_date: dt.date93 proposed_amount: int94 currency: str95 site_lot: str96 town: str97 supplier_id: str98 approved: bool99 decision_date: dt.date100 revisions: list # [(date, amount)]101 cancelled: bool102 postponed_to: dt.date | None103 quantity: tuple # (value, unit_big, unit_small, factor)104 decimal_value: str # unusual decimal, e.g. "17.0463"105106def make_project(rng: random.Random, base: str | None = None, used: set | None = None) -> Project:107 used = used if used is not None else set()108 while True:109 nm = base or _name(rng)110 code = f"{nm}-{rng.randint(1, 19)}"111 if code not in used:112 used.add(code); break113 proposed = _date(rng)114 amt = rng.randint(180_000, 9_900_000)115 amt = amt - amt % rng.choice([1, 5, 25])116 dec = rng.randint(1, 3)117 revisions = []118 d = proposed119 for _ in range(rng.choice([0, 1, 2, 2])):120 d = d + dt.timedelta(days=rng.randint(7, 120))121 amt2 = int(amt * rng.uniform(0.82, 1.18)); amt2 -= amt2 % 5122 revisions.append((d, amt2))123 decision = d + dt.timedelta(days=rng.randint(3, 60))124 approved = rng.random() < 0.7125 cancelled = (rng.random() < 0.3)126 postponed = None if (cancelled or rng.random() < 0.4) else decision + dt.timedelta(days=rng.randint(30, 400))127 q = rng.choice(UNITS)128 qv = round(rng.uniform(0.4, 48.0), rng.choice([1, 2]))129 cur = "CAD" if rng.random() < 0.6 else rng.choice(CURR)130 return Project(nm, code, f"{rng.choice(FIRST)} {rng.choice(LAST)}", proposed, amt, cur, _lot(rng), rng.choice(TOWNS), _ident(rng),131 approved, decision, revisions, cancelled, postponed, (qv, q[0], q[1], q[2]), f"{rng.uniform(1, 99):.4f}")132133def current_amount(p: Project) -> int:134 return p.revisions[-1][1] if p.revisions else p.proposed_amount135136def render_project(p: Project, rng: random.Random, facts: list, shuffle_sentences: bool) -> str:137 S = []138 def F(attr, val, kind, sent):139 facts.append(Fact(f"{p.code}:{attr}", p.code, attr, val, kind, sent)); S.append(sent)140 F("proposer", p.proposer, "person", f"Project {p.code} was proposed by {p.proposer} on {fmt_date(p.proposed_date)} with an initial budget of {fmt_amt(p.proposed_amount)} {p.currency}.")141 facts.append(Fact(f"{p.code}:proposed_date", p.code, "proposed_date", fmt_date(p.proposed_date), "date", S[-1]))142 facts.append(Fact(f"{p.code}:proposed_amount", p.code, "proposed_amount", f"{fmt_amt(p.proposed_amount)} {p.currency}", "amount", S[-1]))143 for i, (d, a) in enumerate(p.revisions):144 F(f"revision_{i+1}", f"{fmt_amt(a)} {p.currency}", "amount", f"On {fmt_date(d)} the budget of {p.code} was revised to {fmt_amt(a)} {p.currency}.")145 verb = "approved" if p.approved else "rejected"146 F("decision", verb, "status", f"The {verb} decision for {p.code} was recorded on {fmt_date(p.decision_date)}.")147 F("site", f"{p.site_lot} in {p.town}", "place", f"The site of {p.code} is {p.site_lot} in {p.town}.")148 F("supplier_id", p.supplier_id, "identifier", f"The supplier identifier for {p.code} is {p.supplier_id}.")149 if p.cancelled:150 F("cancelled", "yes", "status", f"{p.code} was cancelled; it was not postponed.")151 elif p.postponed_to is not None:152 F("cancelled", "no", "status", f"{p.code} was NOT cancelled; its launch was postponed to {fmt_date(p.postponed_to)}.")153 facts.append(Fact(f"{p.code}:postponed_to", p.code, "postponed_to", fmt_date(p.postponed_to), "date", S[-1]))154 else:155 F("cancelled", "no", "status", f"{p.code} remains on schedule and was neither cancelled nor postponed.")156 qv, ub, us, fac = p.quantity157 F("quantity", f"{qv} {ub}", "quantity", f"The rated capacity of {p.code} is {qv} {ub}.")158 F("decimal", p.decimal_value, "identifier", f"The calibration constant filed for {p.code} is {p.decimal_value}.")159 if shuffle_sentences:160 rng.shuffle(S)161 return " ".join(S)162163def _norm_num(x: float) -> str:164 if abs(x - round(x)) < 1e-9:165 return str(int(round(x)))166 return f"{x:.6g}"167168def build_questions(projects: list[Project], distractor_names: list[str], rng: random.Random, doc_text: str, facts: list[Fact]) -> list[Question]:169 Q = []170 pos = {}171 for f in facts:172 i = doc_text.find(f.sentence)173 f.char_start, f.char_end = i, i + len(f.sentence)174 pos[f.fact_id] = i / max(1, len(doc_text))175 def add(qtype, text, ans, atype, fids, tol=0.0, aliases=None):176 Q.append(Question(f"q{len(Q):02d}_{qtype}", qtype, text, ans, atype, tol, fids, (pos.get(fids[0], 0.0) if fids else -1.0), aliases or []))177 # choose target projects for each type (deterministic order by rng)178 P = list(projects); rng.shuffle(P)179 p = P[0]; add("single_fact", f"What is the supplier identifier for {p.code}?", p.supplier_id, "exact", [f"{p.code}:supplier_id"])180 p = P[1 % len(P)]; add("single_fact", f"Where is the site of {p.code}? Give the lot and the town.", f"{p.site_lot} in {p.town}", "exact", [f"{p.code}:site"], aliases=[p.site_lot, f"{p.site_lot}, {p.town}"])181 p = P[2 % len(P)]; add("exact_copy", f"What is the calibration constant filed for {p.code}? Copy it exactly.", p.decimal_value, "exact", [f"{p.code}:decimal"])182 p = P[3 % len(P)]; add("exact_copy", f"Who proposed {p.code}? Give the full name.", p.proposer, "exact", [f"{p.code}:proposer"])183 p = P[4 % len(P)]; add("multi_hop", f"What was the initial budget, in {p.currency}, of the project proposed by {p.proposer}?", fmt_amt(p.proposed_amount), "numeric", [f"{p.code}:proposer", f"{p.code}:proposed_amount"], 0.0)184 p = P[5 % len(P)]; add("multi_hop", f"What is the supplier identifier of the project whose site is {p.site_lot} in {p.town}?", p.supplier_id, "exact", [f"{p.code}:site", f"{p.code}:supplier_id"])185 # negation: cancelled vs postponed186 cand = [x for x in projects if not x.cancelled and x.postponed_to is not None] or projects187 p = rng.choice(cand); add("negation", f"Was {p.code} cancelled? Answer yes or no.", "yes" if p.cancelled else "no", "yesno", [f"{p.code}:cancelled"])188 cand = [x for x in projects if x.cancelled] or projects189 p = rng.choice(cand); add("negation", f"Was {p.code} postponed? Answer yes or no.", "yes" if (p.postponed_to is not None) else "no", "yesno", [f"{p.code}:cancelled"])190 # revision / precedence191 cand = [x for x in projects if len(x.revisions) >= 1] or projects192 p = rng.choice(cand); add("revision", f"According to the ledger's precedence rule, what is the current budget of {p.code}, in {p.currency}? Give the number only.", fmt_amt(current_amount(p)), "numeric", [f"{p.code}:revision_{len(p.revisions)}" if p.revisions else f"{p.code}:proposed_amount"], 0.0)193 cand = [x for x in projects if len(x.revisions) >= 2] or cand194 p = rng.choice(cand); add("revision", f"What was the first revised budget of {p.code}, in {p.currency}? Give the number only.", fmt_amt(p.revisions[0][1]) if p.revisions else fmt_amt(p.proposed_amount), "numeric", [f"{p.code}:revision_1" if p.revisions else f"{p.code}:proposed_amount"], 0.0)195 # temporal order196 a, b = P[0], P[1 % len(P)]197 first = a.code if a.decision_date < b.decision_date else b.code198 add("temporal", f"Which decision was recorded first: the one for {a.code} or the one for {b.code}? Answer with the project code.", first, "exact", [f"{a.code}:decision", f"{b.code}:decision"])199 # aggregation: sum of initial budgets of all projects sharing the most common currency (>= 2 projects when possible)200 curs = {}201 for x in projects: curs.setdefault(x.currency, []).append(x)202 cur = max(curs, key=lambda k: len(curs[k]))203 grp = curs[cur]204 add("aggregation", f"What is the total of the initial budgets of all projects whose budget is denominated in {cur}? Give the number only.", fmt_amt(sum(x.proposed_amount for x in grp)), "numeric", [f"{x.code}:proposed_amount" for x in grp], 0.0)205 # unit conversion206 p = P[6 % len(P)]; qv, ub, us, fac = p.quantity207 add("unit_conversion", f"What is the rated capacity of {p.code} expressed in {us}? Give the number only.", _norm_num(qv * fac), "numeric", [f"{p.code}:quantity"], 1e-6)208 # unanswerable209 dn = distractor_names[0] if distractor_names else "Zorblat-3"210 add("unanswerable", f"What is the supplier identifier for {dn}? If the document does not say, answer exactly: unknown.", "unknown", "abstain", [])211 p = P[7 % len(P)]; add("unanswerable", f"What is the postal code of the head office that issued this ledger? If the document does not say, answer exactly: unknown.", "unknown", "abstain", [])212 return Q213214def generate_document(seed: int, n_projects: int, target_tokens: int, chars_per_token: float = 4.0) -> dict:215 rng = random.Random(seed)216 used = set()217 projects = [make_project(rng, used=used) for _ in range(n_projects)]218 # distractors: similar names (same base, different number) that appear only in a 'proposals under review' list without facts219 distractor_names = [f"{p.name}-{(int(p.code.split('-')[-1]) + rng.randint(1,5))}" for p in projects[: max(1, n_projects // 2)]]220 facts: list[Fact] = []221 blocks = [render_project(p, rng, facts, shuffle_sentences=False) for p in projects]222 header = (f"INTERNAL LEDGER MEMO — Capital Projects Register, issue {rng.randint(100,999)}. "223 "Where two entries conflict, the entry with the later date supersedes the earlier one. "224 "Projects listed under 'proposals under review' have no recorded attributes in this issue.")225 review = "Proposals under review (no attributes recorded): " + ", ".join(distractor_names) + "."226 # interleave filler to reach target length227 target_chars = int(target_tokens * chars_per_token)228 body = [header, review]229 cur = sum(len(b) for b in body) + sum(len(b) for b in blocks)230 n_fill = max(0, int((target_chars - cur) / 95))231 fill_between = max(1, n_fill // max(1, len(blocks)))232 for b in blocks:233 body.append(b)234 body.append(" ".join(rng.choice(FILLER) for _ in range(fill_between)))235 doc = "\n\n".join(body)236 questions = build_questions(projects, distractor_names, rng, doc, facts)237 for q in questions:238 for fid in q.supporting_fact_ids:239 assert any(f.fact_id == fid for f in facts), fid240 return {241 "document_id": f"syn_{seed:05d}_{target_tokens}",242 "seed": seed, "n_projects": n_projects, "target_tokens": target_tokens,243 "text": doc, "char_len": len(doc),244 "questions": [asdict(q) for q in questions],245 "facts": [asdict(f) for f in facts],246 "distractor_names": distractor_names,247 }248