SPB Git
spboucher.ai

spb/firc0001_uqo

Public

N8: rule-based baseline (snapshot rules.py) vs LLM on a stratified seeded sample of 60,000 ledger sections; rules-on-full vs rules-on-focused recall-loss bound

Items: FIRC0001-AI_C-19, THEO-09.
30,000 productive + 30,000 zero-mention sections (inverse-probability weights; DuckDB USING SAMPLE applied in subqueries), 200 s, 4 workers.
P(rules >= 1 | LLM >= 1) = 85.7% (10-K 97.4, 10-Q 94.4, 8-K 69.2); weighted P(LLM >= 1 | rules >= 1) = 32.5%; type overlap on co-detected sections 84.6% (M&A 96.8%);
rules on _focus(text) recover 26.2% of the rules-on-full-text mentions in over-budget sections (10-K MD&A 20.8%, 8-K 84.2%).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 5, 2026) parent f7f4811 Browse files

10 changed files +300 −0

10 changed files
Unified Split
added code/audit/a08_rules_baseline.py +88 −0
@@ -0,0 +1,88 @@
1+"""N8 - rule-based baseline (snapshot rules.py) vs the LLM output on a stratified seeded sample of ledger
2+sections, plus rules-on-full vs rules-on-focused text (recall-loss bound, THEO-09). Read-only; CPU only.
3+"""
4+from __future__ import annotations
5+import json, sys, time
6+from pathlib import Path
7+import multiprocessing as mp
8+sys.path.insert(0, str(Path(__file__).resolve().parent))
9+from common import *
10+from spid.extract.rules import RuleExtractor
11+from spid.extract.llm import _focus
12+
13+N_PROD = 30000; N_ZERO = 30000
14+RX = RuleExtractor()
15+
16+def work(item):
17+ sid, name, text = item
18+ full = RX.extract(text=text, accession_number="a", cik="c", section_id=sid, section_name=name)
19+ foc = RX.extract(text=_focus(text), accession_number="a", cik="c", section_id=sid, section_name=name)
20+ return (sid, len(full), sorted({m.project_type for m in full}), len(foc), sorted({m.project_type for m in foc}), len(text),
21+ [(m.project_type, m.confidence, m.amount_usd, m.description[:200]) for m in full][:20])
22+
23+def main():
24+ r = Run("rules")
25+ con = connect(SPID_DB)
26+ tmin = con.execute("select min(extracted_at) from extracted_sections").fetchone()[0]
27+ con.execute(f"""CREATE TEMP TABLE led AS SELECT e.section_id, e.n_mentions, CASE WHEN e.extracted_at = TIMESTAMP '{tmin}' THEN 'backfilled_june' ELSE 'october' END run,
28+ f.form_type, m.section_name, m.word_count FROM extracted_sections e JOIN src_filings f ON f.accession_number = e.accession_number
29+ JOIN (SELECT section_id, section_name, word_count FROM src_sections UNION ALL SELECT section_id, section_name, word_count FROM spid_sections) m USING (section_id)""")
30+ N_prod_pop = con.execute("select count(*) from led where n_mentions>0").fetchone()[0]
31+ N_zero_pop = con.execute("select count(*) from led where n_mentions=0").fetchone()[0]
32+ # DuckDB applies USING SAMPLE before WHERE: the stratum filter must sit in a subquery
33+ con.execute(f"CREATE TEMP TABLE samp AS SELECT *, 'productive' stratum, {N_prod_pop}/{N_PROD}.0 w FROM (SELECT * FROM led WHERE n_mentions>0) USING SAMPLE reservoir({N_PROD} ROWS) REPEATABLE ({SEED})")
34+ con.execute(f"INSERT INTO samp SELECT *, 'zero', {N_zero_pop}/{N_ZERO}.0 FROM (SELECT * FROM led WHERE n_mentions=0) USING SAMPLE reservoir({N_ZERO} ROWS) REPEATABLE ({SEED})")
35+ assert con.execute("select count(*) filter (where n_mentions>0), count(*) filter (where n_mentions=0) from samp").fetchone() == (N_PROD, N_ZERO)
36+ n_s = con.execute("select count(*) from samp").fetchone()[0]
37+ print(f"sample {n_s} sections (pop productive {N_prod_pop}, zero {N_zero_pop})", flush=True)
38+ cur = con.execute("""SELECT s.section_id, s.section_name, t.text FROM samp s JOIN (SELECT section_id, text FROM src_sections UNION ALL SELECT section_id, text FROM spid_sections) t USING (section_id)""")
39+ out = []; t0 = time.time(); done = 0
40+ ctx = mp.get_context("fork")
41+ with ctx.Pool(4) as pool:
42+ while True:
43+ rows = cur.fetchmany(2000)
44+ if not rows: break
45+ out.extend(pool.imap_unordered(work, rows, chunksize=25))
46+ done += len(rows)
47+ print(f" {done}/{n_s} sections, {time.time()-t0:.0f}s", flush=True)
48+ import pyarrow as pa, pyarrow.parquet as pq
49+ tbl = pa.table({"section_id": [o[0] for o in out], "rules_n": [o[1] for o in out], "rules_types": [o[2] for o in out], "rules_focus_n": [o[3] for o in out],
50+ "rules_focus_types": [o[4] for o in out], "nchar": [o[5] for o in out], "rules_mentions": [json.dumps(o[6]) for o in out]})
51+ pq.write_table(tbl, r.out / "rules_mentions.parquet") # stays on the node
52+ con.register("rt", tbl)
53+ con.execute("""CREATE TEMP TABLE lt AS SELECT section_id, list_distinct(list(project_type)) llm_types FROM project_mentions GROUP BY 1""")
54+ con.execute("""CREATE TEMP TABLE J AS SELECT s.*, r.rules_n, r.rules_types, r.rules_focus_n, r.rules_focus_types, r.nchar, coalesce(l.llm_types, []) llm_types,
55+ r.rules_n > 0 rules_pos, s.n_mentions > 0 llm_pos, len(list_intersect(r.rules_types, coalesce(l.llm_types, []))) > 0 type_overlap
56+ FROM samp s JOIN rt r USING (section_id) LEFT JOIN lt l USING (section_id)""")
57+ agg = """count(*) n_sampled, round(sum(w)) n_weighted,
58+ count(*) FILTER (WHERE llm_pos) llm_pos, count(*) FILTER (WHERE rules_pos) rules_pos,
59+ round(100.0*count(*) FILTER (WHERE llm_pos AND rules_pos)/nullif(count(*) FILTER (WHERE llm_pos),0),2) p_rules_given_llm,
60+ round(100.0*sum(w) FILTER (WHERE llm_pos AND rules_pos)/nullif(sum(w) FILTER (WHERE rules_pos),0),2) p_llm_given_rules_weighted,
61+ round(100.0*sum(w) FILTER (WHERE rules_pos)/sum(w),2) rules_pos_pct_weighted, round(100.0*sum(w) FILTER (WHERE llm_pos)/sum(w),2) llm_pos_pct_weighted,
62+ round(100.0*count(*) FILTER (WHERE llm_pos AND rules_pos AND type_overlap)/nullif(count(*) FILTER (WHERE llm_pos AND rules_pos),0),2) type_overlap_pct_codetected,
63+ round(100.0*sum(w) FILTER (WHERE llm_pos AND NOT rules_pos)/sum(w),2) llm_only_pct_w, round(100.0*sum(w) FILTER (WHERE rules_pos AND NOT llm_pos)/sum(w),2) rules_only_pct_w,
64+ round(100.0*sum(w) FILTER (WHERE llm_pos = rules_pos)/sum(w),2) agreement_pct_w,
65+ round(avg(rules_n) FILTER (WHERE llm_pos),3) rules_mentions_per_llm_pos_section, round(avg(rules_n) FILTER (WHERE NOT llm_pos),3) rules_mentions_per_llm_zero_section"""
66+ r.dump("agreement_by_form_item.csv", con, f"SELECT form_type, section_name item, {agg} FROM J GROUP BY 1,2 ORDER BY 1,2")
67+ r.dump("agreement_by_form.csv", con, f"SELECT form_type, {agg} FROM J GROUP BY 1 ORDER BY 1")
68+ r.dump("agreement_by_run.csv", con, f"SELECT run, {agg} FROM J GROUP BY 1 ORDER BY 1")
69+ r.dump("agreement_overall.csv", con, f"SELECT 'all' scope, {agg} FROM J")
70+ r.dump("type_agreement.csv", con, """
71+ WITH x AS (SELECT unnest(llm_types) t, rules_types FROM J WHERE llm_pos AND rules_pos)
72+ SELECT t llm_type, count(*) n, count(*) FILTER (WHERE list_contains(rules_types, t)) same_type_in_rules, round(100.0*count(*) FILTER (WHERE list_contains(rules_types, t))/count(*),2) pct FROM x GROUP BY 1 ORDER BY 2 DESC""")
73+ r.dump("focus_recall_loss.csv", con, """
74+ SELECT form_type, section_name item, count(*) FILTER (WHERE nchar > 9000) n_over_budget,
75+ sum(rules_n) FILTER (WHERE nchar > 9000) rules_full, sum(rules_focus_n) FILTER (WHERE nchar > 9000) rules_focused,
76+ round(100.0*(1 - sum(rules_focus_n) FILTER (WHERE nchar > 9000)*1.0/nullif(sum(rules_n) FILTER (WHERE nchar > 9000),0)),2) pct_rules_mentions_lost_to_focus,
77+ round(100.0*count(*) FILTER (WHERE nchar > 9000 AND rules_n > 0 AND rules_focus_n = 0)/nullif(count(*) FILTER (WHERE nchar > 9000 AND rules_n > 0),0),2) pct_sections_lost_entirely,
78+ round(100.0*sum(w) FILTER (WHERE nchar > 9000 AND rules_n > 0 AND rules_focus_n = 0)/nullif(sum(w) FILTER (WHERE nchar > 9000 AND rules_n > 0),0),2) pct_sections_lost_entirely_w
79+ FROM J GROUP BY 1,2 ORDER BY 1,2""")
80+ r.dump("focus_recall_loss_overall.csv", con, """
81+ SELECT 'over_9000_chars' scope, count(*) n, sum(rules_n) rules_full, sum(rules_focus_n) rules_focused, round(100.0*(1 - sum(rules_focus_n)*1.0/nullif(sum(rules_n),0)),2) pct_lost,
82+ round(100.0*sum(w*rules_n)/nullif(sum(w*rules_n) + 0,0),2) dummy FROM J WHERE nchar > 9000""")
83+ r.dump("disagreement_strata.csv", con, """SELECT form_type, section_name item, run, llm_pos, rules_pos, count(*) n_sampled, round(sum(w)) n_weighted FROM J GROUP BY 1,2,3,4,5 ORDER BY 1,2,3,4,5""")
84+ r.notes["sample"] = {"productive": N_PROD, "zero": N_ZERO, "pop_productive": N_prod_pop, "pop_zero": N_zero_pop, "seed": SEED}
85+ r.finish()
86+
87+if __name__ == "__main__":
88+ main()
added results/rules/agreement_by_form.csv +4 −0
@@ -0,0 +1,4 @@
1+form_type,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2+10-K,8890,141548.0,5393,7614,97.42,28.17,73.88,21.37,86.24,0.55,53.07,46.38,14.785,5.893
3+10-Q,29399,577609.0,13658,18518,94.35,28.73,43.54,13.26,87.79,0.75,31.03,68.22,12.397,3.501
4+8-K,21711,403941.0,10949,8984,69.18,48.64,21.62,15.2,77.94,4.68,11.1,84.21,1.331,0.198
added results/rules/agreement_by_form_item.csv +16 −0
@@ -0,0 +1,16 @@
1+form_type,item,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2+10-K,business,2434,28674.0,1861,2354,99.79,39.7,91.49,36.4,89.77,0.08,55.17,44.75,10.332,6.503
3+10-K,full_body,81,1057.0,58,81,100.0,30.76,100.0,30.76,98.28,,69.24,30.76,42.948,40.565
4+10-K,md_and_a,2931,28341.0,2477,2890,99.88,51.17,95.67,49.01,90.99,0.06,46.72,53.22,21.193,16.165
5+10-K,properties,1452,28595.0,672,925,81.55,20.39,52.71,13.18,54.2,2.43,41.96,55.6,3.109,1.145
6+10-K,quantitative_disclosures,912,27245.0,68,308,91.18,4.25,30.02,1.4,83.87,0.12,28.74,71.14,11.309,0.868
7+10-K,risk_factors,1080,27636.0,257,1056,99.22,5.31,97.43,5.21,81.18,0.04,92.25,7.71,10.362,8.486
8+10-Q,controls_and_procedures,3732,91568.0,1038,1130,91.71,48.51,12.02,6.36,86.76,0.53,6.19,93.29,2.021,0.351
9+10-Q,financial_statements,5665,90162.0,3438,5552,99.77,22.17,96.24,21.38,91.08,0.05,74.91,25.04,13.171,11.539
10+10-Q,full_document,13,256.0,6,13,100.0,13.12,100.0,13.12,50.0,,86.88,13.12,25.667,28.429
11+10-Q,legal_proceedings,2209,67714.0,99,229,49.49,4.58,8.87,0.82,61.22,0.41,8.46,91.13,1.869,0.147
12+10-Q,md_and_a,8838,92144.0,7214,8688,99.54,45.64,95.76,43.9,89.04,0.2,52.06,47.74,16.222,13.111
13+10-Q,quantitative_disclosures,2674,82305.0,107,492,70.09,3.07,16.64,0.73,65.33,0.22,16.13,83.66,1.879,0.232
14+10-Q,risk_factors,2712,72947.0,510,1099,90.98,11.41,31.27,3.92,89.87,0.35,27.71,71.94,4.982,2.269
15+10-Q,unregistered_sales,3556,80512.0,1246,1315,58.59,18.02,28.21,8.68,64.52,3.59,23.13,73.28,1.471,0.465
16+8-K,full_body,21711,403941.0,10949,8984,69.18,48.64,21.62,15.2,77.94,4.68,11.1,84.21,1.331,0.198
added results/rules/agreement_by_run.csv +3 −0
@@ -0,0 +1,3 @@
1+run,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2+backfilled_june,8370,153998.0,4287,4561,79.75,34.53,36.05,15.61,81.37,3.16,23.6,73.24,6.361,2.055
3+october,51630,969100.0,25713,30555,86.71,32.24,40.02,14.88,85.07,1.98,27.12,70.9,9.192,2.68
added results/rules/agreement_overall.csv +2 −0
@@ -0,0 +1,2 @@
1+scope,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2+all,60000,1123098.0,30000,35116,85.72,32.52,39.48,14.98,84.57,2.14,26.64,71.22,8.788,2.595
added results/rules/disagreement_strata.csv +103 −0
@@ -0,0 +1,103 @@
1+form_type,item,run,llm_pos,rules_pos,n_sampled,n_weighted
2+10-K,business,backfilled_june,False,False,2,64.0
3+10-K,business,backfilled_june,False,True,108,3438.0
4+10-K,business,backfilled_june,True,True,395,2215.0
5+10-K,business,october,False,False,74,2355.0
6+10-K,business,october,False,True,389,12381.0
7+10-K,business,october,True,False,4,22.0
8+10-K,business,october,True,True,1462,8198.0
9+10-K,full_body,backfilled_june,False,True,23,732.0
10+10-K,full_body,backfilled_june,True,True,58,325.0
11+10-K,md_and_a,backfilled_june,False,False,4,127.0
12+10-K,md_and_a,backfilled_june,False,True,82,2610.0
13+10-K,md_and_a,backfilled_june,True,False,3,17.0
14+10-K,md_and_a,backfilled_june,True,True,401,2249.0
15+10-K,md_and_a,october,False,False,34,1082.0
16+10-K,md_and_a,october,False,True,334,10631.0
17+10-K,md_and_a,october,True,True,2073,11625.0
18+10-K,properties,backfilled_june,False,False,65,2069.0
19+10-K,properties,backfilled_june,False,True,80,2546.0
20+10-K,properties,backfilled_june,True,False,13,73.0
21+10-K,properties,backfilled_june,True,True,124,695.0
22+10-K,properties,october,False,False,338,10758.0
23+10-K,properties,october,False,True,297,9453.0
24+10-K,properties,october,True,False,111,622.0
25+10-K,properties,october,True,True,424,2378.0
26+10-K,quantitative_disclosures,backfilled_june,False,False,82,2610.0
27+10-K,quantitative_disclosures,backfilled_june,False,True,53,1687.0
28+10-K,quantitative_disclosures,backfilled_june,True,True,14,79.0
29+10-K,quantitative_disclosures,october,False,False,516,16424.0
30+10-K,quantitative_disclosures,october,False,True,193,6143.0
31+10-K,quantitative_disclosures,october,True,False,6,34.0
32+10-K,quantitative_disclosures,october,True,True,48,269.0
33+10-K,risk_factors,backfilled_june,False,False,3,95.0
34+10-K,risk_factors,backfilled_june,False,True,161,5124.0
35+10-K,risk_factors,backfilled_june,True,True,40,224.0
36+10-K,risk_factors,october,False,False,19,605.0
37+10-K,risk_factors,october,False,True,640,20371.0
38+10-K,risk_factors,october,True,False,2,11.0
39+10-K,risk_factors,october,True,True,215,1206.0
40+10-Q,controls_and_procedures,backfilled_june,False,False,88,2801.0
41+10-Q,controls_and_procedures,backfilled_june,False,True,12,382.0
42+10-Q,controls_and_procedures,backfilled_june,True,False,3,17.0
43+10-Q,controls_and_procedures,backfilled_june,True,True,48,269.0
44+10-Q,controls_and_procedures,october,False,False,2428,77281.0
45+10-Q,controls_and_procedures,october,False,True,166,5284.0
46+10-Q,controls_and_procedures,october,True,False,83,465.0
47+10-Q,controls_and_procedures,october,True,True,904,5069.0
48+10-Q,financial_statements,backfilled_june,False,False,5,159.0
49+10-Q,financial_statements,backfilled_june,False,True,89,2833.0
50+10-Q,financial_statements,backfilled_june,True,True,127,712.0
51+10-Q,financial_statements,october,False,False,100,3183.0
52+10-Q,financial_statements,october,False,True,2033,64708.0
53+10-Q,financial_statements,october,True,False,8,45.0
54+10-Q,financial_statements,october,True,True,3303,18522.0
55+10-Q,full_document,october,False,True,7,223.0
56+10-Q,full_document,october,True,True,6,34.0
57+10-Q,legal_proceedings,backfilled_june,False,False,74,2355.0
58+10-Q,legal_proceedings,backfilled_june,False,True,9,286.0
59+10-Q,legal_proceedings,backfilled_june,True,False,1,6.0
60+10-Q,legal_proceedings,october,False,False,1856,59075.0
61+10-Q,legal_proceedings,october,False,True,171,5443.0
62+10-Q,legal_proceedings,october,True,False,49,275.0
63+10-Q,legal_proceedings,october,True,True,49,275.0
64+10-Q,md_and_a,backfilled_june,False,False,6,191.0
65+10-Q,md_and_a,backfilled_june,False,True,59,1878.0
66+10-Q,md_and_a,backfilled_june,True,False,1,6.0
67+10-Q,md_and_a,backfilled_june,True,True,256,1436.0
68+10-Q,md_and_a,october,False,False,111,3533.0
69+10-Q,md_and_a,october,False,True,1448,46088.0
70+10-Q,md_and_a,october,True,False,32,179.0
71+10-Q,md_and_a,october,True,True,6925,38833.0
72+10-Q,quantitative_disclosures,backfilled_june,False,False,92,2928.0
73+10-Q,quantitative_disclosures,backfilled_june,False,True,7,223.0
74+10-Q,quantitative_disclosures,backfilled_june,True,False,4,22.0
75+10-Q,quantitative_disclosures,backfilled_june,True,True,4,22.0
76+10-Q,quantitative_disclosures,october,False,False,2058,65504.0
77+10-Q,quantitative_disclosures,october,False,True,410,13050.0
78+10-Q,quantitative_disclosures,october,True,False,28,157.0
79+10-Q,quantitative_disclosures,october,True,True,71,398.0
80+10-Q,risk_factors,backfilled_june,False,False,83,2642.0
81+10-Q,risk_factors,backfilled_june,False,True,32,1019.0
82+10-Q,risk_factors,backfilled_june,True,False,1,6.0
83+10-Q,risk_factors,backfilled_june,True,True,20,112.0
84+10-Q,risk_factors,october,False,False,1484,47234.0
85+10-Q,risk_factors,october,False,True,603,19193.0
86+10-Q,risk_factors,october,True,False,45,252.0
87+10-Q,risk_factors,october,True,True,444,2490.0
88+10-Q,unregistered_sales,backfilled_june,False,False,67,2133.0
89+10-Q,unregistered_sales,backfilled_june,False,True,27,859.0
90+10-Q,unregistered_sales,backfilled_june,True,False,18,101.0
91+10-Q,unregistered_sales,backfilled_june,True,True,24,135.0
92+10-Q,unregistered_sales,october,False,False,1658,52772.0
93+10-Q,unregistered_sales,october,False,True,558,17761.0
94+10-Q,unregistered_sales,october,True,False,498,2793.0
95+10-Q,unregistered_sales,october,True,True,706,3959.0
96+8-K,full_body,backfilled_june,False,False,2370,75435.0
97+8-K,full_body,backfilled_june,False,True,400,12732.0
98+8-K,full_body,backfilled_june,True,False,824,4621.0
99+8-K,full_body,backfilled_june,True,True,1908,10699.0
100+8-K,full_body,october,False,False,6983,222261.0
101+8-K,full_body,october,False,True,1009,32115.0
102+8-K,full_body,october,True,False,2550,14300.0
103+8-K,full_body,october,True,True,5667,31779.0
added results/rules/focus_recall_loss.csv +16 −0
@@ -0,0 +1,16 @@
1+form_type,item,n_over_budget,rules_full,rules_focused,pct_rules_mentions_lost_to_focus,pct_sections_lost_entirely,pct_sections_lost_entirely_w
2+10-K,business,2226,22675,8375,63.07,1.44,2.51
3+10-K,full_body,81,3424,344,89.95,6.17,12.57
4+10-K,md_and_a,2866,59769,12445,79.18,2.1,3.71
5+10-K,properties,141,1123,378,66.34,4.35,6.85
6+10-K,quantitative_disclosures,122,1227,223,81.83,4.08,4.43
7+10-K,risk_factors,1046,9612,2697,71.94,8.35,9.75
8+10-Q,controls_and_procedures,111,1464,255,82.58,6.45,9.29
9+10-Q,financial_statements,5549,70907,17249,75.67,6.17,10.19
10+10-Q,full_document,13,353,17,95.18,46.15,54.02
11+10-Q,legal_proceedings,76,265,123,53.58,1.79,2.53
12+10-Q,md_and_a,8577,138076,34652,74.9,3.87,7.46
13+10-Q,quantitative_disclosures,130,265,137,48.3,6.41,7.08
14+10-Q,risk_factors,734,6845,1804,73.64,5.96,8.04
15+10-Q,unregistered_sales,259,1385,375,72.92,5.33,9.49
16+8-K,full_body,2997,6847,5767,15.77,1.78,2.6
added results/rules/focus_recall_loss_overall.csv +2 −0
@@ -0,0 +1,2 @@
1+scope,n,rules_full,rules_focused,pct_lost,dummy
2+over_9000_chars,24928,324237,84841,73.83,100.0
added results/rules/manifest.json +44 −0
@@ -0,0 +1,44 @@
1+{
2+ "package": "rules",
3+ "script": "/Users/simon-pierreboucher/firc0001_review/scripts/a08_rules_baseline.py",
4+ "script_sha256": "7995daa858488f2d67cde688ab4d9c4d3468389c3f781a27c28c85edf8804a63",
5+ "started": "2026-11-05T14:45:00",
6+ "ended": "2026-11-05T14:48:20",
7+ "wall_seconds": 200.2,
8+ "node": "m3u96b",
9+ "python": "3.12.15",
10+ "duckdb": "2.0.0",
11+ "threads": 4,
12+ "memory_limit": "24GB",
13+ "seed": 20261105,
14+ "databases": {
15+ "/Users/simon-pierreboucher/UQO_RESEARCH_DB/project-db/data/db/spid.duckdb": {
16+ "bytes": 3155701760,
17+ "mtime": "2026-10-28T12:21:11.091389"
18+ },
19+ "/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/duckdb/edgar_research.duckdb": {
20+ "bytes": 73500995584,
21+ "mtime": "2026-10-27T21:34:46.297884"
22+ }
23+ },
24+ "read_only": true,
25+ "row_counts": {
26+ "agreement_by_form_item.csv": 15,
27+ "agreement_by_form.csv": 3,
28+ "agreement_by_run.csv": 2,
29+ "agreement_overall.csv": 1,
30+ "type_agreement.csv": 21,
31+ "focus_recall_loss.csv": 15,
32+ "focus_recall_loss_overall.csv": 1,
33+ "disagreement_strata.csv": 102
34+ },
35+ "notes": {
36+ "sample": {
37+ "productive": 30000,
38+ "zero": 30000,
39+ "pop_productive": 168230,
40+ "pop_zero": 954868,
41+ "seed": 20261105
42+ }
43+ }
44+}
\ No newline at end of file
added results/rules/type_agreement.csv +22 −0
@@ -0,0 +1,22 @@
1+llm_type,n,same_type_in_rules,pct
2+ma_integration,15735,15227,96.77
3+geographic_expansion,2289,835,36.48
4+capex_program,2241,1238,55.24
5+partnership,2168,1151,53.09
6+plant_construction,2097,854,40.72
7+drug_pipeline,2021,1870,92.53
8+cost_reduction,1694,1303,76.92
9+rd_program,1597,1369,85.72
10+product_launch,1488,731,49.13
11+manufacturing_expansion,1199,1015,84.65
12+infrastructure,1045,188,17.99
13+technology_deployment,842,221,26.25
14+digital_transformation,815,109,13.37
15+erp_implementation,744,498,66.94
16+sustainability,304,148,48.68
17+supply_chain,169,119,70.41
18+automation,114,67,58.77
19+energy_transition,107,81,75.7
20+data_center,105,91,86.67
21+ai_initiative,84,61,72.62
22+cloud_migration,83,36,43.37
23
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere