SPB Git
spboucher.ai

spb/firc0001_uqo

Public

N8: rule-based baseline (snapshot rules.py) vs LLM on a stratified seeded sample of 60,000 ledger sections; rules-on-full vs rules-on-focused recall-loss bound

Items: FIRC0001-AI_C-19, THEO-09.
30,000 productive + 30,000 zero-mention sections (inverse-probability weights; DuckDB USING SAMPLE applied in subqueries), 200 s, 4 workers.
P(rules >= 1 | LLM >= 1) = 85.7% (10-K 97.4, 10-Q 94.4, 8-K 69.2); weighted P(LLM >= 1 | rules >= 1) = 32.5%; type overlap on co-detected sections 84.6% (M&A 96.8%);
rules on _focus(text) recover 26.2% of the rules-on-full-text mentions in over-budget sections (10-K MD&A 20.8%, 8-K 84.2%).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Nov 5, 2026) parent f7f4811 Browse files

10 changed files +300 −0

10 changed files
Unified Split
added code/audit/a08_rules_baseline.py +88 −0
@@ -0,0 +1,88 @@
1 +"""N8 - rule-based baseline (snapshot rules.py) vs the LLM output on a stratified seeded sample of ledger
2 +sections, plus rules-on-full vs rules-on-focused text (recall-loss bound, THEO-09). Read-only; CPU only.
3 +"""
4 +from __future__ import annotations
5 +import json, sys, time
6 +from pathlib import Path
7 +import multiprocessing as mp
8 +sys.path.insert(0, str(Path(__file__).resolve().parent))
9 +from common import *
10 +from spid.extract.rules import RuleExtractor
11 +from spid.extract.llm import _focus
12 +
13 +N_PROD = 30000; N_ZERO = 30000
14 +RX = RuleExtractor()
15 +
16 +def work(item):
17 + sid, name, text = item
18 + full = RX.extract(text=text, accession_number="a", cik="c", section_id=sid, section_name=name)
19 + foc = RX.extract(text=_focus(text), accession_number="a", cik="c", section_id=sid, section_name=name)
20 + return (sid, len(full), sorted({m.project_type for m in full}), len(foc), sorted({m.project_type for m in foc}), len(text),
21 + [(m.project_type, m.confidence, m.amount_usd, m.description[:200]) for m in full][:20])
22 +
23 +def main():
24 + r = Run("rules")
25 + con = connect(SPID_DB)
26 + tmin = con.execute("select min(extracted_at) from extracted_sections").fetchone()[0]
27 + con.execute(f"""CREATE TEMP TABLE led AS SELECT e.section_id, e.n_mentions, CASE WHEN e.extracted_at = TIMESTAMP '{tmin}' THEN 'backfilled_june' ELSE 'october' END run,
28 + f.form_type, m.section_name, m.word_count FROM extracted_sections e JOIN src_filings f ON f.accession_number = e.accession_number
29 + JOIN (SELECT section_id, section_name, word_count FROM src_sections UNION ALL SELECT section_id, section_name, word_count FROM spid_sections) m USING (section_id)""")
30 + N_prod_pop = con.execute("select count(*) from led where n_mentions>0").fetchone()[0]
31 + N_zero_pop = con.execute("select count(*) from led where n_mentions=0").fetchone()[0]
32 + # DuckDB applies USING SAMPLE before WHERE: the stratum filter must sit in a subquery
33 + con.execute(f"CREATE TEMP TABLE samp AS SELECT *, 'productive' stratum, {N_prod_pop}/{N_PROD}.0 w FROM (SELECT * FROM led WHERE n_mentions>0) USING SAMPLE reservoir({N_PROD} ROWS) REPEATABLE ({SEED})")
34 + con.execute(f"INSERT INTO samp SELECT *, 'zero', {N_zero_pop}/{N_ZERO}.0 FROM (SELECT * FROM led WHERE n_mentions=0) USING SAMPLE reservoir({N_ZERO} ROWS) REPEATABLE ({SEED})")
35 + assert con.execute("select count(*) filter (where n_mentions>0), count(*) filter (where n_mentions=0) from samp").fetchone() == (N_PROD, N_ZERO)
36 + n_s = con.execute("select count(*) from samp").fetchone()[0]
37 + print(f"sample {n_s} sections (pop productive {N_prod_pop}, zero {N_zero_pop})", flush=True)
38 + cur = con.execute("""SELECT s.section_id, s.section_name, t.text FROM samp s JOIN (SELECT section_id, text FROM src_sections UNION ALL SELECT section_id, text FROM spid_sections) t USING (section_id)""")
39 + out = []; t0 = time.time(); done = 0
40 + ctx = mp.get_context("fork")
41 + with ctx.Pool(4) as pool:
42 + while True:
43 + rows = cur.fetchmany(2000)
44 + if not rows: break
45 + out.extend(pool.imap_unordered(work, rows, chunksize=25))
46 + done += len(rows)
47 + print(f" {done}/{n_s} sections, {time.time()-t0:.0f}s", flush=True)
48 + import pyarrow as pa, pyarrow.parquet as pq
49 + tbl = pa.table({"section_id": [o[0] for o in out], "rules_n": [o[1] for o in out], "rules_types": [o[2] for o in out], "rules_focus_n": [o[3] for o in out],
50 + "rules_focus_types": [o[4] for o in out], "nchar": [o[5] for o in out], "rules_mentions": [json.dumps(o[6]) for o in out]})
51 + pq.write_table(tbl, r.out / "rules_mentions.parquet") # stays on the node
52 + con.register("rt", tbl)
53 + con.execute("""CREATE TEMP TABLE lt AS SELECT section_id, list_distinct(list(project_type)) llm_types FROM project_mentions GROUP BY 1""")
54 + con.execute("""CREATE TEMP TABLE J AS SELECT s.*, r.rules_n, r.rules_types, r.rules_focus_n, r.rules_focus_types, r.nchar, coalesce(l.llm_types, []) llm_types,
55 + r.rules_n > 0 rules_pos, s.n_mentions > 0 llm_pos, len(list_intersect(r.rules_types, coalesce(l.llm_types, []))) > 0 type_overlap
56 + FROM samp s JOIN rt r USING (section_id) LEFT JOIN lt l USING (section_id)""")
57 + agg = """count(*) n_sampled, round(sum(w)) n_weighted,
58 + count(*) FILTER (WHERE llm_pos) llm_pos, count(*) FILTER (WHERE rules_pos) rules_pos,
59 + round(100.0*count(*) FILTER (WHERE llm_pos AND rules_pos)/nullif(count(*) FILTER (WHERE llm_pos),0),2) p_rules_given_llm,
60 + round(100.0*sum(w) FILTER (WHERE llm_pos AND rules_pos)/nullif(sum(w) FILTER (WHERE rules_pos),0),2) p_llm_given_rules_weighted,
61 + round(100.0*sum(w) FILTER (WHERE rules_pos)/sum(w),2) rules_pos_pct_weighted, round(100.0*sum(w) FILTER (WHERE llm_pos)/sum(w),2) llm_pos_pct_weighted,
62 + round(100.0*count(*) FILTER (WHERE llm_pos AND rules_pos AND type_overlap)/nullif(count(*) FILTER (WHERE llm_pos AND rules_pos),0),2) type_overlap_pct_codetected,
63 + round(100.0*sum(w) FILTER (WHERE llm_pos AND NOT rules_pos)/sum(w),2) llm_only_pct_w, round(100.0*sum(w) FILTER (WHERE rules_pos AND NOT llm_pos)/sum(w),2) rules_only_pct_w,
64 + round(100.0*sum(w) FILTER (WHERE llm_pos = rules_pos)/sum(w),2) agreement_pct_w,
65 + round(avg(rules_n) FILTER (WHERE llm_pos),3) rules_mentions_per_llm_pos_section, round(avg(rules_n) FILTER (WHERE NOT llm_pos),3) rules_mentions_per_llm_zero_section"""
66 + r.dump("agreement_by_form_item.csv", con, f"SELECT form_type, section_name item, {agg} FROM J GROUP BY 1,2 ORDER BY 1,2")
67 + r.dump("agreement_by_form.csv", con, f"SELECT form_type, {agg} FROM J GROUP BY 1 ORDER BY 1")
68 + r.dump("agreement_by_run.csv", con, f"SELECT run, {agg} FROM J GROUP BY 1 ORDER BY 1")
69 + r.dump("agreement_overall.csv", con, f"SELECT 'all' scope, {agg} FROM J")
70 + r.dump("type_agreement.csv", con, """
71 + WITH x AS (SELECT unnest(llm_types) t, rules_types FROM J WHERE llm_pos AND rules_pos)
72 + SELECT t llm_type, count(*) n, count(*) FILTER (WHERE list_contains(rules_types, t)) same_type_in_rules, round(100.0*count(*) FILTER (WHERE list_contains(rules_types, t))/count(*),2) pct FROM x GROUP BY 1 ORDER BY 2 DESC""")
73 + r.dump("focus_recall_loss.csv", con, """
74 + SELECT form_type, section_name item, count(*) FILTER (WHERE nchar > 9000) n_over_budget,
75 + sum(rules_n) FILTER (WHERE nchar > 9000) rules_full, sum(rules_focus_n) FILTER (WHERE nchar > 9000) rules_focused,
76 + round(100.0*(1 - sum(rules_focus_n) FILTER (WHERE nchar > 9000)*1.0/nullif(sum(rules_n) FILTER (WHERE nchar > 9000),0)),2) pct_rules_mentions_lost_to_focus,
77 + round(100.0*count(*) FILTER (WHERE nchar > 9000 AND rules_n > 0 AND rules_focus_n = 0)/nullif(count(*) FILTER (WHERE nchar > 9000 AND rules_n > 0),0),2) pct_sections_lost_entirely,
78 + round(100.0*sum(w) FILTER (WHERE nchar > 9000 AND rules_n > 0 AND rules_focus_n = 0)/nullif(sum(w) FILTER (WHERE nchar > 9000 AND rules_n > 0),0),2) pct_sections_lost_entirely_w
79 + FROM J GROUP BY 1,2 ORDER BY 1,2""")
80 + r.dump("focus_recall_loss_overall.csv", con, """
81 + SELECT 'over_9000_chars' scope, count(*) n, sum(rules_n) rules_full, sum(rules_focus_n) rules_focused, round(100.0*(1 - sum(rules_focus_n)*1.0/nullif(sum(rules_n),0)),2) pct_lost,
82 + round(100.0*sum(w*rules_n)/nullif(sum(w*rules_n) + 0,0),2) dummy FROM J WHERE nchar > 9000""")
83 + r.dump("disagreement_strata.csv", con, """SELECT form_type, section_name item, run, llm_pos, rules_pos, count(*) n_sampled, round(sum(w)) n_weighted FROM J GROUP BY 1,2,3,4,5 ORDER BY 1,2,3,4,5""")
84 + r.notes["sample"] = {"productive": N_PROD, "zero": N_ZERO, "pop_productive": N_prod_pop, "pop_zero": N_zero_pop, "seed": SEED}
85 + r.finish()
86 +
87 +if __name__ == "__main__":
88 + main()
added results/rules/agreement_by_form.csv +4 −0
@@ -0,0 +1,4 @@
1 +form_type,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2 +10-K,8890,141548.0,5393,7614,97.42,28.17,73.88,21.37,86.24,0.55,53.07,46.38,14.785,5.893
3 +10-Q,29399,577609.0,13658,18518,94.35,28.73,43.54,13.26,87.79,0.75,31.03,68.22,12.397,3.501
4 +8-K,21711,403941.0,10949,8984,69.18,48.64,21.62,15.2,77.94,4.68,11.1,84.21,1.331,0.198
added results/rules/agreement_by_form_item.csv +16 −0
@@ -0,0 +1,16 @@
1 +form_type,item,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2 +10-K,business,2434,28674.0,1861,2354,99.79,39.7,91.49,36.4,89.77,0.08,55.17,44.75,10.332,6.503
3 +10-K,full_body,81,1057.0,58,81,100.0,30.76,100.0,30.76,98.28,,69.24,30.76,42.948,40.565
4 +10-K,md_and_a,2931,28341.0,2477,2890,99.88,51.17,95.67,49.01,90.99,0.06,46.72,53.22,21.193,16.165
5 +10-K,properties,1452,28595.0,672,925,81.55,20.39,52.71,13.18,54.2,2.43,41.96,55.6,3.109,1.145
6 +10-K,quantitative_disclosures,912,27245.0,68,308,91.18,4.25,30.02,1.4,83.87,0.12,28.74,71.14,11.309,0.868
7 +10-K,risk_factors,1080,27636.0,257,1056,99.22,5.31,97.43,5.21,81.18,0.04,92.25,7.71,10.362,8.486
8 +10-Q,controls_and_procedures,3732,91568.0,1038,1130,91.71,48.51,12.02,6.36,86.76,0.53,6.19,93.29,2.021,0.351
9 +10-Q,financial_statements,5665,90162.0,3438,5552,99.77,22.17,96.24,21.38,91.08,0.05,74.91,25.04,13.171,11.539
10 +10-Q,full_document,13,256.0,6,13,100.0,13.12,100.0,13.12,50.0,,86.88,13.12,25.667,28.429
11 +10-Q,legal_proceedings,2209,67714.0,99,229,49.49,4.58,8.87,0.82,61.22,0.41,8.46,91.13,1.869,0.147
12 +10-Q,md_and_a,8838,92144.0,7214,8688,99.54,45.64,95.76,43.9,89.04,0.2,52.06,47.74,16.222,13.111
13 +10-Q,quantitative_disclosures,2674,82305.0,107,492,70.09,3.07,16.64,0.73,65.33,0.22,16.13,83.66,1.879,0.232
14 +10-Q,risk_factors,2712,72947.0,510,1099,90.98,11.41,31.27,3.92,89.87,0.35,27.71,71.94,4.982,2.269
15 +10-Q,unregistered_sales,3556,80512.0,1246,1315,58.59,18.02,28.21,8.68,64.52,3.59,23.13,73.28,1.471,0.465
16 +8-K,full_body,21711,403941.0,10949,8984,69.18,48.64,21.62,15.2,77.94,4.68,11.1,84.21,1.331,0.198
added results/rules/agreement_by_run.csv +3 −0
@@ -0,0 +1,3 @@
1 +run,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2 +backfilled_june,8370,153998.0,4287,4561,79.75,34.53,36.05,15.61,81.37,3.16,23.6,73.24,6.361,2.055
3 +october,51630,969100.0,25713,30555,86.71,32.24,40.02,14.88,85.07,1.98,27.12,70.9,9.192,2.68
added results/rules/agreement_overall.csv +2 −0
@@ -0,0 +1,2 @@
1 +scope,n_sampled,n_weighted,llm_pos,rules_pos,p_rules_given_llm,p_llm_given_rules_weighted,rules_pos_pct_weighted,llm_pos_pct_weighted,type_overlap_pct_codetected,llm_only_pct_w,rules_only_pct_w,agreement_pct_w,rules_mentions_per_llm_pos_section,rules_mentions_per_llm_zero_section
2 +all,60000,1123098.0,30000,35116,85.72,32.52,39.48,14.98,84.57,2.14,26.64,71.22,8.788,2.595
added results/rules/disagreement_strata.csv +103 −0
@@ -0,0 +1,103 @@
1 +form_type,item,run,llm_pos,rules_pos,n_sampled,n_weighted
2 +10-K,business,backfilled_june,False,False,2,64.0
3 +10-K,business,backfilled_june,False,True,108,3438.0
4 +10-K,business,backfilled_june,True,True,395,2215.0
5 +10-K,business,october,False,False,74,2355.0
6 +10-K,business,october,False,True,389,12381.0
7 +10-K,business,october,True,False,4,22.0
8 +10-K,business,october,True,True,1462,8198.0
9 +10-K,full_body,backfilled_june,False,True,23,732.0
10 +10-K,full_body,backfilled_june,True,True,58,325.0
11 +10-K,md_and_a,backfilled_june,False,False,4,127.0
12 +10-K,md_and_a,backfilled_june,False,True,82,2610.0
13 +10-K,md_and_a,backfilled_june,True,False,3,17.0
14 +10-K,md_and_a,backfilled_june,True,True,401,2249.0
15 +10-K,md_and_a,october,False,False,34,1082.0
16 +10-K,md_and_a,october,False,True,334,10631.0
17 +10-K,md_and_a,october,True,True,2073,11625.0
18 +10-K,properties,backfilled_june,False,False,65,2069.0
19 +10-K,properties,backfilled_june,False,True,80,2546.0
20 +10-K,properties,backfilled_june,True,False,13,73.0
21 +10-K,properties,backfilled_june,True,True,124,695.0
22 +10-K,properties,october,False,False,338,10758.0
23 +10-K,properties,october,False,True,297,9453.0
24 +10-K,properties,october,True,False,111,622.0
25 +10-K,properties,october,True,True,424,2378.0
26 +10-K,quantitative_disclosures,backfilled_june,False,False,82,2610.0
27 +10-K,quantitative_disclosures,backfilled_june,False,True,53,1687.0
28 +10-K,quantitative_disclosures,backfilled_june,True,True,14,79.0
29 +10-K,quantitative_disclosures,october,False,False,516,16424.0
30 +10-K,quantitative_disclosures,october,False,True,193,6143.0
31 +10-K,quantitative_disclosures,october,True,False,6,34.0
32 +10-K,quantitative_disclosures,october,True,True,48,269.0
33 +10-K,risk_factors,backfilled_june,False,False,3,95.0
34 +10-K,risk_factors,backfilled_june,False,True,161,5124.0
35 +10-K,risk_factors,backfilled_june,True,True,40,224.0
36 +10-K,risk_factors,october,False,False,19,605.0
37 +10-K,risk_factors,october,False,True,640,20371.0
38 +10-K,risk_factors,october,True,False,2,11.0
39 +10-K,risk_factors,october,True,True,215,1206.0
40 +10-Q,controls_and_procedures,backfilled_june,False,False,88,2801.0
41 +10-Q,controls_and_procedures,backfilled_june,False,True,12,382.0
42 +10-Q,controls_and_procedures,backfilled_june,True,False,3,17.0
43 +10-Q,controls_and_procedures,backfilled_june,True,True,48,269.0
44 +10-Q,controls_and_procedures,october,False,False,2428,77281.0
45 +10-Q,controls_and_procedures,october,False,True,166,5284.0
46 +10-Q,controls_and_procedures,october,True,False,83,465.0
47 +10-Q,controls_and_procedures,october,True,True,904,5069.0
48 +10-Q,financial_statements,backfilled_june,False,False,5,159.0
49 +10-Q,financial_statements,backfilled_june,False,True,89,2833.0
50 +10-Q,financial_statements,backfilled_june,True,True,127,712.0
51 +10-Q,financial_statements,october,False,False,100,3183.0
52 +10-Q,financial_statements,october,False,True,2033,64708.0
53 +10-Q,financial_statements,october,True,False,8,45.0
54 +10-Q,financial_statements,october,True,True,3303,18522.0
55 +10-Q,full_document,october,False,True,7,223.0
56 +10-Q,full_document,october,True,True,6,34.0
57 +10-Q,legal_proceedings,backfilled_june,False,False,74,2355.0
58 +10-Q,legal_proceedings,backfilled_june,False,True,9,286.0
59 +10-Q,legal_proceedings,backfilled_june,True,False,1,6.0
60 +10-Q,legal_proceedings,october,False,False,1856,59075.0
61 +10-Q,legal_proceedings,october,False,True,171,5443.0
62 +10-Q,legal_proceedings,october,True,False,49,275.0
63 +10-Q,legal_proceedings,october,True,True,49,275.0
64 +10-Q,md_and_a,backfilled_june,False,False,6,191.0
65 +10-Q,md_and_a,backfilled_june,False,True,59,1878.0
66 +10-Q,md_and_a,backfilled_june,True,False,1,6.0
67 +10-Q,md_and_a,backfilled_june,True,True,256,1436.0
68 +10-Q,md_and_a,october,False,False,111,3533.0
69 +10-Q,md_and_a,october,False,True,1448,46088.0
70 +10-Q,md_and_a,october,True,False,32,179.0
71 +10-Q,md_and_a,october,True,True,6925,38833.0
72 +10-Q,quantitative_disclosures,backfilled_june,False,False,92,2928.0
73 +10-Q,quantitative_disclosures,backfilled_june,False,True,7,223.0
74 +10-Q,quantitative_disclosures,backfilled_june,True,False,4,22.0
75 +10-Q,quantitative_disclosures,backfilled_june,True,True,4,22.0
76 +10-Q,quantitative_disclosures,october,False,False,2058,65504.0
77 +10-Q,quantitative_disclosures,october,False,True,410,13050.0
78 +10-Q,quantitative_disclosures,october,True,False,28,157.0
79 +10-Q,quantitative_disclosures,october,True,True,71,398.0
80 +10-Q,risk_factors,backfilled_june,False,False,83,2642.0
81 +10-Q,risk_factors,backfilled_june,False,True,32,1019.0
82 +10-Q,risk_factors,backfilled_june,True,False,1,6.0
83 +10-Q,risk_factors,backfilled_june,True,True,20,112.0
84 +10-Q,risk_factors,october,False,False,1484,47234.0
85 +10-Q,risk_factors,october,False,True,603,19193.0
86 +10-Q,risk_factors,october,True,False,45,252.0
87 +10-Q,risk_factors,october,True,True,444,2490.0
88 +10-Q,unregistered_sales,backfilled_june,False,False,67,2133.0
89 +10-Q,unregistered_sales,backfilled_june,False,True,27,859.0
90 +10-Q,unregistered_sales,backfilled_june,True,False,18,101.0
91 +10-Q,unregistered_sales,backfilled_june,True,True,24,135.0
92 +10-Q,unregistered_sales,october,False,False,1658,52772.0
93 +10-Q,unregistered_sales,october,False,True,558,17761.0
94 +10-Q,unregistered_sales,october,True,False,498,2793.0
95 +10-Q,unregistered_sales,october,True,True,706,3959.0
96 +8-K,full_body,backfilled_june,False,False,2370,75435.0
97 +8-K,full_body,backfilled_june,False,True,400,12732.0
98 +8-K,full_body,backfilled_june,True,False,824,4621.0
99 +8-K,full_body,backfilled_june,True,True,1908,10699.0
100 +8-K,full_body,october,False,False,6983,222261.0
101 +8-K,full_body,october,False,True,1009,32115.0
102 +8-K,full_body,october,True,False,2550,14300.0
103 +8-K,full_body,october,True,True,5667,31779.0
added results/rules/focus_recall_loss.csv +16 −0
@@ -0,0 +1,16 @@
1 +form_type,item,n_over_budget,rules_full,rules_focused,pct_rules_mentions_lost_to_focus,pct_sections_lost_entirely,pct_sections_lost_entirely_w
2 +10-K,business,2226,22675,8375,63.07,1.44,2.51
3 +10-K,full_body,81,3424,344,89.95,6.17,12.57
4 +10-K,md_and_a,2866,59769,12445,79.18,2.1,3.71
5 +10-K,properties,141,1123,378,66.34,4.35,6.85
6 +10-K,quantitative_disclosures,122,1227,223,81.83,4.08,4.43
7 +10-K,risk_factors,1046,9612,2697,71.94,8.35,9.75
8 +10-Q,controls_and_procedures,111,1464,255,82.58,6.45,9.29
9 +10-Q,financial_statements,5549,70907,17249,75.67,6.17,10.19
10 +10-Q,full_document,13,353,17,95.18,46.15,54.02
11 +10-Q,legal_proceedings,76,265,123,53.58,1.79,2.53
12 +10-Q,md_and_a,8577,138076,34652,74.9,3.87,7.46
13 +10-Q,quantitative_disclosures,130,265,137,48.3,6.41,7.08
14 +10-Q,risk_factors,734,6845,1804,73.64,5.96,8.04
15 +10-Q,unregistered_sales,259,1385,375,72.92,5.33,9.49
16 +8-K,full_body,2997,6847,5767,15.77,1.78,2.6
added results/rules/focus_recall_loss_overall.csv +2 −0
@@ -0,0 +1,2 @@
1 +scope,n,rules_full,rules_focused,pct_lost,dummy
2 +over_9000_chars,24928,324237,84841,73.83,100.0
added results/rules/manifest.json +44 −0
@@ -0,0 +1,44 @@
1 +{
2 + "package": "rules",
3 + "script": "/Users/simon-pierreboucher/firc0001_review/scripts/a08_rules_baseline.py",
4 + "script_sha256": "7995daa858488f2d67cde688ab4d9c4d3468389c3f781a27c28c85edf8804a63",
5 + "started": "2026-11-05T14:45:00",
6 + "ended": "2026-11-05T14:48:20",
7 + "wall_seconds": 200.2,
8 + "node": "m3u96b",
9 + "python": "3.12.15",
10 + "duckdb": "2.0.0",
11 + "threads": 4,
12 + "memory_limit": "24GB",
13 + "seed": 20261105,
14 + "databases": {
15 + "/Users/simon-pierreboucher/UQO_RESEARCH_DB/project-db/data/db/spid.duckdb": {
16 + "bytes": 3155701760,
17 + "mtime": "2026-10-28T12:21:11.091389"
18 + },
19 + "/Users/simon-pierreboucher/UQO_RESEARCH_DB/data/duckdb/edgar_research.duckdb": {
20 + "bytes": 73500995584,
21 + "mtime": "2026-10-27T21:34:46.297884"
22 + }
23 + },
24 + "read_only": true,
25 + "row_counts": {
26 + "agreement_by_form_item.csv": 15,
27 + "agreement_by_form.csv": 3,
28 + "agreement_by_run.csv": 2,
29 + "agreement_overall.csv": 1,
30 + "type_agreement.csv": 21,
31 + "focus_recall_loss.csv": 15,
32 + "focus_recall_loss_overall.csv": 1,
33 + "disagreement_strata.csv": 102
34 + },
35 + "notes": {
36 + "sample": {
37 + "productive": 30000,
38 + "zero": 30000,
39 + "pop_productive": 168230,
40 + "pop_zero": 954868,
41 + "seed": 20261105
42 + }
43 + }
44 +}
\ No newline at end of file
added results/rules/type_agreement.csv +22 −0
@@ -0,0 +1,22 @@
1 +llm_type,n,same_type_in_rules,pct
2 +ma_integration,15735,15227,96.77
3 +geographic_expansion,2289,835,36.48
4 +capex_program,2241,1238,55.24
5 +partnership,2168,1151,53.09
6 +plant_construction,2097,854,40.72
7 +drug_pipeline,2021,1870,92.53
8 +cost_reduction,1694,1303,76.92
9 +rd_program,1597,1369,85.72
10 +product_launch,1488,731,49.13
11 +manufacturing_expansion,1199,1015,84.65
12 +infrastructure,1045,188,17.99
13 +technology_deployment,842,221,26.25
14 +digital_transformation,815,109,13.37
15 +erp_implementation,744,498,66.94
16 +sustainability,304,148,48.68
17 +supply_chain,169,119,70.41
18 +automation,114,67,58.77
19 +energy_transition,107,81,75.7
20 +data_center,105,91,86.67
21 +ai_initiative,84,61,72.62
22 +cloud_migration,83,36,43.37
23
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere