N11: annotation pilot sampling frames (seed 20261105), weights, design and protocol; no labels produced
Items: FIRC0001-ECON-04, EDIT-01 (c), ADVE-01 (i) (frames only; metrics = author task AD-2). 192 productive strata (item group x run x truncation x confidence x category), n = 1,677 with floor 10 (nested core subsample 510) + 385 zero-mention sections with inclusion probability ~ gate x log length; dev/holdout 1,058 / 1,004; CI half-width +-4.4 pp (precision), +-2.4 pp (false-negative rate at 5%). Full-text frames and the sealed model output stay on M3U96b (~/firc0001_review/frames/); worktree CSVs truncate text to 2,000 chars. docs/annotation_protocol.md (new). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 changed files +20,001 −0
added
code/audit/a11_annotation_frame.py
+148 −0
@@ -0,0 +1,148 @@ | ||
| 1 | +"""N11 - sampling frames and design for the human annotation pilot (cluster A, ECON-04). No labels produced. | |
| 2 | + | |
| 3 | +Strata: item group x run x truncation bin (N5 shown fraction) x productive x confidence bin x category group; | |
| 4 | +productive sections n = 385 allocated proportionally with a floor of 10 and 2x oversampling of the 0.45-0.7 bin; | |
| 5 | +zero-mention sections drawn with inclusion probabilities proportional to (gate pass, log length) with | |
| 6 | +inverse-probability weights; seeded (20261105); development / holdout split. Frames with full text and | |
| 7 | +_focus(text) stay on the node (~/firc0001_review/frames/); the worktree copy truncates text to 2,000 chars and | |
| 8 | +carries no model output (the model's mentions are sealed in a separate file). | |
| 9 | +""" | |
| 10 | +from __future__ import annotations | |
| 11 | +import json, sys, math | |
| 12 | +from pathlib import Path | |
| 13 | +import numpy as np | |
| 14 | +sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| 15 | +from common import * | |
| 16 | +from spid.extract.llm import _focus | |
| 17 | + | |
| 18 | +N_PROD = 385; N_ZERO = 385; FLOOR = 10 | |
| 19 | +rng = np.random.default_rng(SEED) | |
| 20 | +r = Run("annotation_frame") | |
| 21 | +FR = WORK / "frames"; FR.mkdir(exist_ok=True) | |
| 22 | +con = connect(SPID_DB) | |
| 23 | +tmin = con.execute("select min(extracted_at) from extracted_sections").fetchone()[0] | |
| 24 | +feat = RESULTS / "focus" / "section_features.parquet" | |
| 25 | +con.execute(f"CREATE TEMP TABLE ft AS SELECT section_id, nchar, gate_py, shown_chars*1.0/nchar shown_frac FROM read_parquet('{feat}')") | |
| 26 | +con.execute(f"""CREATE TEMP TABLE S AS | |
| 27 | +SELECT e.section_id, e.accession_number, e.n_mentions, CASE WHEN e.extracted_at = TIMESTAMP '{tmin}' THEN 'june' ELSE 'october' END run, | |
| 28 | + f.form_type, f.cik, f.ticker, f.company_name, f.filing_date, m.section_name item, m.word_count, ft.nchar, ft.gate_py gate, ft.shown_frac, | |
| 29 | + CASE WHEN f.form_type='10-K' AND m.section_name='md_and_a' THEN '10-K md_and_a' WHEN f.form_type='10-K' THEN '10-K other' | |
| 30 | + WHEN f.form_type='10-Q' AND m.section_name='md_and_a' THEN '10-Q md_and_a' WHEN f.form_type='10-Q' THEN '10-Q other' ELSE '8-K' END item_group, | |
| 31 | + CASE WHEN ft.shown_frac >= 0.9 THEN 'shown>=0.9' WHEN ft.shown_frac >= 0.25 THEN 'shown0.25-0.9' ELSE 'shown<0.25' END trunc_bin, | |
| 32 | + pm.max_conf, CASE WHEN pm.max_conf IS NULL THEN 'none' WHEN pm.max_conf <= 0.7 THEN 'conf0.45-0.7' ELSE 'conf>0.7' END conf_bin, | |
| 33 | + CASE WHEN pm.types IS NULL THEN 'none' WHEN list_contains(pm.types, 'ma_integration') THEN 'ma_integration' | |
| 34 | + WHEN len(list_intersect(pm.types, ['capex_program','plant_construction','manufacturing_expansion','infrastructure','data_center'])) > 0 THEN 'capex-type' | |
| 35 | + WHEN len(list_intersect(pm.types, ['technology_deployment','ai_initiative','digital_transformation','erp_implementation','cloud_migration','automation'])) > 0 THEN 'technology-type' | |
| 36 | + ELSE 'other' END cat_group | |
| 37 | +FROM extracted_sections e JOIN src_filings f ON f.accession_number = e.accession_number | |
| 38 | +JOIN (SELECT section_id, section_name, word_count FROM src_sections UNION ALL SELECT section_id, section_name, word_count FROM spid_sections) m USING (section_id) | |
| 39 | +LEFT JOIN ft USING (section_id) | |
| 40 | +LEFT JOIN (SELECT section_id, max(confidence) max_conf, list_distinct(list(project_type)) types FROM project_mentions GROUP BY 1) pm USING (section_id)""") | |
| 41 | +rules = RESULTS / "rules" / "rules_mentions.parquet" | |
| 42 | +if rules.exists(): | |
| 43 | + con.execute(f"CREATE TEMP TABLE rl AS SELECT section_id, rules_n FROM read_parquet('{rules}')") | |
| 44 | + con.execute("ALTER TABLE S ADD COLUMN rules_n INTEGER"); con.execute("UPDATE S SET rules_n = rl.rules_n FROM rl WHERE S.section_id = rl.section_id") | |
| 45 | +else: | |
| 46 | + con.execute("ALTER TABLE S ADD COLUMN rules_n INTEGER") | |
| 47 | +# ---- productive strata and allocation --------------------------------------------------------------------------- | |
| 48 | +strata = con.execute("""SELECT item_group, run, trunc_bin, conf_bin, cat_group, count(*) N FROM S WHERE n_mentions > 0 GROUP BY 1,2,3,4,5 ORDER BY 1,2,3,4,5""").fetchall() | |
| 49 | +Ntot = sum(s[5] for s in strata) | |
| 50 | +alloc = [] | |
| 51 | +for ig, rn, tb, cb, cg, N in strata: | |
| 52 | + share = N / Ntot * (2.0 if cb == "conf0.45-0.7" else 1.0) | |
| 53 | + n = max(FLOOR, round(N_PROD * share)); n = min(n, N) | |
| 54 | + alloc.append((ig, rn, tb, cb, cg, N, n, N / n)) | |
| 55 | +n_prod_actual = sum(a[6] for a in alloc) | |
| 56 | +r.write_csv("design_productive_strata.csv", ["item_group", "run", "trunc_bin", "conf_bin", "cat_group", "N_population", "n_sampled", "weight"], alloc) | |
| 57 | +# ---- draw productive -------------------------------------------------------------------------------------------- | |
| 58 | +rows = [] | |
| 59 | +for ig, rn, tb, cb, cg, N, n, w in alloc: | |
| 60 | + ids = con.execute("SELECT section_id FROM S WHERE n_mentions > 0 AND item_group=? AND run=? AND trunc_bin=? AND conf_bin=? AND cat_group=? ORDER BY section_id", [ig, rn, tb, cb, cg]).fetchall() | |
| 61 | + pick = rng.choice(len(ids), size=n, replace=False) | |
| 62 | + for i in pick: | |
| 63 | + rows.append((ids[i][0], "productive", f"{ig}|{rn}|{tb}|{cb}|{cg}", w)) | |
| 64 | +# ---- draw zero-mention with inclusion probabilities ∝ (gate, log length) ------------------------------------------- | |
| 65 | +zero = con.execute("SELECT section_id, gate, nchar, item_group, run, trunc_bin FROM S WHERE n_mentions = 0 ORDER BY section_id").fetchall() | |
| 66 | +score = np.array([(2.0 if g else 1.0) * (1.0 + math.log10(max(30, nc))) for _, g, nc, *_ in zero]) | |
| 67 | +p = score / score.sum() | |
| 68 | +pick = rng.choice(len(zero), size=N_ZERO, replace=False, p=p) | |
| 69 | +pi = N_ZERO * p # first-order inclusion probability (approximation, sampling fraction 4e-4) | |
| 70 | +for i in pick: | |
| 71 | + sid, g, nc, ig, rn, tb = zero[i] | |
| 72 | + rows.append((sid, "zero", f"{ig}|{rn}|{tb}|gate={'pass' if g else 'reject'}", 1.0 / pi[i])) | |
| 73 | +con.execute("CREATE TEMP TABLE pick AS SELECT * FROM (VALUES " + ",".join(["(?,?,?,?)"] * len(rows)) + ") t(section_id, frame, stratum, weight)", [x for row in rows for x in row]) | |
| 74 | +# dev / holdout split, stratified by frame x stratum, seeded | |
| 75 | +sel = con.execute("SELECT p.section_id, p.frame, p.stratum, p.weight, s.accession_number, s.form_type, s.item, s.cik, s.ticker, s.company_name, s.filing_date, s.run, s.word_count, s.nchar, s.gate, s.shown_frac, s.n_mentions, s.max_conf, s.cat_group, s.rules_n FROM pick p JOIN S s USING (section_id) ORDER BY p.frame, p.stratum, p.section_id").fetchall() | |
| 76 | +split = {} | |
| 77 | +by = {} | |
| 78 | +for x in sel: by.setdefault((x[1], x[2]), []).append(x[0]) | |
| 79 | +for k, ids in by.items(): | |
| 80 | + perm = rng.permutation(len(ids)) | |
| 81 | + for j, i in enumerate(perm): | |
| 82 | + split[ids[i]] = "dev" if j % 2 == 0 else "holdout" | |
| 83 | +cols = ["section_id", "frame", "stratum", "weight", "accession_number", "form_type", "item", "cik", "ticker", "company_name", "filing_date", "run", "word_count", "nchar", "gate_pass", "shown_frac", "n_mentions_model", "max_conf_model", "cat_group_model", "rules_n"] | |
| 84 | +# nested core subsample (first annotation pass): ~385 productive sections, proportional to the stratum population, min 1, seeded | |
| 85 | +core = set() | |
| 86 | +prod_by = {} | |
| 87 | +for x in sel: | |
| 88 | + if x[1] == "productive": prod_by.setdefault(x[2], []).append(x[0]) | |
| 89 | +for st, ids in prod_by.items(): | |
| 90 | + N_h = next(a[5] for a in alloc if f"{a[0]}|{a[1]}|{a[2]}|{a[3]}|{a[4]}" == st) | |
| 91 | + k = max(1, min(len(ids), round(385 * N_h / Ntot))) | |
| 92 | + for i in rng.choice(len(ids), size=k, replace=False): core.add(ids[i]) | |
| 93 | +for x in sel: | |
| 94 | + if x[1] == "zero": core.add(x[0]) | |
| 95 | +r.write_csv("weights.csv", ["section_id", "frame", "stratum", "weight", "split", "core"], [(x[0], x[1], x[2], round(x[3], 4), split[x[0]], x[0] in core) for x in sel]) | |
| 96 | +# worktree copies: no model output, text truncated to 2,000 chars | |
| 97 | +con.execute("CREATE TEMP TABLE txt AS SELECT t.section_id, t.text FROM (SELECT section_id, text FROM src_sections UNION ALL SELECT section_id, text FROM spid_sections) t JOIN pick USING (section_id)") | |
| 98 | +texts = dict(con.execute("SELECT section_id, text FROM txt").fetchall()) | |
| 99 | +pub_cols = ["section_id", "split", "core", "frame", "stratum", "weight", "accession_number", "form_type", "item", "cik", "ticker", "company_name", "filing_date", "run", "word_count", "nchar", "gate_pass", "shown_frac", "text_2000"] | |
| 100 | +pub = [(x[0], split[x[0]], x[0] in core, x[1], x[2], round(x[3], 4), x[4], x[5], x[6], x[7], x[8], x[9], str(x[10]), x[11], x[12], x[13], x[14], round(x[15], 4) if x[15] is not None else None, (texts.get(x[0]) or "")[:2000]) for x in sel] | |
| 101 | +r.write_csv("frame_dev.csv", pub_cols, [p_ for p_ in pub if p_[1] == "dev"]) | |
| 102 | +r.write_csv("frame_holdout.csv", pub_cols, [p_ for p_ in pub if p_[1] == "holdout"]) | |
| 103 | +# node-only full frames + sealed model output | |
| 104 | +men = {} | |
| 105 | +for row in con.execute("SELECT section_id, mention_id, project_name, project_type, description, status, amount_usd, locations, partners, years, confidence FROM project_mentions WHERE section_id IN (SELECT section_id FROM pick)").fetchall(): | |
| 106 | + men.setdefault(row[0], []).append(dict(zip(["mention_id", "project_name", "project_type", "description", "status", "amount_usd", "locations", "partners", "years", "confidence"], row[1:]))) | |
| 107 | +for name in ("dev", "holdout"): | |
| 108 | + with open(FR / f"frame_{name}.jsonl", "w") as fh: | |
| 109 | + for x in sel: | |
| 110 | + if split[x[0]] != name: continue | |
| 111 | + t = texts.get(x[0]) or "" | |
| 112 | + fh.write(json.dumps({"section_id": x[0], "frame": x[1], "stratum": x[2], "weight": x[3], "accession_number": x[4], "form_type": x[5], "item": x[6], "cik": x[7], "ticker": x[8], | |
| 113 | + "company_name": x[9], "filing_date": str(x[10]), "run": x[11], "word_count": x[12], "nchar": x[13], "gate_pass": x[14], "shown_frac": x[15], | |
| 114 | + "text": t, "focus_text_seen_by_model": ("SECTION: " + str(x[6]) + "\n\n" + _focus(t))[:14000]}, default=str) + "\n") | |
| 115 | +with open(FR / "model_output_SEALED.jsonl", "w") as fh: | |
| 116 | + for x in sel: | |
| 117 | + fh.write(json.dumps({"section_id": x[0], "split": split[x[0]], "n_mentions": x[16], "mentions": men.get(x[0], [])}, default=str) + "\n") | |
| 118 | +import shutil | |
| 119 | +shutil.copy(r.out / "weights.csv", FR / "weights.csv") | |
| 120 | +# ---- design numbers: CI widths and MDE --------------------------------------------------------------------------------- | |
| 121 | +w_prod = np.array([x[3] for x in sel if x[1] == "productive"]); w_zero = np.array([x[3] for x in sel if x[1] == "zero"]) | |
| 122 | +deff = lambda w: float(1 + (w.std() / w.mean()) ** 2) | |
| 123 | +D = {"seed": SEED, "n_productive": int(len(w_prod)), "n_zero": int(len(w_zero)), "n_strata_productive": len(alloc), "floor": FLOOR, "oversample_conf_0.45_0.7": 2.0, | |
| 124 | + "population_productive": Ntot, "population_zero": len(zero), "deff_productive": deff(w_prod), "deff_zero": deff(w_zero), | |
| 125 | + "half_width_95_precision_p0.5": round(1.96 * math.sqrt(0.25 / len(w_prod) * deff(w_prod)), 4), | |
| 126 | + "half_width_95_precision_p0.8": round(1.96 * math.sqrt(0.16 / len(w_prod) * deff(w_prod)), 4), | |
| 127 | + "half_width_95_zero_fn_rate": {str(p0): round(1.96 * math.sqrt(p0 * (1 - p0) / len(w_zero) * deff(w_zero)), 4) for p0 in (0.02, 0.05, 0.10)}, | |
| 128 | + "mde_gate_recall_note": "false-negative rate among zero-mention sections estimated with inverse-probability weights; MDE (80% power, two-sided 5%) for a difference between gate-pass and gate-reject strata of size n1, n2: 2.8*sqrt(p(1-p)*(1/n1+1/n2)*deff)", | |
| 129 | + "zero_frame_gate_pass": int(sum(1 for x in sel if x[1] == "zero" and x[14])), "zero_frame_gate_reject": int(sum(1 for x in sel if x[1] == "zero" and not x[14])), | |
| 130 | + "split": {"dev": sum(1 for v in split.values() if v == "dev"), "holdout": sum(1 for v in split.values() if v == "holdout")}, | |
| 131 | + "core_subsample": {"productive": sum(1 for x in sel if x[1] == "productive" and x[0] in core), "zero": sum(1 for x in sel if x[1] == "zero" and x[0] in core), | |
| 132 | + "note": "nested first-pass subsample drawn proportionally to the stratum population within the frame (weights = N_h / n_h,core); the full frame keeps the floor-of-10 design"}, | |
| 133 | + "rules_column_available": rules.exists(), | |
| 134 | + "files_on_node": [str(FR / "frame_dev.jsonl"), str(FR / "frame_holdout.jsonl"), str(FR / "model_output_SEALED.jsonl"), str(FR / "weights.csv")]} | |
| 135 | +p0 = 0.05 | |
| 136 | +for n1, n2 in ((D["zero_frame_gate_pass"], D["zero_frame_gate_reject"]),): | |
| 137 | + D["mde_gate_recall_diff_at_p0.05"] = round(2.8 * math.sqrt(p0 * (1 - p0) * (1 / max(1, n1) + 1 / max(1, n2)) * deff(w_zero)), 4) | |
| 138 | +r.write_json("design.json", D) | |
| 139 | +r.dump("frame_composition.csv", con, "SELECT frame, stratum, count(*) n, round(sum(weight)) weighted_n FROM pick GROUP BY 1,2 ORDER BY 1,2") | |
| 140 | +with open(FR / "README.md", "w") as fh: | |
| 141 | + fh.write(f"""# Annotation pilot frames (FIRC-0001, N11) — seed {SEED}, drawn {dt.date.today()} | |
| 142 | + | |
| 143 | +frame_dev.jsonl / frame_holdout.jsonl: one JSON object per section with the full section text and `focus_text_seen_by_model` | |
| 144 | +(exactly what the extractor sent, llm.py:125-126). Annotators label the TEXT; they never see model_output_SEALED.jsonl | |
| 145 | +until the labels are frozen. weights.csv carries the inverse-probability weight and the split. design.json documents n, | |
| 146 | +strata, weights, CI half-widths and the MDE. Protocol: docs/annotation_protocol.md in the paper repository. | |
| 147 | +""") | |
| 148 | +r.finish() | |
added
docs/annotation_protocol.md
+32 −0
@@ -0,0 +1,32 @@ | ||
| 1 | +# Annotation protocol for the SPID human-validation pilot (Stage D, package N11) | |
| 2 | + | |
| 3 | +*Status: protocol and sampling frames delivered (2026-11-05); no label has been produced. The metrics of this protocol are the author task AD-2 (`improvements/FIRC-0001/authorization_requests.md`). Table 10 gate (d) therefore reads "not met — protocol revised, frame drawn, annotation pending".* | |
| 4 | + | |
| 5 | +## 1. Objects and population | |
| 6 | + | |
| 7 | +- **Unit:** one ledgered section (`extracted_sections`, 1,123,098 rows), i.e. exactly the text block the extractor received a verdict for. Two populations: **productive** sections (`n_mentions > 0`, N = 168,230) and **zero-mention** sections (N = 954,868; 419,361 of them never reached the model because `_PROJECT_GATE` rejected them). | |
| 8 | +- **What the annotator sees:** the full section text and, separately, `focus_text_seen_by_model` — the string actually sent (`"SECTION: <item>\n\n" + _focus(text)`, cut at 14,000 characters; `llm.py:125-126`). Both are in `~/firc0001_review/frames/frame_{dev,holdout}.jsonl` (M3U96b). The worktree copies (`results/annotation_frame/frame_{dev,holdout}.csv`) truncate the text to 2,000 characters and are for inspection only. | |
| 9 | +- **What the annotator must not see until labels are frozen:** the model's mentions (`frames/model_output_SEALED.jsonl`). | |
| 10 | + | |
| 11 | +## 2. Sampling design (seed 20261105; `code/audit/a11_annotation_frame.py`) | |
| 12 | + | |
| 13 | +- **Productive frame.** Strata = item group (10-K MD&A / other 10-K / 10-Q MD&A / other 10-Q / 8-K) × run (June backfilled / October) × truncation bin from N5 (shown fraction ≥ 0.9 / 0.25–0.9 / < 0.25) × section confidence bin (max mention confidence ≤ 0.7 / > 0.7) × category group (ma_integration / capex-type / technology-type / other). Allocation: proportional to the stratum population with a **floor of 10** and a **2× oversampling of the ≤ 0.7 bin**; strata smaller than the floor are taken whole. Result: 192 non-empty strata, **n = 1,677** (the floor dominates); weights w_h = N_h / n_h (`design_productive_strata.csv`, `weights.csv`). A nested **core subsample of ≈ 385 sections** (column `core`, proportional within the drawn strata, min 1 per stratum) is the recommended first pass; its weights are N_h / n_h,core. | |
| 14 | +- **Zero-mention frame.** n = 385 drawn without replacement with inclusion probability ∝ (2 if gate passed else 1) × (1 + log10 characters), so that long gate-passing sections (where a missed project is most likely) are over-represented; 285 gate-pass / 100 gate-reject in the draw; inverse-probability weights 1/π_i stored. | |
| 15 | +- **Split.** Every stratum is split at random into *development* (guide refinement, disagreement resolution) and *holdout* (reported metrics): 1,058 / 1,004 sections. | |
| 16 | +- **Precision of the design** (`design.json`): design effect 3.4 (productive) / 1.2 (zero); 95 % half-width ± 4.4 pp for a section-level precision near 0.5 (± 3.5 pp near 0.8) on the full productive frame; ± 2.4 pp for a false-negative rate near 5 % on the zero frame; MDE (80 % power, two-sided 5 %) of a gate-pass vs gate-reject difference in the false-negative rate ≈ 7.8 pp at p = 0.05. | |
| 17 | + | |
| 18 | +## 3. Labels | |
| 19 | + | |
| 20 | +Per section (both frames): **S1** does the text describe at least one concrete initiative under the construct of §5.1 (yes / no / unsure); **S2** list of initiatives found (free text, ≤ 10). Per model mention (productive frame, after unsealing): **M1** the mention corresponds to an initiative in the text (correct / partly / wrong / construct violation — e.g. share repurchase, financing instrument, dividend); **M2** `project_type` correct under the 21-category taxonomy (Appendix A); **M3** each structured field (years, locations, partners, amount) is literally supported by the text (the mechanical rates of N2 are the comparison point); **M4** status correct. Per project (linkage audit, optional): **L1** two mentions resolved to the same `project_id` describe the same initiative (same / different / unsure). | |
| 21 | + | |
| 22 | +## 4. Procedure | |
| 23 | + | |
| 24 | +1. Two annotators, trained on 40 development sections, label independently; the guide is revised once after the development set and frozen before the holdout. | |
| 25 | +2. Blinding: S1–S2 are labelled before `model_output_SEALED.jsonl` is opened; M1–M4 afterwards. The order of sections is shuffled; the stratum, run and confidence are hidden. | |
| 26 | +3. Agreement: Krippendorff's α (nominal) for S1, M1, M2; prevalence-adjusted bias-adjusted κ (PABAK) reported next to Cohen's κ because S1 is rare in the zero frame. Disagreements adjudicated by a third reader; adjudicated labels are the reference. | |
| 27 | +4. Estimators (holdout only, inverse-probability weighted): section-level precision P(S1 = yes | productive), mention-level precision P(M1 = correct), category accuracy P(M2 | M1 = correct), field-support rates (compare with N2), false-negative rate P(S1 = yes | zero) overall and by gate status (gate recall bound), construct-violation share by item (compare with N6). 95 % intervals by stratified bootstrap of sections (1,000 replicates). | |
| 28 | +5. Reporting: Table 10 gate (d) becomes "met / not met" on the holdout numbers; §8.2 prints n per stratum, α / PABAK, and every estimate with its interval; the development-set numbers are reported separately and never pooled. | |
| 29 | + | |
| 30 | +## 5. Files | |
| 31 | + | |
| 32 | +`results/annotation_frame/{design.json, design_productive_strata.csv, frame_composition.csv, weights.csv, frame_dev.csv, frame_holdout.csv}` (worktree); `~/firc0001_review/frames/{frame_dev.jsonl, frame_holdout.jsonl, model_output_SEALED.jsonl, weights.csv, README.md}` (M3U96b, full text). The X1 stability run (AR-LLM-1), if authorised, should reuse the same 500-section stratified subset of this frame so that determinism and human agreement are measured on identical sections. | |
added
results/annotation_frame/design.json
+39 −0
@@ -0,0 +1,39 @@ | ||
| 1 | +{ | |
| 2 | + "seed": 20261105, | |
| 3 | + "n_productive": 1677, | |
| 4 | + "n_zero": 385, | |
| 5 | + "n_strata_productive": 192, | |
| 6 | + "floor": 10, | |
| 7 | + "oversample_conf_0.45_0.7": 2.0, | |
| 8 | + "population_productive": 168230, | |
| 9 | + "population_zero": 954868, | |
| 10 | + "deff_productive": 3.4131363613604115, | |
| 11 | + "deff_zero": 1.2130239948560366, | |
| 12 | + "half_width_95_precision_p0.5": 0.0442, | |
| 13 | + "half_width_95_precision_p0.8": 0.0354, | |
| 14 | + "half_width_95_zero_fn_rate": { | |
| 15 | + "0.02": 0.0154, | |
| 16 | + "0.05": 0.024, | |
| 17 | + "0.1": 0.033 | |
| 18 | + }, | |
| 19 | + "mde_gate_recall_note": "false-negative rate among zero-mention sections estimated with inverse-probability weights; MDE (80% power, two-sided 5%) for a difference between gate-pass and gate-reject strata of size n1, n2: 2.8*sqrt(p(1-p)*(1/n1+1/n2)*deff)", | |
| 20 | + "zero_frame_gate_pass": 285, | |
| 21 | + "zero_frame_gate_reject": 100, | |
| 22 | + "split": { | |
| 23 | + "dev": 1058, | |
| 24 | + "holdout": 1004 | |
| 25 | + }, | |
| 26 | + "core_subsample": { | |
| 27 | + "productive": 510, | |
| 28 | + "zero": 385, | |
| 29 | + "note": "nested first-pass subsample drawn proportionally to the stratum population within the frame (weights = N_h / n_h,core); the full frame keeps the floor-of-10 design" | |
| 30 | + }, | |
| 31 | + "rules_column_available": true, | |
| 32 | + "files_on_node": [ | |
| 33 | + "/Users/simon-pierreboucher/firc0001_review/frames/frame_dev.jsonl", | |
| 34 | + "/Users/simon-pierreboucher/firc0001_review/frames/frame_holdout.jsonl", | |
| 35 | + "/Users/simon-pierreboucher/firc0001_review/frames/model_output_SEALED.jsonl", | |
| 36 | + "/Users/simon-pierreboucher/firc0001_review/frames/weights.csv" | |
| 37 | + ], | |
| 38 | + "mde_gate_recall_diff_at_p0.05": 0.0781 | |
| 39 | +} | |
| \ No newline at end of file | ||
added
results/annotation_frame/design_productive_strata.csv
+193 −0
@@ -0,0 +1,193 @@ | ||
| 1 | +item_group,run,trunc_bin,conf_bin,cat_group,N_population,n_sampled,weight | |
| 2 | +10-K md_and_a,june,shown0.25-0.9,conf>0.7,capex-type,10,10,1.0 | |
| 3 | +10-K md_and_a,june,shown0.25-0.9,conf>0.7,ma_integration,30,10,3.0 | |
| 4 | +10-K md_and_a,june,shown0.25-0.9,conf>0.7,other,5,5,1.0 | |
| 5 | +10-K md_and_a,june,shown0.25-0.9,conf>0.7,technology-type,5,5,1.0 | |
| 6 | +10-K md_and_a,june,shown<0.25,conf0.45-0.7,capex-type,4,4,1.0 | |
| 7 | +10-K md_and_a,june,shown<0.25,conf0.45-0.7,ma_integration,2,2,1.0 | |
| 8 | +10-K md_and_a,june,shown<0.25,conf0.45-0.7,other,2,2,1.0 | |
| 9 | +10-K md_and_a,june,shown<0.25,conf>0.7,capex-type,471,10,47.1 | |
| 10 | +10-K md_and_a,june,shown<0.25,conf>0.7,ma_integration,1502,10,150.2 | |
| 11 | +10-K md_and_a,june,shown<0.25,conf>0.7,other,275,10,27.5 | |
| 12 | +10-K md_and_a,june,shown<0.25,conf>0.7,technology-type,167,10,16.7 | |
| 13 | +10-K md_and_a,june,shown>=0.9,conf>0.7,capex-type,14,10,1.4 | |
| 14 | +10-K md_and_a,june,shown>=0.9,conf>0.7,ma_integration,11,10,1.1 | |
| 15 | +10-K md_and_a,june,shown>=0.9,conf>0.7,other,4,4,1.0 | |
| 16 | +10-K md_and_a,june,shown>=0.9,conf>0.7,technology-type,3,3,1.0 | |
| 17 | +10-K md_and_a,october,shown0.25-0.9,conf0.45-0.7,other,2,2,1.0 | |
| 18 | +10-K md_and_a,october,shown0.25-0.9,conf>0.7,capex-type,88,10,8.8 | |
| 19 | +10-K md_and_a,october,shown0.25-0.9,conf>0.7,ma_integration,174,10,17.4 | |
| 20 | +10-K md_and_a,october,shown0.25-0.9,conf>0.7,other,63,10,6.3 | |
| 21 | +10-K md_and_a,october,shown0.25-0.9,conf>0.7,technology-type,19,10,1.9 | |
| 22 | +10-K md_and_a,october,shown<0.25,conf0.45-0.7,capex-type,10,10,1.0 | |
| 23 | +10-K md_and_a,october,shown<0.25,conf0.45-0.7,ma_integration,4,4,1.0 | |
| 24 | +10-K md_and_a,october,shown<0.25,conf0.45-0.7,other,10,10,1.0 | |
| 25 | +10-K md_and_a,october,shown<0.25,conf0.45-0.7,technology-type,2,2,1.0 | |
| 26 | +10-K md_and_a,october,shown<0.25,conf>0.7,capex-type,1833,10,183.3 | |
| 27 | +10-K md_and_a,october,shown<0.25,conf>0.7,ma_integration,6385,15,425.6666666666667 | |
| 28 | +10-K md_and_a,october,shown<0.25,conf>0.7,other,2405,10,240.5 | |
| 29 | +10-K md_and_a,october,shown<0.25,conf>0.7,technology-type,683,10,68.3 | |
| 30 | +10-K md_and_a,october,shown>=0.9,conf0.45-0.7,ma_integration,2,2,1.0 | |
| 31 | +10-K md_and_a,october,shown>=0.9,conf>0.7,capex-type,26,10,2.6 | |
| 32 | +10-K md_and_a,october,shown>=0.9,conf>0.7,ma_integration,50,10,5.0 | |
| 33 | +10-K md_and_a,october,shown>=0.9,conf>0.7,other,17,10,1.7 | |
| 34 | +10-K md_and_a,october,shown>=0.9,conf>0.7,technology-type,6,6,1.0 | |
| 35 | +10-K other,june,shown0.25-0.9,conf0.45-0.7,ma_integration,1,1,1.0 | |
| 36 | +10-K other,june,shown0.25-0.9,conf0.45-0.7,other,1,1,1.0 | |
| 37 | +10-K other,june,shown0.25-0.9,conf0.45-0.7,technology-type,2,2,1.0 | |
| 38 | +10-K other,june,shown0.25-0.9,conf>0.7,capex-type,72,10,7.2 | |
| 39 | +10-K other,june,shown0.25-0.9,conf>0.7,ma_integration,238,10,23.8 | |
| 40 | +10-K other,june,shown0.25-0.9,conf>0.7,other,71,10,7.1 | |
| 41 | +10-K other,june,shown0.25-0.9,conf>0.7,technology-type,36,10,3.6 | |
| 42 | +10-K other,june,shown<0.25,conf0.45-0.7,capex-type,5,5,1.0 | |
| 43 | +10-K other,june,shown<0.25,conf0.45-0.7,ma_integration,5,5,1.0 | |
| 44 | +10-K other,june,shown<0.25,conf0.45-0.7,other,5,5,1.0 | |
| 45 | +10-K other,june,shown<0.25,conf>0.7,capex-type,545,10,54.5 | |
| 46 | +10-K other,june,shown<0.25,conf>0.7,ma_integration,1252,10,125.2 | |
| 47 | +10-K other,june,shown<0.25,conf>0.7,other,329,10,32.9 | |
| 48 | +10-K other,june,shown<0.25,conf>0.7,technology-type,245,10,24.5 | |
| 49 | +10-K other,june,shown>=0.9,conf0.45-0.7,capex-type,3,3,1.0 | |
| 50 | +10-K other,june,shown>=0.9,conf0.45-0.7,other,9,9,1.0 | |
| 51 | +10-K other,june,shown>=0.9,conf>0.7,capex-type,613,10,61.3 | |
| 52 | +10-K other,june,shown>=0.9,conf>0.7,ma_integration,118,10,11.8 | |
| 53 | +10-K other,june,shown>=0.9,conf>0.7,other,135,10,13.5 | |
| 54 | +10-K other,june,shown>=0.9,conf>0.7,technology-type,14,10,1.4 | |
| 55 | +10-K other,october,shown0.25-0.9,conf0.45-0.7,capex-type,2,2,1.0 | |
| 56 | +10-K other,october,shown0.25-0.9,conf0.45-0.7,other,4,4,1.0 | |
| 57 | +10-K other,october,shown0.25-0.9,conf0.45-0.7,technology-type,1,1,1.0 | |
| 58 | +10-K other,october,shown0.25-0.9,conf>0.7,capex-type,360,10,36.0 | |
| 59 | +10-K other,october,shown0.25-0.9,conf>0.7,ma_integration,610,10,61.0 | |
| 60 | +10-K other,october,shown0.25-0.9,conf>0.7,other,199,10,19.9 | |
| 61 | +10-K other,october,shown0.25-0.9,conf>0.7,technology-type,143,10,14.3 | |
| 62 | +10-K other,october,shown<0.25,conf0.45-0.7,capex-type,13,10,1.3 | |
| 63 | +10-K other,october,shown<0.25,conf0.45-0.7,ma_integration,14,10,1.4 | |
| 64 | +10-K other,october,shown<0.25,conf0.45-0.7,other,8,8,1.0 | |
| 65 | +10-K other,october,shown<0.25,conf0.45-0.7,technology-type,4,4,1.0 | |
| 66 | +10-K other,october,shown<0.25,conf>0.7,capex-type,1604,10,160.4 | |
| 67 | +10-K other,october,shown<0.25,conf>0.7,ma_integration,4222,10,422.2 | |
| 68 | +10-K other,october,shown<0.25,conf>0.7,other,2173,10,217.3 | |
| 69 | +10-K other,october,shown<0.25,conf>0.7,technology-type,662,10,66.2 | |
| 70 | +10-K other,october,shown>=0.9,conf0.45-0.7,capex-type,12,10,1.2 | |
| 71 | +10-K other,october,shown>=0.9,conf0.45-0.7,ma_integration,2,2,1.0 | |
| 72 | +10-K other,october,shown>=0.9,conf0.45-0.7,other,19,10,1.9 | |
| 73 | +10-K other,october,shown>=0.9,conf>0.7,capex-type,1810,10,181.0 | |
| 74 | +10-K other,october,shown>=0.9,conf>0.7,ma_integration,417,10,41.7 | |
| 75 | +10-K other,october,shown>=0.9,conf>0.7,other,468,10,46.8 | |
| 76 | +10-K other,october,shown>=0.9,conf>0.7,technology-type,147,10,14.7 | |
| 77 | +10-Q md_and_a,june,shown0.25-0.9,conf>0.7,capex-type,27,10,2.7 | |
| 78 | +10-Q md_and_a,june,shown0.25-0.9,conf>0.7,ma_integration,46,10,4.6 | |
| 79 | +10-Q md_and_a,june,shown0.25-0.9,conf>0.7,other,20,10,2.0 | |
| 80 | +10-Q md_and_a,june,shown0.25-0.9,conf>0.7,technology-type,6,6,1.0 | |
| 81 | +10-Q md_and_a,june,shown<0.25,conf0.45-0.7,capex-type,2,2,1.0 | |
| 82 | +10-Q md_and_a,june,shown<0.25,conf0.45-0.7,ma_integration,6,6,1.0 | |
| 83 | +10-Q md_and_a,june,shown<0.25,conf0.45-0.7,technology-type,1,1,1.0 | |
| 84 | +10-Q md_and_a,june,shown<0.25,conf>0.7,capex-type,337,10,33.7 | |
| 85 | +10-Q md_and_a,june,shown<0.25,conf>0.7,ma_integration,873,10,87.3 | |
| 86 | +10-Q md_and_a,june,shown<0.25,conf>0.7,other,151,10,15.1 | |
| 87 | +10-Q md_and_a,june,shown<0.25,conf>0.7,technology-type,85,10,8.5 | |
| 88 | +10-Q md_and_a,june,shown>=0.9,conf>0.7,ma_integration,8,8,1.0 | |
| 89 | +10-Q md_and_a,june,shown>=0.9,conf>0.7,other,1,1,1.0 | |
| 90 | +10-Q md_and_a,october,shown0.25-0.9,conf0.45-0.7,capex-type,2,2,1.0 | |
| 91 | +10-Q md_and_a,october,shown0.25-0.9,conf0.45-0.7,ma_integration,4,4,1.0 | |
| 92 | +10-Q md_and_a,october,shown0.25-0.9,conf0.45-0.7,other,5,5,1.0 | |
| 93 | +10-Q md_and_a,october,shown0.25-0.9,conf>0.7,capex-type,820,10,82.0 | |
| 94 | +10-Q md_and_a,october,shown0.25-0.9,conf>0.7,ma_integration,1984,10,198.4 | |
| 95 | +10-Q md_and_a,october,shown0.25-0.9,conf>0.7,other,786,10,78.6 | |
| 96 | +10-Q md_and_a,october,shown0.25-0.9,conf>0.7,technology-type,202,10,20.2 | |
| 97 | +10-Q md_and_a,october,shown<0.25,conf0.45-0.7,capex-type,20,10,2.0 | |
| 98 | +10-Q md_and_a,october,shown<0.25,conf0.45-0.7,ma_integration,45,10,4.5 | |
| 99 | +10-Q md_and_a,october,shown<0.25,conf0.45-0.7,other,59,10,5.9 | |
| 100 | +10-Q md_and_a,october,shown<0.25,conf0.45-0.7,technology-type,2,2,1.0 | |
| 101 | +10-Q md_and_a,october,shown<0.25,conf>0.7,capex-type,6308,14,450.57142857142856 | |
| 102 | +10-Q md_and_a,october,shown<0.25,conf>0.7,ma_integration,17955,41,437.9268292682927 | |
| 103 | +10-Q md_and_a,october,shown<0.25,conf>0.7,other,7471,17,439.47058823529414 | |
| 104 | +10-Q md_and_a,october,shown<0.25,conf>0.7,technology-type,2025,10,202.5 | |
| 105 | +10-Q md_and_a,october,shown>=0.9,conf0.45-0.7,capex-type,2,2,1.0 | |
| 106 | +10-Q md_and_a,october,shown>=0.9,conf0.45-0.7,ma_integration,3,3,1.0 | |
| 107 | +10-Q md_and_a,october,shown>=0.9,conf>0.7,capex-type,161,10,16.1 | |
| 108 | +10-Q md_and_a,october,shown>=0.9,conf>0.7,ma_integration,282,10,28.2 | |
| 109 | +10-Q md_and_a,october,shown>=0.9,conf>0.7,other,74,10,7.4 | |
| 110 | +10-Q md_and_a,october,shown>=0.9,conf>0.7,technology-type,14,10,1.4 | |
| 111 | +10-Q other,june,shown0.25-0.9,conf0.45-0.7,ma_integration,1,1,1.0 | |
| 112 | +10-Q other,june,shown0.25-0.9,conf>0.7,capex-type,7,7,1.0 | |
| 113 | +10-Q other,june,shown0.25-0.9,conf>0.7,ma_integration,46,10,4.6 | |
| 114 | +10-Q other,june,shown0.25-0.9,conf>0.7,other,10,10,1.0 | |
| 115 | +10-Q other,june,shown0.25-0.9,conf>0.7,technology-type,3,3,1.0 | |
| 116 | +10-Q other,june,shown<0.25,conf>0.7,capex-type,44,10,4.4 | |
| 117 | +10-Q other,june,shown<0.25,conf>0.7,ma_integration,618,10,61.8 | |
| 118 | +10-Q other,june,shown<0.25,conf>0.7,other,71,10,7.1 | |
| 119 | +10-Q other,june,shown<0.25,conf>0.7,technology-type,8,8,1.0 | |
| 120 | +10-Q other,june,shown>=0.9,conf0.45-0.7,ma_integration,2,2,1.0 | |
| 121 | +10-Q other,june,shown>=0.9,conf0.45-0.7,other,1,1,1.0 | |
| 122 | +10-Q other,june,shown>=0.9,conf0.45-0.7,technology-type,2,2,1.0 | |
| 123 | +10-Q other,june,shown>=0.9,conf>0.7,capex-type,24,10,2.4 | |
| 124 | +10-Q other,june,shown>=0.9,conf>0.7,ma_integration,364,10,36.4 | |
| 125 | +10-Q other,june,shown>=0.9,conf>0.7,other,55,10,5.5 | |
| 126 | +10-Q other,june,shown>=0.9,conf>0.7,technology-type,150,10,15.0 | |
| 127 | +10-Q other,october,shown0.25-0.9,conf0.45-0.7,capex-type,7,7,1.0 | |
| 128 | +10-Q other,october,shown0.25-0.9,conf0.45-0.7,ma_integration,11,10,1.1 | |
| 129 | +10-Q other,october,shown0.25-0.9,conf0.45-0.7,other,14,10,1.4 | |
| 130 | +10-Q other,october,shown0.25-0.9,conf>0.7,capex-type,300,10,30.0 | |
| 131 | +10-Q other,october,shown0.25-0.9,conf>0.7,ma_integration,1102,10,110.2 | |
| 132 | +10-Q other,october,shown0.25-0.9,conf>0.7,other,306,10,30.6 | |
| 133 | +10-Q other,october,shown0.25-0.9,conf>0.7,technology-type,101,10,10.1 | |
| 134 | +10-Q other,october,shown<0.25,conf0.45-0.7,capex-type,18,10,1.8 | |
| 135 | +10-Q other,october,shown<0.25,conf0.45-0.7,ma_integration,14,10,1.4 | |
| 136 | +10-Q other,october,shown<0.25,conf0.45-0.7,other,36,10,3.6 | |
| 137 | +10-Q other,october,shown<0.25,conf0.45-0.7,technology-type,3,3,1.0 | |
| 138 | +10-Q other,october,shown<0.25,conf>0.7,capex-type,1628,10,162.8 | |
| 139 | +10-Q other,october,shown<0.25,conf>0.7,ma_integration,13740,31,443.2258064516129 | |
| 140 | +10-Q other,october,shown<0.25,conf>0.7,other,2980,10,298.0 | |
| 141 | +10-Q other,october,shown<0.25,conf>0.7,technology-type,302,10,30.2 | |
| 142 | +10-Q other,october,shown>=0.9,conf0.45-0.7,capex-type,22,10,2.2 | |
| 143 | +10-Q other,october,shown>=0.9,conf0.45-0.7,ma_integration,17,10,1.7 | |
| 144 | +10-Q other,october,shown>=0.9,conf0.45-0.7,other,16,10,1.6 | |
| 145 | +10-Q other,october,shown>=0.9,conf0.45-0.7,technology-type,3,3,1.0 | |
| 146 | +10-Q other,october,shown>=0.9,conf>0.7,capex-type,1890,10,189.0 | |
| 147 | +10-Q other,october,shown>=0.9,conf>0.7,ma_integration,7272,17,427.7647058823529 | |
| 148 | +10-Q other,october,shown>=0.9,conf>0.7,other,2196,10,219.6 | |
| 149 | +10-Q other,october,shown>=0.9,conf>0.7,technology-type,2534,10,253.4 | |
| 150 | +8-K,june,shown0.25-0.9,conf0.45-0.7,capex-type,2,2,1.0 | |
| 151 | +8-K,june,shown0.25-0.9,conf0.45-0.7,ma_integration,14,10,1.4 | |
| 152 | +8-K,june,shown0.25-0.9,conf0.45-0.7,other,2,2,1.0 | |
| 153 | +8-K,june,shown0.25-0.9,conf0.45-0.7,technology-type,3,3,1.0 | |
| 154 | +8-K,june,shown0.25-0.9,conf>0.7,capex-type,287,10,28.7 | |
| 155 | +8-K,june,shown0.25-0.9,conf>0.7,ma_integration,1854,10,185.4 | |
| 156 | +8-K,june,shown0.25-0.9,conf>0.7,other,160,10,16.0 | |
| 157 | +8-K,june,shown0.25-0.9,conf>0.7,technology-type,27,10,2.7 | |
| 158 | +8-K,june,shown<0.25,conf0.45-0.7,capex-type,1,1,1.0 | |
| 159 | +8-K,june,shown<0.25,conf0.45-0.7,ma_integration,1,1,1.0 | |
| 160 | +8-K,june,shown<0.25,conf>0.7,capex-type,55,10,5.5 | |
| 161 | +8-K,june,shown<0.25,conf>0.7,ma_integration,593,10,59.3 | |
| 162 | +8-K,june,shown<0.25,conf>0.7,other,50,10,5.0 | |
| 163 | +8-K,june,shown<0.25,conf>0.7,technology-type,14,10,1.4 | |
| 164 | +8-K,june,shown>=0.9,conf0.45-0.7,capex-type,21,10,2.1 | |
| 165 | +8-K,june,shown>=0.9,conf0.45-0.7,ma_integration,59,10,5.9 | |
| 166 | +8-K,june,shown>=0.9,conf0.45-0.7,other,40,10,4.0 | |
| 167 | +8-K,june,shown>=0.9,conf0.45-0.7,technology-type,8,8,1.0 | |
| 168 | +8-K,june,shown>=0.9,conf>0.7,capex-type,2732,10,273.2 | |
| 169 | +8-K,june,shown>=0.9,conf>0.7,ma_integration,7433,17,437.2352941176471 | |
| 170 | +8-K,june,shown>=0.9,conf>0.7,other,1671,10,167.1 | |
| 171 | +8-K,june,shown>=0.9,conf>0.7,technology-type,207,10,20.7 | |
| 172 | +8-K,october,shown0.25-0.9,conf0.45-0.7,capex-type,2,2,1.0 | |
| 173 | +8-K,october,shown0.25-0.9,conf0.45-0.7,ma_integration,48,10,4.8 | |
| 174 | +8-K,october,shown0.25-0.9,conf0.45-0.7,technology-type,1,1,1.0 | |
| 175 | +8-K,october,shown0.25-0.9,conf>0.7,capex-type,614,10,61.4 | |
| 176 | +8-K,october,shown0.25-0.9,conf>0.7,ma_integration,5598,13,430.61538461538464 | |
| 177 | +8-K,october,shown0.25-0.9,conf>0.7,other,838,10,83.8 | |
| 178 | +8-K,october,shown0.25-0.9,conf>0.7,technology-type,87,10,8.7 | |
| 179 | +8-K,october,shown<0.25,conf0.45-0.7,capex-type,3,3,1.0 | |
| 180 | +8-K,october,shown<0.25,conf0.45-0.7,ma_integration,8,8,1.0 | |
| 181 | +8-K,october,shown<0.25,conf0.45-0.7,other,7,7,1.0 | |
| 182 | +8-K,october,shown<0.25,conf>0.7,capex-type,228,10,22.8 | |
| 183 | +8-K,october,shown<0.25,conf>0.7,ma_integration,1665,10,166.5 | |
| 184 | +8-K,october,shown<0.25,conf>0.7,other,213,10,21.3 | |
| 185 | +8-K,october,shown<0.25,conf>0.7,technology-type,26,10,2.6 | |
| 186 | +8-K,october,shown>=0.9,conf0.45-0.7,capex-type,68,10,6.8 | |
| 187 | +8-K,october,shown>=0.9,conf0.45-0.7,ma_integration,142,10,14.2 | |
| 188 | +8-K,october,shown>=0.9,conf0.45-0.7,other,173,10,17.3 | |
| 189 | +8-K,october,shown>=0.9,conf0.45-0.7,technology-type,17,10,1.7 | |
| 190 | +8-K,october,shown>=0.9,conf>0.7,capex-type,7423,17,436.6470588235294 | |
| 191 | +8-K,october,shown>=0.9,conf>0.7,ma_integration,20583,47,437.93617021276594 | |
| 192 | +8-K,october,shown>=0.9,conf>0.7,other,8026,18,445.8888888888889 | |
| 193 | +8-K,october,shown>=0.9,conf>0.7,technology-type,644,10,64.4 | |
added
results/annotation_frame/frame_composition.csv
+221 −0
@@ -0,0 +1,221 @@ | ||
| 1 | +frame,stratum,n,weighted_n | |
| 2 | +productive,10-K md_and_a|june|shown0.25-0.9|conf>0.7|capex-type,10,10.0 | |
| 3 | +productive,10-K md_and_a|june|shown0.25-0.9|conf>0.7|ma_integration,10,30.0 | |
| 4 | +productive,10-K md_and_a|june|shown0.25-0.9|conf>0.7|other,5,5.0 | |
| 5 | +productive,10-K md_and_a|june|shown0.25-0.9|conf>0.7|technology-type,5,5.0 | |
| 6 | +productive,10-K md_and_a|june|shown<0.25|conf0.45-0.7|capex-type,4,4.0 | |
| 7 | +productive,10-K md_and_a|june|shown<0.25|conf0.45-0.7|ma_integration,2,2.0 | |
| 8 | +productive,10-K md_and_a|june|shown<0.25|conf0.45-0.7|other,2,2.0 | |
| 9 | +productive,10-K md_and_a|june|shown<0.25|conf>0.7|capex-type,10,471.0 | |
| 10 | +productive,10-K md_and_a|june|shown<0.25|conf>0.7|ma_integration,10,1502.0 | |
| 11 | +productive,10-K md_and_a|june|shown<0.25|conf>0.7|other,10,275.0 | |
| 12 | +productive,10-K md_and_a|june|shown<0.25|conf>0.7|technology-type,10,167.0 | |
| 13 | +productive,10-K md_and_a|june|shown>=0.9|conf>0.7|capex-type,10,14.0 | |
| 14 | +productive,10-K md_and_a|june|shown>=0.9|conf>0.7|ma_integration,10,11.0 | |
| 15 | +productive,10-K md_and_a|june|shown>=0.9|conf>0.7|other,4,4.0 | |
| 16 | +productive,10-K md_and_a|june|shown>=0.9|conf>0.7|technology-type,3,3.0 | |
| 17 | +productive,10-K md_and_a|october|shown0.25-0.9|conf0.45-0.7|other,2,2.0 | |
| 18 | +productive,10-K md_and_a|october|shown0.25-0.9|conf>0.7|capex-type,10,88.0 | |
| 19 | +productive,10-K md_and_a|october|shown0.25-0.9|conf>0.7|ma_integration,10,174.0 | |
| 20 | +productive,10-K md_and_a|october|shown0.25-0.9|conf>0.7|other,10,63.0 | |
| 21 | +productive,10-K md_and_a|october|shown0.25-0.9|conf>0.7|technology-type,10,19.0 | |
| 22 | +productive,10-K md_and_a|october|shown<0.25|conf0.45-0.7|capex-type,10,10.0 | |
| 23 | +productive,10-K md_and_a|october|shown<0.25|conf0.45-0.7|ma_integration,4,4.0 | |
| 24 | +productive,10-K md_and_a|october|shown<0.25|conf0.45-0.7|other,10,10.0 | |
| 25 | +productive,10-K md_and_a|october|shown<0.25|conf0.45-0.7|technology-type,2,2.0 | |
| 26 | +productive,10-K md_and_a|october|shown<0.25|conf>0.7|capex-type,10,1833.0 | |
| 27 | +productive,10-K md_and_a|october|shown<0.25|conf>0.7|ma_integration,15,6385.0 | |
| 28 | +productive,10-K md_and_a|october|shown<0.25|conf>0.7|other,10,2405.0 | |
| 29 | +productive,10-K md_and_a|october|shown<0.25|conf>0.7|technology-type,10,683.0 | |
| 30 | +productive,10-K md_and_a|october|shown>=0.9|conf0.45-0.7|ma_integration,2,2.0 | |
| 31 | +productive,10-K md_and_a|october|shown>=0.9|conf>0.7|capex-type,10,26.0 | |
| 32 | +productive,10-K md_and_a|october|shown>=0.9|conf>0.7|ma_integration,10,50.0 | |
| 33 | +productive,10-K md_and_a|october|shown>=0.9|conf>0.7|other,10,17.0 | |
| 34 | +productive,10-K md_and_a|october|shown>=0.9|conf>0.7|technology-type,6,6.0 | |
| 35 | +productive,10-K other|june|shown0.25-0.9|conf0.45-0.7|ma_integration,1,1.0 | |
| 36 | +productive,10-K other|june|shown0.25-0.9|conf0.45-0.7|other,1,1.0 | |
| 37 | +productive,10-K other|june|shown0.25-0.9|conf0.45-0.7|technology-type,2,2.0 | |
| 38 | +productive,10-K other|june|shown0.25-0.9|conf>0.7|capex-type,10,72.0 | |
| 39 | +productive,10-K other|june|shown0.25-0.9|conf>0.7|ma_integration,10,238.0 | |
| 40 | +productive,10-K other|june|shown0.25-0.9|conf>0.7|other,10,71.0 | |
| 41 | +productive,10-K other|june|shown0.25-0.9|conf>0.7|technology-type,10,36.0 | |
| 42 | +productive,10-K other|june|shown<0.25|conf0.45-0.7|capex-type,5,5.0 | |
| 43 | +productive,10-K other|june|shown<0.25|conf0.45-0.7|ma_integration,5,5.0 | |
| 44 | +productive,10-K other|june|shown<0.25|conf0.45-0.7|other,5,5.0 | |
| 45 | +productive,10-K other|june|shown<0.25|conf>0.7|capex-type,10,545.0 | |
| 46 | +productive,10-K other|june|shown<0.25|conf>0.7|ma_integration,10,1252.0 | |
| 47 | +productive,10-K other|june|shown<0.25|conf>0.7|other,10,329.0 | |
| 48 | +productive,10-K other|june|shown<0.25|conf>0.7|technology-type,10,245.0 | |
| 49 | +productive,10-K other|june|shown>=0.9|conf0.45-0.7|capex-type,3,3.0 | |
| 50 | +productive,10-K other|june|shown>=0.9|conf0.45-0.7|other,9,9.0 | |
| 51 | +productive,10-K other|june|shown>=0.9|conf>0.7|capex-type,10,613.0 | |
| 52 | +productive,10-K other|june|shown>=0.9|conf>0.7|ma_integration,10,118.0 | |
| 53 | +productive,10-K other|june|shown>=0.9|conf>0.7|other,10,135.0 | |
| 54 | +productive,10-K other|june|shown>=0.9|conf>0.7|technology-type,10,14.0 | |
| 55 | +productive,10-K other|october|shown0.25-0.9|conf0.45-0.7|capex-type,2,2.0 | |
| 56 | +productive,10-K other|october|shown0.25-0.9|conf0.45-0.7|other,4,4.0 | |
| 57 | +productive,10-K other|october|shown0.25-0.9|conf0.45-0.7|technology-type,1,1.0 | |
| 58 | +productive,10-K other|october|shown0.25-0.9|conf>0.7|capex-type,10,360.0 | |
| 59 | +productive,10-K other|october|shown0.25-0.9|conf>0.7|ma_integration,10,610.0 | |
| 60 | +productive,10-K other|october|shown0.25-0.9|conf>0.7|other,10,199.0 | |
| 61 | +productive,10-K other|october|shown0.25-0.9|conf>0.7|technology-type,10,143.0 | |
| 62 | +productive,10-K other|october|shown<0.25|conf0.45-0.7|capex-type,10,13.0 | |
| 63 | +productive,10-K other|october|shown<0.25|conf0.45-0.7|ma_integration,10,14.0 | |
| 64 | +productive,10-K other|october|shown<0.25|conf0.45-0.7|other,8,8.0 | |
| 65 | +productive,10-K other|october|shown<0.25|conf0.45-0.7|technology-type,4,4.0 | |
| 66 | +productive,10-K other|october|shown<0.25|conf>0.7|capex-type,10,1604.0 | |
| 67 | +productive,10-K other|october|shown<0.25|conf>0.7|ma_integration,10,4222.0 | |
| 68 | +productive,10-K other|october|shown<0.25|conf>0.7|other,10,2173.0 | |
| 69 | +productive,10-K other|october|shown<0.25|conf>0.7|technology-type,10,662.0 | |
| 70 | +productive,10-K other|october|shown>=0.9|conf0.45-0.7|capex-type,10,12.0 | |
| 71 | +productive,10-K other|october|shown>=0.9|conf0.45-0.7|ma_integration,2,2.0 | |
| 72 | +productive,10-K other|october|shown>=0.9|conf0.45-0.7|other,10,19.0 | |
| 73 | +productive,10-K other|october|shown>=0.9|conf>0.7|capex-type,10,1810.0 | |
| 74 | +productive,10-K other|october|shown>=0.9|conf>0.7|ma_integration,10,417.0 | |
| 75 | +productive,10-K other|october|shown>=0.9|conf>0.7|other,10,468.0 | |
| 76 | +productive,10-K other|october|shown>=0.9|conf>0.7|technology-type,10,147.0 | |
| 77 | +productive,10-Q md_and_a|june|shown0.25-0.9|conf>0.7|capex-type,10,27.0 | |
| 78 | +productive,10-Q md_and_a|june|shown0.25-0.9|conf>0.7|ma_integration,10,46.0 | |
| 79 | +productive,10-Q md_and_a|june|shown0.25-0.9|conf>0.7|other,10,20.0 | |
| 80 | +productive,10-Q md_and_a|june|shown0.25-0.9|conf>0.7|technology-type,6,6.0 | |
| 81 | +productive,10-Q md_and_a|june|shown<0.25|conf0.45-0.7|capex-type,2,2.0 | |
| 82 | +productive,10-Q md_and_a|june|shown<0.25|conf0.45-0.7|ma_integration,6,6.0 | |
| 83 | +productive,10-Q md_and_a|june|shown<0.25|conf0.45-0.7|technology-type,1,1.0 | |
| 84 | +productive,10-Q md_and_a|june|shown<0.25|conf>0.7|capex-type,10,337.0 | |
| 85 | +productive,10-Q md_and_a|june|shown<0.25|conf>0.7|ma_integration,10,873.0 | |
| 86 | +productive,10-Q md_and_a|june|shown<0.25|conf>0.7|other,10,151.0 | |
| 87 | +productive,10-Q md_and_a|june|shown<0.25|conf>0.7|technology-type,10,85.0 | |
| 88 | +productive,10-Q md_and_a|june|shown>=0.9|conf>0.7|ma_integration,8,8.0 | |
| 89 | +productive,10-Q md_and_a|june|shown>=0.9|conf>0.7|other,1,1.0 | |
| 90 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf0.45-0.7|capex-type,2,2.0 | |
| 91 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf0.45-0.7|ma_integration,4,4.0 | |
| 92 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf0.45-0.7|other,5,5.0 | |
| 93 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf>0.7|capex-type,10,820.0 | |
| 94 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf>0.7|ma_integration,10,1984.0 | |
| 95 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf>0.7|other,10,786.0 | |
| 96 | +productive,10-Q md_and_a|october|shown0.25-0.9|conf>0.7|technology-type,10,202.0 | |
| 97 | +productive,10-Q md_and_a|october|shown<0.25|conf0.45-0.7|capex-type,10,20.0 | |
| 98 | +productive,10-Q md_and_a|october|shown<0.25|conf0.45-0.7|ma_integration,10,45.0 | |
| 99 | +productive,10-Q md_and_a|october|shown<0.25|conf0.45-0.7|other,10,59.0 | |
| 100 | +productive,10-Q md_and_a|october|shown<0.25|conf0.45-0.7|technology-type,2,2.0 | |
| 101 | +productive,10-Q md_and_a|october|shown<0.25|conf>0.7|capex-type,14,6308.0 | |
| 102 | +productive,10-Q md_and_a|october|shown<0.25|conf>0.7|ma_integration,41,17955.0 | |
| 103 | +productive,10-Q md_and_a|october|shown<0.25|conf>0.7|other,17,7471.0 | |
| 104 | +productive,10-Q md_and_a|october|shown<0.25|conf>0.7|technology-type,10,2025.0 | |
| 105 | +productive,10-Q md_and_a|october|shown>=0.9|conf0.45-0.7|capex-type,2,2.0 | |
| 106 | +productive,10-Q md_and_a|october|shown>=0.9|conf0.45-0.7|ma_integration,3,3.0 | |
| 107 | +productive,10-Q md_and_a|october|shown>=0.9|conf>0.7|capex-type,10,161.0 | |
| 108 | +productive,10-Q md_and_a|october|shown>=0.9|conf>0.7|ma_integration,10,282.0 | |
| 109 | +productive,10-Q md_and_a|october|shown>=0.9|conf>0.7|other,10,74.0 | |
| 110 | +productive,10-Q md_and_a|october|shown>=0.9|conf>0.7|technology-type,10,14.0 | |
| 111 | +productive,10-Q other|june|shown0.25-0.9|conf0.45-0.7|ma_integration,1,1.0 | |
| 112 | +productive,10-Q other|june|shown0.25-0.9|conf>0.7|capex-type,7,7.0 | |
| 113 | +productive,10-Q other|june|shown0.25-0.9|conf>0.7|ma_integration,10,46.0 | |
| 114 | +productive,10-Q other|june|shown0.25-0.9|conf>0.7|other,10,10.0 | |
| 115 | +productive,10-Q other|june|shown0.25-0.9|conf>0.7|technology-type,3,3.0 | |
| 116 | +productive,10-Q other|june|shown<0.25|conf>0.7|capex-type,10,44.0 | |
| 117 | +productive,10-Q other|june|shown<0.25|conf>0.7|ma_integration,10,618.0 | |
| 118 | +productive,10-Q other|june|shown<0.25|conf>0.7|other,10,71.0 | |
| 119 | +productive,10-Q other|june|shown<0.25|conf>0.7|technology-type,8,8.0 | |
| 120 | +productive,10-Q other|june|shown>=0.9|conf0.45-0.7|ma_integration,2,2.0 | |
| 121 | +productive,10-Q other|june|shown>=0.9|conf0.45-0.7|other,1,1.0 | |
| 122 | +productive,10-Q other|june|shown>=0.9|conf0.45-0.7|technology-type,2,2.0 | |
| 123 | +productive,10-Q other|june|shown>=0.9|conf>0.7|capex-type,10,24.0 | |
| 124 | +productive,10-Q other|june|shown>=0.9|conf>0.7|ma_integration,10,364.0 | |
| 125 | +productive,10-Q other|june|shown>=0.9|conf>0.7|other,10,55.0 | |
| 126 | +productive,10-Q other|june|shown>=0.9|conf>0.7|technology-type,10,150.0 | |
| 127 | +productive,10-Q other|october|shown0.25-0.9|conf0.45-0.7|capex-type,7,7.0 | |
| 128 | +productive,10-Q other|october|shown0.25-0.9|conf0.45-0.7|ma_integration,10,11.0 | |
| 129 | +productive,10-Q other|october|shown0.25-0.9|conf0.45-0.7|other,10,14.0 | |
| 130 | +productive,10-Q other|october|shown0.25-0.9|conf>0.7|capex-type,10,300.0 | |
| 131 | +productive,10-Q other|october|shown0.25-0.9|conf>0.7|ma_integration,10,1102.0 | |
| 132 | +productive,10-Q other|october|shown0.25-0.9|conf>0.7|other,10,306.0 | |
| 133 | +productive,10-Q other|october|shown0.25-0.9|conf>0.7|technology-type,10,101.0 | |
| 134 | +productive,10-Q other|october|shown<0.25|conf0.45-0.7|capex-type,10,18.0 | |
| 135 | +productive,10-Q other|october|shown<0.25|conf0.45-0.7|ma_integration,10,14.0 | |
| 136 | +productive,10-Q other|october|shown<0.25|conf0.45-0.7|other,10,36.0 | |
| 137 | +productive,10-Q other|october|shown<0.25|conf0.45-0.7|technology-type,3,3.0 | |
| 138 | +productive,10-Q other|october|shown<0.25|conf>0.7|capex-type,10,1628.0 | |
| 139 | +productive,10-Q other|october|shown<0.25|conf>0.7|ma_integration,31,13740.0 | |
| 140 | +productive,10-Q other|october|shown<0.25|conf>0.7|other,10,2980.0 | |
| 141 | +productive,10-Q other|october|shown<0.25|conf>0.7|technology-type,10,302.0 | |
| 142 | +productive,10-Q other|october|shown>=0.9|conf0.45-0.7|capex-type,10,22.0 | |
| 143 | +productive,10-Q other|october|shown>=0.9|conf0.45-0.7|ma_integration,10,17.0 | |
| 144 | +productive,10-Q other|october|shown>=0.9|conf0.45-0.7|other,10,16.0 | |
| 145 | +productive,10-Q other|october|shown>=0.9|conf0.45-0.7|technology-type,3,3.0 | |
| 146 | +productive,10-Q other|october|shown>=0.9|conf>0.7|capex-type,10,1890.0 | |
| 147 | +productive,10-Q other|october|shown>=0.9|conf>0.7|ma_integration,17,7272.0 | |
| 148 | +productive,10-Q other|october|shown>=0.9|conf>0.7|other,10,2196.0 | |
| 149 | +productive,10-Q other|october|shown>=0.9|conf>0.7|technology-type,10,2534.0 | |
| 150 | +productive,8-K|june|shown0.25-0.9|conf0.45-0.7|capex-type,2,2.0 | |
| 151 | +productive,8-K|june|shown0.25-0.9|conf0.45-0.7|ma_integration,10,14.0 | |
| 152 | +productive,8-K|june|shown0.25-0.9|conf0.45-0.7|other,2,2.0 | |
| 153 | +productive,8-K|june|shown0.25-0.9|conf0.45-0.7|technology-type,3,3.0 | |
| 154 | +productive,8-K|june|shown0.25-0.9|conf>0.7|capex-type,10,287.0 | |
| 155 | +productive,8-K|june|shown0.25-0.9|conf>0.7|ma_integration,10,1854.0 | |
| 156 | +productive,8-K|june|shown0.25-0.9|conf>0.7|other,10,160.0 | |
| 157 | +productive,8-K|june|shown0.25-0.9|conf>0.7|technology-type,10,27.0 | |
| 158 | +productive,8-K|june|shown<0.25|conf0.45-0.7|capex-type,1,1.0 | |
| 159 | +productive,8-K|june|shown<0.25|conf0.45-0.7|ma_integration,1,1.0 | |
| 160 | +productive,8-K|june|shown<0.25|conf>0.7|capex-type,10,55.0 | |
| 161 | +productive,8-K|june|shown<0.25|conf>0.7|ma_integration,10,593.0 | |
| 162 | +productive,8-K|june|shown<0.25|conf>0.7|other,10,50.0 | |
| 163 | +productive,8-K|june|shown<0.25|conf>0.7|technology-type,10,14.0 | |
| 164 | +productive,8-K|june|shown>=0.9|conf0.45-0.7|capex-type,10,21.0 | |
| 165 | +productive,8-K|june|shown>=0.9|conf0.45-0.7|ma_integration,10,59.0 | |
| 166 | +productive,8-K|june|shown>=0.9|conf0.45-0.7|other,10,40.0 | |
| 167 | +productive,8-K|june|shown>=0.9|conf0.45-0.7|technology-type,8,8.0 | |
| 168 | +productive,8-K|june|shown>=0.9|conf>0.7|capex-type,10,2732.0 | |
| 169 | +productive,8-K|june|shown>=0.9|conf>0.7|ma_integration,17,7433.0 | |
| 170 | +productive,8-K|june|shown>=0.9|conf>0.7|other,10,1671.0 | |
| 171 | +productive,8-K|june|shown>=0.9|conf>0.7|technology-type,10,207.0 | |
| 172 | +productive,8-K|october|shown0.25-0.9|conf0.45-0.7|capex-type,2,2.0 | |
| 173 | +productive,8-K|october|shown0.25-0.9|conf0.45-0.7|ma_integration,10,48.0 | |
| 174 | +productive,8-K|october|shown0.25-0.9|conf0.45-0.7|technology-type,1,1.0 | |
| 175 | +productive,8-K|october|shown0.25-0.9|conf>0.7|capex-type,10,614.0 | |
| 176 | +productive,8-K|october|shown0.25-0.9|conf>0.7|ma_integration,13,5598.0 | |
| 177 | +productive,8-K|october|shown0.25-0.9|conf>0.7|other,10,838.0 | |
| 178 | +productive,8-K|october|shown0.25-0.9|conf>0.7|technology-type,10,87.0 | |
| 179 | +productive,8-K|october|shown<0.25|conf0.45-0.7|capex-type,3,3.0 | |
| 180 | +productive,8-K|october|shown<0.25|conf0.45-0.7|ma_integration,8,8.0 | |
| 181 | +productive,8-K|october|shown<0.25|conf0.45-0.7|other,7,7.0 | |
| 182 | +productive,8-K|october|shown<0.25|conf>0.7|capex-type,10,228.0 | |
| 183 | +productive,8-K|october|shown<0.25|conf>0.7|ma_integration,10,1665.0 | |
| 184 | +productive,8-K|october|shown<0.25|conf>0.7|other,10,213.0 | |
| 185 | +productive,8-K|october|shown<0.25|conf>0.7|technology-type,10,26.0 | |
| 186 | +productive,8-K|october|shown>=0.9|conf0.45-0.7|capex-type,10,68.0 | |
| 187 | +productive,8-K|october|shown>=0.9|conf0.45-0.7|ma_integration,10,142.0 | |
| 188 | +productive,8-K|october|shown>=0.9|conf0.45-0.7|other,10,173.0 | |
| 189 | +productive,8-K|october|shown>=0.9|conf0.45-0.7|technology-type,10,17.0 | |
| 190 | +productive,8-K|october|shown>=0.9|conf>0.7|capex-type,17,7423.0 | |
| 191 | +productive,8-K|october|shown>=0.9|conf>0.7|ma_integration,47,20583.0 | |
| 192 | +productive,8-K|october|shown>=0.9|conf>0.7|other,18,8026.0 | |
| 193 | +productive,8-K|october|shown>=0.9|conf>0.7|technology-type,10,644.0 | |
| 194 | +zero,10-K md_and_a|october|shown0.25-0.9|gate=pass,1,1712.0 | |
| 195 | +zero,10-K md_and_a|october|shown<0.25|gate=pass,4,5939.0 | |
| 196 | +zero,10-K other|june|shown<0.25|gate=pass,7,11260.0 | |
| 197 | +zero,10-K other|june|shown>=0.9|gate=pass,3,6140.0 | |
| 198 | +zero,10-K other|october|shown0.25-0.9|gate=pass,4,7148.0 | |
| 199 | +zero,10-K other|october|shown<0.25|gate=pass,22,35009.0 | |
| 200 | +zero,10-K other|october|shown>=0.9|gate=pass,23,49230.0 | |
| 201 | +zero,10-K other|october|shown>=0.9|gate=reject,2,9337.0 | |
| 202 | +zero,10-Q md_and_a|june|shown<0.25|gate=pass,1,1579.0 | |
| 203 | +zero,10-Q md_and_a|june|shown>=0.9|gate=pass,1,2310.0 | |
| 204 | +zero,10-Q md_and_a|october|shown0.25-0.9|gate=pass,2,3442.0 | |
| 205 | +zero,10-Q md_and_a|october|shown<0.25|gate=pass,25,39479.0 | |
| 206 | +zero,10-Q md_and_a|october|shown>=0.9|gate=pass,2,4607.0 | |
| 207 | +zero,10-Q other|june|shown<0.25|gate=pass,1,1602.0 | |
| 208 | +zero,10-Q other|june|shown>=0.9|gate=pass,5,10559.0 | |
| 209 | +zero,10-Q other|june|shown>=0.9|gate=reject,1,5677.0 | |
| 210 | +zero,10-Q other|october|shown0.25-0.9|gate=pass,4,7084.0 | |
| 211 | +zero,10-Q other|october|shown<0.25|gate=pass,49,78089.0 | |
| 212 | +zero,10-Q other|october|shown>=0.9|gate=pass,66,136143.0 | |
| 213 | +zero,10-Q other|october|shown>=0.9|gate=reject,52,241507.0 | |
| 214 | +zero,8-K|june|shown0.25-0.9|gate=pass,1,1811.0 | |
| 215 | +zero,8-K|june|shown<0.25|gate=pass,3,5540.0 | |
| 216 | +zero,8-K|june|shown>=0.9|gate=pass,19,37643.0 | |
| 217 | +zero,8-K|june|shown>=0.9|gate=reject,11,44831.0 | |
| 218 | +zero,8-K|october|shown0.25-0.9|gate=pass,4,7151.0 | |
| 219 | +zero,8-K|october|shown<0.25|gate=pass,3,5499.0 | |
| 220 | +zero,8-K|october|shown>=0.9|gate=pass,35,69565.0 | |
| 221 | +zero,8-K|october|shown>=0.9|gate=reject,34,138388.0 | |
added
results/annotation_frame/frame_dev.csv
+19368 −0
Large diff (19,368 lines) is not loaded by default to keep the page fast.Load diff
Diff truncated — file too large.