Stage F B1-B2 N1-N14
Verification sign-off (Agent 11) corrections, no new analysis, no LLM call.
B1 construct.tex §5.1: financing-class amounts printed in trillions (\SPIDfinAmountT / \SPIDfinAmountDenomT; source matched_amount_gusd is in billions), consistent with the Table 5 note and §7.3.
B2 make_tables.py Table 16 note: key read from results/resolver/example_provenance_keys.csv ("san patricio county, texas"), cluster sizes (8 / 1 / 176) from spec_curve.csv at the release threshold; Table 9 note defines "Cheniere cluster size" as the size of the cluster containing the example mention.
N1 data/documented_inputs.json: the six pipeline constants (chunk words/overlap, max file MB, cooldown, SEC rate limit, gateway divisor) recorded with snapshot path + SHA-256; make_tables.py reads them there and cross-checks the archive when reachable; verified in a git-archive copy outside the review tree (844 macros, tables/figures/inline numbers identical, 22/22 tests, latexmk 0 errors).
N2 Table 8 Panel B: retryable lines 94,593 = 94,465 inside the window + 128 in the 20 s tail (no double count); §5.5 prose aligned (\SPIDllmRetryableInsideWindow).
N3 Table 17 note + §8.2: normalisation stated (case-fold, punctuation to spaces, alias list, whole-word search); results/grounding/partners_normalisation_check.json (read-only re-derivation on M3U96b: 99.3 % under the rule vs 96.1 % plain case-fold on a 20,000-mention sample).
N4 abstract: "every structured field verbatim ... for 88.2 % of mentions" (150 words).
N5 Table 6 headers shortened: overfull 20.07 pt removed (max overfull now 3.3 pt).
N6 §5.5 / Table 8 Panel C: log vs ledger bases labelled (5,198 / 4,769 rows; 34 / 30 mentions); the 504 outage-window rows verified productive (results/ledger/outage_window_rows_check.json, 0 zero-mention rows).
N7 Table 9 note prints the branch count 229 as in §6.1 and explains the 286 keys ending in "general".
N8 Appendix E: pooled shares 12.4 / 12.7 % vs per-issuer means 11.9 / 12.5 % (paired 0.6 pp, SE 1.3); stored vs ledger mention bases named once.
N9 cost basis: 775 USD stated as recorded against the October run, alternative per-call figure if both runs (0.0011 USD over 703,725 calls), author to confirm (documented_inputs.json source updated).
N12 reproducibility/validation_report.md: gate regex has 25 alternatives.
N10, N11, N13, N14: no change (author decisions / engine limitation), recorded in improvements/FIRC-0001/unresolved_issues.md.
Build: latexmk 0 errors, 0 undefined references, 0 overfull > 5 pt, 0 bibtex warnings, 84 pages; pytest 22/22.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
18 changed files +274 −60
modified
code/make_tables.py
+79 −23
@@ -15,7 +15,7 @@ Outputs: | |||||
| 15 | No numeric literal is typed in this file; stats.json is never rewritten. | 15 | No numeric literal is typed in this file; stats.json is never rewritten. | ||
| 16 | Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/) | 16 | Usage: python3 -I code/make_tables.py [--out SCRATCH_DIR] (--out is used by tests/) | ||
| 17 | """ | 17 | """ | ||
| 18 | −import argparse, ast, csv, json, math, re, sys | 18 | +import argparse, ast, csv, hashlib, json, math, re, sys | ||
| 19 | from pathlib import Path | 19 | from pathlib import Path | ||
| 20 | import matplotlib | 20 | import matplotlib | ||
| 21 | matplotlib.use("Agg") | 21 | matplotlib.use("Agg") | ||
@@ -430,8 +430,10 @@ if oc_ and tp and rb1.exists(): | |||||
| 430 | tab += f"Ledger rows with a timestamp inside the largest gap ({str(tp.get('october_first_flush',''))[:0]}{str(tp['largest_gaps_seconds'][-1][1])[11:16]}--{str(tp['largest_gaps_seconds'][-1][2])[11:16]}, {tp['largest_gap_hours']:.2f}~h) & {n(oc_.get('ledger_rows_in_largest_gap', 0))} \\\\\n" | 430 | tab += f"Ledger rows with a timestamp inside the largest gap ({str(tp.get('october_first_flush',''))[:0]}{str(tp['largest_gaps_seconds'][-1][1])[11:16]}--{str(tp['largest_gaps_seconds'][-1][2])[11:16]}, {tp['largest_gap_hours']:.2f}~h) & {n(oc_.get('ledger_rows_in_largest_gap', 0))} \\\\\n" | ||
| 431 | tab += f"Log batches returning zero mentions inside the outage window (batch ids {an_ids[0] if an_ids else ''}--{an_ids[-1] if an_ids else ''}) & {len(anomalous)} \\\\\n" | 431 | tab += f"Log batches returning zero mentions inside the outage window (batch ids {an_ids[0] if an_ids else ''}--{an_ids[-1] if an_ids else ''}) & {len(anomalous)} \\\\\n" | ||
| 432 | tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel B: failed-call lines in the October log (" + n(oc_.get('llm_call_failed_total', 0)) + " lines; the ledger records none of them)} \\\\\n" | 432 | tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel B: failed-call lines in the October log (" + n(oc_.get('llm_call_failed_total', 0)) + " lines; the ledger records none of them)} \\\\\n" | ||
| 433 | − tab += f"Retryable HTTP 429/5xx inside the outage window {str(lw[0])[11:16]}--{str(lw[1])[11:16]} & {n(lw[2])} \\\\\n" | 433 | + retry_all = int(rsn.get('retryable_http_429_5xx', 0)); retry_tail = int(owr.get('retryable_http_429_5xx', 0)); win_ = oc_.get("window", lw) | ||
| 434 | − tab += f"Retryable HTTP 429/5xx in the outage tail (first ledger window definition, 20 s after the log window) & {n(owr.get('retryable_http_429_5xx', 0))} \\\\\n" | 434 | + tab += f"Retryable HTTP 429/5xx, all lines & {n(retry_all)} \\\\\n" | ||
| 435 | + tab += f"\\quad of which inside the outage window {str(win_[0])[11:16]}--{str(win_[1])[11:16]} & {n(retry_all - retry_tail)} \\\\\n" | ||||
| 436 | + tab += f"\\quad of which in the twenty seconds after it (to {str(lw[1])[11:19]}, the end of the log window) & {n(retry_tail)} \\\\\n" | ||||
| 435 | tab += f"Connection resets & {n(rsn.get('connection_error', 0))} \\\\\n" | 437 | tab += f"Connection resets & {n(rsn.get('connection_error', 0))} \\\\\n" | ||
| 436 | tab += f"JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & {n(oc_.get('silent_empty_results_outside_outage', 0))} \\\\\n" | 438 | tab += f"JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & {n(oc_.get('silent_empty_results_outside_outage', 0))} \\\\\n" | ||
| 437 | tab += f"``Section failed'' lines & {n(oc_.get('section_failed', 0))} \\\\\n" | 439 | tab += f"``Section failed'' lines & {n(oc_.get('section_failed', 0))} \\\\\n" | ||
@@ -439,6 +441,7 @@ if oc_ and tp and rb1.exists(): | |||||
| 439 | tab += f"JSON-decode failures in the June log (of {n(bb.get('june_log_sections', 0))} sections) & {n(jn_.get('llm_call_failed_total', 0))} \\\\\n" | 441 | tab += f"JSON-decode failures in the June log (of {n(bb.get('june_log_sections', 0))} sections) & {n(jn_.get('llm_call_failed_total', 0))} \\\\\n" | ||
| 440 | tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel C: first batch after the resume (" + str(rs.get('window', ['', ''])[0])[11:16] + "--" + str(rs.get('window', ['', ''])[1])[11:16] + ", " + n(len(rs.get('ciks', []))) + " issuers)} \\\\\n" | 442 | tab += "\\midrule\n\\multicolumn{2}{l}{\\itshape Panel C: first batch after the resume (" + str(rs.get('window', ['', ''])[0])[11:16] + "--" + str(rs.get('window', ['', ''])[1])[11:16] + ", " + n(len(rs.get('ciks', []))) + " issuers)} \\\\\n" | ||
| 441 | tab += f"Sections in the batch (log) / ledgered rows & {n(rs.get('log_sections', 0))} / {n(rb_rows)} \\\\\n" | 443 | tab += f"Sections in the batch (log) / ledgered rows & {n(rs.get('log_sections', 0))} / {n(rb_rows)} \\\\\n" | ||
| 444 | + tab += f"Mentions returned (log) / ledgered & {n(rs.get('log_mentions', 0))} / {n(sum(int(r['mentions']) for r in rb))} \\\\\n" | ||||
| 442 | tab += f"Gate-passing calls & {n(rb_calls)} \\\\\n" | 445 | tab += f"Gate-passing calls & {n(rb_calls)} \\\\\n" | ||
| 443 | tab += f"Productive sections observed & {n(rb_prod)} \\\\\n" | 446 | tab += f"Productive sections observed & {n(rb_prod)} \\\\\n" | ||
| 444 | tab += f"Productive sections expected at the form $\\times$ item rates of the rest of the run & {rb_exp:.1f} \\\\\n" | 447 | tab += f"Productive sections expected at the form $\\times$ item rates of the rest of the run & {rb_exp:.1f} \\\\\n" | ||
@@ -814,16 +817,29 @@ if ex: | |||||
| 814 | macro("provenanceProjectId", ex[0]["project_id"][:8], "data/example_provenance.csv", "project_id[:8]") | 817 | macro("provenanceProjectId", ex[0]["project_id"][:8], "data/example_provenance.csv", "project_id[:8]") | ||
| 815 | json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")}, | 818 | json.dump({k: ex[0][k] for k in ("project_id", "ticker", "project_name", "project_type", "status", "total_amount_usd", "n_mentions", "first_seen", "last_seen")}, | ||
| 816 | open(DATA_OUT / "example_project.json", "w"), indent=2) | 819 | open(DATA_OUT / "example_project.json", "w"), indent=2) | ||
| 817 | − sc_ex = jload("resolver/composition.json") | 820 | + # Stage F (B2): the key and the cluster sizes come from the resolver audit (example_provenance_keys.csv, spec_curve.csv), never typed | ||
| 821 | + epk = R / "resolver" / "example_provenance_keys.csv"; scf = R / "resolver" / "spec_curve.csv" | ||||
| 822 | + ex_key = next((r for r in rows(epk.stem, epk.parent) if r["project_id"] == ex[0]["project_id"]), None) if epk.exists() else None | ||||
| 823 | + key_tok = ex_key["gkey"].split("|")[-1] if ex_key else "" | ||||
| 824 | + sc16 = rows(scf.stem, scf.parent) if scf.exists() else [] | ||||
| 825 | + thr16 = min(float(r["threshold"]) for r in sc16) if sc16 else None | ||||
| 826 | + def cs16(key): | ||||
| 827 | + return next((r["example_cluster_size_cheniere"] for r in sc16 if r["key"] == key and float(r["threshold"]) == thr16), None) | ||||
| 828 | + if key_tok: | ||||
| 829 | + macro("provenanceKeyToken", esc(key_tok), "results/resolver/example_provenance_keys.csv", "last field of gkey (location token) of the example project") | ||||
| 830 | + if key_tok and cs16("k_current") is not None: | ||||
| 831 | + key_note = (f"The key for this issuer and type is the location ``{esc(key_tok)}'' (\\texttt{{results/resolver/example\\_provenance\\_keys.csv}}), so distinct phases at that " | ||||
| 832 | + f"site share one identifier; the cluster that contains the example's highest-confidence mention holds {n(cs16('k_current'))} mentions under this key, " | ||||
| 833 | + f"{n(cs16('k_strict'))} under the strict name $\\times$ location key and {n(cs16('k_cik_type'))} under the issuer $\\times$ type key (Table~\\ref{{tab:spec_curve}}).") | ||||
| 834 | + else: | ||||
| 835 | + key_note = "The key for this issuer and type is a location token, so distinct phases share one identifier (cluster sizes under alternative keys in Table~\\ref{tab:spec_curve})." | ||||
| 818 | float_table("tab_provenance.tex", "tab:provenance", f"Provenance of one resolved project: Cheniere Energy, Corpus Christi liquefaction (project \\texttt{{{ex[0]['project_id'][:8]}\\ldots}})", | 836 | float_table("tab_provenance.tex", "tab:provenance", f"Provenance of one resolved project: Cheniere Energy, Corpus Christi liquefaction (project \\texttt{{{ex[0]['project_id'][:8]}\\ldots}})", | ||
| 819 | tab, f"Snippets are the first {DI['snippet_chars']['value']} characters of the model-generated description stored in \\texttt{{project\\_timeline.snippet}}; " | 837 | tab, f"Snippets are the first {DI['snippet_chars']['value']} characters of the model-generated description stored in \\texttt{{project\\_timeline.snippet}}; " | ||
| 820 | "the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable " | 838 | "the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable " | ||
| 821 | f"through \\texttt{{project\\_mentions.section\\_id}}. All {len(ex[:8])} printed mentions are {' and '.join(forms)} filings. The representative name is that of the " | 839 | f"through \\texttt{{project\\_mentions.section\\_id}}. All {len(ex[:8])} printed mentions are {' and '.join(forms)} filings. The representative name is that of the " | ||
| 822 | f"highest-confidence mention (filed {ex[:8][-1]['filing_date'][:4]}) applied retroactively to the whole timeline, and is one admissible choice among " | 840 | f"highest-confidence mention (filed {ex[:8][-1]['filing_date'][:4]}) applied retroactively to the whole timeline, and is one admissible choice among " | ||
| 823 | − "mentions tied at the maximum confidence. The key for this issuer and type is the location token ``corpus christi'', so distinct phases share one identifier: " | 841 | + "mentions tied at the maximum confidence. " + key_note, | ||
| 824 | − "the same mentions form " | 842 | + "data/example\\_provenance.csv, documented\\_inputs.json:snippet\\_chars, results/resolver/example\\_provenance\\_keys.csv, spec\\_curve.csv", size="\\scriptsize", colsep="3pt", fit=True) | ||
| 825 | − + (f"{sc_ex['project_count_range_045'].get('k_strict', '')} projects under the strict key and {sc_ex['project_count_range_045'].get('k_cik_type', '')} under the issuer $\\times$ type key" if sc_ex.get("project_count_range_045") else "different numbers of projects under alternative keys") | ||||
| 826 | − + " (cluster sizes in Table~\\ref{tab:spec_curve}).", "data/example\\_provenance.csv, documented\\_inputs.json:snippet\\_chars", size="\\scriptsize", colsep="3pt", fit=True) | ||||
| 827 | 843 | ||||
| 828 | # ---- Table 12: data-model audit (Appendix C) as a longtable --------------------------------------------------------- | 844 | # ---- Table 12: data-model audit (Appendix C) as a longtable --------------------------------------------------------- | ||
| 829 | if (D / "datamodel_audit.csv").exists(): | 845 | if (D / "datamodel_audit.csv").exists(): | ||
@@ -934,10 +950,21 @@ if gg.exists() and gas.exists(): | |||||
| 934 | macro("yearsNotInText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text") | 950 | macro("yearsNotInText", n(fbd["years_not_subset_of_text"]), "results/grounding/fallback_bounds.json", "years_not_subset_of_text") | ||
| 935 | macro("techDeployMentionedNoSignal", n(fbd["tech_deploy_mentioned_no_any_signal"]), "results/grounding/fallback_bounds.json", "tech_deploy_mentioned_no_any_signal") | 951 | macro("techDeployMentionedNoSignal", n(fbd["tech_deploy_mentioned_no_any_signal"]), "results/grounding/fallback_bounds.json", "tech_deploy_mentioned_no_any_signal") | ||
| 936 | macro("techDeployMentionedTotal", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]") | 952 | macro("techDeployMentionedTotal", n(fbd["tech_deploy_total"][1]), "results/grounding/fallback_bounds.json", "tech_deploy_total[1]") | ||
| 953 | + pnc = jload("grounding/partners_normalisation_check.json"); pnc_note = "" | ||||
| 954 | + if pnc: | ||||
| 955 | + macro("groundPartnersCheckN", n(pnc["with_partners"]), "results/grounding/partners_normalisation_check.json", "with_partners (mentions with a partner in the sample)") | ||||
| 956 | + macro("groundPartnersCheckAuditPct", pct(pnc["partners_grounded_pct_audit_rule"]), "results/grounding/partners_normalisation_check.json", "partners_grounded_pct_audit_rule") | ||||
| 957 | + macro("groundPartnersPlainCasefoldPct", pct(pnc["partners_grounded_pct_naive_casefold"]), "results/grounding/partners_normalisation_check.json", "partners_grounded_pct_naive_casefold") | ||||
| 958 | + macro("groundPartnersCheckSample", n(pnc["sample_mentions"]), "results/grounding/partners_normalisation_check.json", "sample_mentions (reservoir sample)") | ||||
| 959 | + pnc_note = (f"The normalisation matters for partners: on a reservoir sample of {n(pnc['sample_mentions'])} mentions ({n(pnc['with_partners'])} with a partner), the rate is " | ||||
| 960 | + f"{pct(pnc['partners_grounded_pct_audit_rule'])}\\% under this rule and {pct(pnc['partners_grounded_pct_naive_casefold'])}\\% under a plain case-folded substring search " | ||||
| 961 | + "(\\texttt{results/grounding/partners\\_normalisation\\_check.json}). ") | ||||
| 937 | float_table("tab_grounding.tex", "tab:grounding", "Mechanical grounding of the structured fields of all stored mentions in the stored section text", tab, | 962 | float_table("tab_grounding.tex", "tab:grounding", "Mechanical grounding of the structured fields of all stored mentions in the stored section text", tab, | ||
| 938 | f"Population: all {n(M)} stored mentions, joined to the text of their section as stored (legacy June text for the S\\&P~500 10-Ks). A field is ``grounded'' " | 963 | f"Population: all {n(M)} stored mentions, joined to the text of their section as stored (legacy June text for the S\\&P~500 10-Ks). A field is ``grounded'' " | ||
| 939 | − "when every element of the list (years, partners, locations, technologies, case-folded) or the amount (tested in units, thousands, millions and billions with " | 964 | + "when every element of the list (years, partners, locations, technologies) or the amount (tested in units, thousands, millions and billions with " | ||
| 940 | "one to three significant digits) occurs literally in that text; a currency token is \\$, USD, dollar, million or billion within 40 characters of the number. " | 965 | "one to three significant digits) occurs literally in that text; a currency token is \\$, USD, dollar, million or billion within 40 characters of the number. " | ||
| 966 | + "Before the search, element and text are normalised identically: case-folded, punctuation replaced by spaces, whitespace collapsed, and a short alias list applied " | ||||
| 967 | + "to the element (U.S., USA $\\to$ united states; U.K. $\\to$ united kingdom; UAE; EU); the element must then occur as whole words. " + pnc_note + | ||||
| 941 | "Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a " | 968 | "Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a " | ||
| 942 | "measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); " | 969 | "measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); " | ||
| 943 | "precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field." + fb_note, | 970 | "precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field." + fb_note, | ||
@@ -977,7 +1004,8 @@ if sc_path.exists(): | |||||
| 977 | pb = dict((str(k), v) for k, v in cp["projects_by_key_branch"]); mb_ = dict((str(k), v) for k, v in cp.get("mentions_by_key_branch", [])) | 1004 | pb = dict((str(k), v) for k, v in cp["projects_by_key_branch"]); mb_ = dict((str(k), v) for k, v in cp.get("mentions_by_key_branch", [])) | ||
| 978 | tn = rows("top_name_tokens", R / "resolver") if (R / "resolver" / "top_name_tokens.csv").exists() else [] | 1005 | tn = rows("top_name_tokens", R / "resolver") if (R / "resolver" / "top_name_tokens.csv").exists() else [] | ||
| 979 | comp_note = (f" Under the current key {n(pb.get('name_token', 0))} projects ({100*pb.get('name_token', 0)/P:.1f}\\%) are keyed on a name token because the mention has no " | 1006 | comp_note = (f" Under the current key {n(pb.get('name_token', 0))} projects ({100*pb.get('name_token', 0)/P:.1f}\\%) are keyed on a name token because the mention has no " | ||
| 980 | − f"location, {n(pb.get('location', 0))} on a location and {n(cp.get('projects_keyed_general', 0))} on the fallback ``general''" | 1007 | + f"location, {n(pb.get('location', 0))} on a location and {n(pb.get('general', 0))} on the fallback ``general'' (as in the text; {n(cp.get('projects_keyed_general', 0))} keys end in " | ||
| 1008 | + f"the token ``general'' because a project name whose first token is the word ``general'' is not distinguished from the fallback)" | ||||
| 981 | + (f"; the single token ``{esc(tn[0]['name_tok'])}'' absorbs {n(tn[0]['mentions'])} mentions into {n(tn[0]['projects'])} projects (Table~\\ref{{tab:top_name_tokens}})" if tn else "") | 1009 | + (f"; the single token ``{esc(tn[0]['name_tok'])}'' absorbs {n(tn[0]['mentions'])} mentions into {n(tn[0]['projects'])} projects (Table~\\ref{{tab:top_name_tokens}})" if tn else "") | ||
| 982 | + ".") | 1010 | + ".") | ||
| 983 | tie_note = (f" The stored table is not exactly reproducible: {n(ti['projects_with_tie_at_max_confidence'])} projects have several mentions at the maximum confidence " | 1011 | tie_note = (f" The stored table is not exactly reproducible: {n(ti['projects_with_tie_at_max_confidence'])} projects have several mentions at the maximum confidence " | ||
@@ -985,8 +1013,8 @@ if sc_path.exists(): | |||||
| 985 | "descriptions while reproducing every project identifier." if ti else "") | 1013 | "descriptions while reproducing every project identifier." if ti else "") | ||
| 986 | float_table("tab_spec_curve.tex", "tab:spec_curve", "Specification curve of the project count over alternative resolution keys and confidence thresholds", tab, | 1014 | float_table("tab_spec_curve.tex", "tab:spec_curve", "Specification curve of the project count over alternative resolution keys and confidence thresholds", tab, | ||
| 987 | f"Every key is applied to the same {n(M)} stored mentions, rebuilt from \\texttt{{resolve.py}} in a temporary table on the master node; $\\dagger$ marks the key " | 1015 | f"Every key is applied to the same {n(M)} stored mentions, rebuilt from \\texttt{{resolve.py}} in a temporary table on the master node; $\\dagger$ marks the key " | ||
| 988 | − "of the released \\texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of projects formed by the mentions " | 1016 | + "of the released \\texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of mentions in the cluster that contains the " | ||
| 989 | − f"of the Table~\\ref{{tab:provenance}} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a " | 1017 | + f"highest-confidence mention of the Table~\\ref{{tab:provenance}} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a " | ||
| 990 | f"property of the key: {n(min(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))}--{n(max(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))} " | 1018 | f"property of the key: {n(min(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))}--{n(max(int(float(r['projects'])) for r in sc if float(r['threshold']) == t0_))} " | ||
| 991 | f"at the release threshold, and the single-mention share and projects-per-issuer statistics move with it." + comp_note + tie_note, | 1019 | f"at the release threshold, and the single-mention share and projects-per-issuer statistics move with it." + comp_note + tie_note, | ||
| 992 | "results/resolver/spec\\_curve.csv, composition.json, ties\\_idempotence.json, top\\_name\\_tokens.csv", size="\\scriptsize", colsep="3pt", fit=True) | 1020 | "results/resolver/spec\\_curve.csv, composition.json, ties\\_idempotence.json, top\\_name\\_tokens.csv", size="\\scriptsize", colsep="3pt", fit=True) | ||
@@ -1059,8 +1087,8 @@ if jo.exists(): | |||||
| 1059 | fbf = R / "focus" / "focus_coverage_by_form.csv"; sk = jload("focus/stats_keys.json") | 1087 | fbf = R / "focus" / "focus_coverage_by_form.csv"; sk = jload("focus/stats_keys.json") | ||
| 1060 | if fcp.exists() and fbf.exists(): | 1088 | if fcp.exists() and fbf.exists(): | ||
| 1061 | tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Item & " + H("Ledgered\\\\rows") + " & " + H("Over 9{,}000\\\\chars (\\%)") + " & " + H("Median\\\\chars") + " & " + | 1089 | tab = ("\\begin{tabular}{llrrrrrrrr}\n\\toprule\nForm & Item & " + H("Ledgered\\\\rows") + " & " + H("Over 9{,}000\\\\chars (\\%)") + " & " + H("Median\\\\chars") + " & " + | ||
| 1062 | − H("Share shown\\\\P10") + " & " + H("Share shown\\\\median") + " & " + H("Share shown\\\\P90") + " & " + H("Rows shown\\\\$< 25$\\% (\\%)") + " & " + | 1090 | + H("Share shown\\\\P10") + " & " + H("Share shown\\\\median") + " & " + H("Share shown\\\\P90") + " & " + H("Rows shown\\\\$<25$\\% (\\%)") + " & " + | ||
| 1063 | − H("Mentions from rows\\\\shown $< 25$\\% (\\%)") + " \\\\\n\\midrule\n") | 1091 | + H("Mentions from\\\\such rows (\\%)") + " \\\\\n\\midrule\n") | ||
| 1064 | def frow(lab1, lab2, r): | 1092 | def frow(lab1, lab2, r): | ||
| 1065 | return (f"{lab1} & {lab2} & {n(r['n'])} & {pct(r['pct_over_9000'])} & {n(r['chars_p50'])} & {100*float(r['shown_p10']):.0f} & {100*float(r['shown_p50']):.0f} & " | 1093 | return (f"{lab1} & {lab2} & {n(r['n'])} & {pct(r['pct_over_9000'])} & {n(r['chars_p50'])} & {100*float(r['shown_p10']):.0f} & {100*float(r['shown_p50']):.0f} & " | ||
| 1066 | f"{100*float(r['shown_p90']):.0f} & {pct(r['pct_shown_lt_25'])} & {pct(r['pct_mentions_from_lt25'])} \\\\\n") | 1094 | f"{100*float(r['shown_p90']):.0f} & {pct(r['pct_shown_lt_25'])} & {pct(r['pct_mentions_from_lt25'])} \\\\\n") | ||
@@ -1272,6 +1300,9 @@ jmacro("llmCallFailedInsideOutage", oc_, "llm_call_failed_inside_window", "resul | |||||
| 1272 | jmacro("llmCallFailedOutsideOutage", oc_, "silent_empty_results_outside_outage", "results/ledger/log_summary.json:october") | 1300 | jmacro("llmCallFailedOutsideOutage", oc_, "silent_empty_results_outside_outage", "results/ledger/log_summary.json:october") | ||
| 1273 | if isinstance(oc_.get("outside_window_by_reason"), dict): | 1301 | if isinstance(oc_.get("outside_window_by_reason"), dict): | ||
| 1274 | jmacro("llmRetryableTailLines", oc_["outside_window_by_reason"], "retryable_http_429_5xx", "results/ledger/log_summary.json:october.outside_window_by_reason") | 1302 | jmacro("llmRetryableTailLines", oc_["outside_window_by_reason"], "retryable_http_429_5xx", "results/ledger/log_summary.json:october.outside_window_by_reason") | ||
| 1303 | + if isinstance(oc_.get("llm_call_failed_by_reason"), dict): | ||||
| 1304 | + macro("llmRetryableInsideWindow", n(int(oc_["llm_call_failed_by_reason"].get("retryable_http_429_5xx", 0)) - int(oc_["outside_window_by_reason"].get("retryable_http_429_5xx", 0))), | ||||
| 1305 | + "results/ledger/log_summary.json:october", "llm_call_failed_by_reason.retryable_http_429_5xx - outside_window_by_reason.retryable_http_429_5xx (lines inside the 01:10-06:22 window)") | ||||
| 1275 | if isinstance(oc_.get("llm_call_failed_by_reason"), dict): | 1306 | if isinstance(oc_.get("llm_call_failed_by_reason"), dict): | ||
| 1276 | for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k) | 1307 | for k, v in oc_["llm_call_failed_by_reason"].items(): macro("llmFailed" + "".join(p.capitalize() for p in k.split("_")), n(v), "results/ledger/log_summary.json:october.llm_call_failed_by_reason", k) | ||
| 1277 | if isinstance(oc_.get("outage_window_from_log"), list) and len(oc_["outage_window_from_log"]) >= 2: | 1308 | if isinstance(oc_.get("outage_window_from_log"), list) and len(oc_["outage_window_from_log"]) >= 2: | ||
@@ -1350,6 +1381,8 @@ if oc_: | |||||
| 1350 | rs = oc_.get("resume_batch1", {}) | 1381 | rs = oc_.get("resume_batch1", {}) | ||
| 1351 | if rs: | 1382 | if rs: | ||
| 1352 | macro("resumeBatchMentions", n(rs.get("log_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_mentions", "log") | 1383 | macro("resumeBatchMentions", n(rs.get("log_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.log_mentions", "log") | ||
| 1384 | + macro("resumeBatchLedgerMentions", n(rs.get("ledger_mentions", 0)), "results/ledger/log_summary.json:october.resume_batch1.ledger_mentions", "ledger (sum of resume_batch1_vs_expected.csv:mentions)") | ||||
| 1385 | + macro("resumeBatchLedgerRows", n(rs.get("ledger_rows", 0)), "results/ledger/log_summary.json:october.resume_batch1.ledger_rows", "ledger rows of the batch") | ||||
| 1353 | macro("resumeBatchWindow", f"{str(rs['window'][0])[11:16]}--{str(rs['window'][1])[11:16]}", "results/ledger/log_summary.json:october.resume_batch1.window", "hh:mm") | 1386 | macro("resumeBatchWindow", f"{str(rs['window'][0])[11:16]}--{str(rs['window'][1])[11:16]}", "results/ledger/log_summary.json:october.resume_batch1.window", "hh:mm") | ||
| 1354 | rec = oc_.get("outage_rows_reconciliation", {}) | 1387 | rec = oc_.get("outage_rows_reconciliation", {}) | ||
| 1355 | macro("logExtractionCompleteSum", n(rec.get("extraction_complete_sum_sections", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "extraction_complete_sum_sections") | 1388 | macro("logExtractionCompleteSum", n(rec.get("extraction_complete_sum_sections", 0)), "results/ledger/log_summary.json:october.outage_rows_reconciliation", "extraction_complete_sum_sections") | ||
@@ -1363,6 +1396,10 @@ if oc_: | |||||
| 1363 | if bb.get("rows_in_outage_window"): | 1396 | if bb.get("rows_in_outage_window"): | ||
| 1364 | macro("outageSurvivingRows", n(bb["rows_in_outage_window"][0]), "results/ledger/backfill_block.json", "rows_in_outage_window[0]") | 1397 | macro("outageSurvivingRows", n(bb["rows_in_outage_window"][0]), "results/ledger/backfill_block.json", "rows_in_outage_window[0]") | ||
| 1365 | macro("outageSurvivingMentions", n(bb["rows_in_outage_window"][1]), "results/ledger/backfill_block.json", "rows_in_outage_window[1]") | 1398 | macro("outageSurvivingMentions", n(bb["rows_in_outage_window"][1]), "results/ledger/backfill_block.json", "rows_in_outage_window[1]") | ||
| 1399 | +owc = jload("ledger/outage_window_rows_check.json") | ||||
| 1400 | +if owc: | ||||
| 1401 | + macro("outageSurvivingProductive", n(owc["rows_with_mentions"]), "results/ledger/outage_window_rows_check.json", "rows_with_mentions (read-only re-derivation, Stage F)") | ||||
| 1402 | + macro("outageSurvivingZeroRows", n(owc["rows_zero_mentions"]), "results/ledger/outage_window_rows_check.json", "rows_zero_mentions") | ||||
| 1366 | if bb.get("min_extracted_at"): | 1403 | if bb.get("min_extracted_at"): | ||
| 1367 | macro("backfillTimestamp", longdate(bb["min_extracted_at"]) + ", " + str(bb["min_extracted_at"])[11:16], "results/ledger/backfill_block.json:min_extracted_at", "date, hh:mm") | 1404 | macro("backfillTimestamp", longdate(bb["min_extracted_at"]) + ", " + str(bb["min_extracted_at"])[11:16], "results/ledger/backfill_block.json:min_extracted_at", "date, hh:mm") | ||
| 1368 | if rb1.exists(): | 1405 | if rb1.exists(): | ||
@@ -1381,6 +1418,12 @@ if gr.exists(): | |||||
| 1381 | if gt: | 1418 | if gt: | ||
| 1382 | macro("gateRejectedPct", f"{100*gt['gate_rejected_zero']/gt['ledger_rows']:.1f}", "results/ledger/gate_pass_totals.json", "100*gate_rejected_zero/ledger_rows") | 1419 | macro("gateRejectedPct", f"{100*gt['gate_rejected_zero']/gt['ledger_rows']:.1f}", "results/ledger/gate_pass_totals.json", "100*gate_rejected_zero/ledger_rows") | ||
| 1383 | macro("usdPerOctoberRow", f"{gt['usd_per_october_row_paper_total_775']:.4f}", "results/ledger/gate_pass_totals.json", "usd_per_october_row_paper_total_775") | 1420 | macro("usdPerOctoberRow", f"{gt['usd_per_october_row_paper_total_775']:.4f}", "results/ledger/gate_pass_totals.json", "usd_per_october_row_paper_total_775") | ||
| 1421 | + gpr_ = R / "ledger" / "gate_pass_by_run.csv" | ||||
| 1422 | + if gpr_.exists() and di("total_cost_usd"): | ||||
| 1423 | + calls_both = sum(int(r["calls"]) for r in rows(gpr_.stem, gpr_.parent)) | ||||
| 1424 | + macro("callsBothRuns", n(calls_both), "results/ledger/gate_pass_by_run.csv", "sum(calls) over the two runs") | ||||
| 1425 | + macro("usdPerCallBothRuns", f"{float(str(di('total_cost_usd')).replace('{,}', '')) / calls_both:.4f}", "results/ledger/gate_pass_by_run.csv, data/documented_inputs.json:total_cost_usd", | ||||
| 1426 | + "total_cost_usd / sum(calls) over both runs (alternative basis, author to confirm)") | ||||
| 1384 | if cvd: | 1427 | if cvd: | ||
| 1385 | macro("confExactDefault", n(cvd["confidence_exact_0600_total"]), "results/ledger/code_vs_data_checks.json", "confidence_exact_0600_total") | 1428 | macro("confExactDefault", n(cvd["confidence_exact_0600_total"]), "results/ledger/code_vs_data_checks.json", "confidence_exact_0600_total") | ||
| 1386 | for run_, cnt in cvd.get("confidence_exact_0600_by_run", []): | 1429 | for run_, cnt in cvd.get("confidence_exact_0600_by_run", []): | ||
@@ -1550,6 +1593,8 @@ if wst.exists(): | |||||
| 1550 | wp = jload("runs/within_issuer_paired_10q.json") | 1593 | wp = jload("runs/within_issuer_paired_10q.json") | ||
| 1551 | if wp: | 1594 | if wp: | ||
| 1552 | macro("pairedIssuers", n(wp["issuers_paired"]), "results/runs/within_issuer_paired_10q.json", "issuers_paired"); macro("pairedDeltaPp", f"{100*wp['mean_diff_productive_share_oct_minus_june']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*mean_diff_productive_share_oct_minus_june"); macro("pairedSePp", f"{100*wp['se']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*se") | 1595 | macro("pairedIssuers", n(wp["issuers_paired"]), "results/runs/within_issuer_paired_10q.json", "issuers_paired"); macro("pairedDeltaPp", f"{100*wp['mean_diff_productive_share_oct_minus_june']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*mean_diff_productive_share_oct_minus_june"); macro("pairedSePp", f"{100*wp['se']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*se") | ||
| 1596 | + macro("pairedProductiveJune", f"{100*wp['june_mean_productive_share']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*june_mean_productive_share (mean of per-issuer shares)") | ||||
| 1597 | + macro("pairedProductiveOctober", f"{100*wp['oct_mean_productive_share']:.1f}", "results/runs/within_issuer_paired_10q.json", "100*oct_mean_productive_share (mean of per-issuer shares)") | ||||
| 1553 | if t6.exists() and t6r: | 1598 | if t6.exists() and t6r: | ||
| 1554 | crd = {r["cohort"]: r for r in rows(t6.stem, t6.parent)} | 1599 | crd = {r["cohort"]: r for r in rows(t6.stem, t6.parent)} | ||
| 1555 | macro("cohortBothPerIssuerAll", pct(crd["both"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "both per_issuer_all"); macro("cohortRussellOnlyPerIssuer", pct(crd["russell_only"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "russell_only per_issuer_all"); macro("cohortBothPerIssuerOctober", pct(crd["both"]["per_issuer_october_only"]), "results/runs/table6_cohort.csv", "both per_issuer_october_only") | 1600 | macro("cohortBothPerIssuerAll", pct(crd["both"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "both per_issuer_all"); macro("cohortRussellOnlyPerIssuer", pct(crd["russell_only"]["per_issuer_all"]), "results/runs/table6_cohort.csv", "russell_only per_issuer_all"); macro("cohortBothPerIssuerOctober", pct(crd["both"]["per_issuer_october_only"]), "results/runs/table6_cohort.csv", "both per_issuer_october_only") | ||
@@ -1622,7 +1667,7 @@ if rta.exists(): | |||||
| 1622 | # -- financing class (results/fp_classes) -- | 1667 | # -- financing class (results/fp_classes) -- | ||
| 1623 | if fps: | 1668 | if fps: | ||
| 1624 | mm = fps["mentions_matched"]; macro("finMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]"); macro("finMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]") | 1669 | mm = fps["mentions_matched"]; macro("finMentions", n(mm[0]), "results/fp_classes/summary.json", "mentions_matched[0]"); macro("finMentionsPct", pct(mm[2]), "results/fp_classes/summary.json", "mentions_matched[2]") | ||
| 1625 | − macro("finAmountB", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3 (T USD)"); macro("finAmountDenomB", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3 (T USD)") | 1670 | + macro("finAmountT", f"{fps['matched_amount_gusd'][0]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[0]/1e3 (T USD; the source field is in billions)"); macro("finAmountDenomT", f"{fps['matched_amount_gusd'][1]/1e3:.1f}", "results/fp_classes/summary.json", "matched_amount_gusd[1]/1e3 (T USD; the source field is in billions)") | ||
| 1626 | macro("finAmountSharePct", f"{100*fps['matched_amount_gusd'][0]/fps['matched_amount_gusd'][1]:.1f}", "results/fp_classes/summary.json", "100*matched_amount_gusd[0]/[1]") | 1671 | macro("finAmountSharePct", f"{100*fps['matched_amount_gusd'][0]/fps['matched_amount_gusd'][1]:.1f}", "results/fp_classes/summary.json", "100*matched_amount_gusd[0]/[1]") | ||
| 1627 | us_ = fps["unregistered_sales_matched"]; macro("finUnregPct", f"{100*us_[0]/us_[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]"); macro("finUnregN", n(us_[0]), "results/fp_classes/summary.json", "unregistered_sales_matched[0]"); macro("finUnregDenom", n(us_[1]), "results/fp_classes/summary.json", "unregistered_sales_matched[1]") | 1672 | us_ = fps["unregistered_sales_matched"]; macro("finUnregPct", f"{100*us_[0]/us_[1]:.1f}", "results/fp_classes/summary.json", "100*unregistered_sales_matched[0]/[1]"); macro("finUnregN", n(us_[0]), "results/fp_classes/summary.json", "unregistered_sales_matched[0]"); macro("finUnregDenom", n(us_[1]), "results/fp_classes/summary.json", "unregistered_sales_matched[1]") | ||
| 1628 | fs_ = fps["financial_statements_matched"]; macro("finFinStatementsPct", f"{100*fs_[0]/fs_[1]:.1f}", "results/fp_classes/summary.json", "100*financial_statements_matched[0]/[1]") | 1673 | fs_ = fps["financial_statements_matched"]; macro("finFinStatementsPct", f"{100*fs_[0]/fs_[1]:.1f}", "results/fp_classes/summary.json", "100*financial_statements_matched[0]/[1]") | ||
@@ -1740,8 +1785,8 @@ kge = {r["node_type"]: r["n"] for r in kg if r["k"] == "edges"} | |||||
| 1740 | for e_, nm in (("owns", "kgEdgesOwns"), ("located_in", "kgEdgesLocatedIn"), ("uses", "kgEdgesUses")): | 1785 | for e_, nm in (("owns", "kgEdgesOwns"), ("located_in", "kgEdgesLocatedIn"), ("uses", "kgEdgesUses")): | ||
| 1741 | if e_ in kge: macro(nm, n(kge[e_]), "data/kg.csv", f"edges {e_}") | 1786 | if e_ in kge: macro(nm, n(kge[e_]), "data/kg.csv", f"edges {e_}") | ||
| 1742 | macro("projectsPerIssuerMax", n(rows("projects_per_issuer")[0]["mx"]), "data/projects_per_issuer.csv", "mx") | 1787 | macro("projectsPerIssuerMax", n(rows("projects_per_issuer")[0]["mx"]), "data/projects_per_issuer.csv", "mx") | ||
| 1743 | −# -- pipeline constants read from the frozen snapshot and the archived pipeline code (manuscripts/.../pipeline_code_2324b58) -- | 1788 | +# -- pipeline constants read from the frozen snapshot (code/audit/spid_snapshot, in the repository) -- | ||
| 1744 | −PIPE = ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" | 1789 | +PIPE = ROOT.parent.parent / "manuscripts" / "FIRC-0001" / "pipeline_code_2324b58" # archived pipeline code, outside the repository; only used to cross-check documented inputs | ||
| 1745 | llm_src = _snapshot("llm.py"); cfg_src = _snapshot("config.py"); pipe_src = _snapshot("pipeline.py") | 1790 | llm_src = _snapshot("llm.py"); cfg_src = _snapshot("config.py"); pipe_src = _snapshot("pipeline.py") | ||
| 1746 | v = rxs(llm_src, r"m\.start\(\) - (\d+)", int); macro("focusWindowBefore", n(v), rel(llm_src), "_focus: characters before a gate hit") if v else None | 1791 | v = rxs(llm_src, r"m\.start\(\) - (\d+)", int); macro("focusWindowBefore", n(v), rel(llm_src), "_focus: characters before a gate hit") if v else None | ||
| 1747 | v = rxs(llm_src, r"m\.end\(\) \+ (\d+)", int); macro("focusWindowAfter", n(v), rel(llm_src), "_focus: characters after a gate hit") if v else None | 1792 | v = rxs(llm_src, r"m\.end\(\) \+ (\d+)", int); macro("focusWindowAfter", n(v), rel(llm_src), "_focus: characters after a gate hit") if v else None | ||
@@ -1759,12 +1804,23 @@ v = rxs(llm_src, r"project_name',''\)\)\[:(\d+)\]", int); macro("idNameChars", n | |||||
| 1759 | v = rxs(llm_src, r"retries: int = (\d+)", int); macro("retries", word(v), rel(llm_src), "_call retries") if v else None | 1804 | v = rxs(llm_src, r"retries: int = (\d+)", int); macro("retries", word(v), rel(llm_src), "_call retries") if v else None | ||
| 1760 | v = rxs(cfg_src, r"llm_timeout: float = ([0-9.]+)", float); macro("timeoutSeconds", n(v), rel(cfg_src), "llm_timeout") if v else None | 1805 | v = rxs(cfg_src, r"llm_timeout: float = ([0-9.]+)", float); macro("timeoutSeconds", n(v), rel(cfg_src), "llm_timeout") if v else None | ||
| 1761 | v = rxs(pipe_src, r"min_words: int = (\d+)", int); macro("minWords", n(v), rel(pipe_src), "min_words") if v else None | 1806 | v = rxs(pipe_src, r"min_words: int = (\d+)", int); macro("minWords", n(v), rel(pipe_src), "min_words") if v else None | ||
| 1762 | −v = rxs(PIPE / "src/parsing/chunker.py", r"chunk_size: int = (\d+)", int); macro("chunkWords", n(v), "src/parsing/chunker.py", "chunk_size") if v else None | 1807 | +# -- constants of the archived pipeline code (manuscripts/FIRC-0001/pipeline_code_2324b58, outside this repository): recorded in data/documented_inputs.json with the | ||
| 1763 | −v = rxs(PIPE / "src/parsing/chunker.py", r"overlap: int = (\d+)", int); macro("chunkOverlap", n(v), "src/parsing/chunker.py", "overlap") if v else None | 1808 | +# snapshot path and the SHA-256 of each source file, so that a bare clone regenerates every macro; when the archive is reachable the value is re-read and must agree (Stage F, N1) | ||
| 1764 | −v = rxs(PIPE / "src/parsing/html_parser.py", r"max_file_size_mb\", (\d+)", int); macro("maxFileMb", n(v), "src/parsing/html_parser.py", "max_file_size_mb default") if v else None | 1809 | +def pipe_const(key, name, relpath, pattern, cast, fmt): | ||
| 1765 | −v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"COOLDOWN_SECONDS = ([0-9.]+)", float); macro("cooldownSeconds", n(v), "src/acquisition/bulk_downloader.py", "COOLDOWN_SECONDS") if v else None | 1810 | + e = DI.get(key) | ||
| 1766 | −v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"SEC allows (\d+) req/s", int); macro("secRateLimit", word(v), "src/acquisition/bulk_downloader.py (docstring) / secfairaccess", "requests per second per IP") if v else None | 1811 | + if e is None: return | ||
| 1767 | −v = rxs(PIPE / "src/acquisition/bulk_downloader.py", r"rate: float = ([0-9.]+)", float); macro("acqGatewayDivisor", word(v), "src/acquisition/bulk_downloader.py", "default aggregate rate shared behind one IP") if v else None | 1812 | + p_ = PIPE / relpath | ||
| 1813 | + if p_.exists(): | ||||
| 1814 | + live = rxs(p_, pattern, cast) | ||||
| 1815 | + if live is not None and cast(e["value"]) != live: raise SystemExit(f"documented input {key} = {e['value']} disagrees with {p_}: {live}") | ||||
| 1816 | + if e.get("sha256") and hashlib.sha256(p_.read_bytes()).hexdigest() != e["sha256"]: raise SystemExit(f"documented input {key}: {p_} does not match the recorded SHA-256") | ||||
| 1817 | + macro(name, fmt(cast(e["value"])), f"data/documented_inputs.json:{key}", e.get("source", "")) | ||||
| 1818 | +pipe_const("pipeline_chunk_words", "chunkWords", "src/parsing/chunker.py", r"chunk_size: int = (\d+)", int, n) | ||||
| 1819 | +pipe_const("pipeline_chunk_overlap", "chunkOverlap", "src/parsing/chunker.py", r"overlap: int = (\d+)", int, n) | ||||
| 1820 | +pipe_const("pipeline_max_file_mb", "maxFileMb", "src/parsing/html_parser.py", r"max_file_size_mb\", (\d+)", int, n) | ||||
| 1821 | +pipe_const("pipeline_cooldown_seconds", "cooldownSeconds", "src/acquisition/bulk_downloader.py", r"COOLDOWN_SECONDS = ([0-9.]+)", float, n) | ||||
| 1822 | +pipe_const("pipeline_sec_rate_limit", "secRateLimit", "src/acquisition/bulk_downloader.py", r"SEC allows (\d+) req/s", int, word) | ||||
| 1823 | +pipe_const("pipeline_acq_gateway_divisor", "acqGatewayDivisor", "src/acquisition/bulk_downloader.py", r"rate: float = ([0-9.]+)", float, word) | ||||
| 1768 | for key_, nm in (("acq_new_docs", "acqNewDocs"), ("acq_hours", "acqHours"), ("acq_req_per_sec", "acqReqPerSec"), ("acq_nodes", "acqNodes"), ("acq_node_rate", "acqNodeRate"), ("acq_gateway_rate", "acqGatewayRate"), | 1824 | for key_, nm in (("acq_new_docs", "acqNewDocs"), ("acq_hours", "acqHours"), ("acq_req_per_sec", "acqReqPerSec"), ("acq_nodes", "acqNodes"), ("acq_node_rate", "acqNodeRate"), ("acq_gateway_rate", "acqGatewayRate"), | ||
| 1769 | ("parse_filings_per_sec", "parseFilingsPerSec"), ("parse_minutes_range", "parseMinutesRange"), ("embed_minutes_range", "embedMinutesRange"), ("merge_minutes", "mergeMinutes"), ("total_cost_usd", "totalCostUsd"), | 1825 | ("parse_filings_per_sec", "parseFilingsPerSec"), ("parse_minutes_range", "parseMinutesRange"), ("embed_minutes_range", "embedMinutesRange"), ("merge_minutes", "mergeMinutes"), ("total_cost_usd", "totalCostUsd"), | ||
| 1770 | ("stability_sections", "stabilitySections"), ("stability_calls", "stabilityCalls"), ("embedding_dim", "embeddingDim"), ("factor_anchors", "factorAnchors"), ("tenk_sections_max", "tenKsectionsMax"), ("tenq_sections_n", "tenQsectionsN")): | 1826 | ("stability_sections", "stabilitySections"), ("stability_calls", "stabilityCalls"), ("embedding_dim", "embeddingDim"), ("factor_anchors", "factorAnchors"), ("tenk_sections_max", "tenKsectionsMax"), ("tenq_sections_n", "tenQsectionsN")): | ||
modified
data/documented_inputs.json
+45 −3
@@ -77,8 +77,8 @@ | |||||
| 77 | }, | 77 | }, | ||
| 78 | "total_cost_usd": { | 78 | "total_cost_usd": { | ||
| 79 | "value": "775", | 79 | "value": "775", | ||
| 80 | − "source": "author-reported total of the October 2026 extraction (docs/research_log.md); the API invoice is not archived", | 80 | + "source": "author-reported total recorded in docs/research_log.md against the 2026-10-27/28 entry 'Full LLM extraction of 968,134 new sections ... ≈ 775 USD total' (October run); the review's constraints note (peer_reviews/FIRC-0001/_constraints.md) attributes the same ≈ 775 USD to both runs (153,894 + 968,134 sections); the API invoice is not archived; basis to be confirmed by the authors (Stage F, N9)", | ||
| 81 | − "status": "author-reported" | 81 | + "status": "author-reported; basis (October run only vs both runs) to be confirmed by the authors" | ||
| 82 | }, | 82 | }, | ||
| 83 | "stability_sections": { | 83 | "stability_sections": { | ||
| 84 | "value": "500", | 84 | "value": "500", | ||
@@ -119,5 +119,47 @@ | |||||
| 119 | "value": "29.1", | 119 | "value": "29.1", | ||
| 120 | "source": "N0 schema probe (results/n0_probe, commit 7680606): parquet corpus 1,342,283 sections / 29.1 GB on the master node", | 120 | "source": "N0 schema probe (results/n0_probe, commit 7680606): parquet corpus 1,342,283 sections / 29.1 GB on the master node", | ||
| 121 | "status": "node measurement" | 121 | "status": "node measurement" | ||
| 122 | + }, | ||||
| 123 | + "pipeline_chunk_words": { | ||||
| 124 | + "value": 512, | ||||
| 125 | + "source": "chunk_size default of the embedding chunker (chunk_size: int = 512); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/parsing/chunker.py, SHA-256 716a4013fb839a68ec613aa5f0c91e8da4c8a8135051fb95fec5350d35fc3c79; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable", | ||||
| 126 | + "snapshot_path": "manuscripts/FIRC-0001/pipeline_code_2324b58/src/parsing/chunker.py", | ||||
| 127 | + "sha256": "716a4013fb839a68ec613aa5f0c91e8da4c8a8135051fb95fec5350d35fc3c79", | ||||
| 128 | + "status": "code constant (archived pipeline snapshot)" | ||||
| 129 | + }, | ||||
| 130 | + "pipeline_chunk_overlap": { | ||||
| 131 | + "value": 64, | ||||
| 132 | + "source": "overlap default of the embedding chunker (overlap: int = 64); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/parsing/chunker.py, SHA-256 716a4013fb839a68ec613aa5f0c91e8da4c8a8135051fb95fec5350d35fc3c79; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable", | ||||
| 133 | + "snapshot_path": "manuscripts/FIRC-0001/pipeline_code_2324b58/src/parsing/chunker.py", | ||||
| 134 | + "sha256": "716a4013fb839a68ec613aa5f0c91e8da4c8a8135051fb95fec5350d35fc3c79", | ||||
| 135 | + "status": "code constant (archived pipeline snapshot)" | ||||
| 136 | + }, | ||||
| 137 | + "pipeline_max_file_mb": { | ||||
| 138 | + "value": 50, | ||||
| 139 | + "source": "max_file_size_mb default of the HTML parser (html_cfg.get(\"max_file_size_mb\", 50)); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/parsing/html_parser.py, SHA-256 61aca675ff4652e3479f50a10ce29503f66f7f74ffd4f78df0b87ad517b1908e; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable", | ||||
| 140 | + "snapshot_path": "manuscripts/FIRC-0001/pipeline_code_2324b58/src/parsing/html_parser.py", | ||||
| 141 | + "sha256": "61aca675ff4652e3479f50a10ce29503f66f7f74ffd4f78df0b87ad517b1908e", | ||||
| 142 | + "status": "code constant (archived pipeline snapshot)" | ||||
| 143 | + }, | ||||
| 144 | + "pipeline_cooldown_seconds": { | ||||
| 145 | + "value": 65.0, | ||||
| 146 | + "source": "COOLDOWN_SECONDS = 65.0 after an SEC 403/429; manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/acquisition/bulk_downloader.py, SHA-256 862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable", | ||||
| 147 | + "snapshot_path": "manuscripts/FIRC-0001/pipeline_code_2324b58/src/acquisition/bulk_downloader.py", | ||||
| 148 | + "sha256": "862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159", | ||||
| 149 | + "status": "code constant (archived pipeline snapshot)" | ||||
| 150 | + }, | ||||
| 151 | + "pipeline_sec_rate_limit": { | ||||
| 152 | + "value": 10, | ||||
| 153 | + "source": "module docstring 'SEC allows 10 req/s per IP' (sec.gov fair-access policy); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/acquisition/bulk_downloader.py, SHA-256 862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable", | ||||
| 154 | + "snapshot_path": "manuscripts/FIRC-0001/pipeline_code_2324b58/src/acquisition/bulk_downloader.py", | ||||
| 155 | + "sha256": "862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159", | ||||
| 156 | + "status": "code constant (archived pipeline snapshot)" | ||||
| 157 | + }, | ||||
| 158 | + "pipeline_acq_gateway_divisor": { | ||||
| 159 | + "value": 9.0, | ||||
| 160 | + "source": "BulkDownloader default rate: float = 9.0 requests per second shared behind one IP; manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/acquisition/bulk_downloader.py, SHA-256 862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable", | ||||
| 161 | + "snapshot_path": "manuscripts/FIRC-0001/pipeline_code_2324b58/src/acquisition/bulk_downloader.py", | ||||
| 162 | + "sha256": "862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159", | ||||
| 163 | + "status": "code constant (archived pipeline snapshot)" | ||||
| 122 | } | 164 | } | ||
| 123 | −} | ||||
| \ No newline at end of file | |||||
| 165 | +} | ||||
modified
paper/appendix/appendix.tex
+1 −1
@@ -81,7 +81,7 @@ Author order, affiliations and CRediT contributions & Pending confirmation by th | |||||
| 81 | 81 | ||||
| 82 | \lstinputlisting{../results/runs/git_diff_0304133_2324b58.patch} | 82 | \lstinputlisting{../results/runs/git_diff_0304133_2324b58.patch} | ||
| 83 | 83 | ||||
| 84 | −\paragraph{Mentions and projects by cohort, run and form.} \Cref{tab:june_october} reconciles the mentions of the S\&P~500 cohort (\texttt{in\_sp500} true) and of the Russell-only cohort by run and form; the June block is identified by the backfill timestamp. The S\&P~500 cohort's 10-Q mentions come from both runs for the same issuers, which permits a within-issuer comparison: on the \SPIDpairedIssuers\ issuers with 10-Qs in both runs, the productive share is \SPIDwithinProductiveJune\% in June and \SPIDwithinProductiveOctober\% in October (paired difference \SPIDpairedDeltaPp\ percentage points, standard error \SPIDpairedSePp), mentions per row \SPIDwithinMentionsPerRowJune\ and \SPIDwithinMentionsPerRowOctober, mean confidence \SPIDwithinConfJune\ and \SPIDwithinConfOctober, and the M\&A share \SPIDwithinMaShareJune\% and \SPIDwithinMaShareOctober\% (\texttt{results/runs/within\_sp500\_*.csv}). For 10-Qs, whose text went through the same parser in both runs, the two runs therefore behave alike; the cohort difference of \Cref{sec:time} comes from the volume of sections and from the 10-K segmenter, not from the extractor. | 84 | +\paragraph{Mentions and projects by cohort, run and form.} \Cref{tab:june_october} reconciles the mentions of the S\&P~500 cohort (\texttt{in\_sp500} true) and of the Russell-only cohort by run and form; the June block is identified by the backfill timestamp. The S\&P~500 cohort's 10-Q mentions come from both runs for the same issuers, which permits a within-issuer comparison. Pooled over the cohort's 10-Q ledger rows, the productive share is \SPIDwithinProductiveJune\% in June and \SPIDwithinProductiveOctober\% in October; on the \SPIDpairedIssuers\ issuers with 10-Qs in both runs, the per-issuer means are \SPIDpairedProductiveJune\% and \SPIDpairedProductiveOctober\% (paired difference \SPIDpairedDeltaPp\ percentage points, standard error \SPIDpairedSePp). The cohort is the same in both files (\texttt{in\_sp500} of the issuer), but \Cref{tab:june_october} counts stored mentions whereas the within-cohort files sum the ledger's mention counts, which exceed the stored ones by the identifier collisions of \Cref{sec:ledger}. Mentions per row are \SPIDwithinMentionsPerRowJune\ and \SPIDwithinMentionsPerRowOctober, mean confidence \SPIDwithinConfJune\ and \SPIDwithinConfOctober, and the M\&A share \SPIDwithinMaShareJune\% and \SPIDwithinMaShareOctober\% (\texttt{results/runs/within\_sp500\_*.csv}). For 10-Qs, whose text went through the same parser in both runs, the two runs therefore behave alike; the cohort difference of \Cref{sec:time} comes from the volume of sections and from the 10-K segmenter, not from the extractor. | ||
| 85 | 85 | ||||
| 86 | \input{tables/tab_june_october} | 86 | \input{tables/tab_june_october} | ||
| 87 | 87 | ||||
modified
paper/main.pdf
+0 −0
Binary file not shown.
modified
paper/main.tex
+1 −1
@@ -117,7 +117,7 @@ | |||||
| 117 | \newcommand{\WPversion}{1.1 (revised after peer review; research database under validation)} | 117 | \newcommand{\WPversion}{1.1 (revised after peer review; research database under validation)} | ||
| 118 | \newcommand{\WPfunding}{Research funded by the FIRC (Fonds institutionnel de recherche et de création), Université du Québec en Outaouais.} | 118 | \newcommand{\WPfunding}{Research funded by the FIRC (Fonds institutionnel de recherche et de création), Université du Québec en Outaouais.} | ||
| 119 | \newcommand{\WPabstract}{% | 119 | \newcommand{\WPabstract}{% | ||
| 120 | − Can strategic projects be identified, classified, dated and linked across the SEC filings of a broad cross-section of U.S.-listed issuers? We document SPID, a project-level research database built for a Russell~3000 proxy (iShares IWV holdings, \SPIDrussellDistinctCiks\ of \SPIDrussellIndexConstituents\ index constituents) merged with the S\&P~500: \SPIDfilingsTotal\ Forms 10-K, 10-Q and 8-K of \SPIDuniverseN\ registrants filed since 2010, segmented into \SPIDsectionsTotal\ sections, of which \SPIDgatePassTotal\ reached a large-language-model extractor in JSON mode. The output is \SPIDmentions\ section-linked candidate mentions and \SPIDprojectCountCurrentKey\ candidate projects under the current resolution key (\SPIDprojectsKeyTypeDropped--\SPIDprojectsKeyStrict\ under alternative keys). Mechanical audits find \SPIDgroundAllPct\% of structured fields verbatim in the source section, a median \SPIDmdnaShownMedianPct\% of a 10-K MD\&A shown to the model, and \SPIDfinMentionsPct\% of mentions describing financing instruments. Precision and recall against human labels are not yet measured; the annotation frame is delivered. SPID is a research instrument under validation, not a census of projects.% | 120 | + Can strategic projects be identified, classified, dated and linked across the SEC filings of a broad cross-section of U.S.-listed issuers? We document SPID, a project-level research database built for a Russell~3000 proxy (iShares IWV holdings, \SPIDrussellDistinctCiks\ of \SPIDrussellIndexConstituents\ index constituents) merged with the S\&P~500: \SPIDfilingsTotal\ Forms 10-K, 10-Q and 8-K of \SPIDuniverseN\ registrants filed since 2010, segmented into \SPIDsectionsTotal\ sections, of which \SPIDgatePassTotal\ reached a large-language-model extractor in JSON mode. The output is \SPIDmentions\ section-linked candidate mentions and \SPIDprojectCountCurrentKey\ candidate projects under the current key (\SPIDprojectsKeyTypeDropped--\SPIDprojectsKeyStrict\ under alternative keys). Mechanical audits find every structured field verbatim in the source section for \SPIDgroundAllPct\% of mentions, a median \SPIDmdnaShownMedianPct\% of a 10-K MD\&A shown to the model, and \SPIDfinMentionsPct\% of mentions describing financing instruments. Precision and recall against human labels are not yet measured; the annotation frame is delivered. SPID is an instrument under validation, not a census of projects.% | ||
| 121 | } | 121 | } | ||
| 122 | \newcommand{\WPkeywords}{SEC filings, text as data, corporate investment, strategic projects, information extraction, large language models, entity resolution, Russell 3000, measurement validation} | 122 | \newcommand{\WPkeywords}{SEC filings, text as data, corporate investment, strategic projects, information extraction, large language models, entity resolution, Russell 3000, measurement validation} | ||
| 123 | \newcommand{\WPjel}{C55, C81, G31, G38, M41, O33} | 123 | \newcommand{\WPjel}{C55, C81, G31, G38, M41, O33} | ||
modified
paper/sections/conclusion.tex
+1 −1
@@ -29,7 +29,7 @@ Beyond the identification limits above, the current build has concrete technical | |||||
| 29 | 29 | ||||
| 30 | \paragraph{Code.} The acquisition, parsing and extraction pipeline is revision \texttt{2324b58} of the private repository \texttt{uqo-research-db.git}; the \SPIDsnapshotFilesN\ files that define the extraction instrument and the resolver are archived with the paper (\texttt{code/audit/spid\_snapshot/}, with SHA-256 checksums). The exhibit code (\texttt{code/export\_stats.py}, \texttt{code/make\_tables.py}), the exports (\texttt{data/}), the audit scripts of this revision (\texttt{code/audit/}) and their outputs (\texttt{results/}) are released with the paper; \texttt{run\_all.sh} regenerates every table, figure and inline number from the exports, and \texttt{tests/} (\SPIDtestsN\ checks) verifies that the exports reconcile. The master-node environment is pinned (\texttt{requirements.lock}, \SPIDrequirementsLockPackages\ packages; Python \SPIDpythonVersion; DuckDB \SPIDduckdbVersion); no licence file has been committed pending the authors' decision (MIT for code and CC BY 4.0 for derived tables are proposed). | 30 | \paragraph{Code.} The acquisition, parsing and extraction pipeline is revision \texttt{2324b58} of the private repository \texttt{uqo-research-db.git}; the \SPIDsnapshotFilesN\ files that define the extraction instrument and the resolver are archived with the paper (\texttt{code/audit/spid\_snapshot/}, with SHA-256 checksums). The exhibit code (\texttt{code/export\_stats.py}, \texttt{code/make\_tables.py}), the exports (\texttt{data/}), the audit scripts of this revision (\texttt{code/audit/}) and their outputs (\texttt{results/}) are released with the paper; \texttt{run\_all.sh} regenerates every table, figure and inline number from the exports, and \texttt{tests/} (\SPIDtestsN\ checks) verifies that the exports reconcile. The master-node environment is pinned (\texttt{requirements.lock}, \SPIDrequirementsLockPackages\ packages; Python \SPIDpythonVersion; DuckDB \SPIDduckdbVersion); no licence file has been committed pending the authors' decision (MIT for code and CC BY 4.0 for derived tables are proposed). | ||
| 31 | 31 | ||||
| 32 | −\paragraph{Reproducibility.} The exhibits are reproducible from the released materials. The databases are regenerable by the authors from the repository, the frozen universe artefact, the download logs and the extraction ledger; the paid extraction step is resumable and ledgered but not deterministic (\Cref{sec:contract}), and the complete October run cost about \SPIDtotalCostUsd~USD at October 2026 prices. The repository contains no credentials and no private infrastructure details. The database does not contain personal data; the only names it stores are those of corporations, places and technologies as they appear in public filings. | 32 | +\paragraph{Reproducibility.} The exhibits are reproducible from the released materials. The databases are regenerable by the authors from the repository, the frozen universe artefact, the download logs and the extraction ledger; the paid extraction step is resumable and ledgered but not deterministic (\Cref{sec:contract}), and the extraction cost about \SPIDtotalCostUsd~USD at October 2026 prices (recorded against the October run; whether the total also covers the June block is to be confirmed by the authors). The repository contains no credentials and no private infrastructure details. The database does not contain personal data; the only names it stores are those of corporations, places and technologies as they appear in public filings. | ||
| 33 | 33 | ||||
| 34 | \subsection{Conclusion} | 34 | \subsection{Conclusion} | ||
| 35 | 35 | ||||
modified
paper/sections/construct.tex
+3 −3
@@ -17,7 +17,7 @@ The unit of measurement is a \emph{distinct, identifiable, issuer-associated str | |||||
| 17 | \item \texttt{uncertain\_candidate}: a plausible initiative without enough identity evidence (no name, location, counterparty or amount) to be counted. | 17 | \item \texttt{uncertain\_candidate}: a plausible initiative without enough identity evidence (no name, location, counterparty or amount) to be counted. | ||
| 18 | \end{itemize} | 18 | \end{itemize} | ||
| 19 | 19 | ||||
| 20 | −The current build implements the first two tiers directly. The third is enforced only through the prompt and the confidence threshold; the fourth, fifth and sixth exist as instructions and as a confidence score, not as stored labels. The financing class was added in this revision after a regular-expression pass over the stored mentions (\texttt{repurchase|buy-?back|dividend|credit facility|revolving|term loan|notes? (due|offering)|at-the-market|senior notes|commercial paper} on the project name and description) matched \SPIDfinMentions\ mentions, \SPIDfinMentionsPct\% of the total, carrying \SPIDfinAmountB~B of the \SPIDfinAmountDenomB~B USD of mention amounts; the match concentrates in Part~II Item~2 of the 10-Q (\SPIDfinUnregPct\% of its mentions, where issuer repurchases are tabled) and in 8-K bodies (\SPIDfinEightKPct\%), and by category in \texttt{capex\_program} (\SPIDfinCapexPct\% of its mentions), \texttt{cost\_reduction} (\SPIDfinCostPct\%) and \texttt{ma\_integration} (\SPIDfinMaPct\%). \Cref{sec:results} reports the category counts with and without this class, and \Cref{tab:fp_classes} gives the match by item, category and term. Promoting the labels to explicit columns of the mention table, filled by annotators and eventually by a classifier, is the first schema change proposed in \texttt{docs/project\_ontology.md}. | 20 | +The current build implements the first two tiers directly. The third is enforced only through the prompt and the confidence threshold; the fourth, fifth and sixth exist as instructions and as a confidence score, not as stored labels. The financing class was added in this revision after a regular-expression pass over the stored mentions (\texttt{repurchase|buy-?back|dividend|credit facility|revolving|term loan|notes? (due|offering)|at-the-market|senior notes|commercial paper} on the project name and description) matched \SPIDfinMentions\ mentions, \SPIDfinMentionsPct\% of the total, carrying \SPIDfinAmountT~T of the \SPIDfinAmountDenomT~T USD of mention amounts; the match concentrates in Part~II Item~2 of the 10-Q (\SPIDfinUnregPct\% of its mentions, where issuer repurchases are tabled) and in 8-K bodies (\SPIDfinEightKPct\%), and by category in \texttt{capex\_program} (\SPIDfinCapexPct\% of its mentions), \texttt{cost\_reduction} (\SPIDfinCostPct\%) and \texttt{ma\_integration} (\SPIDfinMaPct\%). \Cref{sec:results} reports the category counts with and without this class, and \Cref{tab:fp_classes} gives the match by item, category and term. Promoting the labels to explicit columns of the mention table, filled by annotators and eventually by a classifier, is the first schema change proposed in \texttt{docs/project\_ontology.md}. | ||
| 21 | 21 | ||||
| 22 | \input{tables/tab_fp_classes} | 22 | \input{tables/tab_fp_classes} | ||
| 23 | 23 | ||||
@@ -57,10 +57,10 @@ The instrument is re-runnable and ledgered, not deterministic. The calls used te | |||||
| 57 | 57 | ||||
| 58 | \subsection{Cost, throughput and the extraction ledger}\label{sec:ledger} | 58 | \subsection{Cost, throughput and the extraction ledger}\label{sec:ledger} | ||
| 59 | 59 | ||||
| 60 | −Every ledgered section is written to the \texttt{extracted\_sections} table (section, accession, backend, number of mentions, timestamp), which makes the step resumable and prevents double billing. Restated on calls rather than rows, the October 2026 run wrote \SPIDoctoberRows\ rows of which \SPIDoctoberCalls\ were calls, and the June block \SPIDbackfilledRows\ rows of which \SPIDjuneCalls\ were calls; the model returned at least one project for \SPIDproductiveShareCalls\% of calls (\Cref{tab:ledger_calls}) (\SPIDproductiveShareLedger\% of rows; \SPIDjuneProductiveShareCalls\% and \SPIDoctoberProductiveShareCalls\% by run), \SPIDmentionsPerCall\ mentions per call (\SPIDmentionsPerLedgerSection\ per row). At the authors' recorded total of about \SPIDtotalCostUsd~USD for the October run at the then-current \texttt{gpt-4o-mini} prices, the cost is about \SPIDusdPerCall~USD per call (\SPIDusdPerOctoberRow~USD per ledger row, the figure the earlier version reported ``per section''). The run was executed with \SPIDworkers\ concurrent workers; the ledger timestamps span \SPIDoctoberWallClockHours\ hours of wall-clock time (\SPIDoctoberFirstFlush\ to \SPIDoctoberLastFlush), of which \SPIDoutageHours\ hours are the largest gap, so the effective throughput over \SPIDoctoberEffectiveHours\ hours was \SPIDoctoberRowsPerSecondEffective\ rows or \SPIDoctoberCallsPerSecondEffective\ calls per second; the ``twelve hours'' and ``27--36 sections per second'' of the earlier version were mutually inconsistent and are withdrawn. | 60 | +Every ledgered section is written to the \texttt{extracted\_sections} table (section, accession, backend, number of mentions, timestamp), which makes the step resumable and prevents double billing. Restated on calls rather than rows, the October 2026 run wrote \SPIDoctoberRows\ rows of which \SPIDoctoberCalls\ were calls, and the June block \SPIDbackfilledRows\ rows of which \SPIDjuneCalls\ were calls; the model returned at least one project for \SPIDproductiveShareCalls\% of calls (\Cref{tab:ledger_calls}) (\SPIDproductiveShareLedger\% of rows; \SPIDjuneProductiveShareCalls\% and \SPIDoctoberProductiveShareCalls\% by run), \SPIDmentionsPerCall\ mentions per call (\SPIDmentionsPerLedgerSection\ per row). At the authors' recorded total of about \SPIDtotalCostUsd~USD at the then-current \texttt{gpt-4o-mini} prices, entered in the research log against the October run, the cost is about \SPIDusdPerCall~USD per call (\SPIDusdPerOctoberRow~USD per ledger row, the figure the earlier version reported ``per section''); the invoice is not archived, and if the recorded total covered the June block as well, as the review's constraints note states, the figure would be \SPIDusdPerCallBothRuns~USD per call over the \SPIDcallsBothRuns\ calls of both runs---the basis is to be confirmed by the authors. The run was executed with \SPIDworkers\ concurrent workers; the ledger timestamps span \SPIDoctoberWallClockHours\ hours of wall-clock time (\SPIDoctoberFirstFlush\ to \SPIDoctoberLastFlush), of which \SPIDoutageHours\ hours are the largest gap, so the effective throughput over \SPIDoctoberEffectiveHours\ hours was \SPIDoctoberRowsPerSecondEffective\ rows or \SPIDoctoberCallsPerSecondEffective\ calls per second; the ``twelve hours'' and ``27--36 sections per second'' of the earlier version were mutually inconsistent and are withdrawn. | ||
| 61 | 61 | ||||
| 62 | \input{tables/tab_ledger_calls} | 62 | \input{tables/tab_ledger_calls} | ||
| 63 | 63 | ||||
| 64 | −The ledger also records a failure mode encountered during the run, which this revision has reconciled against the run log rather than reported from memory (\Cref{tab:outage}). When the API key's credit balance was exhausted (log window \SPIDoutageWindowStart\ to \SPIDoutageWindowEnd), the extractor's retry logic returned empty results that were recorded as ``extracted with zero mentions'': the log carries \SPIDllmFailedRetryableHttpFourTwoNineFivexx\ retryable HTTP errors and \SPIDllmFailedConnectionError\ connection resets in that window, batches \SPIDoutageZeroBatchRange\ (\SPIDoutageZeroBatchWindow) returned no mention at all, and \SPIDoutageRowsRemoved\ rows were removed from the ledger---a number that reconciles exactly with the log as the sum of sections reported complete (\SPIDlogExtractionCompleteSum) minus the sections pending when the run was restarted (\SPIDlogPendingSecondStart). The final ledger has no row inside the largest gap (\SPIDoutageLargestGapStart\ to \SPIDoutageLargestGapEnd); \SPIDoutageSurvivingRows\ productive rows (\SPIDoutageSurvivingMentions\ mentions) stamped in the first quarter of an hour of the window survived because their calls had succeeded. Two residues remain. First, the extractor also returns an empty list after \SPIDretries\ failed attempts on any other error, and the log shows \SPIDllmFailedJsonDecodeError\ JSON-decode failures outside the outage (\SPIDjsonFailBeforeOutage\ before, \SPIDjsonFailAfterOutage\ after; \SPIDjuneLlmCallFailed\ in the June run), each ledgered as ``no project''; the earlier version reported one. Second, the first batch after the restart (\SPIDresumeBatchWindow; \SPIDresumeBatchSections\ sections of \SPIDresumeBatchCiks\ issuers, \SPIDresumeBatchCalls\ calls) returned \SPIDresumeBatchMentions\ mentions with no logged failure: \SPIDresumeBatchProductive\ productive sections against about \SPIDresumeBatchExpectedProductive\ expected at the form-by-item rates of the rest of the run, with \SPIDresumeBatchZeroSharePct\% zero-mention calls in its last minute. The most likely reading is a block of empty answers returned with a success status while the quota was being restored; those sections are ledgered as ``no project'' and were not re-run. A re-submission of the \SPIDresumeBatchCalls\ gate-passing sections of that batch (about \SPIDresumeBatchCalls\ calls) was requested during the review and is not granted in this revision, so the batch stays flagged as suspect in \Cref{tab:evidence} and the issuer list is published with the audit outputs. The general point stands: a ledger that cannot distinguish ``no project'' from ``call failed'' or ``empty answer'' is not sufficient, and an explicit \texttt{call\_status} column, together with the raw response, is part of the proposed schema. | 64 | +The ledger also records a failure mode encountered during the run, which this revision has reconciled against the run log rather than reported from memory (\Cref{tab:outage}). When the API key's credit balance was exhausted (log window \SPIDoutageWindowStart\ to \SPIDoutageWindowEnd), the extractor's retry logic returned empty results that were recorded as ``extracted with zero mentions'': the log carries \SPIDllmRetryableInsideWindow\ retryable HTTP errors and \SPIDllmFailedConnectionError\ connection resets in that window (\SPIDllmFailedRetryableHttpFourTwoNineFivexx\ retryable lines in all, the remaining \SPIDllmRetryableTailLines\ in the twenty seconds after it), batches \SPIDoutageZeroBatchRange\ (\SPIDoutageZeroBatchWindow) returned no mention at all, and \SPIDoutageRowsRemoved\ rows were removed from the ledger---a number that reconciles exactly with the log as the sum of sections reported complete (\SPIDlogExtractionCompleteSum) minus the sections pending when the run was restarted (\SPIDlogPendingSecondStart). The final ledger has no row inside the largest gap (\SPIDoutageLargestGapStart\ to \SPIDoutageLargestGapEnd); the \SPIDoutageSurvivingRows\ rows stamped in the first quarter of an hour of the window survived because their calls had succeeded, and all of them are productive (\SPIDoutageSurvivingMentions\ mentions, \SPIDoutageSurvivingZeroRows\ rows with zero mentions). Two residues remain. First, the extractor also returns an empty list after \SPIDretries\ failed attempts on any other error, and the log shows \SPIDllmFailedJsonDecodeError\ JSON-decode failures outside the outage (\SPIDjsonFailBeforeOutage\ before, \SPIDjsonFailAfterOutage\ after; \SPIDjuneLlmCallFailed\ in the June run), each ledgered as ``no project''; the earlier version reported one. Second, the first batch after the restart (\SPIDresumeBatchWindow; \SPIDresumeBatchSections\ sections in the log, \SPIDresumeBatchLedgerRows\ ledgered rows of \SPIDresumeBatchCiks\ issuers, \SPIDresumeBatchCalls\ calls) returned \SPIDresumeBatchMentions\ mentions in the log (\SPIDresumeBatchLedgerMentions\ ledgered) with no logged failure: \SPIDresumeBatchProductive\ productive sections against about \SPIDresumeBatchExpectedProductive\ expected at the form-by-item rates of the rest of the run, with \SPIDresumeBatchZeroSharePct\% zero-mention calls in its last minute. The most likely reading is a block of empty answers returned with a success status while the quota was being restored; those sections are ledgered as ``no project'' and were not re-run. A re-submission of the \SPIDresumeBatchCalls\ gate-passing sections of that batch (about \SPIDresumeBatchCalls\ calls) was requested during the review and is not granted in this revision, so the batch stays flagged as suspect in \Cref{tab:evidence} and the issuer list is published with the audit outputs. The general point stands: a ledger that cannot distinguish ``no project'' from ``call failed'' or ``empty answer'' is not sufficient, and an explicit \texttt{call\_status} column, together with the raw response, is part of the proposed schema. | ||
| 65 | 65 | ||||
| 66 | \input{tables/tab_outage} | 66 | \input{tables/tab_outage} | ||
modified
paper/sections/validation.tex
+1 −1
@@ -20,7 +20,7 @@ The aim of identifying ``all projects'' of an issuer is a \emph{coverage target} | |||||
| 20 | 20 | ||||
| 21 | \subsection{Mechanical accuracy audits (implemented in this revision)}\label{sec:val_mechanical} | 21 | \subsection{Mechanical accuracy audits (implemented in this revision)}\label{sec:val_mechanical} | ||
| 22 | 22 | ||||
| 23 | −\paragraph{Field-level grounding.} The prompt requires that numbers and locations appear in the text, and the structured fields are literal tokens, so their containment in the stored section text can be tested for every mention. For all \SPIDmentions\ mentions joined to their section text (\texttt{results/grounding/}; \SPIDgroundBootstrapReps\ bootstrap replicates clustered by section), every listed year is found in \SPIDgroundYearsPct\% of mentions with years (\SPIDgroundYearsCiLow--\SPIDgroundYearsCiHigh), every listed partner in \SPIDgroundPartnersPct\%, the amount as a number (in any of the usual renderings) in \SPIDgroundAmountNumberPct\% and with a currency token within forty characters in \SPIDgroundAmountCurrencyPct\%, every listed location in \SPIDgroundLocationsPct\% (10-K \SPIDgroundLocationsTenK\%, 10-Q \SPIDgroundLocationsTenQ\%, 8-K \SPIDgroundLocationsEightK\%), and every technology tag in \SPIDgroundTechPct\%; at the level of individual elements the rates are \SPIDgroundElemYears\% for years, \SPIDgroundElemPartners\% for partners, \SPIDgroundElemLocations\% for locations and \SPIDgroundElemTech\% for technologies. All structured fields of a mention are found in \SPIDgroundAllPct\% of the \SPIDgroundAllN\ mentions that carry at least one (10-K \SPIDgroundAllTenK\%, 10-Q \SPIDgroundAllTenQ\%, 8-K \SPIDgroundAllEightK\%; June block \SPIDgroundAllJune\%, October \SPIDgroundAllOctober\%), with no difference between filings before and after the model's training cut-off beyond one percentage point. Three readings of these rates are needed. First, they are rates of \emph{transcription}, not of correctness: a year, partner or amount can be copied faithfully from a passage that describes a counterparty's initiative, a hypothetical, or a financing instrument, so grounding is necessary for a correct mention and not sufficient. Second, the lower rates for locations and technologies reflect normalization by the model (``Texas'' for ``Austin, TX''; ``cloud computing'' for ``AWS'') as much as invention; the element-level rate for locations is \SPIDgroundElemLocations\%. Third, \SPIDyearsAbsentFromText\ mentions carry a year absent from the section, and the technology and year lists equal the regex output on the model's own description for up to \SPIDtechRegexEqualPct\% and \SPIDyearsRegexEqualPct\% of mentions, so for those fields ``grounded'' may mean ``copied from the paraphrase'' (\Cref{sec:contract}). The breakdown by field and form is in \Cref{tab:grounding} and \Cref{fig:grounding}; the full breakdown by item is in \texttt{results/grounding/grounding\_by\_field\_form\_item.csv}. | 23 | +\paragraph{Field-level grounding.} The prompt requires that numbers and locations appear in the text, and the structured fields are literal tokens, so their containment in the stored section text can be tested for every mention (after case-folding, replacement of punctuation by spaces and a short alias list for country names, as stated in the note of \Cref{tab:grounding}; partners are the field most sensitive to this normalisation). For all \SPIDmentions\ mentions joined to their section text (\texttt{results/grounding/}; \SPIDgroundBootstrapReps\ bootstrap replicates clustered by section), every listed year is found in \SPIDgroundYearsPct\% of mentions with years (\SPIDgroundYearsCiLow--\SPIDgroundYearsCiHigh), every listed partner in \SPIDgroundPartnersPct\%, the amount as a number (in any of the usual renderings) in \SPIDgroundAmountNumberPct\% and with a currency token within forty characters in \SPIDgroundAmountCurrencyPct\%, every listed location in \SPIDgroundLocationsPct\% (10-K \SPIDgroundLocationsTenK\%, 10-Q \SPIDgroundLocationsTenQ\%, 8-K \SPIDgroundLocationsEightK\%), and every technology tag in \SPIDgroundTechPct\%; at the level of individual elements the rates are \SPIDgroundElemYears\% for years, \SPIDgroundElemPartners\% for partners, \SPIDgroundElemLocations\% for locations and \SPIDgroundElemTech\% for technologies. All structured fields of a mention are found in \SPIDgroundAllPct\% of the \SPIDgroundAllN\ mentions that carry at least one (10-K \SPIDgroundAllTenK\%, 10-Q \SPIDgroundAllTenQ\%, 8-K \SPIDgroundAllEightK\%; June block \SPIDgroundAllJune\%, October \SPIDgroundAllOctober\%), with no difference between filings before and after the model's training cut-off beyond one percentage point. Three readings of these rates are needed. First, they are rates of \emph{transcription}, not of correctness: a year, partner or amount can be copied faithfully from a passage that describes a counterparty's initiative, a hypothetical, or a financing instrument, so grounding is necessary for a correct mention and not sufficient. Second, the lower rates for locations and technologies reflect normalization by the model (``Texas'' for ``Austin, TX''; ``cloud computing'' for ``AWS'') as much as invention; the element-level rate for locations is \SPIDgroundElemLocations\%. Third, \SPIDyearsAbsentFromText\ mentions carry a year absent from the section, and the technology and year lists equal the regex output on the model's own description for up to \SPIDtechRegexEqualPct\% and \SPIDyearsRegexEqualPct\% of mentions, so for those fields ``grounded'' may mean ``copied from the paraphrase'' (\Cref{sec:contract}). The breakdown by field and form is in \Cref{tab:grounding} and \Cref{fig:grounding}; the full breakdown by item is in \texttt{results/grounding/grounding\_by\_field\_form\_item.csv}. | ||
| 24 | 24 | ||||
| 25 | \input{tables/tab_grounding} | 25 | \input{tables/tab_grounding} | ||
| 26 | 26 | ||||
modified
paper/tables/numbers.tex
+16 −2
@@ -196,6 +196,7 @@ | |||||
| 196 | \newcommand{\SPIDprovenanceMaxAmountBusd}{11.5} | 196 | \newcommand{\SPIDprovenanceMaxAmountBusd}{11.5} | ||
| 197 | \newcommand{\SPIDprovenanceNameYear}{2022} | 197 | \newcommand{\SPIDprovenanceNameYear}{2022} | ||
| 198 | \newcommand{\SPIDprovenanceProjectId}{3e2705ba} | 198 | \newcommand{\SPIDprovenanceProjectId}{3e2705ba} | ||
| 199 | +\newcommand{\SPIDprovenanceKeyToken}{san patricio county, texas} | ||||
| 199 | \newcommand{\SPIDgateRegexFlags}{re.IGNORECASE} | 200 | \newcommand{\SPIDgateRegexFlags}{re.IGNORECASE} | ||
| 200 | \newcommand{\SPIDgateRegexLength}{215} | 201 | \newcommand{\SPIDgateRegexLength}{215} | ||
| 201 | \newcommand{\SPIDgateRegexAlternatives}{25} | 202 | \newcommand{\SPIDgateRegexAlternatives}{25} | ||
@@ -205,6 +206,10 @@ | |||||
| 205 | \newcommand{\SPIDyearsNotInText}{604} | 206 | \newcommand{\SPIDyearsNotInText}{604} | ||
| 206 | \newcommand{\SPIDtechDeployMentionedNoSignal}{90} | 207 | \newcommand{\SPIDtechDeployMentionedNoSignal}{90} | ||
| 207 | \newcommand{\SPIDtechDeployMentionedTotal}{124} | 208 | \newcommand{\SPIDtechDeployMentionedTotal}{124} | ||
| 209 | +\newcommand{\SPIDgroundPartnersCheckN}{5{,}287} | ||||
| 210 | +\newcommand{\SPIDgroundPartnersCheckAuditPct}{99.3} | ||||
| 211 | +\newcommand{\SPIDgroundPartnersPlainCasefoldPct}{96.1} | ||||
| 212 | +\newcommand{\SPIDgroundPartnersCheckSample}{20{,}000} | ||||
| 208 | \newcommand{\SPIDgroundedYearsPct}{99.8} | 213 | \newcommand{\SPIDgroundedYearsPct}{99.8} | ||
| 209 | \newcommand{\SPIDgroundedYearsCI}{[99.7, 99.8]} | 214 | \newcommand{\SPIDgroundedYearsCI}{[99.7, 99.8]} | ||
| 210 | \newcommand{\SPIDgroundedYearsOneZeroKPct}{99.6} | 215 | \newcommand{\SPIDgroundedYearsOneZeroKPct}{99.6} | ||
@@ -376,6 +381,7 @@ | |||||
| 376 | \newcommand{\SPIDllmCallFailedInsideOutage}{94{,}467} | 381 | \newcommand{\SPIDllmCallFailedInsideOutage}{94{,}467} | ||
| 377 | \newcommand{\SPIDllmCallFailedOutsideOutage}{14} | 382 | \newcommand{\SPIDllmCallFailedOutsideOutage}{14} | ||
| 378 | \newcommand{\SPIDllmRetryableTailLines}{128} | 383 | \newcommand{\SPIDllmRetryableTailLines}{128} | ||
| 384 | +\newcommand{\SPIDllmRetryableInsideWindow}{94{,}465} | ||||
| 379 | \newcommand{\SPIDllmFailedJsonDecodeError}{14} | 385 | \newcommand{\SPIDllmFailedJsonDecodeError}{14} | ||
| 380 | \newcommand{\SPIDllmFailedRetryableHttpFourTwoNineFivexx}{94{,}593} | 386 | \newcommand{\SPIDllmFailedRetryableHttpFourTwoNineFivexx}{94{,}593} | ||
| 381 | \newcommand{\SPIDllmFailedConnectionError}{2} | 387 | \newcommand{\SPIDllmFailedConnectionError}{2} | ||
@@ -415,6 +421,8 @@ | |||||
| 415 | \newcommand{\SPIDoutageZeroBatchRange}{47--60} | 421 | \newcommand{\SPIDoutageZeroBatchRange}{47--60} | ||
| 416 | \newcommand{\SPIDoutageZeroBatchWindow}{01:44--06:05} | 422 | \newcommand{\SPIDoutageZeroBatchWindow}{01:44--06:05} | ||
| 417 | \newcommand{\SPIDresumeBatchMentions}{34} | 423 | \newcommand{\SPIDresumeBatchMentions}{34} | ||
| 424 | +\newcommand{\SPIDresumeBatchLedgerMentions}{30} | ||||
| 425 | +\newcommand{\SPIDresumeBatchLedgerRows}{4{,}769} | ||||
| 418 | \newcommand{\SPIDresumeBatchWindow}{06:38--06:40} | 426 | \newcommand{\SPIDresumeBatchWindow}{06:38--06:40} | ||
| 419 | \newcommand{\SPIDlogExtractionCompleteSum}{1{,}117{,}479} | 427 | \newcommand{\SPIDlogExtractionCompleteSum}{1{,}117{,}479} | ||
| 420 | \newcommand{\SPIDlogPendingSecondStart}{968{,}127} | 428 | \newcommand{\SPIDlogPendingSecondStart}{968{,}127} | ||
@@ -423,6 +431,8 @@ | |||||
| 423 | \newcommand{\SPIDworkers}{48} | 431 | \newcommand{\SPIDworkers}{48} | ||
| 424 | \newcommand{\SPIDoutageSurvivingRows}{504} | 432 | \newcommand{\SPIDoutageSurvivingRows}{504} | ||
| 425 | \newcommand{\SPIDoutageSurvivingMentions}{1{,}011} | 433 | \newcommand{\SPIDoutageSurvivingMentions}{1{,}011} | ||
| 434 | +\newcommand{\SPIDoutageSurvivingProductive}{504} | ||||
| 435 | +\newcommand{\SPIDoutageSurvivingZeroRows}{0} | ||||
| 426 | \newcommand{\SPIDbackfillTimestamp}{October~27, 2026, 13:12} | 436 | \newcommand{\SPIDbackfillTimestamp}{October~27, 2026, 13:12} | ||
| 427 | \newcommand{\SPIDresumeBatchExpectedProductive}{536} | 437 | \newcommand{\SPIDresumeBatchExpectedProductive}{536} | ||
| 428 | \newcommand{\SPIDresumeBatchZeroSharePct}{99.2} | 438 | \newcommand{\SPIDresumeBatchZeroSharePct}{99.2} | ||
@@ -431,6 +441,8 @@ | |||||
| 431 | \newcommand{\SPIDoctoberProductiveShareCalls}{23.6} | 441 | \newcommand{\SPIDoctoberProductiveShareCalls}{23.6} | ||
| 432 | \newcommand{\SPIDgateRejectedPct}{37.3} | 442 | \newcommand{\SPIDgateRejectedPct}{37.3} | ||
| 433 | \newcommand{\SPIDusdPerOctoberRow}{0.0008} | 443 | \newcommand{\SPIDusdPerOctoberRow}{0.0008} | ||
| 444 | +\newcommand{\SPIDcallsBothRuns}{703{,}725} | ||||
| 445 | +\newcommand{\SPIDusdPerCallBothRuns}{0.0011} | ||||
| 434 | \newcommand{\SPIDconfExactDefault}{3{,}278} | 446 | \newcommand{\SPIDconfExactDefault}{3{,}278} | ||
| 435 | \newcommand{\SPIDconfExactDefaultOctober}{2{,}974} | 447 | \newcommand{\SPIDconfExactDefaultOctober}{2{,}974} | ||
| 436 | \newcommand{\SPIDconfExactDefaultJune}{304} | 448 | \newcommand{\SPIDconfExactDefaultJune}{304} | ||
@@ -570,6 +582,8 @@ | |||||
| 570 | \newcommand{\SPIDpairedIssuers}{81} | 582 | \newcommand{\SPIDpairedIssuers}{81} | ||
| 571 | \newcommand{\SPIDpairedDeltaPp}{0.6} | 583 | \newcommand{\SPIDpairedDeltaPp}{0.6} | ||
| 572 | \newcommand{\SPIDpairedSePp}{1.3} | 584 | \newcommand{\SPIDpairedSePp}{1.3} | ||
| 585 | +\newcommand{\SPIDpairedProductiveJune}{11.9} | ||||
| 586 | +\newcommand{\SPIDpairedProductiveOctober}{12.5} | ||||
| 573 | \newcommand{\SPIDcohortBothPerIssuerAll}{55.8} | 587 | \newcommand{\SPIDcohortBothPerIssuerAll}{55.8} | ||
| 574 | \newcommand{\SPIDcohortRussellOnlyPerIssuer}{46.9} | 588 | \newcommand{\SPIDcohortRussellOnlyPerIssuer}{46.9} | ||
| 575 | \newcommand{\SPIDcohortBothPerIssuerOctober}{25.1} | 589 | \newcommand{\SPIDcohortBothPerIssuerOctober}{25.1} | ||
@@ -648,8 +662,8 @@ | |||||
| 648 | \newcommand{\SPIDrulesTypeCost}{76.9} | 662 | \newcommand{\SPIDrulesTypeCost}{76.9} | ||
| 649 | \newcommand{\SPIDfinMentions}{22{,}412} | 663 | \newcommand{\SPIDfinMentions}{22{,}412} | ||
| 650 | \newcommand{\SPIDfinMentionsPct}{7.3} | 664 | \newcommand{\SPIDfinMentionsPct}{7.3} | ||
| 651 | −\newcommand{\SPIDfinAmountB}{17.8} | 665 | +\newcommand{\SPIDfinAmountT}{17.8} | ||
| 652 | −\newcommand{\SPIDfinAmountDenomB}{104.2} | 666 | +\newcommand{\SPIDfinAmountDenomT}{104.2} | ||
| 653 | \newcommand{\SPIDfinAmountSharePct}{17.1} | 667 | \newcommand{\SPIDfinAmountSharePct}{17.1} | ||
| 654 | \newcommand{\SPIDfinUnregPct}{54.5} | 668 | \newcommand{\SPIDfinUnregPct}{54.5} | ||
| 655 | \newcommand{\SPIDfinUnregN}{5{,}179} | 669 | \newcommand{\SPIDfinUnregN}{5{,}179} | ||
modified
paper/tables/tab_focus.tex
+1 −1
@@ -7,7 +7,7 @@ | |||||
| 7 | \begin{adjustbox}{max width=\textwidth} | 7 | \begin{adjustbox}{max width=\textwidth} | ||
| 8 | \begin{tabular}{llrrrrrrrr} | 8 | \begin{tabular}{llrrrrrrrr} | ||
| 9 | \toprule | 9 | \toprule | ||
| 10 | −Form & Item & \multicolumn{1}{c}{\makecell{Ledgered\\rows}} & \multicolumn{1}{c}{\makecell{Over 9{,}000\\chars (\%)}} & \multicolumn{1}{c}{\makecell{Median\\chars}} & \multicolumn{1}{c}{\makecell{Share shown\\P10}} & \multicolumn{1}{c}{\makecell{Share shown\\median}} & \multicolumn{1}{c}{\makecell{Share shown\\P90}} & \multicolumn{1}{c}{\makecell{Rows shown\\$< 25$\% (\%)}} & \multicolumn{1}{c}{\makecell{Mentions from rows\\shown $< 25$\% (\%)}} \\ | 10 | +Form & Item & \multicolumn{1}{c}{\makecell{Ledgered\\rows}} & \multicolumn{1}{c}{\makecell{Over 9{,}000\\chars (\%)}} & \multicolumn{1}{c}{\makecell{Median\\chars}} & \multicolumn{1}{c}{\makecell{Share shown\\P10}} & \multicolumn{1}{c}{\makecell{Share shown\\median}} & \multicolumn{1}{c}{\makecell{Share shown\\P90}} & \multicolumn{1}{c}{\makecell{Rows shown\\$<25$\% (\%)}} & \multicolumn{1}{c}{\makecell{Mentions from\\such rows (\%)}} \\ | ||
| 11 | \midrule | 11 | \midrule | ||
| 12 | 10-K & \texttt{business} & 29{,}285 & 81.1 & 37{,}168 & 5 & 16 & 100 & 66.6 & 86.1 \\ | 12 | 10-K & \texttt{business} & 29{,}285 & 81.1 & 37{,}168 & 5 & 16 & 100 & 66.6 & 86.1 \\ | ||
| 13 | 10-K & \texttt{full\_body} & 1{,}230 & 99.9 & 304{,}265 & 1 & 2 & 4 & 99.8 & 100.0 \\ | 13 | 10-K & \texttt{full\_body} & 1{,}230 & 99.9 & 304{,}265 & 1 & 2 & 4 & 99.8 & 100.0 \\ | ||
modified
paper/tables/tab_grounding.tex
+1 −1
@@ -38,7 +38,7 @@ By run: June (backfilled) / October & 36{,}806 / 239{,}362 & 84.5 / 88.7 & & & \ | |||||
| 38 | \end{tabular} | 38 | \end{tabular} | ||
| 39 | \end{adjustbox} | 39 | \end{adjustbox} | ||
| 40 | \begin{tablenotes}\footnotesize | 40 | \begin{tablenotes}\footnotesize | ||
| 41 | −\item Population: all 308{,}065 stored mentions, joined to the text of their section as stored (legacy June text for the S\&P~500 10-Ks). A field is ``grounded'' when every element of the list (years, partners, locations, technologies, case-folded) or the amount (tested in units, thousands, millions and billions with one to three significant digits) occurs literally in that text; a currency token is \$, USD, dollar, million or billion within 40 characters of the number. Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field. Upper bounds on the regex back-fill from the model's own paraphrase (\texttt{llm.py}): the stored years equal the regex output on the description for 98{,}438 of 252{,}469 mentions with years (39\%) and the technology list for 16{,}949 of 36{,}740 (46\%); 604 mentions cite a year absent from the section text. | 41 | +\item Population: all 308{,}065 stored mentions, joined to the text of their section as stored (legacy June text for the S\&P~500 10-Ks). A field is ``grounded'' when every element of the list (years, partners, locations, technologies) or the amount (tested in units, thousands, millions and billions with one to three significant digits) occurs literally in that text; a currency token is \$, USD, dollar, million or billion within 40 characters of the number. Before the search, element and text are normalised identically: case-folded, punctuation replaced by spaces, whitespace collapsed, and a short alias list applied to the element (U.S., USA $\to$ united states; U.K. $\to$ united kingdom; UAE; EU); the element must then occur as whole words. The normalisation matters for partners: on a reservoir sample of 20{,}000 mentions (5{,}287 with a partner), the rate is 99.3\% under this rule and 96.1\% under a plain case-folded substring search (\texttt{results/grounding/partners\_normalisation\_check.json}). Confidence intervals are percentile bootstraps with 1{,}000 replicates clustered by section. Grounding is a necessary condition for correctness, not a measure of it: a grounded year can belong to another initiative, and an ungrounded location can be a correct normalisation (``Texas'' for ``Austin''); precision and recall require the human labels of gate (d). Panel C counts mentions with at least one structured field. Upper bounds on the regex back-fill from the model's own paraphrase (\texttt{llm.py}): the stored years equal the regex output on the description for 98{,}438 of 252{,}469 mentions with years (39\%) and the technology list for 16{,}949 of 36{,}740 (46\%); 604 mentions cite a year absent from the section text. | ||
| 42 | \end{tablenotes} | 42 | \end{tablenotes} | ||
| 43 | \end{threeparttable} | 43 | \end{threeparttable} | ||
| 44 | \end{table} | 44 | \end{table} | ||
modified
paper/tables/tab_outage.tex
+4 −2
@@ -15,8 +15,9 @@ Ledger rows with a timestamp inside the largest gap (01:25--06:40, 5.25~h) & 0 \ | |||||
| 15 | Log batches returning zero mentions inside the outage window (batch ids 47--60) & 14 \\ | 15 | Log batches returning zero mentions inside the outage window (batch ids 47--60) & 14 \\ | ||
| 16 | \midrule | 16 | \midrule | ||
| 17 | \multicolumn{2}{l}{\itshape Panel B: failed-call lines in the October log (94{,}609 lines; the ledger records none of them)} \\ | 17 | \multicolumn{2}{l}{\itshape Panel B: failed-call lines in the October log (94{,}609 lines; the ledger records none of them)} \\ | ||
| 18 | −Retryable HTTP 429/5xx inside the outage window 01:10--06:22 & 94{,}593 \\ | 18 | +Retryable HTTP 429/5xx, all lines & 94{,}593 \\ | ||
| 19 | −Retryable HTTP 429/5xx in the outage tail (first ledger window definition, 20 s after the log window) & 128 \\ | 19 | +\quad of which inside the outage window 01:10--06:22 & 94{,}465 \\ | ||
| 20 | +\quad of which in the twenty seconds after it (to 06:22:21, the end of the log window) & 128 \\ | ||||
| 20 | Connection resets & 2 \\ | 21 | Connection resets & 2 \\ | ||
| 21 | JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & 14 \\ | 22 | JSON-decode failures outside the outage (ledgered as ``no project'': silent empty results) & 14 \\ | ||
| 22 | ``Section failed'' lines & 0 \\ | 23 | ``Section failed'' lines & 0 \\ | ||
@@ -24,6 +25,7 @@ JSON-decode failures in the June log (of 153{,}894 sections) & 3 \\ | |||||
| 24 | \midrule | 25 | \midrule | ||
| 25 | \multicolumn{2}{l}{\itshape Panel C: first batch after the resume (06:38--06:40, 19 issuers)} \\ | 26 | \multicolumn{2}{l}{\itshape Panel C: first batch after the resume (06:38--06:40, 19 issuers)} \\ | ||
| 26 | Sections in the batch (log) / ledgered rows & 5{,}198 / 4{,}769 \\ | 27 | Sections in the batch (log) / ledgered rows & 5{,}198 / 4{,}769 \\ | ||
| 28 | +Mentions returned (log) / ledgered & 34 / 30 \\ | ||||
| 27 | Gate-passing calls & 2{,}686 \\ | 29 | Gate-passing calls & 2{,}686 \\ | ||
| 28 | Productive sections observed & 23 \\ | 30 | Productive sections observed & 23 \\ | ||
| 29 | Productive sections expected at the form $\times$ item rates of the rest of the run & 536.2 \\ | 31 | Productive sections expected at the form $\times$ item rates of the rest of the run & 536.2 \\ | ||
modified
paper/tables/tab_provenance.tex
+2 −2
@@ -1,4 +1,4 @@ | |||||
| 1 | −% Generated by code/make_tables.py from data/example\_provenance.csv, documented\_inputs.json:snippet\_chars. Do not edit by hand. | 1 | +% Generated by code/make_tables.py from data/example\_provenance.csv, documented\_inputs.json:snippet\_chars, results/resolver/example\_provenance\_keys.csv, spec\_curve.csv. Do not edit by hand. | ||
| 2 | \begin{table}[H] | 2 | \begin{table}[H] | ||
| 3 | \centering\scriptsize | 3 | \centering\scriptsize | ||
| 4 | \setlength{\tabcolsep}{3pt} | 4 | \setlength{\tabcolsep}{3pt} | ||
@@ -21,7 +21,7 @@ Accession & Filed & Form & Status / amount & Evidence snippet (model-generated d | |||||
| 21 | \end{tabular} | 21 | \end{tabular} | ||
| 22 | \end{adjustbox} | 22 | \end{adjustbox} | ||
| 23 | \begin{tablenotes}\footnotesize | 23 | \begin{tablenotes}\footnotesize | ||
| 24 | −\item Snippets are the first 170 characters of the model-generated description stored in \texttt{project\_timeline.snippet}; the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable through \texttt{project\_mentions.section\_id}. All 8 printed mentions are 8-K filings. The representative name is that of the highest-confidence mention (filed 2022) applied retroactively to the whole timeline, and is one admissible choice among mentions tied at the maximum confidence. The key for this issuer and type is the location token ``corpus christi'', so distinct phases share one identifier: the same mentions form 198488 projects under the strict key and 21027 under the issuer $\times$ type key (cluster sizes in Table~\ref{tab:spec_curve}). | 24 | +\item Snippets are the first 170 characters of the model-generated description stored in \texttt{project\_timeline.snippet}; the description is a paraphrase, not a quotation, and the character span of the supporting passage is not stored; the full source section is retrievable through \texttt{project\_mentions.section\_id}. All 8 printed mentions are 8-K filings. The representative name is that of the highest-confidence mention (filed 2022) applied retroactively to the whole timeline, and is one admissible choice among mentions tied at the maximum confidence. The key for this issuer and type is the location ``san patricio county, texas'' (\texttt{results/resolver/example\_provenance\_keys.csv}), so distinct phases at that site share one identifier; the cluster that contains the example's highest-confidence mention holds 8 mentions under this key, 1 under the strict name $\times$ location key and 176 under the issuer $\times$ type key (Table~\ref{tab:spec_curve}). | ||
| 25 | \end{tablenotes} | 25 | \end{tablenotes} | ||
| 26 | \end{threeparttable} | 26 | \end{threeparttable} | ||
| 27 | \end{table} | 27 | \end{table} | ||
modified
paper/tables/tab_spec_curve.tex
+1 −1
@@ -22,7 +22,7 @@ No resolution (one project per mention) & 308{,}065 & 100.0 & 0.0 & 1.00 & 1 & 2 | |||||
| 22 | \end{tabular} | 22 | \end{tabular} | ||
| 23 | \end{adjustbox} | 23 | \end{adjustbox} | ||
| 24 | \begin{tablenotes}\footnotesize | 24 | \begin{tablenotes}\footnotesize | ||
| 25 | −\item Every key is applied to the same 308{,}065 stored mentions, rebuilt from \texttt{resolve.py} in a temporary table on the master node; $\dagger$ marks the key of the released \texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of projects formed by the mentions of the Table~\ref{tab:provenance} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a property of the key: 21{,}027--308{,}065 at the release threshold, and the single-mention share and projects-per-issuer statistics move with it. Under the current key 66{,}405 projects (54.8\%) are keyed on a name token because the mention has no location, 54{,}658 on a location and 286 on the fallback ``general''; the single token ``acquisition'' absorbs 40{,}407 mentions into 2{,}256 projects (Table~\ref{tab:top_name_tokens}). The stored table is not exactly reproducible: 33{,}820 projects have several mentions at the maximum confidence (\texttt{arg\_max} ties), and re-running the authors' SQL changes 394 representative names and 600 descriptions while reproducing every project identifier. | 25 | +\item Every key is applied to the same 308{,}065 stored mentions, rebuilt from \texttt{resolve.py} in a temporary table on the master node; $\dagger$ marks the key of the released \texttt{projects} table, whose identifier set is reproduced exactly. ``Cheniere cluster size'' is the number of mentions in the cluster that contains the highest-confidence mention of the Table~\ref{tab:provenance} example. Thresholds drop mentions below the stated confidence before resolution. The project count is therefore a property of the key: 21{,}027--308{,}065 at the release threshold, and the single-mention share and projects-per-issuer statistics move with it. Under the current key 66{,}405 projects (54.8\%) are keyed on a name token because the mention has no location, 54{,}658 on a location and 229 on the fallback ``general'' (as in the text; 286 keys end in the token ``general'' because a project name whose first token is the word ``general'' is not distinguished from the fallback); the single token ``acquisition'' absorbs 40{,}407 mentions into 2{,}256 projects (Table~\ref{tab:top_name_tokens}). The stored table is not exactly reproducible: 33{,}820 projects have several mentions at the maximum confidence (\texttt{arg\_max} ties), and re-running the authors' SQL changes 394 representative names and 600 descriptions while reproducing every project identifier. | ||
| 26 | \end{tablenotes} | 26 | \end{tablenotes} | ||
| 27 | \end{threeparttable} | 27 | \end{threeparttable} | ||
| 28 | \end{table} | 28 | \end{table} | ||
modified
reproducibility/validation_report.md
+1 −1
@@ -18,7 +18,7 @@ No LLM or embedding call was made; nothing was written on any node; `data/stats. | |||||
| 18 | - **Export reconciliation (21 assertions, all pass):** the marginal CSV tables sum to the headline counts of `stats.json` — ledger rows 1,123,098; stored mentions 308,065 (by form, by year, by confidence); projects 121,099 (by type, status, span, mention bucket, first-seen year, sector, index membership); filings 527,548 (by form, by year × form, by year, by parse status); sections 1,342,283; universe 2,564 = 491 + 9 + 2,064 = 2,499 + 65 = Σ sectors; 10-K and 10-Q item tables sum to the per-form ledger rows and mentions; amount-bearing projects 56,500 in `amount_dist.csv`, `projects_by_type.csv` and `stats.json`. | 18 | - **Export reconciliation (21 assertions, all pass):** the marginal CSV tables sum to the headline counts of `stats.json` — ledger rows 1,123,098; stored mentions 308,065 (by form, by year, by confidence); projects 121,099 (by type, status, span, mention bucket, first-seen year, sector, index membership); filings 527,548 (by form, by year × form, by year, by parse status); sections 1,342,283; universe 2,564 = 491 + 9 + 2,064 = 2,499 + 65 = Σ sectors; 10-K and 10-Q item tables sum to the per-form ledger rows and mentions; amount-bearing projects 56,500 in `amount_dist.csv`, `projects_by_type.csv` and `stats.json`. | ||
| 19 | - **Regeneration:** `make_tables.py --out <scratch>` reproduces every committed `paper/tables/*.tex` byte for byte and the committed `results/inline_numbers.json` (207 macros). Figures are regenerated as PDF (vector; `CreationDate` suppressed) — compared visually, not byte-wise. | 19 | - **Regeneration:** `make_tables.py --out <scratch>` reproduces every committed `paper/tables/*.tex` byte for byte and the committed `results/inline_numbers.json` (207 macros). Figures are regenerated as PDF (vector; `CreationDate` suppressed) — compared visually, not byte-wise. | ||
| 20 | - **Build:** `latexmk` exit 0, 0 LaTeX errors, 0 undefined citations/references, bibtex 0 warnings, 39 pages. Overfull boxes: 17 (baseline 14); the three new ones are the wider Tables 3 and 5 whose notes still sit inside the `adjustbox` (item LITE-10, Stage E/L3 moves the notes outside), and the 835 pt box is the hand-typed regex of `appendix.tex:19` that Stage E replaces by `\input{tables/gate_regex}`. | 20 | - **Build:** `latexmk` exit 0, 0 LaTeX errors, 0 undefined citations/references, bibtex 0 warnings, 39 pages. Overfull boxes: 17 (baseline 14); the three new ones are the wider Tables 3 and 5 whose notes still sit inside the `adjustbox` (item LITE-10, Stage E/L3 moves the notes outside), and the 835 pt box is the hand-typed regex of `appendix.tex:19` that Stage E replaces by `\input{tables/gate_regex}`. | ||
| 21 | −- **Prompt and gate regex:** `paper/tables/prompt_verbatim.tex` and `gate_regex.tex` are now generated from the frozen snapshot (`code/audit/spid_snapshot/llm.py`, `taxonomy.py`, `rules.py`, SHA-256 in `SHA256SUMS`) by parsing the source with `ast`; the printed regex concatenates to the `_PROJECT_GATE` pattern of `rules.py` exactly (three source lines, 26 alternatives), and the prompt lists the 21 taxonomy labels in the order `TYPE_LABELS` sends them. | 21 | +- **Prompt and gate regex:** `paper/tables/prompt_verbatim.tex` and `gate_regex.tex` are now generated from the frozen snapshot (`code/audit/spid_snapshot/llm.py`, `taxonomy.py`, `rules.py`, SHA-256 in `SHA256SUMS`) by parsing the source with `ast`; the printed regex concatenates to the `_PROJECT_GATE` pattern of `rules.py` exactly (three source lines, 25 alternatives), and the prompt lists the 21 taxonomy labels in the order `TYPE_LABELS` sends them. | ||
| 22 | - **Literals removed from the generators:** `make_tables.py` — `2{,}587` (now `data/documented_inputs.json:iwv_equity_lines_reported`, labelled "count reported by the build log, source file not archived"), `(500 CIKs)` (now `stats.json:universe.n_sp500` / `sp500_ciks`), `2010-01-04`/`2026-10-27` (now `stats.json:filing_date_range` or min/max of `filings_by_form.csv`), `128.5` (now `stats.json:amount_total_median_musd` or `amount_dist.csv`), `2700` (y-limit now `universe.n × 1.05`), `300` (label offset now 0.7 % of the maximum), `170` (now `documented_inputs.json:snippet_chars`); `export_stats.py` — the `spid_runs` strings (`2026-06 (gpt-4o-mini, 16 workers, 153,894 sections)`, `… 48 workers, 968,134 sections`) are now read by regex from the two run logs and from the ledger (`None` when absent). `tests/test_tables.py::test_make_tables_has_no_data_literals` tokenises the generator and fails on any integer literal of three or more digits other than the scale `100` (`1e3/1e6/1e9` are exponent forms). | 22 | - **Literals removed from the generators:** `make_tables.py` — `2{,}587` (now `data/documented_inputs.json:iwv_equity_lines_reported`, labelled "count reported by the build log, source file not archived"), `(500 CIKs)` (now `stats.json:universe.n_sp500` / `sp500_ciks`), `2010-01-04`/`2026-10-27` (now `stats.json:filing_date_range` or min/max of `filings_by_form.csv`), `128.5` (now `stats.json:amount_total_median_musd` or `amount_dist.csv`), `2700` (y-limit now `universe.n × 1.05`), `300` (label offset now 0.7 % of the maximum), `170` (now `documented_inputs.json:snippet_chars`); `export_stats.py` — the `spid_runs` strings (`2026-06 (gpt-4o-mini, 16 workers, 153,894 sections)`, `… 48 workers, 968,134 sections`) are now read by regex from the two run logs and from the ledger (`None` when absent). `tests/test_tables.py::test_make_tables_has_no_data_literals` tokenises the generator and fails on any integer literal of three or more digits other than the scale `100` (`1e3/1e6/1e9` are exponent forms). | ||
| 23 | 23 | ||||
| 24 | ## What did not match / is still open | 24 | ## What did not match / is still open | ||
added
results/grounding/partners_normalisation_check.json
+16 −0
@@ -0,0 +1,16 @@ | |||||
| 1 | +{ | ||||
| 2 | + "_about": "Stage F (N3) read-only re-derivation on M3U96b of the verifier's verify_partners.py (reservoir sample of 20,000 project_mentions, repeatable seed 20261105, joined to src_sections/spid_sections text; duckdb read_only, threads=4). Compares the audit rule of code/audit/a02_grounding.py:norm (case-fold, punctuation to spaces, alias list, whole-word search) with a plain case-folded substring search.", | ||||
| 3 | + "date": "2026-11-05", | ||||
| 4 | + "db": "project-db/data/db/spid.duckdb (3155701760 bytes, 2026-10-28 12:21)", | ||||
| 5 | + "sample_mentions": 20000, | ||||
| 6 | + "seed": 20261105, | ||||
| 7 | + "with_partners": 5287, | ||||
| 8 | + "partners_grounded_pct_audit_rule": 99.32, | ||||
| 9 | + "partners_grounded_pct_naive_casefold": 96.1, | ||||
| 10 | + "with_locations": 7638, | ||||
| 11 | + "locations_grounded_pct_audit_rule": 78.16, | ||||
| 12 | + "population_rates_table17": { | ||||
| 13 | + "partners": 99.4, | ||||
| 14 | + "locations": 78.3 | ||||
| 15 | + } | ||||
| 16 | +} | ||||
modified
results/inline_numbers.json
+87 −17
@@ -979,6 +979,11 @@ | |||||
| 979 | "source": "data/example_provenance.csv", | 979 | "source": "data/example_provenance.csv", | ||
| 980 | "formula": "project_id[:8]" | 980 | "formula": "project_id[:8]" | ||
| 981 | }, | 981 | }, | ||
| 982 | + "provenanceKeyToken": { | ||||
| 983 | + "value": "san patricio county, texas", | ||||
| 984 | + "source": "results/resolver/example_provenance_keys.csv", | ||||
| 985 | + "formula": "last field of gkey (location token) of the example project" | ||||
| 986 | + }, | ||||
| 982 | "gateRegexFlags": { | 987 | "gateRegexFlags": { | ||
| 983 | "value": "re.IGNORECASE", | 988 | "value": "re.IGNORECASE", | ||
| 984 | "source": "code/audit/spid_snapshot/rules.py", | 989 | "source": "code/audit/spid_snapshot/rules.py", | ||
@@ -1024,6 +1029,26 @@ | |||||
| 1024 | "source": "results/grounding/fallback_bounds.json", | 1029 | "source": "results/grounding/fallback_bounds.json", | ||
| 1025 | "formula": "tech_deploy_total[1]" | 1030 | "formula": "tech_deploy_total[1]" | ||
| 1026 | }, | 1031 | }, | ||
| 1032 | + "groundPartnersCheckN": { | ||||
| 1033 | + "value": "5{,}287", | ||||
| 1034 | + "source": "results/grounding/partners_normalisation_check.json", | ||||
| 1035 | + "formula": "with_partners (mentions with a partner in the sample)" | ||||
| 1036 | + }, | ||||
| 1037 | + "groundPartnersCheckAuditPct": { | ||||
| 1038 | + "value": "99.3", | ||||
| 1039 | + "source": "results/grounding/partners_normalisation_check.json", | ||||
| 1040 | + "formula": "partners_grounded_pct_audit_rule" | ||||
| 1041 | + }, | ||||
| 1042 | + "groundPartnersPlainCasefoldPct": { | ||||
| 1043 | + "value": "96.1", | ||||
| 1044 | + "source": "results/grounding/partners_normalisation_check.json", | ||||
| 1045 | + "formula": "partners_grounded_pct_naive_casefold" | ||||
| 1046 | + }, | ||||
| 1047 | + "groundPartnersCheckSample": { | ||||
| 1048 | + "value": "20{,}000", | ||||
| 1049 | + "source": "results/grounding/partners_normalisation_check.json", | ||||
| 1050 | + "formula": "sample_mentions (reservoir sample)" | ||||
| 1051 | + }, | ||||
| 1027 | "groundedYearsPct": { | 1052 | "groundedYearsPct": { | ||
| 1028 | "value": "99.8", | 1053 | "value": "99.8", | ||
| 1029 | "source": "results/grounding/grounding_by_group.csv", | 1054 | "source": "results/grounding/grounding_by_group.csv", | ||
@@ -1879,6 +1904,11 @@ | |||||
| 1879 | "source": "results/ledger/log_summary.json:october.outside_window_by_reason", | 1904 | "source": "results/ledger/log_summary.json:october.outside_window_by_reason", | ||
| 1880 | "formula": "retryable_http_429_5xx" | 1905 | "formula": "retryable_http_429_5xx" | ||
| 1881 | }, | 1906 | }, | ||
| 1907 | + "llmRetryableInsideWindow": { | ||||
| 1908 | + "value": "94{,}465", | ||||
| 1909 | + "source": "results/ledger/log_summary.json:october", | ||||
| 1910 | + "formula": "llm_call_failed_by_reason.retryable_http_429_5xx - outside_window_by_reason.retryable_http_429_5xx (lines inside the 01:10-06:22 window)" | ||||
| 1911 | + }, | ||||
| 1882 | "llmFailedJsonDecodeError": { | 1912 | "llmFailedJsonDecodeError": { | ||
| 1883 | "value": "14", | 1913 | "value": "14", | ||
| 1884 | "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | 1914 | "source": "results/ledger/log_summary.json:october.llm_call_failed_by_reason", | ||
@@ -2074,6 +2104,16 @@ | |||||
| 2074 | "source": "results/ledger/log_summary.json:october.resume_batch1.log_mentions", | 2104 | "source": "results/ledger/log_summary.json:october.resume_batch1.log_mentions", | ||
| 2075 | "formula": "log" | 2105 | "formula": "log" | ||
| 2076 | }, | 2106 | }, | ||
| 2107 | + "resumeBatchLedgerMentions": { | ||||
| 2108 | + "value": "30", | ||||
| 2109 | + "source": "results/ledger/log_summary.json:october.resume_batch1.ledger_mentions", | ||||
| 2110 | + "formula": "ledger (sum of resume_batch1_vs_expected.csv:mentions)" | ||||
| 2111 | + }, | ||||
| 2112 | + "resumeBatchLedgerRows": { | ||||
| 2113 | + "value": "4{,}769", | ||||
| 2114 | + "source": "results/ledger/log_summary.json:october.resume_batch1.ledger_rows", | ||||
| 2115 | + "formula": "ledger rows of the batch" | ||||
| 2116 | + }, | ||||
| 2077 | "resumeBatchWindow": { | 2117 | "resumeBatchWindow": { | ||
| 2078 | "value": "06:38--06:40", | 2118 | "value": "06:38--06:40", | ||
| 2079 | "source": "results/ledger/log_summary.json:october.resume_batch1.window", | 2119 | "source": "results/ledger/log_summary.json:october.resume_batch1.window", | ||
@@ -2114,6 +2154,16 @@ | |||||
| 2114 | "source": "results/ledger/backfill_block.json", | 2154 | "source": "results/ledger/backfill_block.json", | ||
| 2115 | "formula": "rows_in_outage_window[1]" | 2155 | "formula": "rows_in_outage_window[1]" | ||
| 2116 | }, | 2156 | }, | ||
| 2157 | + "outageSurvivingProductive": { | ||||
| 2158 | + "value": "504", | ||||
| 2159 | + "source": "results/ledger/outage_window_rows_check.json", | ||||
| 2160 | + "formula": "rows_with_mentions (read-only re-derivation, Stage F)" | ||||
| 2161 | + }, | ||||
| 2162 | + "outageSurvivingZeroRows": { | ||||
| 2163 | + "value": "0", | ||||
| 2164 | + "source": "results/ledger/outage_window_rows_check.json", | ||||
| 2165 | + "formula": "rows_zero_mentions" | ||||
| 2166 | + }, | ||||
| 2117 | "backfillTimestamp": { | 2167 | "backfillTimestamp": { | ||
| 2118 | "value": "October~27, 2026, 13:12", | 2168 | "value": "October~27, 2026, 13:12", | ||
| 2119 | "source": "results/ledger/backfill_block.json:min_extracted_at", | 2169 | "source": "results/ledger/backfill_block.json:min_extracted_at", | ||
@@ -2154,6 +2204,16 @@ | |||||
| 2154 | "source": "results/ledger/gate_pass_totals.json", | 2204 | "source": "results/ledger/gate_pass_totals.json", | ||
| 2155 | "formula": "usd_per_october_row_paper_total_775" | 2205 | "formula": "usd_per_october_row_paper_total_775" | ||
| 2156 | }, | 2206 | }, | ||
| 2207 | + "callsBothRuns": { | ||||
| 2208 | + "value": "703{,}725", | ||||
| 2209 | + "source": "results/ledger/gate_pass_by_run.csv", | ||||
| 2210 | + "formula": "sum(calls) over the two runs" | ||||
| 2211 | + }, | ||||
| 2212 | + "usdPerCallBothRuns": { | ||||
| 2213 | + "value": "0.0011", | ||||
| 2214 | + "source": "results/ledger/gate_pass_by_run.csv, data/documented_inputs.json:total_cost_usd", | ||||
| 2215 | + "formula": "total_cost_usd / sum(calls) over both runs (alternative basis, author to confirm)" | ||||
| 2216 | + }, | ||||
| 2157 | "confExactDefault": { | 2217 | "confExactDefault": { | ||
| 2158 | "value": "3{,}278", | 2218 | "value": "3{,}278", | ||
| 2159 | "source": "results/ledger/code_vs_data_checks.json", | 2219 | "source": "results/ledger/code_vs_data_checks.json", | ||
@@ -2849,6 +2909,16 @@ | |||||
| 2849 | "source": "results/runs/within_issuer_paired_10q.json", | 2909 | "source": "results/runs/within_issuer_paired_10q.json", | ||
| 2850 | "formula": "100*se" | 2910 | "formula": "100*se" | ||
| 2851 | }, | 2911 | }, | ||
| 2912 | + "pairedProductiveJune": { | ||||
| 2913 | + "value": "11.9", | ||||
| 2914 | + "source": "results/runs/within_issuer_paired_10q.json", | ||||
| 2915 | + "formula": "100*june_mean_productive_share (mean of per-issuer shares)" | ||||
| 2916 | + }, | ||||
| 2917 | + "pairedProductiveOctober": { | ||||
| 2918 | + "value": "12.5", | ||||
| 2919 | + "source": "results/runs/within_issuer_paired_10q.json", | ||||
| 2920 | + "formula": "100*oct_mean_productive_share (mean of per-issuer shares)" | ||||
| 2921 | + }, | ||||
| 2852 | "cohortBothPerIssuerAll": { | 2922 | "cohortBothPerIssuerAll": { | ||
| 2853 | "value": "55.8", | 2923 | "value": "55.8", | ||
| 2854 | "source": "results/runs/table6_cohort.csv", | 2924 | "source": "results/runs/table6_cohort.csv", | ||
@@ -3239,15 +3309,15 @@ | |||||
| 3239 | "source": "results/fp_classes/summary.json", | 3309 | "source": "results/fp_classes/summary.json", | ||
| 3240 | "formula": "mentions_matched[2]" | 3310 | "formula": "mentions_matched[2]" | ||
| 3241 | }, | 3311 | }, | ||
| 3242 | − "finAmountB": { | 3312 | + "finAmountT": { | ||
| 3243 | "value": "17.8", | 3313 | "value": "17.8", | ||
| 3244 | "source": "results/fp_classes/summary.json", | 3314 | "source": "results/fp_classes/summary.json", | ||
| 3245 | − "formula": "matched_amount_gusd[0]/1e3 (T USD)" | 3315 | + "formula": "matched_amount_gusd[0]/1e3 (T USD; the source field is in billions)" | ||
| 3246 | }, | 3316 | }, | ||
| 3247 | − "finAmountDenomB": { | 3317 | + "finAmountDenomT": { | ||
| 3248 | "value": "104.2", | 3318 | "value": "104.2", | ||
| 3249 | "source": "results/fp_classes/summary.json", | 3319 | "source": "results/fp_classes/summary.json", | ||
| 3250 | − "formula": "matched_amount_gusd[1]/1e3 (T USD)" | 3320 | + "formula": "matched_amount_gusd[1]/1e3 (T USD; the source field is in billions)" | ||
| 3251 | }, | 3321 | }, | ||
| 3252 | "finAmountSharePct": { | 3322 | "finAmountSharePct": { | ||
| 3253 | "value": "17.1", | 3323 | "value": "17.1", | ||
@@ -3871,33 +3941,33 @@ | |||||
| 3871 | }, | 3941 | }, | ||
| 3872 | "chunkWords": { | 3942 | "chunkWords": { | ||
| 3873 | "value": "512", | 3943 | "value": "512", | ||
| 3874 | − "source": "src/parsing/chunker.py", | 3944 | + "source": "data/documented_inputs.json:pipeline_chunk_words", | ||
| 3875 | − "formula": "chunk_size" | 3945 | + "formula": "chunk_size default of the embedding chunker (chunk_size: int = 512); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/parsing/chunker.py, SHA-256 716a4013fb839a68ec613aa5f0c91e8da4c8a8135051fb95fec5350d35fc3c79; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable" | ||
| 3876 | }, | 3946 | }, | ||
| 3877 | "chunkOverlap": { | 3947 | "chunkOverlap": { | ||
| 3878 | "value": "64", | 3948 | "value": "64", | ||
| 3879 | − "source": "src/parsing/chunker.py", | 3949 | + "source": "data/documented_inputs.json:pipeline_chunk_overlap", | ||
| 3880 | − "formula": "overlap" | 3950 | + "formula": "overlap default of the embedding chunker (overlap: int = 64); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/parsing/chunker.py, SHA-256 716a4013fb839a68ec613aa5f0c91e8da4c8a8135051fb95fec5350d35fc3c79; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable" | ||
| 3881 | }, | 3951 | }, | ||
| 3882 | "maxFileMb": { | 3952 | "maxFileMb": { | ||
| 3883 | "value": "50", | 3953 | "value": "50", | ||
| 3884 | − "source": "src/parsing/html_parser.py", | 3954 | + "source": "data/documented_inputs.json:pipeline_max_file_mb", | ||
| 3885 | − "formula": "max_file_size_mb default" | 3955 | + "formula": "max_file_size_mb default of the HTML parser (html_cfg.get(\"max_file_size_mb\", 50)); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/parsing/html_parser.py, SHA-256 61aca675ff4652e3479f50a10ce29503f66f7f74ffd4f78df0b87ad517b1908e; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable" | ||
| 3886 | }, | 3956 | }, | ||
| 3887 | "cooldownSeconds": { | 3957 | "cooldownSeconds": { | ||
| 3888 | "value": "65", | 3958 | "value": "65", | ||
| 3889 | − "source": "src/acquisition/bulk_downloader.py", | 3959 | + "source": "data/documented_inputs.json:pipeline_cooldown_seconds", | ||
| 3890 | − "formula": "COOLDOWN_SECONDS" | 3960 | + "formula": "COOLDOWN_SECONDS = 65.0 after an SEC 403/429; manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/acquisition/bulk_downloader.py, SHA-256 862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable" | ||
| 3891 | }, | 3961 | }, | ||
| 3892 | "secRateLimit": { | 3962 | "secRateLimit": { | ||
| 3893 | "value": "ten", | 3963 | "value": "ten", | ||
| 3894 | − "source": "src/acquisition/bulk_downloader.py (docstring) / secfairaccess", | 3964 | + "source": "data/documented_inputs.json:pipeline_sec_rate_limit", | ||
| 3895 | − "formula": "requests per second per IP" | 3965 | + "formula": "module docstring 'SEC allows 10 req/s per IP' (sec.gov fair-access policy); manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/acquisition/bulk_downloader.py, SHA-256 862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable" | ||
| 3896 | }, | 3966 | }, | ||
| 3897 | "acqGatewayDivisor": { | 3967 | "acqGatewayDivisor": { | ||
| 3898 | "value": "nine", | 3968 | "value": "nine", | ||
| 3899 | − "source": "src/acquisition/bulk_downloader.py", | 3969 | + "source": "data/documented_inputs.json:pipeline_acq_gateway_divisor", | ||
| 3900 | − "formula": "default aggregate rate shared behind one IP" | 3970 | + "formula": "BulkDownloader default rate: float = 9.0 requests per second shared behind one IP; manuscripts/FIRC-0001/pipeline_code_2324b58 (archived pipeline code at revision 2324b58, outside this repository)/src/acquisition/bulk_downloader.py, SHA-256 862139f47d52cef9c33d6a64e77fae1ae3a31cc9b288762c6a2187579f5a9159; recorded here (Stage F, N1) so that a bare clone regenerates the macro, re-read and checked against the archive by code/make_tables.py when it is reachable" | ||
| 3901 | }, | 3971 | }, | ||
| 3902 | "acqNewDocs": { | 3972 | "acqNewDocs": { | ||
| 3903 | "value": "about 400{,}000", | 3973 | "value": "about 400{,}000", | ||
@@ -3952,7 +4022,7 @@ | |||||
| 3952 | "totalCostUsd": { | 4022 | "totalCostUsd": { | ||
| 3953 | "value": "775", | 4023 | "value": "775", | ||
| 3954 | "source": "data/documented_inputs.json:total_cost_usd", | 4024 | "source": "data/documented_inputs.json:total_cost_usd", | ||
| 3955 | − "formula": "author-reported total of the October 2026 extraction (docs/research_log.md); the API invoice is not archived" | 4025 | + "formula": "author-reported total recorded in docs/research_log.md against the 2026-10-27/28 entry 'Full LLM extraction of 968,134 new sections ... ≈ 775 USD total' (October run); the review's constraints note (peer_reviews/FIRC-0001/_constraints.md) attributes the same ≈ 775 USD to both runs (153,894 + 968,134 sections); the API invoice is not archived; basis to be confirmed by the authors (Stage F, N9)" | ||
| 3956 | }, | 4026 | }, | ||
| 3957 | "stabilitySections": { | 4027 | "stabilitySections": { | ||
| 3958 | "value": "500", | 4028 | "value": "500", | ||
added
results/ledger/outage_window_rows_check.json
+14 −0
@@ -0,0 +1,14 @@ | |||||
| 1 | +{ | ||||
| 2 | + "_about": "Stage F (N6) read-only re-derivation on M3U96b: ledger rows whose extracted_at lies inside the outage window of results/ledger/backfill_block.json:outage_window, split by productivity (duckdb read_only, threads=4).", | ||||
| 3 | + "date": "2026-11-05", | ||||
| 4 | + "db": "project-db/data/db/spid.duckdb (3155701760 bytes, 2026-10-28 12:21)", | ||||
| 5 | + "query": "select count(*), count(*) filter (where n_mentions>0), count(*) filter (where n_mentions=0), sum(n_mentions) from extracted_sections where extracted_at between TIMESTAMP '2026-10-28 01:10:00' and TIMESTAMP '2026-10-28 06:22:00'", | ||||
| 6 | + "window": [ | ||||
| 7 | + "2026-10-28 01:10:00", | ||||
| 8 | + "2026-10-28 06:22:00" | ||||
| 9 | + ], | ||||
| 10 | + "rows": 504, | ||||
| 11 | + "rows_with_mentions": 504, | ||||
| 12 | + "rows_zero_mentions": 0, | ||||
| 13 | + "mentions": 1011 | ||||
| 14 | +} | ||||
| 15 | |||||