SPB Git

Repository Public Research preview

n1_05_alt_anchors.py 34.4 KB 297 lines python

1"""n1_05_alt_anchors.py — X2 alternative-anchor test and X3 negation / boilerplate probe (AR-FIRC0002-2, AUTHORISED).23Embeds ONLY the new anchor paragraphs (and ~40 probe sentences) locally with sentence-transformers/all-MiniLM-L6-v24(cached, offline, CPU) and scores them against the STORED chunk embeddings (read-only parquet copy of M3U96b).5No corpus re-embedding, no LLM call.6Steps: (1) re-embed the 8 original anchors locally and compare with the stored anchor vectors;7       (2) stream the 10.7 M stored chunk vectors, dot products with [stored 8 | local 8 | alt k] anchors -> chunk_scores_alt.parquet;8       (3) check that the stored-anchor recomputation reproduces chunk_scores to float precision;9       (4) filing-level section-weighted means (pipeline weights) and firm-year means (eq. annual) for every column,10           check against filing_scores / annual_final; firm-year rank correlations, composite quintile transitions, top-30 overlap;11       (5) X3 probe sentences vs stored and alternative anchors.12Outputs: results/alt_anchor_results.json, alt_firm_year.csv, alt_corr_by_year.csv, alt_anchor_texts.json, probe_sentences.csv13"""14import os, sys, json, glob, time15import numpy as np, pandas as pd16import pyarrow as pa, pyarrow.parquet as pq, pyarrow.dataset as ds17from scipy.stats import spearmanr, pearsonr18sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))19from n1_common import *  # noqa20os.environ.setdefault("HF_HUB_OFFLINE", "1"); os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")21import torch; torch.set_num_threads(8)22from sentence_transformers import SentenceTransformer2324# ----------------------------------------------------------------------------- original anchors (verbatim, src/factors/anchors.py)25ORIG = {26 "ai_adoption": [27  "The company internally uses artificial intelligence, machine learning, generative AI, large language models, automation, predictive analytics, intelligent agents, and data-driven systems to improve productivity, operations, efficiency, decision-making, customer service, risk management, compliance, cybersecurity, supply chain, software development, finance, human resources, and other business processes.",28  "We have implemented generative AI, large language models, and natural language processing capabilities across our enterprise. Our AI strategy includes deploying intelligent automation, computer vision, and predictive analytics systems to transform operations.",29  "Investment in artificial intelligence research and development, including partnerships with AI technology providers, hiring of machine learning engineers, and deployment of AI infrastructure including GPU computing clusters for internal business applications."],30 "ai_disruption_risk": [31  "Artificial intelligence may reduce the scarcity, defensibility, differentiation, pricing power, or competitive advantage of the company's products or services by enabling rapid replication, automation, synthetic generation, low-cost substitution, or commoditization of capabilities that were previously difficult, expensive, or specialized.",32  "The rapid advancement of AI technologies may render our current offerings less competitive. New market entrants leveraging AI could fundamentally alter industry dynamics, reduce barriers to entry, and erode our market position through AI-native products and services.",33  "Risks associated with artificial intelligence include potential competitive displacement by AI-powered alternatives, obsolescence of our existing products, regulatory uncertainty around AI governance, algorithmic bias, intellectual property challenges, and emergence of AI-native competitors."],34 "ai_infrastructure": [35  "Investment in AI computing infrastructure including data centers, GPU and TPU clusters, cloud computing resources, semiconductor chips, accelerator hardware, and high-performance computing systems dedicated to artificial intelligence and machine learning workloads.",36  "Capital expenditures for AI infrastructure, including networking equipment for distributed AI training, specialized computing platforms for large-scale model development, edge computing for AI inference, and data pipeline infrastructure for machine learning operations.",37  "Cloud infrastructure spending for AI workloads, including partnerships with hyperscale cloud providers, deployment of AI-optimized hardware, GPU supply chain management, and investment in next-generation computing architectures for artificial intelligence applications."],38 "esg_commitment": [39  "Environmental, social, and governance initiatives including sustainability reporting, corporate social responsibility programs, diversity equity and inclusion efforts, stakeholder engagement, and adherence to ESG frameworks and standards such as SASB, GRI, and TCFD.",40  "The company is committed to ESG principles including reducing carbon emissions, improving energy efficiency, promoting workforce diversity, ensuring supply chain responsibility, ethical governance practices, and transparent reporting of environmental and social performance metrics.",41  "Board-level oversight of environmental and social risk management, executive compensation tied to sustainability metrics, corporate governance improvements, and integration of ESG factors into strategic decision-making and capital allocation processes."],42 "climate_risk": [43  "Climate change poses material risks to our operations including physical risks from extreme weather events, transition risks from evolving regulations, carbon pricing mechanisms, and shifting consumer preferences toward sustainable alternatives and low-carbon products.",44  "Exposure to climate-related financial risks including greenhouse gas emission regulations, carbon taxes, stranded asset risk, water scarcity, supply chain disruption from natural disasters, and reputational risk from insufficient climate action and environmental degradation.",45  "Climate risk assessment and scenario analysis aligned with TCFD recommendations, including evaluation of physical and transition risks under different warming scenarios, Scope 1, 2, and 3 greenhouse gas emissions measurement and reduction targets."],46 "cybersecurity": [47  "Cybersecurity risks including data breaches, ransomware attacks, unauthorized access to systems, intellectual property theft, and operational disruption from cyber incidents. Investment in information security controls, threat detection, and incident response capabilities.",48  "Data privacy and security compliance including GDPR, CCPA, and other data protection regulations. Cybersecurity governance, risk management frameworks, security operations centers, and employee security awareness training programs to protect critical infrastructure.",49  "Critical infrastructure protection, network security, endpoint security, identity and access management, zero-trust architecture implementation, penetration testing, cyber insurance, and board-level oversight of cybersecurity risk management and AI-powered threat detection."],50 "automation_labor": [51  "Automation of business processes, robotic process automation, intelligent automation, and workforce transformation. Impact of automation on labor force, reskilling and upskilling programs, and the balance between human workers and automated systems in manufacturing and services.",52  "Workforce restructuring driven by technological automation, including reduction in manual processes, deployment of automated manufacturing and logistics systems, and investment in human-machine collaboration technologies and digital transformation of work.",53  "Labor market dynamics including talent acquisition challenges, employee retention, workforce planning in the context of automation, gig economy participation, remote work infrastructure, and the future of work transformation driven by AI and robotics."],54 "macro_risk": [55  "Macroeconomic risks including inflation, interest rate fluctuations, currency exchange rate volatility, recession risk, geopolitical tensions, trade policy uncertainty, tariffs, and supply chain disruptions affecting business performance and revenue.",56  "Economic uncertainty and market volatility, including sensitivity to GDP growth, consumer spending patterns, credit market conditions, commodity price fluctuations, and the impact of monetary and fiscal policy changes on business operations and profitability.",57  "Global economic risks including sovereign debt concerns, emerging market instability, trade war escalation, sanctions, energy price shocks, pandemic impacts on economic activity, and systemic financial market risks affecting corporate investment and capital markets."],58}59# ----------------------------------------------------------------------------- alternative anchors (X2)60ALT = {61 # (a) AI-adoption anchor stripped of the automation / cybersecurity / data-driven / analytics terms shared with the placebo anchors62 "ai_adoption_strip": {"profile": "operational_measures", "edit": "original 3 paragraphs minus 'automation', 'intelligent automation', 'cybersecurity', 'data-driven systems', 'predictive analytics', 'intelligent agents' and the generic business-function list", "texts": [63  "The company internally uses artificial intelligence, machine learning, generative AI, and large language models to improve productivity, operations, efficiency, decision-making, and customer service.",64  "We have implemented generative AI, large language models, and natural language processing capabilities across our enterprise. Our AI strategy includes deploying machine learning models, computer vision, and neural networks to transform operations.",65  "Investment in artificial intelligence research and development, including partnerships with AI technology providers, hiring of machine learning engineers, and deployment of AI infrastructure including GPU computing clusters for internal business applications."]},66 # (b) a narrow, single-register AI anchor (no infrastructure, no business-function list)67 "ai_adoption_core": {"profile": "operational_measures", "edit": "new narrow paragraphs: only unambiguous AI / machine-learning terms, no automation, analytics, cybersecurity, hardware or business-function vocabulary", "texts": [68  "We use artificial intelligence and machine learning models in our products and operations.",69  "The company has deployed generative artificial intelligence, large language models, and neural network models developed by our data science teams.",70  "Our machine learning and artificial intelligence capabilities, including natural language processing and computer vision, are embedded in our services."]},71 # (c) AI-infrastructure anchor stripped of generic data-centre / cloud / semiconductor / networking vocabulary72 "ai_infrastructure_strip": {"profile": "operational_measures", "edit": "original 3 paragraphs minus standalone 'data centers', 'cloud computing resources', 'semiconductor chips', 'high-performance computing systems', 'networking equipment', 'edge computing', 'hyperscale cloud providers', 'GPU supply chain management'; every remaining hardware term is tied to AI / machine learning", "texts": [73  "Investment in computing infrastructure dedicated to artificial intelligence and machine learning workloads, including GPU and TPU clusters and AI accelerator hardware for model training.",74  "Capital expenditures for AI infrastructure, including specialized computing platforms for large-scale machine learning model development, hardware for AI inference, and data pipeline infrastructure for machine learning operations.",75  "Spending on AI-optimized hardware and next-generation computing architectures for artificial intelligence applications, including partnerships with providers of AI model training capacity."]},76 # (d) placebo anchors stripped of their AI terms77 "cybersecurity_strip": {"profile": "risk_measures", "edit": "original 3 paragraphs minus 'and AI-powered threat detection'", "texts": [78  ORIG["cybersecurity"][0], ORIG["cybersecurity"][1],79  "Critical infrastructure protection, network security, endpoint security, identity and access management, zero-trust architecture implementation, penetration testing, cyber insurance, and board-level oversight of cybersecurity risk management."]},80 "automation_labor_strip": {"profile": "operational_measures", "edit": "original 3 paragraphs minus 'intelligent automation' and 'driven by AI and robotics' -> 'driven by robotics'", "texts": [81  "Automation of business processes, robotic process automation, and workforce transformation. Impact of automation on labor force, reskilling and upskilling programs, and the balance between human workers and automated systems in manufacturing and services.",82  ORIG["automation_labor"][1],83  "Labor market dynamics including talent acquisition challenges, employee retention, workforce planning in the context of automation, gig economy participation, remote work infrastructure, and the future of work transformation driven by robotics."]},84}85# ----------------------------------------------------------------------------- X3 probe sentences86PROBES = [87 ("affirmative_ai", "We use artificial intelligence and machine learning to improve our operations and customer service."),88 ("affirmative_ai", "We are an artificial-intelligence leader and have deployed generative AI across our enterprise."),89 ("affirmative_ai", "During the year we implemented machine learning models to forecast demand and optimize pricing."),90 ("affirmative_ai", "Our products incorporate large language models and natural language processing capabilities."),91 ("affirmative_ai", "We invested in GPU clusters to train our proprietary machine learning models."),92 ("affirmative_ai", "The company has hired machine learning engineers and formed an internal AI center of excellence."),93 ("negated_ai", "We do not currently use artificial intelligence in our operations."),94 ("negated_ai", "We do not use machine learning or generative AI in our products or services."),95 ("negated_ai", "The company has not deployed artificial intelligence and has no plans to do so."),96 ("negated_ai", "We have not implemented large language models or natural language processing capabilities."),97 ("negated_ai", "Our business does not rely on artificial intelligence or machine learning technologies."),98 ("negated_ai", "We have no investments in artificial intelligence infrastructure or GPU computing clusters."),99 ("ai_risk_boilerplate", "The use of artificial intelligence by us or by third parties may expose us to regulatory, reputational and legal risks."),100 ("ai_risk_boilerplate", "Developments in artificial intelligence could render our products obsolete or less competitive."),101 ("ai_risk_boilerplate", "Issues in the development and use of AI, including generative AI, may result in reputational harm or liability."),102 ("ai_risk_boilerplate", "Our competitors may adopt artificial intelligence more quickly than we do, which could adversely affect our results."),103 ("paraphrase_no_keyword", "We deploy neural networks to forecast demand and route inventory in our warehouses."),104 ("paraphrase_no_keyword", "Our platform learns from customer data to personalize recommendations automatically."),105 ("paraphrase_no_keyword", "We trained a transformer model on our proprietary data to classify documents."),106 ("paraphrase_no_keyword", "Our software uses predictive models that improve as more data is collected."),107 ("automation_no_ai", "We automated our invoice processing with robotic process automation software."),108 ("automation_no_ai", "The plant installed automated assembly lines and reduced manual labor by 15 percent."),109 ("automation_no_ai", "Workforce restructuring reflects the deployment of automated logistics systems."),110 ("automation_no_ai", "We continue to invest in process automation and digital transformation of our back office."),111 ("cyber_no_ai", "We experienced a ransomware attack that disrupted our operations for several days."),112 ("cyber_no_ai", "Cybersecurity incidents, data breaches and unauthorized access to our systems could harm our business."),113 ("cyber_no_ai", "We maintain a security operations center and conduct annual penetration testing."),114 ("utility_data_center_load", "Load growth from new data center customers is expected to increase our retail energy sales."),115 ("utility_data_center_load", "We signed agreements to serve several large data centers, which require substantial new generation and transmission capacity."),116 ("utility_data_center_load", "Our service territory is attracting data center and semiconductor manufacturing customers with significant electricity demand."),117 ("utility_data_center_load", "The utility filed a rate case to recover the cost of infrastructure serving large load customers, including data centers."),118 ("it_generic", "We migrated our enterprise resource planning system to the cloud during the year."),119 ("it_generic", "Our information technology systems support order processing, inventory management and financial reporting."),120 ("it_generic", "We invested in network equipment and data storage to support our growing customer base."),121 ("unrelated", "Net sales increased 4 percent due to higher volumes in our beverage segment."),122 ("unrelated", "The company declared a quarterly dividend of 0.25 dollars per share payable in March."),123 ("unrelated", "We lease our corporate headquarters under an agreement that expires in 2029."),124 ("unrelated", "Interest expense decreased because of lower average borrowings under the credit facility."),125 ("unrelated", "Our effective tax rate was 21.4 percent compared with 22.1 percent in the prior year."),126]127128t_all = time.time()129out = {"model": MODEL_ID, "device": "cpu", "seed": SEED}130with Timer("load model + embed anchors (local, anchors only)"):131    model = SentenceTransformer(MODEL_ID, device="cpu")132    out["max_seq_length"] = int(model.max_seq_length)133    def centroid(texts):134        v = model.encode(texts, normalize_embeddings=True, batch_size=len(texts), show_progress_bar=False)135        cvec = v.mean(axis=0); return (cvec / np.linalg.norm(cvec)).astype(np.float32)136    stored = pd.read_parquet(f"{DATA}/anchor_embeddings/anchor_embeddings.parquet")137    S = {r.anchor_id: np.asarray(r.vector, dtype=np.float32) for r in stored.itertuples()}138    L = {k: centroid(v) for k, v in ORIG.items()}139    A = {k: centroid(v["texts"]) for k, v in ALT.items()}140    out["stored_anchor_norms"] = {k: float(np.linalg.norm(v)) for k, v in S.items()}141    out["local_vs_stored_anchor_cosine"] = {k: float(L[k] @ S[k] / np.linalg.norm(S[k])) for k in ORIG}142    out["alt_vs_original_anchor_cosine"] = {k: {o: float(A[k] @ S[o] / np.linalg.norm(S[o])) for o in ORIG} for k in ALT}143    out["original_anchor_cosine_matrix"] = {k: {o: float(S[k] @ S[o] / np.linalg.norm(S[k]) / np.linalg.norm(S[o])) for o in ORIG} for k in ORIG}144    cols = [f"s_{k}" for k in ORIG] + [f"l_{k}" for k in ORIG] + [f"alt_{k}" for k in ALT]145    M = np.stack([S[k] / np.linalg.norm(S[k]) for k in ORIG] + [L[k] for k in ORIG] + [A[k] for k in ALT]).astype(np.float32)  # (21, 384)146    json.dump({"original": ORIG, "alternative": ALT, "columns": cols}, open(f"{RES}/alt_anchor_texts.json", "w"), indent=1)147    print(json.dumps(out["local_vs_stored_anchor_cosine"], indent=1))148149ALT_PATH = f"{RES}/chunk_scores_alt.parquet"150if not os.path.exists(ALT_PATH + ".done"):151    with Timer("score 10.7 M stored chunk vectors against 21 anchors (streaming)"):152        dataset = ds.dataset(f"{DATA}/embeddings", format="parquet", partitioning="hive")153        scanner = dataset.scanner(columns=["source_id", "vector"], batch_size=20000)154        schema = pa.schema([pa.field("chunk_id", pa.string())] + [pa.field(cn, pa.float32()) for cn in cols])155        writer = pq.ParquetWriter(ALT_PATH + ".tmp", schema, compression="zstd")156        n = 0; norm_stats = []; first = True157        for batch in scanner.to_batches():158            if batch.num_rows == 0: continue159            vec = batch.column("vector")160            if isinstance(vec, pa.ChunkedArray): vec = vec.combine_chunks()161            arr = np.asarray(vec.flatten().to_numpy(zero_copy_only=False), dtype=np.float32).reshape(len(vec), -1)162            norms = np.linalg.norm(arr, axis=1)163            if first:164                out["vector_dim"] = int(arr.shape[1]); out["first_batch_norm_mean"] = float(norms.mean()); first = False165            norm_stats.append((float(norms.mean()), float(norms.min()), float(norms.max()), len(norms)))166            if not (0.99 <= norms.mean() <= 1.01):   # the pipeline normalizes only if the first sample is not pre-normalized; we mirror it per batch167                arr = arr / np.clip(norms, 1e-12, None)[:, None]168            sc = arr @ M.T169            tbl = pa.table({"chunk_id": batch.column("source_id"), **{cn: pa.array(sc[:, j], type=pa.float32()) for j, cn in enumerate(cols)}})170            writer.write_table(tbl); n += len(vec)171            if n % 1_000_000 < 20000: print(f"  {n:,} vectors", flush=True)172        writer.close(); os.replace(ALT_PATH + ".tmp", ALT_PATH); open(ALT_PATH + ".done", "w").write(str(n))173        ns = np.array([[a, b, c_, d] for a, b, c_, d in norm_stats])174        out["vectors_scored"] = int(n); out["vector_norm"] = {"mean": float((ns[:, 0] * ns[:, 3]).sum() / ns[:, 3].sum()), "min": float(ns[:, 1].min()), "max": float(ns[:, 2].max())}175else:176    out["vectors_scored"] = int(open(ALT_PATH + ".done").read())177178c = connect(); views(c)179with Timer("validate recomputation vs stored chunk_scores"):180    c.execute(f"CREATE OR REPLACE TABLE alt AS SELECT * FROM read_parquet('{ALT_PATH}') QUALIFY row_number() OVER (PARTITION BY chunk_id ORDER BY chunk_id) = 1")181    out["alt_rows_dedup"] = c.execute("SELECT count(*) FROM alt").fetchone()[0]182    out["alt_join_chunk_scores"] = c.execute("SELECT count(*) FROM alt JOIN chunk_scores USING (chunk_id)").fetchone()[0]183    exprs = ", ".join(f"max(abs(cs.score_{k} - a.s_{k})) AS maxdiff_s_{k}, corr(cs.score_{k}, a.s_{k}) AS corr_s_{k}, max(abs(cs.score_{k} - a.l_{k})) AS maxdiff_l_{k}, corr(cs.score_{k}, a.l_{k}) AS corr_l_{k}" for k in ORIG)184    out["chunk_level_validation"] = c.execute(f"SELECT {exprs} FROM chunk_scores cs JOIN alt a USING (chunk_id)").df().iloc[0].to_dict()185    out["chunk_level_alt_corr"] = c.execute("SELECT " + ", ".join(f"corr(cs.score_ai_adoption, a.alt_{k}) AS adopt_vs_{k}, corr(cs.score_ai_infrastructure, a.alt_{k}) AS infra_vs_{k}" for k in ALT) + " FROM chunk_scores cs JOIN alt a USING (chunk_id)").df().iloc[0].to_dict()186    # Spearman at chunk level on a 2 M hash sample187    out["chunk_level_alt_spearman_2Msample"] = c.execute("WITH s AS (SELECT cs.score_ai_adoption a0, cs.score_ai_infrastructure i0, a.* FROM chunk_scores cs JOIN alt a USING (chunk_id) WHERE hash(chunk_id) % 5 = 0), r AS (SELECT rank() OVER (ORDER BY a0) ra0, rank() OVER (ORDER BY i0) ri0, " + ", ".join(f"rank() OVER (ORDER BY alt_{k}) r_{k}" for k in ALT) + " FROM s) SELECT " + ", ".join(f"corr(ra0, r_{k}) AS adopt_vs_{k}, corr(ri0, r_{k}) AS infra_vs_{k}" for k in ALT) + " FROM r").df().iloc[0].to_dict()188189with Timer("filing-level and firm-year aggregation (pipeline weights)"):190    wop = weight_case("operational_measures", "cs.section_name"); wrk = weight_case("risk_measures", "cs.section_name")191    prof = {**{f"s_{k}": ANCHOR_PROFILE[k] for k in ORIG}, **{f"l_{k}": ANCHOR_PROFILE[k] for k in ORIG}, **{f"alt_{k}": ALT[k]["profile"] for k in ALT}}192    stats = ", ".join(f"sum(a.{cn} * {wop if p == 'operational_measures' else wrk}) / NULLIF(sum({wop if p == 'operational_measures' else wrk}), 0) AS {cn}_wmean, avg(a.{cn}) AS {cn}_mean, max(a.{cn}) AS {cn}_max" for cn, p in prof.items())193    c.execute(f"""CREATE OR REPLACE TABLE filing_alt AS SELECT cs.accession_number, any_value(cs.cik) cik, any_value(cs.ticker) ticker, any_value(cs.form_type) form_type,194        any_value(cs.fiscal_year) fiscal_year, count(*) chunk_count, {stats} FROM chunk_scores cs JOIN alt a USING (chunk_id) GROUP BY 1""")195    out["filing_level_validation"] = c.execute("SELECT count(*) n, max(abs(f.ai_adoption_wmean - x.s_ai_adoption_wmean)) maxdiff_adopt_wmean, corr(f.ai_adoption_wmean, x.s_ai_adoption_wmean) corr_adopt_wmean, max(abs(f.ai_infrastructure_wmean - x.s_ai_infrastructure_wmean)) maxdiff_infra_wmean, max(abs(f.chunk_count - x.chunk_count)) maxdiff_chunks FROM filing_scores f JOIN filing_alt x USING (accession_number)").df().iloc[0].to_dict()196    wcols = [f"{cn}_wmean" for cn in prof]197    fy = c.execute("SELECT cik, any_value(ticker) ticker, fiscal_year, count(*) n_filings, " + ", ".join(f"avg({w}) AS {w}" for w in wcols) + ", " + ", ".join(f"avg(CASE WHEN form_type='10-K' THEN {w} END) AS {w}_10k" for w in wcols if w.startswith("alt_") or w in ("s_ai_adoption_wmean", "s_ai_infrastructure_wmean")) + " FROM filing_alt GROUP BY cik, fiscal_year").df()198    af = c.execute("SELECT cik, ticker, fiscal_year, sector, ai_adoption_wmean, ai_infrastructure_wmean, ai_disruption_risk_wmean, cybersecurity_wmean, automation_labor_wmean FROM annual_final").df()199    m = af.merge(fy.drop(columns=["ticker"]), on=["cik", "fiscal_year"], how="inner")200    m = m[m.fiscal_year.between(2010, 2024)].copy()201    out["firm_year_rows"] = int(len(m)); out["annual_final_rows_2010_2024"] = int(af.fiscal_year.between(2010, 2024).sum())202    out["firm_year_validation"] = {"maxdiff_adopt_wmean_vs_annual_final": float((m.ai_adoption_wmean - m.s_ai_adoption_wmean).abs().max()),203                                   "corr_adopt_wmean_vs_annual_final": float(pearsonr(m.ai_adoption_wmean, m.s_ai_adoption_wmean)[0]),204                                   "maxdiff_infra_wmean_vs_annual_final": float((m.ai_infrastructure_wmean - m.s_ai_infrastructure_wmean).abs().max()),205                                   "maxdiff_adopt_local_vs_stored_anchor": float((m.l_ai_adoption_wmean - m.s_ai_adoption_wmean).abs().max())}206    m.to_csv(f"{RES}/alt_firm_year.csv", index=False, float_format="%.6g")207208def z(g): return (g - g.mean()) / g.std(ddof=0)209with Timer("firm-year comparisons"):210    for col in ["ai_adoption_wmean", "ai_infrastructure_wmean", "l_ai_adoption_wmean", "l_ai_infrastructure_wmean"] + [f"alt_{k}_wmean" for k in ALT] + ["alt_ai_adoption_strip_wmean_10k", "alt_ai_infrastructure_strip_wmean_10k", "s_ai_adoption_wmean_10k", "s_ai_infrastructure_wmean_10k"]:211        m[f"z_{col}"] = m.groupby("fiscal_year")[col].transform(z)212    m["z_compos"] = m[["z_ai_adoption_wmean", "z_ai_infrastructure_wmean"]].mean(axis=1)213    m["z_compos_local"] = m[["z_l_ai_adoption_wmean", "z_l_ai_infrastructure_wmean"]].mean(axis=1)214    m["z_compos_strip"] = m[["z_alt_ai_adoption_strip_wmean", "z_alt_ai_infrastructure_strip_wmean"]].mean(axis=1)215    m["z_compos_core_strip"] = m[["z_alt_ai_adoption_core_wmean", "z_alt_ai_infrastructure_strip_wmean"]].mean(axis=1)216    m["z_compos_strip_10k"] = m[["z_alt_ai_adoption_strip_wmean_10k", "z_alt_ai_infrastructure_strip_wmean_10k"]].mean(axis=1)217    m["z_compos_10k"] = m[["z_s_ai_adoption_wmean_10k", "z_s_ai_infrastructure_wmean_10k"]].mean(axis=1)218    def comp(a, b):219        d = m[[a, b, "fiscal_year", "ticker"]].dropna()220        qa = d.groupby("fiscal_year")[a].transform(lambda s: pd.qcut(s.rank(method="first"), 5, labels=False) + 1)221        qb = d.groupby("fiscal_year")[b].transform(lambda s: pd.qcut(s.rank(method="first"), 5, labels=False) + 1)222        tr = pd.crosstab(qa, qb, normalize="index").round(3)223        return {"n": int(len(d)), "spearman": float(spearmanr(d[a], d[b]).correlation), "pearson": float(pearsonr(d[a], d[b])[0]),224                "share_same_quintile": float((qa == qb).mean()), "share_Q5_stays_Q5": float((qb[qa == 5] == 5).mean()), "share_Q1_stays_Q1": float((qb[qa == 1] == 1).mean()),225                "share_Q5_to_Q45": float((qb[qa == 5] >= 4).mean()), "share_Q5_to_Q12": float((qb[qa == 5] <= 2).mean()),226                "quintile_transition_rows_a_cols_b": {int(i): {int(j): float(tr.loc[i, j]) for j in tr.columns} for i in tr.index},227                "spearman_by_year": {int(y): round(float(spearmanr(g[a], g[b]).correlation), 4) for y, g in d.groupby("fiscal_year")}}228    out["firm_year"] = {229        "adopt_vs_adopt_strip": comp("ai_adoption_wmean", "alt_ai_adoption_strip_wmean"),230        "adopt_vs_adopt_core": comp("ai_adoption_wmean", "alt_ai_adoption_core_wmean"),231        "infra_vs_infra_strip": comp("ai_infrastructure_wmean", "alt_ai_infrastructure_strip_wmean"),232        "cyber_vs_cyber_strip": comp("cybersecurity_wmean", "alt_cybersecurity_strip_wmean"),233        "autom_vs_autom_strip": comp("automation_labor_wmean", "alt_automation_labor_strip_wmean"),234        "compos_vs_compos_strip": comp("z_compos", "z_compos_strip"),235        "compos_vs_compos_core_strip": comp("z_compos", "z_compos_core_strip"),236        "compos_vs_compos_local_reembedded_anchors": comp("z_compos", "z_compos_local"),237        "compos_10k_vs_compos_strip_10k": comp("z_compos_10k", "z_compos_strip_10k"),238        "adopt_vs_cyber_original": comp("ai_adoption_wmean", "cybersecurity_wmean"),239        "adopt_strip_vs_cyber_strip": comp("alt_ai_adoption_strip_wmean", "alt_cybersecurity_strip_wmean"),240        "adopt_vs_autom_original": comp("ai_adoption_wmean", "automation_labor_wmean"),241        "adopt_strip_vs_autom_strip": comp("alt_ai_adoption_strip_wmean", "alt_automation_labor_strip_wmean"),242        "adopt_core_vs_cyber_strip": comp("alt_ai_adoption_core_wmean", "alt_cybersecurity_strip_wmean"),243        "adopt_core_vs_autom_strip": comp("alt_ai_adoption_core_wmean", "alt_automation_labor_strip_wmean"),244        "adopt_vs_infra_original": comp("ai_adoption_wmean", "ai_infrastructure_wmean"),245        "adopt_strip_vs_infra_strip": comp("alt_ai_adoption_strip_wmean", "alt_ai_infrastructure_strip_wmean"),246    }247    by = pd.DataFrame([{"fiscal_year": int(y), "n": int(len(g)),248                        "sp_adopt_strip": spearmanr(g.ai_adoption_wmean, g.alt_ai_adoption_strip_wmean).correlation,249                        "sp_adopt_core": spearmanr(g.ai_adoption_wmean, g.alt_ai_adoption_core_wmean).correlation,250                        "sp_infra_strip": spearmanr(g.ai_infrastructure_wmean, g.alt_ai_infrastructure_strip_wmean).correlation,251                        "sp_compos_strip": spearmanr(g.z_compos, g.z_compos_strip).correlation,252                        "sp_compos_core_strip": spearmanr(g.z_compos, g.z_compos_core_strip).correlation,253                        "sp_adopt_cyber_orig": spearmanr(g.ai_adoption_wmean, g.cybersecurity_wmean).correlation,254                        "sp_adoptstrip_cyberstrip": spearmanr(g.alt_ai_adoption_strip_wmean, g.alt_cybersecurity_strip_wmean).correlation,255                        "sp_adopt_autom_orig": spearmanr(g.ai_adoption_wmean, g.automation_labor_wmean).correlation,256                        "sp_adoptstrip_automstrip": spearmanr(g.alt_ai_adoption_strip_wmean, g.alt_automation_labor_strip_wmean).correlation,257                        "mean_adopt": g.ai_adoption_wmean.mean(), "mean_adopt_strip": g.alt_ai_adoption_strip_wmean.mean(), "mean_adopt_core": g.alt_ai_adoption_core_wmean.mean()}258                       for y, g in m.groupby("fiscal_year")])259    by.to_csv(f"{RES}/alt_corr_by_year.csv", index=False, float_format="%.4f")260    # top-30 firms (>= 5 years) under each composite261    def top(colz):262        t = m.groupby(["ticker", "sector"])[colz].agg(["mean", "count"]).reset_index(); t = t[t["count"] >= 5].nlargest(30, "mean")263        return t264    t0 = top("z_compos"); t1 = top("z_compos_strip"); t2 = top("z_compos_core_strip")265    out["top30_original"] = t0[["ticker", "sector", "mean"]].round(3).to_dict("records")266    out["top30_strip"] = t1[["ticker", "sector", "mean"]].round(3).to_dict("records")267    out["top30_core_strip"] = t2[["ticker", "sector", "mean"]].round(3).to_dict("records")268    out["top30_overlap_strip"] = int(len(set(t0.ticker) & set(t1.ticker))); out["top30_overlap_core_strip"] = int(len(set(t0.ticker) & set(t2.ticker)))269    out["top30_utilities_original"] = int((t0.sector == "Utilities").sum()); out["top30_utilities_strip"] = int((t1.sector == "Utilities").sum()); out["top30_utilities_core_strip"] = int((t2.sector == "Utilities").sum())270    rk = m.groupby(["ticker", "sector"]).agg(z=("z_compos", "mean"), zs=("z_compos_strip", "mean"), zc=("z_compos_core_strip", "mean"), yrs=("fiscal_year", "count")).reset_index()271    rk = rk[rk.yrs >= 5]; rk["r"] = rk.z.rank(ascending=False); rk["rs"] = rk.zs.rank(ascending=False); rk["rc"] = rk.zc.rank(ascending=False)272    out["firm_level_spearman_compos_vs_strip"] = float(spearmanr(rk.z, rk.zs).correlation); out["firm_level_spearman_compos_vs_core_strip"] = float(spearmanr(rk.z, rk.zc).correlation)273    out["ida_he_ranks"] = rk[rk.ticker.isin(["IDA", "HE"])][["ticker", "z", "r", "zs", "rs", "zc", "rc"]].round(3).to_dict("records")274    out["sector_mean_z"] = m.groupby("sector")[["z_compos", "z_compos_strip", "z_compos_core_strip"]].mean().round(3).to_dict("index")275276with Timer("X3 probe sentences"):277    texts = [t for _, t in PROBES]; grp = [g for g, _ in PROBES]278    E = model.encode(texts, normalize_embeddings=True, batch_size=64, show_progress_bar=False)279    P = E @ M.T280    pr = pd.DataFrame(P, columns=cols); pr.insert(0, "group", grp); pr.insert(1, "sentence", texts)281    pr.to_csv(f"{RES}/probe_sentences.csv", index=False, float_format="%.4f")282    keep = ["s_ai_adoption", "s_ai_infrastructure", "s_ai_disruption_risk", "s_cybersecurity", "s_automation_labor", "alt_ai_adoption_strip", "alt_ai_adoption_core", "alt_ai_infrastructure_strip"]283    out["probe_group_means"] = pr.groupby("group")[keep].mean().round(4).to_dict("index")284    aff = pr[pr.group == "affirmative_ai"]; neg = pr[pr.group == "negated_ai"]285    out["probe_negation"] = {k: {"mean_affirmative": float(aff[k].mean()), "mean_negated": float(neg[k].mean()), "diff": float(aff[k].mean() - neg[k].mean()),286                                 "share_pairs_aff_gt_neg": float((aff[k].values[:, None] > neg[k].values[None, :]).mean()),287                                 "min_affirmative": float(aff[k].min()), "max_negated": float(neg[k].max())} for k in ["s_ai_adoption", "alt_ai_adoption_strip", "alt_ai_adoption_core"]}288    # where would the probe sentences fall in the corpus distribution of score_ai_adoption?289    qs = c.execute("SELECT quantile_cont(score_ai_adoption, [0.5, 0.9, 0.95, 0.99, 0.999]) FROM chunk_scores").fetchone()[0]290    out["corpus_adopt_quantiles_50_90_95_99_999"] = [float(x) for x in qs]291    out["probe_pct_rank_in_corpus"] = {g: float(c.execute(f"SELECT avg(CASE WHEN score_ai_adoption < {v} THEN 1.0 ELSE 0 END) FROM chunk_scores").fetchone()[0]) for g, v in pr.groupby("group")["s_ai_adoption"].mean().items()}292293out["runtime_s"] = time.time() - t_all294dump_json(out, f"{RES}/alt_anchor_results.json")295print(json.dumps({k: {kk: vv for kk, vv in v.items() if kk not in ("quintile_transition_rows_a_cols_b", "spearman_by_year")} for k, v in out["firm_year"].items()}, indent=1))296print("probe", json.dumps(out["probe_negation"], indent=1)); print("top30 overlap", out["top30_overlap_strip"], out["top30_overlap_core_strip"], "utilities", out["top30_utilities_original"], out["top30_utilities_strip"], out["top30_utilities_core_strip"])297
Files main
Menu

Theme