1"""n1_common.py — shared settings for the FIRC-0002 Stage D node batch N1 (M4M64a).23Data: read-only rsync copy of M3U96b:~/UQO_RESEARCH_DB/data/parquet/ in ~/firc0002_review/data/4(chmod a-w after the copy). Every query runs on M4M64a; nothing is ever written to M3U96b.5DuckDB 2.0.0, threads <= 8, memory_limit 24 GB, seed 20261105.6"""7from __future__ import annotations8import os, json, time, re9import duckdb1011BASE = os.path.expanduser("~/firc0002_review")12DATA = f"{BASE}/data"13RES = f"{BASE}/results"14LOGS = f"{BASE}/logs"15TMP = f"{BASE}/tmp"16SEED = 2026110517THREADS = int(os.environ.get("N1_THREADS", "8"))18MEM = os.environ.get("N1_MEM", "24GB")19MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2"2021# Acceptance targets quoted in the manuscript / plan22TARGET_CHUNKS = 10_720_89023TARGET_SECTIONS = 1_342_28324TARGET_FILINGS = 527_5482526# Section weights of the pipeline (config/factors.yaml, UQO_RESEARCH_DB) — reproduced verbatim27SECTION_WEIGHTS = {28 "risk_measures": {"risk_factors": 2.0, "md_and_a": 1.5, "legal_proceedings": 1.3, "full_body": 1.0,29 "controls_and_procedures": 0.8, "quantitative_disclosures": 0.8,30 "financial_statements": 0.5, "unregistered_sales": 0.5},31 "operational_measures": {"md_and_a": 2.0, "full_body": 1.0, "risk_factors": 1.2,32 "controls_and_procedures": 0.8, "quantitative_disclosures": 0.8,33 "financial_statements": 0.5, "legal_proceedings": 0.5, "unregistered_sales": 0.5},34}35ANCHOR_PROFILE = {"ai_adoption": "operational_measures", "ai_disruption_risk": "risk_measures",36 "ai_infrastructure": "operational_measures", "esg_commitment": "operational_measures",37 "climate_risk": "risk_measures", "cybersecurity": "risk_measures",38 "automation_labor": "operational_measures", "macro_risk": "risk_measures"}39ORIG_ANCHORS = ["ai_adoption", "ai_disruption_risk", "ai_infrastructure", "esg_commitment",40 "climate_risk", "cybersecurity", "automation_labor", "macro_risk"]414243def weight_case(profile: str, col: str = "section_name") -> str:44 w = SECTION_WEIGHTS[profile]45 cases = " ".join(f"WHEN '{k}' THEN {float(v)}" for k, v in w.items())46 return f"CASE {col} {cases} ELSE 1.0 END"474849# ---------------------------------------------------------------------------50# AI keyword lexicon for the regex baseline (reviewer-assembled; see README in results)51# CORE = unambiguous AI terms shared by the published 10-K / job-posting AI dictionaries52# (Babina, Fedyk, He & Hodson 2024 JFE core terms; Eisfeldt, Schubert & Zhang 2023 generative-AI terms).53# EXT = CORE + the broader "digital / analytics / automation" terms that the original anchor54# paragraphs also contain (the overlap the referees worry about).55# Patterns are applied to lower-cased text (RE2 syntax, DuckDB regexp_extract_all); the bare56# acronym is matched case-sensitively on the raw text (\bAI\b, \bA\.I\.) to avoid French/other noise.57CORE_TERMS = [58 r"artificial intelligence", r"machine[ -]learning", r"deep[ -]learning", r"neural network",59 r"natural[ -]language processing", r"large[ -]language model", r"generative ai\b", r"\bgen ?ai\b",60 r"chatgpt", r"\bopenai\b", r"computer vision", r"reinforcement learning", r"\bllms?\b",61 r"\bai[ -](?:driven|powered|enabled|based|related|assisted)",62]63EXT_EXTRA_TERMS = [64 r"\bautomation\b", r"robotic process automation", r"predictive analytics", r"\bbig data\b",65 r"data analytics", r"\balgorithms?\b", r"\bchatbots?\b", r"\bdata science\b", r"\bautonomous\b",66 r"intelligent automation", r"cloud computing", r"\bdata[ -]driven\b",67]68CORE_RE = "(" + "|".join(CORE_TERMS) + ")"69EXT_RE = "(" + "|".join(CORE_TERMS + EXT_EXTRA_TERMS) + ")"70ACRONYM_RE = r"\bA\.?I\.?\b" # case-sensitive on raw text: AI, A.I.717273def connect(threads: int = THREADS, mem: str = MEM) -> duckdb.DuckDBPyConnection:74 os.makedirs(TMP, exist_ok=True)75 c = duckdb.connect()76 c.execute(f"SET threads={threads}")77 c.execute(f"SET memory_limit='{mem}'")78 c.execute(f"SET temp_directory='{TMP}'")79 c.execute("SET preserve_insertion_order=false")80 return c818283def views(c: duckdb.DuckDBPyConnection) -> None:84 c.execute(f"CREATE OR REPLACE VIEW filings AS SELECT * FROM read_parquet('{DATA}/filings/*.parquet')")85 c.execute(f"CREATE OR REPLACE VIEW companies AS SELECT * FROM read_parquet('{DATA}/companies/*.parquet')")86 c.execute(f"CREATE OR REPLACE VIEW chunks AS SELECT * FROM read_parquet('{DATA}/filing_chunks/*.parquet')")87 c.execute(f"CREATE OR REPLACE VIEW sections AS SELECT * FROM read_parquet('{DATA}/filing_sections/*.parquet')")88 c.execute(f"CREATE OR REPLACE VIEW chunk_scores AS SELECT * FROM read_parquet('{DATA}/chunk_scores/*/*.parquet', hive_partitioning=true)")89 c.execute(f"CREATE OR REPLACE VIEW filing_scores AS SELECT * FROM read_parquet('{DATA}/filing_scores/*/*.parquet', hive_partitioning=true)")90 c.execute(f"CREATE OR REPLACE VIEW annual_final AS SELECT * FROM read_parquet('{DATA}/annual_final/*.parquet')")91 c.execute(f"CREATE OR REPLACE VIEW anchor_embeddings AS SELECT * FROM read_parquet('{DATA}/anchor_embeddings/*.parquet')")929394def dump_json(obj, path: str) -> None:95 os.makedirs(os.path.dirname(path), exist_ok=True)96 with open(path, "w") as f:97 json.dump(obj, f, indent=1, default=_default)9899100def _default(o):101 import numpy as np102 if isinstance(o, (np.integer,)):103 return int(o)104 if isinstance(o, (np.floating,)):105 return float(o)106 if isinstance(o, (np.ndarray,)):107 return o.tolist()108 return str(o)109110111class Timer:112 def __init__(self, label: str):113 self.label = label114 def __enter__(self):115 self.t0 = time.time(); print(f"[{time.strftime('%H:%M:%S')}] {self.label} ...", flush=True); return self116 def __exit__(self, *a):117 print(f"[{time.strftime('%H:%M:%S')}] {self.label} done in {time.time()-self.t0:,.1f}s", flush=True)118