spb/uqo-research-db

1"""Generate showcase figures from the shareable datasets.23Produces high-resolution PNGs in shares/figures/ that demonstrate the4analytical power of the EDGAR research database: scale, time dynamics,5sector structure, cross-construct relationships and firm-level rankings.6"""7from pathlib import Path89import numpy as np10import pandas as pd11import matplotlib.pyplot as plt12import matplotlib.ticker as mticker13from matplotlib.colors import LinearSegmentedColormap1415ROOT = Path(__file__).resolve().parent.parent16DS = ROOT / "shares" / "datasets"17OUT = ROOT / "shares" / "figures"18OUT.mkdir(parents=True, exist_ok=True)1920# ---- style ---------------------------------------------------------------21plt.rcParams.update({22    "figure.dpi": 130,23    "savefig.dpi": 160,24    "font.size": 11,25    "axes.titlesize": 14,26    "axes.titleweight": "bold",27    "axes.labelsize": 11,28    "axes.spines.top": False,29    "axes.spines.right": False,30    "axes.grid": True,31    "grid.alpha": 0.25,32    "grid.linewidth": 0.6,33    "figure.facecolor": "white",34    "axes.facecolor": "white",35})36UQO = "#0033A0"          # UQO blue37ACCENT = "#E04E39"        # warm accent38SEQ = LinearSegmentedColormap.from_list("uqo", ["#f7fbff", "#6baed6", UQO, "#001a52"])3940CONSTRUCTS = {41    "ai_adoption": "AI Adoption",42    "ai_disruption_risk": "AI Disruption Risk",43    "ai_infrastructure": "AI Infrastructure",44    "automation_labor": "Automation & Labor",45    "esg_commitment": "ESG Commitment",46    "climate_risk": "Climate Risk",47    "cybersecurity": "Cybersecurity",48    "macro_risk": "Macro Risk",49}50LABELS = list(CONSTRUCTS.values())51KEYS = list(CONSTRUCTS.keys())5253# ---- load (drop the partial 2026 year for time-series) -------------------54annual = pd.read_parquet(DS / "annual_panel_final.parquet")55annual_ts = annual[annual["fiscal_year"] <= 2025].copy()56quarterly = pd.read_parquet(DS / "quarterly_panel_final.parquet")57filings = pd.read_parquet(DS / "filings_index.parquet")58companies = pd.read_parquet(DS / "companies.parquet")5960YEARS = sorted(annual_ts["fiscal_year"].unique())616263def save(fig, name):64    fig.tight_layout()65    fig.savefig(OUT / f"{name}.png", bbox_inches="tight")66    plt.close(fig)67    print(f"  [fig] {name}.png")686970# 1. Headline: the AI disclosure surge ------------------------------------71def fig_ai_surge():72    g = annual_ts.groupby("fiscal_year")["ai_adoption_mean"].mean()73    fig, ax = plt.subplots(figsize=(10, 5.5))74    ax.fill_between(g.index, g.values, color=UQO, alpha=0.12)75    ax.plot(g.index, g.values, color=UQO, lw=2.8, marker="o", ms=6, zorder=3)76    # annotate the generative-AI inflection77    ax.axvspan(2022, 2025, color=ACCENT, alpha=0.06)78    ax.annotate("ChatGPT era\n(generative-AI surge)",79                xy=(2023, g.loc[2023]), xytext=(2017.5, g.max() * 0.97),80                fontsize=10, color=ACCENT, weight="bold",81                arrowprops=dict(arrowstyle="->", color=ACCENT, lw=1.5))82    ax.set_title("Rise of AI in S&P 500 corporate disclosures (2010–2025)")83    ax.set_xlabel("Fiscal year")84    ax.set_ylabel("Mean AI-adoption score")85    ax.set_xticks(YEARS[::2])86    fig.text(0.5, -0.02, "Source: 134,124 SEC EDGAR filings · 500 S&P 500 firms · UQO EDGAR Research DB",87             ha="center", fontsize=8.5, color="gray")88    save(fig, "01_ai_adoption_surge")899091# 2. All 8 constructs over time (small multiples) -------------------------92def fig_all_constructs():93    fig, axes = plt.subplots(2, 4, figsize=(16, 7.5), sharex=True)94    for ax, key, lab in zip(axes.ravel(), KEYS, LABELS):95        g = annual_ts.groupby("fiscal_year")[f"{key}_mean"].mean()96        ax.plot(g.index, g.values, color=UQO, lw=2.2, marker="o", ms=3.5)97        ax.fill_between(g.index, g.values, g.values.min(), color=UQO, alpha=0.10)98        ax.set_title(lab, fontsize=11.5)99        ax.set_xticks(YEARS[::4])100        ax.tick_params(labelsize=9)101    fig.suptitle("Eight research constructs — mean disclosure intensity, 2010–2025",102                 fontsize=15, weight="bold", y=1.0)103    save(fig, "02_eight_constructs_trends")104105106# 3. Sector × year heatmap of AI adoption ---------------------------------107def fig_sector_heatmap():108    d = annual_ts.dropna(subset=["sector"])109    piv = d.pivot_table(index="sector", columns="fiscal_year",110                        values="ai_adoption_mean", aggfunc="mean")111    piv = piv.loc[piv.mean(axis=1).sort_values(ascending=False).index]112    fig, ax = plt.subplots(figsize=(13, 6))113    im = ax.imshow(piv.values, aspect="auto", cmap=SEQ)114    ax.set_xticks(range(len(piv.columns)))115    ax.set_xticklabels(piv.columns, rotation=45, ha="right", fontsize=9)116    ax.set_yticks(range(len(piv.index)))117    ax.set_yticklabels(piv.index, fontsize=10)118    ax.set_title("AI adoption by GICS sector and year")119    cbar = fig.colorbar(im, ax=ax, fraction=0.025, pad=0.02)120    cbar.set_label("Mean AI-adoption score", fontsize=9)121    ax.grid(False)122    save(fig, "03_ai_sector_year_heatmap")123124125# 4. Disruption vs adoption quadrant (latest full year) -------------------126def fig_quadrant():127    yr = 2024128    d = annual[(annual["fiscal_year"] == yr)].dropna(subset=["sector"]).copy()129    x = d["ai_adoption_mean"]130    y = d["ai_disruption_risk_mean"]131    sectors = sorted(d["sector"].unique())132    cmap = plt.get_cmap("tab10")133    colors = {s: cmap(i % 10) for i, s in enumerate(sectors)}134    fig, ax = plt.subplots(figsize=(10, 7.5))135    for s in sectors:136        sub = d[d["sector"] == s]137        ax.scatter(sub["ai_adoption_mean"], sub["ai_disruption_risk_mean"],138                   s=22, alpha=0.7, color=colors[s], label=s, edgecolor="white", lw=0.3)139    ax.axvline(x.median(), color="gray", ls="--", lw=0.8)140    ax.axhline(y.median(), color="gray", ls="--", lw=0.8)141    ax.set_xlabel("AI adoption →")142    ax.set_ylabel("AI disruption risk →")143    ax.set_title(f"Opportunity vs. threat: how firms frame AI ({yr})")144    ax.text(0.98, 0.98, "High risk\nHigh adoption", transform=ax.transAxes,145            ha="right", va="top", fontsize=9, color="gray")146    ax.text(0.02, 0.02, "Low risk\nLow adoption", transform=ax.transAxes,147            ha="left", va="bottom", fontsize=9, color="gray")148    ax.legend(fontsize=8, ncol=2, frameon=False, loc="upper left")149    save(fig, "04_disruption_vs_adoption_quadrant")150151152# 5. Correlation matrix of the 8 constructs -------------------------------153def fig_correlation():154    cols = [f"{k}_mean" for k in KEYS]155    corr = annual[cols].corr()156    fig, ax = plt.subplots(figsize=(8.5, 7.5))157    im = ax.imshow(corr.values, cmap="RdBu_r", vmin=-1, vmax=1)158    ax.set_xticks(range(len(LABELS)))159    ax.set_xticklabels(LABELS, rotation=45, ha="right", fontsize=9)160    ax.set_yticks(range(len(LABELS)))161    ax.set_yticklabels(LABELS, fontsize=9)162    for i in range(len(LABELS)):163        for j in range(len(LABELS)):164            v = corr.values[i, j]165            ax.text(j, i, f"{v:.2f}", ha="center", va="center",166                    fontsize=8, color="white" if abs(v) > 0.55 else "black")167    ax.set_title("Cross-construct correlations (firm-year)")168    fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02).set_label("Pearson r", fontsize=9)169    ax.grid(False)170    save(fig, "05_construct_correlation")171172173# 6. Top-20 firms by AI adoption ------------------------------------------174def fig_top_firms():175    yr = 2024176    d = annual[annual["fiscal_year"] == yr].copy()177    d["label"] = d["ticker"].fillna(d["company_name"].str.slice(0, 14))178    top = d.nlargest(20, "ai_adoption_mean").sort_values("ai_adoption_mean")179    fig, ax = plt.subplots(figsize=(9, 8))180    ax.barh(top["label"], top["ai_adoption_mean"], color=UQO, alpha=0.85)181    ax.set_title(f"Top 20 S&P 500 firms by AI-adoption disclosure ({yr})")182    ax.set_xlabel("Mean AI-adoption score")183    ax.grid(axis="y", alpha=0)184    save(fig, "06_top20_ai_firms")185186187# 7. Crisis fingerprints: macro risk & the 2020 shock ---------------------188def fig_macro_crisis():189    qm = quarterly.copy()190    qm = qm[qm["fiscal_year"] <= 2025]191    qm["t"] = qm["fiscal_year"] + (qm["fiscal_quarter"] - 1) / 4192    g = qm.groupby("t")[["macro_risk_mean", "cybersecurity_mean"]].mean()193    fig, ax = plt.subplots(figsize=(11, 5.5))194    ax.plot(g.index, g["macro_risk_mean"], color=ACCENT, lw=2.4, label="Macro risk")195    ax.plot(g.index, g["cybersecurity_mean"], color=UQO, lw=2.4, label="Cybersecurity")196    ax.axvspan(2020.0, 2020.75, color="gray", alpha=0.15)197    ax.annotate("COVID-19 shock", xy=(2020.3, g["macro_risk_mean"].max()),198                xytext=(2016.5, g["macro_risk_mean"].max()),199                fontsize=10, color="dimgray", weight="bold",200                arrowprops=dict(arrowstyle="->", color="dimgray"))201    ax.set_title("Disclosure responds to events: macro risk & cybersecurity (quarterly)")202    ax.set_xlabel("Fiscal quarter")203    ax.set_ylabel("Mean score")204    ax.legend(frameon=False)205    save(fig, "07_macro_cyber_quarterly")206207208# 8. Climate vs ESG dual trajectory ---------------------------------------209def fig_climate_esg():210    g = annual_ts.groupby("fiscal_year")[["climate_risk_mean", "esg_commitment_mean"]].mean()211    fig, ax = plt.subplots(figsize=(10, 5.5))212    ax.plot(g.index, g["esg_commitment_mean"], color="#2a9d8f", lw=2.6, marker="o", ms=5, label="ESG commitment")213    ax.plot(g.index, g["climate_risk_mean"], color="#e76f51", lw=2.6, marker="s", ms=5, label="Climate risk")214    ax.set_title("The sustainability decade: ESG & climate disclosure (2010–2025)")215    ax.set_xlabel("Fiscal year")216    ax.set_ylabel("Mean score")217    ax.set_xticks(YEARS[::2])218    ax.legend(frameon=False)219    save(fig, "08_climate_vs_esg")220221222# 9. Database scale overview ----------------------------------------------223def fig_scale():224    fig, axes = plt.subplots(1, 3, figsize=(16, 5))225    # a) filings per year by form type (fiscal_year is empty in the index -> use filing_date)226    f = filings.copy()227    f["year"] = pd.to_datetime(f["filing_date"]).dt.year228    f = f[(f["year"] >= 2010) & (f["year"] <= 2025)]229    piv = f.pivot_table(index="year", columns="form_type",230                        values="accession_number", aggfunc="count").fillna(0)231    order = [c for c in ["8-K", "10-Q", "10-K"] if c in piv.columns]232    cols = {"8-K": "#6baed6", "10-Q": UQO, "10-K": "#001a52"}233    bottom = np.zeros(len(piv))234    for c in order:235        axes[0].bar(piv.index, piv[c], bottom=bottom, label=c, color=cols[c])236        bottom += piv[c].values237    axes[0].set_title("Filings indexed per year")238    axes[0].set_xlabel("Year filed")239    axes[0].set_xticks(range(2010, 2026, 3))240    axes[0].legend(frameon=False, fontsize=9)241    axes[0].yaxis.set_major_formatter(mticker.FuncFormatter(lambda x, _: f"{int(x/1000)}k"))242243    # b) firms per sector244    sc = companies["sector"].value_counts().sort_values()245    axes[1].barh(sc.index, sc.values, color=UQO, alpha=0.85)246    axes[1].set_title("S&P 500 firms by GICS sector")247    axes[1].grid(axis="y", alpha=0)248    axes[1].tick_params(labelsize=9)249250    # c) the headline numbers251    axes[2].axis("off")252    stats = [("500", "companies"), ("134,124", "filings"),253             ("107,636", "scored filings"), ("7,869", "firm-years"),254             ("30,224", "firm-quarters"), ("8", "research constructs"),255             ("2010–2026", "coverage")]256    axes[2].set_title("Database at a glance", pad=14)257    for i, (num, lab) in enumerate(stats):258        y = 0.92 - i * 0.13259        axes[2].text(0.05, y, num, fontsize=20, weight="bold", color=UQO, transform=axes[2].transAxes)260        axes[2].text(0.55, y + 0.01, lab, fontsize=12, color="dimgray", transform=axes[2].transAxes)261    fig.suptitle("Scale of the UQO EDGAR Research Database", fontsize=15, weight="bold", y=1.02)262    save(fig, "09_database_scale")263264265# 10. Sector divergence in AI (ridall small multiples already; do dispersion)266def fig_dispersion():267    g = annual_ts.groupby("fiscal_year")["ai_adoption_mean"]268    mean = g.mean()269    p10 = g.quantile(0.10)270    p90 = g.quantile(0.90)271    fig, ax = plt.subplots(figsize=(10, 5.5))272    ax.fill_between(mean.index, p10.values, p90.values, color=UQO, alpha=0.15, label="10th–90th percentile")273    ax.plot(mean.index, mean.values, color=UQO, lw=2.6, marker="o", ms=5, label="Mean")274    ax.set_title("Widening gap: AI-adoption dispersion across firms")275    ax.set_xlabel("Fiscal year")276    ax.set_ylabel("AI-adoption score")277    ax.set_xticks(YEARS[::2])278    ax.legend(frameon=False)279    save(fig, "10_ai_dispersion")280281282if __name__ == "__main__":283    print("Generating showcase figures ->", OUT)284    fig_ai_surge()285    fig_all_constructs()286    fig_sector_heatmap()287    fig_quadrant()288    fig_correlation()289    fig_top_firms()290    fig_macro_crisis()291    fig_climate_esg()292    fig_scale()293    fig_dispersion()294    print("\nDone.")295