spb/uqo-research-db

1"""Semantic structure of the project corpus via sentence embeddings.23KMeans clustering of the 384-d MiniLM project embeddings into latent themes,4silhouette diagnostics, a t-SNE map, and cluster characterization by dominant5project type / technologies / representative names.6"""7from __future__ import annotations89import warnings10from collections import Counter1112import numpy as np13import pandas as pd14from sklearn.preprocessing import normalize15from sklearn.cluster import KMeans16from sklearn.metrics import silhouette_score17from sklearn.manifold import TSNE1819from common import DATA, set_style, savefig, write_table, save_results, esc20import matplotlib.pyplot as plt2122warnings.filterwarnings("ignore")23set_style()24rng = np.random.default_rng(42)2526emb = pd.read_parquet(DATA / "embeddings.parquet")27proj = pd.read_parquet(DATA / "projects.parquet")28m = emb.merge(proj[["project_id", "project_type", "sector", "project_name",29                    "technologies", "total_amount_usd", "is_ai"]],30              on="project_id", how="inner")31X = normalize(np.vstack(m["vector"].apply(np.asarray).values).astype(np.float32))32print("embedding matrix:", X.shape)3334# ---- choose K via silhouette (on a sample) --------------------------------35samp = rng.choice(len(X), size=min(4000, len(X)), replace=False)36sils = {}37for k in (6, 8, 10, 12, 14):38    km = KMeans(n_clusters=k, n_init=4, random_state=0).fit(X[samp])39    sils[k] = silhouette_score(X[samp], km.labels_)40K = max(sils, key=sils.get)41km = KMeans(n_clusters=K, n_init=8, random_state=0).fit(X)42m["cluster"] = km.labels_4344# ---- characterize clusters ------------------------------------------------45def top_techs(sub, n=3):46    c = Counter()47    for t in sub["technologies"]:48        for x in ([] if t is None else list(t)):49            if str(x).strip():50                c[str(x).strip().title()] += 151    return ", ".join(t for t, _ in c.most_common(n))5253rows = []54for cl in range(K):55    sub = m[m.cluster == cl]56    dom_type = sub["project_type"].mode().iloc[0]57    dom_sec = sub["sector"].mode().iloc[0] if sub["sector"].notna().any() else "-"58    rows.append({59        "cluster": cl + 1, "n": len(sub), "dom_type": dom_type, "dom_sector": dom_sec,60        "ai_share": sub["is_ai"].mean(), "techs": top_techs(sub),61        "capex": np.nansum(sub["total_amount_usd"].values) / 1e9,62    })63cl_df = pd.DataFrame(rows).sort_values("n", ascending=False)6465TYPE_LABEL = {66    "ma_integration": "M&A integration", "capex_program": "Capex program",67    "plant_construction": "Plant construction", "infrastructure": "Infrastructure",68    "data_center": "Data center", "energy_transition": "Energy transition",69    "product_launch": "Product launch", "rd_program": "R&D program",70    "ai_initiative": "AI initiative", "sustainability": "Sustainability",71    "cost_reduction": "Cost reduction", "geographic_expansion": "Geo. expansion",72    "manufacturing_expansion": "Mfg. expansion", "partnership": "Partnership",73    "supply_chain": "Supply chain", "automation": "Automation",74    "cloud_migration": "Cloud migration", "digital_transformation": "Digital transf.",75    "erp_implementation": "ERP", "technology_deployment": "Tech. deployment",76    "drug_pipeline": "Drug pipeline",77}78lt = [r"\begin{tabular}{rlllr}", r"\toprule",79      r"Cl. & Projects & Dominant type & Top technologies & Capital (\$bn) \\", r"\midrule"]80for _, r in cl_df.iterrows():81    lt.append(f"{int(r.cluster)} & {int(r.n):,} & {esc(TYPE_LABEL.get(r.dom_type, r.dom_type))} & "82              f"{esc(r.techs[:42])} & {r.capex:,.1f} \\\\")83lt += [r"\bottomrule", r"\end{tabular}"]84write_table("tab_clusters", "\n".join(lt))8586# ---- t-SNE map (stratified sample for speed) ------------------------------87sviz = rng.choice(len(X), size=min(5000, len(X)), replace=False)88emb2 = TSNE(n_components=2, perplexity=35, init="pca", random_state=0,89            max_iter=750).fit_transform(X[sviz])90fig, ax = plt.subplots(figsize=(8.4, 6.8))91palette = ["#1b3a5b", "#c44e52", "#2e8b57", "#dd8452", "#8172b3", "#937860",92           "#da8bc3", "#64b5cd", "#ccb974", "#55a868", "#b07aa1", "#e15759",93           "#76b7b2", "#59a14f"]94labs = km.labels_[sviz]95for cl in range(K):96    pts = emb2[labs == cl]97    ax.scatter(pts[:, 0], pts[:, 1], s=7, alpha=0.55, color=palette[cl % len(palette)],98               label=f"C{cl+1}: {TYPE_LABEL.get(cl_df.set_index('cluster').loc[cl+1,'dom_type'] if (cl+1) in cl_df['cluster'].values else '', '')[:14]}")99ax.set_title(f"t-SNE map of project embeddings ({K} KMeans themes)")100ax.set_xlabel("t-SNE 1"); ax.set_ylabel("t-SNE 2"); ax.grid(False)101ax.legend(fontsize=6, markerscale=2, ncol=2, loc="upper right")102savefig(fig, "fig_embedding_tsne.pdf")103104# silhouette figure105fig, ax = plt.subplots(figsize=(5.6, 3.4))106ax.plot(list(sils.keys()), list(sils.values()), "o-", color="#1b3a5b")107ax.axvline(K, color="#c44e52", ls="--", lw=1)108ax.set_xlabel("Number of clusters $K$"); ax.set_ylabel("Silhouette score")109ax.set_title("Cluster-count selection")110savefig(fig, "fig_silhouette.pdf")111112save_results("embeddings", {113    "n": int(len(X)), "dim": int(X.shape[1]), "K": int(K),114    "silhouette": {int(k): float(v) for k, v in sils.items()},115    "clusters": cl_df.to_dict("records"),116})117print(f"embeddings done | K={K}, best silhouette={sils[K]:.3f}")118