1#!/usr/bin/env python32"""WP44 — semantic k-NN edges between works (Phase 3, semantic layer) with FAISS HNSW on the SPECTER2 embeddings.3Edges: (work_id, neighbour_id, cosine_similarity, rank) for k = --k neighbours, similarity >= --min-sim; provenance = model tag +4index parameters; verification_status = 'machine-verified' (deterministic from embeddings). Also computes, per work, the share5of its k neighbours that belong to a different primary category (semantic boundary-crossing, used in RQ5) and writes6results/graph/semantic_crossing_yearly.csv. Output: data/processed/graph/semantic_edges_<tag>.parquet."""7import argparse, glob, json, os, time, numpy as np, pandas as pd8910def main():11 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--model", default="allenai/specter2_base"); ap.add_argument("--k", type=int, default=10)12 ap.add_argument("--min-sim", type=float, default=0.80); ap.add_argument("--sample", type=int, default=0, help="query only a random sample of works (0 = all)")13 a = ap.parse_args(); R = a.root; tag = a.model.replace("/", "__"); out = f"{R}/data/processed/graph"; os.makedirs(out, exist_ok=True); os.makedirs(f"{R}/results/graph", exist_ok=True)14 import faiss, duckdb15 E, I = [], []16 for f in sorted(glob.glob(f"{R}/data/processed/embeddings/{tag}/shard_*.npy")):17 if f.endswith(".ids.npy"): continue18 E.append(np.load(f)); I.append(np.load(f.replace(".npy", ".ids.npy")))19 X = np.concatenate(E).astype(np.float32); ids = np.concatenate(I); faiss.normalize_L2(X); n, d = X.shape; print("vectors", n, d, flush=True)20 t0 = time.time(); index = faiss.IndexHNSWFlat(d, 32, faiss.METRIC_INNER_PRODUCT); index.hnsw.efConstruction = 80; index.add(X); print(f"index built in {time.time()-t0:.0f}s", flush=True)21 index.hnsw.efSearch = 6422 q_idx = np.arange(n) if not a.sample else np.random.default_rng(44).choice(n, size=min(a.sample, n), replace=False)23 rows = []; B = 5000024 for s in range(0, len(q_idx), B):25 qi = q_idx[s:s + B]; D, N = index.search(X[qi], a.k + 1)26 for r, (qq, dd, nn) in enumerate(zip(qi, D, N)):27 rk = 028 for sim, j in zip(dd, nn):29 if j == qq or j < 0: continue30 rk += 131 if sim >= a.min_sim:32 rows.append((int(ids[qq]), int(ids[j]), float(sim), rk))33 print(f"{s+len(qi)}/{len(q_idx)} queried, {len(rows)} edges, {time.time()-t0:.0f}s", flush=True)34 df = pd.DataFrame(rows, columns=["work_id", "neighbour_id", "cosine", "rank"]); df.to_parquet(f"{out}/semantic_edges_{tag}.parquet")35 json.dump({"model": a.model, "k": a.k, "min_sim": a.min_sim, "n_vectors": int(n), "n_edges": int(len(df)), "index": "HNSW32 efC80 efS64 inner-product", "seconds": round(time.time() - t0)}, open(f"{out}/semantic_edges_{tag}.json", "w"), indent=1)36 con = duckdb.connect(f"{R}/data/processed/wp44.duckdb", read_only=True)37 con.execute(f"CREATE OR REPLACE VIEW work_primary AS SELECT * FROM read_parquet('{R}/data/processed/classification/primary/*.parquet')")38 con.execute(f"CREATE OR REPLACE VIEW sem AS SELECT * FROM read_parquet('{out}/semantic_edges_{tag}.parquet')")39 con.execute(f"""COPY (SELECT c.publication_year AS year, pa.primary_category, count(*) n_edges, round(avg((pa.primary_category <> pb.primary_category)::int),4) share_cross_category40 FROM sem s JOIN work_primary pa ON pa.work_id = s.work_id JOIN work_primary pb ON pb.work_id = s.neighbour_id JOIN corpus_works c ON c.id = s.work_id41 GROUP BY 1,2 ORDER BY 1,2) TO '{R}/results/graph/semantic_crossing_yearly.csv' (HEADER)""")42 print("done", len(df), "edges")434445if __name__ == "__main__":46 main()47