WP44: landmark selection v2 (read-only DuckDB, dedup, venue and abstract filters)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 changed file +6 −7
modified
literature_synthesis/select_landmarks.py
+6 −7
@@ -14,12 +14,11 @@ def main(): | ||
| 14 | 14 | ap.add_argument("--n-high", type=int, default=10000); ap.add_argument("--n-land", type=int, default=1000); a = ap.parse_args(); R = a.root |
| 15 | 15 | out = f"{R}/literature_synthesis/landmarks"; os.makedirs(out, exist_ok=True) |
| 16 | 16 | ont = yaml.safe_load(open(a.ontology)); area = {c["id"]: c["area"] for c in ont["categories"]}; cats = list(area) |
| 17 | − con = duckdb.connect(f"{R}/data/processed/wp44.duckdb"); con.execute("SET threads TO 16;") | |
| 18 | − con.execute(f"CREATE OR REPLACE VIEW work_primary AS SELECT * FROM read_parquet('{R}/data/processed/classification/primary/*.parquet')") | |
| 19 | − con.execute(f"CREATE OR REPLACE VIEW wcw AS SELECT * FROM read_parquet('{R}/data/processed/corpus/work_citation_windows.parquet')") | |
| 20 | − df = con.execute("""SELECT c.id AS work_id, c.doi, c.arxiv_id_linked AS arxiv_id, c.title, c.source_name, c.publication_year AS year, c.type, p.primary_category, p.n_categories, | |
| 17 | + con = duckdb.connect(f"{R}/data/processed/wp44.duckdb", read_only=True); con.execute("SET threads TO 16;") | |
| 18 | + WP = f"read_parquet('{R}/data/processed/classification/primary/*.parquet')"; WCW = f"read_parquet('{R}/data/processed/corpus/work_citation_windows.parquet')" | |
| 19 | + df = con.execute(f"""SELECT c.id AS work_id, c.doi, c.arxiv_id_linked AS arxiv_id, c.title, c.source_name, c.publication_year AS year, c.type, p.primary_category, p.n_categories, | |
| 21 | 20 | w.c12, w.c36, w.ncs36, c.cited_by_count, c.fwci, c.has_abstract, c.is_oa, c.period |
| 22 | − FROM corpus_works c JOIN work_primary p ON p.work_id = c.id JOIN wcw w ON w.work_id = c.id WHERE c.eligible AND NOT c.e6_retracted""").df() | |
| 21 | + FROM corpus_works c JOIN {WP} p ON p.work_id = c.id JOIN {WCW} w ON w.work_id = c.id WHERE c.eligible AND NOT c.e6_retracted""").df() | |
| 23 | 22 | # cleaning (2026-11-03, after Agent H's audit of the first list): drop venue-like records (title equal to the source name |
| 24 | 23 | # or shorter than four words), records with neither abstract nor reference list, and duplicate titles (keep the earliest |
| 25 | 24 | # year, then the most cited) — OpenAlex carries SSRN/reprint duplicates of landmark papers |
@@ -37,11 +36,11 @@ def main(): | ||
| 37 | 36 | land_parts = [df[df.primary_category == c].nlargest(max(15, a.n_land // len(cats)), ["score", "cited_by_count"]) for c in cats] |
| 38 | 37 | land = pd.concat(land_parts + [df.nlargest(200, ["c36", "cited_by_count"])]).drop_duplicates("work_id"); land["set"] = "landmark"; land["pi_review"] = True |
| 39 | 38 | high.to_parquet(f"{out}/high_priority.parquet"); land.to_csv(f"{out}/landmarks.csv", index=False) |
| 40 | − con.execute(f"CREATE OR REPLACE TEMP TABLE L AS SELECT * FROM read_csv_auto('{out}/landmarks.csv')") | |
| 39 | + L = f"read_csv_auto('{out}/landmarks.csv')" | |
| 41 | 40 | for ar in sorted(set(area.values())): |
| 42 | 41 | ids = [c for c in cats if area[c] == ar]; sub = land[land.primary_category.isin(ids)] |
| 43 | 42 | sub.to_csv(f"{out}/{ar}_landmarks.csv", index=False) |
| 44 | − ab = con.execute(f"SELECT l.work_id, l.title, a.abstract FROM L l LEFT JOIN oa_abstracts a ON a.work_id = l.work_id WHERE l.primary_category IN ({','.join(repr(i) for i in ids)})").fetchall() | |
| 43 | + ab = con.execute(f"SELECT l.work_id, l.title, a.abstract FROM {L} l LEFT JOIN oa_abstracts a ON a.work_id = l.work_id WHERE l.primary_category IN ({','.join(repr(i) for i in ids)})").fetchall() | |
| 45 | 44 | with open(f"{out}/{ar}_abstracts.jsonl", "w") as fh: |
| 46 | 45 | for w, t, abst in ab: |
| 47 | 46 | fh.write(json.dumps({"work_id": int(w), "title": t, "abstract": abst}) + "\n") |
| 48 | 47 | |