spb/datacenterindex

1/**2 * News relevance audit for third-party publishers (sources.kind = news): which stored items would the tightened3 * `news_article_v1` gate still index today? An item is on-topic when its title + summary carry strong data-center4 * vocabulary in the lead (`relevanceOf` = strong and `leadRelevanceOf`), off-topic otherwise (industry chatter,5 * newsletters, job boards, chips, opinion). Also checks the ≤ 400-character summary rule.6 *7 *   set -a; source .env; set +a8 *   node node_modules/tsx/dist/cli.mjs scripts/quality-news.ts               # report (share by connector + sample)9 *   node node_modules/tsx/dist/cli.mjs scripts/quality-news.ts --apply       # delete off-topic kind=news items (+ their events), trim long summaries10 *   node node_modules/tsx/dist/cli.mjs scripts/quality-news.ts --sample 60   # random sample for hand classification11 */12import { closeDb, getDb, sql } from "@dci/db";13import { leadRelevanceOf, relevanceOf, isNonProjectTitle } from "../apps/worker/src/connectors/news/extract-project.js";14import { shortSummary, THIRD_PARTY_SUMMARY_MAX } from "../apps/worker/src/ingest/news.js";1516const args = process.argv.slice(2);17const APPLY = args.includes("--apply");18const SAMPLE = args.includes("--sample") ? Number(args[args.indexOf("--sample") + 1] ?? 60) : 0;1920async function main(): Promise<void> {21  const db = getDb();22  const rows = await db.execute(sql`select n.id, n.connector_id, n.title, n.summary, n.event_type, n.page_type, s.kind from news_items n join sources s on s.id = n.source_id order by n.created_at`);23  const items = rows.map((r) => ({ id: String(r.id), connector: String(r.connector_id), title: String(r.title), summary: r.summary == null ? null : String(r.summary), eventType: r.event_type == null ? null : String(r.event_type), kind: String(r.kind) }));24  const third = items.filter((i) => i.kind === "news");25  if (SAMPLE) {26    for (const i of [...third].sort(() => Math.random() - 0.5).slice(0, SAMPLE)) console.log(`${i.connector} | ${i.eventType ?? "-"} | ${i.title.slice(0, 110)} | ${(i.summary ?? "").slice(0, 120)}`);27    return;28  }29  const judge = (i: { title: string; summary: string | null }) => {30    const text = i.summary ?? "";31    const rel = relevanceOf(i.title, text);32    if (rel !== "strong") return `off-topic:${rel}`;33    if (!leadRelevanceOf(i.title, text)) return "off-topic:body-only";34    return isNonProjectTitle(i.title) ? "on-topic:non-project" : "on-topic";35  };36  const perConnector = new Map<string, { total: number; off: number }>();37  const off: typeof third = [];38  const longSummaries = third.filter((i) => (i.summary?.length ?? 0) > THIRD_PARTY_SUMMARY_MAX);39  for (const i of third) {40    const v = judge(i);41    const s = perConnector.get(i.connector) ?? { total: 0, off: 0 };42    s.total++;43    if (v.startsWith("off-topic")) { s.off++; off.push(i); }44    perConnector.set(i.connector, s);45  }46  console.log(`third-party (kind=news) items: ${third.length} — off-topic by the current gate: ${off.length} (${((off.length / Math.max(1, third.length)) * 100).toFixed(0)} %) — summaries > ${THIRD_PARTY_SUMMARY_MAX} chars: ${longSummaries.length}`);47  for (const [c, s] of [...perConnector].sort((a, b) => b[1].off - a[1].off)) console.log(`  ${c.padEnd(24)} ${s.off}/${s.total}`);48  console.log("\noff-topic sample:");49  for (const i of off.slice(0, 40)) console.log(`  ${i.connector.padEnd(22)} ${i.title.slice(0, 100)}`);50  // operator newsrooms are kept whatever the topic (the source's own news), but report how many carry a non-project headline with a wrong event type51  const opWrong = items.filter((i) => i.kind !== "news" && i.eventType && !["news", "page_changed", "acquisition", "closure"].includes(i.eventType) && isNonProjectTitle(i.title));52  console.log(`\noperator / government items with a people/deal/money headline but a specific event type: ${opWrong.length} (event type is reset to 'news' on reprocess)`);5354  if (!APPLY) { console.log("\n(dry run — pass --apply to delete off-topic third-party items and trim long summaries)"); return; }55  let deleted = 0, trimmed = 0;56  for (const i of off) {57    await db.transaction(async (tx) => {58      await tx.execute(sql`delete from events where entity_type = 'news_event' and entity_id = ${i.id}`);59      await tx.execute(sql`delete from news_items where id = ${i.id}`);60    });61    deleted++;62  }63  for (const i of longSummaries) {64    if (off.includes(i)) continue;65    await db.execute(sql`update news_items set summary = ${shortSummary(i.summary)} where id = ${i.id}`);66    trimmed++;67  }68  console.log(`deleted ${deleted} off-topic items, trimmed ${trimmed} summaries`);69}7071main().catch((e) => { console.error(e); process.exitCode = 1; }).finally(() => closeDb());72