spb/airiskindex Public

1//  File:    rate.ts2//  Path:    apps/worker/src/scripts/rate.ts3//  Project: AI Risk Index — airiskindex.io4//  Author:  Simon-Pierre Boucher5//  Contact: contact@spboucher.ai6//  Copyright © 2026 Simon-Pierre Boucher. All rights reserved.7//8//  Description: One-shot parallel rating run with per-model resume via state file.910import { prisma } from "@airiskindex/db";11import { RATER_MODELS, RATER_PROMPT_VERSION } from "../config";12import {13  ingestBatchResults,14  isBatchComplete,15  submitRatingBatch,16  type RatingTask,17} from "../raters/batch";18import { ratingJobId } from "../raters/job-id";1920// One-shot rating run: submit a Message Batch per model for every task that21// lacks ratings under the current prompt version, poll to completion, ingest.22// Models run in PARALLEL (batches are independent server-side queues).23// Usage: tsx src/scripts/rate.ts [--limit N] [--poll-seconds S]24// Idempotent & resumable: already-rated tasks are excluded, and an in-flight25// batch per model is remembered in a state file so a relaunch resumes polling26// instead of resubmitting (a resubmit would double the spend).2728import { readFileSync, writeFileSync } from "node:fs";2930function argValue(flag: string): string | undefined {31  const index = process.argv.indexOf(flag);32  return index >= 0 ? process.argv[index + 1] : undefined;33}3435const limit = Number(argValue("--limit") ?? "0") || undefined;36const pollSeconds = Number(argValue("--poll-seconds") ?? "60") || 60;37const STATE_PATH = process.env.RATE_STATE ?? "/tmp/aix-batch-state.json";3839function readState(): Record<string, string> {40  try {41    return JSON.parse(readFileSync(STATE_PATH, "utf8")) as Record<string, string>;42  } catch {43    return {};44  }45}4647function writeState(state: Record<string, string>): void {48  writeFileSync(STATE_PATH, JSON.stringify(state, null, 2));49}5051const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));5253async function runForModel(model: string): Promise<void> {54  const tasks = await prisma.task.findMany({55    where: {56      ratings: { none: { model, promptVersion: RATER_PROMPT_VERSION } },57      // exclude demo-seeded synthetic tasks (real O*NET task IDs are numeric)58      NOT: { id: { contains: "-T" } },59    },60    include: { occupation: { select: { title: true } } },61    orderBy: { id: "asc" },62    ...(limit ? { take: limit } : {}),63  });64  if (tasks.length === 0) {65    console.log(`[${model}] nothing to rate`);66    return;67  }6869  const ratingTasks: RatingTask[] = tasks.map((task) => ({70    taskId: task.id,71    occupationTitle: task.occupation.title,72    statement: task.statement,73  }));74  const jobIdToTaskId = new Map(75    ratingTasks.map((task) => [ratingJobId(task.taskId, model, RATER_PROMPT_VERSION), task.taskId]),76  );7778  let batchId = readState()[model];79  if (batchId) {80    console.log(`[${model}] resuming in-flight batch ${batchId}; polling every ${pollSeconds}s`);81  } else {82    console.log(`[${model}] submitting batch of ${ratingTasks.length} tasks…`);83    batchId = await submitRatingBatch(ratingTasks, model);84    writeState({ ...readState(), [model]: batchId });85    console.log(`[${model}] batch ${batchId} submitted; polling every ${pollSeconds}s`);86  }8788  for (;;) {89    await sleep(pollSeconds * 1000);90    if (await isBatchComplete(batchId)) break;91    console.log(`[${model}] batch ${batchId} still processing…`);92  }9394  const { ingested, failed } = await ingestBatchResults(batchId, model, jobIdToTaskId);95  const state = readState();96  delete state[model];97  writeState(state);98  console.log(`[${model}] batch ${batchId} done: ${ingested} dimension ratings ingested, ${failed} requests failed`);99}100101async function main(): Promise<void> {102  if (RATER_MODELS.length < 2) {103    throw new Error("RATER_MODELS must list ≥2 models (multi-model rating, CLAUDE.md §6)");104  }105  console.log(106    `rating run — prompt ${RATER_PROMPT_VERSION}, models: ${RATER_MODELS.join(", ")}${limit ? `, limit ${limit}` : " (all unrated tasks)"}`,107  );108  await Promise.all(RATER_MODELS.map((model) => runForModel(model)));109  console.log("rating run complete — next: pnpm score:recompute");110}111112main()113  .catch((error) => {114    console.error(error);115    process.exitCode = 1;116  })117  .finally(() => prisma.$disconnect());118