Repository Public
1// V7 research runtime — the protocol shared by the API (services/api/src/research), the job runner2// (services/research-runner) and the web app: what a research job is, its budgets, what the runner receives, and the3// submission archive. Pure (zod only): no file system, no network.4//5// Trust: everything that comes back from a job is UNTRUSTED (it was written by an AI coding agent). The schemas below bound6// sizes and shapes; they say nothing about the content being true. The API re-computes every number it shows as measured.7import { z } from 'zod';89export const RESEARCH_STATES = ['queued', 'preparing', 'running', 'evaluating', 'submitted', 'failed', 'cancelled', 'over_budget'] as const;10export type ResearchState = (typeof RESEARCH_STATES)[number];11/** states in which a job is finished: nothing moves it again */12export const RESEARCH_FINAL: readonly ResearchState[] = ['submitted', 'failed', 'cancelled', 'over_budget'];13/** the only transitions that exist (the API enforces them; the runner can only ask for the ones marked runner) */14export const RESEARCH_TRANSITIONS: Record<ResearchState, readonly ResearchState[]> = {15 queued: ['preparing', 'cancelled', 'failed'],16 preparing: ['running', 'failed', 'cancelled', 'over_budget'],17 running: ['evaluating', 'failed', 'cancelled', 'over_budget'],18 evaluating: ['submitted', 'failed'],19 submitted: [], failed: [], cancelled: [], over_budget: [],20};21export const canMove = (from: ResearchState, to: ResearchState): boolean => RESEARCH_TRANSITIONS[from].includes(to);2223/** Tools of the coding agent an owner may allow. There is deliberately NO web tool in this list: a job has no web access. */24export const RESEARCH_TOOLS = ['Read', 'Write', 'Edit', 'Glob', 'Grep', 'Bash'] as const;25export type ResearchTool = (typeof RESEARCH_TOOLS)[number];2627/** Set by the owner per job, enforced by the runner from OUTSIDE the agent process (kill on overrun). The agent cannot change them. */28export const ResearchBudgets = z.strictObject({29 /** wall-clock seconds of the agent process */30 wallSeconds: z.number().int().min(30).max(86_400).default(3600),31 /** lines of the job's trial ledger (every configuration backtested on train + validation, every look at the inner test split) */32 maxTrials: z.number().int().min(1).max(5000).default(200),33 /** size of the job directory */34 maxDiskMb: z.number().int().min(16).max(10_240).default(512),35 /** model tokens (input + output + cache writes and reads) counted by the runner's model gateway */36 maxModelTokens: z.number().int().min(10_000).max(500_000_000).default(5_000_000),37 /** US dollars as reported by the coding agent's CLI (when it reports a cost); also given to the CLI as its own cap */38 maxModelUsd: z.number().min(0.05).max(1000).default(10),39});40export type ResearchBudgets = z.infer<typeof ResearchBudgets>;4142export const RESEARCH_INTERVALS = ['1h', '4h', '1d'] as const;43const Sym = z.string().regex(/^[A-Z0-9]{2,10}$/);44export const ResearchJobCreate = z.strictObject({45 /** the agent the strategy is developed for; without it the result is a library draft */46 agentId: z.uuid().optional(),47 kind: z.enum(['spot', 'perp']).default('spot'),48 /** the owner's own words; given to the coding agent as the task */49 mission: z.string().trim().min(10).max(2000),50 universe: z.array(Sym).min(1).max(12).refine((u) => new Set(u).size === u.length, 'distinct symbols'),51 interval: z.enum(RESEARCH_INTERVALS).default('1h'),52 /** virtual capital of the simulation the owner asks for (reports are computed from this starting cash) */53 capitalUsd: z.number().min(10).max(10_000_000).default(1000),54 budgets: ResearchBudgets.default(ResearchBudgets.parse({})),55 tools: z.array(z.enum(RESEARCH_TOOLS)).min(1).max(RESEARCH_TOOLS.length).default([...RESEARCH_TOOLS]),56});57export type ResearchJobCreate = z.infer<typeof ResearchJobCreate>;5859// ---- what the runner receives when it claims a job ------------------------------------------------------------------------60/** candles the agent may see: train + validation only. The server keeps the test split (see `holdout`). */61export const ResearchData = z.discriminatedUnion('kind', [62 z.object({ kind: z.literal('spot'), source: z.string().regex(/^[a-z]{2,20}$/), interval: z.string(),63 /** CSV text per symbol: header `ts,open,high,low,close,volume`, ts = bar-open time in seconds UTC */64 files: z.array(z.object({ symbol: Sym, csv: z.string().max(40_000_000) })).min(1).max(12) }),65 /** the input of `python -m spq.data.hl_import` (candles + funding of the venue, optional spot candles) */66 z.object({ kind: z.literal('perp'), bundle: z.record(z.string(), z.unknown()) }),67]);68export type ResearchData = z.infer<typeof ResearchData>;6970export const RunnerJob = z.object({71 id: z.uuid(),72 /** secret of THIS job: required to append its log and to push its result; useless for any other job */73 token: z.string().min(32).max(128),74 kind: z.enum(['spot', 'perp']), mission: z.string(), universe: z.array(Sym), interval: z.string(), capitalUsd: z.number(),75 budgets: ResearchBudgets, tools: z.array(z.enum(RESEARCH_TOOLS)),76 /** META["name"] and META["version"] the strategy must carry (job-scoped: the API installs it under strategies/<name>/<version>) */77 strategyName: z.string().regex(/^[a-z0-9_]{1,40}$/), strategyVersion: z.string().regex(/^[a-z0-9_]{1,12}$/),78 /** commit of the laboratory the job must run on (null: whatever the runner has, recorded with the result) */79 labCommit: z.string().regex(/^[0-9a-f]{7,40}$/).nullable(),80 /** what the agent is given, and what it is not: bars before `devEnd` only; the server tests on [devEnd, end) */81 holdout: z.object({ start: z.string(), devEnd: z.string(), end: z.string(), devBars: z.number().int(), testBars: z.number().int() }),82 data: ResearchData,83});84export type RunnerJob = z.infer<typeof RunnerJob>;8586// ---- the submission archive -----------------------------------------------------------------------------------------------87export const SUBMISSION_FILES = {88 'strategy.py': { max: 64 * 1024, required: true },89 'README.md': { max: 32 * 1024, required: true },90 'NOTES.md': { max: 256 * 1024, required: false },91 'claims.json': { max: 16 * 1024, required: false },92} as const;93export type SubmissionFileName = keyof typeof SUBMISSION_FILES;94export const MAX_LEDGER_BYTES = 6 * 1024 * 1024;95export const MAX_SUBMISSION_BODY = 10 * 1024 * 1024;96export const MAX_LOG_LINE = 2000;97export const MAX_LOG_BATCH = 200;9899const B64 = z.string().regex(/^[A-Za-z0-9+/]*={0,2}$/);100/** what the runner measured from outside the agent (the API stores it; it trusts none of it for money decisions) */101export const RunnerUsage = z.object({102 wallMs: z.number().int().min(0), trials: z.number().int().min(0), diskMb: z.number().min(0),103 modelTokens: z.number().int().min(0), modelRequests: z.number().int().min(0), modelUsd: z.number().min(0).nullable(),104 /** the agent's own ledger file was truncated or rewritten while the runner was copying it */105 ledgerTampered: z.boolean(),106 /** 'sandbox-exec' when the agent ran under the macOS sandbox profile; 'none' when the platform offered nothing (said in the UI) */107 sandbox: z.enum(['sandbox-exec', 'none']), agent: z.string().max(40), labCommit: z.string().max(64).nullable(),108 exitCode: z.number().int().nullable(), signal: z.string().max(20).nullable(),109});110export type RunnerUsage = z.infer<typeof RunnerUsage>;111112export const SubmissionArchive = z.strictObject({113 v: z.literal(1),114 /** exactly the expected files, each base64; anything else is refused */115 files: z.array(z.strictObject({ path: z.enum(Object.keys(SUBMISSION_FILES) as [SubmissionFileName, ...SubmissionFileName[]]), b64: B64.max(400_000) })).min(1).max(4),116 /** the runner's append-only copy of the trial ledger (JSON lines) */117 ledger: z.string().max(MAX_LEDGER_BYTES),118});119export type SubmissionArchive = z.infer<typeof SubmissionArchive>;120121export const RunnerResult = z.discriminatedUnion('outcome', [122 z.strictObject({ outcome: z.literal('submission'), usage: RunnerUsage, archive: SubmissionArchive }),123 z.strictObject({ outcome: z.enum(['failed', 'cancelled', 'over_budget']), usage: RunnerUsage, reason: z.string().max(600),124 /** the ledger is kept even when nothing was submitted: the trials were consumed */125 ledger: z.string().max(MAX_LEDGER_BYTES).default('') }),126]);127export type RunnerResult = z.infer<typeof RunnerResult>;128129export interface DecodedSubmission { files: Partial<Record<SubmissionFileName, Uint8Array>>; ledger: string }130const fromB64 = (s: string): Uint8Array => { const bin = atob(s); const out = new Uint8Array(bin.length); for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); return out; };131/** Decode and bound an archive: only the expected names, once each, within their size limits, text files valid UTF-8 without NUL.132 * Throws an Error whose message can be shown to the owner. Nothing is written anywhere. */133export function decodeSubmission(a: SubmissionArchive): DecodedSubmission {134 const files: Partial<Record<SubmissionFileName, Uint8Array>> = {};135 for (const f of a.files) {136 if (files[f.path]) throw new Error(`the archive contains ${f.path} twice`);137 const bytes = fromB64(f.b64), spec = SUBMISSION_FILES[f.path];138 if (bytes.length > spec.max) throw new Error(`${f.path} is ${bytes.length} bytes (limit ${spec.max})`);139 if (bytes.length === 0) throw new Error(`${f.path} is empty`);140 if (bytes.includes(0)) throw new Error(`${f.path} contains a NUL byte`);141 try { new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch { throw new Error(`${f.path} is not UTF-8 text`); }142 files[f.path] = bytes;143 }144 for (const [name, spec] of Object.entries(SUBMISSION_FILES)) if (spec.required && !files[name as SubmissionFileName]) throw new Error(`the submission has no ${name}`);145 return { files, ledger: a.ledger };146}147148// ---- what the agent CLAIMS (claims.json) — shown next to what the server measured, never instead of it --------------------149const Num = z.number().refine((x) => Number.isFinite(x)).nullable().optional();150const ClaimSplit = z.object({ total_return: Num, sharpe: Num, max_drawdown: Num, trades: Num }).partial();151export const AgentClaims = z.object({152 train: ClaimSplit.optional(), validation: ClaimSplit.optional(),153 /** the agent's own inner test split (a part of the data it was given) — NOT the server's test split, which it never saw */154 inner_test: ClaimSplit.optional(),155 trials: z.number().int().min(0).max(1_000_000).optional(),156 summary: z.string().max(4000).optional(),157});158export type AgentClaims = z.infer<typeof AgentClaims>;159/** Parse claims.json leniently: a malformed file is "no claims" plus a note, never an error (it is only a statement). */160export function parseClaims(text: string | null): { claims: AgentClaims | null; problem: string | null } {161 if (text === null) return { claims: null, problem: null };162 try { const r = AgentClaims.safeParse(JSON.parse(text)); return r.success ? { claims: r.data, problem: null } : { claims: null, problem: 'claims.json does not follow the expected shape; it was ignored' }; }163 catch { return { claims: null, problem: 'claims.json is not valid JSON; it was ignored' }; }164}165166/** Compare one claimed metric with the server's (both may be missing). The laboratory is deterministic: an honest claim is the167 * server's number to the last digits, so the tolerance only absorbs floating-point noise between two machines.168 * `differs` only when both exist and disagree beyond it. */169export function compareClaim(claimed: number | null | undefined, measured: number | null | undefined, tol = 1e-4): { claimed: number | null; measured: number | null; differs: boolean } {170 const c = typeof claimed === 'number' && Number.isFinite(claimed) ? claimed : null, m = typeof measured === 'number' && Number.isFinite(measured) ? measured : null;171 return { claimed: c, measured: m, differs: c !== null && m !== null && Math.abs(c - m) > tol * Math.max(1, Math.abs(m)) };172}173174/** job-scoped strategy directory name: `rj_<first 12 hex of the job id>` — the API is the only writer of research/strategies/rj_* */175export const jobStrategyName = (jobId: string): string => `rj_${jobId.replace(/-/g, '').slice(0, 12)}`;176export const JOB_STRATEGY_RE = /^rj_[0-9a-f]{12}$/;177