TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1#!/usr/bin/env tsx2/**3 * RareIndex operator CLI.4 * ri connectors | ri crawl <id> [--mode probe|incremental|backfill] [--limit N]5 * ri backfill <id…> [--start-only] [--reset] [--pause] [--status] [--start YYYY-MM-DD] [--end YYYY-MM-DD]6 * ri certs [--limit N] [--grader psa] | ri certs --backfill (verify cert numbers via cert_lookup connectors / register certs from existing sales+listings)7 * ri normalize [--connector id] [--limit N] | ri resolve [--limit N]8 * ri value [--asset id | --all] [--history] | ri premiums [--category slug] | ri regrade [--kind sale|listing|both] [--dry-run]9 * ri fees --backfill [--limit N] [--connector id] [--force] | ri auctions --assess [--all] [--limit N]10 * ri index | ri snapshots | ri radar | ri health [--probe] | ri fx [--backfill] | ri benchmarks11 * ri expire | ri stats | ri run-all [--limit N] | ri worker12 */13import { loadDotenv } from './lib/env.ts';14loadDotenv();15process.env.RI_SERVICE ??= 'ri-cli';1617import { sql } from 'drizzle-orm';18import { listConnectorMeta } from '@rareindex/connectors';19import { logger } from '@rareindex/shared';20import { closeDb, db } from './lib/db.ts';21import { flushCosts } from './lib/costs.ts';22import { activeBackfill, pauseBackfill, runCrawl, startBackfill } from './crawler/run.ts';23import { normalizeBatch } from './normalizer/index.ts';24import { pendingCount, resolveBatch } from './entity-resolution/index.ts';25import { regradeRecords } from './entity-resolution/regrade.ts';26import { backfillFees } from './fees-backfill.ts';27import { assessLots } from './auctions/assess.ts';28import { assetsNeedingValuation, computePremiums, valueAsset, valueMany } from './valuation/run.ts';29import { runCategorySnapshots, runIndices, runRadar } from './indices/run.ts';30import { syncFx } from './fx.ts';31import { syncBenchmarks } from './benchmarks.ts';32import { computeHealth } from './health.ts';33import { expireListings } from './listings-expire.ts';34import { imageStats, processImages } from './image-processing/index.ts';35import { backfillCertificates, verifyCertificates } from './certs-verify.ts';3637const [cmd = 'help', ...rest] = process.argv.slice(2);38const flags: Record<string, string | boolean> = {};39const positional: string[] = [];40for (let i = 0; i < rest.length; i++) {41 const a = rest[i]!;42 if (a.startsWith('--')) {43 const k = a.slice(2);44 const next = rest[i + 1];45 if (next && !next.startsWith('--')) {46 flags[k] = next;47 i++;48 } else flags[k] = true;49 } else positional.push(a);50}51const num = (k: string, d?: number) => (flags[k] !== undefined ? Number(flags[k]) : d);52const str = (k: string) => (typeof flags[k] === 'string' ? (flags[k] as string) : undefined);53const print = (x: unknown) => console.log(typeof x === 'string' ? x : JSON.stringify(x, null, 2));5455async function drain(fn: () => Promise<{ processed: number }>, batch = 500): Promise<number> {56 let total = 0;57 for (let i = 0; i < 10_000; i++) {58 const r = await fn();59 total += r.processed;60 if (r.processed < batch) break;61 }62 return total;63}6465async function stats() {66 const tables = ['sources', 'connectors', 'connector_runs', 'raw_records', 'normalized_records', 'assets', 'asset_variants', 'sets', 'brands', 'sales', 'listings', 'price_observations', 'auction_lots', 'population_reports', 'valuations', 'asset_stats', 'price_snapshots', 'index_values', 'category_snapshots', 'radar_findings', 'fx_rates', 'benchmarks', 'costs', 'events', 'audit_log'];67 const out: Record<string, number> = {};68 for (const t of tables) {69 const r = (await db().execute(sql.raw(`select count(*)::int as n from ${t}`))) as unknown as Array<{ n: number }>;70 out[t] = r[0]?.n ?? 0;71 }72 const priced = (await db().execute(sql`select count(*)::int as n from asset_stats where riv_usd is not null`)) as unknown as Array<{ n: number }>;73 out['assets_priced'] = priced[0]?.n ?? 0;74 const idx = (await db().execute(sql`select i.ticker, v.date, v.value, v.constituents_count from index_values v join indices i on i.id = v.index_id where v.date = (select max(date) from index_values v2 where v2.index_id = v.index_id) order by i.ticker`)) as unknown as Array<Record<string, unknown>>;75 return { counts: out, indices: idx };76}7778async function main() {79 switch (cmd) {80 case 'connectors': {81 const metas = listConnectorMeta();82 print(metas.map((m) => ({ id: m.id, source: m.sourceName, type: m.sourceType, engines: m.enginePriority.join('>'), categories: m.categories.join(','), enabled: m.enabled, refreshMin: m.refreshFrequencyMinutes })));83 break;84 }85 case 'crawl': {86 const id = positional[0];87 if (!id) throw new Error('usage: ri crawl <connectorId> [--mode ...] [--limit N]');88 print(await runCrawl(id, { mode: (str('mode') as 'probe' | 'incremental' | 'backfill') ?? 'incremental', limit: num('limit'), trigger: 'manual' }));89 break;90 }91 case 'backfill': {92 // ri backfill <id> [--reset] [--pause] [--status] [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--limit N]93 const id = positional[0];94 if (!id) throw new Error('usage: ri backfill <connectorId> [--reset|--pause|--status]');95 if (flags.status) {96 print(await activeBackfill(id));97 break;98 }99 if (flags.pause) {100 print({ paused: await pauseBackfill(id) });101 break;102 }103 const ids = positional.length > 1 ? positional : [id];104 for (const cid of ids) {105 const campaign = await startBackfill(cid, { reset: Boolean(flags.reset), startDate: str('start') ?? null, endDate: str('end') ?? null });106 print({ connector: cid, campaign: campaign.id, status: campaign.status, resumedFrom: campaign.lastCursor });107 // --start-only: register the campaign and let the worker scheduler run the slices (recommended for many sources)108 if (!flags['start-only']) {109 print(await runCrawl(cid, { mode: 'backfill', limit: num('limit'), trigger: 'manual' }));110 print(await activeBackfill(cid));111 }112 }113 break;114 }115 case 'certs':116 if (flags.backfill) print(await backfillCertificates());117 else print(await verifyCertificates({ limit: num('limit', 100), grader: str('grader') }));118 break;119 case 'normalize':120 print({ processed: await drain(() => normalizeBatch({ connectorId: str('connector'), limit: num('limit', 500) }), num('limit', 500)) });121 break;122 case 'resolve': {123 const processed = await drain(() => resolveBatch({ limit: num('limit', 500), connectorId: str('connector') }), num('limit', 500));124 print({ processed, stillPending: await pendingCount() });125 break;126 }127 case 'premiums':128 print({ written: await computePremiums(str('category')) });129 break;130 case 'regrade': {131 // ri regrade [--kind sale|listing|both] [--connector id] [--limit N] [--dry-run]132 const kind = (str('kind') ?? 'both') as 'sale' | 'listing' | 'both';133 print(await regradeRecords({ kind, connectorId: str('connector'), limit: num('limit', 50_000), dryRun: Boolean(flags['dry-run']), recheck: Boolean(flags.recheck) }));134 break;135 }136 case 'fees': {137 // ri fees --backfill [--limit N] [--connector id] [--force] (buyer-pays price on existing sales, §35)138 if (!flags.backfill) {139 print('usage: ri fees --backfill [--limit N] [--connector id] [--force]');140 break;141 }142 print(await backfillFees({ limit: num('limit'), connectorId: str('connector'), force: Boolean(flags.force) }));143 break;144 }145 case 'auctions': {146 // ri auctions --assess [--all] [--limit N] (all-in bid / estimate vs RIV on live lots, §33–§35)147 if (!flags.assess) {148 print('usage: ri auctions --assess [--all] [--limit N]');149 break;150 }151 print(await assessLots({ limit: num('limit', 50_000), all: Boolean(flags.all) }));152 break;153 }154 case 'value': {155 const asset = str('asset');156 if (asset) print(await valueAsset(asset, { rebuildHistory: Boolean(flags.history) }));157 else {158 await computePremiums();159 const ids = await assetsNeedingValuation({ all: Boolean(flags.all), limit: num('limit', 200_000) });160 print(await valueMany(ids, { rebuildHistory: Boolean(flags.history) }));161 }162 break;163 }164 case 'index':165 print(await runIndices());166 break;167 case 'snapshots':168 print({ categories: await runCategorySnapshots() });169 break;170 case 'radar':171 print({ findings: await runRadar() });172 break;173 case 'health':174 print((await computeHealth({ connectorId: positional[0], probe: Boolean(flags.probe) })).map((h) => ({ connector: h.connector, status: h.status, success24h: h.success_rate_24h, records24h: h.records_24h, lastError: h.last_error, anomalies: h.anomalies })));175 break;176 case 'fx':177 print(await syncFx({ backfill: Boolean(flags.backfill) }));178 break;179 case 'benchmarks':180 print(await syncBenchmarks());181 break;182 case 'expire':183 print({ expired: await expireListings() });184 break;185 case 'stats':186 print(await stats());187 break;188 case 'images': {189 const r = await processImages({ limit: num('limit', 2000), recheck: Boolean(flags.recheck) });190 print({ ...r, totals: await imageStats() });191 break;192 }193 case 'run-all': {194 const limit = num('limit', 200);195 const [fx] = (await db().execute(sql`select count(*)::int as n from fx_rates`)) as unknown as Array<{ n: number }>;196 if (!fx || fx.n === 0) print({ fx: await syncFx({ backfill: true }) });197 const summary: Record<string, unknown> = {};198 for (const m of listConnectorMeta({ enabled: true })) {199 try {200 const r = await runCrawl(m.id, { mode: 'incremental', limit, trigger: 'manual' });201 summary[m.id] = { status: r.status, raw: r.recordsRaw, dupes: r.recordsDuplicate, error: r.error };202 } catch (err) {203 summary[m.id] = { status: 'failed', error: err instanceof Error ? err.message : String(err) };204 }205 }206 summary.normalized = await drain(() => normalizeBatch({ limit: 500 }));207 summary.resolved = await drain(() => resolveBatch({ limit: 500 }));208 summary.premiums = await computePremiums();209 const ids = await assetsNeedingValuation({ limit: 200_000 });210 summary.valuation = await valueMany(ids, { rebuildHistory: Boolean(flags.history) });211 summary.indices = await runIndices();212 summary.snapshots = await runCategorySnapshots();213 summary.radar = await runRadar();214 summary.health = (await computeHealth()).map((h) => `${h.connector}:${h.status}`);215 print(summary);216 break;217 }218 case 'worker': {219 const { startWorker } = await import('./main.ts');220 const stop = await startWorker();221 const shutdown = () => void stop().then(() => process.exit(0));222 process.on('SIGINT', shutdown);223 process.on('SIGTERM', shutdown);224 return; // keep running225 }226 default:227 print(`RareIndex CLI\n ri connectors | crawl <id> [--mode probe|incremental|backfill] [--limit N] | normalize [--connector id] | resolve\n ri value [--asset id|--all] [--history] | premiums | regrade [--dry-run] | fees --backfill | auctions --assess | index | snapshots | radar | health [--probe] | fx [--backfill] | benchmarks | expire\n ri stats | run-all [--limit N] [--history] | worker`);228 }229 await flushCosts();230 await closeDb();231}232233main().catch(async (err) => {234 logger.error({ err }, 'ri failed');235 console.error(err);236 await closeDb();237 process.exit(1);238});239