AI package, scanner, research, admin console, public API (agent F2)

modified apps/api/package.json +33 −1

++{
++  "name": "@rareindex/api",
++  "version": "0.1.0",
++  "private": true,
++  "type": "module",
++  "scripts": {
++    "dev": "tsx watch src/server.ts",
++    "start": "tsx src/server.ts",
++    "build": "tsc -p tsconfig.json --noEmit",
++    "typecheck": "tsc -p tsconfig.json --noEmit",
++    "test": "vitest run --passWithNoTests",
++    "create-key": "tsx scripts/create-key.ts"
++  },
++  "dependencies": {
++    "@fastify/compress": "^9.0.0",
++    "@fastify/cors": "^11.0.0",
++    "@fastify/etag": "^6.0.0",
++    "@fastify/rate-limit": "^11.0.0",
++    "@rareindex/connectors": "workspace:*",
++    "@rareindex/database": "workspace:*",
++    "@rareindex/shared": "workspace:*",
++    "@rareindex/taxonomy": "workspace:*",
++    "fastify": "^5.6.0",
++    "fastify-plugin": "^6.0.0",
++    "zod": "^4.0.0"
++  },
++  "devDependencies": {
++    "@types/node": "^24.0.0",
++    "tsx": "^4.20.0",
++    "typescript": "^5.9.3",
++    "vitest": "^3.2.0"
++  }
++}

added apps/api/scripts/create-key.ts +21 −0

++/**
++ * Mint an API key. Prints the plaintext ONCE; only its hash is stored.
++ *   pnpm --filter @rareindex/api create-key -- --name "Research desk" --tier professional [--user usr_xxx]
++ */
++import { closeDb } from '@rareindex/database';
++import { mintApiKey, TIERS, type Tier } from '../src/lib/keys.js';
++
++const args = process.argv.slice(2);
++const get = (flag: string) => {
++  const i = args.indexOf(flag);
++  return i >= 0 ? args[i + 1] : undefined;
++};
++const name = get('--name') ?? 'default';
++const tier = (get('--tier') ?? 'free') as Tier;
++if (!(tier in TIERS)) {
++  console.error(`unknown tier ${tier}; choose one of ${Object.keys(TIERS).join(', ')}`);
++  process.exit(1);
++}
++const minted = await mintApiKey({ name, tier, userId: get('--user') ?? null });
++console.log(`API key created (${minted.tier}, id ${minted.id}). Store it now — it will not be shown again:\n\n  ${minted.key}\n`);
++await closeDb();

added apps/api/scripts/gen-openapi.ts +9 −0

++import { writeFileSync } from 'node:fs';
++import path from 'node:path';
++import { fileURLToPath } from 'node:url';
++import { openapiDocument } from '../src/openapi.js';
++
++const here = path.dirname(fileURLToPath(import.meta.url));
++const out = path.resolve(here, '../../../docs/openapi.json');
++writeFileSync(out, JSON.stringify(openapiDocument(), null, 2) + '\n');
++console.log(`wrote ${out}`);

added apps/api/src/app.test.ts +97 −0

++import { afterAll, beforeAll, describe, expect, it } from 'vitest';
++import type { FastifyInstance } from 'fastify';
++import { closeDb, getDb, apiKeys, eq } from '@rareindex/database';
++import { buildApp } from './app.js';
++import { mintApiKey } from './lib/keys.js';
++import { toCsv, encodeCursor, decodeCursor } from './lib/envelope.js';
++
++/** Integration tests against DATABASE_URL (rareindex_ai locally). They only read except for a temporary API key. */
++let app: FastifyInstance;
++let key: { id: string; key: string };
++
++beforeAll(async () => {
++  process.env.LOG_LEVEL = 'silent';
++  app = await buildApp({ logger: false });
++  key = await mintApiKey({ name: 'vitest', tier: 'hobby' });
++});
++afterAll(async () => {
++  await getDb().delete(apiKeys).where(eq(apiKeys.id, key.id));
++  await app.close();
++  await closeDb();
++});
++
++describe('api', () => {
++  it('health and root', async () => {
++    const h = await app.inject({ method: 'GET', url: '/healthz' });
++    expect(h.statusCode).toBe(200);
++    expect(h.json().ok).toBe(true);
++    const r = await app.inject({ method: 'GET', url: '/' });
++    expect(r.json().openapi).toBe('/v1/openapi.json');
++  });
++  it('serves openapi', async () => {
++    const r = await app.inject({ method: 'GET', url: '/v1/openapi.json' });
++    expect(r.statusCode).toBe(200);
++    expect(r.json().paths['/v1/assets/search']).toBeTruthy();
++  });
++  it('anonymous requests use the public tier with rate-limit headers', async () => {
++    const r = await app.inject({ method: 'GET', url: '/v1/categories' });
++    expect(r.statusCode).toBe(200);
++    expect(r.headers['x-ratelimit-limit']).toBe('20');
++    expect(r.headers['x-request-id']).toBeTruthy();
++    const body = r.json();
++    expect(body.meta.count).toBeGreaterThan(100);
++    expect(body.meta.attribution).toMatch(/RareIndex/);
++    expect(body.data.find((c: { slug: string }) => c.slug === 'pokemon')).toBeTruthy();
++  });
++  it('rejects unknown keys and accepts minted keys with tier limits', async () => {
++    const bad = await app.inject({ method: 'GET', url: '/v1/indices', headers: { authorization: 'Bearer ri_live_nope_nope' } });
++    expect(bad.statusCode).toBe(401);
++    expect(bad.headers['content-type']).toMatch(/problem\+json/);
++    const good = await app.inject({ method: 'GET', url: '/v1/indices', headers: { authorization: `Bearer ${key.key}` } });
++    expect(good.statusCode).toBe(200);
++    expect(good.headers['x-ratelimit-limit']).toBe('120');
++    const idx = good.json();
++    expect(idx.data.find((i: { ticker: string }) => i.ticker === 'RARE')).toBeTruthy();
++    expect(idx.data.every((i: { published: boolean; value: unknown }) => i.published === (i.value !== null))).toBe(true);
++  });
++  it('validates queries and returns problem+json', async () => {
++    const r = await app.inject({ method: 'GET', url: '/v1/assets/search?limit=9999' });
++    expect(r.statusCode).toBe(400);
++    expect(r.json().title).toBe('Invalid query');
++  });
++  it('404 on unknown asset/market', async () => {
++    expect((await app.inject({ method: 'GET', url: '/v1/assets/does-not-exist' })).statusCode).toBe(404);
++    expect((await app.inject({ method: 'GET', url: '/v1/markets/does-not-exist' })).statusCode).toBe(404);
++  });
++  it('search, trending, sales, records, markets and csv export respond', async () => {
++    for (const url of ['/v1/assets/search?q=charizard', '/v1/trending', '/v1/sales/latest', '/v1/records', '/v1/markets', '/v1/markets/pokemon', '/v1/indices/RARE-TCG/history', '/v1/stats']) {
++      const r = await app.inject({ method: 'GET', url });
++      expect(r.statusCode, url).toBe(200);
++    }
++    const csv = await app.inject({ method: 'GET', url: '/v1/categories?format=csv' });
++    expect(csv.headers['content-type']).toMatch(/text\/csv/);
++    expect(csv.body.split('\n')[0]).toContain('slug');
++  });
++  it('rate limits by key', async () => {
++    const small = await mintApiKey({ name: 'vitest-small', tier: 'free' });
++    try {
++      let last = 200;
++      for (let i = 0; i < 65; i++) {
++        const r = await app.inject({ method: 'GET', url: '/v1/stats', headers: { authorization: `Bearer ${small.key}` } });
++        last = r.statusCode;
++        if (last === 429) break;
++      }
++      expect(last).toBe(429);
++    } finally {
++      await getDb().delete(apiKeys).where(eq(apiKeys.id, small.id));
++    }
++  });
++});
++
++describe('envelope helpers', () => {
++  it('csv + cursor', () => {
++    expect(toCsv([{ a: 1, b: 'x,y' }, { a: 2, b: null }])).toBe('a,b\n1,"x,y"\n2,\n');
++    expect(decodeCursor(encodeCursor(150))).toBe(150);
++    expect(decodeCursor('garbage')).toBe(0);
++  });
++});

added apps/api/src/app.ts +71 −0

++import Fastify, { type FastifyInstance } from 'fastify';
++import cors from '@fastify/cors';
++import compress from '@fastify/compress';
++import etag from '@fastify/etag';
++import rateLimit from '@fastify/rate-limit';
++import { randomUUID } from 'node:crypto';
++import { getSql } from '@rareindex/database';
++import { authPlugin, rateKeyFor, rateLimitFor } from './plugins/auth.js';
++import { v1Routes } from './routes/v1.js';
++import { internalRoutes } from './routes/internal.js';
++import { problem } from './lib/envelope.js';
++
++export interface BuildOptions {
++  logger?: boolean;
++}
++
++export async function buildApp(opts: BuildOptions = {}): Promise<FastifyInstance> {
++  const app = Fastify({
++    logger: opts.logger === false ? false : { level: process.env.LOG_LEVEL ?? 'info', redact: ['req.headers.authorization'] },
++    genReqId: (req) => (req.headers['x-request-id'] as string | undefined) ?? randomUUID(),
++    trustProxy: true,
++  });
++
++  await app.register(cors, { origin: true, methods: ['GET', 'HEAD', 'OPTIONS'], exposedHeaders: ['x-request-id', 'x-ratelimit-limit', 'x-ratelimit-remaining', 'x-ratelimit-reset'] });
++  await app.register(compress, { global: true, encodings: ['gzip', 'br', 'deflate'] });
++  await app.register(etag);
++  await app.register(authPlugin);
++  await app.register(rateLimit, {
++    global: true,
++    max: (req) => rateLimitFor(req),
++    timeWindow: '1 minute',
++    keyGenerator: (req) => rateKeyFor(req),
++    errorResponseBuilder: (req, ctx) => ({
++      statusCode: 429,
++      type: 'https://www.rareindex.io/api-docs#errors',
++      title: 'Rate limit exceeded',
++      status: 429,
++      detail: `Limit ${ctx.max} requests per ${ctx.after}. Retry after ${ctx.ttl} ms.`,
++      instance: req.url,
++      request_id: req.id,
++    }),
++  });
++
++  app.addHook('onSend', async (req, reply, payload) => {
++    reply.header('x-request-id', req.id);
++    return payload;
++  });
++
++  app.setErrorHandler((err: Error & { statusCode?: number; status?: number; title?: string; detail?: string }, req, reply) => {
++    const status = err.statusCode ?? err.status ?? 500;
++    if (status >= 500) req.log.error({ err }, 'unhandled api error');
++    const title = status >= 500 ? 'Internal error' : (err.title ?? err.message);
++    const detail = status >= 500 ? 'The request could not be completed.' : (err.detail ?? err.message);
++    return problem(reply, status, title, detail);
++  });
++  app.setNotFoundHandler((req, reply) => problem(reply, 404, 'Not found', `No route for ${req.method} ${req.url}`));
++
++  app.get('/healthz', async (_req, reply) => {
++    try {
++      await getSql()`select 1`;
++      return { ok: true, service: 'rareindex-api', time: new Date().toISOString() };
++    } catch (err) {
++      return reply.code(503).send({ ok: false, service: 'rareindex-api', error: err instanceof Error ? err.message : String(err) });
++    }
++  });
++  app.get('/', async () => ({ name: 'RareIndex Public API', docs: 'https://www.rareindex.io/api-docs', openapi: '/v1/openapi.json' }));
++
++  await app.register(v1Routes);
++  await app.register(internalRoutes);
++  return app;
++}

added apps/api/src/lib/envelope.ts +65 −0

++import type { FastifyReply } from 'fastify';
++
++export const ATTRIBUTION = 'Data © RareIndex.io and its sources; valuations are estimates, listing prices are not confirmed transactions. See https://www.rareindex.io/data for methodology and terms.';
++
++export interface Meta {
++  count: number;
++  cursor?: string | null;
++  as_of: string;
++  attribution: string;
++  [k: string]: unknown;
++}
++
++export function envelope<T>(data: T, meta: Partial<Meta> & { count?: number } = {}) {
++  const count = meta.count ?? (Array.isArray(data) ? data.length : 1);
++  return { data, meta: { count, cursor: meta.cursor ?? null, as_of: new Date().toISOString(), attribution: ATTRIBUTION, ...meta } };
++}
++
++export class ApiProblem extends Error {
++  constructor(
++    public status: number,
++    public title: string,
++    detail?: string,
++    public type = 'about:blank',
++  ) {
++    super(detail ?? title);
++  }
++}
++
++export function problem(reply: FastifyReply, status: number, title: string, detail?: string, extra: Record<string, unknown> = {}) {
++  return reply
++    .code(status)
++    .type('application/problem+json')
++    .send({ type: 'https://www.rareindex.io/api-docs#errors', title, status, detail: detail ?? title, instance: reply.request.url, request_id: reply.request.id, ...extra });
++}
++
++/** Opaque offset cursor. */
++export function encodeCursor(offset: number): string {
++  return Buffer.from(String(offset), 'utf8').toString('base64url');
++}
++export function decodeCursor(cursor: string | undefined | null): number {
++  if (!cursor) return 0;
++  const n = Number(Buffer.from(cursor, 'base64url').toString('utf8'));
++  return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
++}
++
++/** Minimal CSV export for flat row arrays. */
++export function toCsv(rows: Array<Record<string, unknown>>): string {
++  if (rows.length === 0) return '';
++  const cols = [...new Set(rows.flatMap((r) => Object.keys(r)))];
++  const esc = (v: unknown) => {
++    if (v === null || v === undefined) return '';
++    const s = v instanceof Date ? v.toISOString() : typeof v === 'object' ? JSON.stringify(v) : String(v);
++    return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
++  };
++  return [cols.join(','), ...rows.map((r) => cols.map((c) => esc(r[c])).join(','))].join('\n') + '\n';
++}
++
++export function wantsCsv(query: Record<string, unknown>): boolean {
++  return String(query.format ?? '').toLowerCase() === 'csv';
++}
++
++export function sendList(reply: FastifyReply, rows: Array<Record<string, unknown>>, meta: Partial<Meta> = {}, query: Record<string, unknown> = {}) {
++  if (wantsCsv(query)) return reply.type('text/csv; charset=utf-8').header('content-disposition', 'attachment; filename="rareindex-export.csv"').send(toCsv(rows));
++  return reply.send(envelope(rows, meta));
++}

added apps/api/src/lib/keys.ts +54 −0

++import { randomBytes } from 'node:crypto';
++import { sha256, newId } from '@rareindex/shared';
++import { getDb, apiKeys, eq } from '@rareindex/database';
++
++/** API tiers (§158). Limits are per key per minute + a daily quota. */
++export const TIERS = {
++  public: { rateLimitPerMinute: 20, dailyQuota: 500 },
++  free: { rateLimitPerMinute: 60, dailyQuota: 1_000 },
++  hobby: { rateLimitPerMinute: 120, dailyQuota: 10_000 },
++  professional: { rateLimitPerMinute: 600, dailyQuota: 100_000 },
++  research: { rateLimitPerMinute: 600, dailyQuota: 250_000 },
++  enterprise: { rateLimitPerMinute: 3_000, dailyQuota: 5_000_000 },
++} as const;
++export type Tier = keyof typeof TIERS;
++
++export interface MintedKey {
++  id: string;
++  key: string;
++  prefix: string;
++  tier: Tier;
++}
++
++/** Create a key: `ri_live_<prefix>_<secret>`; only the hash is stored. The plaintext is returned once. */
++export async function mintApiKey(input: { name: string; tier?: Tier; userId?: string | null }): Promise<MintedKey> {
++  const tier = input.tier ?? 'free';
++  const prefix = randomBytes(4).toString('hex');
++  const secret = randomBytes(24).toString('base64url');
++  const key = `ri_live_${prefix}_${secret}`;
++  const id = newId('apiKey');
++  const t = TIERS[tier];
++  await getDb().insert(apiKeys).values({ id, userId: input.userId ?? null, name: input.name, prefix, keyHash: sha256(key), tier, rateLimitPerMinute: t.rateLimitPerMinute, dailyQuota: t.dailyQuota });
++  return { id, key, prefix, tier };
++}
++
++export interface ResolvedKey {
++  id: string;
++  tier: Tier;
++  rateLimitPerMinute: number;
++  dailyQuota: number;
++  userId: string | null;
++}
++
++export function parseBearer(header: string | undefined): string | null {
++  if (!header) return null;
++  const m = header.match(/^Bearer\s+(\S+)$/i);
++  return m ? m[1]! : null;
++}
++
++export async function resolveApiKey(key: string): Promise<ResolvedKey | null> {
++  if (!key.startsWith('ri_live_')) return null;
++  const [row] = await getDb().select().from(apiKeys).where(eq(apiKeys.keyHash, sha256(key))).limit(1);
++  if (!row || row.revokedAt) return null;
++  return { id: row.id, tier: (row.tier as Tier) ?? 'free', rateLimitPerMinute: row.rateLimitPerMinute, dailyQuota: row.dailyQuota, userId: row.userId };
++}

added apps/api/src/lib/queries.ts +156 −0

++import { getDb, sql } from '@rareindex/database';
++
++/**
++ * Read-only query helpers for the public API. Plain SQL through drizzle's `sql` tag: every
++ * function returns already-shaped rows so route handlers stay thin. Nothing here writes.
++ */
++type Row = Record<string, unknown>;
++async function rows<T = Row>(q: ReturnType<typeof sql>): Promise<T[]> {
++  const res = await getDb().execute(q);
++  return res as unknown as T[];
++}
++
++export const ASSET_COLUMNS = sql`a.id, a.slug, a.title, a.name, a.category_slug, a.family_slug, a.brand, a.franchise, a.set_name, a.set_code, a.number, a.year, a.edition, a.variant, a.language, a.hero_image_url, a.identifiers, a.data_quality,
++  s.riv_usd, s.riv_low_usd, s.riv_high_usd, s.riv_confidence, s.riv_sample_size, s.latest_sale_usd, s.latest_sale_at, s.change_7d, s.change_30d, s.change_1y, s.ath_usd, s.atl_usd,
++  s.sales_count, s.sales_30d, s.active_listings, s.min_ask_usd, s.liquidity_score, s.rarity_score, s.momentum_30d, s.trending_score, s.value_opportunity, s.updated_at as stats_updated_at`;
++
++export async function searchAssets(opts: { q?: string; category?: string; limit: number; offset: number }) {
++  const q = opts.q?.trim();
++  const catFilter = opts.category ? sql`and (a.category_slug = ${opts.category} or a.family_slug = ${opts.category})` : sql``;
++  if (q) {
++    const tsq = q
++      .split(/\s+/)
++      .filter(Boolean)
++      .map((t) => t.replace(/[^\p{L}\p{N}./-]/gu, ''))
++      .filter(Boolean)
++      .map((t) => `${t}:*`)
++      .join(' & ');
++    return rows(sql`
++      select ${ASSET_COLUMNS},
++        (coalesce(ts_rank(a.search, to_tsquery('simple', ${tsq})), 0) * 2 + similarity(a.title, ${q}) + coalesce(log(1 + s.sales_count), 0) * 0.05) as score
++      from assets a left join asset_stats s on s.asset_id = a.id
++      where (a.search @@ to_tsquery('simple', ${tsq}) or a.title % ${q} or a.identifiers::text ilike ${'%' + q + '%'}) ${catFilter}
++      order by score desc, s.sales_count desc nulls last
++      limit ${opts.limit} offset ${opts.offset}`);
++  }
++  return rows(sql`
++    select ${ASSET_COLUMNS}, 0 as score from assets a left join asset_stats s on s.asset_id = a.id
++    where true ${catFilter}
++    order by s.sales_count desc nulls last, a.updated_at desc
++    limit ${opts.limit} offset ${opts.offset}`);
++}
++
++export async function getAsset(idOrSlug: string) {
++  const [asset] = await rows(sql`select ${ASSET_COLUMNS}, a.description, a.reference, a.model, a.series, a.rarity, a.production_quantity, a.original_msrp, a.original_msrp_currency, a.release_date, a.metadata, a.created_at, a.updated_at
++    from assets a left join asset_stats s on s.asset_id = a.id where a.id = ${idOrSlug} or a.slug = ${idOrSlug} limit 1`);
++  if (!asset) return null;
++  const id = asset.id as string;
++  const variants = await rows(sql`select v.id, v.variant_key, v.label, v.grader, v.grade, v.qualifier, v.condition, v.completeness, v.is_default,
++      vs.riv_usd, vs.riv_low_usd, vs.riv_high_usd, vs.riv_confidence, vs.riv_sample_size, vs.latest_sale_usd, vs.latest_sale_at, vs.change_30d, vs.change_1y, vs.sales_count, vs.sales_30d, vs.active_listings, vs.min_ask_usd, vs.liquidity_score
++    from asset_variants v left join variant_stats vs on vs.variant_id = v.id where v.asset_id = ${id} order by vs.sales_count desc nulls last, v.label`);
++  const [valuation] = await rows(sql`select id, variant_id, computed_at, riv_usd, low_usd, high_usd, confidence, confidence_label, sample_size, window_days, methods, method, notes from valuations where asset_id = ${id} and variant_id is null order by computed_at desc limit 1`);
++  const sources = await rows(sql`select source_id, count(*)::int as sales from sales where asset_id = ${id} and status = 'valid' group by source_id order by sales desc`);
++  return { ...(asset as Row & { id: string }), variants, valuation: valuation ?? null, sources };
++}
++
++export async function assetSales(assetId: string, opts: { limit: number; offset: number; variantId?: string; includeFlagged?: boolean }) {
++  const statusFilter = opts.includeFlagged ? sql`and status <> 'excluded'` : sql`and status = 'valid'`;
++  const variantFilter = opts.variantId ? sql`and variant_id = ${opts.variantId}` : sql``;
++  return rows(sql`select id, variant_id, source_id, source_url, sale_type, sale_date, price, currency, price_usd, buyer_premium_included, condition, grader, grade, certification_number, auction_house, lot_number, image_urls, raw_title, confidence, data_quality, status, flags
++    from sales where asset_id = ${assetId} ${statusFilter} ${variantFilter} order by sale_date desc limit ${opts.limit} offset ${opts.offset}`);
++}
++
++export async function assetListings(assetId: string, opts: { limit: number; offset: number; availability?: string }) {
++  const avail = opts.availability ?? 'available';
++  return rows(sql`select id, variant_id, source_id, source_url, listing_type, price, currency, price_usd, seller, location, shipping_cost, condition, grader, grade, certification_number, image_urls, raw_title, listed_at, ends_at, availability, bid_count, first_seen_at, last_seen_at, discount_to_riv, flags
++    from listings where asset_id = ${assetId} and availability = ${avail} order by price_usd asc nulls last limit ${opts.limit} offset ${opts.offset}`);
++}
++
++export async function assetHistory(assetId: string, opts: { variantId?: string; from?: string; to?: string }) {
++  const variant = opts.variantId ?? '';
++  const from = opts.from ?? '1900-01-01';
++  const to = opts.to ?? '2999-12-31';
++  return rows(sql`select date, riv_usd, latest_sale_usd, median_usd, sales_count, volume_usd, listings_count, min_ask_usd, observation_usd
++    from price_snapshots where asset_id = ${assetId} and variant_id = ${variant} and date between ${from} and ${to} order by date asc`);
++}
++
++export async function listCategories() {
++  return rows(sql`select c.slug, c.parent_slug, c.family_slug, c.name, c.short_name, c.description, c.level, c.phase, c.active, c.index_ticker, c.condition_scale, c.graders,
++      (select count(*)::int from assets a where a.category_slug = c.slug) as tracked_assets
++    from categories c where c.active order by c.sort_order`);
++}
++
++export async function listIndices() {
++  return rows(sql`with latest as (
++      select distinct on (index_id) index_id, date, value, constituents_count, transactions, volume_usd, median_sale_usd, market_cap_est_usd, market_cap_confidence, liquidity_score, momentum, tracked_assets, coverage
++      from index_values order by index_id, date desc)
++    select i.id, i.ticker, i.name, i.description, i.parent_ticker, i.family_slugs, i.methodology, i.weighting, i.base_date, i.base_value, i.min_constituents, i.is_flagship, i.color,
++      l.date as as_of, l.value, l.constituents_count, l.transactions, l.volume_usd, l.median_sale_usd, l.market_cap_est_usd, l.market_cap_confidence, l.liquidity_score, l.momentum, l.tracked_assets, l.coverage,
++      (select value from index_values v where v.index_id = i.id and v.date <= l.date - interval '1 day' order by date desc limit 1) as value_1d,
++      (select value from index_values v where v.index_id = i.id and v.date <= l.date - interval '7 day' order by date desc limit 1) as value_7d,
++      (select value from index_values v where v.index_id = i.id and v.date <= l.date - interval '30 day' order by date desc limit 1) as value_30d,
++      (select value from index_values v where v.index_id = i.id and v.date <= l.date - interval '365 day' order by date desc limit 1) as value_1y,
++      (select value from index_values v where v.index_id = i.id and v.date <= date_trunc('year', l.date)::date order by date desc limit 1) as value_ytd
++    from indices i left join latest l on l.index_id = i.id where i.active order by i.is_flagship desc, i.ticker`);
++}
++
++export async function indexHistory(ticker: string, opts: { from?: string; to?: string }) {
++  const from = opts.from ?? '1900-01-01';
++  const to = opts.to ?? '2999-12-31';
++  return rows(sql`select v.date, v.value, v.constituents_count, v.transactions, v.volume_usd, v.median_sale_usd, v.avg_sale_usd, v.market_cap_est_usd, v.liquidity_score, v.momentum, v.coverage
++    from index_values v join indices i on i.id = v.index_id where i.ticker = ${ticker} and v.date between ${from} and ${to} order by v.date asc`);
++}
++
++export async function listMarkets() {
++  return rows(sql`with latest as (
++      select distinct on (category_slug) * from category_snapshots order by category_slug, date desc)
++    select c.slug, c.name, c.family_slug, c.level, c.index_ticker, l.date as as_of, l.index_value, l.tracked_assets, l.assets_with_valuation, l.sales, l.volume_usd, l.median_sale_usd, l.active_listings, l.market_cap_est_usd, l.liquidity_score, l.change_1d, l.change_7d, l.change_30d, l.change_1y,
++      (select count(*)::int from assets a where a.family_slug = c.slug or a.category_slug = c.slug) as assets_now
++    from categories c left join latest l on l.category_slug = c.slug where c.active and c.level = 0 order by c.sort_order`);
++}
++
++export async function getMarket(slug: string) {
++  const [category] = await rows(sql`select slug, parent_slug, family_slug, name, description, level, phase, index_ticker, condition_scale, graders from categories where slug = ${slug}`);
++  if (!category) return null;
++  const [snapshot] = await rows(sql`select * from category_snapshots where category_slug = ${slug} order by date desc limit 1`);
++  const scope = sql`(a.category_slug = ${slug} or a.family_slug = ${slug})`;
++  const [counts] = await rows(sql`select count(*)::int as tracked_assets, count(s.riv_usd)::int as valued_assets, coalesce(sum(s.sales_count),0)::int as sales_total, coalesce(sum(s.active_listings),0)::int as active_listings from assets a left join asset_stats s on s.asset_id = a.id where ${scope}`);
++  const gainers = await rows(sql`select ${ASSET_COLUMNS} from assets a join asset_stats s on s.asset_id = a.id where ${scope} and s.change_30d is not null and s.riv_sample_size >= 3 order by s.change_30d desc limit 10`);
++  const losers = await rows(sql`select ${ASSET_COLUMNS} from assets a join asset_stats s on s.asset_id = a.id where ${scope} and s.change_30d is not null and s.riv_sample_size >= 3 order by s.change_30d asc limit 10`);
++  const mostValuable = await rows(sql`select ${ASSET_COLUMNS} from assets a join asset_stats s on s.asset_id = a.id where ${scope} and s.riv_usd is not null order by s.riv_usd desc limit 10`);
++  const mostLiquid = await rows(sql`select ${ASSET_COLUMNS} from assets a join asset_stats s on s.asset_id = a.id where ${scope} and s.liquidity_score is not null order by s.liquidity_score desc limit 10`);
++  const recentSales = await rows(sql`select sa.id, sa.asset_id, a.slug as asset_slug, a.title, sa.source_id, sa.source_url, sa.sale_date, sa.price, sa.currency, sa.price_usd, sa.grader, sa.grade from sales sa join assets a on a.id = sa.asset_id where ${scope} and sa.status = 'valid' order by sa.sale_date desc limit 20`);
++  const history = await rows(sql`select date, index_value, sales, volume_usd, median_sale_usd, active_listings, tracked_assets from category_snapshots where category_slug = ${slug} order by date asc`);
++  return { category, snapshot: snapshot ?? null, counts, gainers, losers, most_valuable: mostValuable, most_liquid: mostLiquid, recent_sales: recentSales, history };
++}
++
++export async function trending(opts: { category?: string; limit: number; offset: number }) {
++  const cat = opts.category ? sql`and (a.category_slug = ${opts.category} or a.family_slug = ${opts.category})` : sql``;
++  return rows(sql`select ${ASSET_COLUMNS} from assets a join asset_stats s on s.asset_id = a.id where s.trending_score is not null ${cat} order by s.trending_score desc limit ${opts.limit} offset ${opts.offset}`);
++}
++
++export async function latestSales(opts: { category?: string; minUsd?: number; limit: number; offset: number }) {
++  const cat = opts.category ? sql`and (a.category_slug = ${opts.category} or a.family_slug = ${opts.category})` : sql``;
++  const min = opts.minUsd ? sql`and sa.price_usd >= ${opts.minUsd}` : sql``;
++  return rows(sql`select sa.id, sa.asset_id, a.slug as asset_slug, a.title, a.category_slug, a.hero_image_url, sa.source_id, sa.source_url, sa.sale_type, sa.sale_date, sa.price, sa.currency, sa.price_usd, sa.grader, sa.grade, sa.condition, sa.auction_house, sa.confidence
++    from sales sa join assets a on a.id = sa.asset_id where sa.status = 'valid' ${cat} ${min} order by sa.sale_date desc, sa.created_at desc limit ${opts.limit} offset ${opts.offset}`);
++}
++
++/** Record sales: highest verified transaction per family (§154). */
++export async function recordSales() {
++  return rows(sql`select distinct on (a.family_slug) a.family_slug, c.name as family_name, sa.id as sale_id, sa.asset_id, a.slug as asset_slug, a.title, sa.source_id, sa.source_url, sa.sale_date, sa.price, sa.currency, sa.price_usd, sa.grader, sa.grade, sa.auction_house
++    from sales sa join assets a on a.id = sa.asset_id join categories c on c.slug = a.family_slug
++    where sa.status = 'valid' and sa.confidence >= 0.8 and sa.is_bundle = false
++    order by a.family_slug, sa.price_usd desc`);
++}
++
++export async function platformStats() {
++  const [r] = await rows(sql`select
++    (select count(*)::int from assets) as assets,
++    (select count(*)::int from sales where status = 'valid') as sales,
++    (select count(*)::int from listings where availability = 'available') as listings,
++    (select count(*)::int from sources where active) as sources,
++    (select count(*)::int from connectors where status = 'active') as connectors,
++    (select count(*)::int from categories where active) as categories`);
++  return r;
++}

added apps/api/src/openapi.ts +91 −0

++/** OpenAPI 3.1 document for the public API (§157). Served at /v1/openapi.json and written to docs/openapi.json by scripts/gen-openapi.ts. */
++
++const paging = [
++  { name: 'limit', in: 'query', schema: { type: 'integer', minimum: 1, maximum: 200, default: 50 } },
++  { name: 'cursor', in: 'query', schema: { type: 'string' }, description: 'Opaque cursor from meta.cursor' },
++  { name: 'format', in: 'query', schema: { type: 'string', enum: ['json', 'csv'] }, description: 'csv returns a flat export of the rows' },
++];
++const range = [
++  { name: 'from', in: 'query', schema: { type: 'string', format: 'date' } },
++  { name: 'to', in: 'query', schema: { type: 'string', format: 'date' } },
++  { name: 'format', in: 'query', schema: { type: 'string', enum: ['json', 'csv'] } },
++];
++const envelopeOf = (ref: string, list = true) => ({
++  type: 'object',
++  properties: {
++    data: list ? { type: 'array', items: { $ref: ref } } : { $ref: ref },
++    meta: { $ref: '#/components/schemas/Meta' },
++  },
++});
++const ok = (ref: string, list = true) => ({ 200: { description: 'OK', content: { 'application/json': { schema: envelopeOf(ref, list) } } } });
++const errors = {
++  400: { description: 'Invalid query', content: { 'application/problem+json': { schema: { $ref: '#/components/schemas/Problem' } } } },
++  401: { description: 'Invalid API key', content: { 'application/problem+json': { schema: { $ref: '#/components/schemas/Problem' } } } },
++  404: { description: 'Not found', content: { 'application/problem+json': { schema: { $ref: '#/components/schemas/Problem' } } } },
++  429: { description: 'Rate limit or quota exceeded', content: { 'application/problem+json': { schema: { $ref: '#/components/schemas/Problem' } } } },
++};
++
++export function openapiDocument() {
++  return {
++    openapi: '3.1.0',
++    info: {
++      title: 'RareIndex Public API',
++      version: '1.0.0',
++      description:
++        'Structured market data for collectible assets: canonical assets, observed sales, live listings, price history, category markets and the RareIndex indices. Valuations are estimates with confidence and sample size; listing prices are not confirmed transactions. Attribution to RareIndex and the original sources is required.',
++      contact: { name: 'RareIndex', url: 'https://www.rareindex.io/api-docs', email: 'api@rareindex.io' },
++      termsOfService: 'https://www.rareindex.io/about#terms',
++    },
++    servers: [{ url: 'https://www.rareindex.io/api', description: 'Production (proxied by the web app)' }, { url: 'http://localhost:8211', description: 'Local API service' }],
++    security: [{ bearerAuth: [] }, {}],
++    tags: [
++      { name: 'assets' },
++      { name: 'markets' },
++      { name: 'indices' },
++      { name: 'sales' },
++      { name: 'reference' },
++    ],
++    paths: {
++      '/v1/assets/search': { get: { tags: ['assets'], summary: 'Search canonical assets', parameters: [{ name: 'q', in: 'query', schema: { type: 'string' }, description: 'Natural language query, e.g. "1999 Charizard PSA 10"' }, { name: 'category', in: 'query', schema: { type: 'string' }, description: 'Category or family slug' }, ...paging], responses: { ...ok('#/components/schemas/AssetSummary'), ...errors } } },
++      '/v1/assets/{id}': { get: { tags: ['assets'], summary: 'Asset detail with variants, latest valuation and sources', parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' }, description: 'Asset id (rare_…) or slug' }], responses: { ...ok('#/components/schemas/AssetDetail', false), ...errors } } },
++      '/v1/assets/{id}/sales': { get: { tags: ['assets', 'sales'], summary: 'Observed sales for an asset', parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }, { name: 'variant', in: 'query', schema: { type: 'string' } }, { name: 'include_flagged', in: 'query', schema: { type: 'boolean' } }, ...paging], responses: { ...ok('#/components/schemas/Sale'), ...errors } } },
++      '/v1/assets/{id}/listings': { get: { tags: ['assets'], summary: 'Listings for an asset (asks, not transactions)', parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }, { name: 'availability', in: 'query', schema: { type: 'string', enum: ['available', 'sold', 'ended', 'removed'], default: 'available' } }, ...paging], responses: { ...ok('#/components/schemas/Listing'), ...errors } } },
++      '/v1/assets/{id}/history': { get: { tags: ['assets'], summary: 'Daily price history (RIV, latest sale, median, volume, listings)', parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }, { name: 'variant', in: 'query', schema: { type: 'string' } }, ...range], responses: { ...ok('#/components/schemas/PricePoint'), ...errors } } },
++      '/v1/categories': { get: { tags: ['reference'], summary: 'Taxonomy', parameters: [paging[2]!], responses: { ...ok('#/components/schemas/Category'), ...errors } } },
++      '/v1/indices': { get: { tags: ['indices'], summary: 'RARE and subindices with latest values and changes', parameters: [paging[2]!], responses: { ...ok('#/components/schemas/Index'), ...errors } } },
++      '/v1/indices/{ticker}/history': { get: { tags: ['indices'], summary: 'Index daily history', parameters: [{ name: 'ticker', in: 'path', required: true, schema: { type: 'string' }, example: 'RARE-TCG' }, ...range], responses: { ...ok('#/components/schemas/IndexPoint'), ...errors } } },
++      '/v1/markets': { get: { tags: ['markets'], summary: 'Category markets overview (latest snapshot per family)', parameters: [paging[2]!], responses: { ...ok('#/components/schemas/Market'), ...errors } } },
++      '/v1/markets/{slug}': { get: { tags: ['markets'], summary: 'Category market detail: movers, most valuable, most liquid, recent sales, history', parameters: [{ name: 'slug', in: 'path', required: true, schema: { type: 'string' }, example: 'pokemon' }], responses: { ...ok('#/components/schemas/MarketDetail', false), ...errors } } },
++      '/v1/trending': { get: { tags: ['markets'], summary: 'Trending assets', parameters: [{ name: 'category', in: 'query', schema: { type: 'string' } }, ...paging], responses: { ...ok('#/components/schemas/AssetSummary'), ...errors } } },
++      '/v1/sales/latest': { get: { tags: ['sales'], summary: 'Latest observed sales across the platform', parameters: [{ name: 'category', in: 'query', schema: { type: 'string' } }, { name: 'min_usd', in: 'query', schema: { type: 'number' } }, ...paging], responses: { ...ok('#/components/schemas/SaleWithAsset'), ...errors } } },
++      '/v1/records': { get: { tags: ['sales'], summary: 'Record sale per family (verified transactions only)', parameters: [paging[2]!], responses: { ...ok('#/components/schemas/SaleWithAsset'), ...errors } } },
++      '/v1/stats': { get: { tags: ['reference'], summary: 'Platform counts', responses: { ...ok('#/components/schemas/Stats', false) } } },
++    },
++    components: {
++      securitySchemes: { bearerAuth: { type: 'http', scheme: 'bearer', description: 'API key `ri_live_…`. Requests without a key use the public tier (20 req/min).' } },
++      schemas: {
++        Meta: { type: 'object', properties: { count: { type: 'integer' }, cursor: { type: ['string', 'null'] }, as_of: { type: 'string', format: 'date-time' }, attribution: { type: 'string' } }, additionalProperties: true },
++        Problem: { type: 'object', properties: { type: { type: 'string' }, title: { type: 'string' }, status: { type: 'integer' }, detail: { type: 'string' }, instance: { type: 'string' }, request_id: { type: 'string' } } },
++        AssetSummary: {
++          type: 'object',
++          properties: {
++            id: { type: 'string' }, slug: { type: 'string' }, title: { type: 'string' }, name: { type: 'string' }, category_slug: { type: 'string' }, family_slug: { type: 'string' }, brand: { type: ['string', 'null'] }, franchise: { type: ['string', 'null'] }, set_name: { type: ['string', 'null'] }, number: { type: ['string', 'null'] }, year: { type: ['integer', 'null'] }, variant: { type: ['string', 'null'] }, hero_image_url: { type: ['string', 'null'] },
++            riv_usd: { type: ['number', 'null'], description: 'RareIndex Valuation (estimate)' }, riv_low_usd: { type: ['number', 'null'] }, riv_high_usd: { type: ['number', 'null'] }, riv_confidence: { type: ['number', 'null'] }, riv_sample_size: { type: 'integer' },
++            latest_sale_usd: { type: ['number', 'null'] }, latest_sale_at: { type: ['string', 'null'] }, change_7d: { type: ['number', 'null'] }, change_30d: { type: ['number', 'null'] }, change_1y: { type: ['number', 'null'] }, sales_count: { type: 'integer' }, sales_30d: { type: 'integer' }, active_listings: { type: 'integer' }, min_ask_usd: { type: ['number', 'null'] }, liquidity_score: { type: ['number', 'null'] }, rarity_score: { type: ['number', 'null'] }, trending_score: { type: ['number', 'null'] },
++          },
++        },
++        AssetDetail: { allOf: [{ $ref: '#/components/schemas/AssetSummary' }, { type: 'object', properties: { description: { type: ['string', 'null'] }, identifiers: { type: 'object' }, variants: { type: 'array', items: { type: 'object' } }, valuation: { type: ['object', 'null'] }, sources: { type: 'array', items: { type: 'object' } } } }] },
++        Sale: { type: 'object', properties: { id: { type: 'string' }, source_id: { type: 'string' }, source_url: { type: 'string' }, sale_type: { type: 'string' }, sale_date: { type: 'string' }, price: { type: 'number' }, currency: { type: 'string' }, price_usd: { type: 'number' }, grader: { type: ['string', 'null'] }, grade: { type: ['string', 'null'] }, condition: { type: ['string', 'null'] }, status: { type: 'string' }, confidence: { type: 'number' } } },
++        SaleWithAsset: { allOf: [{ $ref: '#/components/schemas/Sale' }, { type: 'object', properties: { asset_id: { type: 'string' }, asset_slug: { type: 'string' }, title: { type: 'string' } } }] },
++        Listing: { type: 'object', properties: { id: { type: 'string' }, source_id: { type: 'string' }, source_url: { type: 'string' }, listing_type: { type: 'string' }, price: { type: ['number', 'null'] }, currency: { type: ['string', 'null'] }, price_usd: { type: ['number', 'null'] }, availability: { type: 'string' }, discount_to_riv: { type: ['number', 'null'] } } },
++        PricePoint: { type: 'object', properties: { date: { type: 'string', format: 'date' }, riv_usd: { type: ['number', 'null'] }, latest_sale_usd: { type: ['number', 'null'] }, median_usd: { type: ['number', 'null'] }, sales_count: { type: 'integer' }, volume_usd: { type: ['number', 'null'] }, listings_count: { type: 'integer' }, min_ask_usd: { type: ['number', 'null'] } } },
++        Category: { type: 'object', properties: { slug: { type: 'string' }, parent_slug: { type: ['string', 'null'] }, family_slug: { type: 'string' }, name: { type: 'string' }, level: { type: 'integer' }, phase: { type: 'integer' }, index_ticker: { type: ['string', 'null'] }, tracked_assets: { type: 'integer' } } },
++        Index: { type: 'object', properties: { ticker: { type: 'string' }, name: { type: 'string' }, is_flagship: { type: 'boolean' }, as_of: { type: ['string', 'null'] }, value: { type: ['number', 'null'] }, published: { type: 'boolean' }, change_1d: { type: ['number', 'null'] }, change_7d: { type: ['number', 'null'] }, change_30d: { type: ['number', 'null'] }, change_ytd: { type: ['number', 'null'] }, change_1y: { type: ['number', 'null'] }, constituents_count: { type: ['integer', 'null'] }, transactions: { type: ['integer', 'null'] }, market_cap_est_usd: { type: ['number', 'null'] }, market_cap_confidence: { type: ['string', 'null'] } } },
++        IndexPoint: { type: 'object', properties: { date: { type: 'string', format: 'date' }, value: { type: 'number' }, constituents_count: { type: 'integer' }, transactions: { type: 'integer' }, volume_usd: { type: ['number', 'null'] } } },
++        Market: { type: 'object', properties: { slug: { type: 'string' }, name: { type: 'string' }, as_of: { type: ['string', 'null'] }, index_value: { type: ['number', 'null'] }, tracked_assets: { type: ['integer', 'null'] }, sales: { type: ['integer', 'null'] }, volume_usd: { type: ['number', 'null'] }, change_30d: { type: ['number', 'null'] } } },
++        MarketDetail: { type: 'object', properties: { category: { type: 'object' }, snapshot: { type: ['object', 'null'] }, counts: { type: 'object' }, gainers: { type: 'array', items: { $ref: '#/components/schemas/AssetSummary' } }, losers: { type: 'array', items: { $ref: '#/components/schemas/AssetSummary' } }, most_valuable: { type: 'array', items: { $ref: '#/components/schemas/AssetSummary' } }, most_liquid: { type: 'array', items: { $ref: '#/components/schemas/AssetSummary' } }, recent_sales: { type: 'array', items: { type: 'object' } }, history: { type: 'array', items: { type: 'object' } } } },
++        Stats: { type: 'object', properties: { assets: { type: 'integer' }, sales: { type: 'integer' }, listings: { type: 'integer' }, sources: { type: 'integer' }, connectors: { type: 'integer' }, categories: { type: 'integer' } } },
++      },
++    },
++  };
++}

added apps/api/src/plugins/auth.ts +69 −0

++import type { FastifyInstance, FastifyRequest } from 'fastify';
++import fp from 'fastify-plugin';
++import { getDb, apiKeys, apiUsage, eq, sql } from '@rareindex/database';
++import { parseBearer, resolveApiKey, TIERS, type ResolvedKey } from '../lib/keys.js';
++import { problem } from '../lib/envelope.js';
++
++declare module 'fastify' {
++  interface FastifyRequest {
++    apiKey: ResolvedKey | null;
++  }
++}
++
++/**
++ * Bearer API-key authentication. Anonymous requests are allowed on the "public" tier (low limits)
++ * so the docs playground and casual use work; keyed requests get their tier's limits and usage counters.
++ */
++export const authPlugin = fp(async (app: FastifyInstance) => {
++  app.decorateRequest('apiKey', null);
++  const cache = new Map<string, { key: ResolvedKey | null; at: number }>();
++
++  app.addHook('onRequest', async (req, reply) => {
++    const bearer = parseBearer(req.headers.authorization);
++    if (!bearer) {
++      req.apiKey = null;
++      return;
++    }
++    const hit = cache.get(bearer);
++    let resolved: ResolvedKey | null;
++    if (hit && Date.now() - hit.at < 60_000) resolved = hit.key;
++    else {
++      resolved = await resolveApiKey(bearer);
++      cache.set(bearer, { key: resolved, at: Date.now() });
++      if (cache.size > 5000) cache.clear();
++    }
++    if (!resolved) return problem(reply, 401, 'Invalid API key', 'The provided API key is unknown or revoked.');
++    req.apiKey = resolved;
++    // daily quota
++    const today = new Date().toISOString().slice(0, 10);
++    const [row] = await getDb().execute(sql`select coalesce(sum(count),0)::int as used from api_usage where key_id = ${resolved.id} and date = ${today}`);
++    const used = Number((row as { used?: number } | undefined)?.used ?? 0);
++    if (used >= resolved.dailyQuota) return problem(reply, 429, 'Daily quota exceeded', `Tier ${resolved.tier} allows ${resolved.dailyQuota} requests per day.`, { quota: resolved.dailyQuota, used });
++  });
++
++  app.addHook('onResponse', async (req, reply) => {
++    if (!req.apiKey || !req.routeOptions.url?.startsWith('/v1')) return;
++    const today = new Date().toISOString().slice(0, 10);
++    const endpoint = req.routeOptions.url;
++    const latency = reply.elapsedTime;
++    try {
++      await getDb()
++        .insert(apiUsage)
++        .values({ keyId: req.apiKey.id, date: today, endpoint, count: 1, latencyMsAvg: latency })
++        .onConflictDoUpdate({
++          target: [apiUsage.keyId, apiUsage.date, apiUsage.endpoint],
++          set: { count: sql`${apiUsage.count} + 1`, latencyMsAvg: sql`(${apiUsage.latencyMsAvg} * ${apiUsage.count} + ${latency}) / (${apiUsage.count} + 1)` },
++        });
++      await getDb().update(apiKeys).set({ lastUsedAt: new Date() }).where(eq(apiKeys.id, req.apiKey.id));
++    } catch (err) {
++      req.log.warn({ err }, 'api usage counter failed');
++    }
++  });
++});
++
++export function rateLimitFor(req: FastifyRequest): number {
++  return req.apiKey?.rateLimitPerMinute ?? TIERS.public.rateLimitPerMinute;
++}
++export function rateKeyFor(req: FastifyRequest): string {
++  return req.apiKey ? `key:${req.apiKey.id}` : `ip:${req.ip}`;
++}

added apps/api/src/routes/internal.ts +42 −0

++import type { FastifyInstance } from 'fastify';
++import { z } from 'zod';
++import { createHmac } from 'node:crypto';
++import { connectorsForUrl, createRouter, createCrawlContext, listConnectorMeta } from '@rareindex/connectors';
++import type { NormalizedRecord } from '@rareindex/shared';
++import { problem } from '../lib/envelope.js';
++
++/**
++ * Internal (loopback-only) endpoints used by the web app. Connector modules are loaded dynamically
++ * (`import(path)`), which the Next.js bundler cannot do — so URL lookups run here, where tsx can.
++ * Auth: `x-internal-token` = HMAC(SESSION_SECRET, "internal") — both processes share SESSION_SECRET.
++ */
++export function internalToken(secret = process.env.SESSION_SECRET ?? 'dev-only'): string {
++  return createHmac('sha256', secret).update('internal').digest('hex');
++}
++
++export async function internalRoutes(app: FastifyInstance) {
++  app.addHook('onRequest', async (req, reply) => {
++    if (!req.url.startsWith('/internal/')) return;
++    const loopback = req.ip === '127.0.0.1' || req.ip === '::1' || req.ip === '::ffff:127.0.0.1';
++    const ok = loopback && req.headers['x-internal-token'] === internalToken();
++    if (!ok) return problem(reply, 403, 'Forbidden', 'Internal endpoint');
++  });
++
++  app.get('/internal/lookup/sources', async () => ({ data: listConnectorMeta({ enabled: true }).filter((m) => m.supportsLookup).map((m) => ({ id: m.id, sourceName: m.sourceName, sourceUrl: m.sourceUrl })) }));
++
++  app.post('/internal/lookup', async (req, reply) => {
++    const body = z.object({ url: z.string().url().max(2000) }).safeParse(req.body);
++    if (!body.success) return problem(reply, 400, 'Invalid body', 'url required');
++    const url = body.data.url;
++    const connectors = await connectorsForUrl(url);
++    const connector = connectors[0];
++    if (!connector?.lookup) return { data: null, meta: { supported: false } };
++    const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
++    const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: 1 } });
++    const started = Date.now();
++    const raws = await connector.lookup(url, ctx);
++    const records: NormalizedRecord[] = [];
++    for (const raw of raws) records.push(...(await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() })));
++    return { data: { connectorId: connector.meta.id, sourceId: connector.meta.sourceId, records }, meta: { supported: true, durationMs: Date.now() - started, engineStats: ctx.engineStats } };
++  });
++}

added apps/api/src/routes/v1.ts +129 −0

++import type { FastifyInstance } from 'fastify';
++import { z } from 'zod';
++import { decodeCursor, encodeCursor, envelope, problem, sendList } from '../lib/envelope.js';
++import * as q from '../lib/queries.js';
++import { openapiDocument } from '../openapi.js';
++
++const Paging = z.object({ limit: z.coerce.number().int().min(1).max(200).default(50), cursor: z.string().optional(), format: z.enum(['json', 'csv']).optional() });
++const Range = z.object({ from: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(), to: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).optional(), format: z.enum(['json', 'csv']).optional() });
++
++function parse<T>(schema: z.ZodType<T>, input: unknown): { ok: true; data: T } | { ok: false; error: string } {
++  const r = schema.safeParse(input);
++  return r.success ? { ok: true, data: r.data } : { ok: false, error: r.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ') };
++}
++
++export async function v1Routes(app: FastifyInstance) {
++  app.get('/v1/openapi.json', async () => openapiDocument());
++
++  app.get('/v1/stats', async () => envelope(await q.platformStats()));
++
++  app.get('/v1/assets/search', async (req, reply) => {
++    const p = parse(Paging.extend({ q: z.string().max(200).optional(), category: z.string().max(80).optional() }), req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const offset = decodeCursor(p.data.cursor);
++    const rows = await q.searchAssets({ q: p.data.q, category: p.data.category, limit: p.data.limit + 1, offset });
++    const page = rows.slice(0, p.data.limit);
++    return sendList(reply, page, { count: page.length, cursor: rows.length > p.data.limit ? encodeCursor(offset + p.data.limit) : null, query: p.data.q ?? null }, req.query as Record<string, unknown>);
++  });
++
++  app.get<{ Params: { id: string } }>('/v1/assets/:id', async (req, reply) => {
++    const asset = await q.getAsset(req.params.id);
++    if (!asset) return problem(reply, 404, 'Asset not found');
++    reply.header('cache-control', 'public, max-age=60, s-maxage=300');
++    return envelope(asset);
++  });
++
++  app.get<{ Params: { id: string } }>('/v1/assets/:id/sales', async (req, reply) => {
++    const p = parse(Paging.extend({ variant: z.string().optional(), include_flagged: z.coerce.boolean().optional() }), req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const asset = await q.getAsset(req.params.id);
++    if (!asset) return problem(reply, 404, 'Asset not found');
++    const offset = decodeCursor(p.data.cursor);
++    const rows = await q.assetSales(asset.id as string, { limit: p.data.limit + 1, offset, variantId: p.data.variant, includeFlagged: p.data.include_flagged });
++    const page = rows.slice(0, p.data.limit);
++    return sendList(reply, page, { count: page.length, cursor: rows.length > p.data.limit ? encodeCursor(offset + p.data.limit) : null, asset_id: asset.id }, req.query as Record<string, unknown>);
++  });
++
++  app.get<{ Params: { id: string } }>('/v1/assets/:id/listings', async (req, reply) => {
++    const p = parse(Paging.extend({ availability: z.enum(['available', 'sold', 'ended', 'removed']).optional() }), req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const asset = await q.getAsset(req.params.id);
++    if (!asset) return problem(reply, 404, 'Asset not found');
++    const offset = decodeCursor(p.data.cursor);
++    const rows = await q.assetListings(asset.id as string, { limit: p.data.limit + 1, offset, availability: p.data.availability });
++    const page = rows.slice(0, p.data.limit);
++    return sendList(reply, page, { count: page.length, cursor: rows.length > p.data.limit ? encodeCursor(offset + p.data.limit) : null, asset_id: asset.id, note: 'Listing prices are asks, not confirmed transactions.' }, req.query as Record<string, unknown>);
++  });
++
++  app.get<{ Params: { id: string } }>('/v1/assets/:id/history', async (req, reply) => {
++    const p = parse(Range.extend({ variant: z.string().optional() }), req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const asset = await q.getAsset(req.params.id);
++    if (!asset) return problem(reply, 404, 'Asset not found');
++    const rows = await q.assetHistory(asset.id as string, { variantId: p.data.variant, from: p.data.from, to: p.data.to });
++    reply.header('cache-control', 'public, max-age=300');
++    return sendList(reply, rows, { count: rows.length, asset_id: asset.id, variant_id: p.data.variant ?? null }, req.query as Record<string, unknown>);
++  });
++
++  app.get('/v1/categories', async (req, reply) => {
++    const rows = await q.listCategories();
++    reply.header('cache-control', 'public, max-age=3600');
++    return sendList(reply, rows, { count: rows.length }, req.query as Record<string, unknown>);
++  });
++
++  app.get('/v1/indices', async (req, reply) => {
++    const rows = await q.listIndices();
++    const shaped = rows.map((r) => {
++      const v = r.value as number | null;
++      const pct = (prev: unknown) => (v !== null && typeof prev === 'number' && prev > 0 ? (v - prev) / prev : null);
++      return { ...r, change_1d: pct(r.value_1d), change_7d: pct(r.value_7d), change_30d: pct(r.value_30d), change_1y: pct(r.value_1y), change_ytd: pct(r.value_ytd), published: v !== null };
++    });
++    reply.header('cache-control', 'public, max-age=300');
++    return sendList(reply, shaped, { count: shaped.length, note: 'An index is published only once it has enough priced constituents; unpublished indices report value=null.' }, req.query as Record<string, unknown>);
++  });
++
++  app.get<{ Params: { ticker: string } }>('/v1/indices/:ticker/history', async (req, reply) => {
++    const p = parse(Range, req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const rows = await q.indexHistory(req.params.ticker.toUpperCase(), p.data);
++    reply.header('cache-control', 'public, max-age=300');
++    return sendList(reply, rows, { count: rows.length, ticker: req.params.ticker.toUpperCase() }, req.query as Record<string, unknown>);
++  });
++
++  app.get('/v1/markets', async (req, reply) => {
++    const rows = await q.listMarkets();
++    reply.header('cache-control', 'public, max-age=300');
++    return sendList(reply, rows, { count: rows.length }, req.query as Record<string, unknown>);
++  });
++
++  app.get<{ Params: { slug: string } }>('/v1/markets/:slug', async (req, reply) => {
++    const m = await q.getMarket(req.params.slug);
++    if (!m) return problem(reply, 404, 'Market not found');
++    reply.header('cache-control', 'public, max-age=300');
++    return envelope(m);
++  });
++
++  app.get('/v1/trending', async (req, reply) => {
++    const p = parse(Paging.extend({ category: z.string().optional() }), req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const offset = decodeCursor(p.data.cursor);
++    const rows = await q.trending({ category: p.data.category, limit: p.data.limit + 1, offset });
++    const page = rows.slice(0, p.data.limit);
++    return sendList(reply, page, { count: page.length, cursor: rows.length > p.data.limit ? encodeCursor(offset + p.data.limit) : null }, req.query as Record<string, unknown>);
++  });
++
++  app.get('/v1/sales/latest', async (req, reply) => {
++    const p = parse(Paging.extend({ category: z.string().optional(), min_usd: z.coerce.number().nonnegative().optional() }), req.query);
++    if (!p.ok) return problem(reply, 400, 'Invalid query', p.error);
++    const offset = decodeCursor(p.data.cursor);
++    const rows = await q.latestSales({ category: p.data.category, minUsd: p.data.min_usd, limit: p.data.limit + 1, offset });
++    const page = rows.slice(0, p.data.limit);
++    return sendList(reply, page, { count: page.length, cursor: rows.length > p.data.limit ? encodeCursor(offset + p.data.limit) : null }, req.query as Record<string, unknown>);
++  });
++
++  app.get('/v1/records', async (req, reply) => {
++    const rows = await q.recordSales();
++    reply.header('cache-control', 'public, max-age=600');
++    return sendList(reply, rows, { count: rows.length, note: 'Highest verified transaction per family; bundles and low-confidence records excluded.' }, req.query as Record<string, unknown>);
++  });
++}

added apps/api/src/server.ts +20 −0

++import { buildApp } from './app.js';
++
++const port = Number(process.env.API_PORT ?? 8211);
++const host = process.env.API_HOST ?? '127.0.0.1';
++
++const app = await buildApp();
++try {
++  await app.listen({ port, host });
++  app.log.info({ port, host }, 'rareindex-api listening');
++} catch (err) {
++  app.log.error(err);
++  process.exit(1);
++}
++
++for (const sig of ['SIGINT', 'SIGTERM'] as const) {
++  process.on(sig, async () => {
++    await app.close();
++    process.exit(0);
++  });
++}

added apps/api/tsconfig.json +8 −0

++{
++  "extends": "../../tsconfig.base.json",
++  "compilerOptions": {
++    "rootDir": ".",
++    "noEmit": true
++  },
++  "include": ["src", "scripts"]
++}

added apps/api/vitest.config.ts +5 −0

++import { defineConfig } from 'vitest/config';
++
++export default defineConfig({
++  test: { include: ['src/**/*.test.ts'], testTimeout: 30_000 },
++});

modified apps/web/next.config.ts +7 −2

				`@@ -4,8 +4,13 @@ const nextConfig: NextConfig = {`
4	reactStrictMode: true,	4	reactStrictMode: true,
5	poweredByHeader: false,	5	poweredByHeader: false,
6	// Workspace packages are consumed as TypeScript sources.	6	// Workspace packages are consumed as TypeScript sources.
7	− transpilePackages: ['@rareindex/shared', '@rareindex/database', '@rareindex/taxonomy', '@rareindex/connectors'],	7	+ transpilePackages: ['@rareindex/shared', '@rareindex/database', '@rareindex/taxonomy', '@rareindex/connectors', '@rareindex/ai'],
8	− serverExternalPackages: ['postgres', 'pino', 'cheerio'],	8	+ serverExternalPackages: ['postgres', 'pino', 'cheerio', 'pg-boss', '@anthropic-ai/sdk', 'openai'],
		9	+ // Public API: /api/v1/* is proxied to the Fastify service (apps/api) so one public host serves web + API.
		10	+ async rewrites() {
		11	+ const api = process.env.RI_API_URL ?? `http://127.0.0.1:${process.env.API_PORT ?? 8211}`;
		12	+ return [{ source: '/api/v1/:path', destination: `${api}/v1/:path` }];
		13	+ },
9	images: {	14	images: {
10	remotePatterns: [{ protocol: 'https', hostname: '**' }],	15	remotePatterns: [{ protocol: 'https', hostname: '**' }],
11	formats: ['image/avif', 'image/webp'],	16	formats: ['image/avif', 'image/webp'],

modified apps/web/package.json +2 −0

				`@@ -11,6 +11,7 @@`
11	"test": "vitest run --passWithNoTests"	11	"test": "vitest run --passWithNoTests"
12	},	12	},
13	"dependencies": {	13	"dependencies": {
		14	+ "@rareindex/ai": "workspace:*",
14	"@rareindex/connectors": "workspace:*",	15	"@rareindex/connectors": "workspace:*",
15	"@rareindex/database": "workspace:*",	16	"@rareindex/database": "workspace:*",
16	"@rareindex/shared": "workspace:*",	17	"@rareindex/shared": "workspace:*",
				`@@ -18,6 +19,7 @@`
18	"@tanstack/react-query": "^5.90.0",	19	"@tanstack/react-query": "^5.90.0",
19	"lucide-react": "^1.0.0",	20	"lucide-react": "^1.0.0",
20	"next": "16.3.4",	21	"next": "16.3.4",
		22	+ "pg-boss": "^12.30.0",
21	"react": "19.2.8",	23	"react": "19.2.8",
22	"react-dom": "19.2.8",	24	"react-dom": "19.2.8",
23	"server-only": "^0.0.1",	25	"server-only": "^0.0.1",

added apps/web/src/app/admin/audit/page.tsx +35 −0

++import Link from 'next/link';
++import { requireAdmin } from '@/lib/admin/auth';
++import { auditLog } from '@/lib/admin/queries';
++import { AdminShell, fmtTs } from '@/components/admin/shell';
++import { Table, th, td } from '@/components/ui/primitives';
++
++const TYPES = ['', 'sale', 'listing', 'asset', 'connector', 'normalized_record', 'taxonomy_proposal'];
++
++export default async function AuditPage({ searchParams }: { searchParams: Promise<{ type?: string }> }) {
++  await requireAdmin();
++  const sp = await searchParams;
++  const rows = await auditLog({ entityType: sp.type || undefined });
++  return (
++    <AdminShell current="/admin/audit" title="Audit log" subtitle="Every flag, exclusion, restore, merge, edit and admin decision — the platform never silently deletes data" actions={<div className="flex flex-wrap gap-1 text-xs">{TYPES.map((t) => <Link key={t} href={`/admin/audit${t ? `?type=${t}` : ''}`} className={`rounded-md border px-2 py-1 ${(sp.type ?? '') === t ? 'border-accent bg-accent text-accent-fg' : 'border-border hover:bg-inset'}`}>{t || 'all'}</Link>)}</div>}>
++      <div className="card">
++        <Table>
++          <thead><tr><th className={th}>When</th><th className={th}>Entity</th><th className={th}>Action</th><th className={th}>Actor</th><th className={th}>Reason</th><th className={th}>Details</th></tr></thead>
++          <tbody>
++            {rows.length === 0 ? <tr><td className={td} colSpan={6}><span className="text-subtle">No audit entries.</span></td></tr> : null}
++            {rows.map((r) => (
++              <tr key={String(r.id)} className="align-top">
++                <td className={td}>{fmtTs(r.created_at)}</td>
++                <td className={`${td} font-mono text-[11px]`}>{String(r.entity_type)}<br />{String(r.entity_id)}</td>
++                <td className={td}>{String(r.action)}</td>
++                <td className={td}>{String(r.actor)}</td>
++                <td className={`${td} max-w-[320px] whitespace-normal`}>{String(r.reason)}</td>
++                <td className={`${td} max-w-[320px] whitespace-normal font-mono text-[10px] text-muted`}>{JSON.stringify(r.details)}</td>
++              </tr>
++            ))}
++          </tbody>
++        </Table>
++      </div>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/connectors/[id]/page.tsx +133 −0

++import Link from 'next/link';
++import { notFound } from 'next/navigation';
++import { requireAdmin } from '@/lib/admin/auth';
++import { connectorDetail } from '@/lib/admin/queries';
++import { connectorAction, updateConnectorConfig } from '@/lib/admin/actions';
++import { AdminShell, ActionButton, StatusPill, Kpi, fmtTs, n } from '@/components/admin/shell';
++import { Table, th, td, tdNum } from '@/components/ui/primitives';
++
++export default async function ConnectorInspectPage({ params }: { params: Promise<{ id: string }> }) {
++  await requireAdmin();
++  const { id } = await params;
++  const d = await connectorDetail(id);
++  if (!d) notFound();
++  const c = d.connector;
++  const health = (d.health?.health ?? {}) as Record<string, unknown>;
++  const outputs = (d.outputs ?? {}) as Record<string, unknown>;
++  return (
++    <AdminShell
++      current="/admin/connectors"
++      title={String(c.display_name)}
++      subtitle={<>{String(c.id)} · source <span className="font-mono">{String(c.source_id)}</span> · {(c.engine_priority as string[]).join(' → ')} · categories {(c.categories as string[]).join(', ')} · <Link href="/admin/connectors" className="underline">back</Link></>}
++      actions={
++        <div className="flex flex-wrap gap-1">
++          {(['run', 'probe', 'recrawl', 'retry', c.status === 'paused' ? 'resume' : 'pause'] as const).map((a) => (
++            <form key={a} action={connectorAction}>
++              <input type="hidden" name="id" value={String(c.id)} />
++              <input type="hidden" name="action" value={a} />
++              <ActionButton label={a} small={false} tone={a === 'pause' ? 'danger' : a === 'resume' ? 'primary' : 'neutral'} />
++            </form>
++          ))}
++        </div>
++      }
++    >
++      <div className="grid grid-cols-2 gap-3 md:grid-cols-4 xl:grid-cols-6">
++        <Kpi label="Status" value={<StatusPill status={String(c.status)} />} sub={<>health <StatusPill status={(d.health?.status as string) ?? 'unknown'} /></>} />
++        <Kpi label="Success 24h" value={health.success_rate_24h != null ? `${Math.round(Number(health.success_rate_24h) * 100)}%` : '—'} sub={`${n(health.pages_success)}/${n(health.pages_attempted)} pages`} />
++        <Kpi label="Firecrawl / Scrapfly" value={`${health.firecrawl_success_rate != null ? Math.round(Number(health.firecrawl_success_rate) * 100) + '%' : '—'} / ${health.scrapfly_fallback_rate != null ? Math.round(Number(health.scrapfly_fallback_rate) * 100) + '%' : '—'}`} sub="success / fallback share" />
++        <Kpi label="Parse failures" value={health.parse_failure_rate != null ? `${(Number(health.parse_failure_rate) * 100).toFixed(1)}%` : '—'} sub={`${n(health.duplicates_24h)} duplicates 24h`} />
++        <Kpi label="Outputs" value={`${n(outputs.sales)} sales`} sub={`${n(outputs.listings)} listings · ${n(outputs.observations)} observations`} />
++        <Kpi label="Last success" value={fmtTs(c.last_success_at).slice(0, 16)} sub={`next ${fmtTs(c.next_run_at).slice(0, 16)}`} />
++      </div>
++
++      {d.anomalies.length ? (
++        <section className="card mt-6">
++          <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold text-alert">Anomalies (recent runs)</h2></header>
++          <ul className="divide-y divide-border px-4 text-xs">{d.anomalies.map((a, i) => <li key={i} className="py-1.5 font-mono">{a}</li>)}</ul>
++        </section>
++      ) : null}
++
++      <section className="card mt-6">
++        <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Runs</h2></header>
++        <Table>
++          <thead><tr><th className={th}>Started</th><th className={th}>Status</th><th className={th}>Trigger</th><th className={`${th} text-right`}>Pages</th><th className={`${th} text-right`}>Raw</th><th className={`${th} text-right`}>Norm.</th><th className={`${th} text-right`}>Dup.</th><th className={`${th} text-right`}>Rej.</th><th className={`${th} text-right`}>Credits</th><th className={th}>Engines</th><th className={th}>Error</th></tr></thead>
++          <tbody>
++            {d.runs.length === 0 ? <tr><td className={td} colSpan={11}><span className="text-subtle">No runs yet.</span></td></tr> : null}
++            {d.runs.map((r) => (
++              <tr key={String(r.id)}>
++                <td className={td}>{fmtTs(r.started_at)}<div className="text-[10px] text-subtle">{r.finished_at ? `${Math.round((new Date(String(r.finished_at)).getTime() - new Date(String(r.started_at)).getTime()) / 1000)} s` : 'running'}</div></td>
++                <td className={td}><StatusPill status={String(r.status)} /></td>
++                <td className={td}>{String(r.trigger)}</td>
++                <td className={tdNum}>{n(r.pages_success)}/{n(r.pages_attempted)}</td>
++                <td className={tdNum}>{n(r.records_raw)}</td>
++                <td className={tdNum}>{n(r.records_normalized)}</td>
++                <td className={tdNum}>{n(r.records_duplicate)}</td>
++                <td className={tdNum}>{n(r.records_rejected)}</td>
++                <td className={tdNum}>{n(r.cost_credits, 1)}</td>
++                <td className={`${td} font-mono text-[10px]`}>{Object.entries((r.engine_stats ?? {}) as Record<string, { attempts: number; success: number }>).map(([k, v]) => `${k} ${v.success}/${v.attempts}`).join(' · ')}</td>
++                <td className={`${td} max-w-[260px] truncate text-loss`} title={String(r.error ?? '')}>{r.error ? String(r.error) : ''}</td>
++              </tr>
++            ))}
++          </tbody>
++        </Table>
++      </section>
++
++      <div className="mt-6 grid gap-6 lg:grid-cols-2">
++        <section className="card">
++          <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Raw records (latest)</h2></header>
++          <ul className="divide-y divide-border text-xs">
++            {d.rawSample.length === 0 ? <li className="px-4 py-3 text-subtle">None yet.</li> : null}
++            {d.rawSample.map((r) => (
++              <li key={String(r.id)} className="px-4 py-2">
++                <div className="flex items-center justify-between gap-2"><span className="font-mono text-[11px]">{String(r.kind)} · {String(r.engine)} · {String(r.http_status ?? '')}</span><span className="text-subtle">{fmtTs(r.fetched_at)}</span></div>
++                <a href={String(r.url)} target="_blank" rel="noreferrer noopener" className="block truncate text-muted hover:text-fg">{String(r.url)}</a>
++                {r.process_error ? <p className="text-loss">{String(r.process_error)}</p> : <p className="text-[10px] text-subtle">{r.processed_at ? `processed ${fmtTs(r.processed_at)}` : 'unprocessed'}</p>}
++              </li>
++            ))}
++          </ul>
++        </section>
++        <section className="card">
++          <header className="flex items-center justify-between border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Normalized records (latest)</h2><span className="text-[11px] text-muted">{d.normalized.map((x) => `${x.status} ${n(x.n)}`).join(' · ')}</span></header>
++          <ul className="divide-y divide-border text-xs">
++            {d.normalizedSample.length === 0 ? <li className="px-4 py-3 text-subtle">None yet.</li> : null}
++            {d.normalizedSample.map((r) => (
++              <li key={String(r.id)} className="px-4 py-2">
++                <div className="flex items-center justify-between gap-2"><span className="truncate">{String(r.raw_title ?? '')}</span><StatusPill status={String(r.status)} /></div>
++                <p className="text-[10px] text-subtle">{String(r.kind)} · {r.price ? `${String(r.price)} ${String(r.currency ?? '')}` : ''} · {r.match_method ? `${String(r.match_method)} ${r.match_confidence ? Math.round(Number(r.match_confidence) * 100) + '%' : ''}` : ''} {r.reject_reason ? `· ${String(r.reject_reason)}` : ''}</p>
++              </li>
++            ))}
++          </ul>
++        </section>
++      </div>
++
++      <div className="mt-6 grid gap-6 lg:grid-cols-2">
++        <section className="card">
++          <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Costs (30d)</h2></header>
++          <Table>
++            <thead><tr><th className={th}>Day</th><th className={th}>Kind</th><th className={`${th} text-right`}>Credits</th><th className={`${th} text-right`}>USD est.</th></tr></thead>
++            <tbody>
++              {d.costs.length === 0 ? <tr><td className={td} colSpan={4}><span className="text-subtle">No cost events.</span></td></tr> : null}
++              {d.costs.map((r, i) => <tr key={i}><td className={td}>{String(r.day)}</td><td className={td}>{String(r.kind)}</td><td className={tdNum}>{n(r.credits, 1)}</td><td className={tdNum}>${n(r.usd, 3)}</td></tr>)}
++            </tbody>
++          </Table>
++        </section>
++        <section className="card">
++          <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Mapping & schedule</h2></header>
++          <form action={updateConnectorConfig} className="space-y-3 p-4 text-xs">
++            <input type="hidden" name="id" value={String(c.id)} />
++            <div className="grid grid-cols-2 gap-3">
++              <label className="block text-muted">Refresh (minutes)<input name="refresh" type="number" min={5} defaultValue={Number(c.refresh_frequency_minutes)} className="mt-1 w-full rounded-md border border-border bg-sunken px-2 py-1.5 text-sm" /></label>
++              <label className="block text-muted">Priority
++                <select name="priority" defaultValue={String(c.priority)} className="mt-1 w-full rounded-md border border-border bg-sunken px-2 py-1.5 text-sm"><option value="high">high</option><option value="medium">medium</option><option value="low">low</option></select>
++              </label>
++            </div>
++            <label className="block text-muted">Config JSON (seeds, mappings, selectors — connector-specific)
++              <textarea name="config" rows={10} defaultValue={JSON.stringify(c.config ?? {}, null, 2)} className="mt-1 w-full rounded-md border border-border bg-sunken px-2 py-1.5 font-mono text-[11px]" />
++            </label>
++            <div className="flex items-center justify-between"><span className="text-subtle">schema v{String(c.schema_version)} · connector v{String(c.connector_version)} · source trust {d.source ? Math.round(Number(d.source.trust_score) * 100) + '%' : '—'}</span><ActionButton label="Save mapping" small={false} tone="primary" /></div>
++          </form>
++        </section>
++      </div>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/connectors/page.tsx +86 −0

++import Link from 'next/link';
++import { requireAdmin } from '@/lib/admin/auth';
++import { connectorsOverview } from '@/lib/admin/queries';
++import { connectorAction } from '@/lib/admin/actions';
++import { AdminShell, ActionButton, StatusPill, fmtTs, n } from '@/components/admin/shell';
++import { Table, th, td, tdNum } from '@/components/ui/primitives';
++
++function pct(v: unknown): string {
++  const x = Number(v);
++  return Number.isFinite(x) ? `${Math.round(x * 100)}%` : '—';
++}
++
++export default async function ConnectorsPage() {
++  await requireAdmin();
++  const rows = await connectorsOverview();
++  return (
++    <AdminShell current="/admin/connectors" title="Connector control center" subtitle={`${rows.length} connectors registered · actions enqueue jobs on the crawl.run queue consumed by the worker`}>
++      <div className="card">
++        <Table>
++          <thead>
++            <tr>
++              <th className={th}>Connector</th>
++              <th className={th}>Health</th>
++              <th className={th}>Status</th>
++              <th className={`${th} text-right`}>Pages/min</th>
++              <th className={`${th} text-right`}>Errors</th>
++              <th className={th}>Last crawl</th>
++              <th className={`${th} text-right`}>Records 24h</th>
++              <th className={`${th} text-right`}>Dup.</th>
++              <th className={`${th} text-right`}>FC rate</th>
++              <th className={`${th} text-right`}>SF rate</th>
++              <th className={`${th} text-right`}>Parser conf.</th>
++              <th className={th}>Schema</th>
++              <th className={th}>Actions</th>
++            </tr>
++          </thead>
++          <tbody>
++            {rows.length === 0 ? <tr><td className={td} colSpan={13}><span className="text-subtle">No connectors in the registry yet. Add meta.json files under connectors/ and run pnpm db:seed.</span></td></tr> : null}
++            {rows.map((r) => {
++              const health = (r.health ?? {}) as Record<string, unknown>;
++              const anomalies = Array.isArray(r.last_anomalies) ? (r.last_anomalies as string[]) : [];
++              const drift = anomalies.some((a) => /schema|selector|missing_field|redesign/.test(a));
++              return (
++                <tr key={String(r.id)} className="align-top">
++                  <td className={td}>
++                    <Link href={`/admin/connectors/${String(r.id)}`} className="font-medium hover:underline">{String(r.display_name)}</Link>
++                    <div className="text-[11px] text-subtle">{String(r.id)} · {(r.engine_priority as string[]).join('→')} · {String(r.priority)} · every {n(r.refresh_frequency_minutes)} min</div>
++                  </td>
++                  <td className={td}><StatusPill status={(r.health_status as string) ?? 'unknown'} />{r.health_at ? <div className="text-[10px] text-subtle">{fmtTs(r.health_at)}</div> : null}</td>
++                  <td className={td}><StatusPill status={String(r.status)} />{r.last_run_status ? <div className="text-[10px] text-subtle">run: {String(r.last_run_status)}</div> : null}</td>
++                  <td className={tdNum}>{n(r.pages_per_min, 1)}</td>
++                  <td className={`${tdNum} ${r.last_error ? 'text-loss' : ''}`} title={String(r.last_error ?? '')}>{r.last_error ? '1' : '0'}{anomalies.length ? <div className="text-[10px] text-alert">{anomalies.length} anomal.</div> : null}</td>
++                  <td className={td}>{fmtTs(r.last_started_at)}<div className="text-[10px] text-subtle">ok: {fmtTs(r.last_success_at)}</div></td>
++                  <td className={tdNum}>{n(r.records_24h)}<div className="text-[10px] text-subtle">{n(r.normalized_24h)} norm.</div></td>
++                  <td className={tdNum}>{n(r.duplicates_24h)}</td>
++                  <td className={tdNum}>{pct(health.firecrawl_success_rate)}</td>
++                  <td className={tdNum}>{pct(health.scrapfly_fallback_rate)}</td>
++                  <td className={tdNum}>{pct(r.parser_confidence)}</td>
++                  <td className={td}>v{String(r.schema_version)}{drift ? <div className="text-[10px] text-alert">drift?</div> : null}</td>
++                  <td className={td}>
++                    <div className="flex flex-wrap gap-1">
++                      {(['run', 'probe', 'recrawl', 'retry'] as const).map((a) => (
++                        <form key={a} action={connectorAction}>
++                          <input type="hidden" name="id" value={String(r.id)} />
++                          <input type="hidden" name="action" value={a} />
++                          <ActionButton label={a} />
++                        </form>
++                      ))}
++                      <form action={connectorAction}>
++                        <input type="hidden" name="id" value={String(r.id)} />
++                        <input type="hidden" name="action" value={r.status === 'paused' ? 'resume' : 'pause'} />
++                        <ActionButton label={r.status === 'paused' ? 'resume' : 'pause'} tone={r.status === 'paused' ? 'primary' : 'danger'} />
++                      </form>
++                      <Link href={`/admin/connectors/${String(r.id)}`} className="rounded-md border border-border px-2 py-0.5 text-[11px] font-medium hover:bg-inset">inspect</Link>
++                    </div>
++                  </td>
++                </tr>
++              );
++            })}
++          </tbody>
++        </Table>
++      </div>
++      <p className="mt-3 text-[11px] text-subtle">run = incremental crawl · probe = 25-record smoke test · recrawl = backfill mode · retry = re-run from the last cursor. FC = Firecrawl success rate, SF = share of pages that needed Scrapfly (from the 24 h health snapshot).</p>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/costs/page.tsx +62 −0

++import Link from 'next/link';
++import { requireAdmin } from '@/lib/admin/auth';
++import { costsOverview } from '@/lib/admin/queries';
++import { AdminShell, Bars, Kpi, n } from '@/components/admin/shell';
++import { Table, th, td, tdNum } from '@/components/ui/primitives';
++
++export default async function CostsPage({ searchParams }: { searchParams: Promise<{ days?: string }> }) {
++  await requireAdmin();
++  const sp = await searchParams;
++  const days = [7, 30, 90].includes(Number(sp.days)) ? Number(sp.days) : 30;
++  const c = await costsOverview(days);
++  const t = (c.totals ?? {}) as Record<string, unknown>;
++  const raw = Number((c.perRecord as Record<string, unknown> | null)?.raw_records ?? 0);
++  const dayMap = new Map<string, { usd: number; credits: number }>();
++  for (const r of c.byDay) {
++    const k = String(r.day);
++    const cur = dayMap.get(k) ?? { usd: 0, credits: 0 };
++    cur.usd += Number(r.usd);
++    cur.credits += Number(r.credits);
++    dayMap.set(k, cur);
++  }
++  const series = [...dayMap.entries()].sort(([a], [b]) => (a < b ? -1 : 1)).map(([label, v]) => ({ label, value: v.usd }));
++  const creditSeries = [...dayMap.entries()].sort(([a], [b]) => (a < b ? -1 : 1)).map(([label, v]) => ({ label, value: v.credits }));
++  return (
++    <AdminShell
++      current="/admin/costs"
++      title="Costs"
++      subtitle="Crawler credits and AI spend by day, kind, provider, connector, category and endpoint (estimates from list prices)"
++      actions={<div className="flex gap-1 text-xs">{[7, 30, 90].map((d) => <Link key={d} href={`/admin/costs?days=${d}`} className={`rounded-md border px-2 py-1 ${d === days ? 'border-accent bg-accent text-accent-fg' : 'border-border hover:bg-inset'}`}>{d}d</Link>)}</div>}
++    >
++      <div className="grid grid-cols-2 gap-3 md:grid-cols-5">
++        <Kpi label={`Spend ${days}d`} value={`$${n(t.usd, 2)}`} sub={`${n(t.events)} cost events`} />
++        <Kpi label="AI" value={`$${n(t.ai_usd, 2)}`} />
++        <Kpi label="Firecrawl credits" value={n(t.firecrawl_credits)} />
++        <Kpi label="Scrapfly credits" value={n(t.scrapfly_credits)} />
++        <Kpi label="Per raw record" value={raw ? `$${(Number(t.usd ?? 0) / raw).toFixed(4)}` : '—'} sub={`${n(raw)} raw records`} />
++      </div>
++      <div className="mt-6 grid gap-6 lg:grid-cols-2">
++        <section className="card p-4"><h2 className="mb-2 text-sm font-semibold">USD per day</h2><Bars points={series} format={(v) => `$${v.toFixed(3)}`} /></section>
++        <section className="card p-4"><h2 className="mb-2 text-sm font-semibold">Crawler credits per day</h2><Bars points={creditSeries} format={(v) => `${Math.round(v)} credits`} /></section>
++      </div>
++      <div className="mt-6 grid gap-6 lg:grid-cols-2">
++        <section className="card"><header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">By kind / provider</h2></header>
++          <Table><thead><tr><th className={th}>Kind</th><th className={th}>Provider</th><th className={`${th} text-right`}>Credits</th><th className={`${th} text-right`}>USD</th><th className={`${th} text-right`}>Events</th></tr></thead>
++            <tbody>{c.byKind.length === 0 ? <tr><td className={td} colSpan={5}><span className="text-subtle">No cost events yet.</span></td></tr> : c.byKind.map((r, i) => <tr key={i}><td className={td}>{String(r.kind)}</td><td className={td}>{String(r.provider)}</td><td className={tdNum}>{n(r.credits, 1)}</td><td className={tdNum}>${n(r.usd, 3)}</td><td className={tdNum}>{n(r.events)}</td></tr>)}</tbody></Table>
++        </section>
++        <section className="card"><header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">By connector</h2></header>
++          <Table><thead><tr><th className={th}>Connector</th><th className={`${th} text-right`}>Credits</th><th className={`${th} text-right`}>USD</th><th className={`${th} text-right`}>Events</th></tr></thead>
++            <tbody>{c.byConnector.length === 0 ? <tr><td className={td} colSpan={4}><span className="text-subtle">—</span></td></tr> : c.byConnector.map((r, i) => <tr key={i}><td className={td}><Link href={`/admin/connectors/${String(r.connector_id)}`} className="hover:underline">{String(r.connector_id)}</Link></td><td className={tdNum}>{n(r.credits, 1)}</td><td className={tdNum}>${n(r.usd, 3)}</td><td className={tdNum}>{n(r.events)}</td></tr>)}</tbody></Table>
++        </section>
++        <section className="card"><header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">By category</h2></header>
++          <Table><thead><tr><th className={th}>Category</th><th className={`${th} text-right`}>Credits</th><th className={`${th} text-right`}>USD</th></tr></thead>
++            <tbody>{c.byCategory.length === 0 ? <tr><td className={td} colSpan={3}><span className="text-subtle">—</span></td></tr> : c.byCategory.map((r, i) => <tr key={i}><td className={td}>{String(r.category_slug)}</td><td className={tdNum}>{n(r.credits, 1)}</td><td className={tdNum}>${n(r.usd, 3)}</td></tr>)}</tbody></Table>
++        </section>
++        <section className="card"><header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">AI by endpoint / model</h2></header>
++          <Table><thead><tr><th className={th}>Endpoint</th><th className={th}>Model</th><th className={`${th} text-right`}>Calls</th><th className={`${th} text-right`}>In tok</th><th className={`${th} text-right`}>Out tok</th><th className={`${th} text-right`}>USD</th></tr></thead>
++            <tbody>{c.byEndpoint.length === 0 ? <tr><td className={td} colSpan={6}><span className="text-subtle">—</span></td></tr> : c.byEndpoint.map((r, i) => <tr key={i}><td className={td}>{String(r.endpoint)}</td><td className={`${td} font-mono text-[11px]`}>{String(r.model)}</td><td className={tdNum}>{n(r.events)}</td><td className={tdNum}>{n(r.input_tokens)}</td><td className={tdNum}>{n(r.output_tokens)}</td><td className={tdNum}>${n(r.usd, 4)}</td></tr>)}</tbody></Table>
++        </section>
++      </div>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/data-quality/page.tsx +105 −0

++import Link from 'next/link';
++import { requireAdmin } from '@/lib/admin/auth';
++import { dataQuality, matchCandidatesForRecord } from '@/lib/admin/queries';
++import { manualMatchAction, saleStatusAction } from '@/lib/admin/actions';
++import { AdminShell, ActionButton, Kpi, StatusPill, fmtTs, n } from '@/components/admin/shell';
++import { Table, th, td, tdNum } from '@/components/ui/primitives';
++
++export default async function DataQualityPage({ searchParams }: { searchParams: Promise<{ match?: string }> }) {
++  await requireAdmin();
++  const sp = await searchParams;
++  const dq = await dataQuality();
++  const q = (dq.quality ?? {}) as Record<string, unknown>;
++  const matching = sp.match ? await matchCandidatesForRecord(sp.match) : null;
++  return (
++    <AdminShell current="/admin/data-quality" title="Data quality" subtitle="Flagged/excluded sales with audit reasons, unmatched records with manual matching, cross-listing duplicates. Nothing is deleted; every change is written to the audit log.">
++      <div className="grid grid-cols-2 gap-3 md:grid-cols-4">
++        <Kpi label="Sales data quality (avg)" value={n(q.sales_avg, 1)} sub={`p10 ${n(q.sales_p10, 1)}`} />
++        <Kpi label="Assets data quality (avg)" value={n(q.assets_avg, 1)} />
++        <Kpi label="Low-confidence sales" value={n(q.low_confidence_sales)} sub="confidence < 0.6" />
++        <Kpi label="Flagged / excluded" value={dq.flagged.length} sub="latest 100 shown" />
++      </div>
++
++      {matching?.record ? (
++        <section className="card mt-6 border-index/40">
++          <header className="flex items-center justify-between border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Manual match</h2><Link href="/admin/data-quality" className="text-xs text-muted hover:text-fg">close</Link></header>
++          <div className="p-4 text-sm">
++            <p className="font-medium">{String((matching.record.payload as { rawTitle?: string }).rawTitle ?? matching.record.id)}</p>
++            <ul className="mt-3 divide-y divide-border text-xs">
++              {matching.candidates.length === 0 ? <li className="py-2 text-subtle">No similar asset titles. Reject or leave for the resolver.</li> : null}
++              {matching.candidates.map((c) => (
++                <li key={String(c.id)} className="flex items-center justify-between gap-3 py-1.5">
++                  <span className="truncate"><Link href={`/asset/${String(c.slug)}`} className="hover:underline">{String(c.title)}</Link> <span className="text-subtle">{String(c.category_slug)} · {Math.round(Number(c.score) * 100)}%</span></span>
++                  <form action={manualMatchAction}><input type="hidden" name="recordId" value={String(matching.record!.id)} /><input type="hidden" name="assetId" value={String(c.id)} /><ActionButton label="match" tone="primary" /></form>
++                </li>
++              ))}
++            </ul>
++            <form action={manualMatchAction} className="mt-3 flex items-center gap-2 text-xs"><input type="hidden" name="recordId" value={String(matching.record.id)} /><input name="assetId" placeholder="or paste an asset id (rare_…)" className="flex-1 rounded-md border border-border bg-sunken px-2 py-1" /><ActionButton label="match id" /></form>
++            <form action={manualMatchAction} className="mt-2"><input type="hidden" name="recordId" value={String(matching.record.id)} /><input type="hidden" name="assetId" value="" /><ActionButton label="reject record" tone="danger" /></form>
++          </div>
++        </section>
++      ) : null}
++
++      <section className="card mt-6">
++        <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Flagged & excluded sales</h2></header>
++        <Table>
++          <thead><tr><th className={th}>Asset</th><th className={th}>Status</th><th className={th}>Flags</th><th className={`${th} text-right`}>Price</th><th className={th}>Date</th><th className={th}>Source</th><th className={th}>Last reason</th><th className={th}>Actions</th></tr></thead>
++          <tbody>
++            {dq.flagged.length === 0 ? <tr><td className={td} colSpan={8}><span className="text-subtle">No flagged or excluded sales.</span></td></tr> : null}
++            {dq.flagged.map((s) => (
++              <tr key={String(s.id)}>
++                <td className={`${td} max-w-[260px] truncate`}><Link href={`/asset/${String(s.asset_slug)}`} className="hover:underline">{String(s.title)}</Link><div className="truncate text-[10px] text-subtle">{String(s.raw_title)}</div></td>
++                <td className={td}><StatusPill status={String(s.status)} /></td>
++                <td className={`${td} font-mono text-[10px]`}>{(s.flags as string[]).join(', ')}</td>
++                <td className={tdNum}>{n(s.price, 2)} {String(s.currency)}<div className="text-[10px] text-subtle">${n(s.price_usd)}</div></td>
++                <td className={td}>{fmtTs(s.sale_date).slice(0, 10)}</td>
++                <td className={td}><a href={String(s.source_url)} target="_blank" rel="noreferrer noopener" className="hover:underline">{String(s.source_id)}</a></td>
++                <td className={`${td} max-w-[220px] truncate`} title={String(s.last_reason ?? '')}>{String(s.last_reason ?? '')}</td>
++                <td className={td}>
++                  <div className="flex gap-1">
++                    {(['valid', 'flagged', 'excluded'] as const).filter((x) => x !== s.status).map((x) => (
++                      <form key={x} action={saleStatusAction} className="flex items-center gap-1"><input type="hidden" name="id" value={String(s.id)} /><input type="hidden" name="status" value={x} /><input type="hidden" name="reason" value={`admin review → ${x}`} /><ActionButton label={x === 'valid' ? 'restore' : x} tone={x === 'excluded' ? 'danger' : 'neutral'} /></form>
++                    ))}
++                  </div>
++                </td>
++              </tr>
++            ))}
++          </tbody>
++        </Table>
++      </section>
++
++      <section className="card mt-6">
++        <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Unmatched / rejected normalized records</h2></header>
++        <Table>
++          <thead><tr><th className={th}>Title</th><th className={th}>Connector</th><th className={th}>Kind</th><th className={th}>Category</th><th className={`${th} text-right`}>Price</th><th className={th}>Reason</th><th className={th}>Created</th><th className={th}></th></tr></thead>
++          <tbody>
++            {dq.unmatched.length === 0 ? <tr><td className={td} colSpan={8}><span className="text-subtle">Nothing waiting for a manual decision.</span></td></tr> : null}
++            {dq.unmatched.map((r) => (
++              <tr key={String(r.id)}>
++                <td className={`${td} max-w-[300px] truncate`}><a href={String(r.source_url ?? '#')} target="_blank" rel="noreferrer noopener" className="hover:underline">{String(r.raw_title ?? r.id)}</a></td>
++                <td className={td}>{String(r.connector_id)}</td>
++                <td className={td}>{String(r.kind)}</td>
++                <td className={td}>{String(r.category_slug ?? '—')}</td>
++                <td className={tdNum}>{r.price ? `${n(r.price, 2)} ${String(r.currency ?? '')}` : '—'}</td>
++                <td className={`${td} max-w-[200px] truncate`} title={String(r.reject_reason ?? '')}>{String(r.reject_reason ?? '')}</td>
++                <td className={td}>{fmtTs(r.created_at)}</td>
++                <td className={td}><Link href={`/admin/data-quality?match=${String(r.id)}`} className="rounded-md border border-border px-2 py-0.5 text-[11px] font-medium hover:bg-inset">match</Link></td>
++              </tr>
++            ))}
++          </tbody>
++        </Table>
++      </section>
++
++      <section className="card mt-6">
++        <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Cross-listing groups</h2></header>
++        <Table>
++          <thead><tr><th className={th}>Group</th><th className={`${th} text-right`}>Listings</th><th className={th}>Sources</th><th className={th}>Sample</th></tr></thead>
++          <tbody>
++            {dq.duplicates.length === 0 ? <tr><td className={td} colSpan={4}><span className="text-subtle">No cross-listing groups detected yet.</span></td></tr> : null}
++            {dq.duplicates.map((d) => <tr key={String(d.cross_listing_group_id)}><td className={`${td} font-mono text-[11px]`}>{String(d.cross_listing_group_id)}</td><td className={tdNum}>{n(d.n)}</td><td className={td}>{(d.sources as string[]).join(', ')}</td><td className={`${td} max-w-[320px] truncate`}>{String(d.sample_title)}</td></tr>)}
++          </tbody>
++        </Table>
++      </section>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/events/page.tsx +31 −0

++import Link from 'next/link';
++import { requireAdmin } from '@/lib/admin/auth';
++import { eventsLog } from '@/lib/admin/queries';
++import { AdminShell, fmtTs, n } from '@/components/admin/shell';
++import { Table, th, td } from '@/components/ui/primitives';
++
++export default async function EventsPage({ searchParams }: { searchParams: Promise<{ type?: string }> }) {
++  await requireAdmin();
++  const sp = await searchParams;
++  const { rows, types } = await eventsLog({ type: sp.type || undefined });
++  return (
++    <AdminShell current="/admin/events" title="Domain events" subtitle="page_crawled, listing_created, sale_detected, entity_matched, valuation_updated, index_updated… (last 200)" actions={<div className="flex flex-wrap gap-1 text-xs"><Link href="/admin/events" className={`rounded-md border px-2 py-1 ${!sp.type ? 'border-accent bg-accent text-accent-fg' : 'border-border hover:bg-inset'}`}>all</Link>{types.map((t) => <Link key={String(t.type)} href={`/admin/events?type=${String(t.type)}`} className={`rounded-md border px-2 py-1 ${sp.type === t.type ? 'border-accent bg-accent text-accent-fg' : 'border-border hover:bg-inset'}`}>{String(t.type)} <span className="text-subtle">{n(t.n)}</span></Link>)}</div>}>
++      <div className="card">
++        <Table>
++          <thead><tr><th className={th}>When</th><th className={th}>Type</th><th className={th}>Entity</th><th className={th}>Payload</th></tr></thead>
++          <tbody>
++            {rows.length === 0 ? <tr><td className={td} colSpan={4}><span className="text-subtle">No events recorded yet.</span></td></tr> : null}
++            {rows.map((r) => (
++              <tr key={String(r.id)} className="align-top">
++                <td className={td}>{fmtTs(r.created_at)}</td>
++                <td className={`${td} font-mono text-[11px]`}>{String(r.type)}</td>
++                <td className={`${td} font-mono text-[11px]`}>{String(r.entity_type ?? '')} {String(r.entity_id ?? '')}</td>
++                <td className={`${td} max-w-[520px] whitespace-normal font-mono text-[10px] text-muted`}>{JSON.stringify(r.payload)}</td>
++              </tr>
++            ))}
++          </tbody>
++        </Table>
++      </div>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/layout.tsx +8 −0

++import type { Metadata } from 'next';
++
++export const metadata: Metadata = { title: 'Admin', robots: { index: false, follow: false } };
++export const dynamic = 'force-dynamic';
++
++export default function AdminLayout({ children }: { children: React.ReactNode }) {
++  return <>{children}</>;
++}

added apps/web/src/app/admin/login/page.tsx +20 −0

++import { adminLogin } from '@/lib/admin/actions';
++
++export default async function AdminLoginPage({ searchParams }: { searchParams: Promise<{ error?: string; next?: string }> }) {
++  const sp = await searchParams;
++  return (
++    <div className="mx-auto mt-16 max-w-sm">
++      <div className="card p-6">
++        <p className="text-[10px] font-semibold uppercase tracking-wider text-subtle">Admin</p>
++        <h1 className="mt-1 text-lg font-semibold">Connector control center</h1>
++        <p className="mt-1 text-xs text-muted">Enter the server admin token. The token is exchanged for a signed, httpOnly cookie valid 12 hours.</p>
++        <form action={adminLogin} className="mt-4 space-y-3">
++          <input type="hidden" name="next" value={sp.next ?? '/admin'} />
++          <input name="token" type="password" autoComplete="current-password" required placeholder="ADMIN_TOKEN" className="w-full rounded-md border border-border bg-sunken px-3 py-2 text-sm focus:border-border-strong focus:outline-none" />
++          {sp.error ? <p className="rounded-md bg-loss-bg px-3 py-2 text-xs text-loss">{sp.error}</p> : null}
++          <button type="submit" className="w-full rounded-md bg-accent px-3 py-2 text-sm font-medium text-accent-fg">Sign in</button>
++        </form>
++      </div>
++    </div>
++  );
++}

added apps/web/src/app/admin/page.tsx +98 −0

++import Link from 'next/link';
++import { requireAdmin } from '@/lib/admin/auth';
++import { dashboard } from '@/lib/admin/queries';
++import { queueDepths } from '@/lib/admin/queue';
++import { adminLogout } from '@/lib/admin/actions';
++import { AdminShell, Kpi, StatusPill, fmtTs, n } from '@/components/admin/shell';
++import { Table, th, td, tdNum } from '@/components/ui/primitives';
++import { getRouter } from '@rareindex/ai';
++
++export default async function AdminHome() {
++  await requireAdmin();
++  const [{ counts, recentRuns, recentEvents }, queues] = await Promise.all([dashboard(), queueDepths()]);
++  const c = (counts ?? {}) as Record<string, unknown>;
++  const ai = getRouter();
++  return (
++    <AdminShell
++      current="/admin"
++      title="Pipeline overview"
++      subtitle="Counts, lag and freshness across ingestion, canonical data and analytics"
++      actions={
++        <form action={adminLogout}>
++          <button type="submit" className="rounded-md border border-border px-2.5 py-1 text-xs hover:bg-inset">Sign out</button>
++        </form>
++      }
++    >
++      <div className="grid grid-cols-2 gap-3 md:grid-cols-4 xl:grid-cols-6">
++        <Kpi label="Assets" value={n(c.assets)} sub={`${n(c.variants)} variants`} />
++        <Kpi label="Sales" value={n(c.sales)} sub={`${n(c.sales_flagged)} flagged · ${n(c.sales_excluded)} excluded`} />
++        <Kpi label="Live listings" value={n(c.listings_live)} />
++        <Kpi label="Price observations" value={n(c.observations)} />
++        <Kpi label="Raw records" value={n(c.raw_records)} sub={`${n(c.raw_unprocessed)} unprocessed · ${n(c.raw_errors)} errors`} tone={Number(c.raw_errors) > 0 ? 'alert' : undefined} />
++        <Kpi label="Normalized pending" value={n(c.normalized_pending)} sub={`${n(c.normalized_unmatched)} unmatched`} tone={Number(c.normalized_pending) > 1000 ? 'alert' : undefined} />
++        <Kpi label="Connectors" value={`${n(c.connectors_active)} active`} sub={`${n(c.connectors_paused)} paused · ${n(c.connectors_unhealthy)} unhealthy`} tone={Number(c.connectors_unhealthy) > 0 ? 'loss' : undefined} />
++        <Kpi label="Taxonomy proposals" value={n(c.proposals_pending)} sub="pending review" />
++        <Kpi label="Latest fetch" value={fmtTs(c.latest_fetch).slice(0, 16)} />
++        <Kpi label="Latest sale" value={fmtTs(c.latest_sale).slice(0, 10)} sub={`valuation ${fmtTs(c.latest_valuation).slice(0, 16)}`} />
++        <Kpi label="Latest index date" value={c.latest_index ? String(c.latest_index).slice(0, 10) : '—'} />
++        <Kpi label="Spend 24h" value={`$${n(c.cost_24h_usd, 2)}`} sub={`FC ${n(c.firecrawl_credits_24h)} · SF ${n(c.scrapfly_credits_24h)} credits`} />
++      </div>
++
++      <div className="mt-6 grid gap-6 lg:grid-cols-3">
++        <section className="card lg:col-span-2">
++          <header className="flex items-center justify-between border-b border-border px-4 py-2.5">
++            <h2 className="text-sm font-semibold">Recent connector runs</h2>
++            <Link href="/admin/connectors" className="text-xs text-muted hover:text-fg">Control center →</Link>
++          </header>
++          <Table>
++            <thead><tr><th className={th}>Connector</th><th className={th}>Status</th><th className={th}>Trigger</th><th className={th}>Started</th><th className={`${th} text-right`}>Pages</th><th className={`${th} text-right`}>Raw</th><th className={`${th} text-right`}>Norm.</th><th className={`${th} text-right`}>Dup.</th><th className={th}>Error</th></tr></thead>
++            <tbody>
++              {recentRuns.length === 0 ? <tr><td className={td} colSpan={9}><span className="text-subtle">No runs yet. Trigger one from the control center.</span></td></tr> : null}
++              {recentRuns.map((r) => (
++                <tr key={String(r.id)}>
++                  <td className={td}><Link href={`/admin/connectors/${String(r.connector_id)}`} className="font-medium hover:underline">{String(r.connector_id)}</Link></td>
++                  <td className={td}><StatusPill status={String(r.status)} /></td>
++                  <td className={td}>{String(r.trigger)}</td>
++                  <td className={td}>{fmtTs(r.started_at)}</td>
++                  <td className={tdNum}>{n(r.pages_success)}/{n(r.pages_attempted)}</td>
++                  <td className={tdNum}>{n(r.records_raw)}</td>
++                  <td className={tdNum}>{n(r.records_normalized)}</td>
++                  <td className={tdNum}>{n(r.records_duplicate)}</td>
++                  <td className={`${td} max-w-[240px] truncate text-loss`} title={String(r.error ?? '')}>{r.error ? String(r.error) : ''}</td>
++                </tr>
++              ))}
++            </tbody>
++          </Table>
++        </section>
++        <div className="space-y-6">
++          <section className="card">
++            <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Queues (pg-boss)</h2></header>
++            <ul className="divide-y divide-border text-xs">
++              {queues.length === 0 ? <li className="px-4 py-3 text-subtle">Queue tables not initialised yet (the worker creates them on first start).</li> : null}
++              {queues.map((q) => (
++                <li key={q.name} className="flex items-center justify-between px-4 py-2"><span className="font-mono">{q.name}</span><span className="num text-muted">{q.queued} queued · {q.active} active · {q.failed} failed</span></li>
++              ))}
++            </ul>
++          </section>
++          <section className="card">
++            <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Events (24h)</h2></header>
++            <ul className="divide-y divide-border text-xs">
++              {recentEvents.length === 0 ? <li className="px-4 py-3 text-subtle">No events yet.</li> : null}
++              {recentEvents.map((e) => (
++                <li key={String(e.type)} className="flex items-center justify-between px-4 py-2"><span className="font-mono">{String(e.type)}</span><span className="num text-muted">{n(e.n)}</span></li>
++              ))}
++            </ul>
++          </section>
++          <section className="card">
++            <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">AI model routing</h2></header>
++            <ul className="divide-y divide-border text-xs">
++              {ai.table().map((r) => (
++                <li key={r.role} className="flex items-center justify-between px-4 py-2"><span>{r.role}</span><span className="font-mono text-muted">{r.provider ? `${r.provider} · ${r.model}` : 'not configured'}</span></li>
++              ))}
++            </ul>
++          </section>
++        </div>
++      </div>
++    </AdminShell>
++  );
++}

added apps/web/src/app/admin/taxonomy/page.tsx +61 −0

++import { requireAdmin } from '@/lib/admin/auth';
++import { taxonomyQueue } from '@/lib/admin/queries';
++import { proposeTaxonomyNode, taxonomyDecision } from '@/lib/admin/actions';
++import { AdminShell, ActionButton, StatusPill, fmtTs, n } from '@/components/admin/shell';
++
++export default async function TaxonomyAdminPage() {
++  await requireAdmin();
++  const { pending, decided, families } = await taxonomyQueue();
++  return (
++    <AdminShell current="/admin/taxonomy" title="Taxonomy proposals" subtitle="Category discovery queue (§101): approve to insert a node under the chosen parent; the category becomes available to connectors and pages immediately">
++      <div className="grid gap-6 lg:grid-cols-[1fr_320px]">
++        <section className="card">
++          <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Pending ({pending.length})</h2></header>
++          <ul className="divide-y divide-border">
++            {pending.length === 0 ? <li className="px-4 py-6 text-center text-xs text-subtle">Queue empty. The discovery worker files proposals here when it finds recurring product clusters outside the taxonomy.</li> : null}
++            {pending.map((p) => (
++              <li key={String(p.id)} className="px-4 py-3 text-sm">
++                <div className="flex flex-wrap items-start justify-between gap-3">
++                  <div>
++                    <p className="font-medium">{String(p.name)} <span className="font-mono text-xs text-muted">{String(p.proposed_slug)}</span></p>
++                    <p className="text-xs text-muted">parent {String(p.parent_slug ?? '—')} · est. volume {n(p.volume_estimate)} · {fmtTs(p.created_at)}</p>
++                    <pre className="mt-1 max-h-32 overflow-auto rounded-md bg-sunken p-2 text-[10px] text-muted">{JSON.stringify(p.evidence ?? {}, null, 1)}</pre>
++                  </div>
++                  <div className="flex items-center gap-2">
++                    <form action={taxonomyDecision} className="flex items-center gap-1">
++                      <input type="hidden" name="id" value={String(p.id)} />
++                      <input type="hidden" name="decision" value="approved" />
++                      <select name="parent" defaultValue={String(p.parent_slug ?? '')} className="rounded-md border border-border bg-sunken px-1.5 py-0.5 text-xs">
++                        <option value="">(new family)</option>
++                        {families.map((f) => <option key={String(f.slug)} value={String(f.slug)}>{String(f.name)}</option>)}
++                      </select>
++                      <ActionButton label="approve" tone="primary" />
++                    </form>
++                    <form action={taxonomyDecision}>
++                      <input type="hidden" name="id" value={String(p.id)} />
++                      <input type="hidden" name="decision" value="rejected" />
++                      <ActionButton label="reject" tone="danger" />
++                    </form>
++                  </div>
++                </div>
++              </li>
++            ))}
++          </ul>
++          <header className="border-y border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Decided</h2></header>
++          <ul className="divide-y divide-border text-xs">
++            {decided.map((p) => <li key={String(p.id)} className="flex items-center justify-between px-4 py-2"><span>{String(p.name)} <span className="font-mono text-muted">{String(p.proposed_slug)}</span></span><span className="flex items-center gap-2 text-subtle"><StatusPill status={String(p.status)} />{fmtTs(p.decided_at)}</span></li>)}
++          </ul>
++        </section>
++        <section className="card self-start">
++          <header className="border-b border-border px-4 py-2.5"><h2 className="text-sm font-semibold">Propose manually</h2></header>
++          <form action={proposeTaxonomyNode} className="space-y-2 p-4 text-xs">
++            <label className="block text-muted">Slug<input name="slug" required placeholder="riftbound" className="mt-1 w-full rounded-md border border-border bg-sunken px-2 py-1.5 text-sm" /></label>
++            <label className="block text-muted">Name<input name="name" required placeholder="Riftbound (League of Legends TCG)" className="mt-1 w-full rounded-md border border-border bg-sunken px-2 py-1.5 text-sm" /></label>
++            <label className="block text-muted">Parent<select name="parent" className="mt-1 w-full rounded-md border border-border bg-sunken px-2 py-1.5 text-sm"><option value="">(new family)</option>{families.map((f) => <option key={String(f.slug)} value={String(f.slug)}>{String(f.name)}</option>)}</select></label>
++            <ActionButton label="Add to queue" small={false} tone="primary" />
++          </form>
++        </section>
++      </div>
++    </AdminShell>
++  );
++}

added apps/web/src/app/api/admin/connectors/[id]/[action]/route.ts +18 −0

Large diff (18 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/api/admin/status/route.ts +17 −0

Large diff (17 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/api/research/[id]/route.ts +16 −0

Large diff (16 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/api/research/route.ts +40 −0

Large diff (40 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/api/scanner/[id]/choose/route.ts +13 −0

Large diff (13 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/api/scanner/route.ts +48 −0

Large diff (48 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/research/page.tsx +31 −0

Large diff (31 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/app/scanner/page.tsx +23 −0

Large diff (23 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/components/admin/shell.tsx +105 −0

Large diff (105 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/components/research/markdown.tsx +71 −0

Large diff (71 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/components/research/research-client.tsx +212 −0

Large diff (212 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/components/scanner/scanner-client.tsx +350 −0

Large diff (350 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/admin/actions.ts +156 −0

Large diff (156 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/admin/auth.ts +35 −0

Large diff (35 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/admin/queries.ts +121 −0

Large diff (121 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/admin/queue.ts +57 −0

Large diff (57 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/ai/candidates.ts +80 −0

Large diff (80 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/ai/lookup.ts +46 −0

Large diff (46 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/ai/request.ts +75 −0

Large diff (75 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/ai/research-tools.ts +259 −0

Large diff (259 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/ai/research.ts +135 −0

Large diff (135 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/lib/ai/scanner.ts +169 −0

Large diff (169 lines) is not loaded by default to keep the page fast.Load diff

added apps/web/src/proxy.ts +41 −0

Large diff (41 lines) is not loaded by default to keep the page fast.Load diff

modified apps/web/tsconfig.json +2 −2

Large diff (10 lines) is not loaded by default to keep the page fast.Load diff

added docs/API.md +78 −0

Large diff (78 lines) is not loaded by default to keep the page fast.Load diff

added docs/PENDING-SCHEMA-ai.md +14 −0

Large diff (14 lines) is not loaded by default to keep the page fast.Load diff

added docs/openapi.json +2075 −0

Large diff (2,075 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/package.json +30 −0

Large diff (30 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/ai.test.ts +101 −0

Large diff (101 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/costs.ts +61 −0

Large diff (61 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/helpers/identify.ts +89 −0

Large diff (89 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/helpers/news.ts +37 −0

Large diff (37 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/helpers/verify-match.ts +38 −0

Large diff (38 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/index.ts +9 −0

Large diff (9 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/pricing.ts +55 −0

Large diff (55 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/providers/anthropic.ts +275 −0

Large diff (275 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/providers/openai-compatible.ts +206 −0

Large diff (206 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/router.ts +125 −0

Large diff (125 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/src/types.ts +150 −0

Large diff (150 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/tsconfig.json +9 −0

Large diff (9 lines) is not loaded by default to keep the page fast.Load diff

added packages/ai/vitest.config.ts +5 −0

Large diff (5 lines) is not loaded by default to keep the page fast.Load diff

added packages/database/src/schema/ai.ts +82 −0

Large diff (82 lines) is not loaded by default to keep the page fast.Load diff

modified packages/database/src/schema/index.ts +1 −0

Large diff (4 lines) is not loaded by default to keep the page fast.Load diff

modified pnpm-lock.yaml +677 −2

Large diff (1,240 lines) is not loaded by default to keep the page fast.Load diff

added scripts/admin-cookie.ts +5 −0

Large diff (5 lines) is not loaded by default to keep the page fast.Load diff

added scripts/ai-smoke.ts +11 −0

Large diff (11 lines) is not loaded by default to keep the page fast.Load diff

added scripts/internal-token.ts +3 −0

Large diff (3 lines) is not loaded by default to keep the page fast.Load diff

added scripts/scan-smoke.py +27 −0

Large diff (27 lines) is not loaded by default to keep the page fast.Load diff

added scripts/with-env.sh +6 −0

Large diff (7 lines) is not loaded by default to keep the page fast.Load diff

spb/rareindex

Theme