TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Live fixture capture for the g10 connectors (real router + crawl context, trimmed payloads, trimmed HTML3 * snapshots for parser tests). Usage: pnpm tsx connectors/api/_g10-lib/capture.ts [id…]4 * Gated connectors (ebay-browse, etsy, trademe) are captured only when their env vars are present.5 */6import { readFileSync } from 'node:fs';7import path from 'node:path';8import { fileURLToPath } from 'node:url';9import { createCrawlContext, createRouter, type ConnectorMeta, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors';10import { saveFixture, type Fixture } from '@rareindex/connectors/testing';11import { childLogger } from '@rareindex/shared';12import { localMeta } from '../_lib/local-meta.js';1314const here = path.dirname(fileURLToPath(import.meta.url));15const apiDir = path.resolve(here, '..');16const router = createRouter({});17const ids = process.argv.slice(2).length ? process.argv.slice(2) : ['whisky-auction', 'chairish', 'pamono', 'peyton-street-pens', '1stdibs', 'chatterley-luxuries'];1819async function load(id: string): Promise<{ connector: RareIndexConnector; meta: ConnectorMeta; mod: Record<string, unknown> }> {20 const meta = localMeta(JSON.parse(readFileSync(path.join(apiDir, id, 'meta.json'), 'utf8')));21 const mod = (await import(path.join(apiDir, id, 'index.ts'))) as Record<string, unknown> & { default: (m: ConnectorMeta) => RareIndexConnector };22 return { connector: mod.default(meta), meta, mod };23}2425function ctxFor(meta: ConnectorMeta, seeds?: string[], limit = 2) {26 return createCrawlContext({ router, meta, options: { mode: 'probe', limit, ...(seeds ? { seeds } : {}) }, log: childLogger({ connector: meta.id, level: 'warn' }) });27}2829async function firstRaw(connector: RareIndexConnector, meta: ConnectorMeta, seeds?: string[]): Promise<RawRecordInput | null> {30 for await (const raw of connector.crawl(ctxFor(meta, seeds, 1))) return raw;31 return null;32}3334function fixture(raw: RawRecordInput, expect: Fixture['expect'], note: string): Fixture {35 return { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, payload: raw.payload, fetchedAt: raw.fetchedAt ?? new Date() }, expect, note };36}3738const today = new Date().toISOString().slice(0, 10);3940for (const id of ids) {41 const { connector, meta, mod } = await load(id);42 console.log(`[capture] ${id}`);43 try {44 if (id === 'whisky-auction') {45 const parseAuctionList = mod.parseAuctionList as (h: string) => Array<{ id: string }>;46 const parseLotPage = mod.parseLotPage as (h: string) => { lots: unknown[]; total: number | null; auctionTitle: string | null };47 const trimLotHtml = mod.trimLotHtml as (h: string, n?: number) => string;48 const pageUrl = mod.pageUrl as (a: string, s: { category: string; slug: 'whisky' | 'rum' | 'cognac' | 'wine' }, i: number, n: number) => string;49 const ctx = ctxFor(meta);50 const list = await ctx.fetch('https://whisky.auction/auctions', { engines: ['api'], responseType: 'text', minQuality: 0 });51 const entries = parseAuctionList(list.html!);52 // keep the first 4 auction entries of the real list page as the HTML snapshot53 const m = list.html!.match(/<div class="wa-grid-item">[\s\S]*?<\/div>\s*<\/div>\s*<\/div>\s*<\/div>/g) ?? [];54 const listHtml = `<!doctype html><html><body><div id="pastauctions" class="wa-grid top">${m.slice(0, 4).join('\n')}</div></body></html>`;55 saveFixture(id, 'auctions-list', { raw: { url: 'https://whisky.auction/auctions', externalId: 'auctions-list', kind: 'sale', engine: 'api', fetchedAt: list.fetchedAt, payload: { kind: 'lot_page', url: 'https://whisky.auction/auctions', auctionId: entries[0]!.id, auctionTitle: null, seed: { category: '', slug: 'whisky' }, pageIndex: 0, pageSize: 90, total: 0, lots: [], snapshot: listHtml } }, expect: { count: 0 }, note: `Live capture ${today}: /auctions past-auction list trimmed to 4 entries (parser test only; no lots).` });56 const seeds = meta.config.seeds as Array<{ category: string; slug: 'whisky' | 'rum' | 'cognac' | 'wine' }>;57 const auctionId = '201';58 for (const [name, seed, pick] of [59 ['auction-201-whisky-p0', seeds[0]!, (lots: unknown[]) => lots.slice(0, 6)],60 ['auction-201-rum-p0', seeds[1]!, (lots: unknown[]) => [...lots.slice(0, 4), ...lots.slice(-3)]],61 ] as const) {62 const url = pageUrl(auctionId, seed, 0, 90);63 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 });64 const parsed = parseLotPage(res.html!);65 const payload = { kind: 'lot_page', url, auctionId, auctionTitle: parsed.auctionTitle, seed, pageIndex: 0, pageSize: 90, total: parsed.total, lots: pick(parsed.lots), snapshot: trimLotHtml(res.html!, 3) };66 saveFixture(id, name, { raw: { url, externalId: `auction:${auctionId}:${seed.slug}:p0`, kind: 'sale', engine: 'api', fetchedAt: res.fetchedAt, payload }, expect: { minCount: 1, kinds: ['sale'], requiredFields: ['price', 'currency', 'saleDate', 'attributes.identifiers.whisky_auction_lot'] }, note: `Live capture ${today} of whisky.auction auction 201 (May 2026), ${seed.slug} filter, first grid page sorted by price desc; lots trimmed (${name.includes('rum') ? 'first 4 + last 3, the tail includes unsold NotMet lots' : 'first 6'}), HTML snapshot trimmed to 3 lot cards.` });67 }68 continue;69 }70 if (id === 'chairish' || id === 'pamono' || id === 'peyton-street-pens' || id === '1stdibs') {71 const seeds = meta.config.seeds as Array<{ path: string }>;72 const picks: Array<[string, string, number]> = id === 'chairish' ? [['vintage-furniture-p2', '/collection/vintage-furniture', 2], ['decor-p1', '/collection/decor', 1]] : id === 'pamono' ? [['furniture-p1', '/furniture', 1], ['jewelry-watches-p1', '/jewelry-watches', 1]] : id === 'peyton-street-pens' ? [['parker-p1', '/pens-by-brand/parker/', 1], ['montblanc-p1', '/pen-makers-europe/montblanc/', 1]] : [['seating-chairs-p1', '/furniture/seating/chairs/', 1], ['wrist-watches-p1', '/jewelry/watches/wrist-watches/', 1]];73 void seeds;74 const pageUrl = mod.pageUrl as (p: string, n: number) => string;75 const ctx = ctxFor(meta);76 for (const [name, seedPath, page] of picks) {77 const url = pageUrl(seedPath, page);78 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 90_000 });79 if (!res.success || !res.html) throw new Error(`${url}: ${res.error ?? res.httpStatus}`);80 const seedCfg = (meta.config.seeds as Array<Record<string, unknown>>).find((s) => s.path === seedPath) ?? { path: seedPath };81 let payload: Record<string, unknown>;82 let snapshot: string | undefined;83 if (id === 'chairish') {84 const items = (mod.parseBrowseHtml as (h: string) => unknown[])(res.html).slice(0, 6);85 snapshot = (mod.trimBrowseHtml as (h: string, n: number) => string)(res.html, 3);86 payload = { kind: 'listing_page', url, seed: seedCfg, page, items, snapshot };87 } else if (id === 'pamono') {88 const cards = (mod.parseCategoryHtml as (h: string) => unknown[])(res.html).slice(0, 6);89 snapshot = (mod.trimCategoryHtml as (h: string, n: number) => string)(res.html, 3);90 payload = { kind: 'listing_page', url, seed: seedCfg, page, currency: (mod.parseStoreCurrency as (h: string) => string | null)(res.html), cards, snapshot };91 } else if (id === 'peyton-street-pens') {92 const parsed = (mod.parseCategoryHtml as (h: string) => { cards: unknown[]; totalPages: number | null })(res.html);93 snapshot = (mod.trimCategoryHtml as (h: string, n: number) => string)(res.html, 3);94 payload = { kind: 'listing_page', url, seed: seedCfg, page, totalPages: parsed.totalPages, cards: parsed.cards.slice(0, 6), snapshot };95 } else {96 const parsed = (mod.parseBrowseHtml as (h: string) => { items: unknown[]; totalResults: number | null; maxPages: number | null })(res.html)!;97 snapshot = (mod.trimStoreHtml as (h: string, n: number) => string)(res.html, 3);98 payload = { kind: 'listing_page', url, seed: seedCfg, page, totalResults: parsed.totalResults, maxPages: parsed.maxPages, items: parsed.items.slice(0, 6) };99 // the Relay snapshot is stored as a separate .html next to the fixture100 const fx = fixture({ url, externalId: `${seedPath}:p${page}`, kind: 'listing', engine: 'api', payload: { ...payload, snapshot }, fetchedAt: res.fetchedAt }, { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', 'attributes.identifiers.firstdibs_item_id'] }, `Live capture ${today} of ${url}; items trimmed to 6, Relay store snapshot reduced to the records reachable from the first 3 items.`);101 saveFixture(id, name, fx);102 continue;103 }104 const req = id === 'chairish' ? 'attributes.identifiers.chairish_product_id' : id === 'pamono' ? 'attributes.identifiers.pamono_sku' : 'seller';105 saveFixture(id, name, fixture({ url, externalId: `${seedPath}:p${page}`, kind: 'listing', engine: 'api', payload, fetchedAt: res.fetchedAt }, { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', req] }, `Live capture ${today} of ${url}; items trimmed to 6, HTML snapshot trimmed to 3 cards.`));106 }107 continue;108 }109 if (id === 'chatterley-luxuries') {110 for (const [name, handle, wantSku] of [['pens-p1', 'pens', true], ['lighters-p1', 'lighters', false]] as const) {111 const m2 = { ...meta, config: { ...meta.config, collections: (meta.config.collections as Array<{ handle: string }>).filter((c) => c.handle === handle) } };112 const c2 = (mod.default as (m: ConnectorMeta) => RareIndexConnector)(m2);113 let chosen: RawRecordInput | null = null;114 for await (const raw of c2.crawl(ctxFor(m2, undefined, 12))) {115 const sku = ((raw.payload as { product: { sku?: string | null; prices?: { price?: string } } }).product.sku ?? '').trim();116 const priced = Number((raw.payload as { product: { prices?: { price?: string } } }).product.prices?.price ?? 0) > 0;117 if (priced && (!wantSku || sku)) {118 chosen = raw;119 break;120 }121 }122 if (!chosen) throw new Error(`no product captured for ${handle}`);123 const p = chosen.payload as { product: Record<string, unknown> };124 p.product = { ...p.product, description: String(p.product.description ?? '').slice(0, 1500), images: (p.product.images as unknown[]).slice(0, 2) };125 saveFixture(id, name, fixture(chosen, { count: 1, kinds: ['listing'], requiredFields: ['price', 'currency'] }, `Live capture ${today}: first priced product of the '${handle}' category via the WooCommerce Store API (description/images trimmed).`));126 }127 continue;128 }129 // gated official APIs: capture only when credentials exist130 const raw = await firstRaw(connector, meta);131 if (!raw) {132 console.log(` [${id}] nothing captured (gated / no credentials)`);133 continue;134 }135 saveFixture(id, `live-${today}`, fixture(raw, { minCount: 1 }, `Live capture ${today}.`));136 } catch (err) {137 console.error(` [${id}] capture failed:`, err instanceof Error ? err.message : err);138 }139}140