TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Connector scaffolder (SPEC §15).3 *4 * pnpm connector:new <id> --url https://www.example.com --name "Example" [--engine api|firecrawl|scrapfly]5 * [--adapter shopify|woocommerce] [--type marketplace|auction_house|dealer|…]6 * [--categories pokemon,magic_the_gathering] [--country US] [--currency USD]7 * [--kind listing|sale|auction_lot|price_observation|catalog_item|population_report] [--group <entries-file>]8 *9 * Creates connectors/<engine>/<id>/{meta.json,index.ts,index.test.ts,README.md}, data/fixtures/<id>/,10 * a stub entry in data/sources/sources.json, then rebuilds the registry.11 */12import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';13import path from 'node:path';14import { fileURLToPath } from 'node:url';15import { execSync } from 'node:child_process';1617const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');18const [id, ...rest] = process.argv.slice(2);19const flags: Record<string, string> = {};20for (let i = 0; i < rest.length; i++) {21 const a = rest[i]!;22 if (a.startsWith('--')) {23 flags[a.slice(2)] = rest[i + 1] && !rest[i + 1]!.startsWith('--') ? rest[++i]! : 'true';24 }25}26if (!id || !/^[a-z0-9-]+$/.test(id) || !flags.url) {27 console.error('usage: pnpm connector:new <id> --url https://… --name "Name" [--engine api|firecrawl|scrapfly] [--adapter shopify|woocommerce] [--type marketplace] [--categories a,b] [--country US] [--currency USD] [--kind listing]');28 process.exit(1);29}30const adapter = flags.adapter as 'shopify' | 'woocommerce' | undefined;31const engine = (flags.engine ?? 'api') as 'api' | 'firecrawl' | 'scrapfly';32const name = flags.name ?? id;33const url = flags.url.replace(/\/+$/, '');34const host = new URL(url).hostname.replace(/^www\./, '');35const sourceType = flags.type ?? (adapter ? 'dealer' : 'marketplace');36const categories = (flags.categories ?? 'other_tcg').split(',').map((s) => s.trim()).filter(Boolean);37const country = flags.country ?? 'US';38const currency = flags.currency ?? 'USD';39const kind = flags.kind ?? (adapter ? 'listing' : 'sale');40const dir = path.join(root, 'connectors', engine, id);41if (existsSync(dir)) {42 console.error(`connector ${engine}/${id} already exists`);43 process.exit(1);44}45mkdirSync(dir, { recursive: true });46mkdirSync(path.join(root, 'data/fixtures', id), { recursive: true });4748const meta = {49 id,50 displayName: name,51 sourceId: id,52 sourceName: name,53 sourceType,54 sourceUrl: url,55 module: `${engine}/${id}`,56 enginePriority: adapter ? ['api'] : engine === 'api' ? ['api'] : engine === 'firecrawl' ? ['firecrawl', 'scrapfly'] : ['scrapfly'],57 categories,58 regions: [country],59 languages: ['en'],60 currency: [currency],61 supportsListings: kind === 'listing',62 supportsSold: kind === 'sale',63 supportsAuctions: kind === 'auction_lot',64 supportsImages: true,65 supportsCatalog: kind === 'catalog_item',66 supportsPopulation: kind === 'population_report',67 supportsLookup: Boolean(adapter),68 refreshFrequencyMinutes: kind === 'listing' ? 720 : 1440,69 priority: 'medium',70 trustScore: 0.7,71 attributionRequired: true,72 termsUrl: `${url}/terms`,73 accessNotes: `TODO (SPEC §36): describe exactly which public pages/endpoints are read, robots.txt findings, rate limit, anti-bot observations and what is deliberately NOT fetched.`,74 enabled: true,75 schemaVersion: '1.0',76 acquisitionMethod: adapter === 'shopify' ? 'Shopify storefront products.json' : adapter === 'woocommerce' ? 'WooCommerce Store API' : engine === 'api' ? 'public JSON/HTML (direct HTTP)' : engine === 'firecrawl' ? 'Firecrawl markdown/HTML' : 'Scrapfly render',77 historicalDepth: kind === 'sale' || kind === 'auction_lot' ? 'years' : 'none',78 requires: [],79 config: adapter80 ? { currency, seller: name, collections: [{ handle: 'TODO-collection-handle', categorySlug: categories[0] }], rules: [], defaultCategory: null }81 : { seeds: [], pagesPerSeed: 2 },82};83writeFileSync(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n');8485const adapterIndex = (cls: string) => `import { ${cls}, type ConnectorMeta } from '@rareindex/connectors';8687/**88 * ${name} — ${adapter} storefront read through the shared ${adapter} adapter.89 * All source-specific knowledge lives in meta.json (collections → taxonomy mapping, rules, currency).90 */91export default (meta: ConnectorMeta) => new ${cls}(meta);92`;9394const customIndex = `import { z } from 'zod';95import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';96import { ${kind === 'sale' ? 'NormalizedSaleSchema' : kind === 'listing' ? 'NormalizedListingSchema' : kind === 'auction_lot' ? 'NormalizedAuctionLotSchema' : kind === 'price_observation' ? 'NormalizedPriceObservationSchema' : kind === 'population_report' ? 'NormalizedPopulationReportSchema' : 'NormalizedCatalogItemSchema'}, type NormalizedRecord } from '@rareindex/shared';9798/**99 * ${name} (${host}) — ${kind} connector.100 * Acquisition: ${meta.acquisitionMethod}. See meta.json accessNotes for what is (not) fetched and why.101 */102const PARSER_VERSION = '1.0.0';103const SITE = '${url}';104105/** One captured page. Keep the payload small and source-shaped; parsing happens in normalize(). */106export const RawPayloadSchema = z.object({107 seed: z.string(),108 page: z.number().int(),109 items: z.array(z.object({ id: z.string(), title: z.string(), url: z.string(), priceText: z.string().nullable(), dateText: z.string().nullable(), image: z.string().nullable() })),110});111export type RawPayload = z.infer<typeof RawPayloadSchema>;112113/** Parse a fetched document into items. Exported so tests can exercise it on saved HTML/markdown. */114export function parsePage(doc: { html: string | null; markdown: string | null; json: unknown }): RawPayload['items'] {115 // TODO: prefer JSON (embedded __NEXT_DATA__/JSON-LD via html.jsonLd / adapters.productsFromHtml) over selectors.116 void doc;117 return [];118}119120export class ${toClass(id)}Connector extends BaseConnector {121 readonly version = '1.0.0';122 readonly parserVersion = PARSER_VERSION;123 protected override minIntervalMs = 2000;124125 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {126 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []);127 const maxPages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? this.policy.crawlDepth);128 const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number };129 let count = 0;130 for (const seed of seeds) {131 if (cursor.seed && cursor.seed !== seed && seeds.indexOf(seed) < seeds.indexOf(cursor.seed)) continue;132 for (let page = cursor.seed === seed && cursor.page ? cursor.page : 1; page <= maxPages; page++) {133 if (ctx.signal?.aborted || this.reached(ctx, count)) return;134 const url = \`\${SITE}/\${seed}?page=\${page}\`;135 await this.throttle(url);136 const res = await ctx.fetch(url, { expect: ['title', 'price'], parse: (r) => { const items = parsePage(r); return { title: items[0]?.title ?? null, price: items[0]?.priceText ?? null }; } });137 if (!res.success) {138 ctx.anomaly('page_fetch_failed', \`\${url}: \${res.error ?? res.httpStatus}\`);139 break;140 }141 const items = parsePage(res);142 if (!items.length) {143 ctx.anomaly(page === 1 ? 'parse_failure_page' : 'pagination_end', url);144 break;145 }146 count++;147 const payload: RawPayload = { seed, page, items };148 yield { url, externalId: \`\${seed}:p\${page}\`, kind: '${kind}', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };149 await ctx.setCursor({ seed, page: page + 1, at: new Date().toISOString() });150 await ctx.progress({ page, itemsProcessed: count });151 }152 }153 await ctx.setCursor({ done: true, at: new Date().toISOString() });154 }155156 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {157 const p = RawPayloadSchema.parse(raw.payload);158 const out: NormalizedRecord[] = [];159 for (const it of p.items) {160 void it; // TODO: map to the canonical schema; keep native currency and the source's own date; never invent fields.161 }162 return out;163 }164}165166function toClass(s: string): string {167 return s.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('');168}169170export default (meta: ConnectorMeta) => new ${toClass(id)}Connector(meta);171`;172173function toClass(s: string): string {174 return s.split('-').map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join('');175}176177writeFileSync(path.join(dir, 'index.ts'), adapter ? adapterIndex(adapter === 'shopify' ? 'ShopifyStoreConnector' : 'WooCommerceStoreConnector') : customIndex.replace(/\nfunction toClass[\s\S]*?\n}\n/, '\n'));178179const test = `import { describe, expect, it } from 'vitest';180import meta from './meta.json' with { type: 'json' };181import { localMeta } from '../../api/_lib/local-meta.js';182import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';183import createConnector from './index.js';184185const connector = createConnector(localMeta(meta));186187describe('${id}', () => {188 // Standard invariants on every saved fixture (schema validity, prices > 0, source dates, currency, dedupe keys).189 runFixtureSuite(connector, it, expect);190191 it('normalises fixtures into ${kind} records with native currency', async () => {192 for (const name of listFixtures('${id}')) {193 const out = await connector.normalize(loadFixture('${id}', name).raw);194 expect(out.length).toBeGreaterThan(0);195 for (const r of out) {196 expect(r.kind).toBe('${kind}');197 if ('currency' in r && r.currency) expect(meta.currency).toContain(r.currency);198 }199 }200 });201});202`;203writeFileSync(path.join(dir, 'index.test.ts'), test);204205writeFileSync(206 path.join(dir, 'README.md'),207 `# ${name} connector (\`${id}\`)208209- Source: ${url} (${host}) · type: ${sourceType} · country: ${country} · currency: ${currency}210- Acquisition: ${meta.acquisitionMethod}211- Records: \`${kind}\`212- Status: scaffolded — complete the TODOs, capture at least one real fixture, run \`pnpm vitest run connectors/${engine}/${id}\`.213214## Access & compliance215Fill \`accessNotes\` in meta.json: which public pages/endpoints are read, robots.txt findings, rate limit, anti-bot behaviour, what is intentionally not fetched (logins, paywalls, personal data).216217## Fixtures218\`data/fixtures/${id}/<name>.json\` = \`{ raw: { url, externalId, kind, engine, fetchedAt, payload }, expect: { minCount, kinds, requiredFields } }\` captured from a live page (never hand-written). Large HTML goes next to it as \`<name>.html\` (\`payload.snapshot\`).219220## Checklist (SPEC §35)221discovery · pagination · normalisation · stable ids · error handling · rate limiting (domains.json) · retries · fixtures · tests · health · logging · persistence · refresh class · source metadata (data/sources/sources.json) · documentation222`,223);224225// source registry stub → data/sources/entries/<group>.json (fragments are merged into sources.json by build-sources)226const group = flags.group ?? 'scaffold';227mkdirSync(path.join(root, 'data/sources/entries'), { recursive: true });228const sourcesPath = path.join(root, 'data/sources/entries', `${group}.json`);229const sources = { sources: existsSync(sourcesPath) ? (JSON.parse(readFileSync(sourcesPath, 'utf8')) as Array<Record<string, unknown>>) : [] };230if (!sources.sources.some((s) => s.id === id)) {231 sources.sources.push({ id, name, domain: host, country, supportedCountries: [country], languages: ['en'], sourceType, categories, api: { available: engine === 'api' && !adapter, docsUrl: null, auth: 'none', notes: null }, urls: { search: null, product: null, sold: null, sitemap: `${url}/sitemap.xml`, robots: `${url}/robots.txt` }, data: { liveListings: kind === 'listing', soldResults: kind === 'sale', auctionResults: kind === 'auction_lot', historical: meta.historicalDepth, catalog: kind === 'catalog_item', population: kind === 'population_report' }, access: { pagination: 'page_number', jsRendering: engine !== 'api', antiBot: 'none', cloudflare: false, preferredEngine: adapter ?? engine }, currency: [currency], freshness: 'daily', priority: 'wave3', difficulty: adapter ? 'easy' : 'medium', reliability: 0.6, legal: null, status: 'planned', connectors: [id], statusReason: 'scaffolded, not yet verified', verified: false, verifiedAt: null, notes: null });232 sources.sources.sort((a, b) => String(a.id).localeCompare(String(b.id)));233 writeFileSync(sourcesPath, JSON.stringify(sources.sources, null, 2) + '\n');234}235236execSync('pnpm tsx scripts/build-registry.ts', { cwd: root, stdio: 'inherit' });237console.log(`\n✔ scaffolded connectors/${engine}/${id}\n next: fill meta.json (collections/seeds, accessNotes), implement parsePage/normalize, capture a fixture, run:\n pnpm vitest run connectors/${engine}/${id} && pnpm tsx scripts/build-sources.ts`);238