TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Validate data/sources/sources.json (SPEC §4) and generate docs/connectors/SOURCES.md.3 * Run: pnpm tsx scripts/build-sources.ts4 */5import { readFileSync, writeFileSync, existsSync, readdirSync } from 'node:fs';6import path from 'node:path';7import { fileURLToPath } from 'node:url';8import { SourcesFileSchema, type SourceEntry } from '../packages/connectors/src/sources.ts';910const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');11// Base catalogue + fragments: data/sources/entries/<group>.json = SourceEntry[] (fragments override the base by id).12const basePath = path.join(root, 'data/sources/sources.json');13const file = SourcesFileSchema.parse(existsSync(basePath) ? JSON.parse(readFileSync(basePath, 'utf8')) : { version: '1.0', sources: [] });14const entriesDir = path.join(root, 'data/sources/entries');15if (existsSync(entriesDir)) {16 const byId = new Map(file.sources.map((s) => [s.id, s] as const));17 for (const f of readdirSync(entriesDir).filter((x) => x.endsWith('.json')).sort()) {18 const arr = JSON.parse(readFileSync(path.join(entriesDir, f), 'utf8')) as unknown[];19 if (!Array.isArray(arr)) throw new Error(`${f}: expected an array of SourceEntry`);20 for (const raw of arr) {21 const parsed = SourcesFileSchema.shape.sources.element.safeParse(raw);22 if (!parsed.success) throw new Error(`${f}: invalid entry ${JSON.stringify((raw as { id?: string })?.id)}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`);23 byId.set(parsed.data.id, { ...(byId.get(parsed.data.id) ?? {}), ...parsed.data });24 }25 }26 file.sources = [...byId.values()];27}28const registryPath = path.join(root, 'connectors/registry.json');29const registry = existsSync(registryPath) ? (JSON.parse(readFileSync(registryPath, 'utf8')) as { connectors: Array<{ id: string; sourceId: string; enabled: boolean }> }) : { connectors: [] };30const implemented = new Set(registry.connectors.map((c) => c.id));31const bySource = new Map<string, string[]>();32for (const c of registry.connectors) bySource.set(c.sourceId, [...(bySource.get(c.sourceId) ?? []), c.id]);3334// Synthesize a catalogue entry for every registered connector whose source has no hand-written entry,35// so SOURCES.md always lists 100% of the connectors (facts come from meta.json; marked verified because36// the connector was built against the live source).37const fullRegistry = existsSync(registryPath) ? (JSON.parse(readFileSync(registryPath, 'utf8')) as { connectors: Array<Record<string, any>> }) : { connectors: [] };38const known = new Set(file.sources.map((s) => s.id));39for (const c of fullRegistry.connectors) {40 if (known.has(c.sourceId) || known.has(c.id)) continue;41 const engine = String(c.enginePriority?.[0] ?? 'api');42 const entry = SourcesFileSchema.shape.sources.element.parse({43 id: c.sourceId,44 name: c.sourceName ?? c.displayName,45 domain: c.domain ?? new URL(c.sourceUrl).hostname.replace(/^www\./, ''),46 country: c.country ?? c.regions?.[0] ?? 'global',47 supportedCountries: c.regions ?? [],48 languages: c.languages ?? ['en'],49 sourceType: c.sourceType,50 categories: c.categories,51 api: { available: engine === 'api' && /api|json|feed|bulk/i.test(String(c.acquisitionMethod ?? c.accessNotes ?? '')), docsUrl: null, auth: c.requires?.length ? 'api_key' : 'none', notes: c.acquisitionMethod ?? null },52 urls: { robots: `${c.sourceUrl.replace(/\/$/, '')}/robots.txt` },53 data: { liveListings: Boolean(c.supportsListings), soldResults: Boolean(c.supportsSold), auctionResults: Boolean(c.supportsAuctions), historical: c.historicalDepth ?? (c.supportsSold ? 'years' : 'none'), images: c.supportsImages !== false, saleDate: Boolean(c.supportsSold), realizedPrice: Boolean(c.supportsSold), population: Boolean(c.supportsPopulation), catalog: Boolean(c.supportsCatalog), grading: /grade|psa|bgs|cgc|slab/i.test(String(c.accessNotes ?? '')), certNumber: /cert/i.test(String(c.accessNotes ?? '')) },54 access: { pagination: 'page_number', jsRendering: engine !== 'api', antiBot: engine === 'scrapfly' ? 'high' : engine === 'firecrawl' ? 'low' : 'none', cloudflare: /cloudflare/i.test(String(c.accessNotes ?? '')), robots: null, rateLimit: null, preferredEngine: engine },55 currency: c.currency ?? ['USD'],56 freshness: (c.refreshFrequencyMinutes ?? 1440) <= 60 ? 'hourly' : (c.refreshFrequencyMinutes ?? 1440) <= 1440 ? 'daily' : 'weekly',57 priority: c.priority === 'high' ? 'wave1' : c.priority === 'low' ? 'wave3' : 'wave2',58 difficulty: engine === 'scrapfly' ? 'hard' : engine === 'firecrawl' ? 'medium' : 'easy',59 reliability: c.trustScore ?? 0.6,60 legal: c.accessNotes ? String(c.accessNotes).slice(0, 400) : null,61 status: c.enabled === false ? 'partial' : c.requires?.length ? 'gated' : 'implemented',62 connectors: [c.id],63 statusReason: c.enabled === false ? 'connector disabled in registry' : null,64 verified: true,65 verifiedAt: null,66 notes: 'entry synthesized from connector meta.json',67 });68 file.sources.push(entry);69 known.add(entry.id);70}7172// Reconcile status with the registry: a source with a registered connector is at least "partial".73const ids = new Set<string>();74for (const s of file.sources) {75 if (ids.has(s.id)) throw new Error(`duplicate source id ${s.id}`);76 ids.add(s.id);77 const cons = [...new Set([...(bySource.get(s.id) ?? []), ...s.connectors.filter((c) => implemented.has(c))])];78 s.connectors = cons;79 if (cons.length && (s.status === 'planned' || s.status === 'researching')) s.status = 'partial';80 if (!cons.length && s.status === 'implemented') s.status = 'planned';81}82file.updatedAt = new Date().toISOString();83file.sources.sort((a, b) => a.id.localeCompare(b.id));84writeFileSync(path.join(root, 'data/sources/sources.json'), JSON.stringify(file, null, 2) + '\n');8586const count = (f: (s: SourceEntry) => boolean) => file.sources.filter(f).length;87const statuses = ['implemented', 'partial', 'gated', 'planned', 'researching', 'blocked', 'rejected'] as const;88const groups = new Map<string, SourceEntry[]>();89for (const s of file.sources) {90 const key = s.categories[0] ?? 'other';91 groups.set(key, [...(groups.get(key) ?? []), s]);92}93const flag = (b: boolean) => (b ? '✓' : '·');94const esc = (s: string | null | undefined) => (s ?? '').replace(/\|/g, '\\|').replace(/\n/g, ' ');9596let md = `# Source catalog (SPEC §4)\n\n_Generated from \`data/sources/sources.json\` by \`scripts/build-sources.ts\` on ${file.updatedAt.slice(0, 10)}. Edit the JSON, not this file._\n\n`;97md += `**${file.sources.length} sources researched** · ${statuses.map((st) => `${st}: ${count((s) => s.status === st)}`).join(' · ')}\n\n`;98md += `Status legend: **implemented** connector in production · **partial** connector exists but incomplete (see reason) · **gated** needs an API key/partner access we do not hold · **planned** researched, connector to build · **researching** facts not yet verified · **blocked** login/anti-bot/robots prevents lawful public access · **rejected** not worth building (see reason).\n\n`;99100md += `## By country\n\n| Country | Sources | Implemented |\n|---|---:|---:|\n`;101const countries = [...new Set(file.sources.map((s) => s.country))].sort();102for (const c of countries) md += `| ${c} | ${count((s) => s.country === c)} | ${count((s) => s.country === c && (s.status === 'implemented' || s.status === 'partial'))} |\n`;103104md += `\n## By category\n\n`;105for (const [cat, list] of [...groups.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {106 md += `### ${cat} (${list.length})\n\n| Source | Domain | Country | Type | Status | Connectors | API | Live | Sold | Auction | Hist. | Pop. | Cert | GTIN | JS | Anti-bot | Pagination | Engine | Currency | Priority | Difficulty | Notes |\n|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n`;107 for (const s of list.sort((a, b) => a.name.localeCompare(b.name))) {108 md += `| ${esc(s.name)} | ${s.domain} | ${s.country} | ${s.sourceType} | **${s.status}** | ${s.connectors.join(', ') || '—'} | ${s.api.available ? `✓ (${s.api.auth})` : '·'} | ${flag(s.data.liveListings)} | ${flag(s.data.soldResults)} | ${flag(s.data.auctionResults)} | ${s.data.historical} | ${flag(s.data.population)} | ${flag(s.data.certNumber)} | ${flag(s.data.gtin)} | ${flag(s.access.jsRendering)} | ${s.access.antiBot}${s.access.cloudflare ? ' (CF)' : ''} | ${s.access.pagination} | ${s.access.preferredEngine} | ${s.currency.join('/')} | ${s.priority} | ${s.difficulty} | ${esc(s.statusReason ?? s.notes)} |\n`;109 }110 md += '\n';111}112113md += `## Rejected / blocked sources and why\n\n| Source | Status | Reason |\n|---|---|---|\n`;114for (const s of file.sources.filter((x) => x.status === 'rejected' || x.status === 'blocked').sort((a, b) => a.name.localeCompare(b.name))) md += `| ${esc(s.name)} | ${s.status} | ${esc(s.statusReason)} |\n`;115116md += `\n## Legal / compliance notes\n\n| Source | Notes |\n|---|---|\n`;117for (const s of file.sources.filter((x) => x.legal).sort((a, b) => a.name.localeCompare(b.name))) md += `| ${esc(s.name)} | ${esc(s.legal)} |\n`;118119writeFileSync(path.join(root, 'docs/connectors/SOURCES.md'), md);120console.log(`[sources] ${file.sources.length} sources → docs/connectors/SOURCES.md (${statuses.map((st) => `${st}=${count((s) => s.status === st)}`).join(' ')})`);121