TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Live fixture capture for the g3-shops-na Shopify store connectors (SPEC §14: fixtures are real captures, trimmed).3 *4 * pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts <connector-id> [<connector-id>…] [--max-collections 8]5 *6 * For each connector it reads meta.json, walks the configured collections' public7 * /collections/<handle>/products.json?limit=100&page=1 (honest UA, 1.5 s apart), and saves up to three8 * trimmed single-product payloads into data/fixtures/<id>/: a graded item when the store lists any, a9 * product with an out-of-stock variant, and an in-stock product — spread over distinct collections.10 * Payload shape = ShopifyPayload ({ collection, product }); trimming keeps everything `normalize` reads.11 */12import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';13import path from 'node:path';14import { fileURLToPath } from 'node:url';15import { adapters, type ConnectorMeta } from '@rareindex/connectors';16import { fixtureDir, saveFixture } from '@rareindex/connectors/testing';17import { parseGradeFromTitle } from '@rareindex/taxonomy';18import { localMeta } from '../_lib/local-meta.js';1920const UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)';21const here = path.dirname(fileURLToPath(import.meta.url));22const root = path.resolve(here, '../../..');23const args = process.argv.slice(2);24const maxIdx = args.indexOf('--max-collections');25const maxCollections = maxIdx >= 0 ? Number(args[maxIdx + 1]) : 8;26const ids = args.filter((a, i) => !a.startsWith('--') && (maxIdx < 0 || i !== maxIdx + 1));27const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));2829interface Candidate {30 handle: string;31 product: adapters.ShopifyProduct;32 categorySlug: string;33 graded: boolean;34 hasOos: boolean;35 hasAvailable: boolean;36}3738function trim(p: adapters.ShopifyProduct, keepVariantIds: Set<number>): adapters.ShopifyProduct {39 const variants = p.variants.filter((v) => keepVariantIds.has(v.id)).slice(0, 4);40 return {41 id: p.id,42 title: p.title,43 handle: p.handle,44 body_html: p.body_html ? p.body_html.slice(0, 400) : p.body_html,45 published_at: p.published_at,46 updated_at: p.updated_at,47 vendor: p.vendor,48 product_type: p.product_type,49 tags: Array.isArray(p.tags) ? p.tags.slice(0, 12) : p.tags,50 variants: variants.map((v) => ({ id: v.id, title: v.title, sku: v.sku, barcode: v.barcode, price: v.price, compare_at_price: v.compare_at_price, available: v.available, featured_image: v.featured_image ? { src: v.featured_image.src } : v.featured_image })),51 images: p.images.slice(0, 2).map((i) => ({ src: i.src })),52 };53}5455async function captureStore(id: string): Promise<void> {56 const dir = path.join(root, 'connectors/api', id);57 const meta: ConnectorMeta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));58 const cfg = adapters.ShopifyConfigSchema.parse(meta.config);59 const site = meta.sourceUrl.replace(/\/+$/, '');60 const candidates: Candidate[] = [];61 const handles = cfg.collections.map((c) => c.handle).slice(0, maxCollections);62 for (const handle of handles) {63 const url = `${site}/collections/${handle}/products.json?limit=100&page=1`;64 await sleep(1500);65 const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' } });66 if (!res.ok) {67 console.warn(` [${id}] ${handle}: HTTP ${res.status}`);68 continue;69 }70 const json = (await res.json()) as { products?: unknown[] };71 const products = (json.products ?? []).map((x) => adapters.ShopifyProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data);72 let mapped = 0;73 for (const product of products) {74 const sp = adapters.shopifyToStorefrontProduct(site, { collection: handle, product });75 const m = adapters.mapProduct(cfg, sp);76 if (!m) continue;77 const priced = product.variants.filter((v) => Number(v.price) > 0);78 if (!priced.length) continue;79 mapped++;80 const g = parseGradeFromTitle(product.title);81 // "graded" = a real third-party grader with a grade (or an ICCS/PMG/PCGS/NGC coin/note grade the parser does not read yet);82 // parseGradeFromTitle also answers grader "raw" for colourways like "Raw Indigo" — not a grade.83 const graded = Boolean(g.grader && g.grader !== 'raw' && (g.grade !== null || /\b(iccs|pmg|pcgs|ngc|anacs)\b/i.test(product.title)));84 candidates.push({ handle, product, categorySlug: m.categorySlug, graded, hasOos: priced.some((v) => v.available === false), hasAvailable: priced.some((v) => v.available === true) });85 }86 console.log(` [${id}] ${handle}: ${products.length} products, ${mapped} mapped`);87 }88 if (!candidates.length) {89 console.error(` [${id}] no candidates — nothing saved`);90 return;91 }92 const picks: Array<{ kind: string; c: Candidate; why: string }> = [];93 const used = new Set<string>();94 const fresh = (x: Candidate) => !picks.some((p) => p.c.product.id === x.product.id);95 const take = (kind: string, pred: (c: Candidate) => boolean, why: string) => {96 const c = candidates.find((x) => pred(x) && fresh(x) && !used.has(x.handle)) ?? candidates.find((x) => pred(x) && fresh(x));97 if (!c) return;98 picks.push({ kind, c, why });99 used.add(c.handle);100 };101 take('graded', (c) => c.graded, 'title carries a third-party grade (grader parsed by parseGradeFromTitle)');102 take('out-of-stock', (c) => c.hasOos && c.hasAvailable, 'mixed availability: at least one variant sold out → ended listing');103 take('out-of-stock', (c) => c.hasOos && !picks.some((p) => p.kind === 'out-of-stock'), 'sold-out product → ended listing');104 take('available', (c) => c.hasAvailable, 'in-stock product with asking price');105 while (picks.length < 3) {106 const next = candidates.find((c) => !used.has(c.handle) && !picks.some((p) => p.c.product.id === c.product.id));107 if (!next) break;108 picks.push({ kind: 'available', c: next, why: 'additional collection coverage' });109 used.add(next.handle);110 }111 // fresh capture replaces the previous one112 const fdir = fixtureDir(id);113 if (existsSync(fdir)) for (const f of readdirSync(fdir)) if (f.endsWith('.json')) rmSync(path.join(fdir, f));114 const seenNames = new Set<string>();115 for (const { kind, c, why } of picks) {116 const priced = c.product.variants.filter((v) => Number(v.price) > 0);117 const keep = new Set<number>();118 const oos = priced.find((v) => v.available === false);119 const avail = priced.find((v) => v.available === true);120 if (avail) keep.add(avail.id);121 if (oos) keep.add(oos.id);122 for (const v of priced) if (keep.size < 4) keep.add(v.id);123 const product = trim(c.product, keep);124 let name = `${c.handle}-${kind}`.slice(0, 80);125 while (seenNames.has(name)) name = `${name}-2`;126 seenNames.add(name);127 const now = new Date();128 saveFixture(id, name, {129 raw: { url: `${site}/products/${product.handle}`, externalId: String(product.id), kind: 'listing', engine: 'api', fetchedAt: now, payload: { collection: c.handle, product } },130 expect: { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', 'sourceUrl', 'attributes.categorySlug'], first: { currency: cfg.currency, 'attributes.categorySlug': c.categorySlug } },131 note: `Live capture ${now.toISOString().slice(0, 10)} of ${site}/collections/${c.handle}/products.json?limit=100&page=1 (product ${product.id}); trimmed to what normalize() reads: body_html ≤400 chars, ≤2 images, ≤4 variants (${keep.size} kept${oos ? ', incl. one sold-out' : ''}). Chosen because: ${why}.`,132 });133 console.log(` [${id}] saved ${name} (${c.categorySlug}) ← ${product.title.slice(0, 70)}`);134 }135}136137for (const id of ids) {138 console.log(`== ${id}`);139 try {140 await captureStore(id);141 } catch (err) {142 console.error(` [${id}] failed: ${(err as Error).message}`);143 }144}145