/** * Live fixture capture for the g3-shops-na Shopify store connectors (SPEC §14: fixtures are real captures, trimmed). * * pnpm tsx connectors/api/_g3-shops-na-lib/capture.ts […] [--max-collections 8] * * For each connector it reads meta.json, walks the configured collections' public * /collections//products.json?limit=100&page=1 (honest UA, 1.5 s apart), and saves up to three * trimmed single-product payloads into data/fixtures//: a graded item when the store lists any, a * product with an out-of-stock variant, and an in-stock product — spread over distinct collections. * Payload shape = ShopifyPayload ({ collection, product }); trimming keeps everything `normalize` reads. */ import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { adapters, type ConnectorMeta } from '@rareindex/connectors'; import { fixtureDir, saveFixture } from '@rareindex/connectors/testing'; import { parseGradeFromTitle } from '@rareindex/taxonomy'; import { localMeta } from '../_lib/local-meta.js'; const UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)'; const here = path.dirname(fileURLToPath(import.meta.url)); const root = path.resolve(here, '../../..'); const args = process.argv.slice(2); const maxIdx = args.indexOf('--max-collections'); const maxCollections = maxIdx >= 0 ? Number(args[maxIdx + 1]) : 8; const ids = args.filter((a, i) => !a.startsWith('--') && (maxIdx < 0 || i !== maxIdx + 1)); const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); interface Candidate { handle: string; product: adapters.ShopifyProduct; categorySlug: string; graded: boolean; hasOos: boolean; hasAvailable: boolean; } function trim(p: adapters.ShopifyProduct, keepVariantIds: Set): adapters.ShopifyProduct { const variants = p.variants.filter((v) => keepVariantIds.has(v.id)).slice(0, 4); return { id: p.id, title: p.title, handle: p.handle, body_html: p.body_html ? p.body_html.slice(0, 400) : p.body_html, published_at: p.published_at, updated_at: p.updated_at, vendor: p.vendor, product_type: p.product_type, tags: Array.isArray(p.tags) ? p.tags.slice(0, 12) : p.tags, variants: variants.map((v) => ({ id: v.id, title: v.title, sku: v.sku, barcode: v.barcode, price: v.price, compare_at_price: v.compare_at_price, available: v.available, featured_image: v.featured_image ? { src: v.featured_image.src } : v.featured_image })), images: p.images.slice(0, 2).map((i) => ({ src: i.src })), }; } async function captureStore(id: string): Promise { const dir = path.join(root, 'connectors/api', id); const meta: ConnectorMeta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8'))); const cfg = adapters.ShopifyConfigSchema.parse(meta.config); const site = meta.sourceUrl.replace(/\/+$/, ''); const candidates: Candidate[] = []; const handles = cfg.collections.map((c) => c.handle).slice(0, maxCollections); for (const handle of handles) { const url = `${site}/collections/${handle}/products.json?limit=100&page=1`; await sleep(1500); const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' } }); if (!res.ok) { console.warn(` [${id}] ${handle}: HTTP ${res.status}`); continue; } const json = (await res.json()) as { products?: unknown[] }; const products = (json.products ?? []).map((x) => adapters.ShopifyProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data); let mapped = 0; for (const product of products) { const sp = adapters.shopifyToStorefrontProduct(site, { collection: handle, product }); const m = adapters.mapProduct(cfg, sp); if (!m) continue; const priced = product.variants.filter((v) => Number(v.price) > 0); if (!priced.length) continue; mapped++; const g = parseGradeFromTitle(product.title); // "graded" = a real third-party grader with a grade (or an ICCS/PMG/PCGS/NGC coin/note grade the parser does not read yet); // parseGradeFromTitle also answers grader "raw" for colourways like "Raw Indigo" — not a grade. const graded = Boolean(g.grader && g.grader !== 'raw' && (g.grade !== null || /\b(iccs|pmg|pcgs|ngc|anacs)\b/i.test(product.title))); candidates.push({ handle, product, categorySlug: m.categorySlug, graded, hasOos: priced.some((v) => v.available === false), hasAvailable: priced.some((v) => v.available === true) }); } console.log(` [${id}] ${handle}: ${products.length} products, ${mapped} mapped`); } if (!candidates.length) { console.error(` [${id}] no candidates — nothing saved`); return; } const picks: Array<{ kind: string; c: Candidate; why: string }> = []; const used = new Set(); const fresh = (x: Candidate) => !picks.some((p) => p.c.product.id === x.product.id); const take = (kind: string, pred: (c: Candidate) => boolean, why: string) => { const c = candidates.find((x) => pred(x) && fresh(x) && !used.has(x.handle)) ?? candidates.find((x) => pred(x) && fresh(x)); if (!c) return; picks.push({ kind, c, why }); used.add(c.handle); }; take('graded', (c) => c.graded, 'title carries a third-party grade (grader parsed by parseGradeFromTitle)'); take('out-of-stock', (c) => c.hasOos && c.hasAvailable, 'mixed availability: at least one variant sold out → ended listing'); take('out-of-stock', (c) => c.hasOos && !picks.some((p) => p.kind === 'out-of-stock'), 'sold-out product → ended listing'); take('available', (c) => c.hasAvailable, 'in-stock product with asking price'); while (picks.length < 3) { const next = candidates.find((c) => !used.has(c.handle) && !picks.some((p) => p.c.product.id === c.product.id)); if (!next) break; picks.push({ kind: 'available', c: next, why: 'additional collection coverage' }); used.add(next.handle); } // fresh capture replaces the previous one const fdir = fixtureDir(id); if (existsSync(fdir)) for (const f of readdirSync(fdir)) if (f.endsWith('.json')) rmSync(path.join(fdir, f)); const seenNames = new Set(); for (const { kind, c, why } of picks) { const priced = c.product.variants.filter((v) => Number(v.price) > 0); const keep = new Set(); const oos = priced.find((v) => v.available === false); const avail = priced.find((v) => v.available === true); if (avail) keep.add(avail.id); if (oos) keep.add(oos.id); for (const v of priced) if (keep.size < 4) keep.add(v.id); const product = trim(c.product, keep); let name = `${c.handle}-${kind}`.slice(0, 80); while (seenNames.has(name)) name = `${name}-2`; seenNames.add(name); const now = new Date(); saveFixture(id, name, { raw: { url: `${site}/products/${product.handle}`, externalId: String(product.id), kind: 'listing', engine: 'api', fetchedAt: now, payload: { collection: c.handle, product } }, expect: { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', 'sourceUrl', 'attributes.categorySlug'], first: { currency: cfg.currency, 'attributes.categorySlug': c.categorySlug } }, note: `Live capture ${now.toISOString().slice(0, 10)} of ${site}/collections/${c.handle}/products.json?limit=100&page=1 (product ${product.id}); trimmed to what normalize() reads: body_html ≤400 chars, ≤2 images, ≤4 variants (${keep.size} kept${oos ? ', incl. one sold-out' : ''}). Chosen because: ${why}.`, }); console.log(` [${id}] saved ${name} (${c.categorySlug}) ← ${product.title.slice(0, 70)}`); } } for (const id of ids) { console.log(`== ${id}`); try { await captureStore(id); } catch (err) { console.error(` [${id}] failed: ${(err as Error).message}`); } }