/** * Live fixture capture for the g4-shops-intl storefront connectors. * * pnpm tsx connectors/api/_g4-shops-intl-lib/capture.ts [--pick ] [--index ] [--note "..."] * * Fetches one public storefront page (Shopify `/collections//products.json?limit=50` or the * WooCommerce Store API `/wp-json/wc/store/v1/products?category=`), picks one product (first, by * index, or first whose title matches --pick), trims it to what `normalize` needs (short body, ≤ 3 * images, ≤ 6 variants, ≤ 12 tags) and saves data/fixtures//.json with an `expect` block * derived from the connector's own normalisation of that live record. Never hand-written (SPEC §14). */ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { ConnectorMetaSchema } from '@rareindex/connectors'; import { saveFixture } from '@rareindex/connectors/testing'; const UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)'; const [id, handle, name, ...rest] = process.argv.slice(2); if (!id || !handle || !name) { console.error('usage: capture.ts [--pick ] [--index ] [--note "..."]'); process.exit(1); } const flags: Record = {}; for (let i = 0; i < rest.length; i++) if (rest[i]!.startsWith('--')) flags[rest[i]!.slice(2)] = rest[i + 1] ?? 'true', i++; const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..'); const metaPath = path.join(root, 'connectors/api', id, 'meta.json'); const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(metaPath, 'utf8'))); const mod = (await import(path.join(root, 'connectors/api', id, 'index.ts'))) as { default: (m: typeof meta) => { normalize(raw: any): Promise } }; const connector = mod.default(meta); const site = meta.sourceUrl.replace(/\/+$/, ''); const isWoo = /WooCommerce/i.test(meta.acquisitionMethod ?? ''); /** Shopify occasionally emits raw control characters inside description strings; blank them before parsing. */ function tolerantJson(s: string): unknown { return JSON.parse(s.replace(/[\x00-\x1f]/g, " ")); } async function get(url: string): Promise<{ status: number; body: unknown }> { const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' } }); const text = await res.text(); return { status: res.status, body: tolerantJson(text) }; } const fetchedAt = new Date(); let url: string; let payload: unknown; let recordUrl: string; let externalId: string; if (isWoo) { const cats = (await get(`${site}/wp-json/wc/store/v1/products/categories?per_page=100`)).body as Array<{ id: number; slug: string }>; const cat = cats.find((c) => c.slug === handle); if (!cat) throw new Error(`category ${handle} not found`); url = `${site}/wp-json/wc/store/v1/products?per_page=50&category=${cat.id}`; const list = (await get(url)).body as Array>; const pick = choose(list, (p) => String(p.name)); const p = pick as Record; const trimmed = { id: p.id, name: p.name, slug: p.slug, permalink: p.permalink, sku: p.sku ?? null, short_description: String(p.short_description ?? '').slice(0, 400), prices: p.prices, images: (p.images ?? []).slice(0, 3).map((i: { src: string }) => ({ src: i.src })), categories: (p.categories ?? []).map((c: { id: number; name: string; slug: string }) => ({ id: c.id, name: c.name, slug: c.slug })), tags: (p.tags ?? []).slice(0, 12), is_in_stock: p.is_in_stock ?? null, stock_availability: p.stock_availability, type: p.type, brands: p.brands, }; payload = { category: handle, product: trimmed }; recordUrl = p.permalink; externalId = String(p.id); } else { url = `${site}/collections/${handle}/products.json?limit=50`; const body = (await get(url)).body as { products: Array> }; const p = choose(body.products, (x) => String(x.title)) as Record; const trimmed = { id: p.id, title: p.title, handle: p.handle, body_html: p.body_html ? String(p.body_html).replace(/\s+/g, ' ').slice(0, 400) : null, published_at: p.published_at ?? null, updated_at: p.updated_at ?? null, vendor: p.vendor ?? null, product_type: p.product_type ?? null, tags: (Array.isArray(p.tags) ? p.tags : String(p.tags ?? '').split(',').map((t: string) => t.trim()).filter(Boolean)).slice(0, 12), variants: (p.variants ?? []).slice(0, 6).map((v: Record) => ({ id: v.id, title: v.title ?? null, sku: v.sku ?? null, barcode: v.barcode ?? null, price: v.price ?? null, compare_at_price: v.compare_at_price ?? null, available: v.available ?? null, featured_image: v.featured_image ? { src: v.featured_image.src } : null })), images: (p.images ?? []).slice(0, 3).map((i: { src: string }) => ({ src: i.src })), }; payload = { collection: handle, product: trimmed }; recordUrl = `${site}/products/${p.handle}`; externalId = String(p.id); } function choose(list: T[], title: (x: T) => string): T { if (!list.length) throw new Error(`no products returned by ${url}`); if (flags.pick) { const re = new RegExp(flags.pick, 'i'); const hit = list.find((x) => re.test(title(x))); if (!hit) throw new Error(`no product matching /${flags.pick}/ among: ${list.slice(0, 15).map(title).join(' · ')}`); return hit; } const i = Number(flags.index ?? 0); return list[Math.min(i, list.length - 1)]!; } const raw = { url: recordUrl, externalId, kind: 'listing' as const, engine: 'api' as const, fetchedAt, payload }; const out = (await connector.normalize(raw)) as Array>; if (!out.length) throw new Error(`normalize() produced no listing for "${(payload as any).product.title ?? (payload as any).product.name}" — mapping/exclude rejects it; pick another product`); const first = out[0]!; const expect = { minCount: 1, kinds: ['listing'], requiredFields: ['price', 'currency', 'attributes.categorySlug', 'sourceUrl', 'externalId'], first: { currency: first.currency, 'attributes.categorySlug': first.attributes.categorySlug, listingType: 'fixed_price', ...(first.grade?.grader ? { 'grade.grader': first.grade.grader, 'grade.grade': first.grade.grade } : {}) }, }; saveFixture(id, name, { raw, expect, note: `${flags.note ?? 'Live capture'} — ${url} (trimmed: body ≤ 400 chars, ≤ 3 images, ≤ 6 variants, ≤ 12 tags). Captured ${fetchedAt.toISOString().slice(0, 10)} with the RareIndexBot UA.` }); console.log(`✔ data/fixtures/${id}/${name}.json ← "${(payload as any).product.title ?? (payload as any).product.name}" → ${out.length} listing(s): ${first.attributes.categorySlug} ${first.price} ${first.currency}${first.grade?.grader ? ` grade=${first.grade.grader} ${first.grade.grade}` : ''} avail=${first.availability}`);