spb/rareindex

1/**2 * Live fixture capture for the g4-shops-intl storefront connectors.3 *4 *   pnpm tsx connectors/api/_g4-shops-intl-lib/capture.ts <connectorId> <collectionHandle> <fixtureName> [--pick <regex>] [--index <n>] [--note "..."]5 *6 * Fetches one public storefront page (Shopify `/collections/<handle>/products.json?limit=50` or the7 * WooCommerce Store API `/wp-json/wc/store/v1/products?category=<id>`), picks one product (first, by8 * index, or first whose title matches --pick), trims it to what `normalize` needs (short body, ≤ 39 * images, ≤ 6 variants, ≤ 12 tags) and saves data/fixtures/<id>/<name>.json with an `expect` block10 * derived from the connector's own normalisation of that live record. Never hand-written (SPEC §14).11 */12import { readFileSync } from 'node:fs';13import path from 'node:path';14import { ConnectorMetaSchema } from '@rareindex/connectors';15import { saveFixture } from '@rareindex/connectors/testing';1617const UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)';18const [id, handle, name, ...rest] = process.argv.slice(2);19if (!id || !handle || !name) {20  console.error('usage: capture.ts <connectorId> <collectionHandle> <fixtureName> [--pick <regex>] [--index <n>] [--note "..."]');21  process.exit(1);22}23const flags: Record<string, string> = {};24for (let i = 0; i < rest.length; i++) if (rest[i]!.startsWith('--')) flags[rest[i]!.slice(2)] = rest[i + 1] ?? 'true', i++;2526const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..');27const metaPath = path.join(root, 'connectors/api', id, 'meta.json');28const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync(metaPath, 'utf8')));29const mod = (await import(path.join(root, 'connectors/api', id, 'index.ts'))) as { default: (m: typeof meta) => { normalize(raw: any): Promise<unknown[]> } };30const connector = mod.default(meta);31const site = meta.sourceUrl.replace(/\/+$/, '');32const isWoo = /WooCommerce/i.test(meta.acquisitionMethod ?? '');3334/** Shopify occasionally emits raw control characters inside description strings; blank them before parsing. */35function tolerantJson(s: string): unknown {36  return JSON.parse(s.replace(/[\x00-\x1f]/g, " "));37}3839async function get(url: string): Promise<{ status: number; body: unknown }> {40  const res = await fetch(url, { headers: { 'user-agent': UA, accept: 'application/json' } });41  const text = await res.text();42  return { status: res.status, body: tolerantJson(text) };43}4445const fetchedAt = new Date();46let url: string;47let payload: unknown;48let recordUrl: string;49let externalId: string;50if (isWoo) {51  const cats = (await get(`${site}/wp-json/wc/store/v1/products/categories?per_page=100`)).body as Array<{ id: number; slug: string }>;52  const cat = cats.find((c) => c.slug === handle);53  if (!cat) throw new Error(`category ${handle} not found`);54  url = `${site}/wp-json/wc/store/v1/products?per_page=50&category=${cat.id}`;55  const list = (await get(url)).body as Array<Record<string, any>>;56  const pick = choose(list, (p) => String(p.name));57  const p = pick as Record<string, any>;58  const trimmed = {59    id: p.id, name: p.name, slug: p.slug, permalink: p.permalink, sku: p.sku ?? null,60    short_description: String(p.short_description ?? '').slice(0, 400), prices: p.prices,61    images: (p.images ?? []).slice(0, 3).map((i: { src: string }) => ({ src: i.src })),62    categories: (p.categories ?? []).map((c: { id: number; name: string; slug: string }) => ({ id: c.id, name: c.name, slug: c.slug })),63    tags: (p.tags ?? []).slice(0, 12), is_in_stock: p.is_in_stock ?? null, stock_availability: p.stock_availability, type: p.type, brands: p.brands,64  };65  payload = { category: handle, product: trimmed };66  recordUrl = p.permalink;67  externalId = String(p.id);68} else {69  url = `${site}/collections/${handle}/products.json?limit=50`;70  const body = (await get(url)).body as { products: Array<Record<string, any>> };71  const p = choose(body.products, (x) => String(x.title)) as Record<string, any>;72  const trimmed = {73    id: p.id, title: p.title, handle: p.handle,74    body_html: p.body_html ? String(p.body_html).replace(/\s+/g, ' ').slice(0, 400) : null,75    published_at: p.published_at ?? null, updated_at: p.updated_at ?? null, vendor: p.vendor ?? null, product_type: p.product_type ?? null,76    tags: (Array.isArray(p.tags) ? p.tags : String(p.tags ?? '').split(',').map((t: string) => t.trim()).filter(Boolean)).slice(0, 12),77    variants: (p.variants ?? []).slice(0, 6).map((v: Record<string, any>) => ({ id: v.id, title: v.title ?? null, sku: v.sku ?? null, barcode: v.barcode ?? null, price: v.price ?? null, compare_at_price: v.compare_at_price ?? null, available: v.available ?? null, featured_image: v.featured_image ? { src: v.featured_image.src } : null })),78    images: (p.images ?? []).slice(0, 3).map((i: { src: string }) => ({ src: i.src })),79  };80  payload = { collection: handle, product: trimmed };81  recordUrl = `${site}/products/${p.handle}`;82  externalId = String(p.id);83}8485function choose<T>(list: T[], title: (x: T) => string): T {86  if (!list.length) throw new Error(`no products returned by ${url}`);87  if (flags.pick) {88    const re = new RegExp(flags.pick, 'i');89    const hit = list.find((x) => re.test(title(x)));90    if (!hit) throw new Error(`no product matching /${flags.pick}/ among: ${list.slice(0, 15).map(title).join(' · ')}`);91    return hit;92  }93  const i = Number(flags.index ?? 0);94  return list[Math.min(i, list.length - 1)]!;95}9697const raw = { url: recordUrl, externalId, kind: 'listing' as const, engine: 'api' as const, fetchedAt, payload };98const out = (await connector.normalize(raw)) as Array<Record<string, any>>;99if (!out.length) throw new Error(`normalize() produced no listing for "${(payload as any).product.title ?? (payload as any).product.name}" — mapping/exclude rejects it; pick another product`);100const first = out[0]!;101const expect = {102  minCount: 1,103  kinds: ['listing'],104  requiredFields: ['price', 'currency', 'attributes.categorySlug', 'sourceUrl', 'externalId'],105  first: { currency: first.currency, 'attributes.categorySlug': first.attributes.categorySlug, listingType: 'fixed_price', ...(first.grade?.grader ? { 'grade.grader': first.grade.grader, 'grade.grade': first.grade.grade } : {}) },106};107saveFixture(id, name, { raw, expect, note: `${flags.note ?? 'Live capture'} — ${url} (trimmed: body ≤ 400 chars, ≤ 3 images, ≤ 6 variants, ≤ 12 tags). Captured ${fetchedAt.toISOString().slice(0, 10)} with the RareIndexBot UA.` });108console.log(`✔ data/fixtures/${id}/${name}.json ← "${(payload as any).product.title ?? (payload as any).product.name}" → ${out.length} listing(s): ${first.attributes.categorySlug} ${first.price} ${first.currency}${first.grade?.grader ? ` grade=${first.grade.grader} ${first.grade.grade}` : ''} avail=${first.availability}`);109