import { z } from 'zod'; import type { NormalizedRecord } from '@rareindex/shared'; import { BaseConnector } from '../base.js'; import type { ConnectorMeta, CrawlContext, RawRecordInput, RawRecordLike } from '../types.js'; import { StorefrontConfigSchema, storefrontListings, type StorefrontProduct } from './storefront.js'; /** * Generic Shopify storefront adapter (SPEC §1 "regional storefronts", §5 reusable adapters). * Reads the public, unauthenticated storefront JSON that every Shopify shop serves: * /collections//products.json?limit=250&page=N (or /products.json for the whole shop) * One raw record per product; `normalize` yields one listing per variant with SKU/barcode identifiers. * Barcodes are only present on the per-product /products/.js endpoint; set * `config.fetchBarcodes: true` to enrich (1 extra request per product — use for small catalogues). * Asking prices are listings, never sales (§111). */ export const ShopifyConfigSchema = StorefrontConfigSchema.extend({ fetchBarcodes: z.boolean().default(false), /** crawl the whole shop (/products.json) when no collections are configured */ wholeShop: z.boolean().default(false), pageSize: z.number().int().min(1).max(250).default(250), /** * Shopify Markets pin (ISO-3166 alpha-2). Markets-enabled shops geolocate the requester and convert prices * (content-language: en-CA) while products.json carries no currency field → the `localization=` cookie * selects the shop's home market deterministically so prices match `currency`. Defaults to meta.regions[0]. */ market: z.string().regex(/^[A-Z]{2}$/).optional(), }); export type ShopifyConfig = z.infer; const ShopifyVariant = z.object({ id: z.number(), title: z.string().nullable().optional(), sku: z.string().nullable().optional(), barcode: z.string().nullable().optional(), price: z.union([z.string(), z.number()]).nullable().optional(), compare_at_price: z.union([z.string(), z.number()]).nullable().optional(), available: z.boolean().nullable().optional(), featured_image: z.object({ src: z.string().optional() }).nullable().optional(), inventory_quantity: z.number().nullable().optional() }); export const ShopifyProductSchema = z.object({ id: z.number(), title: z.string(), handle: z.string(), body_html: z.string().nullable().optional(), published_at: z.string().nullable().optional(), updated_at: z.string().nullable().optional(), vendor: z.string().nullable().optional(), product_type: z.string().nullable().optional(), tags: z.union([z.array(z.string()), z.string()]).optional(), variants: z.array(ShopifyVariant).default([]), images: z.array(z.object({ src: z.string() })).default([]), }); export type ShopifyProduct = z.infer; export const ShopifyPayloadSchema = z.object({ collection: z.string().nullable(), product: ShopifyProductSchema, barcodes: z.record(z.string(), z.string()).optional() }); export type ShopifyPayload = z.infer; function priceNum(v: string | number | null | undefined, cents = false): number | null { if (v === null || v === undefined || v === '') return null; const n = typeof v === 'number' ? v : Number.parseFloat(v); if (!Number.isFinite(n) || n <= 0) return null; return cents ? n / 100 : n; } export function toStorefrontProduct(site: string, payload: ShopifyPayload): StorefrontProduct { const p = payload.product; const tags = Array.isArray(p.tags) ? p.tags : (p.tags ?? '').split(',').map((t) => t.trim()).filter(Boolean); return { id: String(p.id), title: p.title, url: `${site}/products/${p.handle}`, description: p.body_html ?? null, vendor: p.vendor ?? null, productType: p.product_type ?? null, tags, collection: payload.collection, images: p.images.map((i) => i.src), publishedAt: p.published_at ?? null, updatedAt: p.updated_at ?? null, variants: p.variants.map((v) => ({ id: String(v.id), title: v.title ?? null, sku: v.sku ?? null, barcode: v.barcode ?? payload.barcodes?.[String(v.id)] ?? null, price: priceNum(v.price), compareAtPrice: priceNum(v.compare_at_price), available: v.available ?? null, quantity: v.inventory_quantity ?? null, image: v.featured_image?.src ?? null })), }; } export class ShopifyStoreConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = '1.0.0'; protected override minIntervalMs = 1500; protected readonly cfg: ShopifyConfig; protected readonly site: string; /** market the storefront is pinned to (null when the region is unknown) */ readonly market: string | null; constructor(meta: ConnectorMeta) { super(meta); this.cfg = ShopifyConfigSchema.parse(meta.config); this.site = meta.sourceUrl.replace(/\/+$/, ''); const region = this.cfg.market ?? meta.regions[0] ?? null; this.market = region && /^[A-Z]{2}$/.test(region) ? region : null; this.urlPatterns = [new RegExp(`^${this.site.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/(?:collections/[^/]+/)?products/`, 'i')]; } override urlPatterns?: RegExp[]; /** Every fetch carries the market cookie so Markets-enabled shops answer in their home currency (callers' headers win). */ protected pinMarket(ctx: CrawlContext): CrawlContext { if (!this.market) return ctx; const pinned = { cookie: `localization=${this.market}`, 'accept-language': `en-${this.market},en;q=0.9` }; return { ...ctx, fetch: (url, opts) => ctx.fetch(url, { ...(opts ?? {}), headers: { ...pinned, ...(opts?.headers ?? {}) } }) }; } private collectionUrl(handle: string | null, page: number): string { const base = handle ? `${this.site}/collections/${handle}/products.json` : `${this.site}/products.json`; return `${base}?limit=${this.cfg.pageSize}&page=${page}`; } async *crawl(rawCtx: CrawlContext): AsyncIterable { const ctx = this.pinMarket(rawCtx); // ctx.options.seeds (manual runs / probes) may name collection handles to restrict the crawl. const seeds = ctx.options.seeds?.length ? ctx.options.seeds : null; const configured = this.cfg.collections.length ? this.cfg.collections : this.cfg.wholeShop ? [{ handle: null as string | null, pages: undefined as number | undefined }] : []; const collections = seeds ? seeds.map((h) => configured.find((c) => c.handle === h) ?? { handle: h, pages: undefined as number | undefined }) : configured; if (!collections.length) { ctx.anomaly('config_missing', 'no collections configured and wholeShop=false'); return; } const backfill = ctx.options.mode === 'backfill'; const cursor = (ctx.options.cursor ?? {}) as { collection?: string | null; page?: number }; let resume = cursor.collection !== undefined; let count = 0; for (const col of collections) { const handle = col.handle; if (resume && cursor.collection !== handle) continue; const maxPages = backfill ? this.policy.backfillMaxPages : (col.pages ?? this.policy.crawlDepth); let page = resume && cursor.page ? cursor.page : 1; resume = false; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = this.collectionUrl(handle, page); await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0 }); if (!res.success) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const parsed = z.object({ products: z.array(z.unknown()) }).safeParse(res.json); if (!parsed.success) { ctx.anomaly('schema_drift', `${url}: products[] missing`); break; } const products = parsed.data.products.map((x) => ShopifyProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data); if (products.length < parsed.data.products.length) ctx.anomaly('parse_failure', `${url}: ${parsed.data.products.length - products.length} products rejected by schema`); for (const product of products) { let barcodes: Record | undefined; if (this.cfg.fetchBarcodes) { await this.throttle(); const pj = await ctx.fetch(`${this.site}/products/${product.handle}.js`, { engines: ['api'], responseType: 'json', minQuality: 0 }); const vars = (pj.json as { variants?: Array<{ id: number; barcode?: string | null }> } | null)?.variants; if (vars) barcodes = Object.fromEntries(vars.filter((v) => v.barcode).map((v) => [String(v.id), String(v.barcode)])); } const payload: ShopifyPayload = { collection: handle, product, barcodes }; count++; yield { url: `${this.site}/products/${product.handle}`, externalId: String(product.id), kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (this.reached(ctx, count)) return; } await ctx.setCursor({ collection: handle, page: page + 1, at: new Date().toISOString() }); await ctx.progress({ page, itemsProcessed: count }); if (products.length < this.cfg.pageSize) break; } } await ctx.setCursor({ done: true, at: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const payload = ShopifyPayloadSchema.parse(raw.payload); return storefrontListings({ connectorId: this.meta.id, sourceId: this.meta.sourceId, cfg: this.cfg, product: toStorefrontProduct(this.site, payload), observedAt: raw.fetchedAt, parserVersion: this.parserVersion }); } async lookup(url: string, rawCtx: CrawlContext): Promise { const ctx = this.pinMarket(rawCtx); const handle = url.match(/\/products\/([^/?#]+)/)?.[1]; if (!handle) return []; const res = await ctx.fetch(`${this.site}/products/${handle}.json`, { engines: ['api'], responseType: 'json', minQuality: 0 }); const parsed = z.object({ product: ShopifyProductSchema }).safeParse(res.json); if (!res.success || !parsed.success) return []; const payload: ShopifyPayload = { collection: url.match(/\/collections\/([^/]+)\//)?.[1] ?? null, product: parsed.data.product }; return [{ url: `${this.site}/products/${handle}`, externalId: String(parsed.data.product.id), kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } }