TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import type { NormalizedRecord } from '@rareindex/shared';3import { BaseConnector } from '../base.js';4import type { ConnectorMeta, CrawlContext, RawRecordInput, RawRecordLike } from '../types.js';5import { StorefrontConfigSchema, storefrontListings, type StorefrontProduct } from './storefront.js';67/**8 * Generic Shopify storefront adapter (SPEC §1 "regional storefronts", §5 reusable adapters).9 * Reads the public, unauthenticated storefront JSON that every Shopify shop serves:10 * /collections/<handle>/products.json?limit=250&page=N (or /products.json for the whole shop)11 * One raw record per product; `normalize` yields one listing per variant with SKU/barcode identifiers.12 * Barcodes are only present on the per-product /products/<handle>.js endpoint; set13 * `config.fetchBarcodes: true` to enrich (1 extra request per product — use for small catalogues).14 * Asking prices are listings, never sales (§111).15 */16export const ShopifyConfigSchema = StorefrontConfigSchema.extend({17 fetchBarcodes: z.boolean().default(false),18 /** crawl the whole shop (/products.json) when no collections are configured */19 wholeShop: z.boolean().default(false),20 pageSize: z.number().int().min(1).max(250).default(250),21 /**22 * Shopify Markets pin (ISO-3166 alpha-2). Markets-enabled shops geolocate the requester and convert prices23 * (content-language: en-CA) while products.json carries no currency field → the `localization=<CC>` cookie24 * selects the shop's home market deterministically so prices match `currency`. Defaults to meta.regions[0].25 */26 market: z.string().regex(/^[A-Z]{2}$/).optional(),27});28export type ShopifyConfig = z.infer<typeof ShopifyConfigSchema>;2930const ShopifyVariant = z.object({ id: z.number(), title: z.string().nullable().optional(), sku: z.string().nullable().optional(), barcode: z.string().nullable().optional(), price: z.union([z.string(), z.number()]).nullable().optional(), compare_at_price: z.union([z.string(), z.number()]).nullable().optional(), available: z.boolean().nullable().optional(), featured_image: z.object({ src: z.string().optional() }).nullable().optional(), inventory_quantity: z.number().nullable().optional() });31export const ShopifyProductSchema = z.object({32 id: z.number(),33 title: z.string(),34 handle: z.string(),35 body_html: z.string().nullable().optional(),36 published_at: z.string().nullable().optional(),37 updated_at: z.string().nullable().optional(),38 vendor: z.string().nullable().optional(),39 product_type: z.string().nullable().optional(),40 tags: z.union([z.array(z.string()), z.string()]).optional(),41 variants: z.array(ShopifyVariant).default([]),42 images: z.array(z.object({ src: z.string() })).default([]),43});44export type ShopifyProduct = z.infer<typeof ShopifyProductSchema>;4546export const ShopifyPayloadSchema = z.object({ collection: z.string().nullable(), product: ShopifyProductSchema, barcodes: z.record(z.string(), z.string()).optional() });47export type ShopifyPayload = z.infer<typeof ShopifyPayloadSchema>;4849function priceNum(v: string | number | null | undefined, cents = false): number | null {50 if (v === null || v === undefined || v === '') return null;51 const n = typeof v === 'number' ? v : Number.parseFloat(v);52 if (!Number.isFinite(n) || n <= 0) return null;53 return cents ? n / 100 : n;54}5556export function toStorefrontProduct(site: string, payload: ShopifyPayload): StorefrontProduct {57 const p = payload.product;58 const tags = Array.isArray(p.tags) ? p.tags : (p.tags ?? '').split(',').map((t) => t.trim()).filter(Boolean);59 return {60 id: String(p.id),61 title: p.title,62 url: `${site}/products/${p.handle}`,63 description: p.body_html ?? null,64 vendor: p.vendor ?? null,65 productType: p.product_type ?? null,66 tags,67 collection: payload.collection,68 images: p.images.map((i) => i.src),69 publishedAt: p.published_at ?? null,70 updatedAt: p.updated_at ?? null,71 variants: p.variants.map((v) => ({ id: String(v.id), title: v.title ?? null, sku: v.sku ?? null, barcode: v.barcode ?? payload.barcodes?.[String(v.id)] ?? null, price: priceNum(v.price), compareAtPrice: priceNum(v.compare_at_price), available: v.available ?? null, quantity: v.inventory_quantity ?? null, image: v.featured_image?.src ?? null })),72 };73}7475export class ShopifyStoreConnector extends BaseConnector {76 readonly version = '1.0.0';77 readonly parserVersion = '1.0.0';78 protected override minIntervalMs = 1500;79 protected readonly cfg: ShopifyConfig;80 protected readonly site: string;81 /** market the storefront is pinned to (null when the region is unknown) */82 readonly market: string | null;8384 constructor(meta: ConnectorMeta) {85 super(meta);86 this.cfg = ShopifyConfigSchema.parse(meta.config);87 this.site = meta.sourceUrl.replace(/\/+$/, '');88 const region = this.cfg.market ?? meta.regions[0] ?? null;89 this.market = region && /^[A-Z]{2}$/.test(region) ? region : null;90 this.urlPatterns = [new RegExp(`^${this.site.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/(?:collections/[^/]+/)?products/`, 'i')];91 }9293 override urlPatterns?: RegExp[];9495 /** Every fetch carries the market cookie so Markets-enabled shops answer in their home currency (callers' headers win). */96 protected pinMarket(ctx: CrawlContext): CrawlContext {97 if (!this.market) return ctx;98 const pinned = { cookie: `localization=${this.market}`, 'accept-language': `en-${this.market},en;q=0.9` };99 return { ...ctx, fetch: (url, opts) => ctx.fetch(url, { ...(opts ?? {}), headers: { ...pinned, ...(opts?.headers ?? {}) } }) };100 }101102 private collectionUrl(handle: string | null, page: number): string {103 const base = handle ? `${this.site}/collections/${handle}/products.json` : `${this.site}/products.json`;104 return `${base}?limit=${this.cfg.pageSize}&page=${page}`;105 }106107 async *crawl(rawCtx: CrawlContext): AsyncIterable<RawRecordInput> {108 const ctx = this.pinMarket(rawCtx);109 // ctx.options.seeds (manual runs / probes) may name collection handles to restrict the crawl.110 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : null;111 const configured = this.cfg.collections.length ? this.cfg.collections : this.cfg.wholeShop ? [{ handle: null as string | null, pages: undefined as number | undefined }] : [];112 const collections = seeds ? seeds.map((h) => configured.find((c) => c.handle === h) ?? { handle: h, pages: undefined as number | undefined }) : configured;113 if (!collections.length) {114 ctx.anomaly('config_missing', 'no collections configured and wholeShop=false');115 return;116 }117 const backfill = ctx.options.mode === 'backfill';118 const cursor = (ctx.options.cursor ?? {}) as { collection?: string | null; page?: number };119 let resume = cursor.collection !== undefined;120 let count = 0;121 for (const col of collections) {122 const handle = col.handle;123 if (resume && cursor.collection !== handle) continue;124 const maxPages = backfill ? this.policy.backfillMaxPages : (col.pages ?? this.policy.crawlDepth);125 let page = resume && cursor.page ? cursor.page : 1;126 resume = false;127 for (; page <= maxPages; page++) {128 if (ctx.signal?.aborted || this.reached(ctx, count)) return;129 const url = this.collectionUrl(handle, page);130 await this.throttle(url);131 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0 });132 if (!res.success) {133 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);134 break;135 }136 const parsed = z.object({ products: z.array(z.unknown()) }).safeParse(res.json);137 if (!parsed.success) {138 ctx.anomaly('schema_drift', `${url}: products[] missing`);139 break;140 }141 const products = parsed.data.products.map((x) => ShopifyProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data);142 if (products.length < parsed.data.products.length) ctx.anomaly('parse_failure', `${url}: ${parsed.data.products.length - products.length} products rejected by schema`);143 for (const product of products) {144 let barcodes: Record<string, string> | undefined;145 if (this.cfg.fetchBarcodes) {146 await this.throttle();147 const pj = await ctx.fetch(`${this.site}/products/${product.handle}.js`, { engines: ['api'], responseType: 'json', minQuality: 0 });148 const vars = (pj.json as { variants?: Array<{ id: number; barcode?: string | null }> } | null)?.variants;149 if (vars) barcodes = Object.fromEntries(vars.filter((v) => v.barcode).map((v) => [String(v.id), String(v.barcode)]));150 }151 const payload: ShopifyPayload = { collection: handle, product, barcodes };152 count++;153 yield { url: `${this.site}/products/${product.handle}`, externalId: String(product.id), kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };154 if (this.reached(ctx, count)) return;155 }156 await ctx.setCursor({ collection: handle, page: page + 1, at: new Date().toISOString() });157 await ctx.progress({ page, itemsProcessed: count });158 if (products.length < this.cfg.pageSize) break;159 }160 }161 await ctx.setCursor({ done: true, at: new Date().toISOString() });162 }163164 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {165 const payload = ShopifyPayloadSchema.parse(raw.payload);166 return storefrontListings({ connectorId: this.meta.id, sourceId: this.meta.sourceId, cfg: this.cfg, product: toStorefrontProduct(this.site, payload), observedAt: raw.fetchedAt, parserVersion: this.parserVersion });167 }168169 async lookup(url: string, rawCtx: CrawlContext): Promise<RawRecordInput[]> {170 const ctx = this.pinMarket(rawCtx);171 const handle = url.match(/\/products\/([^/?#]+)/)?.[1];172 if (!handle) return [];173 const res = await ctx.fetch(`${this.site}/products/${handle}.json`, { engines: ['api'], responseType: 'json', minQuality: 0 });174 const parsed = z.object({ product: ShopifyProductSchema }).safeParse(res.json);175 if (!res.success || !parsed.success) return [];176 const payload: ShopifyPayload = { collection: url.match(/\/collections\/([^/]+)\//)?.[1] ?? null, product: parsed.data.product };177 return [{ url: `${this.site}/products/${handle}`, externalId: String(parsed.data.product.id), kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];178 }179}180