import { z } from 'zod'; import type { NormalizedRecord } from '@rareindex/shared'; import { BaseConnector } from '../base.js'; import type { ConnectorMeta, CrawlContext, RawRecordInput, RawRecordLike } from '../types.js'; import { StorefrontConfigSchema, storefrontListings, type StorefrontProduct } from './storefront.js'; /** * Generic WooCommerce adapter using the public, unauthenticated Store API that ships with * WooCommerce Blocks: /wp-json/wc/store/v1/products?per_page=100&page=N&category=. * Prices come in minor units with an explicit currency; categories are resolved by slug through * /wp-json/wc/store/v1/products/categories. One raw record per product. */ export const WooConfigSchema = StorefrontConfigSchema.extend({ perPage: z.number().int().min(1).max(100).default(100), /** extra query params (e.g. { orderby: 'date', order: 'desc' }) */ query: z.record(z.string(), z.string()).default({}), }); export type WooConfig = z.infer; const WooPrices = z.object({ price: z.string().nullable().optional(), regular_price: z.string().nullable().optional(), sale_price: z.string().nullable().optional(), currency_code: z.string().optional(), currency_minor_unit: z.number().optional() }); export const WooProductSchema = z.object({ id: z.number(), name: z.string(), slug: z.string().optional(), permalink: z.string(), sku: z.string().nullable().optional(), description: z.string().nullable().optional(), short_description: z.string().nullable().optional(), prices: WooPrices.optional(), images: z.array(z.object({ src: z.string() })).default([]), categories: z.array(z.object({ id: z.number().optional(), name: z.string().optional(), slug: z.string().optional() })).default([]), tags: z.array(z.object({ name: z.string().optional(), slug: z.string().optional() })).default([]), is_in_stock: z.boolean().nullable().optional(), stock_availability: z.object({ text: z.string().optional() }).optional(), type: z.string().optional(), brands: z.array(z.object({ name: z.string().optional() })).optional(), }); export type WooProduct = z.infer; export const WooPayloadSchema = z.object({ category: z.string().nullable(), product: WooProductSchema }); export type WooPayload = z.infer; function minor(v: string | null | undefined, unit = 2): number | null { if (!v) return null; const n = Number.parseInt(v, 10); return Number.isFinite(n) && n > 0 ? n / 10 ** unit : null; } export function toStorefrontProduct(payload: WooPayload): StorefrontProduct { const p = payload.product; const unit = p.prices?.currency_minor_unit ?? 2; const price = minor(p.prices?.price, unit); return { id: String(p.id), title: p.name, url: p.permalink, description: p.description || p.short_description || null, vendor: p.brands?.[0]?.name ?? null, productType: p.categories.map((c) => c.name).filter(Boolean).join(' / ') || null, tags: p.tags.map((t) => t.name ?? '').filter(Boolean), collection: payload.category, images: p.images.map((i) => i.src), publishedAt: null, updatedAt: null, variants: [{ id: String(p.id), title: null, sku: p.sku ?? null, barcode: null, price, compareAtPrice: minor(p.prices?.regular_price, unit), available: p.is_in_stock ?? null, quantity: null, image: null }], }; } export class WooCommerceStoreConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = '1.0.0'; protected override minIntervalMs = 1500; protected readonly cfg: WooConfig; protected readonly site: string; private categoryIds: Map | null = null; constructor(meta: ConnectorMeta) { super(meta); this.cfg = WooConfigSchema.parse(meta.config); this.site = meta.sourceUrl.replace(/\/+$/, ''); } private async resolveCategoryId(ctx: CrawlContext, slug: string): Promise { if (/^\d+$/.test(slug)) return Number(slug); if (!this.categoryIds) { this.categoryIds = new Map(); for (let page = 1; page <= 10; page++) { const res = await ctx.fetch(`${this.site}/wp-json/wc/store/v1/products/categories?per_page=100&page=${page}`, { engines: ['api'], responseType: 'json', minQuality: 0 }); const cats = z.array(z.object({ id: z.number(), slug: z.string() })).safeParse(res.json); if (!res.success || !cats.success) break; for (const c of cats.data) this.categoryIds.set(c.slug, c.id); if (cats.data.length < 100) break; } } return this.categoryIds.get(slug) ?? null; } async *crawl(ctx: CrawlContext): AsyncIterable { const collections = this.cfg.collections.length ? this.cfg.collections : [{ handle: '*', pages: undefined as number | undefined }]; const backfill = ctx.options.mode === 'backfill'; const cursor = (ctx.options.cursor ?? {}) as { collection?: string; page?: number }; let resume = cursor.collection !== undefined; let count = 0; for (const col of collections) { if (resume && cursor.collection !== col.handle) continue; const catId = col.handle === '*' ? null : await this.resolveCategoryId(ctx, col.handle); if (col.handle !== '*' && catId === null) { ctx.anomaly('category_missing', col.handle); continue; } const maxPages = backfill ? this.policy.backfillMaxPages : (col.pages ?? this.policy.crawlDepth); let page = resume && cursor.page ? cursor.page : 1; resume = false; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const q = new URLSearchParams({ per_page: String(this.cfg.perPage), page: String(page), ...this.cfg.query, ...(catId !== null ? { category: String(catId) } : {}) }); const url = `${this.site}/wp-json/wc/store/v1/products?${q.toString()}`; await this.throttle(url); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0, failOnHttpError: false }); if (!res.success && res.httpStatus !== 400) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const list = z.array(z.unknown()).safeParse(res.json); if (!list.success) break; // WooCommerce answers 400 past the last page const products = list.data.map((x) => WooProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data); for (const product of products) { count++; const payload: WooPayload = { category: col.handle === '*' ? null : col.handle, product }; yield { url: product.permalink, externalId: String(product.id), kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (this.reached(ctx, count)) return; } await ctx.setCursor({ collection: col.handle, page: page + 1, at: new Date().toISOString() }); await ctx.progress({ page, itemsProcessed: count }); if (products.length < this.cfg.perPage) break; } } await ctx.setCursor({ done: true, at: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const payload = WooPayloadSchema.parse(raw.payload); const cfg = payload.product.prices?.currency_code ? { ...this.cfg, currency: payload.product.prices.currency_code as WooConfig['currency'] } : this.cfg; return storefrontListings({ connectorId: this.meta.id, sourceId: this.meta.sourceId, cfg, product: toStorefrontProduct(payload), observedAt: raw.fetchedAt, parserVersion: this.parserVersion }); } }