spb/rareindex

1import { z } from 'zod';2import type { NormalizedRecord } from '@rareindex/shared';3import { BaseConnector } from '../base.js';4import type { ConnectorMeta, CrawlContext, RawRecordInput, RawRecordLike } from '../types.js';5import { StorefrontConfigSchema, storefrontListings, type StorefrontProduct } from './storefront.js';67/**8 * Generic WooCommerce adapter using the public, unauthenticated Store API that ships with9 * WooCommerce Blocks: /wp-json/wc/store/v1/products?per_page=100&page=N&category=<id>.10 * Prices come in minor units with an explicit currency; categories are resolved by slug through11 * /wp-json/wc/store/v1/products/categories. One raw record per product.12 */13export const WooConfigSchema = StorefrontConfigSchema.extend({14  perPage: z.number().int().min(1).max(100).default(100),15  /** extra query params (e.g. { orderby: 'date', order: 'desc' }) */16  query: z.record(z.string(), z.string()).default({}),17});18export type WooConfig = z.infer<typeof WooConfigSchema>;1920const WooPrices = z.object({ price: z.string().nullable().optional(), regular_price: z.string().nullable().optional(), sale_price: z.string().nullable().optional(), currency_code: z.string().optional(), currency_minor_unit: z.number().optional() });21export const WooProductSchema = z.object({22  id: z.number(),23  name: z.string(),24  slug: z.string().optional(),25  permalink: z.string(),26  sku: z.string().nullable().optional(),27  description: z.string().nullable().optional(),28  short_description: z.string().nullable().optional(),29  prices: WooPrices.optional(),30  images: z.array(z.object({ src: z.string() })).default([]),31  categories: z.array(z.object({ id: z.number().optional(), name: z.string().optional(), slug: z.string().optional() })).default([]),32  tags: z.array(z.object({ name: z.string().optional(), slug: z.string().optional() })).default([]),33  is_in_stock: z.boolean().nullable().optional(),34  stock_availability: z.object({ text: z.string().optional() }).optional(),35  type: z.string().optional(),36  brands: z.array(z.object({ name: z.string().optional() })).optional(),37});38export type WooProduct = z.infer<typeof WooProductSchema>;3940export const WooPayloadSchema = z.object({ category: z.string().nullable(), product: WooProductSchema });41export type WooPayload = z.infer<typeof WooPayloadSchema>;4243function minor(v: string | null | undefined, unit = 2): number | null {44  if (!v) return null;45  const n = Number.parseInt(v, 10);46  return Number.isFinite(n) && n > 0 ? n / 10 ** unit : null;47}4849export function toStorefrontProduct(payload: WooPayload): StorefrontProduct {50  const p = payload.product;51  const unit = p.prices?.currency_minor_unit ?? 2;52  const price = minor(p.prices?.price, unit);53  return {54    id: String(p.id),55    title: p.name,56    url: p.permalink,57    description: p.description || p.short_description || null,58    vendor: p.brands?.[0]?.name ?? null,59    productType: p.categories.map((c) => c.name).filter(Boolean).join(' / ') || null,60    tags: p.tags.map((t) => t.name ?? '').filter(Boolean),61    collection: payload.category,62    images: p.images.map((i) => i.src),63    publishedAt: null,64    updatedAt: null,65    variants: [{ id: String(p.id), title: null, sku: p.sku ?? null, barcode: null, price, compareAtPrice: minor(p.prices?.regular_price, unit), available: p.is_in_stock ?? null, quantity: null, image: null }],66  };67}6869export class WooCommerceStoreConnector extends BaseConnector {70  readonly version = '1.0.0';71  readonly parserVersion = '1.0.0';72  protected override minIntervalMs = 1500;73  protected readonly cfg: WooConfig;74  protected readonly site: string;75  private categoryIds: Map<string, number> | null = null;7677  constructor(meta: ConnectorMeta) {78    super(meta);79    this.cfg = WooConfigSchema.parse(meta.config);80    this.site = meta.sourceUrl.replace(/\/+$/, '');81  }8283  private async resolveCategoryId(ctx: CrawlContext, slug: string): Promise<number | null> {84    if (/^\d+$/.test(slug)) return Number(slug);85    if (!this.categoryIds) {86      this.categoryIds = new Map();87      for (let page = 1; page <= 10; page++) {88        const res = await ctx.fetch(`${this.site}/wp-json/wc/store/v1/products/categories?per_page=100&page=${page}`, { engines: ['api'], responseType: 'json', minQuality: 0 });89        const cats = z.array(z.object({ id: z.number(), slug: z.string() })).safeParse(res.json);90        if (!res.success || !cats.success) break;91        for (const c of cats.data) this.categoryIds.set(c.slug, c.id);92        if (cats.data.length < 100) break;93      }94    }95    return this.categoryIds.get(slug) ?? null;96  }9798  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {99    const collections = this.cfg.collections.length ? this.cfg.collections : [{ handle: '*', pages: undefined as number | undefined }];100    const backfill = ctx.options.mode === 'backfill';101    const cursor = (ctx.options.cursor ?? {}) as { collection?: string; page?: number };102    let resume = cursor.collection !== undefined;103    let count = 0;104    for (const col of collections) {105      if (resume && cursor.collection !== col.handle) continue;106      const catId = col.handle === '*' ? null : await this.resolveCategoryId(ctx, col.handle);107      if (col.handle !== '*' && catId === null) {108        ctx.anomaly('category_missing', col.handle);109        continue;110      }111      const maxPages = backfill ? this.policy.backfillMaxPages : (col.pages ?? this.policy.crawlDepth);112      let page = resume && cursor.page ? cursor.page : 1;113      resume = false;114      for (; page <= maxPages; page++) {115        if (ctx.signal?.aborted || this.reached(ctx, count)) return;116        const q = new URLSearchParams({ per_page: String(this.cfg.perPage), page: String(page), ...this.cfg.query, ...(catId !== null ? { category: String(catId) } : {}) });117        const url = `${this.site}/wp-json/wc/store/v1/products?${q.toString()}`;118        await this.throttle(url);119        const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0, failOnHttpError: false });120        if (!res.success && res.httpStatus !== 400) {121          ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);122          break;123        }124        const list = z.array(z.unknown()).safeParse(res.json);125        if (!list.success) break; // WooCommerce answers 400 past the last page126        const products = list.data.map((x) => WooProductSchema.safeParse(x)).filter((r) => r.success).map((r) => r.data);127        for (const product of products) {128          count++;129          const payload: WooPayload = { category: col.handle === '*' ? null : col.handle, product };130          yield { url: product.permalink, externalId: String(product.id), kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };131          if (this.reached(ctx, count)) return;132        }133        await ctx.setCursor({ collection: col.handle, page: page + 1, at: new Date().toISOString() });134        await ctx.progress({ page, itemsProcessed: count });135        if (products.length < this.cfg.perPage) break;136      }137    }138    await ctx.setCursor({ done: true, at: new Date().toISOString() });139  }140141  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {142    const payload = WooPayloadSchema.parse(raw.payload);143    const cfg = payload.product.prices?.currency_code ? { ...this.cfg, currency: payload.product.prices.currency_code as WooConfig['currency'] } : this.cfg;144    return storefrontListings({ connectorId: this.meta.id, sourceId: this.meta.sourceId, cfg, product: toStorefrontProduct(payload), observedAt: raw.fetchedAt, parserVersion: this.parserVersion });145  }146}147