TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { AssetAttributesSchema, CurrencySchema, NormalizedListingSchema, type AssetAttributes, type NormalizedListing } from '@rareindex/shared';3import { parseGradeFromTitle } from '@rareindex/taxonomy';45/**6 * Shared configuration and normalisation for storefront-platform adapters (Shopify, WooCommerce…).7 * A store connector is metadata only: which collections/categories to read and how they map onto the8 * RareIndex taxonomy. Anything unmapped is skipped — never guessed (§192).9 */10export const StorefrontRuleSchema = z.object({11 /** regex tested against "title | product_type | tags | collection" (case-insensitive) */12 match: z.string(),13 /** optional extra conditions: regex on product_type only / on vendor only */14 typeMatch: z.string().optional(),15 vendorMatch: z.string().optional(),16 categorySlug: z.string(),17 brand: z.string().nullable().optional(),18 franchise: z.string().nullable().optional(),19 series: z.string().nullable().optional(),20});2122export const StorefrontCollectionSchema = z.object({23 /** collection handle (Shopify) or category slug/id (WooCommerce) */24 handle: z.string(),25 categorySlug: z.string().nullable().optional(),26 brand: z.string().nullable().optional(),27 franchise: z.string().nullable().optional(),28 series: z.string().nullable().optional(),29 /** pages to read per incremental run (defaults to the domain crawlDepth) */30 pages: z.number().int().positive().optional(),31});3233export const StorefrontConfigSchema = z.object({34 currency: CurrencySchema,35 collections: z.array(StorefrontCollectionSchema).default([]),36 /** title/tag rules applied after the collection mapping (first match wins) */37 rules: z.array(StorefrontRuleSchema).default([]),38 /** used when neither collection nor rule maps the product; null = skip */39 defaultCategory: z.string().nullable().default(null),40 /** products whose title/type/tags match are skipped (accessories, supplies, gift cards…) */41 exclude: z.string().default('gift card|sleeves?|deck box|binder|playmat|toploader|top loader|storage box|dice|tokens? only|bundle of|mystery'),42 /** seller label stored on listings */43 seller: z.string().optional(),44 /** treat out-of-stock items as `ended` listings (default) or drop them */45 keepOutOfStock: z.boolean().default(true),46 /** regex with named groups name/set/number/style/year applied to titles, e.g. "^(?<name>.+?) - (?<set>.+?) #(?<number>\\S+)"; `style` → identifiers.style_code */47 titlePattern: z.string().optional(),48 location: z.string().nullable().default(null),49});50export type StorefrontConfig = z.infer<typeof StorefrontConfigSchema>;5152export interface StorefrontProduct {53 id: string;54 title: string;55 url: string;56 description: string | null;57 vendor: string | null;58 productType: string | null;59 tags: string[];60 collection: string | null;61 images: string[];62 publishedAt: string | null;63 updatedAt: string | null;64 variants: Array<{ id: string; title: string | null; sku: string | null; barcode: string | null; price: number | null; compareAtPrice: number | null; available: boolean | null; quantity: number | null; image: string | null }>;65}6667export interface Mapping {68 categorySlug: string;69 brand: string | null;70 franchise: string | null;71 series: string | null;72}7374export function mapProduct(cfg: StorefrontConfig, p: StorefrontProduct): Mapping | null {75 const hay = [p.title, p.productType ?? '', p.tags.join(' '), p.collection ?? ''].join(' | ');76 if (cfg.exclude && new RegExp(cfg.exclude, 'i').test(hay)) return null;77 const col = p.collection ? cfg.collections.find((c) => c.handle === p.collection) : undefined;78 let m: Mapping | null = col?.categorySlug ? { categorySlug: col.categorySlug, brand: col.brand ?? null, franchise: col.franchise ?? null, series: col.series ?? null } : null;79 for (const r of cfg.rules) {80 if (r.typeMatch && !new RegExp(r.typeMatch, 'i').test(p.productType ?? '')) continue;81 if (r.vendorMatch && !new RegExp(r.vendorMatch, 'i').test(p.vendor ?? '')) continue;82 if (new RegExp(r.match, 'i').test(hay)) {83 m = { categorySlug: r.categorySlug, brand: r.brand ?? m?.brand ?? null, franchise: r.franchise ?? m?.franchise ?? null, series: r.series ?? m?.series ?? null };84 break;85 }86 }87 if (!m && cfg.defaultCategory) m = { categorySlug: cfg.defaultCategory, brand: null, franchise: null, series: null };88 if (!m && col && !col.categorySlug) return null;89 return m;90}9192/** UPC (12), EAN-13, EAN-8, JAN (EAN-13 starting 45/49), ISBN-13 (978/979) from a barcode string. */93export function identifiersFromBarcode(barcode: string | null | undefined): Record<string, string> {94 if (!barcode) return {};95 const d = barcode.replace(/\D/g, '');96 if (d.length === 12) return { upc: d };97 if (d.length === 13) {98 if (/^97[89]/.test(d)) return { isbn: d, ean: d };99 if (/^4[59]/.test(d)) return { jan: d, ean: d };100 return { ean: d };101 }102 if (d.length === 8) return { ean: d };103 if (d.length === 10 && /^\d{9}[\dXx]$/.test(barcode)) return { isbn: barcode.toUpperCase() };104 return {};105}106107const COND_RE = /\b(near mint|nm|lightly played|lp|moderately played|mp|heavily played|hp|damaged|dmg|mint|sealed|new|used|cib|loose|complete|graded)\b/i;108109export function variantCondition(variantTitle: string | null): string | null {110 if (!variantTitle || /^default title$/i.test(variantTitle)) return null;111 const m = variantTitle.match(COND_RE);112 return m ? m[0] : null;113}114115/** Normalise one storefront product into listings (one per variant). */116export function storefrontListings(input: { connectorId: string; sourceId: string; cfg: StorefrontConfig; product: StorefrontProduct; observedAt: Date; parserVersion: string; confidence?: number }): NormalizedListing[] {117 const { cfg, product: p } = input;118 const mapping = mapProduct(cfg, p);119 if (!mapping) return [];120 const out: NormalizedListing[] = [];121 const grade = parseGradeFromTitle(p.title);122 let name = p.title.replace(/\s+/g, ' ').trim();123 let set: string | null = null;124 let number: string | null = null;125 let styleCode: string | null = null;126 let patternYear: number | null = null;127 if (cfg.titlePattern) {128 const m = p.title.match(new RegExp(cfg.titlePattern, 'i'));129 if (m?.groups) {130 name = m.groups.name?.trim() || name;131 set = m.groups.set?.trim() || null;132 number = m.groups.number?.trim() || null;133 styleCode = m.groups.style?.trim().toUpperCase() || null;134 patternYear = m.groups.year ? Number(m.groups.year) : null;135 }136 }137 const yearMatch = p.title.match(/\b(19[0-9]{2}|20[0-4][0-9])\b/);138 for (const v of p.variants) {139 if (v.price === null) continue;140 const available = v.available === null ? 'unknown' : v.available ? 'available' : 'ended';141 if (!cfg.keepOutOfStock && available === 'ended') continue;142 const variantTitle = v.title && !/^default title$/i.test(v.title) ? v.title : null;143 const vGrade = variantTitle ? parseGradeFromTitle(variantTitle) : { grader: null, grade: null, qualifier: null };144 const condRaw = variantCondition(variantTitle);145 const attributes: AssetAttributes = AssetAttributesSchema.parse({146 categorySlug: mapping.categorySlug,147 name,148 brand: mapping.brand ?? p.vendor ?? null,149 franchise: mapping.franchise,150 series: mapping.series,151 set,152 number,153 year: patternYear ?? (yearMatch ? Number(yearMatch[1]) : null),154 identifiers: { ...(v.sku ? { sku: v.sku } : {}), ...(styleCode ? { style_code: styleCode } : {}), ...identifiersFromBarcode(v.barcode) },155 metadata: { productType: p.productType, tags: p.tags.slice(0, 20), collection: p.collection, variantTitle, compareAtPrice: v.compareAtPrice },156 });157 out.push(158 NormalizedListingSchema.parse({159 kind: 'listing',160 connectorId: input.connectorId,161 sourceId: input.sourceId,162 sourceUrl: p.url,163 externalId: p.variants.length > 1 ? `${p.id}:${v.id}` : p.id,164 rawTitle: variantTitle ? `${p.title} — ${variantTitle}` : p.title,165 description: p.description ? p.description.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 2000) || null : null,166 imageUrls: v.image ? [v.image, ...p.images.filter((i) => i !== v.image)].slice(0, 8) : p.images.slice(0, 8),167 attributes,168 grade: { grader: vGrade.grader ?? grade.grader, grade: vGrade.grade ?? grade.grade, qualifier: vGrade.qualifier ?? grade.qualifier, certificationNumber: null },169 condition: { conditionRaw: condRaw, completeness: /\bsealed\b/i.test(`${p.title} ${variantTitle ?? ''}`) ? 'sealed' : null },170 observedAt: input.observedAt,171 confidence: input.confidence ?? 0.7,172 parserVersion: input.parserVersion,173 listingType: 'fixed_price',174 price: v.price,175 currency: cfg.currency,176 seller: cfg.seller ?? null,177 location: cfg.location,178 quantity: v.quantity,179 listedAt: p.publishedAt ? new Date(p.publishedAt) : null,180 availability: available,181 }),182 );183 }184 return out;185}186