import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js'; const BASE = 'https://www.trainz.com'; const PARSER_VERSION = '1.0.0'; export const ProductSchema = z.object({ id: z.number(), title: z.string(), handle: z.string(), vendor: z.string().nullable().default(null), product_type: z.string().nullable().default(null), tags: z.array(z.string()).default([]), created_at: z.string().nullable().default(null), updated_at: z.string().nullable().default(null), variants: z.array(z.object({ id: z.number(), sku: z.string().nullable().default(null), price: z.string(), compare_at_price: z.string().nullable().default(null), available: z.boolean().default(true) })).default([]), images: z.array(z.object({ src: z.string() })).default([]), }); export type Product = z.infer; export const PayloadSchema = z.object({ kind: z.literal('collection_page'), collection: z.string(), page: z.number(), products: z.array(ProductSchema) }); export type Payload = z.infer; /** Keep only the fields we use so raw payloads stay small. */ export function trimProduct(p: Record): Product | null { const parsed = ProductSchema.safeParse(p); if (!parsed.success) return null; const v = parsed.data; return { ...v, variants: v.variants.slice(0, 3), images: v.images.slice(0, 2), tags: v.tags.filter((t) => /^(condition|class|era|scale|gauge|Inventory Type2|roadname|road_name|manufacturer)[:_]/i.test(t) || /^in-stock$|^sold-out$/.test(t)).slice(0, 12) }; } function tag(p: Product, prefix: string): string | null { const t = p.tags.find((x) => x.toLowerCase().startsWith(`${prefix.toLowerCase()}:`)); return t ? t.slice(prefix.length + 1).trim() : null; } /** "Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive LN/Box" → name without the trailing condition code. */ export function splitTitle(title: string): { name: string; conditionCode: string | null } { const m = title.match(/\s+(LN|EX|VG|GD|G|PR|FR|NM|MT|C-?\d{1,2})(?:\/(Box|OB|Sealed|No Box))?\s*$/i); if (!m) return { name: title.trim(), conditionCode: null }; return { name: title.slice(0, m.index).trim(), conditionCode: m[0].trim() }; } const CODE_WORDS: Record = { LN: 'Like New', EX: 'Excellent', VG: 'Very Good', GD: 'Good', G: 'Good', PR: 'Poor', FR: 'Fair', NM: 'Near Mint', MT: 'Mint' }; /** * Trainz.com (world's largest model-train dealer): public Shopify product feed per collection * (/collections//products.json). Catalog facts + dealer asking price with graded condition. */ export class TrainzConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.collections as string[] | undefined) ?? ['lionel-postwar-trains', 'american-flyer-postwar-trains']; const pagesPerCollection = Number(this.meta.config.pagesPerCollection ?? 2); const startIdx = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length; const perRun = Number(this.meta.config.collectionsPerRun ?? 4); let count = 0; for (let k = 0; k < Math.min(perRun, seeds.length); k++) { const handle = seeds[(startIdx + k) % seeds.length]!; for (let page = 1; page <= pagesPerCollection; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}/collections/${handle}/products.json?limit=250&page=${page}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price'], parse: (r) => ({ title: (r.json as { products?: Array<{ title: string }> })?.products?.[0]?.title ?? null, price: (r.json as { products?: Array<{ variants?: Array<{ price: string }> }> })?.products?.[0]?.variants?.[0]?.price ?? null }) }); const list = (res.json as { products?: Record[] } | null)?.products; if (!res.success || !Array.isArray(list)) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const products = list.map(trimProduct).filter((p): p is Product => Boolean(p)); if (products.length === 0) break; count++; yield { url, externalId: `${handle}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'collection_page', collection: handle, page, products } satisfies Payload, fetchedAt: res.fetchedAt }; if (products.length < 250) break; } } await ctx.setCursor({ seedIndex: (startIdx + perRun) % seeds.length, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const pr of p.products) { const v = pr.variants[0]; if (!v) continue; const { name, conditionCode } = splitTitle(pr.title); const condTag = tag(pr, 'condition'); const condRaw = condTag ?? (conditionCode ? CODE_WORDS[conditionCode.split('/')[0]!.toUpperCase()] ?? conditionCode : null); const cond = dealerCondition(condRaw ?? ''); const scale = tag(pr, 'scale') ?? tag(pr, 'gauge') ?? name.match(/\b(HO|N|O|S|G|Z|TT|O27|Standard)\s+(?:Scale|Gauge)\b/i)?.[0] ?? null; const url = `${BASE}/products/${pr.handle}`; const attributes = lotAttributes({ categorySlug: 'model_trains', name, brand: pr.vendor, series: scale, identifiers: { trainz_sku: v.sku ?? String(pr.id), shopify_product_id: String(pr.id) }, metadata: { product_type: pr.product_type, era: tag(pr, 'era'), class: tag(pr, 'class'), tags: pr.tags } }); const price = Number(v.price); const common = { meta: this.meta, sourceUrl: url, externalId: String(pr.id), rawTitle: pr.title, attributes, imageUrls: pr.images.map((i) => i.src), observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness }; out.push(makeCatalogItem({ ...common, confidence: 0.8 })); if (Number.isFinite(price) && price > 0) { out.push(makeListing({ ...common, price, currency: 'USD', listingType: 'fixed_price', seller: 'Trainz', location: 'US', availability: v.available ? 'available' : 'sold', listedAt: pr.created_at ? new Date(pr.created_at) : null, quantity: 1 })); } } return out; } } export default (meta: ConnectorMeta) => new TrainzConnector(meta);