TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js';56const BASE = 'https://www.trainz.com';7const PARSER_VERSION = '1.0.0';89export const ProductSchema = z.object({10 id: z.number(),11 title: z.string(),12 handle: z.string(),13 vendor: z.string().nullable().default(null),14 product_type: z.string().nullable().default(null),15 tags: z.array(z.string()).default([]),16 created_at: z.string().nullable().default(null),17 updated_at: z.string().nullable().default(null),18 variants: z.array(z.object({ id: z.number(), sku: z.string().nullable().default(null), price: z.string(), compare_at_price: z.string().nullable().default(null), available: z.boolean().default(true) })).default([]),19 images: z.array(z.object({ src: z.string() })).default([]),20});21export type Product = z.infer<typeof ProductSchema>;22export const PayloadSchema = z.object({ kind: z.literal('collection_page'), collection: z.string(), page: z.number(), products: z.array(ProductSchema) });23export type Payload = z.infer<typeof PayloadSchema>;2425/** Keep only the fields we use so raw payloads stay small. */26export function trimProduct(p: Record<string, unknown>): Product | null {27 const parsed = ProductSchema.safeParse(p);28 if (!parsed.success) return null;29 const v = parsed.data;30 return { ...v, variants: v.variants.slice(0, 3), images: v.images.slice(0, 2), tags: v.tags.filter((t) => /^(condition|class|era|scale|gauge|Inventory Type2|roadname|road_name|manufacturer)[:_]/i.test(t) || /^in-stock$|^sold-out$/.test(t)).slice(0, 12) };31}3233function tag(p: Product, prefix: string): string | null {34 const t = p.tags.find((x) => x.toLowerCase().startsWith(`${prefix.toLowerCase()}:`));35 return t ? t.slice(prefix.length + 1).trim() : null;36}3738/** "Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive LN/Box" → name without the trailing condition code. */39export function splitTitle(title: string): { name: string; conditionCode: string | null } {40 const m = title.match(/\s+(LN|EX|VG|GD|G|PR|FR|NM|MT|C-?\d{1,2})(?:\/(Box|OB|Sealed|No Box))?\s*$/i);41 if (!m) return { name: title.trim(), conditionCode: null };42 return { name: title.slice(0, m.index).trim(), conditionCode: m[0].trim() };43}4445const CODE_WORDS: Record<string, string> = { LN: 'Like New', EX: 'Excellent', VG: 'Very Good', GD: 'Good', G: 'Good', PR: 'Poor', FR: 'Fair', NM: 'Near Mint', MT: 'Mint' };4647/**48 * Trainz.com (world's largest model-train dealer): public Shopify product feed per collection49 * (/collections/<handle>/products.json). Catalog facts + dealer asking price with graded condition.50 */51export class TrainzConnector extends BaseConnector {52 readonly version = '1.0.0';53 readonly parserVersion = PARSER_VERSION;54 protected override minIntervalMs = 1500;5556 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {57 const seeds = (this.meta.config.collections as string[] | undefined) ?? ['lionel-postwar-trains', 'american-flyer-postwar-trains'];58 const pagesPerCollection = Number(this.meta.config.pagesPerCollection ?? 2);59 const startIdx = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length;60 const perRun = Number(this.meta.config.collectionsPerRun ?? 4);61 let count = 0;62 for (let k = 0; k < Math.min(perRun, seeds.length); k++) {63 const handle = seeds[(startIdx + k) % seeds.length]!;64 for (let page = 1; page <= pagesPerCollection; page++) {65 if (ctx.signal?.aborted || this.reached(ctx, count)) break;66 const url = `${BASE}/collections/${handle}/products.json?limit=250&page=${page}`;67 await this.throttle();68 const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price'], parse: (r) => ({ title: (r.json as { products?: Array<{ title: string }> })?.products?.[0]?.title ?? null, price: (r.json as { products?: Array<{ variants?: Array<{ price: string }> }> })?.products?.[0]?.variants?.[0]?.price ?? null }) });69 const list = (res.json as { products?: Record<string, unknown>[] } | null)?.products;70 if (!res.success || !Array.isArray(list)) {71 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);72 break;73 }74 const products = list.map(trimProduct).filter((p): p is Product => Boolean(p));75 if (products.length === 0) break;76 count++;77 yield { url, externalId: `${handle}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'collection_page', collection: handle, page, products } satisfies Payload, fetchedAt: res.fetchedAt };78 if (products.length < 250) break;79 }80 }81 await ctx.setCursor({ seedIndex: (startIdx + perRun) % seeds.length, updatedAt: new Date().toISOString() });82 }8384 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {85 const p = PayloadSchema.parse(raw.payload);86 const out: NormalizedRecord[] = [];87 for (const pr of p.products) {88 const v = pr.variants[0];89 if (!v) continue;90 const { name, conditionCode } = splitTitle(pr.title);91 const condTag = tag(pr, 'condition');92 const condRaw = condTag ?? (conditionCode ? CODE_WORDS[conditionCode.split('/')[0]!.toUpperCase()] ?? conditionCode : null);93 const cond = dealerCondition(condRaw ?? '');94 const scale = tag(pr, 'scale') ?? tag(pr, 'gauge') ?? name.match(/\b(HO|N|O|S|G|Z|TT|O27|Standard)\s+(?:Scale|Gauge)\b/i)?.[0] ?? null;95 const url = `${BASE}/products/${pr.handle}`;96 const attributes = lotAttributes({ categorySlug: 'model_trains', name, brand: pr.vendor, series: scale, identifiers: { trainz_sku: v.sku ?? String(pr.id), shopify_product_id: String(pr.id) }, metadata: { product_type: pr.product_type, era: tag(pr, 'era'), class: tag(pr, 'class'), tags: pr.tags } });97 const price = Number(v.price);98 const common = { meta: this.meta, sourceUrl: url, externalId: String(pr.id), rawTitle: pr.title, attributes, imageUrls: pr.images.map((i) => i.src), observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness };99 out.push(makeCatalogItem({ ...common, confidence: 0.8 }));100 if (Number.isFinite(price) && price > 0) {101 out.push(makeListing({ ...common, price, currency: 'USD', listingType: 'fixed_price', seller: 'Trainz', location: 'US', availability: v.available ? 'available' : 'sold', listedAt: pr.created_at ? new Date(pr.created_at) : null, quantity: 1 }));102 }103 }104 return out;105 }106}107108export default (meta: ConnectorMeta) => new TrainzConnector(meta);109