import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { decodeEntities } from '../_wlib/index.js'; /** * Lukie Games — US retro video-game dealer. Its storefront search is the public SearchSpring JSON * feed the site itself calls (siteId dytuzo); one result = one product with platform (extrafield5), * genre (extrafield3), price, list price, SKU, stock message and image. Dealer asking prices only. */ const API = 'https://dytuzo.a.searchspring.io/api/search/search.json'; const PARSER_VERSION = '1.0.0'; export const ResultSchema = z.object({ uid: z.string(), sku: z.string().nullable().default(null), name: z.string(), brand: z.string().nullable().default(null), platform: z.string().nullable().default(null), genre: z.string().nullable().default(null), price: z.number().nullable(), msrp: z.number().nullable(), url: z.string(), image: z.string().nullable().default(null), stock: z.string().nullable().default(null), onsale: z.boolean().default(false), }); export type Result = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), platform: z.string(), page: z.number(), total: z.number().nullable(), results: z.array(ResultSchema) }); export type PagePayload = z.infer; const num = (v: unknown) => { const n = Number(v); return Number.isFinite(n) && n > 0 ? n : null; }; export function parseApiPage(json: unknown, url: string, platform: string, page: number): PagePayload | null { const j = json as { results?: Array>; pagination?: { totalResults?: number } } | null; if (!j || !Array.isArray(j.results)) return null; const results: Result[] = []; for (const r of j.results) { const parsed = ResultSchema.safeParse({ uid: String(r.uid ?? r.id ?? ''), sku: r.sku ? String(r.sku) : null, name: decodeEntities(String(r.name ?? '')), brand: r.brand ? String(r.brand) : null, platform: r.extrafield5 ? decodeEntities(String(r.extrafield5)) : null, genre: r.extrafield3 ? decodeEntities(String(r.extrafield3)) : null, price: num(r.price), msrp: num(r.msrp), url: String(r.url ?? ''), image: r.imageUrl ? decodeEntities(String(r.imageUrl)) : null, stock: r.stock_message ? String(r.stock_message) : null, onsale: r.onsale === '1' || r.onsale === 1 || r.onsale === true, }); if (parsed.success && parsed.data.name && parsed.data.url) results.push(parsed.data); } return { kind: 'search_page', url, platform, page, total: typeof j.pagination?.totalResults === 'number' ? j.pagination.totalResults : null, results }; } /** Lukie platform label → PriceCharting-style console name + taxonomy slug. */ export const PLATFORMS: Record = { 'Nintendo 64': { set: 'Nintendo 64', slug: 'nintendo_games' }, 'Super Nintendo': { set: 'Super Nintendo', slug: 'nintendo_games' }, 'Nintendo NES': { set: 'NES', slug: 'nintendo_games' }, NES: { set: 'NES', slug: 'nintendo_games' }, Gamecube: { set: 'Gamecube', slug: 'nintendo_games' }, 'Nintendo Gamecube': { set: 'Gamecube', slug: 'nintendo_games' }, 'Nintendo Wii': { set: 'Wii', slug: 'nintendo_games' }, 'Wii U': { set: 'Wii U', slug: 'nintendo_games' }, 'Nintendo Switch': { set: 'Nintendo Switch', slug: 'nintendo_games' }, Gameboy: { set: 'GameBoy', slug: 'nintendo_games' }, 'Gameboy Color': { set: 'GameBoy Color', slug: 'nintendo_games' }, 'Gameboy Advance': { set: 'GameBoy Advance', slug: 'nintendo_games' }, 'Nintendo DS': { set: 'Nintendo DS', slug: 'nintendo_games' }, 'Nintendo 3DS': { set: 'Nintendo 3DS', slug: 'nintendo_games' }, 'Virtual Boy': { set: 'Virtual Boy', slug: 'nintendo_games' }, Playstation: { set: 'Playstation', slug: 'playstation_games' }, 'Playstation 2': { set: 'Playstation 2', slug: 'playstation_games' }, 'Playstation 3': { set: 'Playstation 3', slug: 'playstation_games' }, 'Playstation 4': { set: 'Playstation 4', slug: 'playstation_games' }, PSP: { set: 'PSP', slug: 'playstation_games' }, 'PS Vita': { set: 'Playstation Vita', slug: 'playstation_games' }, Xbox: { set: 'Xbox', slug: 'xbox_games' }, 'Xbox 360': { set: 'Xbox 360', slug: 'xbox_games' }, 'Xbox One': { set: 'Xbox One', slug: 'xbox_games' }, 'Sega Genesis': { set: 'Sega Genesis', slug: 'sega_games' }, 'Sega Dreamcast': { set: 'Sega Dreamcast', slug: 'sega_games' }, 'Sega Saturn': { set: 'Sega Saturn', slug: 'sega_games' }, 'Sega Master System': { set: 'Sega Master System', slug: 'sega_games' }, 'Sega Game Gear': { set: 'Sega Game Gear', slug: 'sega_games' }, 'Sega CD': { set: 'Sega CD', slug: 'sega_games' }, 'Sega 32X': { set: 'Sega 32X', slug: 'sega_games' }, 'Atari 2600': { set: 'Atari 2600', slug: 'atari_retro_games' }, 'Atari 5200': { set: 'Atari 5200', slug: 'atari_retro_games' }, 'Atari 7800': { set: 'Atari 7800', slug: 'atari_retro_games' }, 'Atari Jaguar': { set: 'Jaguar', slug: 'atari_retro_games' }, 'Neo Geo Pocket': { set: 'Neo Geo Pocket Color', slug: 'atari_retro_games' }, TurboGrafx: { set: 'TurboGrafx-16', slug: 'atari_retro_games' }, Intellivision: { set: 'Intellivision', slug: 'atari_retro_games' }, Colecovision: { set: 'Colecovision', slug: 'atari_retro_games' }, }; const HARDWARE = /\b(system|console|controller|adapter|cable|memory card|expansion pak|rumble pak|power supply|av cable|accessor(?:y|ies)|carrying case|cover|skin|stylus|charger|headset|strategy guide|magazine)\b/i; const NOT_A_GAME = /^\s*(manual|box|instructions?|poster|insert|map|sleeve)\b|\b(manual only|box only|instructions only)\b/i; /** "GoldenEye 007" / "Super Mario 64 Game" / "Zelda Ocarina of Time N64 Game Complete" → { name, completeness } */ export function splitName(name: string, platform: string | null): { name: string; completeness: 'loose' | 'cib' | 'sealed' | null; condition: string | null } { let n = name.replace(/\s+/g, ' ').trim(); let completeness: 'loose' | 'cib' | 'sealed' | null = null; if (/\b(brand new|new sealed|factory sealed|sealed)\b/i.test(n)) completeness = 'sealed'; else if (/\b(complete in box|complete|cib|boxed|with box)\b/i.test(n)) completeness = 'cib'; else if (/\b(cartridge only|cart only|disc only|game only|loose)\b/i.test(n)) completeness = 'loose'; n = n.replace(/\b(brand new|new sealed|factory sealed|sealed|complete in box|complete|cib|boxed|with box|cartridge only|cart only|disc only|game only|loose)\b/gi, ' '); if (platform) n = n.replace(new RegExp(`\\b${platform.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i'), ' '); n = n.replace(/\bNintendo\s+(N64|NES|SNES|DS|3DS|Wii)\b/gi, ' ').replace(/\b(N64|SNES|NES|GBA|GBC|PS1|PS2|PS3|PS4|PSP|GC|Wii U|Wii)\b\s*(Game)?/gi, ' ').replace(/\bGame\b\s*$/i, ' ').replace(/\s+/g, ' ').replace(/^[\s\-–:]+|[\s\-–:]+$/g, '').trim(); return { name: n || name, completeness, condition: completeness }; } export class LukieGamesConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; async *crawl(ctx: CrawlContext): AsyncIterable { const platforms = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.platforms as string[] | undefined)) ?? Object.keys(PLATFORMS); const pages = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillPages ?? 10) : Number(this.meta.config.pagesPerPlatform ?? 2); const perPage = Number(this.meta.config.perPage ?? 100); let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.platformIndex ?? 0) : 0; for (let pi = start; pi < platforms.length; pi++) { const platform = platforms[pi]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${API}?siteId=dytuzo&resultsFormat=native&resultsPerPage=${perPage}&page=${page}&bgfilter.extrafield5=${encodeURIComponent(platform)}&sort.current_price=desc`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => (parseApiPage(r.json, url, platform, page)?.results.length ? { title: 'ok', price: 1 } : null) }); const payload = res.success ? parseApiPage(res.json, url, platform, page) : null; if (!payload) { ctx.anomaly('page_fetch_failed', `${platform} p${page}: ${res.error ?? res.httpStatus}`); break; } if (!payload.results.length) break; count++; yield { url, externalId: `platform:${platform}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; if (payload.results.length < perPage) break; } await ctx.setCursor({ platformIndex: pi + 1 >= platforms.length ? 0 : pi + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const r of p.results) { const platform = r.platform ?? p.platform; const map = PLATFORMS[platform] ?? PLATFORMS[p.platform]; if (!map || !r.price) continue; if (HARDWARE.test(r.name) || NOT_A_GAME.test(r.name) || /SYS|_CONTROLLER|_ACC|_MANUAL|_BOX/i.test(r.sku ?? '')) continue; const { name, completeness } = splitName(r.name, platform); const attributes = AssetAttributesSchema.parse({ categorySlug: map.slug, brand: r.brand, set: map.set, name, identifiers: { lukie_sku: r.sku ?? r.uid }, metadata: { platform, genre: r.genre, lukie_list_price: r.msrp, on_sale: r.onsale }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: r.url, externalId: r.uid, rawTitle: r.name, imageUrls: r.image ? [r.image] : [], attributes, condition: { condition: completeness ?? 'loose', conditionRaw: completeness ? null : 'Lukie default (game only unless stated)', completeness: completeness ?? 'loose' }, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: r.price, currency: 'USD', seller: 'Lukie Games', location: 'US', availability: r.stock && /out of stock|sold out/i.test(r.stock) ? 'ended' : 'available', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new LukieGamesConnector(meta); }