TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { decodeEntities } from '../_wlib/index.js';56/**7 * Lukie Games — US retro video-game dealer. Its storefront search is the public SearchSpring JSON8 * feed the site itself calls (siteId dytuzo); one result = one product with platform (extrafield5),9 * genre (extrafield3), price, list price, SKU, stock message and image. Dealer asking prices only.10 */11const API = 'https://dytuzo.a.searchspring.io/api/search/search.json';12const PARSER_VERSION = '1.0.0';1314export const ResultSchema = z.object({15 uid: z.string(),16 sku: z.string().nullable().default(null),17 name: z.string(),18 brand: z.string().nullable().default(null),19 platform: z.string().nullable().default(null),20 genre: z.string().nullable().default(null),21 price: z.number().nullable(),22 msrp: z.number().nullable(),23 url: z.string(),24 image: z.string().nullable().default(null),25 stock: z.string().nullable().default(null),26 onsale: z.boolean().default(false),27});28export type Result = z.infer<typeof ResultSchema>;29export const PagePayloadSchema = z.object({ kind: z.literal('search_page'), url: z.string(), platform: z.string(), page: z.number(), total: z.number().nullable(), results: z.array(ResultSchema) });30export type PagePayload = z.infer<typeof PagePayloadSchema>;3132const num = (v: unknown) => {33 const n = Number(v);34 return Number.isFinite(n) && n > 0 ? n : null;35};3637export function parseApiPage(json: unknown, url: string, platform: string, page: number): PagePayload | null {38 const j = json as { results?: Array<Record<string, unknown>>; pagination?: { totalResults?: number } } | null;39 if (!j || !Array.isArray(j.results)) return null;40 const results: Result[] = [];41 for (const r of j.results) {42 const parsed = ResultSchema.safeParse({43 uid: String(r.uid ?? r.id ?? ''),44 sku: r.sku ? String(r.sku) : null,45 name: decodeEntities(String(r.name ?? '')),46 brand: r.brand ? String(r.brand) : null,47 platform: r.extrafield5 ? decodeEntities(String(r.extrafield5)) : null,48 genre: r.extrafield3 ? decodeEntities(String(r.extrafield3)) : null,49 price: num(r.price),50 msrp: num(r.msrp),51 url: String(r.url ?? ''),52 image: r.imageUrl ? decodeEntities(String(r.imageUrl)) : null,53 stock: r.stock_message ? String(r.stock_message) : null,54 onsale: r.onsale === '1' || r.onsale === 1 || r.onsale === true,55 });56 if (parsed.success && parsed.data.name && parsed.data.url) results.push(parsed.data);57 }58 return { kind: 'search_page', url, platform, page, total: typeof j.pagination?.totalResults === 'number' ? j.pagination.totalResults : null, results };59}6061/** Lukie platform label → PriceCharting-style console name + taxonomy slug. */62export const PLATFORMS: Record<string, { set: string; slug: string }> = {63 'Nintendo 64': { set: 'Nintendo 64', slug: 'nintendo_games' },64 'Super Nintendo': { set: 'Super Nintendo', slug: 'nintendo_games' },65 'Nintendo NES': { set: 'NES', slug: 'nintendo_games' },66 NES: { set: 'NES', slug: 'nintendo_games' },67 Gamecube: { set: 'Gamecube', slug: 'nintendo_games' },68 'Nintendo Gamecube': { set: 'Gamecube', slug: 'nintendo_games' },69 'Nintendo Wii': { set: 'Wii', slug: 'nintendo_games' },70 'Wii U': { set: 'Wii U', slug: 'nintendo_games' },71 'Nintendo Switch': { set: 'Nintendo Switch', slug: 'nintendo_games' },72 Gameboy: { set: 'GameBoy', slug: 'nintendo_games' },73 'Gameboy Color': { set: 'GameBoy Color', slug: 'nintendo_games' },74 'Gameboy Advance': { set: 'GameBoy Advance', slug: 'nintendo_games' },75 'Nintendo DS': { set: 'Nintendo DS', slug: 'nintendo_games' },76 'Nintendo 3DS': { set: 'Nintendo 3DS', slug: 'nintendo_games' },77 'Virtual Boy': { set: 'Virtual Boy', slug: 'nintendo_games' },78 Playstation: { set: 'Playstation', slug: 'playstation_games' },79 'Playstation 2': { set: 'Playstation 2', slug: 'playstation_games' },80 'Playstation 3': { set: 'Playstation 3', slug: 'playstation_games' },81 'Playstation 4': { set: 'Playstation 4', slug: 'playstation_games' },82 PSP: { set: 'PSP', slug: 'playstation_games' },83 'PS Vita': { set: 'Playstation Vita', slug: 'playstation_games' },84 Xbox: { set: 'Xbox', slug: 'xbox_games' },85 'Xbox 360': { set: 'Xbox 360', slug: 'xbox_games' },86 'Xbox One': { set: 'Xbox One', slug: 'xbox_games' },87 'Sega Genesis': { set: 'Sega Genesis', slug: 'sega_games' },88 'Sega Dreamcast': { set: 'Sega Dreamcast', slug: 'sega_games' },89 'Sega Saturn': { set: 'Sega Saturn', slug: 'sega_games' },90 'Sega Master System': { set: 'Sega Master System', slug: 'sega_games' },91 'Sega Game Gear': { set: 'Sega Game Gear', slug: 'sega_games' },92 'Sega CD': { set: 'Sega CD', slug: 'sega_games' },93 'Sega 32X': { set: 'Sega 32X', slug: 'sega_games' },94 'Atari 2600': { set: 'Atari 2600', slug: 'atari_retro_games' },95 'Atari 5200': { set: 'Atari 5200', slug: 'atari_retro_games' },96 'Atari 7800': { set: 'Atari 7800', slug: 'atari_retro_games' },97 'Atari Jaguar': { set: 'Jaguar', slug: 'atari_retro_games' },98 'Neo Geo Pocket': { set: 'Neo Geo Pocket Color', slug: 'atari_retro_games' },99 TurboGrafx: { set: 'TurboGrafx-16', slug: 'atari_retro_games' },100 Intellivision: { set: 'Intellivision', slug: 'atari_retro_games' },101 Colecovision: { set: 'Colecovision', slug: 'atari_retro_games' },102};103104const HARDWARE = /\b(system|console|controller|adapter|cable|memory card|expansion pak|rumble pak|power supply|av cable|accessor(?:y|ies)|carrying case|cover|skin|stylus|charger|headset|strategy guide|magazine)\b/i;105const NOT_A_GAME = /^\s*(manual|box|instructions?|poster|insert|map|sleeve)\b|\b(manual only|box only|instructions only)\b/i;106107/** "GoldenEye 007" / "Super Mario 64 Game" / "Zelda Ocarina of Time N64 Game Complete" → { name, completeness } */108export function splitName(name: string, platform: string | null): { name: string; completeness: 'loose' | 'cib' | 'sealed' | null; condition: string | null } {109 let n = name.replace(/\s+/g, ' ').trim();110 let completeness: 'loose' | 'cib' | 'sealed' | null = null;111 if (/\b(brand new|new sealed|factory sealed|sealed)\b/i.test(n)) completeness = 'sealed';112 else if (/\b(complete in box|complete|cib|boxed|with box)\b/i.test(n)) completeness = 'cib';113 else if (/\b(cartridge only|cart only|disc only|game only|loose)\b/i.test(n)) completeness = 'loose';114 n = n.replace(/\b(brand new|new sealed|factory sealed|sealed|complete in box|complete|cib|boxed|with box|cartridge only|cart only|disc only|game only|loose)\b/gi, ' ');115 if (platform) n = n.replace(new RegExp(`\\b${platform.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i'), ' ');116 n = n.replace(/\bNintendo\s+(N64|NES|SNES|DS|3DS|Wii)\b/gi, ' ').replace(/\b(N64|SNES|NES|GBA|GBC|PS1|PS2|PS3|PS4|PSP|GC|Wii U|Wii)\b\s*(Game)?/gi, ' ').replace(/\bGame\b\s*$/i, ' ').replace(/\s+/g, ' ').replace(/^[\s\-–:]+|[\s\-–:]+$/g, '').trim();117 return { name: n || name, completeness, condition: completeness };118}119120export class LukieGamesConnector extends BaseConnector {121 readonly version = '1.0.0';122 readonly parserVersion = PARSER_VERSION;123 protected override minIntervalMs = 1500;124125 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {126 const platforms = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.platforms as string[] | undefined)) ?? Object.keys(PLATFORMS);127 const pages = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillPages ?? 10) : Number(this.meta.config.pagesPerPlatform ?? 2);128 const perPage = Number(this.meta.config.perPage ?? 100);129 let count = 0;130 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.platformIndex ?? 0) : 0;131 for (let pi = start; pi < platforms.length; pi++) {132 const platform = platforms[pi]!;133 for (let page = 1; page <= pages; page++) {134 if (ctx.signal?.aborted || this.reached(ctx, count)) return;135 const url = `${API}?siteId=dytuzo&resultsFormat=native&resultsPerPage=${perPage}&page=${page}&bgfilter.extrafield5=${encodeURIComponent(platform)}&sort.current_price=desc`;136 await this.throttle();137 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', expect: ['title', 'price'], parse: (r) => (parseApiPage(r.json, url, platform, page)?.results.length ? { title: 'ok', price: 1 } : null) });138 const payload = res.success ? parseApiPage(res.json, url, platform, page) : null;139 if (!payload) {140 ctx.anomaly('page_fetch_failed', `${platform} p${page}: ${res.error ?? res.httpStatus}`);141 break;142 }143 if (!payload.results.length) break;144 count++;145 yield { url, externalId: `platform:${platform}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };146 if (payload.results.length < perPage) break;147 }148 await ctx.setCursor({ platformIndex: pi + 1 >= platforms.length ? 0 : pi + 1, updatedAt: new Date().toISOString() });149 }150 }151152 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {153 const p = PagePayloadSchema.parse(raw.payload);154 const out: NormalizedRecord[] = [];155 for (const r of p.results) {156 const platform = r.platform ?? p.platform;157 const map = PLATFORMS[platform] ?? PLATFORMS[p.platform];158 if (!map || !r.price) continue;159 if (HARDWARE.test(r.name) || NOT_A_GAME.test(r.name) || /SYS|_CONTROLLER|_ACC|_MANUAL|_BOX/i.test(r.sku ?? '')) continue;160 const { name, completeness } = splitName(r.name, platform);161 const attributes = AssetAttributesSchema.parse({162 categorySlug: map.slug,163 brand: r.brand,164 set: map.set,165 name,166 identifiers: { lukie_sku: r.sku ?? r.uid },167 metadata: { platform, genre: r.genre, lukie_list_price: r.msrp, on_sale: r.onsale },168 });169 out.push(170 NormalizedListingSchema.parse({171 kind: 'listing',172 connectorId: this.meta.id,173 sourceId: this.meta.sourceId,174 sourceUrl: r.url,175 externalId: r.uid,176 rawTitle: r.name,177 imageUrls: r.image ? [r.image] : [],178 attributes,179 condition: { condition: completeness ?? 'loose', conditionRaw: completeness ? null : 'Lukie default (game only unless stated)', completeness: completeness ?? 'loose' },180 observedAt: raw.fetchedAt,181 confidence: 0.8,182 parserVersion: PARSER_VERSION,183 listingType: 'fixed_price',184 price: r.price,185 currency: 'USD',186 seller: 'Lukie Games',187 location: 'US',188 availability: r.stock && /out of stock|sold out/i.test(r.stock) ? 'ended' : 'available',189 }),190 );191 }192 return out;193 }194}195196export default function createConnector(meta: ConnectorMeta) {197 return new LukieGamesConnector(meta);198}199