1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { extractYear, parsePrice, parseSourceDate, type NormalizedCatalogItem, type NormalizedPriceObservation, type NormalizedRecord, type NormalizedSale, type AssetAttributes } from '@rareindex/shared';45/**6 * PriceCharting connector — video games, LEGO sets, Funko Pops and comic books.7 *8 * A product page carries (a) PriceCharting's own guide value per condition column and9 * (b) a table of recently completed eBay sales per condition tab (date · title · price · eBay id).10 * We store the parsed page compactly as one raw record and normalise it into:11 * - one catalog_item (canonical identity + identifiers),12 * - one price_observation per non-empty guide column ('guide_value'),13 * - one sale per completed-sale row (sale_type unknown: eBay auction/BIN not distinguished).14 */1516const BASE = 'https://www.pricecharting.com';17const PARSER_VERSION = '1.0.0';1819export const PriceCellSchema = z.object({ key: z.string(), value: z.number().nullable(), raw: z.string() });20export const SaleRowSchema = z.object({ tab: z.string(), date: z.string(), title: z.string(), price: z.number(), ebayId: z.string().nullable(), listedPrice: z.number().nullable() });21export const ProductPayloadSchema = z.object({22 kind: z.literal('product'),23 url: z.string(),24 productId: z.string().nullable(),25 consoleUri: z.string(),26 consoleName: z.string(),27 title: z.string(),28 flags: z.object({ isComic: z.boolean(), isLegoSet: z.boolean(), isFunkoPop: z.boolean(), isCard: z.boolean(), isCoin: z.boolean(), isSystem: z.boolean() }),29 columnLabels: z.array(z.string()),30 prices: z.array(PriceCellSchema),31 sales: z.array(SaleRowSchema),32 details: z.record(z.string(), z.string()),33 images: z.array(z.string()),34});35export type ProductPayload = z.infer<typeof ProductPayloadSchema>;3637type Family = 'video_games' | 'lego' | 'funko' | 'comics';3839interface ColumnMeaning {40 condition: string | null;41 completeness: string | null;42 conditionRaw: string;43 grade?: string | null;44}4546const PRICE_KEYS = ['used_price', 'complete_price', 'new_price', 'graded_price', 'box_only_price', 'manual_only_price'] as const;47const TAB_TO_KEY: Record<string, (typeof PRICE_KEYS)[number]> = {48 used: 'used_price',49 cib: 'complete_price',50 new: 'new_price',51 graded: 'graded_price',52 'box-only': 'box_only_price',53 'manual-only': 'manual_only_price',54 'loose-and-manual': 'graded_price',55};5657/** Column semantics per family (PriceCharting reuses the same six cells with different labels). */58const COLUMN_MEANING: Record<Family, Partial<Record<(typeof PRICE_KEYS)[number], ColumnMeaning>>> = {59 video_games: {60 used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Loose' },61 complete_price: { condition: 'cib', completeness: 'cib', conditionRaw: 'Complete in box' },62 new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },63 graded_price: { condition: null, completeness: null, conditionRaw: 'Graded (any grader)' },64 box_only_price: { condition: null, completeness: 'box_only', conditionRaw: 'Box only' },65 manual_only_price: { condition: null, completeness: 'manual_only', conditionRaw: 'Manual only' },66 },67 lego: {68 used_price: { condition: 'used_complete', completeness: 'used_complete', conditionRaw: 'Pieces only (used, complete pieces)' },69 complete_price: { condition: 'opened_complete', completeness: 'opened_complete', conditionRaw: 'Complete (pieces, box, manual)' },70 new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },71 manual_only_price: { condition: null, completeness: 'instructions_only', conditionRaw: 'Manual only' },72 },73 funko: {74 used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Out of box' },75 complete_price: { condition: 'boxed', completeness: 'boxed', conditionRaw: 'In damaged box' },76 new_price: { condition: 'mint_in_box', completeness: 'boxed', conditionRaw: 'New (mint in box)' },77 },78 comics: {79 used_price: { condition: null, completeness: null, conditionRaw: 'Ungraded (raw)', grade: null },80 complete_price: { condition: 'very_good', completeness: null, conditionRaw: 'Graded 4.0 / VG (any grader)', grade: '4.0' },81 new_price: { condition: 'fine', completeness: null, conditionRaw: 'Graded 6.0 / Fine (any grader)', grade: '6.0' },82 graded_price: { condition: 'very_fine', completeness: null, conditionRaw: 'Graded 8.0 / VF (any grader)', grade: '8.0' },83 box_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.2 / NM- (any grader)', grade: '9.2' },84 manual_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.8 (any grader)', grade: '9.8' },85 },86};8788const NINTENDO = /^(nes|famicom|super-nintendo|super-famicom|nintendo-64|gamecube|wii|wii-u|nintendo-switch|gameboy|gameboy-color|gameboy-advance|nintendo-ds|nintendo-3ds|virtual-boy|jp-|pal-)/;89const SEGA = /^(sega-|pal-sega|jp-sega)/;90const PLAYSTATION = /^(playstation|psp|playstation-vita|jp-playstation|pal-playstation)/;91const XBOX = /^xbox/;92const RETRO = /^(atari|intellivision|colecovision|neo-geo|turbografx|pc-engine|commodore|amiga|vectrex|3do|jaguar|philips-cd-i|magnavox|odyssey|msx|sharp|wonderswan|n-gage|game-com|tiger|evercade|super-cassette|fairchild|bally|arcadia|action-max|amiga-cd32)/;9394function familyOf(p: ProductPayload): Family | null {95 if (p.flags.isComic || p.consoleUri.startsWith('comic-books')) return 'comics';96 if (p.flags.isLegoSet || p.consoleUri.startsWith('lego')) return 'lego';97 if (p.flags.isFunkoPop || p.consoleUri.startsWith('funko')) return 'funko';98 if (p.flags.isCard || p.flags.isCoin) return null; // cards & coins are handled by dedicated connectors99 return 'video_games';100}101102function categorySlug(p: ProductPayload, family: Family): string | null {103 const c = p.consoleUri;104 switch (family) {105 case 'lego':106 return 'lego_sets';107 case 'funko':108 return 'funko';109 case 'comics': {110 const pub = (p.details['Publisher'] ?? '').toLowerCase();111 if (/marvel/.test(pub)) return 'marvel_comics';112 if (/\bdc\b|dc comics|vertigo|wildstorm/.test(pub)) return 'dc_comics';113 return 'independent_comics';114 }115 case 'video_games':116 if (c === 'pc-games') return 'pc_games';117 if (NINTENDO.test(c)) return 'nintendo_games';118 if (SEGA.test(c)) return 'sega_games';119 if (PLAYSTATION.test(c)) return 'playstation_games';120 if (XBOX.test(c)) return 'xbox_games';121 if (RETRO.test(c)) return 'atari_retro_games';122 return 'video_games';123 }124}125126/** "Super Mario 64 [Player's Choice]" → { name, variant } ; "Cloud City #10123" → { name, number } */127function splitTitle(title: string): { name: string; variant: string | null; number: string | null } {128 let t = title.trim();129 const variants: string[] = [];130 t = t.replace(/\[([^\]]+)\]/g, (_m, v: string) => {131 variants.push(v.trim());132 return ' ';133 });134 let number: string | null = null;135 const num = t.match(/#\s*([A-Za-z0-9.\-/]+)/);136 if (num) {137 number = num[1]!;138 t = t.replace(num[0], ' ');139 }140 return { name: t.replace(/\s+/g, ' ').trim(), variant: variants.length ? variants.join(' · ') : null, number };141}142143function parseProductPage(htmlText: string, url: string): ProductPayload {144 const $ = H.load(htmlText);145 const h1 = $('h1#product_name');146 const consoleName = H.text(h1.find('a')) ?? '';147 const consoleUri = (h1.find('a').attr('href') ?? '').replace(/^\/console\//, '') || (url.match(/\/game\/([^/]+)\//)?.[1] ?? '');148 const title = h1149 .clone()150 .children('a')151 .remove()152 .end()153 .text()154 .replace(/\s+/g, ' ')155 .trim();156 const flagsBlock = htmlText.match(/VGPC\.product\s*=\s*\{([\s\S]*?)\}/)?.[1] ?? '';157 const flag = (k: string) => new RegExp(`${k}\\s*:\\s*true`).test(flagsBlock);158 const productId = flagsBlock.match(/id\s*:\s*(\d+)/)?.[1] ?? null;159160 const prices: z.infer<typeof PriceCellSchema>[] = [];161 const seen = new Set<string>();162 for (const key of PRICE_KEYS) {163 if (seen.has(key)) continue;164 const cell = $(`td#${key}`).first();165 if (!cell.length) continue;166 seen.add(key);167 const raw = H.text(cell.find('.price').first()) ?? H.text(cell) ?? '';168 const parsed = parsePrice(raw, 'USD');169 prices.push({ key, value: parsed && parsed.amount > 0 ? parsed.amount : null, raw });170 }171 const columnLabels: string[] = [];172 $('#price_data thead th, table.price_data thead th').each((_, el) => {173 const t = H.text($(el));174 if (t) columnLabels.push(t);175 });176177 const sales: z.infer<typeof SaleRowSchema>[] = [];178 $('div[class^="completed-auctions-"]').each((_, sec) => {179 const cls = ($(sec).attr('class') ?? '').split(/\s+/).find((c) => c.startsWith('completed-auctions-')) ?? '';180 const tab = cls.replace('completed-auctions-', '');181 if (!tab || tab === 'condition' || !$(sec).find('table').length) return;182 $(sec)183 .find('tbody tr')184 .each((__, tr) => {185 const $tr = $(tr);186 const date = H.text($tr.find('td.date')) ?? '';187 const titleEl = $tr.find('td.title a').first();188 const rowTitle = H.text(titleEl) ?? H.text($tr.find('td.title')) ?? '';189 const priceTxt = H.text($tr.find('td.numeric .js-price').first()) ?? H.text($tr.find('td.numeric').first()) ?? '';190 const price = parsePrice(priceTxt, 'USD');191 const listedTxt = H.text($tr.find('td.listed-price'));192 const listed = listedTxt ? parsePrice(listedTxt, 'USD') : null;193 const ebayId = ($tr.attr('id') ?? '').match(/ebay-(\d+)/)?.[1] ?? titleEl.attr('href')?.match(/\/itm\/(\d+)/)?.[1] ?? null;194 if (!date || !rowTitle || !price || price.amount <= 0) return;195 sales.push({ tab, date, title: rowTitle, price: price.amount, ebayId, listedPrice: listed && listed.amount > 0 ? listed.amount : null });196 });197 });198199 const details: Record<string, string> = {};200 $('td.title').each((_, el) => {201 const k = (H.text($(el)) ?? '').replace(/:$/, '').trim();202 const v = H.text($(el).next('td.details'));203 if (k && v && v !== 'none' && v !== 'n/a') details[k] = v;204 });205 const images: string[] = [];206 $('img[src*="images.pricecharting.com"]').each((_, el) => {207 const src = $(el).attr('src');208 if (src && /\/(240|1600|400)\.jpg$/.test(src) && !images.includes(src)) images.push(src);209 });210211 return {212 kind: 'product',213 url,214 productId,215 consoleUri,216 consoleName,217 title,218 flags: { isComic: flag('is_comic'), isLegoSet: flag('is_lego_set'), isFunkoPop: flag('is_funko_pop'), isCard: flag('is_card'), isCoin: flag('is_coin'), isSystem: flag('is_system') },219 columnLabels,220 prices,221 sales,222 details,223 images: images.slice(0, 4),224 };225}226227interface ConsoleProduct {228 id: string;229 productUri: string;230 productName: string;231 consoleUri: string;232}233234export class PriceChartingConnector extends BaseConnector {235 readonly version = '1.0.0';236 readonly parserVersion = PARSER_VERSION;237 override readonly urlPatterns = [/^https?:\/\/(www\.)?pricecharting\.com\/game\/[^/]+\/[^/?#]+/i];238 protected override minIntervalMs = 1500;239240 private get seeds(): string[] {241 const s = this.meta.config.seeds;242 return Array.isArray(s) ? (s as string[]) : [];243 }244245 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {246 const perConsole = Number(this.meta.config.productsPerConsole ?? 200);247 const sort = String(this.meta.config.sort ?? 'popularity');248 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : this.seeds;249 const cursor = ctx.options.cursor ?? {};250 const doneConsoles = new Set<string>((cursor.doneConsoles as string[] | undefined) ?? []);251 let count = 0;252253 for (const consoleUri of seeds) {254 if (doneConsoles.has(consoleUri) && ctx.options.mode !== 'backfill') continue;255 if (ctx.signal?.aborted) return;256 const products = await this.listConsole(ctx, consoleUri, perConsole, sort);257 if (products.length === 0) ctx.anomaly('empty_console', consoleUri);258 for (const p of products) {259 if (this.reached(ctx, count)) return;260 const url = `${BASE}/game/${p.consoleUri}/${p.productUri}`;261 if (!(await ctx.shouldFetch(url))) continue;262 await this.throttle();263 const rec = await this.fetchProduct(ctx, url);264 if (rec) {265 count++;266 yield rec;267 }268 }269 doneConsoles.add(consoleUri);270 await ctx.setCursor({ doneConsoles: [...doneConsoles], updatedAt: new Date().toISOString() });271 }272 // Full pass complete: reset so the next incremental run starts again.273 await ctx.setCursor({ doneConsoles: [], updatedAt: new Date().toISOString() });274 }275276 private async listConsole(ctx: CrawlContext, consoleUri: string, max: number, sort: string): Promise<ConsoleProduct[]> {277 const out: ConsoleProduct[] = [];278 let cursorPos = 0;279 while (out.length < max) {280 await this.throttle();281 const url = `${BASE}/console/${consoleUri}?sort=${encodeURIComponent(sort)}&cursor=${cursorPos}&format=json`;282 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'json', minQuality: 0 });283 let json = res.json as { products?: Array<Record<string, unknown>>; cursor?: string } | null;284 if (!json && res.html) {285 try {286 json = JSON.parse(res.html) as typeof json;287 } catch {288 json = null;289 }290 }291 if (!res.success || !json?.products) {292 if (cursorPos === 0) ctx.anomaly('console_list_failed', `${consoleUri}: ${res.error ?? 'no products'}`);293 break;294 }295 for (const p of json.products) {296 if (typeof p.productUri === 'string' && typeof p.consoleUri === 'string') {297 out.push({ id: String(p.id ?? ''), productUri: p.productUri, productName: String(p.productName ?? ''), consoleUri: p.consoleUri });298 }299 }300 const next = Number(json.cursor);301 if (!Number.isFinite(next) || next <= cursorPos || json.products.length === 0) break;302 cursorPos = next;303 }304 return out.slice(0, max);305 }306307 private async fetchProduct(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {308 const res = await ctx.fetch(url, {309 responseType: 'text',310 expect: ['title', 'price', 'identifiers', 'category'],311 parse: (r) => {312 if (!r.html) return null;313 const p = parseProductPage(r.html, url);314 return { title: p.title, price: p.prices.find((x) => x.value !== null)?.value ?? null, identifiers: p.productId ? { id: p.productId } : null, category: p.consoleUri };315 },316 });317 if (!res.success || !res.html) {318 ctx.anomaly('product_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);319 return null;320 }321 const payload = parseProductPage(res.html, url);322 if (!payload.title) {323 ctx.anomaly('parse_failure_title', url);324 return null;325 }326 return { url, externalId: payload.productId, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };327 }328329 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {330 const clean = url.split('?')[0]!;331 const rec = await this.fetchProduct(ctx, clean);332 return rec ? [rec] : [];333 }334335 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {336 const p = ProductPayloadSchema.parse(raw.payload);337 const family = familyOf(p);338 if (!family) return [];339 const slug = categorySlug(p, family);340 if (!slug) return [];341 const { name, variant, number } = splitTitle(p.title);342 const year = extractYear(p.details['Release Date'] ?? '') ?? extractYear(p.title) ?? null;343 const identifiers: Record<string, string> = {};344 if (p.productId) identifiers.pricecharting_id = p.productId;345 if (p.details['UPC']) identifiers.upc = p.details['UPC'].replace(/\s+/g, '');346 if (p.details['ASIN (Amazon)']) identifiers.asin = p.details['ASIN (Amazon)'];347 if (p.details['ePID (eBay)']) identifiers.ebay_epid = p.details['ePID (eBay)'];348 if (p.details['Comic.org ID']) identifiers.comics_org_id = p.details['Comic.org ID'];349 if (family === 'lego' && number) identifiers.lego_set_number = number;350 if (p.details['Model Number']) identifiers.model_number = p.details['Model Number'];351 if (family === 'funko' && p.details['Box Number']) identifiers.funko_box_number = p.details['Box Number'];352353 const attributes: AssetAttributes = {354 categorySlug: slug,355 subcategorySlug: null,356 franchise: family === 'funko' || family === 'lego' ? p.consoleName.replace(/^(Funko POP|LEGO)\s*/i, '') || null : null,357 brand: family === 'lego' ? 'LEGO' : family === 'funko' ? 'Funko' : p.details['Publisher'] ?? null,358 series: family === 'funko' ? p.details['Series'] ?? p.consoleName : null,359 set: family === 'comics' ? p.consoleName : family === 'lego' ? p.consoleName.replace(/^LEGO\s*/i, '') : p.consoleName,360 setCode: null,361 name: family === 'comics' ? p.consoleName : name,362 model: null,363 reference: null,364 number: number ?? (family === 'funko' ? p.details['Box Number'] ?? null : null),365 year,366 edition: null,367 variant,368 language: 'English',369 region: /^(jp|pal)-/.test(p.consoleUri) ? p.consoleUri.split('-')[0]!.toUpperCase() : 'NTSC-U',370 country: null,371 material: null,372 size: null,373 color: null,374 rarity: null,375 productionQuantity: null,376 originalMsrp: null,377 originalMsrpCurrency: null,378 identifiers,379 metadata: { pricecharting_console: p.consoleUri, key_issue: p.details['Is Key Issue'] === 'Yes' ? true : undefined, genre: p.details['Genre'], notes: p.details['Notes'] },380 };381 if (family === 'comics') {382 // "Amazing Spider-Man #1 (1963)" → name = series (from console), number = issue, year from title383 attributes.name = p.consoleName;384 attributes.year = extractYear(p.title) ?? year;385 }386387 const observedAt = raw.fetchedAt;388 const base = {389 connectorId: this.meta.id,390 sourceId: this.meta.sourceId,391 sourceUrl: p.url,392 externalId: p.productId,393 rawTitle: family === 'comics' ? p.title : `${p.title} (${p.consoleName})`,394 description: p.details['Description'] ?? null,395 imageUrls: p.images,396 attributes,397 observedAt,398 confidence: 0.9,399 parserVersion: PARSER_VERSION,400 };401 const meanings = COLUMN_MEANING[family];402 const out: NormalizedRecord[] = [];403 const catalog: NormalizedCatalogItem = { kind: 'catalog_item', ...base, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, releaseDate: parseSourceDate(p.details['Release Date']) ?? null };404 out.push(catalog);405406 const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));407 for (const cell of p.prices) {408 const m = meanings[cell.key as (typeof PRICE_KEYS)[number]];409 if (!m || cell.value === null) continue;410 const obs: NormalizedPriceObservation = {411 kind: 'price_observation',412 ...base,413 grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },414 condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },415 priceKind: 'guide_value',416 price: cell.value,417 currency: 'USD',418 observationDate: obsDate,419 sampleSize: null,420 confidence: 0.85,421 };422 out.push(obs);423 }424 const seenSale = new Set<string>();425 for (const row of p.sales) {426 const key = TAB_TO_KEY[row.tab];427 const m = key ? meanings[key] : undefined;428 if (!m) continue;429 const saleDate = parseSourceDate(row.date);430 if (!saleDate) continue;431 const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;432 if (seenSale.has(dedupe)) continue;433 seenSale.add(dedupe);434 const sale: NormalizedSale = {435 kind: 'sale',436 ...base,437 externalId: row.ebayId ? `ebay:${row.ebayId}` : `${p.productId ?? p.url}:${row.date}:${row.price}`,438 rawTitle: row.title,439 description: null,440 grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },441 condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },442 saleType: 'unknown',443 saleDate,444 price: row.price,445 currency: 'USD',446 buyerPremiumIncluded: false,447 quantity: 1,448 isBundle: /\b(lot|bundle)\b/i.test(row.title),449 location: null,450 auctionHouse: null,451 lotNumber: null,452 confidence: 0.75,453 };454 out.push(sale);455 }456 return out;457 }458}459460export default (meta: ConnectorMeta) => new PriceChartingConnector(meta);461export { parseProductPage };462