TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import { extractYear, parsePrice, parseSourceDate, type NormalizedCatalogItem, type NormalizedPriceObservation, type NormalizedRecord, type NormalizedSale, type AssetAttributes, type Grade } from '@rareindex/shared';56/**7 * Shared core for the PriceCharting family of sites (pricecharting.com, sportscardspro.com).8 *9 * A product page carries (a) guide values per condition/grade column and (b) tables of recently10 * completed eBay sales per condition/grade tab (date · title · price · eBay id). The page is11 * stored compactly as one raw record and normalised into one catalog_item, one price_observation12 * per non-empty guide cell ('guide_value') and one sale per completed-sale row.13 *14 * Families: video_games · lego · funko · comics (PriceCharting) · cards (Pokémon / Magic / Yu-Gi-Oh!15 * on PriceCharting) · sports (SportsCardsPro). Card pages use grade columns instead of conditions.16 */1718export const PARSER_VERSION = '2.0.0';1920export const PriceCellSchema = z.object({ key: z.string(), value: z.number().nullable(), raw: z.string() });21export const SaleRowSchema = z.object({ tab: z.string(), date: z.string(), title: z.string(), price: z.number(), ebayId: z.string().nullable(), listedPrice: z.number().nullable() });22export const SetRefSchema = z.object({ id: z.string().nullable(), code: z.string().nullable(), name: z.string().nullable(), source: z.string() });23export const CardRefSchema = z.object({ scryfall_id: z.string().optional(), number: z.string().optional(), pokemontcg_id: z.string().optional(), tcgplayer_id: z.string().optional() });24export const ProductPayloadSchema = z.object({25 kind: z.literal('product'),26 url: z.string(),27 productId: z.string().nullable(),28 consoleUri: z.string(),29 consoleName: z.string(),30 title: z.string(),31 flags: z.object({ isComic: z.boolean(), isLegoSet: z.boolean(), isFunkoPop: z.boolean(), isCard: z.boolean(), isCoin: z.boolean(), isSystem: z.boolean() }),32 columnLabels: z.array(z.string()),33 prices: z.array(PriceCellSchema),34 /** card pages: every row of the "full prices" table (Ungraded, Grade 1 … PSA 10, BGS 10 Black…) */35 fullPrices: z.array(z.object({ label: z.string(), value: z.number().nullable() })).default([]),36 /** tab id → human label ("grade-seventeen" → "CGC 10"), parsed from the page's tab selector */37 tabLabels: z.record(z.string(), z.string()).default({}),38 sales: z.array(SaleRowSchema),39 details: z.record(z.string(), z.string()),40 images: z.array(z.string()),41 /** set resolved against a reference catalog at crawl time (TCGdex / pokemontcg / Scryfall) */42 setRef: SetRefSchema.nullable().default(null),43 /** card resolved against a reference catalog at crawl time (Scryfall exact-name lookup) */44 cardRef: CardRefSchema.nullable().default(null),45 site: z.enum(['pricecharting', 'sportscardspro']).default('pricecharting'),46});47export type ProductPayload = z.infer<typeof ProductPayloadSchema>;4849export type Family = 'video_games' | 'lego' | 'funko' | 'comics' | 'cards' | 'sports';5051interface ColumnMeaning {52 condition: string | null;53 completeness: string | null;54 conditionRaw: string;55 grade?: string | null;56}5758export const PRICE_KEYS = ['used_price', 'complete_price', 'new_price', 'graded_price', 'box_only_price', 'manual_only_price'] as const;59export const TAB_TO_KEY: Record<string, (typeof PRICE_KEYS)[number]> = {60 used: 'used_price',61 cib: 'complete_price',62 new: 'new_price',63 graded: 'graded_price',64 'box-only': 'box_only_price',65 'manual-only': 'manual_only_price',66 'loose-and-manual': 'graded_price',67};6869/** Column semantics per family (the site reuses the same six cells with different labels). */70export const COLUMN_MEANING: Record<Exclude<Family, 'cards' | 'sports'>, Partial<Record<(typeof PRICE_KEYS)[number], ColumnMeaning>>> = {71 video_games: {72 used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Loose' },73 complete_price: { condition: 'cib', completeness: 'cib', conditionRaw: 'Complete in box' },74 new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },75 graded_price: { condition: null, completeness: null, conditionRaw: 'Graded (any grader)' },76 box_only_price: { condition: null, completeness: 'box_only', conditionRaw: 'Box only' },77 manual_only_price: { condition: null, completeness: 'manual_only', conditionRaw: 'Manual only' },78 },79 lego: {80 used_price: { condition: 'used_complete', completeness: 'used_complete', conditionRaw: 'Pieces only (used, complete pieces)' },81 complete_price: { condition: 'opened_complete', completeness: 'opened_complete', conditionRaw: 'Complete (pieces, box, manual)' },82 new_price: { condition: 'sealed', completeness: 'sealed', conditionRaw: 'New / sealed' },83 manual_only_price: { condition: null, completeness: 'instructions_only', conditionRaw: 'Manual only' },84 },85 funko: {86 used_price: { condition: 'loose', completeness: 'loose', conditionRaw: 'Out of box' },87 complete_price: { condition: 'boxed', completeness: 'boxed', conditionRaw: 'In damaged box' },88 new_price: { condition: 'mint_in_box', completeness: 'boxed', conditionRaw: 'New (mint in box)' },89 },90 comics: {91 used_price: { condition: null, completeness: null, conditionRaw: 'Ungraded (raw)', grade: null },92 complete_price: { condition: 'very_good', completeness: null, conditionRaw: 'Graded 4.0 / VG (any grader)', grade: '4.0' },93 new_price: { condition: 'fine', completeness: null, conditionRaw: 'Graded 6.0 / Fine (any grader)', grade: '6.0' },94 graded_price: { condition: 'very_fine', completeness: null, conditionRaw: 'Graded 8.0 / VF (any grader)', grade: '8.0' },95 box_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.2 / NM- (any grader)', grade: '9.2' },96 manual_only_price: { condition: 'near_mint', completeness: null, conditionRaw: 'Graded 9.8 (any grader)', grade: '9.8' },97 },98};99100/** Card pages: the six main cells map to Ungraded / Grade 7 / Grade 8 / Grade 9 / Grade 9.5 / PSA 10. */101export const CARD_CELL_LABEL: Record<(typeof PRICE_KEYS)[number], string> = {102 used_price: 'Ungraded',103 complete_price: 'Grade 7',104 new_price: 'Grade 8',105 graded_price: 'Grade 9',106 box_only_price: 'Grade 9.5',107 manual_only_price: 'PSA 10',108};109const CARD_TAB_LABEL: Record<string, string> = { used: 'Ungraded', cib: 'Grade 7', new: 'Grade 8', graded: 'Grade 9', 'box-only': 'Grade 9.5', 'manual-only': 'PSA 10' };110111/** Generic grader used when the site reports a grade without a grading company ("Grade 9"). */112export const GENERIC_GRADER = 'graded';113114export interface CardGrade {115 grader: string | null;116 grade: string | null;117 qualifier: string | null;118 conditionRaw: string;119}120121/** "PSA 10" | "BGS 10 Black" | "CGC 10 Pristine" | "Grade 9.5" | "Ungraded" → structured grade. */122export function gradeFromLabel(label: string): CardGrade | null {123 const l = label.replace(/\s*\(\d+\)\s*$/, '').trim();124 if (!l) return null;125 if (/^ungraded$/i.test(l)) return { grader: null, grade: null, qualifier: null, conditionRaw: 'Ungraded (raw)' };126 let m = l.match(/^grade\s+(\d+(?:\.\d)?)$/i);127 if (m) return { grader: GENERIC_GRADER, grade: m[1]!, qualifier: null, conditionRaw: `Grade ${m[1]} (any grader)` };128 m = l.match(/^(PSA|BGS|CGC|SGC|TAG|ACE)\s+(\d+(?:\.\d)?)\s*(Black|Pristine|Black Label)?$/i);129 if (m) {130 const q = m[3] ? (/black/i.test(m[3]) ? 'Black Label' : 'Pristine') : null;131 return { grader: m[1]!.toLowerCase(), grade: m[2]!, qualifier: q, conditionRaw: l };132 }133 return null;134}135136/** Refine a generic grade with the grader named in the eBay row title when it agrees on the grade. */137export function refineGradeFromTitle(base: CardGrade, rowTitle: string): CardGrade {138 if (base.grader !== GENERIC_GRADER || !base.grade) return base;139 const t = parseGradeFromTitle(rowTitle);140 if (t.grader && t.grader !== 'raw' && t.grade && Number(t.grade) === Number(base.grade)) {141 return { grader: t.grader, grade: base.grade, qualifier: t.qualifier ?? null, conditionRaw: `${t.grader.toUpperCase()} ${base.grade}` };142 }143 return base;144}145146const NINTENDO = /^(nes|famicom|super-nintendo|super-famicom|nintendo-64|gamecube|wii|wii-u|nintendo-switch|gameboy|gameboy-color|gameboy-advance|nintendo-ds|nintendo-3ds|virtual-boy|jp-|pal-)/;147const SEGA = /^(sega-|pal-sega|jp-sega)/;148const PLAYSTATION = /^(playstation|psp|playstation-vita|jp-playstation|pal-playstation)/;149const XBOX = /^xbox/;150const RETRO = /^(atari|intellivision|colecovision|neo-geo|turbografx|pc-engine|commodore|amiga|vectrex|3do|jaguar|philips-cd-i|magnavox|odyssey|msx|sharp|wonderswan|n-gage|game-com|tiger|evercade|super-cassette|fairchild|bally|arcadia|action-max|amiga-cd32)/;151const SPORTS = ['basketball', 'baseball', 'football', 'hockey', 'soccer', 'wrestling', 'golf', 'racing', 'boxing', 'tennis', 'ufc', 'mma', 'formula', 'multi-sport', 'non-sport'] as const;152153export function familyOf(p: ProductPayload): Family | null {154 if (p.site === 'sportscardspro') return 'sports';155 if (p.flags.isComic || p.consoleUri.startsWith('comic-books')) return 'comics';156 if (p.flags.isLegoSet || p.consoleUri.startsWith('lego')) return 'lego';157 if (p.flags.isFunkoPop || p.consoleUri.startsWith('funko')) return 'funko';158 if (p.flags.isCoin) return null; // coins handled elsewhere159 if (p.flags.isCard || /^(pokemon|magic|yugioh)-/.test(p.consoleUri)) {160 return /^(pokemon|magic|yugioh)-/.test(p.consoleUri) ? 'cards' : null; // other card games not mapped yet161 }162 return 'video_games';163}164165export function categorySlug(p: ProductPayload, family: Family): string | null {166 const c = p.consoleUri;167 switch (family) {168 case 'lego':169 return 'lego_sets';170 case 'funko':171 return 'funko';172 case 'comics': {173 const pub = (p.details['Publisher'] ?? '').toLowerCase();174 if (/marvel/.test(pub)) return 'marvel_comics';175 if (/\bdc\b|dc comics|vertigo|wildstorm/.test(pub)) return 'dc_comics';176 return 'independent_comics';177 }178 case 'cards':179 if (c.startsWith('pokemon-')) return 'pokemon';180 if (c.startsWith('magic-')) return 'magic_the_gathering';181 if (c.startsWith('yugioh-')) return 'yugioh';182 return null;183 case 'sports': {184 const sport = SPORTS.find((s) => c.startsWith(s));185 switch (sport) {186 case 'basketball':187 return 'basketball_cards';188 case 'baseball':189 return 'baseball_cards';190 case 'football':191 return 'football_cards';192 case 'hockey':193 return 'hockey_cards';194 case 'soccer':195 return 'soccer_cards';196 case 'racing':197 case 'formula':198 return 'f1_cards';199 case 'non-sport':200 return 'non_sport_cards';201 default:202 return 'other_sports_cards';203 }204 }205 case 'video_games':206 if (c === 'pc-games') return 'pc_games';207 if (NINTENDO.test(c)) return 'nintendo_games';208 if (SEGA.test(c)) return 'sega_games';209 if (PLAYSTATION.test(c)) return 'playstation_games';210 if (XBOX.test(c)) return 'xbox_games';211 if (RETRO.test(c)) return 'atari_retro_games';212 return 'video_games';213 }214}215216const YGO_CODE = /\b([A-Z0-9]{2,6}-[A-Z]{0,3}\d{2,4})\b/;217/** Tags that describe the card's status, not a distinct printing (kept in metadata, not in variant). */218const NON_VARIANT_TAGS = /^(rookie|rc|key issue|hall of fame|hof|error|misprint)$/i;219220/** "Super Mario 64 [Player's Choice]" → { name, variant } ; "Cloud City #10123" → { name, number } ; "Blue-Eyes White Dragon [1st Edition] LOB-001" → number LOB-001 */221export function splitTitle(title: string, family?: Family | null): { name: string; variant: string | null; number: string | null; tags: string[] } {222 let t = title.trim();223 const variants: string[] = [];224 const tags: string[] = [];225 t = t.replace(/\[([^\]]+)\]/g, (_m, v: string) => {226 const tag = v.trim();227 if (NON_VARIANT_TAGS.test(tag)) tags.push(tag);228 else variants.push(tag);229 return ' ';230 });231 let number: string | null = null;232 const num = t.match(/#\s*([A-Za-z0-9.\-/]+)/);233 if (num) {234 number = num[1]!;235 t = t.replace(num[0], ' ');236 } else if (family === 'cards') {237 const code = t.match(YGO_CODE);238 if (code) {239 number = code[1]!;240 t = t.replace(code[0], ' ');241 }242 }243 return { name: t.replace(/\s+/g, ' ').trim(), variant: variants.length ? variants.join(' · ') : null, number, tags };244}245246/** Pokémon/Magic set name from a console name: "Pokemon Base Set" → "Base Set"; "Magic Alpha" → "Alpha". */247export function setNameFromConsole(consoleName: string, family: Family): string {248 if (family === 'cards') return consoleName.replace(/^(Pokemon|Pokémon|Magic|YuGiOh|Yu-Gi-Oh!?)\s+/i, '').trim();249 return consoleName.trim();250}251252export function parseProductPage(htmlText: string, url: string, site: ProductPayload['site'] = 'pricecharting'): ProductPayload {253 const $ = H.load(htmlText);254 const h1 = $('h1#product_name');255 const consoleName = H.text(h1.find('a')) ?? '';256 const consoleUri = (h1.find('a').attr('href') ?? '').replace(/^(https?:\/\/[^/]+)?\/console\//, '') || (url.match(/\/game\/([^/]+)\//)?.[1] ?? '');257 const title = h1258 .clone()259 .children('a')260 .remove()261 .end()262 .text()263 .replace(/\s+/g, ' ')264 .trim();265 const flagsBlock = htmlText.match(/VGPC\.product\s*=\s*\{([\s\S]*?)\}/)?.[1] ?? '';266 const flag = (k: string) => new RegExp(`${k}\\s*:\\s*true`).test(flagsBlock);267 let productId = flagsBlock.match(/id\s*:\s*(\d+)/)?.[1] ?? null;268269 const prices: z.infer<typeof PriceCellSchema>[] = [];270 const seen = new Set<string>();271 for (const key of PRICE_KEYS) {272 if (seen.has(key)) continue;273 const cell = $(`td#${key}`).first();274 if (!cell.length) continue;275 seen.add(key);276 const raw = H.text(cell.find('.price').first()) ?? H.text(cell) ?? '';277 const parsed = parsePrice(raw, 'USD');278 prices.push({ key, value: parsed && parsed.amount > 0 ? parsed.amount : null, raw });279 }280 const columnLabels: string[] = [];281 $('#price_data thead th, table.price_data thead th').each((_, el) => {282 const t = H.text($(el));283 if (t) columnLabels.push(t);284 });285 const fullPrices: Array<{ label: string; value: number | null }> = [];286 $('#full-prices tr').each((_, tr) => {287 const tds = $(tr).find('td');288 if (tds.length < 2) return;289 const label = H.text(tds.eq(0)) ?? '';290 const v = parsePrice(H.text(tds.eq(1)) ?? '', 'USD');291 if (label) fullPrices.push({ label, value: v && v.amount > 0 ? v.amount : null });292 });293 const tabLabels: Record<string, string> = {};294 $('option[value^="completed-auctions-"]').each((_, el) => {295 const tab = ($(el).attr('value') ?? '').replace('completed-auctions-', '');296 const label = (H.text($(el)) ?? '').replace(/\s*\(\d+\)\s*$/, '').trim();297 if (tab && label) tabLabels[tab] = label;298 });299300 const sales: z.infer<typeof SaleRowSchema>[] = [];301 $('div[class^="completed-auctions-"]').each((_, sec) => {302 const cls = ($(sec).attr('class') ?? '').split(/\s+/).find((c) => c.startsWith('completed-auctions-')) ?? '';303 const tab = cls.replace('completed-auctions-', '');304 if (!tab || tab === 'condition' || !$(sec).find('table').length) return;305 $(sec)306 .find('tbody tr')307 .each((__, tr) => {308 const $tr = $(tr);309 const date = H.text($tr.find('td.date')) ?? '';310 const titleEl = $tr.find('td.title a').first();311 const rowTitle = H.text(titleEl) ?? H.text($tr.find('td.title')) ?? '';312 const priceTxt = H.text($tr.find('td.numeric .js-price').first()) ?? H.text($tr.find('td.numeric').first()) ?? '';313 const price = parsePrice(priceTxt, 'USD');314 const listedTxt = H.text($tr.find('td.listed-price'));315 const listed = listedTxt ? parsePrice(listedTxt, 'USD') : null;316 const ebayId = ($tr.attr('id') ?? '').match(/ebay-(\d+)/)?.[1] ?? titleEl.attr('href')?.match(/\/itm\/(\d+)/)?.[1] ?? null;317 if (!date || !rowTitle || !price || price.amount <= 0) return;318 sales.push({ tab, date, title: rowTitle, price: price.amount, ebayId, listedPrice: listed && listed.amount > 0 ? listed.amount : null });319 });320 });321322 const details: Record<string, string> = {};323 $('td.title').each((_, el) => {324 const k = (H.text($(el)) ?? '').replace(/:$/, '').trim();325 const v = H.text($(el).next('td.details'));326 if (k && v && v !== 'none' && v !== 'n/a') details[k] = v;327 });328 if (!productId && details['PriceCharting ID']) productId = details['PriceCharting ID'].trim();329 const images: string[] = [];330 $('img[src*="images.pricecharting.com"]').each((_, el) => {331 const src = $(el).attr('src');332 if (src && /\/(240|1600|400)\.jpg$/.test(src) && !images.includes(src)) images.push(src);333 });334335 return {336 kind: 'product',337 url,338 productId,339 consoleUri,340 consoleName,341 title,342 flags: { isComic: flag('is_comic'), isLegoSet: flag('is_lego_set'), isFunkoPop: flag('is_funko_pop'), isCard: flag('is_card'), isCoin: flag('is_coin'), isSystem: flag('is_system') },343 columnLabels,344 prices,345 fullPrices,346 tabLabels,347 sales,348 details,349 images: images.slice(0, 4),350 setRef: null,351 cardRef: null,352 site,353 };354}355356export interface ConsoleProduct {357 id: string;358 productUri: string;359 productName: string;360 consoleUri: string;361}362363/** Parse a JSON body that may arrive wrapped in <html><body>…</body></html> by a rendering engine. */364export function parseJsonBody<T>(res: { json: unknown; html: string | null }): T | null {365 if (res.json && typeof res.json === 'object') return res.json as T;366 const text = res.html ?? '';367 if (!text) return null;368 const stripped = text.replace(/^[\s\S]*?<body[^>]*>/i, '').replace(/<\/body>[\s\S]*$/i, '').replace(/<[^>]+>/g, '').trim();369 const candidate = stripped.startsWith('{') || stripped.startsWith('[') ? stripped : text.trim();370 try {371 return JSON.parse(candidate) as T;372 } catch {373 return null;374 }375}376377export interface SiteOptions {378 base: string;379 site: ProductPayload['site'];380 /** category slugs (site "/category/<slug>") whose console lists are discovered automatically */381 categoryPrefix: (categorySlug: string) => string;382}383384/**385 * Base connector for PriceCharting-like sites. Subclasses provide the site and may enrich a386 * parsed product with reference-catalog lookups (setRef / cardRef) before it is yielded.387 */388export abstract class PriceChartingLikeConnector extends BaseConnector {389 readonly version = '2.0.0';390 readonly parserVersion = PARSER_VERSION;391 protected override minIntervalMs = 1500;392 protected abstract readonly siteOptions: SiteOptions;393394 protected get seeds(): string[] {395 const s = this.meta.config.seeds;396 return Array.isArray(s) ? (s as string[]) : [];397 }398 protected get cardCategories(): string[] {399 const s = this.meta.config.cardCategories;400 return Array.isArray(s) ? (s as string[]) : [];401 }402403 /** Hook: enrich a product payload with reference-catalog data (set code, card number, ids). */404 protected async enrich(_ctx: CrawlContext, payload: ProductPayload): Promise<ProductPayload> {405 return payload;406 }407 /** Hook: called once per crawl before iterating consoles (load reference maps). */408 protected async prepare(_ctx: CrawlContext): Promise<void> {}409410 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {411 const perConsole = Number(this.meta.config.productsPerConsole ?? 200);412 const sort = String(this.meta.config.sort ?? 'popularity');413 const maxConsoles = Number(this.meta.config.maxConsolesPerCategory ?? 40);414 const cursor = ctx.options.cursor ?? {};415 const doneConsoles = new Set<string>((cursor.doneConsoles as string[] | undefined) ?? []);416 let count = 0;417418 await this.prepare(ctx);419 let seeds = ctx.options.seeds?.length ? [...ctx.options.seeds] : [...this.seeds];420 if (!ctx.options.seeds?.length) {421 for (const cat of this.cardCategories) {422 const discovered = await this.discoverConsoles(ctx, cat, maxConsoles);423 for (const c of discovered) if (!seeds.includes(c)) seeds.push(c);424 }425 }426 seeds = [...new Set(seeds)];427428 for (const consoleUri of seeds) {429 if (doneConsoles.has(consoleUri) && ctx.options.mode !== 'backfill') continue;430 if (ctx.signal?.aborted) return;431 const products = await this.listConsole(ctx, consoleUri, perConsole, sort);432 if (products.length === 0) ctx.anomaly('empty_console', consoleUri);433 for (const p of products) {434 if (this.reached(ctx, count)) return;435 const url = `${this.siteOptions.base}/game/${p.consoleUri}/${p.productUri}`;436 if (!(await ctx.shouldFetch(url))) continue;437 await this.throttle();438 const rec = await this.fetchProduct(ctx, url);439 if (rec) {440 count++;441 yield rec;442 }443 }444 doneConsoles.add(consoleUri);445 await ctx.setCursor({ doneConsoles: [...doneConsoles], updatedAt: new Date().toISOString() });446 }447 // Full pass complete: reset so the next incremental run starts again.448 await ctx.setCursor({ doneConsoles: [], updatedAt: new Date().toISOString() });449 }450451 /** Console (set) slugs listed on a category page, in page order (most popular first). */452 protected async discoverConsoles(ctx: CrawlContext, category: string, max: number): Promise<string[]> {453 await this.throttle();454 const url = `${this.siteOptions.base}/category/${category}`;455 const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0 });456 if (!res.success || !res.html) {457 ctx.anomaly('category_discovery_failed', `${category}: ${res.error ?? res.httpStatus}`);458 return [];459 }460 const prefix = this.siteOptions.categoryPrefix(category);461 const out: string[] = [];462 for (const m of res.html.matchAll(/href="(?:https?:\/\/[^/"]+)?\/console\/([a-z0-9-]+)"/g)) {463 const slug = m[1]!;464 if (slug.startsWith(prefix) && !out.includes(slug)) out.push(slug);465 if (out.length >= max) break;466 }467 return out;468 }469470 protected async listConsole(ctx: CrawlContext, consoleUri: string, max: number, sort: string): Promise<ConsoleProduct[]> {471 const out: ConsoleProduct[] = [];472 let cursorPos = 0;473 while (out.length < max) {474 await this.throttle();475 const url = `${this.siteOptions.base}/console/${consoleUri}?sort=${encodeURIComponent(sort)}&cursor=${cursorPos}&format=json`;476 const res = await ctx.fetch(url, { responseType: 'json', minQuality: 0 });477 const json = parseJsonBody<{ products?: Array<Record<string, unknown>>; cursor?: string }>(res);478 if (!res.success || !json?.products) {479 if (cursorPos === 0) ctx.anomaly('console_list_failed', `${consoleUri}: ${res.error ?? 'no products'}`);480 break;481 }482 for (const p of json.products) {483 if (typeof p.productUri === 'string' && typeof p.consoleUri === 'string') {484 out.push({ id: String(p.id ?? ''), productUri: p.productUri, productName: String(p.productName ?? ''), consoleUri: p.consoleUri });485 }486 }487 const next = Number(json.cursor);488 if (!Number.isFinite(next) || next <= cursorPos || json.products.length === 0) break;489 cursorPos = next;490 }491 return out.slice(0, max);492 }493494 protected async fetchProduct(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {495 const site = this.siteOptions.site;496 const res = await ctx.fetch(url, {497 responseType: 'text',498 expect: ['title', 'price', 'identifiers', 'category'],499 parse: (r) => {500 if (!r.html) return null;501 const p = parseProductPage(r.html, url, site);502 return { title: p.title, price: p.prices.find((x) => x.value !== null)?.value ?? p.fullPrices.find((x) => x.value !== null)?.value ?? null, identifiers: p.productId ? { id: p.productId } : null, category: p.consoleUri };503 },504 });505 if (!res.success || !res.html) {506 ctx.anomaly('product_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);507 return null;508 }509 let payload = parseProductPage(res.html, url, site);510 if (!payload.title) {511 ctx.anomaly('parse_failure_title', url);512 return null;513 }514 try {515 payload = await this.enrich(ctx, payload);516 } catch (err) {517 ctx.anomaly('enrich_failed', `${url}: ${err instanceof Error ? err.message : String(err)}`);518 }519 return { url, externalId: payload.productId, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };520 }521522 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {523 const clean = url.split('?')[0]!;524 await this.prepare(ctx);525 const rec = await this.fetchProduct(ctx, clean);526 return rec ? [rec] : [];527 }528529 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {530 const p = ProductPayloadSchema.parse(raw.payload);531 return normalizeProduct(p, { connectorId: this.meta.id, sourceId: this.meta.sourceId, fetchedAt: raw.fetchedAt });532 }533}534535// ---------------------------------------------------------------------------------------------536// Normalisation537// ---------------------------------------------------------------------------------------------538539export interface NormalizeCtx {540 connectorId: string;541 sourceId: string;542 fetchedAt: Date;543}544545const RAW_GRADE: Grade = { grader: null, grade: null, qualifier: null, certificationNumber: null };546547export function normalizeProduct(p: ProductPayload, nctx: NormalizeCtx): NormalizedRecord[] {548 const family = familyOf(p);549 if (!family) return [];550 const slug = categorySlug(p, family);551 if (!slug) return [];552 const { name, variant, number, tags } = splitTitle(p.title, family);553 const isCardFamily = family === 'cards' || family === 'sports';554 const setName = isCardFamily ? (p.setRef?.name ?? setNameFromConsole(p.consoleName, family)) : null;555 const year = isCardFamily ? (extractYear(p.consoleName) ?? extractYear(p.details['Release Date'] ?? '') ?? null) : (extractYear(p.details['Release Date'] ?? '') ?? extractYear(p.title) ?? null);556557 const identifiers: Record<string, string> = {};558 if (p.productId) identifiers.pricecharting_id = p.site === 'sportscardspro' ? `scp:${p.productId}` : p.productId;559 if (p.details['UPC']) identifiers.upc = p.details['UPC'].replace(/\s+/g, '');560 if (p.details['ASIN (Amazon)']) identifiers.asin = p.details['ASIN (Amazon)'];561 if (p.details['ePID (eBay)']) identifiers.ebay_epid = p.details['ePID (eBay)'];562 if (p.details['Comic.org ID']) identifiers.comics_org_id = p.details['Comic.org ID'];563 if (p.details['TCGPlayer ID']) identifiers.tcgplayer_id = p.details['TCGPlayer ID'];564 if (family === 'lego' && number) identifiers.lego_set_number = number;565 if (p.details['Model Number']) identifiers.model_number = p.details['Model Number'];566 if (family === 'funko' && p.details['Box Number']) identifiers.funko_box_number = p.details['Box Number'];567 if (p.cardRef?.scryfall_id) identifiers.scryfall_id = p.cardRef.scryfall_id;568 if (p.cardRef?.pokemontcg_id) identifiers.pokemontcg_id = p.cardRef.pokemontcg_id;569 if (p.cardRef?.tcgplayer_id) identifiers.tcgplayer_id = p.cardRef.tcgplayer_id;570571 let cardNumber = number;572 if (isCardFamily && !cardNumber && p.cardRef?.number) cardNumber = p.cardRef.number;573 const language = /japanese/i.test(p.consoleName) ? 'Japanese' : /korean/i.test(p.consoleName) ? 'Korean' : /chinese/i.test(p.consoleName) ? 'Chinese' : 'English';574 const cardVariant = isCardFamily ? normalizeCardVariant(variant, slug) : variant;575 const setCode = isCardFamily ? (p.setRef?.code ?? (slug === 'yugioh' && cardNumber ? cardNumber.split('-')[0]! : null)) : null;576577 const attributes: AssetAttributes = {578 categorySlug: slug,579 subcategorySlug: null,580 franchise: family === 'funko' || family === 'lego' ? p.consoleName.replace(/^(Funko POP|LEGO)\s*/i, '') || null : slug === 'pokemon' ? 'Pokémon' : slug === 'magic_the_gathering' ? 'Magic: The Gathering' : slug === 'yugioh' ? 'Yu-Gi-Oh!' : null,581 brand: family === 'lego' ? 'LEGO' : family === 'funko' ? 'Funko' : family === 'sports' ? brandFromSet(setName ?? '') : slug === 'pokemon' ? 'The Pokémon Company' : slug === 'magic_the_gathering' ? 'Wizards of the Coast' : slug === 'yugioh' ? 'Konami' : (p.details['Publisher'] ?? null),582 series: family === 'funko' ? (p.details['Series'] ?? p.consoleName) : null,583 set: isCardFamily ? setName : family === 'comics' ? p.consoleName : family === 'lego' ? p.consoleName.replace(/^LEGO\s*/i, '') : p.consoleName,584 setCode,585 name: family === 'comics' ? p.consoleName : name,586 model: null,587 reference: null,588 number: cardNumber ?? (family === 'funko' ? (p.details['Box Number'] ?? null) : null),589 year,590 edition: null,591 variant: cardVariant,592 language: isCardFamily ? language : 'English',593 region: isCardFamily ? null : /^(jp|pal)-/.test(p.consoleUri) ? p.consoleUri.split('-')[0]!.toUpperCase() : 'NTSC-U',594 country: null,595 material: null,596 size: null,597 color: null,598 rarity: null,599 productionQuantity: null,600 originalMsrp: null,601 originalMsrpCurrency: null,602 identifiers,603 metadata: {604 pricecharting_console: p.consoleUri,605 pricecharting_site: p.site,606 key_issue: p.details['Is Key Issue'] === 'Yes' ? true : undefined,607 rookie: family === 'sports' && (p.details['Is Rookie Card'] === 'Yes' || tags.some((t) => /rookie|^rc$/i.test(t))) ? true : undefined,608 tags: tags.length ? tags : undefined,609 genre: p.details['Genre'],610 notes: p.details['Notes'],611 set_ref_source: p.setRef?.source,612 },613 };614 if (family === 'comics') {615 attributes.name = p.consoleName;616 attributes.year = extractYear(p.title) ?? year;617 }618619 const observedAt = nctx.fetchedAt;620 const base = {621 connectorId: nctx.connectorId,622 sourceId: nctx.sourceId,623 sourceUrl: p.url,624 externalId: p.productId ? (p.site === 'sportscardspro' ? `scp:${p.productId}` : p.productId) : null,625 rawTitle: family === 'comics' ? p.title : `${p.title} (${p.consoleName})`,626 description: p.details['Description'] ?? null,627 imageUrls: p.images,628 attributes,629 observedAt,630 confidence: 0.9,631 parserVersion: PARSER_VERSION,632 };633 const out: NormalizedRecord[] = [];634 const catalog: NormalizedCatalogItem = { kind: 'catalog_item', ...base, grade: RAW_GRADE, condition: { condition: null, conditionRaw: null, completeness: null }, releaseDate: parseSourceDate(p.details['Release Date']) ?? null };635 out.push(catalog);636637 const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));638639 if (isCardFamily) {640 // Guide values: prefer the full price table (all grades); fall back to the six main cells.641 const cells: Array<{ label: string; value: number | null }> = p.fullPrices.length ? p.fullPrices : p.prices.map((c) => ({ label: CARD_CELL_LABEL[c.key as (typeof PRICE_KEYS)[number]] ?? c.key, value: c.value }));642 const seenLabel = new Set<string>();643 for (const cell of cells) {644 const g = gradeFromLabel(cell.label);645 if (!g || cell.value === null || seenLabel.has(cell.label)) continue;646 seenLabel.add(cell.label);647 out.push({648 kind: 'price_observation',649 ...base,650 externalId: `${base.externalId ?? p.url}:guide:${cell.label.toLowerCase().replace(/[^a-z0-9.]+/g, '-')}`,651 grade: { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: null },652 condition: { condition: null, conditionRaw: g.conditionRaw, completeness: null },653 priceKind: 'guide_value',654 price: cell.value,655 currency: 'USD',656 observationDate: obsDate,657 sampleSize: null,658 confidence: 0.85,659 } satisfies NormalizedPriceObservation);660 }661 const seenSale = new Set<string>();662 for (const row of p.sales) {663 const label = p.tabLabels[row.tab] ?? CARD_TAB_LABEL[row.tab];664 if (!label) continue;665 const g0 = gradeFromLabel(label);666 if (!g0) continue;667 const g = refineGradeFromTitle(g0, row.title);668 const saleDate = parseSourceDate(row.date);669 if (!saleDate) continue;670 const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;671 if (seenSale.has(dedupe)) continue;672 seenSale.add(dedupe);673 out.push(makeSale(base, p, row, saleDate, { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: null }, { condition: null, conditionRaw: g.conditionRaw, completeness: null }));674 }675 return out;676 }677678 const meanings = COLUMN_MEANING[family];679 for (const cell of p.prices) {680 const m = meanings[cell.key as (typeof PRICE_KEYS)[number]];681 if (!m || cell.value === null) continue;682 out.push({683 kind: 'price_observation',684 ...base,685 externalId: `${base.externalId ?? p.url}:guide:${cell.key}`,686 grade: { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null },687 condition: { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness },688 priceKind: 'guide_value',689 price: cell.value,690 currency: 'USD',691 observationDate: obsDate,692 sampleSize: null,693 confidence: 0.85,694 } satisfies NormalizedPriceObservation);695 }696 const seenSale = new Set<string>();697 for (const row of p.sales) {698 const key = TAB_TO_KEY[row.tab];699 const m = key ? meanings[key] : undefined;700 if (!m) continue;701 const saleDate = parseSourceDate(row.date);702 if (!saleDate) continue;703 const dedupe = `${row.ebayId ?? row.title}|${row.date}|${row.price}`;704 if (seenSale.has(dedupe)) continue;705 seenSale.add(dedupe);706 out.push(makeSale(base, p, row, saleDate, { grader: null, grade: m.grade ?? null, qualifier: null, certificationNumber: null }, { condition: m.condition, conditionRaw: m.conditionRaw, completeness: m.completeness }));707 }708 return out;709}710711function makeSale(base: Omit<NormalizedCatalogItem, 'kind' | 'grade' | 'condition' | 'releaseDate'>, p: ProductPayload, row: z.infer<typeof SaleRowSchema>, saleDate: Date, grade: Grade, condition: NormalizedSale['condition']): NormalizedSale {712 return {713 kind: 'sale',714 ...base,715 externalId: row.ebayId ? `ebay:${row.ebayId}` : `${p.productId ?? p.url}:${row.date}:${row.price}`,716 rawTitle: row.title,717 description: null,718 grade,719 condition,720 saleType: 'unknown',721 saleDate,722 price: row.price,723 currency: 'USD',724 buyerPremiumIncluded: false,725 quantity: 1,726 isBundle: /\b(lot|bundle)\b/i.test(row.title),727 location: null,728 auctionHouse: null,729 lotNumber: null,730 confidence: 0.75,731 };732}733734/** Align bracket tags with the API catalogs' variant vocabulary (pokemontcg: 'Holo', 'Reverse Holo', '1st Edition', '1st Edition Holo'; scryfall: 'Foil'). */735export function normalizeCardVariant(variant: string | null, slug: string): string | null {736 if (!variant) return null;737 const parts = variant.split(' · ').map((v) => v.trim()).filter(Boolean);738 const mapped = parts.map((v) => {739 const l = v.toLowerCase();740 if (slug === 'pokemon') {741 if (l === 'reverse holo' || l === 'reverse holofoil') return 'Reverse Holo';742 if (l === 'holo' || l === 'holofoil') return 'Holo';743 if (l === '1st edition') return '1st Edition';744 if (l === 'shadowless') return 'Shadowless';745 }746 if (slug === 'magic_the_gathering') {747 if (l === 'foil') return 'Foil';748 if (l === 'etched foil' || l === 'foil etched') return 'Etched Foil';749 }750 return v;751 });752 // Pokémon: "1st Edition · Holo" → "1st Edition Holo" (pokemontcg vocabulary)753 if (slug === 'pokemon' && mapped.includes('1st Edition') && mapped.includes('Holo')) {754 return ['1st Edition Holo', ...mapped.filter((v) => v !== '1st Edition' && v !== 'Holo')].join(' · ');755 }756 return mapped.join(' · ');757}758759const CARD_BRANDS = ['Topps', 'Panini', 'Upper Deck', 'Bowman', 'Fleer', 'Donruss', 'Score', 'Leaf', 'Futera', 'O-Pee-Chee', 'Skybox', 'Hoops', 'Pinnacle', 'Pro Set', 'Goudey', 'Playoff', 'Pacific', 'Select', 'Prizm', 'Mosaic', 'Optic'];760export function brandFromSet(setName: string): string | null {761 for (const b of CARD_BRANDS) if (new RegExp(`\\b${b.replace('-', '\\-')}\\b`, 'i').test(setName)) return b === 'Prizm' || b === 'Select' || b === 'Mosaic' || b === 'Optic' ? 'Panini' : b;762 return null;763}764