TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { type NormalizedRecord } from '@rareindex/shared';4import { attrs, catalogItem, makeTitle, num, priceObservation, withRetries } from '../_lib/shared.js';5import { dayOf, isoDay, parseCompactGrade } from '../_lib/tcg-shared.js';67/** PokemonPrice.com — per-grade fair-value model for Pokémon cards, embedded in server-rendered pages. */8const BASE = 'https://www.pokemonprice.com';9const PARSER_VERSION = '1.0.0';10const HEADERS = { 'user-agent': 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data)', accept: 'text/html,application/xhtml+xml' };1112const GradeRowSchema = z.object({13 grade: z.string(),14 fair_price: z.union([z.string(), z.number()]).nullable().optional(),15 low_price: z.union([z.string(), z.number()]).nullable().optional(),16 high_price: z.union([z.string(), z.number()]).nullable().optional(),17 confidence: z.union([z.string(), z.number()]).nullable().optional(),18 last_sale_date: z.string().nullable().optional(),19});20const TxSchema = z.object({ month: z.string(), grade: z.string(), count: z.number() });21const PayloadSchema = z.object({22 slug: z.string(),23 setSlug: z.string(),24 name: z.string(),25 tag: z.string().nullable(),26 number: z.string().nullable(),27 total: z.string().nullable(),28 setName: z.string().nullable(),29 image: z.string().nullable(),30 rows: z.array(GradeRowSchema),31 transactions: z.array(TxSchema),32});33export type PokemonPricePayload = z.infer<typeof PayloadSchema>;3435/** Decode the RSC flight strings of a Next.js app-router page into one searchable string. */36export function flightText(html: string): string {37 const parts: string[] = [];38 for (const m of html.matchAll(/self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g)) {39 try {40 parts.push(JSON.parse(`"${m[1]}"`) as string);41 } catch {42 /* skip undecodable chunk */43 }44 }45 return parts.join('');46}4748/** Extract a balanced JSON array/object that starts right after `key":`. */49export function extractJson(text: string, key: string): unknown | null {50 const idx = text.indexOf(`"${key}":`);51 if (idx < 0) return null;52 let i = text.indexOf(':', idx) + 1;53 while (text[i] === ' ') i++;54 const open = text[i];55 if (open !== '[' && open !== '{') return null;56 const close = open === '[' ? ']' : '}';57 let depth = 0;58 let inStr = false;59 for (let j = i; j < text.length; j++) {60 const ch = text[j]!;61 if (inStr) {62 if (ch === '\\') j++;63 else if (ch === '"') inStr = false;64 continue;65 }66 if (ch === '"') inStr = true;67 else if (ch === open) depth++;68 else if (ch === close) {69 depth--;70 if (depth === 0) {71 try {72 return JSON.parse(text.slice(i, j + 1));73 } catch {74 return null;75 }76 }77 }78 }79 return null;80}8182export function parseCardPage(html: string, url: string): PokemonPricePayload | null {83 const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/);84 const spans = h1 ? [...h1[1]!.matchAll(/<span[^>]*>([^<]*)<\/span>/g)].map((m) => m[1]!.trim()).filter(Boolean) : [];85 const titleSpan = spans[0] ?? '';86 const nm = titleSpan.match(/^(.*?)\s*(?:\[(.+?)\])?$/);87 const name = (nm?.[1] ?? titleSpan).trim();88 const tag = nm?.[2]?.trim() ?? null;89 const numSpan = spans.find((s) => /^\d+[A-Za-z]?\s*\/\s*\w+$/.test(s) || /^[A-Za-z0-9]+\s*\/\s*[A-Za-z0-9]+$/.test(s));90 const [number, total] = numSpan ? numSpan.split('/').map((s) => s.trim()) : [null, null];91 const setName = spans.filter((s) => s !== titleSpan && s !== numSpan).pop() ?? null;92 const image = html.match(/https:\/\/cdn\.pokemonprice\.com\/cards\/[^"\\\s]+/)?.[0] ?? null;93 const text = flightText(html);94 const rowsRaw = extractJson(text, 'rows');95 const rows = Array.isArray(rowsRaw) ? rowsRaw.filter((r) => r && typeof r === 'object' && 'grade' in (r as object) && 'fair_price' in (r as object)).map((r) => {96 const o = r as Record<string, unknown>;97 return { grade: String(o.grade), fair_price: o.fair_price as string, low_price: o.low_price as string, high_price: o.high_price as string, confidence: o.confidence as string, last_sale_date: (o.last_sale_date as string) ?? null };98 }) : [];99 const txRaw = extractJson(text, 'transactions');100 const transactions = Array.isArray(txRaw) ? txRaw.filter((t) => t && typeof t === 'object' && 'month' in (t as object)).map((t) => ({ month: String((t as Record<string, unknown>).month), grade: String((t as Record<string, unknown>).grade), count: Number((t as Record<string, unknown>).count) || 0 })) : [];101 if (!name) return null;102 const parts = new URL(url).pathname.split('/').filter(Boolean);103 const parsed = PayloadSchema.safeParse({ slug: parts.slice(-1)[0] ?? '', setSlug: parts[0] ?? '', name, tag, number: number ?? null, total: total ?? null, setName, image, rows, transactions });104 return parsed.success ? parsed.data : null;105}106107/** "Base Set 1st Edition" + tag "Holo 1st edition" → { set: "Base Set", variant: "1st Edition Holo" } */108export function setAndVariant(setName: string | null, tag: string | null): { set: string | null; variant: string | null } {109 let set = setName?.trim() ?? null;110 const flags = new Set<string>();111 if (set) {112 const m = set.match(/^(.*?)\s+(1st Edition|Shadowless|Unlimited)$/i);113 if (m) {114 set = m[1]!.trim();115 const f = m[2]!.toLowerCase();116 if (f === '1st edition') flags.add('1st Edition');117 if (f === 'shadowless') flags.add('Shadowless');118 }119 }120 for (const t of (tag ?? '').split(/[,·]/).map((s) => s.trim().toLowerCase()).filter(Boolean)) {121 if (t.includes('reverse')) flags.add('Reverse Holo');122 else if (t.includes('1st') && t.includes('holo')) {123 flags.add('1st Edition');124 flags.add('Holo');125 } else if (t.includes('1st')) flags.add('1st Edition');126 else if (t.includes('holo')) flags.add('Holo');127 else if (t.includes('shadowless')) flags.add('Shadowless');128 else if (t !== 'unlimited') flags.add(t.replace(/\b\w/g, (c) => c.toUpperCase()));129 }130 const ordered: string[] = [];131 if (flags.has('1st Edition') && flags.has('Holo')) ordered.push('1st Edition Holo');132 else {133 if (flags.has('1st Edition')) ordered.push('1st Edition');134 if (flags.has('Holo')) ordered.push('Holo');135 }136 if (flags.has('Shadowless')) ordered.push('Shadowless');137 if (flags.has('Reverse Holo')) ordered.push('Reverse Holo');138 for (const f of flags) if (!['1st Edition', 'Holo', 'Shadowless', 'Reverse Holo'].includes(f)) ordered.push(f);139 return { set, variant: ordered.length ? ordered.join(' · ') : null };140}141142export class PokemonPriceConnector extends BaseConnector {143 readonly version = '1.0.0';144 readonly parserVersion = PARSER_VERSION;145 protected override minIntervalMs = 1500;146 override readonly urlPatterns = [/pokemonprice\.com\/[a-z0-9-]+\/[a-z0-9-]+$/i];147148 private async page(ctx: CrawlContext, url: string) {149 await this.throttle();150 return withRetries(() => ctx.fetch(url, { engines: ['api', 'firecrawl'], headers: HEADERS, responseType: 'text', minQuality: 0.3 }), (r) => r.success && Boolean(r.html), 3, 2500);151 }152153 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {154 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];155 let sets = seeds;156 if (!sets.length) {157 const idx = await this.page(ctx, `${BASE}/card-lists`);158 sets = [...new Set([...(idx.html ?? '').matchAll(/href="\/card-lists\/([a-z0-9-]+)"/g)].map((m) => m[1]!))];159 }160 const maxCards = ctx.options.mode === 'backfill' ? Infinity : Number(this.meta.config.maxCardsPerRun ?? 300);161 let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);162 if (setIdx >= sets.length) setIdx = 0;163 let count = 0;164 for (; setIdx < sets.length; setIdx++) {165 const setSlug = sets[setIdx]!;166 const cardSlugs: string[] = [];167 for (let offset = 0; offset < 1200; offset += 30) {168 const list = await this.page(ctx, `${BASE}/card-lists/${setSlug}${offset ? `?offset=${offset}` : ''}`);169 if (!list.success || !list.html) {170 ctx.anomaly('page_fetch_failed', `list ${setSlug}@${offset}: ${list.error ?? list.httpStatus}`);171 break;172 }173 const found = [...new Set([...list.html.matchAll(new RegExp(`href="/${setSlug}/([a-z0-9-]+)"`, 'g'))].map((m) => m[1]!))];174 const fresh = found.filter((s) => !cardSlugs.includes(s));175 if (!fresh.length) break;176 cardSlugs.push(...fresh);177 if (!list.html.includes(`offset=${offset + 30}`)) break;178 }179 for (const cardSlug of cardSlugs) {180 if (ctx.signal?.aborted) return;181 if (this.reached(ctx, count) || count >= maxCards) {182 await ctx.setCursor({ setIdx, updatedAt: new Date().toISOString() });183 return;184 }185 const url = `${BASE}/${setSlug}/${cardSlug}`;186 const res = await this.page(ctx, url);187 if (!res.success || !res.html) {188 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);189 continue;190 }191 const payload = parseCardPage(res.html, url);192 if (!payload) {193 ctx.anomaly('parse_failure_card', url);194 continue;195 }196 count++;197 yield { url, externalId: `${setSlug}/${cardSlug}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };198 }199 await ctx.setCursor({ setIdx: setIdx + 1, updatedAt: new Date().toISOString() });200 }201 await ctx.setCursor({ setIdx: 0, updatedAt: new Date().toISOString() });202 }203204 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {205 if (!this.urlPatterns[0]!.test(url)) return [];206 const res = await this.page(ctx, url);207 if (!res.success || !res.html) return [];208 const payload = parseCardPage(res.html, url);209 if (!payload) return [];210 return [{ url, externalId: `${payload.setSlug}/${payload.slug}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];211 }212213 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {214 const p = PayloadSchema.parse(raw.payload);215 const { set, variant } = setAndVariant(p.setName, p.tag);216 const a = attrs({217 categorySlug: 'pokemon',218 franchise: 'Pokémon',219 brand: 'The Pokémon Company',220 set,221 name: p.name,222 number: p.number,223 year: null,224 variant,225 language: 'English',226 identifiers: { pokemonprice_slug: `${p.setSlug}/${p.slug}` },227 metadata: { total: p.total, set_label: p.setName, tag: p.tag },228 });229 const images = p.image ? [p.image] : [];230 const rawTitle = makeTitle({ name: p.name, set, number: p.number, total: p.total, variant });231 const observedAt = raw.fetchedAt;232 const out: NormalizedRecord[] = [233 catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${p.setSlug}/${p.slug}`, rawTitle, imageUrls: images, attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION, releaseDate: null }),234 ];235 const kinds = new Set(((this.meta.config.priceKinds as string[] | undefined) ?? ['guide_value', 'low', 'high']));236 const txByGrade = new Map<string, number>();237 for (const t of p.transactions) txByGrade.set(t.grade, (txByGrade.get(t.grade) ?? 0) + t.count);238 for (const row of p.rows) {239 const { grader, grade } = parseCompactGrade(row.grade);240 if (!grader && !/^raw$/i.test(row.grade)) continue;241 const fair = num(row.fair_price);242 if (!fair) continue;243 const conf = Math.min(0.8, Math.max(0.2, (Number(row.confidence) || 40) / 100));244 // The fair price is the model's current estimate → dated by fetch day; the last sale date is kept in the title context only.245 const obsDate = dayOf(observedAt);246 const lastSale = isoDay(row.last_sale_date);247 const sample = txByGrade.get(row.grade) ?? null;248 const gradeLabel = grader === 'raw' ? 'Raw' : `${grader!.toUpperCase()} ${grade}`;249 const title = `${rawTitle} — ${gradeLabel}${lastSale ? ` (last sale ${lastSale.toISOString().slice(0, 10)})` : ''}`;250 const emit = (kind: 'guide_value' | 'low' | 'high', price: number | null) => {251 if (!price || !kinds.has(kind)) return;252 out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `${p.setSlug}/${p.slug}:${row.grade}:${kind}`, rawTitle: title, imageUrls: images, attributes: a, grade: { grader: grader === 'raw' ? 'raw' : grader, grade, qualifier: null, certificationNumber: null }, observedAt, confidence: conf, parserVersion: PARSER_VERSION, priceKind: kind, price, currency: 'USD', observationDate: obsDate, sampleSize: sample }));253 };254 emit('guide_value', fair);255 emit('low', num(row.low_price));256 emit('high', num(row.high_price));257 }258 return out;259 }260}261262export default (meta: ConnectorMeta) => new PokemonPriceConnector(meta);263