TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * PCGS Price Guide — US coin series pages: one row per coin (PCGS #, description, designation)7 * with guide values per grade column. Raw payload = parsed table; normalise → catalog item +8 * one guide_value observation per (grade, plus-grade).9 */1011const BASE = 'https://www.pcgs.com';12const PARSER_VERSION = '1.0.0';1314export const RowSchema = z.object({15 pcgsNumber: z.string(),16 description: z.string(),17 designation: z.string().nullable(),18 /** grade column label (e.g. "65") → { value, plus } */19 prices: z.record(z.string(), z.object({ value: z.number().nullable(), plus: z.number().nullable() })),20});21export const PagePayloadSchema = z.object({22 kind: z.literal('price_guide_page'),23 url: z.string(),24 seriesName: z.string(),25 seriesSlug: z.string(),26 seriesId: z.string(),27 lastUpdate: z.string().nullable(),28 grades: z.array(z.string()),29 rows: z.array(RowSchema),30});31export type PagePayload = z.infer<typeof PagePayloadSchema>;3233function money(s: string | null): number | null {34 if (!s) return null;35 const n = Number(s.replace(/[^\d.]/g, ''));36 return Number.isFinite(n) && n > 0 ? n : null;37}3839export function parsePriceGuidePage(htmlText: string, url: string): PagePayload | null {40 const $ = H.load(htmlText);41 const h1 = H.text($('h1').first()) ?? '';42 const seriesName = h1.replace(/\s*Price Guide$/i, '').trim();43 const m = url.match(/\/prices\/detail\/([^/]+)\/(\d+)/);44 if (!m || !seriesName) return null;45 const lastUpdate = $('main').text().match(/Last Update:\s*([0-9]{2}-[0-9]{2}\s+[0-9:]+\s*[AP]M\s*[A-Z]{3})/)?.[1] ?? null;46 // header: find the row containing "PCGS #" and numeric grade cells47 let grades: string[] = [];48 let table: ReturnType<typeof $> | null = null;49 $('table').each((_, t) => {50 const txt = $(t).text();51 if (/PCGS\s*#/.test(txt) && /Desig/.test(txt)) {52 table = $(t);53 return false;54 }55 return undefined;56 });57 if (!table) return null;58 const tbl = table as ReturnType<typeof $>;59 tbl.find('tr').each((_, tr) => {60 const cells = $(tr).find('th,td').map((__, c) => H.text($(c)) ?? '').get();61 if (cells.some((c) => /PCGS\s*#/.test(c))) {62 grades = cells.filter((c) => /^\d{1,2}$/.test(c));63 return false;64 }65 return undefined;66 });67 if (!grades.length) return null;68 const rows: z.infer<typeof RowSchema>[] = [];69 tbl.find('tr').each((_, tr) => {70 const $tr = $(tr);71 const numLink = $tr.find('a[href*="/coinfacts/coin/detail/"]').first();72 const pcgsNumber = (H.text(numLink) ?? '').trim();73 if (!/^\d+$/.test(pcgsNumber)) return;74 const tds = $tr.children('td');75 const descCell = tds.eq(1).clone();76 descCell.find('.hidden-print, a[data-func]').remove();77 const description = H.text(descCell) ?? '';78 const desigCell = tds.eq(2).clone();79 desigCell.find('br').replaceWith(' ');80 const designation = (H.text(desigCell) ?? '').replace(/\s*\+\s*$/, '').trim() || null;81 const prices: Record<string, { value: number | null; plus: number | null }> = {};82 const priceCells = tds.slice(3);83 grades.forEach((g, i) => {84 const cell = priceCells.eq(i);85 if (!cell.length) return;86 const anchors = cell.find('a');87 const value = money(H.text(anchors.eq(0)));88 const plus = anchors.length > 1 ? money(H.text(anchors.eq(1))) : null;89 if (value !== null || plus !== null) prices[g] = { value, plus };90 });91 if (description && Object.keys(prices).length) rows.push({ pcgsNumber, description, designation, prices });92 });93 return { kind: 'price_guide_page', url, seriesName, seriesSlug: m[1]!, seriesId: m[2]!, lastUpdate, grades, rows };94}9596/** "09-06 10:59 PM EST" + fetch date → Date (year inferred from the fetch date, never after it). */97export function lastUpdateDate(stamp: string | null, fetchedAt: Date): Date {98 const m = stamp?.match(/^(\d{2})-(\d{2})/);99 if (!m) return fetchedAt;100 let year = fetchedAt.getUTCFullYear();101 let d = new Date(Date.UTC(year, Number(m[1]) - 1, Number(m[2])));102 if (d.getTime() > fetchedAt.getTime() + 86_400_000) d = new Date(Date.UTC(--year, Number(m[1]) - 1, Number(m[2])));103 return d;104}105106/** "1878 8TF" → { year: 1878, mintMark: null, variety: '8TF' }; "1893-S" → { year: 1893, mintMark: 'S' } */107export function parseDescription(desc: string): { year: number | null; mintMark: string | null; variety: string | null } {108 const m = desc.match(/^(\d{4})(?:-([A-Z]{1,2}))?\s*(.*)$/);109 if (!m) return { year: null, mintMark: null, variety: desc || null };110 return { year: Number(m[1]), mintMark: m[2] ?? null, variety: m[3]?.trim() || null };111}112113export class PcgsPriceGuideConnector extends BaseConnector {114 readonly version = '1.0.0';115 readonly parserVersion = PARSER_VERSION;116 protected override minIntervalMs = 2000;117118 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {119 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];120 const desigs = (this.meta.config.designations as string[] | undefined) ?? ['ms'];121 let count = 0;122 for (const seed of seeds) {123 for (const d of desigs) {124 if (ctx.signal?.aborted || this.reached(ctx, count)) return;125 const url = `${BASE}/prices/detail/${seed}/most-active/${d}`;126 if (ctx.options.mode !== 'backfill' && !(await ctx.shouldFetch(url))) continue;127 await this.throttle();128 const res = await ctx.fetch(url, {129 engines: ['firecrawl', 'scrapfly'],130 expect: ['title', 'price', 'identifiers'],131 parse: (r) => {132 const p = r.html ? parsePriceGuidePage(r.html, url) : null;133 const row = p?.rows[0];134 return p && row ? { title: p.seriesName, price: Object.values(row.prices)[0]?.value ?? null, identifiers: { pcgs: row.pcgsNumber } } : null;135 },136 });137 const payload = res.success && res.html ? parsePriceGuidePage(res.html, url) : null;138 if (!payload || payload.rows.length === 0) {139 ctx.anomaly(payload ? 'empty_page' : 'page_parse_failed', `${url}: ${res.error ?? res.httpStatus}`);140 continue;141 }142 count++;143 yield { url, externalId: `series:${payload.seriesId}:${d}`, kind: 'price_observation', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };144 }145 }146 }147148 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {149 const p = PagePayloadSchema.parse(raw.payload);150 const obsDate = lastUpdateDate(p.lastUpdate, raw.fetchedAt);151 const out: NormalizedRecord[] = [];152 for (const row of p.rows) {153 const { year, mintMark, variety } = parseDescription(row.description);154 const desig = row.designation?.split(/\s+/)[0]?.toUpperCase() ?? null; // MS | PR | SP155 const name = `${row.description} ${p.seriesName}`.trim();156 const rawTitle = `${row.description} ${p.seriesName}${desig ? ` (${desig})` : ''} · PCGS #${row.pcgsNumber}`;157 const attributes = AssetAttributesSchema.parse({158 categorySlug: 'coins',159 brand: 'United States Mint',160 series: p.seriesName,161 set: p.seriesName,162 name,163 number: row.pcgsNumber,164 year,165 variant: [mintMark ? `${mintMark} mint` : null, variety, desig && desig !== 'MS' ? desig : null].filter(Boolean).join(' · ') || null,166 country: 'US',167 identifiers: { pcgs_number: row.pcgsNumber },168 metadata: { mint_mark: mintMark, variety, designation: desig, pcgs_series_id: p.seriesId, pcgs_series_slug: p.seriesSlug },169 });170 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${BASE}/coinfacts/coin/detail/${row.pcgsNumber}`, rawTitle, imageUrls: [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };171 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `coin:${row.pcgsNumber}`, confidence: 0.95 }));172 for (const [g, pr] of Object.entries(row.prices)) {173 for (const [suffix, value] of [['', pr.value], ['+', pr.plus]] as const) {174 if (!value) continue;175 const grade = `${desig ?? 'MS'}${g}${suffix}`;176 out.push(177 NormalizedPriceObservationSchema.parse({178 kind: 'price_observation',179 ...base,180 externalId: `coin:${row.pcgsNumber}:${grade}`,181 confidence: 0.85,182 grade: { grader: 'pcgs', grade, qualifier: null, certificationNumber: null },183 priceKind: 'guide_value',184 price: value,185 currency: 'USD',186 observationDate: obsDate,187 sampleSize: null,188 }),189 );190 }191 }192 }193 return out;194 }195}196197export default function createConnector(meta: ConnectorMeta) {198 return new PcgsPriceGuideConnector(meta);199}200