TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';45/**6 * NGC US Coin Price Guide — one server-rendered page per series (subcategory): a pinned table with the7 * coin rows (NGC coin id, year/mint, description, denomination, designation, "Updated: m/d/yyyy") and a8 * price table with one cell per grade (`price-history-link coin-id grade`). Values are NGC retail guide9 * values in USD. Subcategories come from the public JSON /coin-explorer/data/categories/.10 */1112const SITE = 'https://www.ngccoin.com';13const PARSER_VERSION = '1.0.0';1415export const CoinSchema = z.object({16 coinId: z.string(),17 yearMint: z.string().nullable(),18 description: z.string(),19 denomination: z.string().nullable(),20 designation: z.string().nullable(),21 strike: z.enum(['ms', 'pf']).nullable(),22 updatedText: z.string().nullable(),23 /** grade label as NGC prints it ("PrAg", "VF", "XF+", "50", "65", "65+", "70★") → USD */24 prices: z.record(z.string(), z.number()),25});26export const PayloadSchema = z.object({27 kind: z.literal('price_guide_series'),28 url: z.string(),29 subcategoryId: z.string(),30 categorySeo: z.string().nullable(),31 seriesName: z.string(),32 coins: z.array(CoinSchema),33});34export type Payload = z.infer<typeof PayloadSchema>;3536const CategoriesSchema = z.array(z.object({ CategoryID: z.number(), Name: z.string(), SeoName: z.string(), Subcategories: z.array(z.object({ SubcategoryID: z.number(), Name: z.string(), SubcategorySeoName: z.string().nullable().optional(), HasMS: z.boolean().nullable().optional(), HasPF: z.boolean().nullable().optional() })) }));37export type Subcategory = { id: string; name: string; categorySeo: string; categoryName: string };3839export function parseCategories(json: unknown): Subcategory[] {40 const parsed = CategoriesSchema.safeParse(json);41 if (!parsed.success) return [];42 return parsed.data.flatMap((c) => c.Subcategories.map((s) => ({ id: String(s.SubcategoryID), name: s.Name, categorySeo: c.SeoName, categoryName: c.Name })));43}4445function money(s: string | null | undefined): number | null {46 if (!s) return null;47 const n = Number(s.replace(/[^\d.]/g, ''));48 return Number.isFinite(n) && n > 0 ? n : null;49}5051export function parseSeriesPage(htmlText: string, url: string, categorySeo: string | null): Payload | null {52 const $ = H.load(htmlText);53 const seriesName = H.text($('h1.census-header-title').first()) ?? '';54 const subcategoryId = url.match(/\/(\d+)\/?(?:[?#].*)?$/)?.[1] ?? $('[subcategory-id]').first().attr('subcategory-id') ?? '';55 if (!seriesName || !subcategoryId) return null;56 const coins = new Map<string, z.infer<typeof CoinSchema>>();57 $('td[parent-coin-id]').each((_, td) => {58 const cell = $(td);59 const coinId = cell.attr('parent-coin-id');60 if (!coinId || coins.has(coinId)) return;61 const tr = cell.closest('tr');62 const cls = tr.attr('class') ?? '';63 const strike = /\bpf\b/.test(cls) ? 'pf' : /\bms\b/.test(cls) ? 'ms' : null;64 const description = H.text(cell.find('.merged').first()) ?? H.text(cell.find('a').first()) ?? '';65 if (!description) return;66 const tds = tr.children('td');67 coins.set(coinId, {68 coinId,69 yearMint: H.text(cell.find('.standard').first()),70 description,71 denomination: H.text(tds.eq(1)),72 designation: H.text(tds.eq(2)),73 strike,74 updatedText: (H.text(cell.find('.last-updated').first()) ?? '').replace(/^Updated:\s*/i, '') || null,75 prices: {},76 });77 });78 $('td[price-history-link]').each((_, td) => {79 const cell = $(td);80 const coinId = cell.attr('coin-id');81 const grade = cell.attr('grade');82 const value = money(H.text(cell));83 if (!coinId || !grade || value === null) return;84 const coin = coins.get(coinId);85 if (coin) coin.prices[grade] = value;86 });87 return { kind: 'price_guide_series', url, subcategoryId, categorySeo, seriesName, coins: [...coins.values()].filter((c) => Object.keys(c.prices).length) };88}8990/** "9/8/2026" (US m/d/yyyy) → UTC date; null when absent. */91export function updatedDate(s: string | null): Date | null {92 const m = s?.match(/(\d{1,2})\/(\d{1,2})\/(\d{4})/);93 if (!m) return null;94 return new Date(Date.UTC(Number(m[3]), Number(m[1]) - 1, Number(m[2])));95}9697/** "1878 8TF" → { year, mintMark, variety }; "1893-S" → S; "1921 D" → D. */98export function parseYearMint(s: string | null): { year: number | null; mintMark: string | null; variety: string | null } {99 const m = s?.match(/^(\d{4})(?:[-\s]([A-Z]{1,2}))?\s*(.*)$/);100 if (!m) return { year: null, mintMark: null, variety: s?.trim() || null };101 return { year: Number(m[1]), mintMark: m[2] ?? null, variety: m[3]?.trim() || null };102}103104/** NGC grade label + designation → grade token ("65" + "MS" → MS65, "65+" → MS65+, "XF+" stays, "PrAg" stays). */105export function gradeToken(label: string, designation: string | null, strike: 'ms' | 'pf' | null): string {106 const star = label.includes('★') || /star/i.test(label);107 const base = label.replace(/★|star/gi, '').trim();108 const prefix = designation?.match(/^(MS|PF|SP|PL)/)?.[1] ?? (strike === 'pf' ? 'PF' : 'MS');109 const token = /^\d{1,2}\+?$/.test(base) ? `${prefix}${base}` : base;110 return star ? `${token}★` : token;111}112113interface Cursor {114 index?: number;115 done?: boolean;116 updatedAt?: string;117}118119export class NgcPriceGuideConnector extends BaseConnector {120 readonly version = '1.0.0';121 readonly parserVersion = PARSER_VERSION;122 protected override minIntervalMs = 4000;123124 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {125 const perRun = Number(this.meta.config.subcategoriesPerRun ?? 10);126 const only = (this.meta.config.onlySubcategories as string[] | undefined) ?? [];127 const cursor = { ...((ctx.options.cursor ?? {}) as Cursor) };128 await this.throttle();129 const cats = await ctx.fetch(`${SITE}/coin-explorer/data/categories/`, { engines: ['api'], minQuality: 0 });130 let subs = cats.success && cats.json ? parseCategories(cats.json) : [];131 if (!subs.length) {132 ctx.anomaly(cats.success ? 'schema_drift' : 'page_fetch_failed', `categories: ${cats.error ?? cats.httpStatus ?? 'unexpected shape'}`);133 return;134 }135 if (only.length) subs = subs.filter((s) => only.includes(s.id));136 let i = Math.min(cursor.index ?? 0, subs.length);137 let count = 0;138 let fetched = 0;139 while (i < subs.length && fetched < perRun && !ctx.signal?.aborted && !this.reached(ctx, count)) {140 const sub = subs[i]!;141 const url = `${SITE}/price-guide/united-states/${sub.categorySeo}/${sub.id}/`;142 i++;143 if (ctx.options.mode !== 'backfill' && !(await ctx.shouldFetch(url))) continue;144 await this.throttle();145 const res = await ctx.fetch(url, {146 engines: ['api', 'firecrawl'],147 responseType: 'text',148 expect: ['title', 'price', 'identifiers'],149 parse: (r) => {150 const p = r.html ? parseSeriesPage(r.html, url, sub.categorySeo) : null;151 const c = p?.coins[0];152 return p && c ? { title: p.seriesName, price: Object.values(c.prices)[0] ?? null, identifiers: { ngc: c.coinId } } : null;153 },154 });155 fetched++;156 const payload = res.success && res.html ? parseSeriesPage(res.html, url, sub.categorySeo) : null;157 if (!payload || !payload.coins.length) {158 ctx.anomaly(res.success ? 'parse_failure_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no priced rows'}`);159 continue;160 }161 count++;162 yield { url, externalId: `subcategory:${sub.id}`, kind: 'price_observation', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };163 await ctx.setCursor({ index: i >= subs.length ? 0 : i, updatedAt: new Date().toISOString() });164 if (ctx.options.mode === 'backfill') await ctx.progress({ page: i, totalPages: subs.length, itemsProcessed: count });165 }166 if (i >= subs.length) await ctx.setCursor({ index: 0, done: ctx.options.mode === 'backfill', updatedAt: new Date().toISOString() });167 }168169 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {170 const p = PayloadSchema.parse(raw.payload);171 const out: NormalizedRecord[] = [];172 for (const coin of p.coins) {173 const { year, mintMark, variety } = parseYearMint(coin.yearMint);174 const obsDate = updatedDate(coin.updatedText);175 if (!obsDate) continue; // never date a guide value with the fetch time176 const desig = coin.designation?.toUpperCase() ?? null;177 const attributes = AssetAttributesSchema.parse({178 categorySlug: 'coins',179 brand: 'United States Mint',180 series: p.seriesName,181 set: p.seriesName,182 name: coin.description,183 number: coin.coinId,184 year,185 variant: [mintMark ? `${mintMark} mint` : null, variety, desig && desig !== 'MS' ? desig : null].filter(Boolean).join(' · ') || null,186 country: 'US',187 identifiers: { ngc_coin_id: coin.coinId },188 metadata: { mint_mark: mintMark, variety, designation: desig, strike: coin.strike, denomination: coin.denomination, ngc_subcategory_id: p.subcategoryId, ngc_category_seo: p.categorySeo, guide: 'NGC US Coin Price Guide (retail)' },189 });190 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: `${SITE}/coin-explorer/${coin.coinId}/`, rawTitle: `${coin.description} · NGC #${coin.coinId}`, imageUrls: [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };191 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `coin:${coin.coinId}`, confidence: 0.95 }));192 for (const [label, value] of Object.entries(coin.prices)) {193 const grade = gradeToken(label, desig, coin.strike);194 out.push(195 NormalizedPriceObservationSchema.parse({196 kind: 'price_observation',197 ...base,198 externalId: `coin:${coin.coinId}:${grade}`,199 confidence: 0.85,200 grade: { grader: 'ngc', grade, qualifier: null, certificationNumber: null },201 priceKind: 'guide_value',202 price: value,203 currency: 'USD',204 observationDate: obsDate,205 sampleSize: null,206 }),207 );208 }209 }210 return out;211 }212}213214export default (meta: ConnectorMeta) => new NgcPriceGuideConnector(meta);215