TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';45const PARSER_VERSION = '1.0.0';67export const CardSchema = z.object({8 id: z.string(),9 url: z.string(),10 title: z.string(),11 price: z.number().nullable(),12 message: z.string().nullable(),13 image: z.string().nullable(),14 badge: z.string().nullable(),15});16export type Card = z.infer<typeof CardSchema>;17export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), series: z.string().nullable(), country: z.string().nullable(), items: z.array(CardSchema) });18export type PagePayload = z.infer<typeof PagePayloadSchema>;1920export function parseCategoryPage(htmlText: string, pageUrl: string, series: string | null, country: string | null): PagePayload {21 const $ = H.load(htmlText);22 const items: Card[] = [];23 const seen = new Set<string>();24 $('.mod-product-card').each((_, el) => {25 const e = $(el);26 const a = e.find('a.item-link').first();27 const url = a.attr('href');28 const id = a.attr('data-product-id') ?? url?.match(/\/product\/(\d+)\//)?.[1];29 if (!url || !id || seen.has(id)) return;30 seen.add(id);31 const title = (a.attr('data-product-name') ?? H.text(e.find('.mod-product-title').first()) ?? '').replace(/\s+/g, ' ').trim();32 if (!title) return;33 const price = parsePrice(H.text(e.find('.mod-product-pricing .price').first()), 'USD')?.amount ?? null;34 const message = H.text(e.find('.mod-product-message').first());35 const image = e.find('.mod-product-img img').attr('src') ?? null;36 const badge = H.text(e.find('.product-badge').first());37 items.push({ id, url, title, price, message, image, badge });38 });39 return { kind: 'category_page', url: pageUrl, series, country, items };40}4142/** "1887-S $10 Liberty Gold Eagle MS-63 NGC" → parts. Unknown fields stay null. */43export function parseCoinTitle(title: string): { year: number | null; mintMark: string | null; grader: string | null; grade: string | null; designation: string | null; cleaned: boolean; random: boolean } {44 const y = title.match(/\b(1[6-9]\d{2}|20\d{2})(?:-([A-Z]{1,2}))?\b/);45 const g = title.match(/\b(MS|AU|XF|EF|VF|F|VG|G|AG|PR|PF|PL|SP|BU)-?(\d{1,2})(\+?)(?!\d)/i);46 const grader = title.match(/\b(PCGS|NGC|ANACS|ICG|CAC)\b/i)?.[1]?.toLowerCase() ?? null;47 const designation = title.match(/\b(CAC|DMPL|PL|FBL|FB|RD|RB|BN|Cameo|Ultra Cameo|DCAM|First Strike|Early Releases)\b/i)?.[1] ?? null;48 return {49 year: y ? Number(y[1]) : null,50 mintMark: y?.[2] ?? null,51 grader,52 grade: g ? `${g[1]!.toUpperCase()}${g[2]}${g[3] ?? ''}` : null,53 designation,54 cleaned: /\(cleaned\)|cleaned/i.test(title),55 random: /random/i.test(title),56 };57}5859export class ApmexConnector extends BaseConnector {60 readonly version = '1.0.0';61 readonly parserVersion = PARSER_VERSION;62 protected override minIntervalMs = 2500;6364 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {65 const seeds = (this.meta.config.seeds as Array<{ url: string; series?: string; country?: string }> | undefined) ?? [];66 const pages = Number(this.meta.config.pagesPerSeed ?? 1);67 const cap = ctx.options.limit;68 let count = 0;69 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;70 for (let i = start; i < seeds.length; i++) {71 const seed = seeds[i]!;72 for (let page = 1; page <= pages; page++) {73 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;74 const url = page > 1 ? `${seed.url}?page=${page}` : seed.url;75 await this.throttle();76 const res = await ctx.fetch(url, {77 engines: ['firecrawl', 'scrapfly'],78 expect: ['title', 'price'],79 parse: (r) => {80 const p = r.html ? parseCategoryPage(r.html, url, seed.series ?? null, seed.country ?? null) : null;81 return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null;82 },83 });84 const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.series ?? null, seed.country ?? null) : null;85 if (!payload?.items.length) {86 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product cards'}`);87 break;88 }89 count++;90 yield { url, externalId: `${seed.url}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };91 }92 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });93 }94 }9596 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {97 const p = PagePayloadSchema.parse(raw.payload);98 const out: NormalizedRecord[] = [];99 for (const c of p.items) {100 const t = parseCoinTitle(c.title);101 const name = c.title.replace(/\s*\(cleaned\)/i, '').replace(/\b(PCGS|NGC|ANACS|ICG)\b/gi, '').replace(/\b(MS|AU|XF|EF|VF|F|VG|G|PR|PF|BU)-?\d{1,2}\+?\b/gi, '').replace(/\s+/g, ' ').trim();102 const attributes = AssetAttributesSchema.parse({103 categorySlug: 'coins',104 series: p.series,105 name,106 year: t.year,107 variant: [t.mintMark ? `${t.mintMark} mint` : null, t.designation, t.cleaned ? 'Cleaned' : null].filter(Boolean).join(' · ') || null,108 country: p.country,109 identifiers: { apmex_sku: c.id },110 metadata: { mint_mark: t.mintMark, designation: t.designation, random_year: t.random, badge: c.badge },111 });112 const grade = t.grader && t.grade ? { grader: t.grader, grade: t.grade, qualifier: t.designation, certificationNumber: null } : { grader: null, grade: t.grade, qualifier: null, certificationNumber: null };113 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };114 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.85, releaseDate: null }));115 if (c.price) {116 const availability = /out of stock|sold out|notify/i.test(c.message ?? '') ? 'sold' : 'available';117 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'USD', seller: 'APMEX', location: 'US', condition: { condition: t.grade ? null : /\bBU\b|brilliant uncirculated/i.test(c.title) ? 'mint_state' : null, conditionRaw: c.title.match(/\b(BU|Brilliant Uncirculated|Cull|Cleaned|Uncirculated|Proof)\b/i)?.[1] ?? null, completeness: null }, availability }));118 }119 }120 return out;121 }122}123124export default function createConnector(meta: ConnectorMeta) {125 return new ApmexConnector(meta);126}127