import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; const PARSER_VERSION = '1.0.0'; export const CardSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), price: z.number().nullable(), message: z.string().nullable(), image: z.string().nullable(), badge: z.string().nullable(), }); export type Card = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), series: z.string().nullable(), country: z.string().nullable(), items: z.array(CardSchema) }); export type PagePayload = z.infer; export function parseCategoryPage(htmlText: string, pageUrl: string, series: string | null, country: string | null): PagePayload { const $ = H.load(htmlText); const items: Card[] = []; const seen = new Set(); $('.mod-product-card').each((_, el) => { const e = $(el); const a = e.find('a.item-link').first(); const url = a.attr('href'); const id = a.attr('data-product-id') ?? url?.match(/\/product\/(\d+)\//)?.[1]; if (!url || !id || seen.has(id)) return; seen.add(id); const title = (a.attr('data-product-name') ?? H.text(e.find('.mod-product-title').first()) ?? '').replace(/\s+/g, ' ').trim(); if (!title) return; const price = parsePrice(H.text(e.find('.mod-product-pricing .price').first()), 'USD')?.amount ?? null; const message = H.text(e.find('.mod-product-message').first()); const image = e.find('.mod-product-img img').attr('src') ?? null; const badge = H.text(e.find('.product-badge').first()); items.push({ id, url, title, price, message, image, badge }); }); return { kind: 'category_page', url: pageUrl, series, country, items }; } /** "1887-S $10 Liberty Gold Eagle MS-63 NGC" → parts. Unknown fields stay null. */ export function parseCoinTitle(title: string): { year: number | null; mintMark: string | null; grader: string | null; grade: string | null; designation: string | null; cleaned: boolean; random: boolean } { const y = title.match(/\b(1[6-9]\d{2}|20\d{2})(?:-([A-Z]{1,2}))?\b/); const g = title.match(/\b(MS|AU|XF|EF|VF|F|VG|G|AG|PR|PF|PL|SP|BU)-?(\d{1,2})(\+?)(?!\d)/i); const grader = title.match(/\b(PCGS|NGC|ANACS|ICG|CAC)\b/i)?.[1]?.toLowerCase() ?? null; const designation = title.match(/\b(CAC|DMPL|PL|FBL|FB|RD|RB|BN|Cameo|Ultra Cameo|DCAM|First Strike|Early Releases)\b/i)?.[1] ?? null; return { year: y ? Number(y[1]) : null, mintMark: y?.[2] ?? null, grader, grade: g ? `${g[1]!.toUpperCase()}${g[2]}${g[3] ?? ''}` : null, designation, cleaned: /\(cleaned\)|cleaned/i.test(title), random: /random/i.test(title), }; } export class ApmexConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2500; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (this.meta.config.seeds as Array<{ url: string; series?: string; country?: string }> | undefined) ?? []; const pages = Number(this.meta.config.pagesPerSeed ?? 1); const cap = ctx.options.limit; let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0; for (let i = start; i < seeds.length; i++) { const seed = seeds[i]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return; const url = page > 1 ? `${seed.url}?page=${page}` : seed.url; await this.throttle(); const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], expect: ['title', 'price'], parse: (r) => { const p = r.html ? parseCategoryPage(r.html, url, seed.series ?? null, seed.country ?? null) : null; return p?.items.length ? { title: 'ok', price: p.items.some((x) => x.price) ? 1 : null } : null; }, }); const payload = res.success && res.html ? parseCategoryPage(res.html, url, seed.series ?? null, seed.country ?? null) : null; if (!payload?.items.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product cards'}`); break; } count++; yield { url, externalId: `${seed.url}|p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const c of p.items) { const t = parseCoinTitle(c.title); const name = c.title.replace(/\s*\(cleaned\)/i, '').replace(/\b(PCGS|NGC|ANACS|ICG)\b/gi, '').replace(/\b(MS|AU|XF|EF|VF|F|VG|G|PR|PF|BU)-?\d{1,2}\+?\b/gi, '').replace(/\s+/g, ' ').trim(); const attributes = AssetAttributesSchema.parse({ categorySlug: 'coins', series: p.series, name, year: t.year, variant: [t.mintMark ? `${t.mintMark} mint` : null, t.designation, t.cleaned ? 'Cleaned' : null].filter(Boolean).join(' · ') || null, country: p.country, identifiers: { apmex_sku: c.id }, metadata: { mint_mark: t.mintMark, designation: t.designation, random_year: t.random, badge: c.badge }, }); const grade = t.grader && t.grade ? { grader: t.grader, grade: t.grade, qualifier: t.designation, certificationNumber: null } : { grader: null, grade: t.grade, qualifier: null, certificationNumber: null }; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, rawTitle: c.title, imageUrls: c.image ? [c.image] : [], attributes, grade, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: c.id, confidence: 0.85, releaseDate: null })); if (c.price) { const availability = /out of stock|sold out|notify/i.test(c.message ?? '') ? 'sold' : 'available'; out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: c.id, confidence: 0.85, listingType: 'fixed_price', price: c.price, currency: 'USD', seller: 'APMEX', location: 'US', condition: { condition: t.grade ? null : /\bBU\b|brilliant uncirculated/i.test(c.title) ? 'mint_state' : null, conditionRaw: c.title.match(/\b(BU|Brilliant Uncirculated|Cull|Cleaned|Uncirculated|Proof)\b/i)?.[1] ?? null, completeness: null }, availability })); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new ApmexConnector(meta); }