TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Shared base for grading-company certificate-verification connectors (SPEC §3 "Grading/certification3 * data", §22 cert tracking; group g2-grading).4 *5 * A cert connector verifies ONE public certification page per cert number: it never enumerates the6 * grader's cert space. Certs come from `ctx.options.seeds` (bare numbers or public verify URLs) or from7 * `meta.config.certs`; `lookup(url)` resolves a single verify URL (the worker calls8 * `lookup(certUrl(cert))` for cert numbers seen on marketplaces). Each found cert yields one raw record9 * whose payload is a trimmed snapshot of the public page; `normalize` turns it into a `catalog_item`10 * (the graded object + grade + cert identifiers) and, when the page shows it, a `population_report`.11 */12import { z } from 'zod';13import { BaseConnector, type ConnectorMeta, type CrawlContext, type FetchOptions, type QualityField, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';14import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPopulationReportSchema, type ExtractionResult, type NormalizedRecord } from '@rareindex/shared';15import { dayOf } from '../../api/_lib/wave4.js';1617export const CertPayloadSchema = z.object({18 grader: z.string(),19 cert: z.string(),20 /** the public verify URL that was fetched */21 url: z.string(),22 format: z.enum(['html', 'markdown']),23 status: z.enum(['found', 'not_found']),24 /** trimmed page fragment (HTML or markdown) — everything `normalize` needs */25 snapshot: z.string(),26});27export type CertPayload = z.infer<typeof CertPayloadSchema>;2829export type AttrInput = z.input<typeof AssetAttributesSchema>;3031/** What a connector's page parser returns for a found certificate. */32export interface ParsedCert {33 /** the grader's headline description, kept verbatim as rawTitle */34 title: string;35 attributes: AttrInput;36 grade: string | null;37 qualifier: string | null;38 /** grader's printed grade label (e.g. "GEM MT 10", "MS 61", "67 EPQ") */39 gradeLabel: string | null;40 description?: string | null;41 images?: string[];42 /**43 * Population shown on the page for this exact grade, plus either the count in higher grades44 * (PSA/CGC/PCGS/NGC/PMG) or the total graded across all grades (TAG). Partial by nature: the45 * report keeps `population_scope` in metadata so consumers never mistake it for a full distribution.46 */47 population?: { gradeKey: string; atGrade: number | null; higher: number | null; total?: number | null; url: string | null; asOf?: Date | null } | null;48 confidence: number;49}5051export type PageStatus = 'found' | 'not_found' | 'unknown';5253export const CertConfigSchema = z.object({54 /** taxonomy grader slug this connector verifies (coordinator convention) */55 grader: z.string(),56 /** bounded list of cert numbers (or verify URLs) verified on every scheduled run */57 certs: z.array(z.string()).default([]),58 /** hard cap of pages per run */59 maxPerRun: z.number().int().positive().default(50),60});6162export abstract class CertLookupConnector extends BaseConnector {63 /** taxonomy grader slug (data/taxonomy/graders.json) */64 abstract readonly grader: string;65 /** identifiers key carrying the cert number (e.g. psa_cert) */66 abstract readonly idKey: string;67 /** which engine output the parser consumes */68 abstract readonly format: 'html' | 'markdown';69 /** Firecrawl wait for client-rendered pages */70 protected waitForMs = 3000;71 protected override minIntervalMs = 4000;72 /** URL patterns understood by `lookup` (subclasses assign their own list). */73 abstract override readonly urlPatterns: RegExp[];7475 /** Canonical public verify URL for a cert number (the worker builds these). */76 abstract certUrl(cert: string): string;77 /** Extract the cert number from any public URL variant; null when the URL is not a cert page. */78 abstract certFromUrl(url: string): string | null;79 /** found = the page shows a certificate; not_found = the grader says the cert does not exist; unknown = unexpected page. */80 abstract classify(doc: string): PageStatus;81 /** Trim the full document to the fragment `parse` needs (fixtures store this). */82 abstract trim(doc: string): string;83 /** Pure parser over the trimmed snapshot. Null when the fragment carries no certificate. */84 abstract parse(snapshot: string, info: { cert: string; url: string }): ParsedCert | null;8586 protected get config() {87 return CertConfigSchema.parse({ grader: this.grader, ...this.meta.config });88 }8990 /** The value stored in identifiers[idKey] / grade.certificationNumber (subclasses strip URL-only parts such as NGC's grade segment). */91 certIdentifier(cert: string): string {92 return cert;93 }9495 /** Normalise a seed (bare cert number or URL) to a cert number. */96 protected seedToCert(seed: string): string | null {97 const s = seed.trim();98 if (!s) return null;99 if (/^https?:\/\//i.test(s)) return this.certFromUrl(s);100 return s;101 }102103 protected docOf(res: ExtractionResult): string | null {104 return this.format === 'markdown' ? (res.markdown ?? null) : (res.html ?? res.markdown ?? null);105 }106107 protected fetchOptions(): Partial<FetchOptions> {108 return {};109 }110111 /**112 * Fetch and classify one certificate page. Returns a raw record for a found cert; for a not-found113 * cert returns null (or a `status: not_found` record when `includeNotFound`, used to build fixtures).114 */115 async fetchCert(ctx: CrawlContext, cert: string, opts: { includeNotFound?: boolean } = {}): Promise<RawRecordInput | null> {116 const url = this.certUrl(cert);117 await this.throttle(url);118 const expect: QualityField[] = ['title', 'identifiers', 'status'];119 const res = await ctx.fetch(url, {120 engines: this.meta.enginePriority,121 waitForMs: this.waitForMs,122 timeoutMs: 90_000,123 minQuality: 0.3,124 expect,125 parse: (r) => {126 const doc = this.docOf(r);127 if (!doc) return null;128 const status = this.classify(doc);129 if (status === 'not_found') return { title: 'not found', status: 'not_found' };130 if (status === 'unknown') return null;131 const parsed = this.parse(this.trim(doc), { cert, url });132 return parsed ? { title: parsed.title, identifiers: cert, status: 'found' } : null;133 },134 ...this.fetchOptions(),135 });136 const gone = res.httpStatus === 404 || res.httpStatus === 410;137 if (!res.success && !gone) {138 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);139 return null;140 }141 const doc = this.docOf(res);142 // A 404/410 from the grader is a definitive answer (PSA answers unknown certs with 404), never a host failure.143 const status: PageStatus = gone ? 'not_found' : doc ? this.classify(doc) : 'unknown';144 if (status === 'unknown') {145 ctx.anomaly('selector_missing', `${url}: page is neither a certificate nor a not-found response`);146 return null;147 }148 const snapshot = doc && !gone ? this.trim(doc) : doc ? `<div class="g2-not-found" data-http-status="${res.httpStatus}">${this.trim(doc).slice(0, 4000)}</div>` : '';149 if (status === 'not_found') {150 ctx.log.info({ cert, url }, 'cert not found at grader');151 if (!opts.includeNotFound) return null;152 }153 const payload: CertPayload = { grader: this.grader, cert, url, format: this.format, status, snapshot };154 return { url, externalId: `${this.grader}:${cert}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };155 }156157 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {158 const cfg = this.config;159 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : cfg.certs;160 const certs = [...new Set(seeds.map((s) => this.seedToCert(s)).filter((c): c is string => Boolean(c)))];161 if (!certs.length) {162 ctx.log.info({ connector: this.meta.id }, 'no certs to verify (pass seeds or config.certs)');163 return;164 }165 let idx = Number(ctx.options.cursor?.idx ?? 0);166 if (!Number.isFinite(idx) || idx < 0 || idx >= certs.length) idx = 0;167 let count = 0;168 let pages = 0;169 for (; idx < certs.length && pages < cfg.maxPerRun; idx++) {170 if (ctx.signal?.aborted) return;171 if (this.reached(ctx, count)) break;172 const cert = certs[idx]!;173 pages++;174 const raw = await this.fetchCert(ctx, cert);175 if (raw) {176 count++;177 yield raw;178 }179 await ctx.setCursor({ idx: idx + 1, total: certs.length });180 if (ctx.options.mode === 'backfill') await ctx.progress({ page: idx + 1, totalPages: certs.length, itemsProcessed: count });181 }182 if (idx >= certs.length) await ctx.setCursor({ idx: 0, total: certs.length, completedAt: new Date().toISOString(), ...(ctx.options.mode === 'backfill' ? { done: true } : {}) });183 }184185 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {186 const cert = this.certFromUrl(url);187 if (!cert) return [];188 const raw = await this.fetchCert(ctx, cert);189 return raw ? [raw] : [];190 }191192 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {193 const p = CertPayloadSchema.parse(raw.payload);194 if (p.status === 'not_found') return [];195 const parsed = this.parse(p.snapshot, { cert: p.cert, url: p.url });196 if (!parsed) return [];197 const certId = this.certIdentifier(p.cert);198 const identifiers = { ...(parsed.attributes.identifiers ?? {}), [this.idKey]: certId };199 const attributes = AssetAttributesSchema.parse({ ...parsed.attributes, identifiers });200 const out: NormalizedRecord[] = [];201 out.push(202 NormalizedCatalogItemSchema.parse({203 kind: 'catalog_item',204 connectorId: this.meta.id,205 sourceId: this.meta.sourceId,206 sourceUrl: p.url,207 externalId: `${this.grader}:${certId}`,208 rawTitle: parsed.title,209 description: parsed.description ?? null,210 imageUrls: parsed.images ?? [],211 attributes,212 grade: { grader: this.grader, grade: parsed.grade, qualifier: parsed.qualifier, certificationNumber: certId },213 condition: {},214 observedAt: raw.fetchedAt,215 confidence: parsed.confidence,216 parserVersion: this.parserVersion,217 releaseDate: null,218 }),219 );220 const pop = parsed.population;221 if (pop && pop.atGrade !== null) {222 const byGrade: Record<string, number> = { [pop.gradeKey]: pop.atGrade };223 if (pop.higher !== null) byGrade.higher = pop.higher;224 const hasTotal = pop.total !== undefined && pop.total !== null && pop.total >= pop.atGrade;225 const total = hasTotal ? (pop.total as number) : pop.atGrade + (pop.higher ?? 0);226 if (hasTotal && total > pop.atGrade) byGrade.other = total - pop.atGrade;227 out.push(228 NormalizedPopulationReportSchema.parse({229 kind: 'population_report',230 connectorId: this.meta.id,231 sourceId: this.meta.sourceId,232 sourceUrl: pop.url ?? p.url,233 grader: this.grader,234 attributes: AssetAttributesSchema.parse({ ...attributes, metadata: { ...attributes.metadata, population_scope: hasTotal ? 'grade_and_total' : 'grade_and_higher', grade_label: parsed.gradeLabel } }),235 reportDate: pop.asOf ?? dayOf(raw.fetchedAt),236 total,237 byGrade,238 parserVersion: this.parserVersion,239 confidence: Math.min(parsed.confidence, 0.9),240 }),241 );242 }243 return out;244 }245}246247/** Shared meta factory signature used by every cert connector module. */248export type CertConnectorFactory = (meta: ConnectorMeta) => CertLookupConnector;249