TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedPopulationReportSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../../api/_lib/shared.js';5import { BOT_HEADERS, cardCategorySlug, dayOf, markdownTables, mdLink, mdText, parseSitemap, toInt } from '../../api/_lib/wave4.js';67/** TAG Grading population report — set pages rendered by Firecrawl, parsed from the markdown table. */8const PARSER_VERSION = '1.0.0';9const SITE = 'https://my.taggrading.com';1011const RowSchema = z.object({ number: z.string().nullable(), name: z.string(), variation: z.string().nullable(), url: z.string().nullable(), counts: z.record(z.string(), z.number()), total: z.number().nullable() });12const RawPayloadSchema = z.object({ category: z.string(), year: z.string().nullable(), company: z.string().nullable(), setName: z.string().nullable(), grades: z.array(z.string()), rows: z.array(RowSchema) });13export type TagPopPayload = z.infer<typeof RawPayloadSchema>;1415/** Parse the set-page markdown into rows. Exported for tests. */16export function parseSetMarkdown(md: string): { grades: string[]; rows: z.infer<typeof RowSchema>[] } {17 const tables = markdownTables(md);18 for (const t of tables) {19 const headerIdx = t.findIndex((r) => r[0]?.replace(/\s/g, '').toLowerCase() === 'card#');20 if (headerIdx < 0) continue;21 const header = t[headerIdx]!;22 const gradeCols: Array<{ idx: number; label: string }> = [];23 let totalIdx = -1;24 header.forEach((h, i) => {25 const label = mdText(h);26 if (i < 2) return;27 if (/^total$/i.test(label)) totalIdx = i;28 else if (label) gradeCols.push({ idx: i, label: label === 'VA' ? 'authentic' : label });29 });30 const rows: z.infer<typeof RowSchema>[] = [];31 for (const r of t.slice(headerIdx + 1)) {32 if (r.length < 3) continue;33 const number = mdText(r[0] ?? '') || null;34 const nameCell = r[1] ?? '';35 const link = mdLink(nameCell);36 const nameText = mdText(nameCell.replace(/<br\s*\/?>/gi, ' ¦ '));37 const [namePart, ...rest] = nameText.split('¦').map((s) => s.trim());38 const name = namePart ?? nameText;39 if (!name || /^totals?$/i.test(name)) continue;40 const variation = rest.filter(Boolean).join(' ') || null;41 const counts: Record<string, number> = {};42 for (const g of gradeCols) {43 const n = toInt(mdText(r[g.idx] ?? ''));44 if (n !== null && n > 0) counts[g.label] = n;45 }46 const total = totalIdx >= 0 ? toInt(mdText(r[totalIdx] ?? '')) : Object.values(counts).reduce((a, b) => a + b, 0);47 rows.push({ number, name, variation, url: link, counts, total });48 }49 return { grades: gradeCols.map((g) => g.label), rows };50 }51 return { grades: [], rows: [] };52}5354function parseSetUrl(u: string): { category: string; year: string | null; company: string | null; setName: string | null } | null {55 try {56 const url = new URL(u);57 const parts = url.pathname.split('/').filter(Boolean).map((p) => decodeURIComponent(p).trim());58 if (parts[0] !== 'pop-report' || parts.length < 4) return null;59 return { category: parts[1]!, year: parts[2] ?? null, company: parts[3] ?? null, setName: url.searchParams.get('setName') };60 } catch {61 return null;62 }63}6465export class TagPopConnector extends BaseConnector {66 readonly version = '1.0.0';67 readonly parserVersion = PARSER_VERSION;68 protected override minIntervalMs = 1500;6970 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {71 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined) ?? ['Pokemon']).map((s) => s.toLowerCase());72 const maxPages = ctx.options.mode === 'backfill' ? Infinity : Number(this.meta.config.maxPagesPerRun ?? 40);73 let urls: string[] = [];74 if (ctx.options.seeds?.some((s) => s.startsWith('http'))) urls = ctx.options.seeds.filter((s) => s.startsWith('http'));75 else {76 const sm = await ctx.fetch(`${SITE}/pop.xml`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml' }, responseType: 'text', timeoutMs: 120_000 });77 if (!sm.success || !sm.html) throw new Error(`tag pop sitemap failed: ${sm.error ?? sm.httpStatus}`);78 urls = parseSitemap(sm.html)79 .map((e) => e.loc)80 .filter((u) => u.includes('setName=') && !/\t|%09/.test(u))81 .filter((u) => {82 const p = parseSetUrl(u);83 return p && seeds.some((s) => p.category.toLowerCase() === s || p.category.toLowerCase().startsWith(s));84 });85 }86 let idx = Number(ctx.options.cursor?.idx ?? 0);87 if (idx >= urls.length) idx = 0;88 let count = 0;89 let pages = 0;90 for (; idx < urls.length && pages < maxPages; idx++) {91 if (ctx.signal?.aborted) return;92 if (this.reached(ctx, count)) break;93 const u = urls[idx]!;94 const info = parseSetUrl(u);95 if (!info) continue;96 await this.throttle();97 const res = await ctx.fetch(u, { engines: ['firecrawl'], waitForMs: 8000, timeoutMs: 90_000, expect: ['title'], parse: (r) => ({ title: r.markdown && /Card #/i.test(r.markdown) ? 'ok' : null }) });98 pages++;99 if (!res.success || !res.markdown) {100 ctx.anomaly('page_fetch_failed', `${u}: ${res.error ?? res.httpStatus}`);101 continue;102 }103 const { grades, rows } = parseSetMarkdown(res.markdown);104 if (!rows.length) {105 ctx.anomaly('parse_failure_table', u);106 continue;107 }108 count++;109 const payload: TagPopPayload = { ...info, grades, rows };110 yield { url: u, externalId: `${info.category}|${info.year ?? ''}|${info.company ?? ''}|${info.setName ?? ''}`, kind: 'population_report', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };111 await ctx.setCursor({ idx: idx + 1, total: urls.length });112 }113 if (idx >= urls.length) await ctx.setCursor({ idx: 0, total: urls.length, completedAt: new Date().toISOString() });114 }115116 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {117 const p = RawPayloadSchema.parse(raw.payload);118 const categorySlug = cardCategorySlug(p.category);119 if (!categorySlug) return [];120 const year = p.year && /^\d{4}$/.test(p.year) ? Number(p.year) : null;121 const reportDate = dayOf(raw.fetchedAt);122 const isPokemon = categorySlug === 'pokemon';123 const out: NormalizedRecord[] = [];124 for (const r of p.rows) {125 const total = r.total ?? Object.values(r.counts).reduce((a, b) => a + b, 0);126 if (!total) continue;127 const number = r.number ? r.number.split('/')[0]!.trim() : null;128 const totalInSet = r.number?.includes('/') ? toInt(r.number.split('/')[1]!) : null;129 const a = attrs({130 categorySlug,131 franchise: isPokemon ? 'Pokémon' : null,132 brand: isPokemon ? 'The Pokémon Company' : p.company,133 set: p.setName,134 name: r.name,135 number,136 year,137 variant: r.variation,138 language: isPokemon && /japanese/i.test(p.company ?? '') ? 'Japanese' : isPokemon ? 'English' : null,139 identifiers: {},140 metadata: { total_in_set: totalInSet, company: p.company },141 });142 out.push(NormalizedPopulationReportSchema.parse({ kind: 'population_report', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: r.url ?? raw.url, grader: 'tag', attributes: a, reportDate, total, byGrade: r.counts, parserVersion: PARSER_VERSION, confidence: 0.95 }));143 }144 return out;145 }146}147148export default (meta: ConnectorMeta) => new TagPopConnector(meta);149