TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { gunzipSync } from 'node:zlib';2import { z } from 'zod';3import { adapters, BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';4import type { NormalizedRecord } from '@rareindex/shared';5import { amount, cardHouseCategory, certFromTitle, isBundleTitle, isCurrency, isoDate, jsonAfterKey, lotAttributes, makeSale, nextFlightText, safeYear, saleGrade } from '../_g7-auctions-na-lib/index.js';67const SITE = 'https://www.fanaticscollect.com';8const HOUSE = 'Fanatics Collect';9const PARSER_VERSION = '1.0.0';1011const MoneySchema = z.object({ amountInCents: z.number().nullable().optional(), currency: z.string().nullable().optional() }).nullable().optional();12export const ListingSchema = z.object({13 id: z.string(),14 title: z.string(),15 listingType: z.string().nullable().optional(),16 status: z.string().nullable().optional(),17 bidCount: z.number().nullable().optional(),18 currentBid: MoneySchema,19 startingPrice: MoneySchema,20 collectSales: z.array(z.object({ soldDate: z.string().nullable().optional(), soldFor: MoneySchema })).default([]),21 auction: z.object({ id: z.string().nullable().optional(), name: z.string().nullable().optional(), shortName: z.string().nullable().optional(), startsAt: z.string().nullable().optional(), endsAt: z.string().nullable().optional(), status: z.string().nullable().optional() }).nullable().optional(),22 lotString: z.string().nullable().optional(),23 slug: z.string().nullable().optional(),24 description: z.string().nullable().optional(),25 integerId: z.number().nullable().optional(),26 insertedAt: z.string().nullable().optional(),27 updatedAt: z.string().nullable().optional(),28 vaultItem: z.object({ id: z.string().nullable().optional(), integerId: z.number().nullable().optional() }).nullable().optional(),29 images: z.array(z.string()).default([]),30});31export type Listing = z.infer<typeof ListingSchema>;32export const PayloadSchema = z.object({ kind: z.literal('fc_item'), url: z.string(), lastmod: z.string().nullable(), listing: ListingSchema });33export type Payload = z.infer<typeof PayloadSchema>;3435type RawListing = Record<string, unknown> & { imageSets?: Array<{ large?: string | null; medium?: string | null; small?: string | null }> | null; description?: string | null; vaultItem?: Record<string, unknown> | null; auction?: Record<string, unknown> | null };3637/** Item page HTML → trimmed CollectListing (RSC flight payload, JSON-LD Product as fallback for title/images). */38export function parseItemPage(html: string, url: string, lastmod: string | null = null): Payload | null {39 const text = nextFlightText(html);40 const raw = jsonAfterKey<RawListing>(text, '"prefetchedItemData":');41 const ld = H.jsonLd(html, 'Product')[0];42 if (!raw && !ld) return null;43 const images: string[] = [];44 for (const s of raw?.imageSets ?? []) {45 const u = s?.large ?? s?.medium ?? s?.small;46 if (u && !images.includes(u)) images.push(u);47 }48 if (!images.length && ld?.image) for (const u of Array.isArray(ld.image) ? ld.image : [ld.image]) if (typeof u === 'string') images.push(u);49 const pick = (o: Record<string, unknown> | null | undefined, keys: string[]) => (o ? Object.fromEntries(keys.filter((k) => k in o).map((k) => [k, o[k]])) : o ?? null);50 const listing = {51 ...pick(raw ?? {}, ['id', 'title', 'listingType', 'status', 'bidCount', 'currentBid', 'startingPrice', 'collectSales', 'lotString', 'slug', 'integerId', 'insertedAt', 'updatedAt']),52 id: (raw?.id as string | undefined) ?? (typeof ld?.sku === 'string' ? ld.sku : url.match(/\/(?:weekly|fixed|premier)\/([0-9a-f-]{36})/i)?.[1]),53 title: (raw?.title as string | undefined) ?? (typeof ld?.name === 'string' ? ld.name : undefined),54 auction: pick(raw?.auction ?? null, ['id', 'name', 'shortName', 'startsAt', 'endsAt', 'status']),55 vaultItem: pick(raw?.vaultItem ?? null, ['id', 'integerId']),56 description: typeof raw?.description === 'string' ? raw.description.slice(0, 2000) : null,57 images: images.slice(0, 6),58 };59 const parsed = ListingSchema.safeParse(listing);60 if (!parsed.success) return null;61 return { kind: 'fc_item', url, lastmod, listing: parsed.data };62}6364/** Sales-history children from the sitemap index, newest first (higher N = newer; fixed-price after weekly of the same N). */65export function salesHistorySitemaps(indexXml: string): string[] {66 const locs = adapters.parseSitemapIndex(indexXml).map((e) => e.loc).filter((l) => /sales-history/.test(l));67 const n = (l: string) => Number(l.match(/-(\d+)\.xml/)?.[1] ?? 0);68 const kind = (l: string) => (/fixed-price/.test(l) ? 1 : 0);69 return locs.sort((a, b) => n(b) - n(a) || kind(b) - kind(a));70}7172function gunzipMaybe(buf: Uint8Array | null | undefined, text: string | null | undefined): string | null {73 if (buf && buf.byteLength) {74 const b = Buffer.from(buf);75 return (b.length >= 2 && b[0] === 0x1f && b[1] === 0x8b ? gunzipSync(b) : b).toString('utf8');76 }77 return text ?? null;78}7980/**81 * Fanatics Collect (formerly PWCC) — sold results of the weekly/premier auctions and fixed-price sales.82 * Discovery through the public sitemap index (sales-history-*.xml.gz children with lastmod); each public83 * item page embeds the listing (incl. collectSales) in its Next.js RSC payload. See meta.json accessNotes.84 */85export class FanaticsCollectConnector extends BaseConnector {86 readonly version = '1.0.0';87 readonly parserVersion = PARSER_VERSION;88 override readonly urlPatterns = [/^https?:\/\/(www\.)?fanaticscollect\.com\/(weekly|fixed|premier)\/[0-9a-f-]{36}/i];89 protected override minIntervalMs = 1500;9091 private async fetchText(ctx: CrawlContext, url: string, binary = false): Promise<{ text: string | null; status: number | null; fetchedAt: Date }> {92 await this.throttle(url);93 const res = await ctx.fetch(url, { engines: ['api'], responseType: binary ? 'binary' : 'text', minQuality: 0, force: binary, timeoutMs: 45_000 });94 if (!res.success) {95 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);96 return { text: null, status: res.httpStatus, fetchedAt: res.fetchedAt };97 }98 return { text: gunzipMaybe(res.buffer, res.html), status: res.httpStatus, fetchedAt: res.fetchedAt };99 }100101 private async childEntries(ctx: CrawlContext, url: string): Promise<Array<{ loc: string; lastmod: string | null }>> {102 const r = await this.fetchText(ctx, url, true);103 if (!r.text) return [];104 const entries = adapters.parseUrlset(r.text).map((e) => ({ loc: e.loc, lastmod: e.lastmod }));105 if (!entries.length) ctx.anomaly('pagination_failure', `${url}: empty sitemap`);106 return entries;107 }108109 private async itemRecord(ctx: CrawlContext, url: string, lastmod: string | null): Promise<RawRecordInput | null> {110 const r = await this.fetchText(ctx, url);111 if (!r.text) return null;112 const payload = parseItemPage(r.text, url, lastmod);113 if (!payload) {114 ctx.anomaly('parse_failure_page', `${url}: no prefetchedItemData / Product JSON-LD`);115 return null;116 }117 return { url, externalId: payload.listing.id, kind: 'sale', engine: 'api', httpStatus: r.status, payload, fetchedAt: r.fetchedAt };118 }119120 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {121 const mode = ctx.options.mode;122 const cfg = this.meta.config;123 const pagesPerRun = mode === 'probe' ? Number(cfg.probePages ?? 3) : Number(cfg.pagesPerRun ?? 300);124 let fetched = 0;125 let noSale = 0;126 let count = 0;127 const track = (rec: RawRecordInput | null) => {128 fetched++;129 if (rec && (rec.payload as Payload).listing.collectSales.length === 0) noSale++;130 };131 if (ctx.options.seeds?.length) {132 for (const url of ctx.options.seeds) {133 if (ctx.signal?.aborted || this.reached(ctx, count)) break;134 const rec = await this.itemRecord(ctx, url, null);135 track(rec);136 if (rec) {137 count++;138 yield rec;139 }140 }141 return;142 }143 const idx = await this.fetchText(ctx, `${SITE}/sitemap.xml`);144 if (!idx.text) return;145 const children = salesHistorySitemaps(idx.text);146 if (!children.length) {147 ctx.anomaly('pagination_failure', 'sitemap index has no sales-history children');148 return;149 }150 const cursor = ctx.options.cursor ?? {};151 if (mode === 'backfill') {152 // Oldest → newest, resumable by child url + offset.153 const order = [...children].reverse();154 let startIdx = typeof cursor.sitemap === 'string' ? Math.max(0, order.indexOf(cursor.sitemap)) : 0;155 let offset = startIdx === order.indexOf(cursor.sitemap as string) && typeof cursor.offset === 'number' ? cursor.offset : 0;156 let urlsDone = typeof cursor.urlsDone === 'number' ? cursor.urlsDone : 0;157 let items = typeof cursor.itemsProcessed === 'number' ? cursor.itemsProcessed : 0;158 for (let ci = startIdx; ci < order.length; ci++) {159 if (ctx.signal?.aborted || fetched >= pagesPerRun) break;160 const child = order[ci]!;161 const entries = await this.childEntries(ctx, child);162 for (let i = offset; i < entries.length; i++) {163 if (ctx.signal?.aborted || fetched >= pagesPerRun || this.reached(ctx, count)) {164 await ctx.setCursor({ sitemap: child, offset: i, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() });165 await ctx.progress({ page: urlsDone, totalPages: null, itemsProcessed: items, cursor: { sitemap: child, offset: i, urlsDone, itemsProcessed: items } });166 this.reportNoSale(ctx, noSale, fetched);167 return;168 }169 const e = entries[i]!;170 const rec = await this.itemRecord(ctx, e.loc, e.lastmod);171 track(rec);172 urlsDone++;173 if (rec) {174 items++;175 count++;176 yield rec;177 }178 if (urlsDone % 25 === 0) {179 await ctx.setCursor({ sitemap: child, offset: i + 1, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() });180 await ctx.progress({ page: urlsDone, totalPages: null, itemsProcessed: items });181 }182 }183 offset = 0;184 startIdx = ci + 1;185 await ctx.setCursor({ sitemap: order[ci + 1] ?? child, offset: 0, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() });186 }187 if (startIdx >= order.length) await ctx.setCursor({ done: true, urlsDone, itemsProcessed: items, updatedAt: new Date().toISOString() });188 this.reportNoSale(ctx, noSale, fetched);189 return;190 }191 // Incremental / probe: newest children first, only entries newer than the last fully processed lastmod.192 const since = typeof cursor.since === 'string' ? cursor.since : '';193 let newest = since;194 const maxChildren = mode === 'probe' ? 1 : children.length;195 for (const child of children.slice(0, maxChildren)) {196 if (ctx.signal?.aborted || fetched >= pagesPerRun) break;197 const entries = await this.childEntries(ctx, child);198 const fresh = entries.filter((e) => !since || (e.lastmod ?? '') > since).sort((a, b) => (b.lastmod ?? '').localeCompare(a.lastmod ?? ''));199 if (!fresh.length) break; // whole child older than the cursor → stop descending200 for (const e of fresh) {201 if (ctx.signal?.aborted || fetched >= pagesPerRun || this.reached(ctx, count)) break;202 const rec = await this.itemRecord(ctx, e.loc, e.lastmod);203 track(rec);204 if (e.lastmod && e.lastmod > newest) newest = e.lastmod;205 if (rec) {206 count++;207 yield rec;208 }209 }210 }211 if (mode !== 'probe' && newest && newest !== since) await ctx.setCursor({ since: newest, updatedAt: new Date().toISOString() });212 this.reportNoSale(ctx, noSale, fetched);213 }214215 private reportNoSale(ctx: CrawlContext, noSale: number, fetched: number): void {216 if (fetched > 0 && noSale / fetched > 0.3) ctx.anomaly('no_sale_on_page', `${noSale} of ${fetched} item pages carried no collectSales`);217 }218219 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {220 const rec = await this.itemRecord(ctx, url, null);221 return rec ? [rec] : [];222 }223224 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {225 const p = PayloadSchema.parse(raw.payload);226 const l = p.listing;227 const sale = l.collectSales.find((s) => s.soldFor?.amountInCents && s.soldDate) ?? l.collectSales[0];228 if (!sale) return [];229 const price = amount((sale.soldFor?.amountInCents ?? 0) / 100);230 const saleDate = isoDate(sale.soldDate);231 if (!price || !saleDate || saleDate.getTime() > Date.now() + 86_400_000) return [];232 const currency = isCurrency(sale.soldFor?.currency) ? sale.soldFor!.currency! : 'USD';233 const type = (l.listingType ?? '').toUpperCase();234 const isAuction = type === 'WEEKLY' || type === 'PREMIER' || type === 'FLASH' || type === 'AUCTION';235 const hammer = isAuction ? amount((l.currentBid?.amountInCents ?? 0) / 100) : null;236 const g = saleGrade(l.title);237 const cert = certFromTitle(l.title) ?? (l.description ? certFromTitle(l.description) : null);238 const identifiers: Record<string, string> = { fanatics_listing_id: l.id };239 if (l.vaultItem?.id) identifiers.fanatics_vault_item_id = l.vaultItem.id;240 const attributes = lotAttributes({241 categorySlug: cardHouseCategory(l.title),242 name: l.title,243 year: safeYear(l.title),244 identifiers,245 metadata: { auction_name: l.auction?.name ?? null, auction_short_name: l.auction?.shortName ?? null, auction_ends_at: l.auction?.endsAt ?? null, listing_type: l.listingType ?? null, bid_count: l.bidCount ?? null, fanatics_status: l.status ?? null, hammer_price: hammer, buyer_premium_pct: isAuction && hammer && price > hammer ? Math.round(((price / hammer) - 1) * 1000) / 10 : null, sitemap_lastmod: p.lastmod },246 });247 const record = makeSale({248 meta: this.meta,249 sourceUrl: p.url,250 externalId: l.id,251 rawTitle: l.title,252 description: l.description ?? null,253 attributes,254 price,255 currency,256 saleDate,257 buyerPremiumIncluded: isAuction ? true : null,258 auctionHouse: HOUSE,259 lotNumber: l.lotString?.match(/Lot:?\s*([A-Za-z0-9-]+)/i)?.[1] ?? null,260 imageUrls: l.images,261 location: 'US',262 observedAt: raw.fetchedAt,263 parserVersion: PARSER_VERSION,264 grader: g.grader,265 grade: g.grade,266 isBundle: isBundleTitle(l.title),267 saleType: isAuction ? 'auction' : 'fixed_price',268 confidence: g.grader ? 0.9 : 0.85,269 });270 record.grade.qualifier = g.qualifier;271 record.grade.certificationNumber = cert;272 return [record];273 }274}275276export default (meta: ConnectorMeta) => new FanaticsCollectConnector(meta);277