TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parseGradeFromTitle } from '@rareindex/taxonomy';4import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';5import { dateMDY, lotAttributes, makeSale, md, money, splitMarkdownItems } from '../_carlib/index.js';67const BASE = 'https://pbagalleries.com';8const PARSER_VERSION = '1.0.0';910export const AuctionSchema = z.object({ catalogId: z.string(), saleNumber: z.string().nullable(), title: z.string(), dateText: z.string().nullable(), lots: z.number().nullable(), closed: z.boolean() });11export const LotSchema = z.object({12 lotNumber: z.string().nullable(),13 lotId: z.string(),14 title: z.string(),15 url: z.string(),16 soldText: z.string().nullable(),17 status: z.string().nullable(),18 estimateText: z.string().nullable(),19 fields: z.record(z.string(), z.string()),20 image: z.string().nullable(),21});22export const CatalogPayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });23export type CatalogPayload = z.infer<typeof CatalogPayloadSchema>;2425export function parseAuctionList(markdown: string): z.infer<typeof AuctionSchema>[] {26 const out: z.infer<typeof AuctionSchema>[] = [];27 const chunks = splitMarkdownItems(markdown, /^- \[!\[/m);28 for (const c of chunks) {29 const info = c.match(/\((https:\/\/pbagalleries\.com\/auctions\/info\/id\/(\d+))\)/);30 if (!info) continue;31 const catalogId = info[2]!;32 const head = c.match(/######\s*\[(?:(\d+)\s+)?([^\]]+)\]/);33 const title = md.clean(head?.[2] ?? '');34 if (!title) continue;35 const dateText = c.match(/\[(\d{2}\/\d{2}\/\d{4})[^\]]*\]/)?.[1] ?? null;36 const lots = c.match(/Lots:\s*(\d+)/)?.[1];37 const closed = /Sale closed/i.test(c);38 if (!out.some((a) => a.catalogId === catalogId)) out.push({ catalogId, saleNumber: head?.[1] ?? null, title, dateText, lots: lots ? Number(lots) : null, closed });39 }40 return out;41}4243export function parseCatalogPage(markdown: string, auction: z.infer<typeof AuctionSchema>, page: number): CatalogPayload {44 const chunks = splitMarkdownItems(markdown, /^- \[!\[/m);45 const lots: z.infer<typeof LotSchema>[] = [];46 for (const c of chunks) {47 const url = c.match(/\((https:\/\/pbagalleries\.com\/lot-details\/index\/catalog\/\d+\/lot\/(\d+)\/[^)?\s]+)/);48 if (!url) continue;49 const title = c.match(/##\s*\[([^\]]+)\]/)?.[1];50 if (!title) continue;51 const fields: Record<string, string> = {};52 const fre = /^\s*-\s+([A-Z][A-Za-z /]+)\n\n\s+(.+)$/gm;53 let fm: RegExpExecArray | null;54 while ((fm = fre.exec(c))) fields[fm[1]!.trim()] = md.clean(fm[2]!);55 lots.push({56 lotNumber: c.match(/\[Lot #(\d+)\]/)?.[1] ?? null,57 lotId: url[2]!,58 title: md.clean(title),59 url: url[1]!,60 soldText: c.match(/Sold for\s*(\$[\d,]+(?:\.\d+)?)/)?.[1] ?? null,61 status: c.match(/Status\s*([A-Za-z ]+)/)?.[1]?.trim() ?? null,62 estimateText: md.clean(c.match(/Estimate\s*(\$[^\n]+)/)?.[1] ?? '') || null,63 fields,64 image: md.image(c),65 });66 }67 return { kind: 'catalog_page', auction, page, lots };68}6970export function pbaCategory(saleTitle: string, lot: { title: string; fields: Record<string, string> }): string {71 const s = `${saleTitle}`.toLowerCase();72 const t = `${lot.title} ${Object.values(lot.fields).join(' ')}`.toLowerCase();73 if (/comic|pre-code|ec,|mad,|graphic novel/.test(s) || /cgc|cbcs|no\. \d+ \*|comic/.test(t)) {74 const pub = (lot.fields.Publisher ?? '').toLowerCase();75 if (/marvel|timely|atlas/.test(pub)) return 'marvel_comics';76 if (/\bdc\b|national|vertigo/.test(pub)) return 'dc_comics';77 return 'independent_comics';78 }79 if (/photograph/.test(s) && !/book/.test(t)) return 'photography';80 if (/map|atlas|cartograph/.test(s)) return 'maps';81 if (/poster/.test(s) || /poster/.test(t)) return 'movie_posters';82 if (/\b(autograph letter|letter signed|typed letter|manuscript (?:leaf|page|document)|signed document|archive of|telegram|deed|land grant)\b/.test(t)) return 'historical_documents';83 if (/fine art|print|painting/.test(s) && !/book/.test(t)) return 'art';84 return 'books';85}8687export class PbaGalleriesConnector extends BaseConnector {88 readonly version = '1.0.0';89 readonly parserVersion = PARSER_VERSION;90 protected override minIntervalMs = 2000;9192 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {93 const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);94 const pagesPerAuction = Number(this.meta.config.catalogPagesPerAuction ?? 5);95 const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneCatalogs) ? (ctx.options.cursor!.doneCatalogs as string[]) : []);96 const listPage = ctx.options.mode === 'backfill' ? Number(ctx.options.cursor?.listPage ?? 1) : 1;97 const listUrl = `${BASE}/auctions/${listPage > 1 ? `?page=${listPage}` : ''}`;98 await this.throttle();99 const list = await ctx.fetch(listUrl, { expect: ['title', 'date'], parse: (r) => (r.markdown ? { title: parseAuctionList(r.markdown)[0]?.title ?? null, date: parseAuctionList(r.markdown)[0]?.dateText ?? null } : null) });100 if (!list.success || !list.markdown) {101 ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);102 return;103 }104 const auctions = parseAuctionList(list.markdown).filter((a) => a.closed && !done.has(a.catalogId));105 let count = 0;106 let processed = 0;107 for (const auction of auctions) {108 if (processed >= auctionsPerRun || ctx.signal?.aborted) break;109 for (let page = 1; page <= pagesPerAuction; page++) {110 if (ctx.signal?.aborted || this.reached(ctx, count)) break;111 const url = `${BASE}/auctions/catalog/id/${auction.catalogId}${page > 1 ? `?page=${page}` : ''}`;112 await this.throttle();113 const res = await ctx.fetch(url, {114 expect: ['title', 'price', 'date', 'status'],115 parse: (r) => {116 const f = r.markdown ? parseCatalogPage(r.markdown, auction, page).lots.find((l) => l.soldText) : null;117 return f ? { title: f.title, price: money(f.soldText, 'USD')?.amount ?? null, date: auction.dateText, status: f.status } : null;118 },119 });120 if (!res.success || !res.markdown) {121 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);122 break;123 }124 const payload = parseCatalogPage(res.markdown, auction, page);125 if (payload.lots.length === 0) break;126 count++;127 yield { url, externalId: `catalog:${auction.catalogId}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };128 if (!res.markdown.includes(`catalog/id/${auction.catalogId}?page=${page + 1}`)) break;129 }130 processed++;131 done.add(auction.catalogId);132 await ctx.setCursor({ doneCatalogs: [...done].slice(-300), listPage: ctx.options.mode === 'backfill' && auctions.every((a) => done.has(a.catalogId)) ? listPage + 1 : listPage, updatedAt: new Date().toISOString() });133 }134 }135136 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {137 const p = CatalogPayloadSchema.parse(raw.payload);138 const saleDate = dateMDY(p.auction.dateText);139 if (!saleDate) return [];140 const out: NormalizedSale[] = [];141 for (const lot of p.lots) {142 if (!lot.soldText || (lot.status && !/sold/i.test(lot.status))) continue;143 const m = money(lot.soldText, 'USD');144 if (!m) continue;145 const categorySlug = pbaCategory(p.auction.title, lot);146 const g = parseGradeFromTitle(lot.title);147 const issue = lot.title.match(/#\s?(\d+[A-Za-z]?)/)?.[1] ?? lot.fields.Title?.match(/No\.\s*(\d+)/)?.[1] ?? null;148 const yearField = Object.entries(lot.fields).find(([k]) => /date|year/i.test(k))?.[1] ?? null;149 const year = yearField?.match(/\b(1[6-9]\d{2}|20\d{2})\b/)?.[1];150 const attributes = lotAttributes({151 categorySlug,152 name: lot.title,153 brand: lot.fields.Publisher ?? lot.fields.Author ?? null,154 series: categorySlug.endsWith('_comics') ? (lot.fields.Title?.replace(/\s*No\.\s*\d+.*$/i, '') ?? null) : null,155 set: categorySlug.endsWith('_comics') ? (lot.fields.Title?.replace(/\s*No\.\s*\d+.*$/i, '') ?? null) : null,156 number: categorySlug.endsWith('_comics') ? issue : null,157 year: year ? Number(year) : null,158 identifiers: { pba_lot: lot.lotId },159 metadata: { sale_number: p.auction.saleNumber, sale_title: p.auction.title, estimate: lot.estimateText, fields: lot.fields },160 });161 out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: lot.lotId, rawTitle: lot.title, attributes, price: m.amount, currency: 'USD', saleDate, buyerPremiumIncluded: true, auctionHouse: 'PBA Galleries', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader && g.grader !== 'raw' ? g.grader : null, grade: g.grader && g.grader !== 'raw' ? g.grade : null, location: 'US' }));162 }163 return out;164 }165}166167export default (meta: ConnectorMeta) => new PbaGalleriesConnector(meta);168