import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { AssetAttributes, NormalizedRecord } from '@rareindex/shared'; import { lotAttributes, makeSale, vehicleAttributes } from '../../firecrawl/_carlib/index.js'; /** * PCARMARKET sold auctions (Porsche-centric enthusiast marketplace; cars, watches, automobilia). * One raw record per API page (compact items); one sale per item with status "Sold". */ const BASE = 'https://www.pcarmarket.com'; const API = `${BASE}/api/auctions/`; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ id: z.number(), title: z.string(), slug: z.string(), vehicle: z.object({ make: z.string().nullable().optional(), model: z.string().nullable().optional(), year: z.number().nullable().optional() }).nullable().optional(), high_bid: z.number().nullable().optional(), end_date: z.string().nullable().optional(), status: z.string().nullable().optional(), country: z.string().nullable().optional(), zip_code: z.string().nullable().optional(), mileage_body: z.number().nullable().optional(), odometer_type: z.string().nullable().optional(), bid_count: z.number().nullable().optional(), reserve_status: z.string().nullable().optional(), is_marketplace: z.boolean().nullable().optional(), featured_image_large_url: z.string().nullable().optional(), }); export type Item = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), count: z.number().nullable(), items: z.array(ItemSchema) }); const KEEP = ['id', 'title', 'slug', 'vehicle', 'high_bid', 'end_date', 'status', 'country', 'zip_code', 'mileage_body', 'odometer_type', 'bid_count', 'reserve_status', 'is_marketplace', 'featured_image_large_url'] as const; export function trimItem(raw: Record): Item | null { const out: Record = {}; for (const k of KEEP) if (raw[k] !== undefined) out[k] = raw[k]; if (out.vehicle && typeof out.vehicle === 'object') { const v = out.vehicle as Record; out.vehicle = { make: v.make ?? null, model: v.model ?? null, year: v.year ?? null }; } const p = ItemSchema.safeParse(out); return p.success ? p.data : null; } const WATCH_BRANDS: Array<[RegExp, string]> = [ [/\brolex\b/i, 'rolex'], [/\bomega\b/i, 'omega'], [/\bpatek\b/i, 'patek_philippe'], [/\baudemars\b/i, 'audemars_piguet'], [/\b(tag heuer|heuer|tissot|panerai|cartier|breitling|iwc|tudor|hublot|zenith|seiko|grand seiko|franck muller|chopard|longines|oris|bell & ross|jaeger|montblanc|richard mille|vacheron|a\.? lange|girard|chronograph watch|watch ref)\b/i, 'other_watches'], ]; /** Classify a sold lot: vehicle object → car/moto; watch keywords → watch slugs; else automobilia. */ export function classify(it: Item): { kind: 'vehicle' | 'watch' | 'memorabilia'; categorySlug: string; brand: string | null } { const t = it.title; if (it.vehicle && (it.vehicle.make || it.vehicle.year)) return { kind: 'vehicle', categorySlug: 'automobiles', brand: it.vehicle.make ?? null }; if (/\bwatch(es)?\b|\bref\.? ?[a-z0-9.-]{4,}\b.*(full set|box)/i.test(t) || WATCH_BRANDS.slice(0, 4).some(([re]) => re.test(t) && /watch|ref\b|ref\.|full set|dial|bracelet/i.test(t))) { for (const [re, slug] of WATCH_BRANDS) if (re.test(t)) return { kind: 'watch', categorySlug: slug, brand: t.match(re)?.[0]?.replace(/\bwatch ref\b|\bchronograph watch\b/i, '').trim() || null }; return { kind: 'watch', categorySlug: 'other_watches', brand: null }; } return { kind: 'memorabilia', categorySlug: 'automotive_memorabilia', brand: null }; } export function watchReference(title: string): string | null { const m = title.match(/\bRef\.?\s*([A-Z0-9][A-Z0-9.\-/]{3,})/i); return m ? m[1]!.replace(/[.,]$/, '') : null; } export class PcarmarketConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1200; async *crawl(ctx: CrawlContext): AsyncIterable { const limit = Number(this.meta.config.limit ?? 50); const pages = Number(this.meta.config.pagesPerRun ?? 10); const backfill = ctx.options.mode === 'backfill'; const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1; const newestSeen = !backfill && typeof ctx.options.cursor?.newestEnd === 'string' ? String(ctx.options.cursor.newestEnd) : ''; let count = 0; let maxEnd = newestSeen; for (let page = start; page < start + pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${API}?limit=${limit}&page=${page}&sort_by=ending_soon&status=sold&type=all`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price', 'date', 'status'], parse: (r) => { const first = (r.json as { results?: Array> } | null)?.results?.[0]; return first ? { title: first.title, price: first.high_bid, date: first.end_date, status: first.status } : null; }, }); const data = res.json as { results?: Array>; count?: number; next?: string | null } | null; if (!res.success || !data?.results) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const items = data.results.map(trimItem).filter((x): x is Item => Boolean(x)); if (!items.length) { ctx.anomaly('empty_page', `page ${page}`); break; } for (const it of items) if (it.end_date && it.end_date > maxEnd) maxEnd = it.end_date; count++; yield { url: `${BASE}/results/?page=${page}`, externalId: `results:${page}:${items[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_page' as const, page, count: data.count ?? null, items }, fetchedAt: res.fetchedAt }; const oldest = items.map((i) => i.end_date ?? '').filter(Boolean).sort()[0] ?? ''; if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() }); else if (newestSeen && oldest && oldest <= newestSeen) break; if (!data.next) break; } if (!backfill && maxEnd) await ctx.setCursor({ newestEnd: maxEnd, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; for (const it of p.items) { if (it.status !== 'Sold' || !it.high_bid || it.high_bid <= 0 || !it.end_date) continue; const saleDate = new Date(it.end_date); if (Number.isNaN(saleDate.getTime())) continue; const c = classify(it); const country = it.country === 'United States of America' ? 'US' : it.country === 'Canada' ? 'CA' : (it.country ?? null); const meta = { bid_count: it.bid_count ?? null, reserve_status: it.reserve_status ?? null, mileage: it.mileage_body ?? null, odometer_type: it.odometer_type ?? null, zip_code: it.zip_code ?? null, marketplace: it.is_marketplace ?? null }; let attributes: AssetAttributes; if (c.kind === 'vehicle') { attributes = vehicleAttributes(it.title, { country, identifiers: { pcarmarket_id: String(it.id) }, metadata: meta }); if (it.vehicle?.make) attributes.brand = it.vehicle.make; if (it.vehicle?.model) attributes.model = it.vehicle.model; if (it.vehicle?.year) attributes.year = it.vehicle.year; } else if (c.kind === 'watch') { const ref = watchReference(it.title); attributes = lotAttributes({ categorySlug: c.categorySlug, name: it.title.replace(/^No Reserve\s+/i, ''), brand: c.brand, country, identifiers: { pcarmarket_id: String(it.id), ...(ref ? { reference: ref } : {}) }, metadata: meta }); attributes.reference = ref; } else { attributes = lotAttributes({ categorySlug: 'automotive_memorabilia', name: it.title.replace(/^No Reserve\s+/i, ''), country, identifiers: { pcarmarket_id: String(it.id) }, metadata: meta }); } out.push( makeSale({ meta: this.meta, sourceUrl: `${BASE}/auction/${it.slug}/`, externalId: String(it.id), rawTitle: it.title, attributes, price: it.high_bid, currency: 'USD', saleDate, buyerPremiumIncluded: false, auctionHouse: 'PCARMARKET', imageUrls: it.featured_image_large_url ? [it.featured_image_large_url] : [], location: [it.zip_code, country].filter(Boolean).join(', ') || null, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, confidence: c.kind === 'vehicle' ? 0.92 : 0.85, }), ); } return out; } } export default function createConnector(meta: ConnectorMeta): PcarmarketConnector { return new PcarmarketConnector(meta); }