TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { AssetAttributes, NormalizedRecord } from '@rareindex/shared';4import { lotAttributes, makeSale, vehicleAttributes } from '../../firecrawl/_carlib/index.js';56/**7 * PCARMARKET sold auctions (Porsche-centric enthusiast marketplace; cars, watches, automobilia).8 * One raw record per API page (compact items); one sale per item with status "Sold".9 */10const BASE = 'https://www.pcarmarket.com';11const API = `${BASE}/api/auctions/`;12const PARSER_VERSION = '1.0.0';1314export const ItemSchema = z.object({15 id: z.number(),16 title: z.string(),17 slug: z.string(),18 vehicle: z.object({ make: z.string().nullable().optional(), model: z.string().nullable().optional(), year: z.number().nullable().optional() }).nullable().optional(),19 high_bid: z.number().nullable().optional(),20 end_date: z.string().nullable().optional(),21 status: z.string().nullable().optional(),22 country: z.string().nullable().optional(),23 zip_code: z.string().nullable().optional(),24 mileage_body: z.number().nullable().optional(),25 odometer_type: z.string().nullable().optional(),26 bid_count: z.number().nullable().optional(),27 reserve_status: z.string().nullable().optional(),28 is_marketplace: z.boolean().nullable().optional(),29 featured_image_large_url: z.string().nullable().optional(),30});31export type Item = z.infer<typeof ItemSchema>;32export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), count: z.number().nullable(), items: z.array(ItemSchema) });3334const KEEP = ['id', 'title', 'slug', 'vehicle', 'high_bid', 'end_date', 'status', 'country', 'zip_code', 'mileage_body', 'odometer_type', 'bid_count', 'reserve_status', 'is_marketplace', 'featured_image_large_url'] as const;3536export function trimItem(raw: Record<string, unknown>): Item | null {37 const out: Record<string, unknown> = {};38 for (const k of KEEP) if (raw[k] !== undefined) out[k] = raw[k];39 if (out.vehicle && typeof out.vehicle === 'object') {40 const v = out.vehicle as Record<string, unknown>;41 out.vehicle = { make: v.make ?? null, model: v.model ?? null, year: v.year ?? null };42 }43 const p = ItemSchema.safeParse(out);44 return p.success ? p.data : null;45}4647const WATCH_BRANDS: Array<[RegExp, string]> = [48 [/\brolex\b/i, 'rolex'],49 [/\bomega\b/i, 'omega'],50 [/\bpatek\b/i, 'patek_philippe'],51 [/\baudemars\b/i, 'audemars_piguet'],52 [/\b(tag heuer|heuer|tissot|panerai|cartier|breitling|iwc|tudor|hublot|zenith|seiko|grand seiko|franck muller|chopard|longines|oris|bell & ross|jaeger|montblanc|richard mille|vacheron|a\.? lange|girard|chronograph watch|watch ref)\b/i, 'other_watches'],53];5455/** Classify a sold lot: vehicle object → car/moto; watch keywords → watch slugs; else automobilia. */56export function classify(it: Item): { kind: 'vehicle' | 'watch' | 'memorabilia'; categorySlug: string; brand: string | null } {57 const t = it.title;58 if (it.vehicle && (it.vehicle.make || it.vehicle.year)) return { kind: 'vehicle', categorySlug: 'automobiles', brand: it.vehicle.make ?? null };59 if (/\bwatch(es)?\b|\bref\.? ?[a-z0-9.-]{4,}\b.*(full set|box)/i.test(t) || WATCH_BRANDS.slice(0, 4).some(([re]) => re.test(t) && /watch|ref\b|ref\.|full set|dial|bracelet/i.test(t))) {60 for (const [re, slug] of WATCH_BRANDS) if (re.test(t)) return { kind: 'watch', categorySlug: slug, brand: t.match(re)?.[0]?.replace(/\bwatch ref\b|\bchronograph watch\b/i, '').trim() || null };61 return { kind: 'watch', categorySlug: 'other_watches', brand: null };62 }63 return { kind: 'memorabilia', categorySlug: 'automotive_memorabilia', brand: null };64}6566export function watchReference(title: string): string | null {67 const m = title.match(/\bRef\.?\s*([A-Z0-9][A-Z0-9.\-/]{3,})/i);68 return m ? m[1]!.replace(/[.,]$/, '') : null;69}7071export class PcarmarketConnector extends BaseConnector {72 readonly version = '1.0.0';73 readonly parserVersion = PARSER_VERSION;74 protected override minIntervalMs = 1200;7576 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {77 const limit = Number(this.meta.config.limit ?? 50);78 const pages = Number(this.meta.config.pagesPerRun ?? 10);79 const backfill = ctx.options.mode === 'backfill';80 const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;81 const newestSeen = !backfill && typeof ctx.options.cursor?.newestEnd === 'string' ? String(ctx.options.cursor.newestEnd) : '';82 let count = 0;83 let maxEnd = newestSeen;84 for (let page = start; page < start + pages; page++) {85 if (ctx.signal?.aborted || this.reached(ctx, count)) break;86 const url = `${API}?limit=${limit}&page=${page}&sort_by=ending_soon&status=sold&type=all`;87 await this.throttle();88 const res = await ctx.fetch(url, {89 engines: ['api'],90 expect: ['title', 'price', 'date', 'status'],91 parse: (r) => {92 const first = (r.json as { results?: Array<Record<string, unknown>> } | null)?.results?.[0];93 return first ? { title: first.title, price: first.high_bid, date: first.end_date, status: first.status } : null;94 },95 });96 const data = res.json as { results?: Array<Record<string, unknown>>; count?: number; next?: string | null } | null;97 if (!res.success || !data?.results) {98 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);99 break;100 }101 const items = data.results.map(trimItem).filter((x): x is Item => Boolean(x));102 if (!items.length) {103 ctx.anomaly('empty_page', `page ${page}`);104 break;105 }106 for (const it of items) if (it.end_date && it.end_date > maxEnd) maxEnd = it.end_date;107 count++;108 yield { url: `${BASE}/results/?page=${page}`, externalId: `results:${page}:${items[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_page' as const, page, count: data.count ?? null, items }, fetchedAt: res.fetchedAt };109 const oldest = items.map((i) => i.end_date ?? '').filter(Boolean).sort()[0] ?? '';110 if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });111 else if (newestSeen && oldest && oldest <= newestSeen) break;112 if (!data.next) break;113 }114 if (!backfill && maxEnd) await ctx.setCursor({ newestEnd: maxEnd, updatedAt: new Date().toISOString() });115 }116117 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {118 const p = PagePayloadSchema.parse(raw.payload);119 const out: NormalizedRecord[] = [];120 for (const it of p.items) {121 if (it.status !== 'Sold' || !it.high_bid || it.high_bid <= 0 || !it.end_date) continue;122 const saleDate = new Date(it.end_date);123 if (Number.isNaN(saleDate.getTime())) continue;124 const c = classify(it);125 const country = it.country === 'United States of America' ? 'US' : it.country === 'Canada' ? 'CA' : (it.country ?? null);126 const meta = { bid_count: it.bid_count ?? null, reserve_status: it.reserve_status ?? null, mileage: it.mileage_body ?? null, odometer_type: it.odometer_type ?? null, zip_code: it.zip_code ?? null, marketplace: it.is_marketplace ?? null };127 let attributes: AssetAttributes;128 if (c.kind === 'vehicle') {129 attributes = vehicleAttributes(it.title, { country, identifiers: { pcarmarket_id: String(it.id) }, metadata: meta });130 if (it.vehicle?.make) attributes.brand = it.vehicle.make;131 if (it.vehicle?.model) attributes.model = it.vehicle.model;132 if (it.vehicle?.year) attributes.year = it.vehicle.year;133 } else if (c.kind === 'watch') {134 const ref = watchReference(it.title);135 attributes = lotAttributes({ categorySlug: c.categorySlug, name: it.title.replace(/^No Reserve\s+/i, ''), brand: c.brand, country, identifiers: { pcarmarket_id: String(it.id), ...(ref ? { reference: ref } : {}) }, metadata: meta });136 attributes.reference = ref;137 } else {138 attributes = lotAttributes({ categorySlug: 'automotive_memorabilia', name: it.title.replace(/^No Reserve\s+/i, ''), country, identifiers: { pcarmarket_id: String(it.id) }, metadata: meta });139 }140 out.push(141 makeSale({142 meta: this.meta,143 sourceUrl: `${BASE}/auction/${it.slug}/`,144 externalId: String(it.id),145 rawTitle: it.title,146 attributes,147 price: it.high_bid,148 currency: 'USD',149 saleDate,150 buyerPremiumIncluded: false,151 auctionHouse: 'PCARMARKET',152 imageUrls: it.featured_image_large_url ? [it.featured_image_large_url] : [],153 location: [it.zip_code, country].filter(Boolean).join(', ') || null,154 observedAt: raw.fetchedAt,155 parserVersion: PARSER_VERSION,156 confidence: c.kind === 'vehicle' ? 0.92 : 0.85,157 }),158 );159 }160 return out;161 }162}163164export default function createConnector(meta: ConnectorMeta): PcarmarketConnector {165 return new PcarmarketConnector(meta);166}167