import { z } from 'zod'; import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared'; import { dateDMY, makeSale, md, money, splitMarkdownItems, vehicleAttributes } from '../_carlib/index.js'; const BASE = 'https://collectingcars.com'; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ slug: z.string(), url: z.string(), title: z.string(), priceText: z.string(), dateText: z.string().nullable(), country: z.string().nullable(), town: z.string().nullable(), image: z.string().nullable(), }); export const PagePayloadSchema = z.object({ kind: z.literal('sold_page'), page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) }); export type PagePayload = z.infer; const CURRENCY_LINE = /^(?:£|€|A\$|US\$|CA\$|NZ\$|AED|CHF|kr|\$)\s?[0-9][0-9,.]*/; export function parseSoldPage(markdown: string, page: number): PagePayload { const total = markdown.match(/Showing\s+([\d,]+)\s+lots/i)?.[1]; const chunks = splitMarkdownItems(markdown, /^- \[!\[/m); const items: z.infer[] = []; for (const c of chunks) { const href = c.match(/\((https:\/\/collectingcars\.com\/for-sale\/[a-z0-9-]+)\)/)?.[1]; if (!href) continue; const slug = href.split('/').pop()!; const title = c.match(/\*\*([^*]+)\*\*/)?.[1]?.trim() ?? c.match(/^- \[!\[([^\]]+)\]/)?.[1] ?? null; if (!title) continue; const lines = c.split('\n').map((l) => l.replace(/\\+$/, '').trim()).filter(Boolean); const priceText = lines.find((l) => CURRENCY_LINE.test(l)) ?? null; if (!priceText) continue; // no price → not sold const dateText = lines.find((l) => /^\d{2}\/\d{2}\/\d{4}$/.test(l)) ?? null; const flag = c.match(/!\[([^\]]+)\]\(https:\/\/flagcdn\.com\/[a-z]{2}\.svg\)([^\]\n]*)/); items.push({ slug, url: href, title: md.clean(title), priceText, dateText, country: flag?.[1]?.trim() ?? null, town: flag?.[2]?.trim() || null, image: md.image(c) }); } return { kind: 'sold_page', page, total: total ? Number(total.replace(/,/g, '')) : null, items }; } const MEMORABILIA = /\b(watch|helmet|poster|sign|number plate|registration|memorabilia|model|artwork|painting|print|petrol pump|engine|wheel set|wheels|seat|suit|jacket|book|literature|steering wheel|trophy)\b/i; export class CollectingCarsConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const pages = Number(this.meta.config.pagesPerRun ?? 10); const backfill = ctx.options.mode === 'backfill'; const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1; const newest = !backfill && typeof ctx.options.cursor?.newestSold === 'string' ? new Date(ctx.options.cursor.newestSold as string) : null; let count = 0; let maxSold: Date | null = newest; for (let page = start; page < start + pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; const url = `${BASE}/sold${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { expect: ['title', 'price', 'date', 'status'], parse: (r) => { const f = r.markdown ? parseSoldPage(r.markdown, page).items[0] : null; return f ? { title: f.title, price: money(f.priceText)?.amount ?? null, date: f.dateText, status: 'sold' } : null; }, }); if (!res.success || !res.markdown) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parseSoldPage(res.markdown, page); if (payload.items.length === 0) { ctx.anomaly('empty_page', url); break; } count++; yield { url, externalId: `sold:${page}:${payload.items[0]!.slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; const dates = payload.items.map((i) => dateDMY(i.dateText)).filter((d): d is Date => Boolean(d)); for (const d of dates) if (!maxSold || d > maxSold) maxSold = d; const oldest = dates.length ? new Date(Math.min(...dates.map((d) => d.getTime()))) : null; if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() }); else if (newest && oldest && oldest < newest) break; } if (!backfill && maxSold) await ctx.setCursor({ newestSold: maxSold.toISOString(), updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedSale[] = []; for (const it of p.items) { const m = money(it.priceText, 'GBP'); const saleDate = dateDMY(it.dateText); if (!m || !saleDate) continue; const memorabilia = MEMORABILIA.test(it.title) && !/^\d{4}\s/.test(it.title); const attributes = vehicleAttributes(it.title, { country: it.country, identifiers: { collectingcars_slug: it.slug }, metadata: { town: it.town }, ...(memorabilia ? { categorySlug: 'automotive_memorabilia' as const } : {}) }); out.push(makeSale({ meta: this.meta, sourceUrl: it.url, externalId: it.slug, rawTitle: it.title, attributes, price: m.amount, currency: m.currency, saleDate, buyerPremiumIncluded: false, auctionHouse: 'Collecting Cars', imageUrls: it.image ? [it.image.replace(/\?.*$/, '')] : [], location: [it.town, it.country].filter(Boolean).join(', ') || null, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION })); } return out; } } export default (meta: ConnectorMeta) => new CollectingCarsConnector(meta);