spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateDMY, makeSale, md, money, splitMarkdownItems, vehicleAttributes } from '../_carlib/index.js';56const BASE = 'https://collectingcars.com';7const PARSER_VERSION = '1.0.0';89export const ItemSchema = z.object({10  slug: z.string(),11  url: z.string(),12  title: z.string(),13  priceText: z.string(),14  dateText: z.string().nullable(),15  country: z.string().nullable(),16  town: z.string().nullable(),17  image: z.string().nullable(),18});19export const PagePayloadSchema = z.object({ kind: z.literal('sold_page'), page: z.number(), total: z.number().nullable(), items: z.array(ItemSchema) });20export type PagePayload = z.infer<typeof PagePayloadSchema>;2122const CURRENCY_LINE = /^(?:£|€|A\$|US\$|CA\$|NZ\$|AED|CHF|kr|\$)\s?[0-9][0-9,.]*/;2324export function parseSoldPage(markdown: string, page: number): PagePayload {25  const total = markdown.match(/Showing\s+([\d,]+)\s+lots/i)?.[1];26  const chunks = splitMarkdownItems(markdown, /^- \[!\[/m);27  const items: z.infer<typeof ItemSchema>[] = [];28  for (const c of chunks) {29    const href = c.match(/\((https:\/\/collectingcars\.com\/for-sale\/[a-z0-9-]+)\)/)?.[1];30    if (!href) continue;31    const slug = href.split('/').pop()!;32    const title = c.match(/\*\*([^*]+)\*\*/)?.[1]?.trim() ?? c.match(/^- \[!\[([^\]]+)\]/)?.[1] ?? null;33    if (!title) continue;34    const lines = c.split('\n').map((l) => l.replace(/\\+$/, '').trim()).filter(Boolean);35    const priceText = lines.find((l) => CURRENCY_LINE.test(l)) ?? null;36    if (!priceText) continue; // no price → not sold37    const dateText = lines.find((l) => /^\d{2}\/\d{2}\/\d{4}$/.test(l)) ?? null;38    const flag = c.match(/!\[([^\]]+)\]\(https:\/\/flagcdn\.com\/[a-z]{2}\.svg\)([^\]\n]*)/);39    items.push({ slug, url: href, title: md.clean(title), priceText, dateText, country: flag?.[1]?.trim() ?? null, town: flag?.[2]?.trim() || null, image: md.image(c) });40  }41  return { kind: 'sold_page', page, total: total ? Number(total.replace(/,/g, '')) : null, items };42}4344const MEMORABILIA = /\b(watch|helmet|poster|sign|number plate|registration|memorabilia|model|artwork|painting|print|petrol pump|engine|wheel set|wheels|seat|suit|jacket|book|literature|steering wheel|trophy)\b/i;4546export class CollectingCarsConnector extends BaseConnector {47  readonly version = '1.0.0';48  readonly parserVersion = PARSER_VERSION;49  protected override minIntervalMs = 2000;5051  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {52    const pages = Number(this.meta.config.pagesPerRun ?? 10);53    const backfill = ctx.options.mode === 'backfill';54    const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;55    const newest = !backfill && typeof ctx.options.cursor?.newestSold === 'string' ? new Date(ctx.options.cursor.newestSold as string) : null;56    let count = 0;57    let maxSold: Date | null = newest;58    for (let page = start; page < start + pages; page++) {59      if (ctx.signal?.aborted || this.reached(ctx, count)) break;60      const url = `${BASE}/sold${page > 1 ? `?page=${page}` : ''}`;61      await this.throttle();62      const res = await ctx.fetch(url, {63        expect: ['title', 'price', 'date', 'status'],64        parse: (r) => {65          const f = r.markdown ? parseSoldPage(r.markdown, page).items[0] : null;66          return f ? { title: f.title, price: money(f.priceText)?.amount ?? null, date: f.dateText, status: 'sold' } : null;67        },68      });69      if (!res.success || !res.markdown) {70        ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);71        break;72      }73      const payload = parseSoldPage(res.markdown, page);74      if (payload.items.length === 0) {75        ctx.anomaly('empty_page', url);76        break;77      }78      count++;79      yield { url, externalId: `sold:${page}:${payload.items[0]!.slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };80      const dates = payload.items.map((i) => dateDMY(i.dateText)).filter((d): d is Date => Boolean(d));81      for (const d of dates) if (!maxSold || d > maxSold) maxSold = d;82      const oldest = dates.length ? new Date(Math.min(...dates.map((d) => d.getTime()))) : null;83      if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });84      else if (newest && oldest && oldest < newest) break;85    }86    if (!backfill && maxSold) await ctx.setCursor({ newestSold: maxSold.toISOString(), updatedAt: new Date().toISOString() });87  }8889  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {90    const p = PagePayloadSchema.parse(raw.payload);91    const out: NormalizedSale[] = [];92    for (const it of p.items) {93      const m = money(it.priceText, 'GBP');94      const saleDate = dateDMY(it.dateText);95      if (!m || !saleDate) continue;96      const memorabilia = MEMORABILIA.test(it.title) && !/^\d{4}\s/.test(it.title);97      const attributes = vehicleAttributes(it.title, { country: it.country, identifiers: { collectingcars_slug: it.slug }, metadata: { town: it.town }, ...(memorabilia ? { categorySlug: 'automotive_memorabilia' as const } : {}) });98      out.push(makeSale({ meta: this.meta, sourceUrl: it.url, externalId: it.slug, rawTitle: it.title, attributes, price: m.amount, currency: m.currency, saleDate, buyerPremiumIncluded: false, auctionHouse: 'Collecting Cars', imageUrls: it.image ? [it.image.replace(/\?.*$/, '')] : [], location: [it.town, it.country].filter(Boolean).join(', ') || null, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }));99    }100    return out;101  }102}103104export default (meta: ConnectorMeta) => new CollectingCarsConnector(meta);105