spb/rareindex

1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord, NormalizedSale } from '@rareindex/shared';4import { dateMDY, makeSale, md, money, splitMarkdownItems, vehicleAttributes } from '../_carlib/index.js';56const BASE = 'https://carsandbids.com';7const PARSER_VERSION = '1.0.0';89export const ItemSchema = z.object({10  id: z.string(),11  url: z.string(),12  title: z.string(),13  statusText: z.string(),14  ended: z.string().nullable(),15  subtitle: z.string().nullable(),16  image: z.string().nullable(),17});18export const PagePayloadSchema = z.object({ kind: z.literal('past_page'), page: z.number(), items: z.array(ItemSchema) });19export type PagePayload = z.infer<typeof PagePayloadSchema>;2021/** Parse the Firecrawl markdown of /past-auctions/ into compact items. */22export function parsePastPage(markdown: string, page: number): PagePayload {23  const chunks = splitMarkdownItems(markdown, /^- \[!\[/m);24  const items: z.infer<typeof ItemSchema>[] = [];25  for (const c of chunks) {26    const link = md.link(c, /https:\/\/carsandbids\.com\/auctions\/([A-Za-z0-9]+)\/[a-z0-9-]+/);27    if (!link) continue;28    const id = link.href.match(/\/auctions\/([A-Za-z0-9]+)\//)?.[1];29    if (!id) continue;30    const title = c.match(/\]\(https:\/\/carsandbids\.com\/auctions\/[A-Za-z0-9]+\/[a-z0-9-]+\s+"([^"]+)"\)/)?.[1] ?? link.text;31    const statusText = c.match(/-\s*((?:Sold for|Bid to)\s+[^\]\n]+)\]/)?.[1]?.trim() ?? '';32    const ended = c.match(/Ended\s+(\d{1,2}\/\d{1,2}\/\d{2,4})/)?.[1] ?? null;33    const lines = c.split('\n').map((l) => l.trim()).filter(Boolean);34    const titleLineIdx = lines.findIndex((l) => l.startsWith(`[${title}](`) || l.startsWith(`[${title.replace(/"/g, '')}]`));35    const subtitle = titleLineIdx >= 0 ? lines.slice(titleLineIdx + 1).find((l) => !/^Ended|^Featured|^Watch|^!\[|^- \[/.test(l) && !l.startsWith('[')) ?? null : null;36    if (!statusText) continue;37    items.push({ id, url: link.href, title: md.clean(title), statusText, ended, subtitle: subtitle ? md.clean(subtitle) : null, image: md.image(c) });38  }39  return { kind: 'past_page', page, items };40}4142export class CarsAndBidsConnector extends BaseConnector {43  readonly version = '1.0.0';44  readonly parserVersion = PARSER_VERSION;45  protected override minIntervalMs = 2000;4647  async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {48    const pages = Number(this.meta.config.pagesPerRun ?? 10);49    const backfill = ctx.options.mode === 'backfill';50    const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;51    const newest = !backfill && typeof ctx.options.cursor?.newestEnded === 'string' ? new Date(ctx.options.cursor.newestEnded as string) : null;52    let count = 0;53    let maxEnded: Date | null = newest;54    for (let page = start; page < start + pages; page++) {55      if (ctx.signal?.aborted || this.reached(ctx, count)) break;56      const url = `${BASE}/past-auctions/${page > 1 ? `?page=${page}` : ''}`;57      await this.throttle();58      const res = await ctx.fetch(url, {59        expect: ['title', 'price', 'date', 'status'],60        parse: (r) => {61          const p = r.markdown ? parsePastPage(r.markdown, page) : null;62          const f = p?.items[0];63          return f ? { title: f.title, price: money(f.statusText, 'USD')?.amount ?? null, date: f.ended, status: f.statusText } : null;64        },65      });66      if (!res.success || !res.markdown) {67        ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);68        break;69      }70      const payload = parsePastPage(res.markdown, page);71      if (payload.items.length === 0) {72        ctx.anomaly('empty_page', url);73        break;74      }75      count++;76      yield { url, externalId: `past:${page}:${payload.items[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };77      const dates = payload.items.map((i) => dateMDY(i.ended)).filter((d): d is Date => Boolean(d));78      for (const d of dates) if (!maxEnded || d > maxEnded) maxEnded = d;79      const oldest = dates.length ? new Date(Math.min(...dates.map((d) => d.getTime()))) : null;80      if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });81      else if (newest && oldest && oldest < newest) break;82    }83    if (!backfill && maxEnded) await ctx.setCursor({ newestEnded: maxEnded.toISOString(), updatedAt: new Date().toISOString() });84  }8586  async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {87    const p = PagePayloadSchema.parse(raw.payload);88    const out: NormalizedSale[] = [];89    for (const it of p.items) {90      if (!/^Sold for/i.test(it.statusText)) continue;91      const m = money(it.statusText, 'USD');92      const saleDate = dateMDY(it.ended);93      if (!m || !saleDate) continue;94      const attributes = vehicleAttributes(it.title, { identifiers: { carsandbids_id: it.id }, metadata: { highlights: it.subtitle } });95      out.push(makeSale({ meta: this.meta, sourceUrl: it.url, externalId: it.id, rawTitle: it.title, attributes, price: m.amount, currency: m.currency, saleDate, buyerPremiumIncluded: false, auctionHouse: 'Cars & Bids', imageUrls: it.image ? [it.image] : [], description: it.subtitle, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }));96    }97    return out;98  }99}100101export default (meta: ConnectorMeta) => new CarsAndBidsConnector(meta);102