SPB Git
spboucher.ai

spb/rareindex

Public

Connectors: cars, whisky, wine auction results (agent V); registry 93 connectors

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 7, 2026) parent d185538 Browse files

36 changed files +4,971 −136

36 changed files
Load all 36 files Unified Split
added connectors/api/gooding/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv[2] === 'capture') await captureFixture(dir, process.argv[3] ?? 'realized-auction', Number(process.argv[4] ?? 5));
7 +else await runSmoke(dir);
added connectors/api/gooding/index.test.ts +38 −0
@@ -0,0 +1,38 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import metaJson from './meta.json' with { type: 'json' };
5 +import createConnector, { auctionSaleDate, discoverRealizedSlugs, imageUrl, parseAuctionPageData } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('gooding', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('maps realized lots to auction sales in the auction currency', async () => {
13 + const fx = loadFixture('gooding', 'realized-auction');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThan(0);
16 + for (const r of out) {
17 + if (r.kind !== 'sale') throw new Error('expected sale');
18 + expect(['USD', 'GBP', 'EUR']).toContain(r.currency);
19 + expect(r.saleType).toBe('auction');
20 + expect(r.buyerPremiumIncluded).toBeNull();
21 + expect(['automobiles', 'motorcycles', 'automotive_memorabilia']).toContain(r.attributes.categorySlug);
22 + expect(r.attributes.identifiers.gooding_lot).toBeTruthy();
23 + expect(r.sourceUrl).toMatch(/^https:\/\/www\.goodingco\.com\/lot\//);
24 + expect(r.auctionHouse).toBe('Gooding & Company');
25 + }
26 + });
27 +
28 + it('parses page-data JSON, dates and images', () => {
29 + const json = { result: { data: { contentfulWebPageAuction: { title: 'x', auction: { name: 'Pebble Beach Auctions', currency: 'USD', sellThroughRate: 0.94, subEvents: [{ __typename: 'ContentfulSubEventViewing' }, { __typename: 'ContentfulSubEventAuction', startDate: '2026-08-14T16:00-08:00', endDate: '2026-08-14T21:00-08:00' }, { __typename: 'ContentfulSubEventAuction', startDate: '2026-08-15T11:00-08:00', endDate: '2026-08-15T16:00-08:00' }], lot: [{ slug: '1961-jaguar-xk150', lotNumber: 116, salePrice: 72800, privateSalesPrice: false, item: { __typename: 'ContentfulVehicle', title: '1961 Jaguar XK150 3.8-Litre Fixed Head Coupe', modelYear: 1961, make: { name: 'Jaguar' }, model: 'XK150', cloudinaryImagesCombined: [{ public_id: 'Prod/PB26 Pebble/1961_Jaguar' }] } }, { slug: 'unsold', lotNumber: 1, salePrice: null, item: { title: 'Unsold car' } }] } } } } };
30 + const p = parseAuctionPageData(json, 'pebble-beach-auctions-2026')!;
31 + expect(p.saleDate).toBe('2026-08-15T00:00:00.000Z');
32 + expect(p.lots).toHaveLength(2);
33 + expect(p.lots[0]!.image).toBe('https://res.cloudinary.com/goodingco/image/upload/c_fill,w_1200/Prod/PB26%20Pebble/1961_Jaguar');
34 + expect(auctionSaleDate(undefined)).toBeNull();
35 + expect(imageUrl(null)).toBeNull();
36 + expect(discoverRealizedSlugs('<a href="/auction/realized/pebble-beach-auctions-2026">x</a><a href="https://www.goodingco.com/auction/realized/amelia-island-auctions-2026/">y</a>')).toEqual(['pebble-beach-auctions-2026', 'amelia-island-auctions-2026']);
37 + });
38 +});
added connectors/api/gooding/index.ts +197 −0
@@ -0,0 +1,197 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { CurrencyCode, NormalizedRecord } from '@rareindex/shared';
4 +import { makeSale, vehicleAttributes } from '../../firecrawl/_carlib/index.js';
5 +
6 +/**
7 + * Gooding & Company prices realized. The Gatsby site serves every realized auction's lot list as
8 + * static JSON (page-data); one raw record per auction (compact lots), one sale per lot with a price.
9 + */
10 +const BASE = 'https://www.goodingco.com';
11 +const CLOUDINARY = 'https://res.cloudinary.com/goodingco/image/upload/c_fill,w_1200';
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +export const LotSchema = z.object({
15 + slug: z.string(),
16 + lotNumber: z.union([z.number(), z.string()]).nullable(),
17 + salePrice: z.number().nullable(),
18 + privateSalesPrice: z.boolean().nullable().optional(),
19 + title: z.string(),
20 + modelYear: z.number().nullable().optional(),
21 + make: z.string().nullable().optional(),
22 + model: z.string().nullable().optional(),
23 + itemType: z.string().nullable().optional(),
24 + image: z.string().nullable().optional(),
25 +});
26 +export type Lot = z.infer<typeof LotSchema>;
27 +export const AuctionPayloadSchema = z.object({
28 + kind: z.literal('realized_auction'),
29 + slug: z.string(),
30 + url: z.string(),
31 + name: z.string(),
32 + currency: z.string(),
33 + saleDate: z.string().nullable(),
34 + sellThroughRate: z.number().nullable(),
35 + lots: z.array(LotSchema),
36 +});
37 +export type AuctionPayload = z.infer<typeof AuctionPayloadSchema>;
38 +
39 +interface PageData {
40 + result?: { data?: { contentfulWebPageAuction?: { title?: string; auction?: RawAuction }; contentfulLot?: RawLot & { auction?: RawAuction } } };
41 +}
42 +interface RawAuction {
43 + name?: string;
44 + currency?: string;
45 + sellThroughRate?: number | null;
46 + subEvents?: Array<{ __typename?: string; startDate?: string; endDate?: string }>;
47 + lot?: RawLot[];
48 +}
49 +interface RawLot {
50 + slug?: string;
51 + lotNumber?: number | string | null;
52 + salePrice?: number | null;
53 + privateSalesPrice?: boolean | null;
54 + item?: { __typename?: string; title?: string; modelYear?: number | null; make?: { name?: string } | null; model?: string | null; cloudinaryImagesCombined?: Array<{ public_id?: string }> | null } | null;
55 +}
56 +
57 +export function imageUrl(publicId: string | undefined | null): string | null {
58 + if (!publicId) return null;
59 + return `${CLOUDINARY}/${publicId.split('/').map(encodeURIComponent).join('/')}`;
60 +}
61 +
62 +/** Sale date = end of the last auction sub-event (calendar day in the venue's offset, stored as UTC midnight). */
63 +export function auctionSaleDate(a: RawAuction | undefined): string | null {
64 + const days = (a?.subEvents ?? []).filter((s) => s.__typename === 'ContentfulSubEventAuction' && (s.endDate || s.startDate)).map((s) => (s.endDate ?? s.startDate)!.slice(0, 10));
65 + if (!days.length) return null;
66 + const last = days.sort().at(-1)!;
67 + return `${last}T00:00:00.000Z`;
68 +}
69 +
70 +export function trimLot(l: RawLot): Lot | null {
71 + if (!l.slug || !l.item?.title) return null;
72 + return LotSchema.parse({
73 + slug: l.slug,
74 + lotNumber: l.lotNumber ?? null,
75 + salePrice: typeof l.salePrice === 'number' ? l.salePrice : null,
76 + privateSalesPrice: l.privateSalesPrice ?? null,
77 + title: l.item.title,
78 + modelYear: l.item.modelYear ?? null,
79 + make: l.item.make?.name ?? null,
80 + model: l.item.model ?? null,
81 + itemType: l.item.__typename ?? null,
82 + image: imageUrl(l.item.cloudinaryImagesCombined?.[0]?.public_id),
83 + });
84 +}
85 +
86 +export function parseAuctionPageData(json: unknown, slug: string): AuctionPayload | null {
87 + const page = (json as PageData)?.result?.data?.contentfulWebPageAuction;
88 + const a = page?.auction;
89 + if (!a) return null;
90 + const lots = (a.lot ?? []).map(trimLot).filter((x): x is Lot => Boolean(x));
91 + return { kind: 'realized_auction', slug, url: `${BASE}/auction/realized/${slug}`, name: a.name ?? page?.title ?? slug, currency: (a.currency ?? 'USD').toUpperCase(), saleDate: auctionSaleDate(a), sellThroughRate: a.sellThroughRate ?? null, lots };
92 +}
93 +
94 +export function discoverRealizedSlugs(html: string): string[] {
95 + return [...new Set([...html.matchAll(/\/auction\/realized\/([a-z0-9-]+)/g)].map((m) => m[1]!))];
96 +}
97 +
98 +export class GoodingConnector extends BaseConnector {
99 + readonly version = '1.0.0';
100 + readonly parserVersion = PARSER_VERSION;
101 + protected override minIntervalMs = 1500;
102 + override readonly urlPatterns = [/^https?:\/\/www\.goodingco\.com\/lot\/[a-z0-9-]+\/?$/i];
103 +
104 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
105 + const seeds = (this.meta.config.seeds as string[] | undefined) ?? [];
106 + const perRun = Number(this.meta.config.auctionsPerRun ?? 3);
107 + const done = new Set<string>((ctx.options.cursor?.done as string[] | undefined) ?? []);
108 + const backfill = ctx.options.mode === 'backfill';
109 + // Discover currently linked realized auctions from the homepage (plain HTML).
110 + let discovered: string[] = [];
111 + const home = await ctx.fetch(`${BASE}/`, { engines: ['api'], responseType: 'text', minQuality: 0 });
112 + if (home.success && home.html) discovered = discoverRealizedSlugs(home.html);
113 + const slugs = [...new Set([...discovered, ...seeds])];
114 + // Incremental: newest first, skip auctions already ingested unless backfilling.
115 + const todo = slugs.filter((s) => backfill || !done.has(s)).slice(0, perRun);
116 + let count = 0;
117 + for (const slug of todo) {
118 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
119 + const url = `${BASE}/page-data/auction/realized/${slug}/page-data.json`;
120 + await this.throttle();
121 + const res = await ctx.fetch(url, {
122 + engines: ['api'],
123 + expect: ['title', 'price', 'currency', 'date'],
124 + parse: (r) => {
125 + const p = parseAuctionPageData(r.json, slug);
126 + const sold = p?.lots.find((l) => l.salePrice);
127 + return p ? { title: p.name, price: sold?.salePrice ?? null, currency: p.currency, date: p.saleDate } : null;
128 + },
129 + });
130 + const payload = res.success ? parseAuctionPageData(res.json, slug) : null;
131 + if (!payload) {
132 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
133 + continue;
134 + }
135 + if (!payload.lots.length) ctx.anomaly('empty_page', url);
136 + count++;
137 + yield { url: payload.url, externalId: `auction:${slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
138 + done.add(slug);
139 + await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() });
140 + }
141 + }
142 +
143 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
144 + const slug = url.match(/\/lot\/([a-z0-9-]+)/i)?.[1];
145 + if (!slug) return [];
146 + const res = await ctx.fetch(`${BASE}/page-data/lot/${slug}/page-data.json`, { engines: ['api'], minQuality: 0 });
147 + const lot = (res.json as PageData)?.result?.data?.contentfulLot;
148 + if (!res.success || !lot?.auction) return [];
149 + const item = trimLot({ ...lot, slug });
150 + if (!item) return [];
151 + const payload: AuctionPayload = { kind: 'realized_auction', slug: `lot-${slug}`, url, name: lot.auction.name ?? '', currency: (lot.auction.currency ?? 'USD').toUpperCase(), saleDate: auctionSaleDate(lot.auction), sellThroughRate: null, lots: [item] };
152 + return [{ url, externalId: `lot:${slug}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
153 + }
154 +
155 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
156 + const p = AuctionPayloadSchema.parse(raw.payload);
157 + if (!p.saleDate) return [];
158 + const saleDate = new Date(p.saleDate);
159 + const currency = p.currency as CurrencyCode;
160 + const out: NormalizedRecord[] = [];
161 + for (const lot of p.lots) {
162 + if (!lot.salePrice || lot.salePrice <= 0 || lot.privateSalesPrice) continue;
163 + const isVehicle = lot.itemType !== 'ContentfulAutomobilia' && Boolean(lot.make || lot.modelYear);
164 + const attributes = vehicleAttributes(lot.title, {
165 + ...(isVehicle ? {} : { categorySlug: 'automotive_memorabilia' }),
166 + identifiers: { gooding_lot: lot.slug },
167 + metadata: { auction: p.name, lot_number: lot.lotNumber, make_field: lot.make, model_field: lot.model, model_year_field: lot.modelYear, sell_through_rate: p.sellThroughRate },
168 + });
169 + if (lot.make) attributes.brand = lot.make;
170 + if (lot.modelYear) attributes.year = lot.modelYear;
171 + if (lot.model) attributes.model = lot.model;
172 + out.push(
173 + makeSale({
174 + meta: this.meta,
175 + sourceUrl: `${BASE}/lot/${lot.slug}`,
176 + externalId: lot.slug,
177 + rawTitle: lot.title,
178 + attributes,
179 + price: lot.salePrice,
180 + currency,
181 + saleDate,
182 + buyerPremiumIncluded: null,
183 + auctionHouse: 'Gooding & Company',
184 + lotNumber: lot.lotNumber === null ? null : String(lot.lotNumber),
185 + imageUrls: lot.image ? [lot.image] : [],
186 + observedAt: raw.fetchedAt,
187 + parserVersion: PARSER_VERSION,
188 + }),
189 + );
190 + }
191 + return out;
192 + }
193 +}
194 +
195 +export default function createConnector(meta: ConnectorMeta): GoodingConnector {
196 + return new GoodingConnector(meta);
197 +}
added connectors/api/gooding/meta.json +43 −0
@@ -0,0 +1,43 @@
1 +{
2 + "id": "gooding",
3 + "displayName": "Gooding & Company (prices realized)",
4 + "sourceId": "gooding",
5 + "sourceName": "Gooding & Company",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://www.goodingco.com",
8 + "module": "api/gooding",
9 + "enginePriority": ["api"],
10 + "categories": ["automobiles", "motorcycles", "automotive_memorabilia"],
11 + "regions": ["US", "GB", "FR"],
12 + "languages": ["en"],
13 + "currency": ["USD", "GBP", "EUR"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "medium",
23 + "trustScore": 0.9,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.goodingco.com/terms-of-use",
26 + "accessNotes": "Gooding & Company's public 'Prices Realized' pages are a Gatsby site; each realized auction page ships its full lot list as static JSON at /page-data/auction/realized/<slug>/page-data.json (one plain HTTPS request per auction, 0 credits; robots.txt has no disallow rules). Each lot carries salePrice (auction currency), lotNumber, title, model year, make, model and Cloudinary image ids (cloud 'goodingco'). Sale date = the last ContentfulSubEventAuction end date of the auction. Lots without a salePrice (unsold/withdrawn) and private-sale prices are skipped. The page does not state whether salePrice includes the buyer's premium, so buyer_premium_included is null (Gooding's published totals are customarily quoted inclusive of premium). Auction slugs come from config.seeds plus /auction/realized/ links found on the homepage; lookup() handles /lot/<slug> pages via their page-data JSON.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "seeds": [
31 + "pebble-beach-auctions-2026",
32 + "amelia-island-auctions-2026",
33 + "retromobile-paris-2026",
34 + "retromobile-new-york-auctions-2026",
35 + "pebble-beach-auctions-2025",
36 + "pebble-beach-auctions-2024",
37 + "london-auction-2024",
38 + "pebble-beach-auctions-2023",
39 + "london-auction-2023"
40 + ],
41 + "auctionsPerRun": 3
42 + }
43 +}
added connectors/api/historics/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv[2] === 'capture') await captureFixture(dir, process.argv[3] ?? 'lots-page', Number(process.argv[4] ?? 6));
7 +else await runSmoke(dir);
added connectors/api/historics/index.test.ts +42 −0
@@ -0,0 +1,42 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import metaJson from './meta.json' with { type: 'json' };
5 +import createConnector, { categoryFor, parseLotsPage, parseResultsList, parseUkDate } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('historics', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('maps lots to GBP hammer sales dated by the sale', async () => {
13 + const fx = loadFixture('historics', 'lots-page');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThan(0);
16 + for (const r of out) {
17 + if (r.kind !== 'sale') throw new Error('expected sale');
18 + expect(r.currency).toBe('GBP');
19 + expect(r.buyerPremiumIncluded).toBe(false);
20 + expect(r.attributes.identifiers.historics_lot).toMatch(/^\d+$/);
21 + expect(r.sourceUrl).toMatch(/^https:\/\/www\.historics\.co\.uk\/auction\/lot\//);
22 + expect(r.auctionHouse).toBe('Historics Auctioneers');
23 + }
24 + });
25 +
26 + it('parses dates, the results list and lot cards', () => {
27 + expect(parseUkDate('18th Jul, 2026 9:30')).toEqual(new Date(Date.UTC(2026, 6, 18)));
28 + expect(parseUkDate('6th Aug, 2026 19:30')).toEqual(new Date(Date.UTC(2026, 7, 6)));
29 + const list = `<div class="auction-calendar-item"><a href='/auction/details/a075-the-summer-serenade-windsorview-lakes?au=107'><img/></a><div class="auction-calendar-text"><h3>The Summer Serenade; Windsorview Lakes</h3> Date: 18th Jul, 2026 9:30 Sale number: A075 Lots: 194 blah</div></div><hr />`;
30 + const a = parseResultsList(list);
31 + expect(a).toHaveLength(1);
32 + expect(a[0]).toMatchObject({ au: '107', slug: 'a075-the-summer-serenade-windsorview-lakes', saleNumber: 'A075', lotCount: 194, endedOn: '2026-07-18T00:00:00.000Z' });
33 + const lots = `<div class="auction-lot"><div class="auction-lot-image"><a href="/auction/lot/lot-101---1955-phillips-panda-autocycle/?lot=20016&amp;so=0&amp;au=107"><img src="https://storagegohistorics.goauction.co.uk/stock/19969-0-small.jpg?v=1" /></a></div><div class="auction-lot-text"><p class="auction-lot-title"><a href="/auction/lot/lot-101---1955-phillips-panda-autocycle/?lot=20016&amp;so=0&amp;au=107"><span class='lot-title cat-29'>Lot 101 - <span class="req-tag"></span>1955 Phillips Panda Autocycle <span class='sub-title cat-29'>Offered without reserve</span></span></a></p><p><strong>Sold £801</strong></p></div></div><div class="auction-lot"><div class="auction-lot-text"><p class="auction-lot-title"><a href="/auction/lot/x/?lot=1&amp;au=107"><span class='lot-title'>Lot 102 - 2006 Nissan Micra</span></a></p><p><strong>Sold for an undisclosed fee</strong></p></div></div>`;
34 + const l = parseLotsPage(lots);
35 + expect(l).toHaveLength(2);
36 + expect(l[0]).toMatchObject({ lotId: '20016', lotNo: '101', title: '1955 Phillips Panda Autocycle', subtitle: 'Offered without reserve', priceGbp: 801, image: 'https://storagegohistorics.goauction.co.uk/stock/19969-0-small.jpg' });
37 + expect(l[1]!.priceGbp).toBeNull();
38 + expect(categoryFor(a[0]!, '1955 Phillips Panda Autocycle')).toMatchObject({ vehicle: true });
39 + expect(categoryFor({ ...a[0]!, saleNumber: 'W017' }, 'Porsche dealership sign')).toMatchObject({ categorySlug: 'automotive_memorabilia' });
40 + expect(categoryFor({ ...a[0]!, saleNumber: 'W015' }, 'Registration number 1 ABC')).toMatchObject({ categorySlug: 'license_plates' });
41 + });
42 +});
added connectors/api/historics/index.ts +174 −0
@@ -0,0 +1,174 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { lotAttributes, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js';
5 +
6 +/**
7 + * Historics Auctioneers — results list → per-sale "Past lots" grid (96 lots per page) with "Sold £X".
8 + * One raw record per lots page; one sale per lot with a price.
9 + */
10 +const BASE = 'https://www.historics.co.uk';
11 +const PARSER_VERSION = '1.0.0';
12 +
13 +export const AuctionSchema = z.object({ au: z.string(), slug: z.string(), title: z.string(), saleNumber: z.string().nullable(), endedOn: z.string().nullable(), lotCount: z.number().nullable() });
14 +export type Auction = z.infer<typeof AuctionSchema>;
15 +export const LotSchema = z.object({ lotId: z.string(), url: z.string(), lotNo: z.string().nullable(), title: z.string(), subtitle: z.string().nullable(), soldText: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable() });
16 +export type Lot = z.infer<typeof LotSchema>;
17 +export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), auction: AuctionSchema, page: z.number(), lots: z.array(LotSchema) });
18 +
19 +const MONTHS: Record<string, number> = { jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11 };
20 +/** "18th Jul, 2026 9:30" | "6th Aug, 2026 19:30" → UTC midnight. */
21 +export function parseUkDate(s: string | null | undefined): Date | null {
22 + const m = s?.match(/(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3})[a-z]*,?\s+(\d{4})/);
23 + if (!m) return null;
24 + const mo = MONTHS[m[2]!.toLowerCase()];
25 + return mo === undefined ? null : new Date(Date.UTC(Number(m[3]), mo, Number(m[1])));
26 +}
27 +
28 +/** Parse /auction-results: calendar items with title, "Date:"/"Ends:" and "Sale number:". */
29 +export function parseResultsList(htmlText: string): Auction[] {
30 + const $ = H.load(htmlText);
31 + const out: Auction[] = [];
32 + const seen = new Set<string>();
33 + $('.auction-calendar-item').each((_, el) => {
34 + const href = $(el).find('a[href*="/auction/details/"]').first().attr('href') ?? '';
35 + const m = href.match(/\/auction\/details\/([^/?]+)\?au=(\d+)/);
36 + if (!m || seen.has(m[2]!)) return;
37 + const text = $(el).text().replace(/\s+/g, ' ').trim();
38 + const title = H.text($(el).find('.auction-calendar-text h2, .auction-calendar-text h3, .auction-calendar-text h4').first()) ?? text.split(/\s(?:Date|Ends):/)[0]!.trim();
39 + const dateTxt = text.match(/(?:Date|Ends):\s*([^S]+?)\s+Sale number/i)?.[1] ?? text.match(/(?:Date|Ends):\s*(\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]{3,9},?\s+\d{4})/i)?.[1] ?? null;
40 + const saleNumber = text.match(/Sale number:\s*([A-Z]{1,2}O?\d{2,4})/i)?.[1] ?? null;
41 + const lots = text.match(/Lots:\s*(\d+)/i)?.[1];
42 + seen.add(m[2]!);
43 + out.push({ au: m[2]!, slug: m[1]!, title, saleNumber, endedOn: parseUkDate(dateTxt)?.toISOString() ?? null, lotCount: lots ? Number(lots) : null });
44 + });
45 + return out;
46 +}
47 +
48 +/** Parse a lots page: cards with lot number, title, "Sold £X". */
49 +export function parseLotsPage(htmlText: string): Lot[] {
50 + const $ = H.load(htmlText);
51 + const out: Lot[] = [];
52 + $('.auction-lot').each((_, el) => {
53 + const a = $(el).find('.auction-lot-title a').first();
54 + const href = a.attr('href') ?? '';
55 + const lotId = href.match(/[?&]lot=(\d+)/)?.[1];
56 + if (!lotId) return;
57 + const titleEl = a.find('.lot-title').clone();
58 + const subtitle = H.text(titleEl.find('.sub-title'));
59 + titleEl.find('.sub-title').remove();
60 + const full = H.text(titleEl) ?? '';
61 + const lotNo = full.match(/^Lot\s+([A-Z]?\d+[A-Z]?)\s*-\s*/i)?.[1] ?? null;
62 + const title = full.replace(/^Lot\s+[A-Z]?\d+[A-Z]?\s*-\s*/i, '').trim();
63 + if (!title) return;
64 + const soldText = H.text($(el).find('strong').filter((_, s) => /^Sold/i.test($(s).text().trim())).first());
65 + const priceTxt = soldText?.match(/Sold\s*£\s*([\d,]+(?:\.\d+)?)/i)?.[1] ?? null;
66 + const img = $(el).find('.auction-lot-image img').attr('src') ?? null;
67 + out.push({ lotId, url: `${BASE}${href.split('&so=')[0]!.replace(/&amp;/g, '&')}`, lotNo, title, subtitle, soldText, priceGbp: priceTxt ? Number(priceTxt.replace(/,/g, '')) : null, image: img ? img.replace(/\?v=.*$/, '') : null });
68 + });
69 + return out;
70 +}
71 +
72 +export function categoryFor(auction: Auction, title: string): { categorySlug: string; vehicle: boolean } {
73 + const sale = (auction.saleNumber ?? '').toUpperCase();
74 + if (/registration|number plate|cherished/i.test(title) || /^[A-Z]{1,2}\d*\s*[A-Z]{0,3}$/.test(title.replace(/\s+/g, ' ').trim()) && sale.startsWith('W')) return { categorySlug: 'license_plates', vehicle: false };
75 + if (sale.startsWith('W') && !/^(19|20)\d{2}\b/.test(title)) return { categorySlug: 'automotive_memorabilia', vehicle: false };
76 + return { categorySlug: 'automobiles', vehicle: true };
77 +}
78 +
79 +export class HistoricsConnector extends BaseConnector {
80 + readonly version = '1.0.0';
81 + readonly parserVersion = PARSER_VERSION;
82 + protected override minIntervalMs = 10_000; // robots.txt crawl-delay: 10
83 +
84 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
85 + const perRun = Number(this.meta.config.auctionsPerRun ?? 1);
86 + const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 3);
87 + const progress = { ...((ctx.options.cursor?.progress as Record<string, number> | undefined) ?? {}) };
88 + const complete = new Set<string>((ctx.options.cursor?.complete as string[] | undefined) ?? []);
89 + await this.throttle();
90 + const list = await ctx.fetch(`${BASE}/auction-results`, { engines: ['api'], responseType: 'text', expect: ['title', 'date'], parse: (r) => (r.html ? { title: parseResultsList(r.html)[0]?.title, date: parseResultsList(r.html)[0]?.endedOn } : null) });
91 + if (!list.success || !list.html) {
92 + ctx.anomaly('page_fetch_failed', `results list: ${list.error ?? list.httpStatus}`);
93 + return;
94 + }
95 + const auctions = parseResultsList(list.html).filter((a) => a.endedOn && new Date(a.endedOn).getTime() < Date.now());
96 + const todo = auctions.filter((a) => !complete.has(a.au)).slice(0, perRun);
97 + let pagesFetched = 0;
98 + let count = 0;
99 + for (const auction of todo) {
100 + let page = (progress[auction.au] ?? 0) + 1;
101 + while (true) {
102 + if (ctx.signal?.aborted || this.reached(ctx, count) || pagesFetched >= pagesPerRun) return void (await ctx.setCursor({ progress, complete: [...complete] }));
103 + const url = `${BASE}/auction/details/${auction.slug}?au=${auction.au}&pp=96&pn=${page}`;
104 + await this.throttle();
105 + const res = await ctx.fetch(url, {
106 + engines: ['api'],
107 + responseType: 'text',
108 + expect: ['title', 'price', 'currency', 'status'],
109 + parse: (r) => {
110 + const lots = r.html ? parseLotsPage(r.html) : [];
111 + const sold = lots.find((l) => l.priceGbp);
112 + return lots.length ? { title: lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, status: sold ? 'sold' : null } : null;
113 + },
114 + });
115 + pagesFetched++;
116 + const lots = res.success && res.html ? parseLotsPage(res.html) : [];
117 + if (!lots.length) {
118 + if (!res.success) ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
119 + complete.add(auction.au);
120 + break;
121 + }
122 + count++;
123 + yield { url, externalId: `auction:${auction.au}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, auction, page, lots }, fetchedAt: res.fetchedAt };
124 + progress[auction.au] = page;
125 + if (lots.length < 96) {
126 + complete.add(auction.au);
127 + break;
128 + }
129 + page++;
130 + }
131 + }
132 + await ctx.setCursor({ progress, complete: [...complete] });
133 + }
134 +
135 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
136 + const p = PagePayloadSchema.parse(raw.payload);
137 + if (!p.auction.endedOn) return [];
138 + const saleDate = new Date(p.auction.endedOn);
139 + const out: NormalizedRecord[] = [];
140 + for (const lot of p.lots) {
141 + if (!lot.priceGbp || lot.priceGbp <= 0) continue;
142 + const m = money(`£${lot.priceGbp}`, 'GBP');
143 + if (!m) continue;
144 + const cat = categoryFor(p.auction, lot.title);
145 + const meta = { auction: p.auction.title, sale_number: p.auction.saleNumber, subtitle: lot.subtitle, sold_text: lot.soldText };
146 + const attributes = cat.vehicle ? vehicleAttributes(lot.title, { country: 'GB', identifiers: { historics_lot: lot.lotId }, metadata: meta }) : lotAttributes({ categorySlug: cat.categorySlug, name: lot.title, country: 'GB', identifiers: { historics_lot: lot.lotId }, metadata: meta });
147 + out.push(
148 + makeSale({
149 + meta: this.meta,
150 + sourceUrl: lot.url,
151 + externalId: lot.lotId,
152 + rawTitle: lot.title,
153 + attributes,
154 + price: m.amount,
155 + currency: 'GBP',
156 + saleDate,
157 + buyerPremiumIncluded: false,
158 + auctionHouse: 'Historics Auctioneers',
159 + lotNumber: lot.lotNo,
160 + imageUrls: lot.image ? [lot.image] : [],
161 + description: lot.subtitle,
162 + location: 'United Kingdom',
163 + observedAt: raw.fetchedAt,
164 + parserVersion: PARSER_VERSION,
165 + }),
166 + );
167 + }
168 + return out;
169 + }
170 +}
171 +
172 +export default function createConnector(meta: ConnectorMeta): HistoricsConnector {
173 + return new HistoricsConnector(meta);
174 +}
added connectors/api/historics/meta.json +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "id": "historics",
3 + "displayName": "Historics Auctioneers (results)",
4 + "sourceId": "historics",
5 + "sourceName": "Historics Auctioneers",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://www.historics.co.uk",
8 + "module": "api/historics",
9 + "enginePriority": ["api"],
10 + "categories": ["automobiles", "motorcycles", "automotive_memorabilia", "license_plates"],
11 + "regions": ["GB"],
12 + "languages": ["en"],
13 + "currency": ["GBP"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.historics.co.uk/terms-and-conditions",
26 + "accessNotes": "Historics (UK classic-car auctions, Ascot/Windsor/Brooklands + online automobilia & registration sales) lists past sales at /auction-results (title, sale number, end date) and renders each sale's 'Past lots' server-side at /auction/details/<slug>?au=<id>&pp=96&pn=<page> with 'Sold £X' per lot card (plain HTTPS, 0 credits). robots.txt sets crawl-delay 10, so the connector waits 10 s between requests and caps pages per run. 'Sold £X' is the hammer price (Historics adds a buyer's premium + VAT on top) → buyer_premium_included=false; lots 'sold for an undisclosed fee' and unsold lots are skipped. Sale numbers starting with A = vehicle sales (cars/motorcycles from the title), W = online automobilia/registration sales (automotive_memorabilia; registration plates → license_plates). Sale date = the end date printed on the results list.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "auctionsPerRun": 1,
31 + "pagesPerRun": 3
32 + }
33 +}
added connectors/api/just-whisky/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv[2] === 'capture') await captureFixture(dir, process.argv[3] ?? 'lots-page', Number(process.argv[4] ?? 8));
7 +else await runSmoke(dir);
added connectors/api/just-whisky/index.test.ts +39 −0
@@ -0,0 +1,39 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import metaJson from './meta.json' with { type: 'json' };
5 +import createConnector, { lotsUrl, trimLot } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('just-whisky', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('maps reserve-met lots to GBP hammer sales', async () => {
13 + const fx = loadFixture('just-whisky', 'lots-page');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThan(0);
16 + for (const r of out) {
17 + if (r.kind !== 'sale') throw new Error('expected sale');
18 + expect(r.currency).toBe('GBP');
19 + expect(r.buyerPremiumIncluded).toBe(false);
20 + expect(['whisky', 'rum', 'cognac']).toContain(r.attributes.categorySlug);
21 + expect(r.attributes.identifiers.justwhisky_lot).toMatch(/^\d+$/);
22 + expect(r.sourceUrl).toMatch(/^https:\/\/www\.just-whisky\.co\.uk\/lot\//);
23 + }
24 + });
25 +
26 + it('skips reserve-not-met lots and builds API urls', async () => {
27 + const met = trimLot({ id: 1, slug: 'port-ellen-1981-feis-ile-2008-1', reserve_met: true, hammer_price: '2750.00', bid_stats: { current_bid: 2750 }, seller_sheet: { auction: { id: 1160, end_date: '2026-07-20T19:00:00' } }, item: { title: 'Port Ellen 1981', subtitle: 'Feis Ile 2008', strength: { name: '54.7%' }, size: { name: '70 cl' } } })!;
28 + const notMet = trimLot({ id: 2, slug: 'springbank-36-2', reserve_met: false, hammer_price: null, bid_stats: { current_bid: 3150 }, seller_sheet: { auction: { id: 1160, end_date: '2026-07-20T19:00:00' } }, item: { title: 'Springbank 36 Years Old 1965' } })!;
29 + const out = await connector.normalize({ url: 'u', externalId: null, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'lots_page', url: 'u', page: 1, count: 2, lots: [met, notMet] } });
30 + expect(out).toHaveLength(1);
31 + const s = out[0]!;
32 + if (s.kind !== 'sale') throw new Error('sale');
33 + expect(s.price).toBe(2750);
34 + expect(s.saleDate.toISOString()).toBe('2026-07-20T19:00:00.000Z');
35 + expect(s.attributes.size).toBe('70cl');
36 + expect(s.attributes.year).toBe(1981);
37 + expect(lotsUrl(new Date(Date.UTC(2026, 6, 14)), new Date(Date.UTC(2026, 6, 21)), 2, 200)).toBe('https://www.just-whisky.co.uk/api/lots/?min_end_date=14%2F07%2F2026&max_end_date=21%2F07%2F2026&ordering=-price&page_size=200&page=2');
38 + });
39 +});
added connectors/api/just-whisky/index.ts +210 −0
@@ -0,0 +1,210 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';
4 +import { whiskyFacts } from '../scotch-whisky-auctions/index.js';
5 +
6 +/**
7 + * Just Whisky past auctions — public JSON API behind the Past Auctions page. One raw record per API page
8 + * (compact lots); one sale per lot whose reserve was met (hammer price).
9 + */
10 +const BASE = 'https://www.just-whisky.co.uk';
11 +const PARSER_VERSION = '1.0.0';
12 +
13 +export const LotSchema = z.object({
14 + id: z.number(),
15 + slug: z.string(),
16 + title: z.string(),
17 + subtitle: z.string().nullable(),
18 + reserveMet: z.boolean(),
19 + hammerPrice: z.number().nullable(),
20 + currentBid: z.number().nullable(),
21 + isGroupLot: z.boolean(),
22 + auctionId: z.number().nullable(),
23 + auctionEnd: z.string().nullable(),
24 + strength: z.string().nullable(),
25 + size: z.string().nullable(),
26 + distillery: z.string().nullable(),
27 + bottler: z.string().nullable(),
28 + region: z.string().nullable(),
29 + estimatedValue: z.number().nullable(),
30 + image: z.string().nullable(),
31 +});
32 +export type Lot = z.infer<typeof LotSchema>;
33 +export const PagePayloadSchema = z.object({ kind: z.literal('lots_page'), url: z.string(), page: z.number(), count: z.number().nullable(), lots: z.array(LotSchema) });
34 +
35 +interface ApiLot {
36 + id: number;
37 + slug: string;
38 + reserve_met?: boolean;
39 + hammer_price?: string | null;
40 + is_group_lot?: boolean;
41 + custom_title?: string | null;
42 + custom_subtitle?: string | null;
43 + bid_stats?: { current_bid?: number | null };
44 + seller_sheet?: { auction?: { id?: number; end_date?: string } };
45 + item?: { title?: string; subtitle?: string; strength?: { name?: string } | null; size?: { name?: string } | null; distillery?: { name?: string } | string | null; bottler?: { name?: string } | string | null; region?: { name?: string } | string | null; estimated_value?: string | null; photo?: { file?: string } | null };
46 + photos?: Array<{ file?: string }>;
47 +}
48 +const name = (v: { name?: string } | string | null | undefined): string | null => (typeof v === 'string' ? v : v?.name ?? null) || null;
49 +
50 +export function trimLot(l: ApiLot): Lot | null {
51 + const title = (l.custom_title || l.item?.title || '').trim();
52 + if (!l.id || !l.slug || !title) return null;
53 + const hp = l.hammer_price ? Number(l.hammer_price) : null;
54 + return LotSchema.parse({
55 + id: l.id,
56 + slug: l.slug,
57 + title,
58 + subtitle: (l.custom_subtitle || l.item?.subtitle || null)?.trim() || null,
59 + reserveMet: Boolean(l.reserve_met),
60 + hammerPrice: hp && Number.isFinite(hp) ? hp : null,
61 + currentBid: typeof l.bid_stats?.current_bid === 'number' ? l.bid_stats.current_bid : null,
62 + isGroupLot: Boolean(l.is_group_lot),
63 + auctionId: l.seller_sheet?.auction?.id ?? null,
64 + auctionEnd: l.seller_sheet?.auction?.end_date ?? null,
65 + strength: name(l.item?.strength),
66 + size: name(l.item?.size),
67 + distillery: name(l.item?.distillery),
68 + bottler: name(l.item?.bottler),
69 + region: name(l.item?.region),
70 + estimatedValue: l.item?.estimated_value ? Number(l.item.estimated_value) || null : null,
71 + image: l.photos?.[0]?.file ?? l.item?.photo?.file ?? null,
72 + });
73 +}
74 +
75 +const ddmmyyyy = (d: Date) => `${String(d.getUTCDate()).padStart(2, '0')}/${String(d.getUTCMonth() + 1).padStart(2, '0')}/${d.getUTCFullYear()}`;
76 +
77 +export function lotsUrl(from: Date, to: Date, page: number, pageSize: number): string {
78 + return `${BASE}/api/lots/?min_end_date=${encodeURIComponent(ddmmyyyy(from))}&max_end_date=${encodeURIComponent(ddmmyyyy(to))}&ordering=-price&page_size=${pageSize}&page=${page}`;
79 +}
80 +
81 +export class JustWhiskyConnector extends BaseConnector {
82 + readonly version = '1.0.0';
83 + readonly parserVersion = PARSER_VERSION;
84 + protected override minIntervalMs = 1500;
85 + override readonly urlPatterns = [/^https?:\/\/www\.just-whisky\.co\.uk\/lot\/[a-z0-9-]+/i];
86 +
87 + /** Auction windows: [start-1d, end+1d] for each completed auction (newest first). */
88 + private async windows(ctx: CrawlContext): Promise<Array<{ id: number; from: Date; to: Date; name: string }>> {
89 + const res = await ctx.fetch(`${BASE}/api/auctions/?page_size=200`, { engines: ['api'], minQuality: 0 });
90 + const data = (res.json as { data?: { results?: Array<{ id: number; name: string; start_date: string; end_date: string; is_published: boolean }> } } | null)?.data?.results ?? [];
91 + const now = Date.now();
92 + return data
93 + .filter((a) => a.is_published && new Date(a.end_date).getTime() < now && new Date(a.end_date).getUTCFullYear() >= 2013)
94 + .map((a) => ({ id: a.id, name: a.name, from: new Date(new Date(a.start_date).getTime() - 86_400_000), to: new Date(new Date(a.end_date).getTime() + 86_400_000) }))
95 + .sort((a, b) => b.to.getTime() - a.to.getTime());
96 + }
97 +
98 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
99 + const pageSize = Number(this.meta.config.pageSize ?? 200);
100 + const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 8);
101 + const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);
102 + const backfill = ctx.options.mode === 'backfill';
103 + const done = new Set<number>((ctx.options.cursor?.done as number[] | undefined) ?? []);
104 + const all = await this.windows(ctx);
105 + if (!all.length) {
106 + ctx.anomaly('empty_page', 'no auctions from /api/auctions/');
107 + return;
108 + }
109 + const todo = (backfill ? [...all].reverse() : all).filter((w) => !done.has(w.id)).slice(0, auctionsPerRun);
110 + let pages = 0;
111 + let count = 0;
112 + for (const w of todo) {
113 + let page = 1;
114 + while (pages < pagesPerRun && !ctx.signal?.aborted && !this.reached(ctx, count)) {
115 + const url = lotsUrl(w.from, w.to, page, pageSize);
116 + await this.throttle();
117 + const res = await ctx.fetch(url, {
118 + engines: ['api'],
119 + expect: ['title', 'price', 'date', 'status'],
120 + parse: (r) => {
121 + const first = (r.json as { data?: { results?: ApiLot[] } } | null)?.data?.results?.[0];
122 + return first ? { title: first.item?.title ?? first.custom_title, price: first.hammer_price ?? first.bid_stats?.current_bid, date: first.seller_sheet?.auction?.end_date, status: first.reserve_met } : null;
123 + },
124 + });
125 + pages++;
126 + const data = (res.json as { data?: { results?: ApiLot[]; count?: number; total_pages?: number } } | null)?.data;
127 + if (!res.success || !data?.results) {
128 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
129 + break;
130 + }
131 + const lots = data.results.map(trimLot).filter((x): x is Lot => Boolean(x));
132 + if (!lots.length) break;
133 + count++;
134 + yield { url, externalId: `auction:${w.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, page, count: data.count ?? null, lots }, fetchedAt: res.fetchedAt };
135 + if (!data.total_pages || page >= data.total_pages) {
136 + done.add(w.id);
137 + break;
138 + }
139 + page++;
140 + }
141 + await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() });
142 + }
143 + }
144 +
145 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
146 + const slug = url.match(/\/lot\/([a-z0-9-]+)/i)?.[1];
147 + const id = slug?.match(/(\d+)$/)?.[1];
148 + if (!id) return [];
149 + const res = await ctx.fetch(`${BASE}/api/lots/${id}/`, { engines: ['api'], minQuality: 0 });
150 + const data = (res.json as { data?: ApiLot } | null)?.data;
151 + const lot = data ? trimLot(data) : null;
152 + if (!res.success || !lot) return [];
153 + return [{ url, externalId: `lot:${id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'lots_page' as const, url, page: 0, count: 1, lots: [lot] }, fetchedAt: res.fetchedAt }];
154 + }
155 +
156 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
157 + const p = PagePayloadSchema.parse(raw.payload);
158 + const out: NormalizedRecord[] = [];
159 + for (const lot of p.lots) {
160 + const price = lot.hammerPrice ?? (lot.reserveMet ? lot.currentBid : null);
161 + if (!lot.reserveMet || !price || price <= 0 || !lot.auctionEnd) continue;
162 + const saleDate = new Date(lot.auctionEnd.endsWith('Z') ? lot.auctionEnd : `${lot.auctionEnd}Z`);
163 + if (Number.isNaN(saleDate.getTime())) continue;
164 + const fullTitle = lot.subtitle ? `${lot.title} ${lot.subtitle}` : lot.title;
165 + const f = whiskyFacts(fullTitle);
166 + const attributes = AssetAttributesSchema.parse({
167 + categorySlug: f.categorySlug,
168 + brand: lot.distillery ?? f.brand,
169 + name: fullTitle,
170 + year: f.vintage,
171 + size: lot.size && /\d/.test(lot.size) ? lot.size.replace(/\s+/g, '') : f.size,
172 + country: /scotch|islay|speyside|highland|campbeltown|lowland|scotland/i.test(`${fullTitle} ${lot.region ?? ''}`) ? 'GB' : null,
173 + identifiers: { justwhisky_lot: String(lot.id) },
174 + metadata: { age_statement: f.age, strength: lot.strength, bottler: lot.bottler, region: lot.region, estimated_value_gbp: lot.estimatedValue, auction_id: lot.auctionId, group_lot: lot.isGroupLot },
175 + });
176 + out.push(
177 + NormalizedSaleSchema.parse({
178 + kind: 'sale',
179 + connectorId: this.meta.id,
180 + sourceId: this.meta.sourceId,
181 + sourceUrl: `${BASE}/lot/${lot.slug}`,
182 + externalId: String(lot.id),
183 + rawTitle: fullTitle,
184 + imageUrls: lot.image ? [lot.image] : [],
185 + attributes,
186 + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
187 + condition: { condition: null, conditionRaw: null, completeness: null },
188 + observedAt: raw.fetchedAt,
189 + confidence: 0.9,
190 + parserVersion: PARSER_VERSION,
191 + saleType: 'auction',
192 + saleDate,
193 + price,
194 + currency: 'GBP',
195 + buyerPremiumIncluded: false,
196 + quantity: 1,
197 + isBundle: lot.isGroupLot || /\bx\s?\d|\(x\d+\)|\bset of\b/i.test(fullTitle),
198 + location: 'Scotland, United Kingdom',
199 + auctionHouse: 'Just Whisky',
200 + lotNumber: null,
201 + }),
202 + );
203 + }
204 + return out;
205 + }
206 +}
207 +
208 +export default function createConnector(meta: ConnectorMeta): JustWhiskyConnector {
209 + return new JustWhiskyConnector(meta);
210 +}
added connectors/api/just-whisky/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "just-whisky",
3 + "displayName": "Just Whisky (past auctions)",
4 + "sourceId": "just-whisky",
5 + "sourceName": "Just Whisky",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://www.just-whisky.co.uk",
8 + "module": "api/just-whisky",
9 + "enginePriority": ["api"],
10 + "categories": ["whisky", "rum", "cognac"],
11 + "regions": ["GB"],
12 + "languages": ["en"],
13 + "currency": ["GBP"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "high",
23 + "trustScore": 0.9,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.just-whisky.co.uk/terms-and-conditions",
26 + "accessNotes": "Just Whisky (Scotland, monthly online whisky auctions since 2013) renders its Past Auctions page from a public JSON API that we read directly: GET /api/lots/?min_end_date=dd/mm/yyyy&max_end_date=dd/mm/yyyy&ordering=-price&page_size=200&page=N and GET /api/auctions/ (plain HTTPS, 0 credits; robots.txt disallows only /account/ and /checkout). A lot is a sale when reserve_met is true (hammer_price is then populated); price = hammer price in GBP, buyer's commission is charged separately → buyer_premium_included=false. Sale date = the lot's auction end_date (seller_sheet.auction). Item facts (strength, size, distillery/bottler when filled) go to attributes/metadata; titles are parsed with the shared whisky heuristics. Incremental runs read the last two auction windows; backfill walks auctions oldest→newest from /api/auctions/. 1.5 s politeness.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "pageSize": 200,
31 + "pagesPerRun": 8,
32 + "auctionsPerRun": 2
33 + }
34 +}
added connectors/api/pcarmarket/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv[2] === 'capture') await captureFixture(dir, process.argv[3] ?? 'results-page', Number(process.argv[4] ?? 6));
7 +else await runSmoke(dir);
added connectors/api/pcarmarket/index.test.ts +42 −0
@@ -0,0 +1,42 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import metaJson from './meta.json' with { type: 'json' };
5 +import createConnector, { classify, trimItem, watchReference } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('pcarmarket', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('maps sold items to USD auction sales without premium', async () => {
13 + const fx = loadFixture('pcarmarket', 'results-page');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThan(0);
16 + for (const r of out) {
17 + if (r.kind !== 'sale') throw new Error('expected sale');
18 + expect(r.currency).toBe('USD');
19 + expect(r.buyerPremiumIncluded).toBe(false);
20 + expect(r.attributes.identifiers.pcarmarket_id).toMatch(/^\d+$/);
21 + expect(r.sourceUrl).toMatch(/^https:\/\/www\.pcarmarket\.com\/auction\//);
22 + expect(r.saleDate.getTime()).toBeLessThan(Date.now());
23 + }
24 + });
25 +
26 + it('classifies cars, watches and automobilia', () => {
27 + const car = trimItem({ id: 1, title: 'One-Owner 1983 Porsche 911SC Coupe', slug: 'x', vehicle: { id: 5, make: 'Porsche', model: '911SC', year: 1983, slug_model: 's' }, high_bid: 65000, end_date: '2026-09-04T15:00:07-04:00', status: 'Sold' })!;
28 + expect(classify(car)).toMatchObject({ kind: 'vehicle', categorySlug: 'automobiles', brand: 'Porsche' });
29 + const watch = trimItem({ id: 2, title: 'TAG Heuer Gulf Special Edition Watch Ref. CAVZ101 Full Set', slug: 'y', vehicle: null, high_bid: 1250, end_date: '2026-09-03T15:00:00-04:00', status: 'Sold' })!;
30 + expect(classify(watch)).toMatchObject({ kind: 'watch', categorySlug: 'other_watches' });
31 + expect(watchReference(watch.title)).toBe('CAVZ101');
32 + const rolex = trimItem({ id: 3, title: '2023 Rolex Daytona Panda Watch Ref. 116500LN Full Set', slug: 'z', vehicle: null, high_bid: 30000, end_date: '2026-09-03T15:00:00-04:00', status: 'Sold' })!;
33 + expect(classify(rolex)).toMatchObject({ kind: 'watch', categorySlug: 'rolex' });
34 + const sign = trimItem({ id: 4, title: 'No Reserve Illuminated Porsche Eye Chart Sign', slug: 'w', vehicle: null, high_bid: 250, end_date: '2026-09-03T15:00:00-04:00', status: 'Sold' })!;
35 + expect(classify(sign)).toMatchObject({ kind: 'memorabilia', categorySlug: 'automotive_memorabilia' });
36 + });
37 +
38 + it('drops unsold and priceless items', async () => {
39 + const out = await connector.normalize({ url: 'u', externalId: null, kind: 'sale', engine: 'api', fetchedAt: new Date(), payload: { kind: 'results_page', page: 1, count: 1, items: [{ id: 9, title: '1990 Porsche 964', slug: 'a', vehicle: { make: 'Porsche', model: '964', year: 1990 }, high_bid: 50000, end_date: '2026-09-01T00:00:00Z', status: 'Reserve Not Met' }] } });
40 + expect(out).toHaveLength(0);
41 + });
42 +});
added connectors/api/pcarmarket/index.ts +166 −0
@@ -0,0 +1,166 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { AssetAttributes, NormalizedRecord } from '@rareindex/shared';
4 +import { lotAttributes, makeSale, vehicleAttributes } from '../../firecrawl/_carlib/index.js';
5 +
6 +/**
7 + * PCARMARKET sold auctions (Porsche-centric enthusiast marketplace; cars, watches, automobilia).
8 + * One raw record per API page (compact items); one sale per item with status "Sold".
9 + */
10 +const BASE = 'https://www.pcarmarket.com';
11 +const API = `${BASE}/api/auctions/`;
12 +const PARSER_VERSION = '1.0.0';
13 +
14 +export const ItemSchema = z.object({
15 + id: z.number(),
16 + title: z.string(),
17 + slug: z.string(),
18 + vehicle: z.object({ make: z.string().nullable().optional(), model: z.string().nullable().optional(), year: z.number().nullable().optional() }).nullable().optional(),
19 + high_bid: z.number().nullable().optional(),
20 + end_date: z.string().nullable().optional(),
21 + status: z.string().nullable().optional(),
22 + country: z.string().nullable().optional(),
23 + zip_code: z.string().nullable().optional(),
24 + mileage_body: z.number().nullable().optional(),
25 + odometer_type: z.string().nullable().optional(),
26 + bid_count: z.number().nullable().optional(),
27 + reserve_status: z.string().nullable().optional(),
28 + is_marketplace: z.boolean().nullable().optional(),
29 + featured_image_large_url: z.string().nullable().optional(),
30 +});
31 +export type Item = z.infer<typeof ItemSchema>;
32 +export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), count: z.number().nullable(), items: z.array(ItemSchema) });
33 +
34 +const KEEP = ['id', 'title', 'slug', 'vehicle', 'high_bid', 'end_date', 'status', 'country', 'zip_code', 'mileage_body', 'odometer_type', 'bid_count', 'reserve_status', 'is_marketplace', 'featured_image_large_url'] as const;
35 +
36 +export function trimItem(raw: Record<string, unknown>): Item | null {
37 + const out: Record<string, unknown> = {};
38 + for (const k of KEEP) if (raw[k] !== undefined) out[k] = raw[k];
39 + if (out.vehicle && typeof out.vehicle === 'object') {
40 + const v = out.vehicle as Record<string, unknown>;
41 + out.vehicle = { make: v.make ?? null, model: v.model ?? null, year: v.year ?? null };
42 + }
43 + const p = ItemSchema.safeParse(out);
44 + return p.success ? p.data : null;
45 +}
46 +
47 +const WATCH_BRANDS: Array<[RegExp, string]> = [
48 + [/\brolex\b/i, 'rolex'],
49 + [/\bomega\b/i, 'omega'],
50 + [/\bpatek\b/i, 'patek_philippe'],
51 + [/\baudemars\b/i, 'audemars_piguet'],
52 + [/\b(tag heuer|heuer|tissot|panerai|cartier|breitling|iwc|tudor|hublot|zenith|seiko|grand seiko|franck muller|chopard|longines|oris|bell & ross|jaeger|montblanc|richard mille|vacheron|a\.? lange|girard|chronograph watch|watch ref)\b/i, 'other_watches'],
53 +];
54 +
55 +/** Classify a sold lot: vehicle object → car/moto; watch keywords → watch slugs; else automobilia. */
56 +export function classify(it: Item): { kind: 'vehicle' | 'watch' | 'memorabilia'; categorySlug: string; brand: string | null } {
57 + const t = it.title;
58 + if (it.vehicle && (it.vehicle.make || it.vehicle.year)) return { kind: 'vehicle', categorySlug: 'automobiles', brand: it.vehicle.make ?? null };
59 + if (/\bwatch(es)?\b|\bref\.? ?[a-z0-9.-]{4,}\b.*(full set|box)/i.test(t) || WATCH_BRANDS.slice(0, 4).some(([re]) => re.test(t) && /watch|ref\b|ref\.|full set|dial|bracelet/i.test(t))) {
60 + for (const [re, slug] of WATCH_BRANDS) if (re.test(t)) return { kind: 'watch', categorySlug: slug, brand: t.match(re)?.[0]?.replace(/\bwatch ref\b|\bchronograph watch\b/i, '').trim() || null };
61 + return { kind: 'watch', categorySlug: 'other_watches', brand: null };
62 + }
63 + return { kind: 'memorabilia', categorySlug: 'automotive_memorabilia', brand: null };
64 +}
65 +
66 +export function watchReference(title: string): string | null {
67 + const m = title.match(/\bRef\.?\s*([A-Z0-9][A-Z0-9.\-/]{3,})/i);
68 + return m ? m[1]!.replace(/[.,]$/, '') : null;
69 +}
70 +
71 +export class PcarmarketConnector extends BaseConnector {
72 + readonly version = '1.0.0';
73 + readonly parserVersion = PARSER_VERSION;
74 + protected override minIntervalMs = 1200;
75 +
76 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
77 + const limit = Number(this.meta.config.limit ?? 50);
78 + const pages = Number(this.meta.config.pagesPerRun ?? 10);
79 + const backfill = ctx.options.mode === 'backfill';
80 + const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;
81 + const newestSeen = !backfill && typeof ctx.options.cursor?.newestEnd === 'string' ? String(ctx.options.cursor.newestEnd) : '';
82 + let count = 0;
83 + let maxEnd = newestSeen;
84 + for (let page = start; page < start + pages; page++) {
85 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
86 + const url = `${API}?limit=${limit}&page=${page}&sort_by=ending_soon&status=sold&type=all`;
87 + await this.throttle();
88 + const res = await ctx.fetch(url, {
89 + engines: ['api'],
90 + expect: ['title', 'price', 'date', 'status'],
91 + parse: (r) => {
92 + const first = (r.json as { results?: Array<Record<string, unknown>> } | null)?.results?.[0];
93 + return first ? { title: first.title, price: first.high_bid, date: first.end_date, status: first.status } : null;
94 + },
95 + });
96 + const data = res.json as { results?: Array<Record<string, unknown>>; count?: number; next?: string | null } | null;
97 + if (!res.success || !data?.results) {
98 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
99 + break;
100 + }
101 + const items = data.results.map(trimItem).filter((x): x is Item => Boolean(x));
102 + if (!items.length) {
103 + ctx.anomaly('empty_page', `page ${page}`);
104 + break;
105 + }
106 + for (const it of items) if (it.end_date && it.end_date > maxEnd) maxEnd = it.end_date;
107 + count++;
108 + yield { url: `${BASE}/results/?page=${page}`, externalId: `results:${page}:${items[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_page' as const, page, count: data.count ?? null, items }, fetchedAt: res.fetchedAt };
109 + const oldest = items.map((i) => i.end_date ?? '').filter(Boolean).sort()[0] ?? '';
110 + if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });
111 + else if (newestSeen && oldest && oldest <= newestSeen) break;
112 + if (!data.next) break;
113 + }
114 + if (!backfill && maxEnd) await ctx.setCursor({ newestEnd: maxEnd, updatedAt: new Date().toISOString() });
115 + }
116 +
117 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
118 + const p = PagePayloadSchema.parse(raw.payload);
119 + const out: NormalizedRecord[] = [];
120 + for (const it of p.items) {
121 + if (it.status !== 'Sold' || !it.high_bid || it.high_bid <= 0 || !it.end_date) continue;
122 + const saleDate = new Date(it.end_date);
123 + if (Number.isNaN(saleDate.getTime())) continue;
124 + const c = classify(it);
125 + const country = it.country === 'United States of America' ? 'US' : it.country === 'Canada' ? 'CA' : (it.country ?? null);
126 + const meta = { bid_count: it.bid_count ?? null, reserve_status: it.reserve_status ?? null, mileage: it.mileage_body ?? null, odometer_type: it.odometer_type ?? null, zip_code: it.zip_code ?? null, marketplace: it.is_marketplace ?? null };
127 + let attributes: AssetAttributes;
128 + if (c.kind === 'vehicle') {
129 + attributes = vehicleAttributes(it.title, { country, identifiers: { pcarmarket_id: String(it.id) }, metadata: meta });
130 + if (it.vehicle?.make) attributes.brand = it.vehicle.make;
131 + if (it.vehicle?.model) attributes.model = it.vehicle.model;
132 + if (it.vehicle?.year) attributes.year = it.vehicle.year;
133 + } else if (c.kind === 'watch') {
134 + const ref = watchReference(it.title);
135 + attributes = lotAttributes({ categorySlug: c.categorySlug, name: it.title.replace(/^No Reserve\s+/i, ''), brand: c.brand, country, identifiers: { pcarmarket_id: String(it.id), ...(ref ? { reference: ref } : {}) }, metadata: meta });
136 + attributes.reference = ref;
137 + } else {
138 + attributes = lotAttributes({ categorySlug: 'automotive_memorabilia', name: it.title.replace(/^No Reserve\s+/i, ''), country, identifiers: { pcarmarket_id: String(it.id) }, metadata: meta });
139 + }
140 + out.push(
141 + makeSale({
142 + meta: this.meta,
143 + sourceUrl: `${BASE}/auction/${it.slug}/`,
144 + externalId: String(it.id),
145 + rawTitle: it.title,
146 + attributes,
147 + price: it.high_bid,
148 + currency: 'USD',
149 + saleDate,
150 + buyerPremiumIncluded: false,
151 + auctionHouse: 'PCARMARKET',
152 + imageUrls: it.featured_image_large_url ? [it.featured_image_large_url] : [],
153 + location: [it.zip_code, country].filter(Boolean).join(', ') || null,
154 + observedAt: raw.fetchedAt,
155 + parserVersion: PARSER_VERSION,
156 + confidence: c.kind === 'vehicle' ? 0.92 : 0.85,
157 + }),
158 + );
159 + }
160 + return out;
161 + }
162 +}
163 +
164 +export default function createConnector(meta: ConnectorMeta): PcarmarketConnector {
165 + return new PcarmarketConnector(meta);
166 +}
added connectors/api/pcarmarket/meta.json +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "id": "pcarmarket",
3 + "displayName": "PCARMARKET (sold auctions)",
4 + "sourceId": "pcarmarket",
5 + "sourceName": "PCARMARKET",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://www.pcarmarket.com",
8 + "module": "api/pcarmarket",
9 + "enginePriority": ["api"],
10 + "categories": ["automobiles", "motorcycles", "automotive_memorabilia", "rolex", "omega", "other_watches"],
11 + "regions": ["US", "CA"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 360,
22 + "priority": "high",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.pcarmarket.com/terms/",
26 + "accessNotes": "PCARMARKET's public results page (/results/) is fed by the same JSON endpoint we read: GET /api/auctions/?status=sold&type=all&sort_by=ending_soon&limit=50&page=N (plain HTTPS, 0 credits; robots.txt: 'User-agent: * Allow: /'). Each item has title, vehicle {make, model, year}, high_bid (USD winning bid), end_date, status 'Sold', country, images. Price = winning bid; PCARMARKET charges the buyer a separate 5% fee, so buyer_premium_included=false. Items with a vehicle object are cars/motorcycles; other lots are classified from the title (watches by brand keyword → watch slugs, everything else → automotive_memorabilia: signs, models, parts, helmets). Incremental runs stop at the newest end_date seen previously; backfill mode walks older pages (≈7,900 sold lots as of 2026-09).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "pagesPerRun": 10,
31 + "limit": 50
32 + }
33 +}
added connectors/api/the-market-bonhams/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv[2] === 'capture') await captureFixture(dir, process.argv[3] ?? 'results-page', Number(process.argv[4] ?? 6));
7 +else await runSmoke(dir);
added connectors/api/the-market-bonhams/index.test.ts +36 −0
@@ -0,0 +1,36 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import metaJson from './meta.json' with { type: 'json' };
5 +import createConnector, { parseResultsPage, parseSoldText } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('the-market-bonhams', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('maps result cards to sales dated by the card', async () => {
13 + const fx = loadFixture('the-market-bonhams', 'results-page');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThan(0);
16 + for (const r of out) {
17 + if (r.kind !== 'sale') throw new Error('expected sale');
18 + expect(['GBP', 'EUR', 'AUD', 'USD']).toContain(r.currency);
19 + expect(r.buyerPremiumIncluded).toBe(false);
20 + expect(['automobiles', 'motorcycles']).toContain(r.attributes.categorySlug);
21 + expect(r.attributes.identifiers.themarket_id).toMatch(/^[0-9a-f-]{36}$/);
22 + expect(r.sourceUrl).toMatch(/^https:\/\/www\.themarket\.co\.uk\/listings\//);
23 + }
24 + });
25 +
26 + it('parses sold text and cards', () => {
27 + expect(parseSoldText('Sold for £56,000 on 17 Aug 2026')).toMatchObject({ amount: 56000, currency: 'GBP', date: new Date(Date.UTC(2026, 7, 17)) });
28 + expect(parseSoldText('Sold for €25,100 on 05 Jan 2023')).toMatchObject({ amount: 25100, currency: 'EUR' });
29 + expect(parseSoldText('Sold for A$41,000 on 1 Feb 2024')).toMatchObject({ amount: 41000, currency: 'AUD' });
30 + expect(parseSoldText('Bid to £10,000')).toBeNull();
31 + const html = `<a href="/listings/range-rover/classic/4f12faa9-7308-4e71-a17f-e3ca8db972c3" data-qa="listing card"><div class="listing-card"><div class="listing-card__image"><img src="https://cdn.themarket.co.uk/x/y.jpg?optimizer=image"/></div><div class="listing-card__heading"><p class="heading-text">Sold for £56,000 on 17 Aug 2026</p></div><div class="listing-card__content"><h3 class="listing-title">1991 Range Rover Classic</h3><p class="listing-intro">£170k Restoration</p><span class="icon-text listing-location"><span class="text">THE MARKET HQ, GB</span></span></div><div class="listing-card__footer"><p class="bids-count footer__item">43 bids</p></div></div></a>`;
32 + const cards = parseResultsPage(html);
33 + expect(cards).toHaveLength(1);
34 + expect(cards[0]).toMatchObject({ id: '4f12faa9-7308-4e71-a17f-e3ca8db972c3', title: '1991 Range Rover Classic', bids: 43, location: 'THE MARKET HQ, GB', image: 'https://cdn.themarket.co.uk/x/y.jpg' });
35 + });
36 +});
added connectors/api/the-market-bonhams/index.ts +130 −0
@@ -0,0 +1,130 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { dateWords, makeSale, money, vehicleAttributes } from '../../firecrawl/_carlib/index.js';
5 +
6 +/**
7 + * The Market by Bonhams — server-rendered results grid. One raw record per results page; one sale per card
8 + * carrying "Sold for <price> on <date>".
9 + */
10 +const BASE = 'https://www.themarket.co.uk';
11 +const PARSER_VERSION = '1.0.0';
12 +
13 +export const CardSchema = z.object({ id: z.string(), url: z.string(), title: z.string(), intro: z.string().nullable(), soldText: z.string(), bids: z.number().nullable(), location: z.string().nullable(), image: z.string().nullable() });
14 +export type Card = z.infer<typeof CardSchema>;
15 +export const PagePayloadSchema = z.object({ kind: z.literal('results_page'), page: z.number(), url: z.string(), cards: z.array(CardSchema) });
16 +
17 +export function parseResultsPage(htmlText: string): Card[] {
18 + const $ = H.load(htmlText);
19 + const out: Card[] = [];
20 + $('a[data-qa="listing card"]').each((_, a) => {
21 + const href = $(a).attr('href') ?? '';
22 + const id = href.match(/\/listings\/[^/]+\/[^/]+\/([0-9a-f-]{36})/)?.[1];
23 + const title = H.text($(a).find('.listing-title'));
24 + const soldText = H.text($(a).find('.listing-card__heading .heading-text'));
25 + if (!id || !title || !soldText || !/^Sold for/i.test(soldText)) return;
26 + const bidsTxt = H.text($(a).find('.bids-count'));
27 + const img = $(a).find('img').attr('src') ?? null;
28 + out.push({
29 + id,
30 + url: BASE + href,
31 + title,
32 + intro: H.text($(a).find('.listing-intro')),
33 + soldText,
34 + bids: bidsTxt ? Number(bidsTxt.replace(/\D/g, '')) || null : null,
35 + location: H.text($(a).find('.listing-location .text')),
36 + image: img ? img.replace(/\?.*$/, '') : null,
37 + });
38 + });
39 + return out;
40 +}
41 +
42 +/** "Sold for £56,000 on 17 Aug 2026" → price/currency/date. */
43 +export function parseSoldText(s: string): { amount: number; currency: 'GBP' | 'EUR' | 'AUD' | 'USD'; date: Date } | null {
44 + const m = s.match(/Sold for\s+(A\$|US\$|\$|£|€)\s?([\d,]+(?:\.\d+)?)\s+on\s+(.+)$/i);
45 + if (!m) return null;
46 + const sym = m[1]!;
47 + const currency = sym === '£' ? 'GBP' : sym === '€' ? 'EUR' : sym === 'A$' ? 'AUD' : 'USD';
48 + const amt = money(`${m[2]} ${currency}`, currency);
49 + const date = dateWords(m[3]!);
50 + if (!amt || !date) return null;
51 + return { amount: amt.amount, currency, date };
52 +}
53 +
54 +export class TheMarketConnector extends BaseConnector {
55 + readonly version = '1.0.0';
56 + readonly parserVersion = PARSER_VERSION;
57 + protected override minIntervalMs = 1500;
58 +
59 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
60 + const pages = Number(this.meta.config.pagesPerRun ?? 8);
61 + const backfill = ctx.options.mode === 'backfill';
62 + const start = backfill ? Number(ctx.options.cursor?.nextPage ?? 1) : 1;
63 + const newestSeen = !backfill && typeof ctx.options.cursor?.newestDate === 'string' ? String(ctx.options.cursor.newestDate) : '';
64 + let maxDate = newestSeen;
65 + let count = 0;
66 + for (let page = start; page < start + pages; page++) {
67 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
68 + const url = `${BASE}/auctions/results?page=${page}`;
69 + await this.throttle();
70 + const res = await ctx.fetch(url, {
71 + engines: ['api'],
72 + responseType: 'text',
73 + expect: ['title', 'price', 'date', 'status'],
74 + parse: (r) => {
75 + const c = r.html ? parseResultsPage(r.html) : [];
76 + const p = c[0] ? parseSoldText(c[0].soldText) : null;
77 + return c.length ? { title: c[0]!.title, price: p?.amount ?? null, date: p?.date ?? null, status: 'sold' } : null;
78 + },
79 + });
80 + const cards = res.success && res.html ? parseResultsPage(res.html) : [];
81 + if (!cards.length) {
82 + ctx.anomaly(res.success ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
83 + break;
84 + }
85 + const dates = cards.map((c) => parseSoldText(c.soldText)?.date.toISOString() ?? '').filter(Boolean);
86 + for (const d of dates) if (d > maxDate) maxDate = d;
87 + count++;
88 + yield { url, externalId: `results:${page}:${cards[0]!.id}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_page' as const, page, url, cards }, fetchedAt: res.fetchedAt };
89 + const oldest = dates.sort()[0] ?? '';
90 + if (backfill) await ctx.setCursor({ nextPage: page + 1, updatedAt: new Date().toISOString() });
91 + else if (newestSeen && oldest && oldest <= newestSeen) break;
92 + }
93 + if (!backfill && maxDate) await ctx.setCursor({ newestDate: maxDate, updatedAt: new Date().toISOString() });
94 + }
95 +
96 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
97 + const p = PagePayloadSchema.parse(raw.payload);
98 + const out: NormalizedRecord[] = [];
99 + for (const c of p.cards) {
100 + const sold = parseSoldText(c.soldText);
101 + if (!sold) continue;
102 + const country = c.location?.match(/,\s*([A-Z]{2})$/)?.[1] ?? null;
103 + const attributes = vehicleAttributes(c.title, { country, identifiers: { themarket_id: c.id }, metadata: { intro: c.intro, bids: c.bids, location: c.location } });
104 + out.push(
105 + makeSale({
106 + meta: this.meta,
107 + sourceUrl: c.url,
108 + externalId: c.id,
109 + rawTitle: c.title,
110 + attributes,
111 + price: sold.amount,
112 + currency: sold.currency,
113 + saleDate: sold.date,
114 + buyerPremiumIncluded: false,
115 + auctionHouse: 'The Market by Bonhams',
116 + imageUrls: c.image ? [c.image] : [],
117 + location: c.location,
118 + description: c.intro,
119 + observedAt: raw.fetchedAt,
120 + parserVersion: PARSER_VERSION,
121 + }),
122 + );
123 + }
124 + return out;
125 + }
126 +}
127 +
128 +export default function createConnector(meta: ConnectorMeta): TheMarketConnector {
129 + return new TheMarketConnector(meta);
130 +}
added connectors/api/the-market-bonhams/meta.json +32 −0
@@ -0,0 +1,32 @@
1 +{
2 + "id": "the-market-bonhams",
3 + "displayName": "The Market by Bonhams (results)",
4 + "sourceId": "the-market-bonhams",
5 + "sourceName": "The Market by Bonhams",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://www.themarket.co.uk",
8 + "module": "api/the-market-bonhams",
9 + "enginePriority": ["api"],
10 + "categories": ["automobiles", "motorcycles"],
11 + "regions": ["GB", "EU", "AU"],
12 + "languages": ["en"],
13 + "currency": ["GBP", "EUR", "AUD", "USD"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.themarket.co.uk/terms-and-conditions",
26 + "accessNotes": "The Market by Bonhams (online collector-car auctions, UK/EU/AU) publishes results server-side at /auctions/results?page=N (18 cards per page, ≈330 pages back to 2019; plain HTTPS with the RareIndex user agent; robots.txt 'User-agent: * Allow: /'). Each card: title, 'Sold for £56,000 on 17 Aug 2026' (native currency symbol £/€/A$), listing URL, bid count, location. Price = winning bid; The Market charges the buyer a separate fee, so buyer_premium_included=false. Sale date = the date printed on the card. Incremental runs stop once cards older than the newest previously seen date appear; backfill walks older pages. 1.5 s politeness.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "pagesPerRun": 8
31 + }
32 +}
added connectors/firecrawl/hdh-wine/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv[2] === 'capture') await captureFixture(dir, process.argv[3] ?? 'results-pdf', Number(process.argv[4] ?? 12));
7 +else await runSmoke(dir);
added connectors/firecrawl/hdh-wine/index.test.ts +49 −0
@@ -0,0 +1,49 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { ConnectorMetaSchema } from '@rareindex/connectors';
3 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
4 +import metaJson from './meta.json' with { type: 'json' };
5 +import createConnector, { parseArchive, parseResultsPdf, wineFacts } from './index.js';
6 +
7 +const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
8 +
9 +describe('hdh-wine', () => {
10 + runFixtureSuite(connector, it, expect);
11 +
12 + it('maps PDF rows to USD hammer sales, grouping multi-row lots', async () => {
13 + const fx = loadFixture('hdh-wine', 'results-pdf');
14 + const out = await connector.normalize(fx.raw);
15 + expect(out.length).toBeGreaterThan(0);
16 + for (const r of out) {
17 + if (r.kind !== 'sale') throw new Error('expected sale');
18 + expect(r.currency).toBe('USD');
19 + expect(r.buyerPremiumIncluded).toBe(false);
20 + expect(['wine', 'whisky', 'cognac']).toContain(r.attributes.categorySlug);
21 + expect(r.attributes.identifiers.hdh_lot).toMatch(/:\d+/);
22 + expect(r.auctionHouse).toBe('Hart Davis Hart Wine Co.');
23 + }
24 + });
25 +
26 + it('parses the PDF table, header date and archive links', () => {
27 + const md = `## Aggregate: 3,892,330 August 27-28, 2026\n\n| Lot | Qty | Description | Estimate | Hammer | Aggregate |\n| --- | --- | --- | --- | --- | --- |\n| 1 | 6 | 1982 Château Ducru-Beaucaillou | 1,600 - 2,400 | 2,200 | 2,629.00 |\n| 5 | 1 | 2009 Château Cos d'Estournel | 500 - 750 | 900 | 1,075.50 |\n| 5 | 1 | 2010 Château Léoville Poyferré | | | |\n| 8 | 1 | 2009 Château d'Yquem (375ml) | 280 - 420 | 500 | 597.50 |\n`;
28 + const p = parseResultsPdf(md);
29 + expect(p.saleDate).toBe('2026-08-27T00:00:00.000Z');
30 + expect(p.rows).toHaveLength(4);
31 + expect(p.rows[0]).toMatchObject({ lot: '1', qty: 6, hammer: 2200, aggregate: 2629 });
32 + expect(p.rows[2]!.hammer).toBeNull();
33 + expect(wineFacts('2009 Château d\'Yquem (375ml)')).toMatchObject({ vintage: 2009, size: '375ml', name: "Château d'Yquem", categorySlug: 'wine' });
34 + expect(wineFacts('2017 La Tâche, Domaine de la Romanée-Conti')).toMatchObject({ producer: 'Domaine de la Romanée-Conti' });
35 + const arch = parseArchive('**An Auction of Finest & Rarest Wines**\n\nJune 25 & 26, 2026\n\n**100% SOLD**\n\n[View auction results](https://hdhauctions.com/wp-content/uploads/2026/07/2606_AuctionResults_PDF.pdf)');
36 + expect(arch).toHaveLength(1);
37 + expect(arch[0]).toMatchObject({ pdfUrl: 'https://hdhauctions.com/wp-content/uploads/2026/07/2606_AuctionResults_PDF.pdf', title: 'An Auction of Finest & Rarest Wines', dateText: 'June 25 & 26, 2026' });
38 + });
39 +
40 + it('groups a two-row lot into one bundle sale', async () => {
41 + const out = await connector.normalize({ url: 'https://hdhauctions.com/wp-content/uploads/2026/08/2608_AuctionResults.pdf', externalId: null, kind: 'sale', engine: 'firecrawl', fetchedAt: new Date(), payload: { kind: 'results_pdf', url: 'https://hdhauctions.com/wp-content/uploads/2026/08/2608_AuctionResults.pdf', auctionName: 'x', saleDate: '2026-08-27T00:00:00.000Z', rows: [{ lot: '5', qty: 1, description: "2009 Château Cos d'Estournel", estimate: '500 - 750', hammer: 900, aggregate: 1075.5 }, { lot: '5', qty: 1, description: '2010 Château Montrose', estimate: null, hammer: null, aggregate: null }] } });
42 + expect(out).toHaveLength(1);
43 + const s = out[0]!;
44 + if (s.kind !== 'sale') throw new Error('sale');
45 + expect(s.isBundle).toBe(true);
46 + expect(s.price).toBe(900);
47 + expect(s.lotNumber).toBe('5');
48 + });
49 +});
added connectors/firecrawl/hdh-wine/index.ts +163 −0
@@ -0,0 +1,163 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import { AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';
4 +import { dateWords } from '../_carlib/index.js';
5 +
6 +/**
7 + * Hart Davis Hart prices realized (PDF per auction, parsed to markdown tables by Firecrawl).
8 + * One raw record per PDF (compact rows); one sale per lot with a hammer price.
9 + */
10 +const PARSER_VERSION = '1.0.0';
11 +
12 +export const RowSchema = z.object({ lot: z.string(), qty: z.number().nullable(), description: z.string(), estimate: z.string().nullable(), hammer: z.number().nullable(), aggregate: z.number().nullable() });
13 +export type Row = z.infer<typeof RowSchema>;
14 +export const PdfPayloadSchema = z.object({ kind: z.literal('results_pdf'), url: z.string(), auctionName: z.string().nullable(), saleDate: z.string().nullable(), rows: z.array(RowSchema) });
15 +export type PdfPayload = z.infer<typeof PdfPayloadSchema>;
16 +
17 +const num = (s: string | undefined): number | null => {
18 + const v = Number((s ?? '').replace(/[,$\s]/g, ''));
19 + return Number.isFinite(v) && v > 0 ? v : null;
20 +};
21 +
22 +/** Archive page markdown → [{ pdfUrl, title, dateText }] newest first. */
23 +export function parseArchive(md: string): Array<{ pdfUrl: string; title: string | null; dateText: string | null }> {
24 + const out: Array<{ pdfUrl: string; title: string | null; dateText: string | null }> = [];
25 + const seen = new Set<string>();
26 + for (const m of md.matchAll(/\[View auction results\]\((https:\/\/hdhauctions\.com\/wp-content\/uploads\/[^)\s]+\.pdf)\)/gi)) {
27 + const url = m[1]!;
28 + if (seen.has(url)) continue;
29 + seen.add(url);
30 + const before = md.slice(Math.max(0, m.index! - 1200), m.index);
31 + const title = [...before.matchAll(/\*\*([^*\n]{6,120})\*\*/g)].map((x) => x[1]!.trim()).find((t) => !/SOLD|\$|estimate/i.test(t)) ?? null;
32 + const dateText = [...before.matchAll(/([A-Z][a-z]+ \d{1,2}(?:\s*[-–&]\s*\d{1,2})?,? \d{4})/g)].at(-1)?.[1] ?? null;
33 + out.push({ pdfUrl: url, title, dateText });
34 + }
35 + return out;
36 +}
37 +
38 +/** PDF markdown → header facts + table rows (Lot | Qty | Description | Estimate | Hammer | Aggregate). */
39 +export function parseResultsPdf(md: string): { auctionName: string | null; saleDate: string | null; rows: Row[] } {
40 + const rows: Row[] = [];
41 + for (const line of md.split('\n')) {
42 + const m = line.match(/^\|\s*(\d+[A-Z]?)\s*\|\s*(\d*)\s*\|\s*(.+?)\s*\|\s*([\d,]*\s*-?\s*[\d,]*)\s*\|\s*([\d,]*)\s*\|\s*([\d,.]*)\s*\|/);
43 + if (!m) continue;
44 + const description = m[3]!.replace(/\\/g, '').trim();
45 + if (!description || /^Description$/i.test(description)) continue;
46 + rows.push({ lot: m[1]!, qty: m[2] ? Number(m[2]) : null, description, estimate: m[4]?.trim() || null, hammer: num(m[5]), aggregate: num(m[6]) });
47 + }
48 + const header = md.slice(0, 4000);
49 + const dateText = header.match(/([A-Z][a-z]+ \d{1,2}(?:\s*[-–]\s*\d{1,2})?,? \d{4})/)?.[1] ?? null;
50 + const saleDate = dateText ? dateWords(dateText)?.toISOString() ?? null : null;
51 + const auctionName = header.match(/^#+\s*(.+)$/m)?.[1]?.replace(/Aggregate:.*$/, '').trim() || null;
52 + return { auctionName, saleDate, rows };
53 +}
54 +
55 +const SIZE_RE = /\((\d+(?:\.\d+)?\s?(?:ml|L|l|cl))\)/;
56 +/** "2010 Château Ducru-Beaucaillou" → vintage, producer/name, bottle size. */
57 +export function wineFacts(description: string): { vintage: number | null; name: string; size: string; producer: string | null; categorySlug: 'wine' | 'whisky' | 'cognac' } {
58 + const vintage = Number(description.match(/^((?:18|19|20)\d{2})\b/)?.[1]) || null;
59 + const sizeM = description.match(SIZE_RE);
60 + const size = sizeM ? sizeM[1]!.replace(/\s/g, '') : '750ml';
61 + const name = description.replace(/^(?:18|19|20)\d{2}\s+/, '').replace(SIZE_RE, '').replace(/\s+/g, ' ').trim();
62 + const producer = name.includes(',') ? name.split(',').at(-1)!.trim() : name.split(/\s+/).slice(0, 3).join(' ');
63 + const categorySlug = /whisky|whiskey|scotch|bourbon/i.test(description) ? 'whisky' : /cognac|armagnac/i.test(description) ? 'cognac' : 'wine';
64 + return { vintage, name, size, producer: producer || null, categorySlug };
65 +}
66 +
67 +export class HdhWineConnector extends BaseConnector {
68 + readonly version = '1.0.0';
69 + readonly parserVersion = PARSER_VERSION;
70 + protected override minIntervalMs = 3000;
71 +
72 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
73 + const archiveUrl = String(this.meta.config.archiveUrl ?? 'https://hdhauctions.com/auction-archives/');
74 + const perRun = Number(this.meta.config.pdfsPerRun ?? 1);
75 + const done = new Set<string>((ctx.options.cursor?.done as string[] | undefined) ?? []);
76 + const archive = await ctx.fetch(archiveUrl, { engines: ['firecrawl'], expect: ['title'], parse: (r) => (r.markdown ? { title: parseArchive(r.markdown)[0]?.pdfUrl ?? null } : null) });
77 + const entries = archive.success && archive.markdown ? parseArchive(archive.markdown) : [];
78 + if (!entries.length) {
79 + ctx.anomaly('page_fetch_failed', `archive: ${archive.error ?? archive.httpStatus}`);
80 + return;
81 + }
82 + const order = ctx.options.mode === 'backfill' ? [...entries].reverse() : entries;
83 + let count = 0;
84 + for (const e of order.filter((x) => !done.has(x.pdfUrl)).slice(0, perRun)) {
85 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
86 + await this.throttle();
87 + const res = await ctx.fetch(e.pdfUrl, { engines: ['firecrawl'], timeoutMs: 180_000, expect: ['title', 'price', 'date'], parse: (r) => {
88 + const p = r.markdown ? parseResultsPdf(r.markdown) : null;
89 + const sold = p?.rows.find((x) => x.hammer);
90 + return p && p.rows.length ? { title: sold?.description ?? p.rows[0]!.description, price: sold?.hammer ?? null, date: p.saleDate ?? e.dateText } : null;
91 + } });
92 + const parsed = res.success && res.markdown ? parseResultsPdf(res.markdown) : null;
93 + if (!parsed || !parsed.rows.length) {
94 + ctx.anomaly(parsed ? 'empty_page' : 'page_fetch_failed', `${e.pdfUrl}: ${res.error ?? res.httpStatus}`);
95 + continue;
96 + }
97 + const saleDate = parsed.saleDate ?? (e.dateText ? dateWords(e.dateText)?.toISOString() ?? null : null);
98 + count++;
99 + yield { url: e.pdfUrl, externalId: `pdf:${e.pdfUrl.split('/').pop()}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'results_pdf' as const, url: e.pdfUrl, auctionName: e.title ?? parsed.auctionName, saleDate, rows: parsed.rows }, fetchedAt: res.fetchedAt };
100 + done.add(e.pdfUrl);
101 + await ctx.setCursor({ done: [...done], updatedAt: new Date().toISOString() });
102 + }
103 + }
104 +
105 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
106 + const p = PdfPayloadSchema.parse(raw.payload);
107 + if (!p.saleDate) return [];
108 + const saleDate = new Date(p.saleDate);
109 + const pdfName = p.url.split('/').pop()!.replace(/\.pdf$/i, '');
110 + // Group multi-row lots: the first row of a lot carries the prices; following rows list the other wines.
111 + const groups = new Map<string, Row[]>();
112 + for (const r of p.rows) (groups.get(r.lot) ?? groups.set(r.lot, []).get(r.lot)!).push(r);
113 + const out: NormalizedRecord[] = [];
114 + for (const [lot, rows] of groups) {
115 + const head = rows.find((r) => r.hammer) ?? rows[0]!;
116 + if (!head.hammer || head.hammer <= 0) continue;
117 + const f = wineFacts(head.description);
118 + const bundle = rows.length > 1;
119 + const title = bundle ? `${head.description} (+${rows.length - 1} more)` : head.description;
120 + const attributes = AssetAttributesSchema.parse({
121 + categorySlug: f.categorySlug,
122 + brand: f.producer,
123 + name: f.name,
124 + year: f.vintage,
125 + size: f.size,
126 + identifiers: { hdh_lot: `${pdfName}:${lot}` },
127 + metadata: { auction: p.auctionName, estimate: head.estimate, aggregate_usd: head.aggregate, bottles: head.qty, lines: rows.map((r) => `${r.qty ?? ''} × ${r.description}`.trim()) },
128 + });
129 + out.push(
130 + NormalizedSaleSchema.parse({
131 + kind: 'sale',
132 + connectorId: this.meta.id,
133 + sourceId: this.meta.sourceId,
134 + sourceUrl: `${p.url}#lot-${lot}`,
135 + externalId: `${pdfName}:${lot}`,
136 + rawTitle: title,
137 + imageUrls: [],
138 + attributes,
139 + grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
140 + condition: { condition: null, conditionRaw: null, completeness: null },
141 + observedAt: raw.fetchedAt,
142 + confidence: 0.92,
143 + parserVersion: PARSER_VERSION,
144 + saleType: 'auction',
145 + saleDate,
146 + price: head.hammer,
147 + currency: 'USD',
148 + buyerPremiumIncluded: false,
149 + quantity: head.qty ?? 1,
150 + isBundle: bundle || (head.qty ?? 1) > 1,
151 + location: 'Chicago, IL, United States',
152 + auctionHouse: 'Hart Davis Hart Wine Co.',
153 + lotNumber: lot,
154 + }),
155 + );
156 + }
157 + return out;
158 + }
159 +}
160 +
161 +export default function createConnector(meta: ConnectorMeta): HdhWineConnector {
162 + return new HdhWineConnector(meta);
163 +}
added connectors/firecrawl/hdh-wine/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/whisky-hammer/_smoke.ts +7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/whisky-hammer/index.test.ts +36 −0
Large diff (36 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/whisky-hammer/index.ts +162 −0
Large diff (162 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/whisky-hammer/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
modified connectors/registry.json +2158 −136
Large diff (2,492 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/gooding/realized-auction.json +88 −0
Large diff (89 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/hdh-wine/results-pdf.json +345 −0
Large diff (346 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/historics/lots-page.json +92 −0
Large diff (93 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/just-whisky/lots-page.json +177 −0
Large diff (178 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/pcarmarket/results-page.json +172 −0
Large diff (173 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/the-market-bonhams/results-page.json +84 −0
Large diff (85 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/whisky-hammer/lot-page.json +81 −0
Large diff (83 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere