spb/rareindex

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

added connectors/api/idealwine/_smoke.ts +19 −0

		@@ -0,0 +1,19 @@
	1	+import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
	2	+import { saveFixture } from '@rareindex/connectors/testing';
	3	+import metaJson from './meta.json' with { type: 'json' };
	4	+import createConnector from './index.js';
	5	+
	6	+const save = process.argv.includes('--save');
	7	+const meta = ConnectorMetaSchema.parse(metaJson);
	8	+const connector = createConnector(meta);
	9	+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
	10	+const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2 } });
	11	+let i = 0;
	12	+for await (const raw of connector.crawl(ctx)) {
	13	+ const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
	14	+ console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} records (${out.filter((r) => r.kind === 'price_observation').length} observations)`);
	15	+ for (const r of out.filter((r) => r.kind === 'price_observation').slice(0, 3)) console.log(' ', JSON.stringify(r).slice(0, 420));
	16	+ if (save && i < 2) saveFixture(meta.id, i === 0 ? 'bordeaux-page-1' : 'bordeaux-page-2', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 40, kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.name'] }, note: 'Captured live from idealwine.com /en/cote/bordeaux (plain HTTPS)' });
	17	+ i++;
	18	+}
	19	+console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);

added connectors/api/idealwine/index.test.ts +43 −0

		@@ -0,0 +1,43 @@
	1	+import { describe, expect, it } from 'vitest';
	2	+import { ConnectorMetaSchema } from '@rareindex/connectors';
	3	+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	4	+import metaJson from './meta.json' with { type: 'json' };
	5	+import createConnector, { parseCotePage, producerFromSlug } from './index.js';
	6	+
	7	+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
	8	+
	9	+describe('idealwine', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('emits a catalog item + EUR guide_value observation per wine vintage', async () => {
	13	+ const fx = loadFixture('idealwine', 'bordeaux-page-1');
	14	+ const out = await connector.normalize(fx.raw);
	15	+ const obs = out.filter((r) => r.kind === 'price_observation');
	16	+ const items = out.filter((r) => r.kind === 'catalog_item');
	17	+ expect(obs.length).toBe(items.length);
	18	+ expect(obs.length).toBeGreaterThan(20);
	19	+ for (const r of obs) {
	20	+ if (r.kind !== 'price_observation') continue;
	21	+ expect(r.currency).toBe('EUR');
	22	+ expect(r.priceKind).toBe('guide_value');
	23	+ expect(r.price).toBeGreaterThan(0);
	24	+ expect(r.attributes.categorySlug).toBe('wine');
	25	+ expect(r.attributes.set).toBeTruthy();
	26	+ expect(r.attributes.identifiers.idealwine_ref).toMatch(/^\d+-/);
	27	+ expect(r.sourceUrl).toMatch(/^https:\/\/www\.idealwine\.com\/en\/wine-prices\//);
	28	+ }
	29	+ });
	30	+
	31	+ it('parses cards and producer slugs', () => {
	32	+ const html = `<div class="cote_gridItem___dC3b"><div class="cote_breadCrumpRegion__xzjeq">Bordeaux<!-- --> &gt; Pessac-Léognan</div><div class="cote_nameWine__v1UwE">Abeille de Fieuzal<!-- --> <span class="cote_colorDisplay__VWf8N">(<!-- -->Red<!-- -->)</span></div><img src="https://www.idealwine.com/media/x.jpg"/><div class="VintageRatingsCards_container__m_TZj"><a href="/en/wine-prices/120055-2021-Bottle-Bordeaux-Pessac-Leognan-Chateau-de-Fieuzal-red"><div class="VintageRatingsCards_cote__H0Z2A">2021<!-- --> <span><span class="Price_units__UHpM6">€<!-- -->13</span></span></div></a><a href="/en/wine-prices/120055-2020-Bottle-Bordeaux-Pessac-Leognan-Chateau-de-Fieuzal-red"><div class="VintageRatingsCards_cote__H0Z2A">2020 <span><span>€1,250</span></span></div></a></div></div><a href="/en/cote/bordeaux?page=58">58</a>`;
	33	+ const p = parseCotePage(html, 'u', 'bordeaux', 1);
	34	+ expect(p.totalPages).toBe(58);
	35	+ expect(p.wines).toHaveLength(1);
	36	+ expect(p.wines[0]).toMatchObject({ wineId: '120055', name: 'Abeille de Fieuzal', color: 'Red', region: 'Bordeaux', appellation: 'Pessac-Léognan' });
	37	+ expect(p.wines[0]!.vintages).toEqual([
	38	+ { vintage: 2021, priceEur: 13, url: 'https://www.idealwine.com/en/wine-prices/120055-2021-Bottle-Bordeaux-Pessac-Leognan-Chateau-de-Fieuzal-red', ref: '120055-2021' },
	39	+ { vintage: 2020, priceEur: 1250, url: 'https://www.idealwine.com/en/wine-prices/120055-2020-Bottle-Bordeaux-Pessac-Leognan-Chateau-de-Fieuzal-red', ref: '120055-2020' },
	40	+ ]);
	41	+ expect(producerFromSlug('Bordeaux-Pessac-Leognan-Chateau-de-Fieuzal-red')).toBe('Bordeaux Pessac Leognan Chateau de Fieuzal');
	42	+ });
	43	+});

added connectors/api/idealwine/index.ts +149 −0

		@@ -0,0 +1,149 @@
	1	+import { z } from 'zod';
	2	+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';
	4	+
	5	+/**
	6	+ * iDealwine "cote" (Price Estimate) pages — per-vintage auction-derived reference prices for
	7	+ * French and Italian wines. One raw record per index page; normalise → one catalog_item per
	8	+ * wine-vintage plus one guide_value observation (EUR / bottle).
	9	+ */
	10	+
	11	+const BASE = 'https://www.idealwine.com';
	12	+const PARSER_VERSION = '1.0.0';
	13	+
	14	+export const VintageSchema = z.object({ vintage: z.number().nullable(), priceEur: z.number(), url: z.string(), ref: z.string() });
	15	+export const WineSchema = z.object({ wineId: z.string(), name: z.string(), color: z.string().nullable(), region: z.string().nullable(), appellation: z.string().nullable(), producerSlug: z.string().nullable(), image: z.string().nullable(), vintages: z.array(VintageSchema) });
	16	+export const PagePayloadSchema = z.object({ kind: z.literal('cote_page'), url: z.string(), region: z.string(), page: z.number(), totalPages: z.number().nullable(), wines: z.array(WineSchema) });
	17	+export type PagePayload = z.infer<typeof PagePayloadSchema>;
	18	+
	19	+/** Parse a /en/cote/<region> page (cards with per-vintage estimates). */
	20	+export function parseCotePage(htmlText: string, url: string, region: string, page: number): PagePayload {
	21	+ const $ = H.load(htmlText);
	22	+ const wines: z.infer<typeof WineSchema>[] = [];
	23	+ const pages = [...htmlText.matchAll(/\/en\/cote\/[a-z-]+\?page=(\d+)/g)].map((m) => Number(m[1]));
	24	+ const totalPages = pages.length ? Math.max(...pages) : null;
	25	+ $('[class*="cote_gridItem"]').each((_, el) => {
	26	+ const $el = $(el);
	27	+ const crumb = H.text($el.find('[class*="cote_breadCrumpRegion"]')) ?? '';
	28	+ const [reg, app] = crumb.split('>').map((s) => s.trim());
	29	+ const nameEl = $el.find('[class*="cote_nameWine"]').first();
	30	+ const color = H.text(nameEl.find('[class*="cote_colorDisplay"]'))?.replace(/[()]/g, '').trim() ?? null;
	31	+ nameEl.find('[class*="cote_colorDisplay"]').remove();
	32	+ const name = H.text(nameEl);
	33	+ const image = $el.find('img').attr('src') ?? null;
	34	+ const vintages: z.infer<typeof VintageSchema>[] = [];
	35	+ let wineId: string | null = null;
	36	+ let producerSlug: string | null = null;
	37	+ $el.find('a[href*="/en/wine-prices/"]').each((__, a) => {
	38	+ const href = $(a).attr('href') ?? '';
	39	+ const m = href.match(/\/en\/wine-prices\/(\d+)-(\d{4}|NV|[^-]+)-Bottle-(.+)$/i);
	40	+ const id = href.match(/\/en\/wine-prices\/(\d+)-/)?.[1] ?? null;
	41	+ const vTxt = H.text($(a).find('[class*="VintageRatingsCards_cote"]')) ?? '';
	42	+ const vintage = vTxt.match(/^(\d{4})/)?.[1] ?? null;
	43	+ const priceTxt = vTxt.match(/€\s*[\d\s.,]+/)?.[0] ?? null;
	44	+ const price = priceTxt ? parsePrice(priceTxt.replace(/\s/g, ''), 'EUR')?.amount ?? null : null;
	45	+ if (!id || !price) return;
	46	+ wineId ??= id;
	47	+ if (!producerSlug && m?.[3]) producerSlug = m[3]!;
	48	+ vintages.push({ vintage: vintage ? Number(vintage) : null, priceEur: price, url: BASE + href, ref: href.replace(/^\/en\/wine-prices\//, '').split('-Bottle-')[0]! });
	49	+ });
	50	+ if (!name || !wineId || vintages.length === 0) return;
	51	+ wines.push({ wineId, name, color, region: reg ?? null, appellation: app ?? null, producerSlug, image: image && !image.includes('_no_picture') ? (image.startsWith('http') ? image : BASE + image) : null, vintages });
	52	+ });
	53	+ return { kind: 'cote_page', url, region, page, totalPages, wines };
	54	+}
	55	+
	56	+/** "Chateau-de-Fieuzal-red" → "Château de Fieuzal" is not recoverable exactly; keep a readable producer label without guessing accents. */
	57	+export function producerFromSlug(slug: string | null): string | null {
	58	+ if (!slug) return null;
	59	+ const s = slug.replace(/-(red|white|rose|sparkling|sweet)$/i, '').replace(/^(Second-Wine-|Cru-bourgeois-)/i, '');
	60	+ const label = s.replace(/-/g, ' ').trim();
	61	+ return label || null;
	62	+}
	63	+
	64	+export class IdealwineConnector extends BaseConnector {
	65	+ readonly version = '1.0.0';
	66	+ readonly parserVersion = PARSER_VERSION;
	67	+ protected override minIntervalMs = 2000;
	68	+
	69	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	70	+ const regions = z.array(z.string()).parse(this.meta.config.regions ?? ['bordeaux']);
	71	+ const pages = Number(this.meta.config.pagesPerRegion ?? 3);
	72	+ const cursor = (ctx.options.cursor ?? {}) as { next?: Record<string, number> };
	73	+ const next: Record<string, number> = { ...(cursor.next ?? {}) };
	74	+ let count = 0;
	75	+ for (const region of regions) {
	76	+ const start = ctx.options.mode === 'backfill' ? (next[region] ?? 1) : 1;
	77	+ for (let page = start; page < start + pages; page++) {
	78	+ if (ctx.signal?.aborted || this.reached(ctx, count)) return void (await ctx.setCursor({ next }));
	79	+ const url = `${BASE}/en/cote/${region}${page > 1 ? `?page=${page}` : ''}`;
	80	+ await this.throttle();
	81	+ const res = await ctx.fetch(url, {
	82	+ engines: ['api'],
	83	+ responseType: 'text',
	84	+ expect: ['title', 'price', 'currency'],
	85	+ parse: (r) => {
	86	+ const p = r.html ? parseCotePage(r.html, url, region, page) : null;
	87	+ const w = p?.wines[0];
	88	+ return w ? { title: w.name, price: w.vintages[0]?.priceEur ?? null, currency: 'EUR' } : null;
	89	+ },
	90	+ });
	91	+ const payload = res.success && res.html ? parseCotePage(res.html, url, region, page) : null;
	92	+ if (!payload || payload.wines.length === 0) {
	93	+ ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
	94	+ next[region] = 1;
	95	+ break;
	96	+ }
	97	+ count++;
	98	+ yield { url, externalId: `cote:${region}:${page}`, kind: 'price_observation', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	99	+ next[region] = payload.totalPages && page >= payload.totalPages ? 1 : page + 1;
	100	+ }
	101	+ }
	102	+ await ctx.setCursor({ next });
	103	+ }
	104	+
	105	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	106	+ const p = PagePayloadSchema.parse(raw.payload);
	107	+ const out: NormalizedRecord[] = [];
	108	+ const observationDate = new Date(Date.UTC(raw.fetchedAt.getUTCFullYear(), raw.fetchedAt.getUTCMonth(), raw.fetchedAt.getUTCDate()));
	109	+ for (const w of p.wines) {
	110	+ for (const v of w.vintages) {
	111	+ const attributes = AssetAttributesSchema.parse({
	112	+ categorySlug: 'wine',
	113	+ brand: producerFromSlug(w.producerSlug),
	114	+ series: w.region,
	115	+ set: w.appellation,
	116	+ name: w.name,
	117	+ year: v.vintage,
	118	+ color: w.color,
	119	+ size: '75cl',
	120	+ country: /italie|italy/i.test(p.region) ? 'IT' : 'FR',
	121	+ identifiers: { idealwine_wine_id: w.wineId, idealwine_ref: v.ref },
	122	+ metadata: { region_page: p.region },
	123	+ });
	124	+ const base = {
	125	+ connectorId: this.meta.id,
	126	+ sourceId: this.meta.sourceId,
	127	+ sourceUrl: v.url,
	128	+ externalId: v.ref,
	129	+ rawTitle: `${w.name}${v.vintage ? ` ${v.vintage}` : ''}${w.appellation ? ` · ${w.appellation}` : ''}${w.color ? ` (${w.color})` : ''}`,
	130	+ description: null,
	131	+ imageUrls: w.image ? [w.image] : [],
	132	+ attributes,
	133	+ grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
	134	+ condition: { condition: null, conditionRaw: null, completeness: null },
	135	+ observedAt: raw.fetchedAt,
	136	+ confidence: 0.75,
	137	+ parserVersion: PARSER_VERSION,
	138	+ };
	139	+ out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, releaseDate: null }));
	140	+ out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, priceKind: 'guide_value', price: v.priceEur, currency: 'EUR', observationDate, sampleSize: null }));
	141	+ }
	142	+ }
	143	+ return out;
	144	+ }
	145	+}
	146	+
	147	+export default function createConnector(meta: ConnectorMeta) {
	148	+ return new IdealwineConnector(meta);
	149	+}

added connectors/api/idealwine/meta.json +33 −0

		@@ -0,0 +1,33 @@
	1	+{
	2	+ "id": "idealwine",
	3	+ "displayName": "iDealwine Price Estimate (wine price index)",
	4	+ "sourceId": "idealwine",
	5	+ "sourceName": "iDealwine",
	6	+ "sourceType": "pricing_guide",
	7	+ "sourceUrl": "https://www.idealwine.com",
	8	+ "module": "api/idealwine",
	9	+ "enginePriority": ["api", "firecrawl", "scrapfly"],
	10	+ "categories": ["wine"],
	11	+ "regions": ["FR", "EU"],
	12	+ "languages": ["en", "fr"],
	13	+ "currency": ["EUR"],
	14	+ "supportsListings": false,
	15	+ "supportsSold": false,
	16	+ "supportsAuctions": false,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": true,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": false,
	21	+ "refreshFrequencyMinutes": 10080,
	22	+ "priority": "medium",
	23	+ "trustScore": 0.85,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://www.idealwine.com/en/cgv",
	26	+ "accessNotes": "Public price-index pages (/en/cote/<region>?page=N) of iDealwine, the leading French wine auction house: each card lists a wine (region > appellation, colour, producer picture) with the iDealwine Price Estimate per vintage in EUR per 75 cl bottle. The estimate is computed by iDealwine from auction results (hammer price INCLUDING buyer's premium, updated weekly from 3M+ French auction prices since 1992) — we store it as a guide_value price observation, never as a transaction. Plain HTTPS with the RareIndex user agent; robots.txt disallows only /api/, account and login paths. No timestamp is published per estimate, so observationDate = fetch date (confidence 0.75). 2 s politeness delay; regions and pages per run are configurable.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "regions": ["bordeaux", "bourgogne", "rhone", "champagne", "loire", "italie", "alsace", "languedoc", "provence", "jura"],
	31	+ "pagesPerRegion": 3
	32	+ }
	33	+}

added connectors/api/lyon-turnbull/_smoke.ts +23 −0

		@@ -0,0 +1,23 @@
	1	+import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
	2	+import { saveFixture } from '@rareindex/connectors/testing';
	3	+import metaJson from './meta.json' with { type: 'json' };
	4	+import createConnector from './index.js';
	5	+
	6	+const save = process.argv.includes('--save');
	7	+const meta = ConnectorMetaSchema.parse(metaJson);
	8	+const connector = createConnector(meta);
	9	+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
	10	+const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2 } });
	11	+let i = 0;
	12	+for await (const raw of connector.crawl(ctx)) {
	13	+ const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
	14	+ const sales = out.filter((r) => r.kind === 'sale');
	15	+ console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} records, ${sales.length} sales`);
	16	+ for (const r of sales.slice(0, 3)) console.log(' ', JSON.stringify(r).slice(0, 420));
	17	+ if (save && i < 2) {
	18	+ const p = raw.payload as { lots: unknown[] };
	19	+ saveFixture(meta.id, `auction-${raw.externalId}`, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: { ...(raw.payload as object), lots: p.lots.slice(0, 60) } }, expect: { minCount: 20, kinds: ['sale', 'auction_lot'], requiredFields: ['attributes.categorySlug', 'rawTitle'] }, note: 'Captured live from lyonandturnbull.com (plain HTTPS, lots trimmed to 60)' });
	20	+ }
	21	+ i++;
	22	+}
	23	+console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);

added connectors/api/lyon-turnbull/index.test.ts +38 −0

		@@ -0,0 +1,38 @@
	1	+import { describe, expect, it } from 'vitest';
	2	+import { ConnectorMetaSchema } from '@rareindex/connectors';
	3	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	4	+import metaJson from './meta.json' with { type: 'json' };
	5	+import createConnector, { categoryForAuction, parsePastAuctions, premiumFor } from './index.js';
	6	+
	7	+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
	8	+
	9	+describe('lyon-turnbull', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('emits GBP hammer-price sales for sold lots and auction_lot records for unsold ones', async () => {
	13	+ const name = listFixtures('lyon-turnbull')[0]!;
	14	+ const fx = loadFixture('lyon-turnbull', name);
	15	+ const out = await connector.normalize(fx.raw);
	16	+ const sales = out.filter((r) => r.kind === 'sale');
	17	+ expect(sales.length).toBeGreaterThan(5);
	18	+ for (const r of sales) {
	19	+ if (r.kind !== 'sale') continue;
	20	+ expect(r.currency).toBe('GBP');
	21	+ expect(r.buyerPremiumIncluded).toBe(false);
	22	+ expect(r.auctionHouse).toBe('Lyon & Turnbull');
	23	+ expect(Array.isArray(r.attributes.metadata.premium_tiers)).toBe(true);
	24	+ if (r.attributes.metadata.premium_estimate !== null) expect(r.attributes.metadata.premium_estimate).toBeGreaterThan(0);
	25	+ expect(r.saleDate.getFullYear()).toBeGreaterThanOrEqual(2020);
	26	+ }
	27	+ });
	28	+
	29	+ it('helpers', () => {
	30	+ const html = `<script id="__NEXT_DATA__" type="application/json">${JSON.stringify({ props: { pageProps: { auctionsListData: [{ title: 'Scotland Collected ', saleNumber: '908', date: '19 & 20 August 2026', amsDate: '2026-08-19T09:00:00.000Z', arrowLink: '/auctions/scotland-collected-908' }] } } })}</script>`;
	31	+ expect(parsePastAuctions(html)).toEqual([{ saleNumber: '908', title: 'Scotland Collected', subtitle: null, date: '19 & 20 August 2026', amsDate: '2026-08-19T09:00:00.000Z', link: '/auctions/scotland-collected-908' }]);
	32	+ expect(categoryForAuction('Fine Jewellery')).toBe('jewelry');
	33	+ expect(categoryForAuction('1925: Celebrating Art Deco')).toBe('design_furniture');
	34	+ expect(categoryForAuction('Rare Books, Manuscripts, Maps & Photographs')).toBe('books');
	35	+ expect(premiumFor(10000, [{ percent: 27, amount_over: 0 }, { percent: 26, amount_over: 20000 }])).toBe(2700);
	36	+ expect(premiumFor(30000, [{ percent: 27, amount_over: 0 }, { percent: 26, amount_over: 20000 }])).toBe(8000);
	37	+ });
	38	+});

added connectors/api/lyon-turnbull/index.ts +257 −0

		@@ -0,0 +1,257 @@
	1	+import { z } from 'zod';
	2	+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { AssetAttributesSchema, NormalizedAuctionLotSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';
	4	+
	5	+/**
	6	+ * Lyon & Turnbull — auction results embedded as JSON in each auction page. One raw record per
	7	+ * auction (compact lot list); normalise → one sale per sold lot (hammer price, GBP) and one
	8	+ * auction_lot for unsold lots.
	9	+ */
	10	+
	11	+const BASE = 'https://www.lyonandturnbull.com';
	12	+const PARSER_VERSION = '1.0.0';
	13	+
	14	+export const PastAuctionSchema = z.object({ saleNumber: z.string(), title: z.string(), subtitle: z.string().nullable(), date: z.string().nullable(), amsDate: z.string().nullable(), link: z.string() });
	15	+export type PastAuction = z.infer<typeof PastAuctionSchema>;
	16	+
	17	+export const LotSchema = z.object({
	18	+ uuid: z.string(),
	19	+ lotNo: z.string(),
	20	+ title: z.string(),
	21	+ subTitle: z.string().nullable(),
	22	+ low: z.number().nullable(),
	23	+ high: z.number().nullable(),
	24	+ hammer: z.number().nullable(),
	25	+ status: z.string().nullable(),
	26	+ sessionDate: z.string().nullable(),
	27	+ image: z.string().nullable(),
	28	+});
	29	+export const AuctionPayloadSchema = z.object({
	30	+ kind: z.literal('auction_results'),
	31	+ url: z.string(),
	32	+ auction: PastAuctionSchema,
	33	+ currency: z.string(),
	34	+ status: z.string().nullable(),
	35	+ premiumTiers: z.array(z.object({ percent: z.number(), amount_over: z.number() })),
	36	+ lots: z.array(LotSchema),
	37	+});
	38	+export type AuctionPayload = z.infer<typeof AuctionPayloadSchema>;
	39	+
	40	+/** Past auctions from the /auctions/past-auctions __NEXT_DATA__ (`auctionsListData`). */
	41	+export function parsePastAuctions(htmlText: string): PastAuction[] {
	42	+ const data = H.nextData(htmlText) as { props?: { pageProps?: { auctionsListData?: Array<Record<string, unknown>> } } } | null;
	43	+ const list = data?.props?.pageProps?.auctionsListData ?? [];
	44	+ return list
	45	+ .map((a) => ({
	46	+ saleNumber: String(a.saleNumber ?? ''),
	47	+ title: String(a.title ?? '').trim(),
	48	+ subtitle: a.subtitle ? String(a.subtitle) : null,
	49	+ date: a.date ? String(a.date) : null,
	50	+ amsDate: a.amsDate ? String(a.amsDate) : null,
	51	+ link: String(a.arrowLink ?? ''),
	52	+ }))
	53	+ .filter((a) => a.link.startsWith('/auctions/') && a.saleNumber);
	54	+}
	55	+
	56	+function num(v: unknown): number | null {
	57	+ if (v === null || v === undefined || v === '') return null;
	58	+ const n = Number(v);
	59	+ return Number.isFinite(n) && n > 0 ? n : null;
	60	+}
	61	+
	62	+/** Extract the embedded lots JSON ("lots":[...]) plus auction status/currency/premium tiers from an auction page. */
	63	+export function parseAuctionPage(htmlText: string, url: string, auction: PastAuction): AuctionPayload | null {
	64	+ const data = H.nextData(htmlText) as Record<string, unknown> | null;
	65	+ let lotsRaw: Array<Record<string, unknown>> | null = null;
	66	+ let auctionRaw: Record<string, unknown> | null = null;
	67	+ const visit = (o: unknown, depth: number): void => {
	68	+ if (!o || typeof o !== 'object' || depth > 8 || (lotsRaw && auctionRaw)) return;
	69	+ if (Array.isArray(o)) {
	70	+ for (const x of o) visit(x, depth + 1);
	71	+ return;
	72	+ }
	73	+ const rec = o as Record<string, unknown>;
	74	+ if (!lotsRaw && Array.isArray(rec.lots) && rec.lots.length && typeof (rec.lots[0] as Record<string, unknown>).lot_no !== 'undefined') {
	75	+ lotsRaw = rec.lots as Array<Record<string, unknown>>;
	76	+ auctionRaw = rec;
	77	+ }
	78	+ for (const v of Object.values(rec)) visit(v, depth + 1);
	79	+ };
	80	+ if (data) visit(data, 0);
	81	+ if (!lotsRaw) {
	82	+ // fallback: the JSON may be inlined outside __NEXT_DATA__
	83	+ const i = htmlText.indexOf('"lots":[');
	84	+ if (i < 0) return null;
	85	+ const literal = H.inlineJson(htmlText.slice(i - 1), '"lots"');
	86	+ if (!literal || !Array.isArray((literal as { lots?: unknown }).lots)) return null;
	87	+ lotsRaw = (literal as { lots: Array<Record<string, unknown>> }).lots;
	88	+ auctionRaw = literal as Record<string, unknown>;
	89	+ }
	90	+ const a = auctionRaw as Record<string, unknown>;
	91	+ const first = lotsRaw![0]!;
	92	+ // Premium tiers live on the auction object (sibling of `lots`) or on each lot's financeItem.
	93	+ const tierSource = (Array.isArray(a.premiums) ? a.premiums : Array.isArray((a.auction as { premiums?: unknown } | undefined)?.premiums) ? (a.auction as { premiums: unknown[] }).premiums : Array.isArray((first.financeItem as { premium_tiers?: unknown } | undefined)?.premium_tiers) ? (first.financeItem as { premium_tiers: unknown[] }).premium_tiers : []) as Array<{ percent?: number; amount_over?: number }>;
	94	+ const premiums = tierSource.map((p) => ({ percent: Number(p.percent ?? 0), amount_over: Number(p.amount_over ?? 0) }));
	95	+ const currencySymbol = ((first.auction as { currency?: { symbol?: string } } | undefined)?.currency?.symbol ?? (a.currency as { symbol?: string } | undefined)?.symbol ?? '£');
	96	+ const currency = currencySymbol === '£' ? 'GBP' : currencySymbol === '€' ? 'EUR' : currencySymbol === '$' ? 'USD' : 'GBP';
	97	+ const status = typeof a.status === 'string' ? a.status : ((first.auction as { status?: string } | undefined)?.status ?? null);
	98	+ const lots = lotsRaw!.map((l) => {
	99	+ const title = (l.title as { en?: string } | undefined)?.en ?? (typeof l.title === 'string' ? l.title : '');
	100	+ const sub = ((l.dynamic_fields as { en?: { sub_title?: string } } | undefined)?.en?.sub_title ?? null) as string | null;
	101	+ const img = (l.images as Array<{ data?: { original?: { url?: string }; sm?: { url?: string } } }> | undefined)?.[0]?.data;
	102	+ return {
	103	+ uuid: String(l.uuid ?? ''),
	104	+ lotNo: String(l.lot_no ?? ''),
	105	+ title: String(title).trim(),
	106	+ subTitle: sub ? String(sub).trim() : null,
	107	+ low: num(l.low),
	108	+ high: num(l.high),
	109	+ hammer: num(l.hammer_price) ?? num((l.financeItem as { hammer_price?: unknown } | undefined)?.hammer_price),
	110	+ status: typeof l.status === 'string' ? l.status : null,
	111	+ sessionDate: typeof l.session_date === 'string' ? l.session_date : null,
	112	+ image: img?.original?.url ?? img?.sm?.url ?? null,
	113	+ };
	114	+ });
	115	+ return { kind: 'auction_results', url, auction, currency, status, premiumTiers: premiums, lots: lots.filter((l) => l.title && l.lotNo) };
	116	+}
	117	+
	118	+const TITLE_CATEGORY: Array<[RegExp, string]> = [
	119	+ [/jewel/i, 'jewelry'],
	120	+ [/silver|objets de vertu/i, 'silver'],
	121	+ [/whisky|spirits|wine/i, 'whisky'],
	122	+ [/watch|clock|horolog/i, 'watches'],
	123	+ [/book|manuscript|map/i, 'books'],
	124	+ [/photograph/i, 'photography'],
	125	+ [/contemporary|modern made|post-war|prints|editions|modern british/i, 'contemporary_art'],
	126	+ [/design|art deco|art nouveau|decorative arts|furniture|form through time/i, 'design_furniture'],
	127	+ [/asian|chinese|japanese|islamic|african|tribal|ceramic|porcelain|glass/i, 'antiques'],
	128	+ [/scottish|paintings|pictures|old master|fine art|art/i, 'art'],
	129	+];
	130	+
	131	+export function categoryForAuction(title: string): string {
	132	+ return TITLE_CATEGORY.find(([re]) => re.test(title))?.[1] ?? 'antiques';
	133	+}
	134	+
	135	+/** Lot-level refinements for cross-category sales (natural history, watches inside mixed auctions…). */
	136	+const LOT_CATEGORY: Array<[RegExp, string]> = [
	137	+ [/meteorite/i, 'meteorites'],
	138	+ [/fossil|ammonite|trilobite|dinosaur|mammoth|megalodon/i, 'fossils'],
	139	+ [/mineral|crystal|quartz|amethyst|geode|specimen/i, 'minerals'],
	140	+ [/wristwatch|pocket watch|chronograph/i, 'other_watches'],
	141	+ [/rolex/i, 'rolex'],
	142	+ [/patek philippe/i, 'patek_philippe'],
	143	+ [/omega/i, 'omega'],
	144	+];
	145	+export function categoryForLot(auctionCategory: string, title: string): string {
	146	+ return LOT_CATEGORY.find(([re]) => re.test(title))?.[1] ?? auctionCategory;
	147	+}
	148	+
	149	+/** Buyer's premium for a hammer price from tiered percentages (each tier applies to the slice above `amount_over`). */
	150	+export function premiumFor(hammer: number, tiers: Array<{ percent: number; amount_over: number }>): number | null {
	151	+ if (!tiers.length) return null;
	152	+ const sorted = [...tiers].sort((a, b) => a.amount_over - b.amount_over);
	153	+ let total = 0;
	154	+ for (let i = 0; i < sorted.length; i++) {
	155	+ const from = sorted[i]!.amount_over;
	156	+ const to = sorted[i + 1]?.amount_over ?? Infinity;
	157	+ if (hammer <= from) break;
	158	+ total += (Math.min(hammer, to) - from) * (sorted[i]!.percent / 100);
	159	+ }
	160	+ return Math.round(total * 100) / 100;
	161	+}
	162	+
	163	+export class LyonTurnbullConnector extends BaseConnector {
	164	+ readonly version = '1.0.0';
	165	+ readonly parserVersion = PARSER_VERSION;
	166	+ protected override minIntervalMs = 2000;
	167	+
	168	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	169	+ const pastUrl = String(this.meta.config.pastAuctionsUrl ?? `${BASE}/auctions/past-auctions`);
	170	+ const perRun = Number(this.meta.config.auctionsPerRun ?? 3);
	171	+ const cursor = (ctx.options.cursor ?? {}) as { done?: string[] };
	172	+ const done = new Set<string>(cursor.done ?? []);
	173	+ const list = await ctx.fetch(pastUrl, { engines: ['api'], responseType: 'text', minQuality: 0, timeoutMs: 60_000 });
	174	+ if (!list.success || !list.html) {
	175	+ ctx.anomaly('past_list_failed', list.error ?? String(list.httpStatus));
	176	+ return;
	177	+ }
	178	+ const auctions = parsePastAuctions(list.html).sort((a, b) => (b.amsDate ?? '').localeCompare(a.amsDate ?? ''));
	179	+ let fetched = 0;
	180	+ let count = 0;
	181	+ for (const auction of auctions) {
	182	+ if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun) break;
	183	+ if (done.has(auction.saleNumber) && ctx.options.mode !== 'backfill') continue;
	184	+ const url = BASE + auction.link;
	185	+ await this.throttle();
	186	+ fetched++;
	187	+ const res = await ctx.fetch(url, {
	188	+ engines: ['api'],
	189	+ responseType: 'text',
	190	+ timeoutMs: 60_000,
	191	+ expect: ['title', 'price', 'currency', 'status'],
	192	+ parse: (r) => {
	193	+ const p = r.html ? parseAuctionPage(r.html, url, auction) : null;
	194	+ const sold = p?.lots.find((l) => l.hammer);
	195	+ return p ? { title: p.lots[0]?.title, price: sold?.hammer ?? null, currency: sold ? p.currency : null, status: p.status } : null;
	196	+ },
	197	+ });
	198	+ const payload = res.success && res.html ? parseAuctionPage(res.html, url, auction) : null;
	199	+ if (!payload) {
	200	+ ctx.anomaly('auction_parse_failed', `${auction.saleNumber}: ${res.error ?? res.httpStatus}`);
	201	+ continue;
	202	+ }
	203	+ if (payload.status && payload.status !== 'completed') {
	204	+ // still live/published — results not final yet
	205	+ continue;
	206	+ }
	207	+ count++;
	208	+ done.add(auction.saleNumber);
	209	+ await ctx.setCursor({ done: [...done].slice(-400) });
	210	+ yield { url, externalId: auction.saleNumber, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	211	+ }
	212	+ }
	213	+
	214	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	215	+ const p = AuctionPayloadSchema.parse(raw.payload);
	216	+ const categorySlug = categoryForAuction(`${p.auction.title} ${p.auction.subtitle ?? ''}`);
	217	+ const out: NormalizedRecord[] = [];
	218	+ for (const lot of p.lots) {
	219	+ const saleDateIso = lot.sessionDate ?? p.auction.amsDate;
	220	+ const saleDate = saleDateIso ? new Date(saleDateIso) : null;
	221	+ const lotUrl = `${BASE}${p.auction.link}/lot/${lot.lotNo}`;
	222	+ const yearHint = lot.subTitle ? extractYear(lot.subTitle) : null;
	223	+ const attributes = AssetAttributesSchema.parse({
	224	+ categorySlug: categoryForLot(categorySlug, `${lot.title} ${lot.subTitle ?? ''}`),
	225	+ name: lot.title,
	226	+ year: yearHint,
	227	+ identifiers: { lt_lot_uuid: lot.uuid, lt_lot: `${p.auction.saleNumber}/${lot.lotNo}` },
	228	+ metadata: { sale_number: p.auction.saleNumber, auction_title: p.auction.title, sub_title: lot.subTitle, estimate_low: lot.low, estimate_high: lot.high, premium_tiers: p.premiumTiers, premium_estimate: lot.hammer ? premiumFor(lot.hammer, p.premiumTiers) : null },
	229	+ });
	230	+ const base = {
	231	+ connectorId: this.meta.id,
	232	+ sourceId: this.meta.sourceId,
	233	+ sourceUrl: lotUrl,
	234	+ externalId: `${p.auction.saleNumber}:${lot.lotNo}`,
	235	+ rawTitle: lot.subTitle ? `${lot.title} — ${lot.subTitle}` : lot.title,
	236	+ description: null,
	237	+ imageUrls: lot.image ? [lot.image] : [],
	238	+ attributes,
	239	+ grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
	240	+ condition: { condition: null, conditionRaw: null, completeness: null },
	241	+ observedAt: raw.fetchedAt,
	242	+ confidence: 0.9,
	243	+ parserVersion: PARSER_VERSION,
	244	+ };
	245	+ if (lot.hammer && saleDate && lot.status !== 'unsold') {
	246	+ out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, saleType: 'auction', saleDate, price: lot.hammer, currency: p.currency, buyerPremiumIncluded: false, quantity: 1, isBundle: false, location: 'United Kingdom', auctionHouse: 'Lyon & Turnbull', lotNumber: lot.lotNo }));
	247	+ } else {
	248	+ out.push(NormalizedAuctionLotSchema.parse({ kind: 'auction_lot', ...base, auctionHouse: 'Lyon & Turnbull', auctionName: p.auction.title, lotNumber: lot.lotNo, startsAt: saleDate, endsAt: saleDate, estimateLow: lot.low, estimateHigh: lot.high, currentBid: null, currency: p.currency, status: 'ended', location: 'United Kingdom' }));
	249	+ }
	250	+ }
	251	+ return out;
	252	+ }
	253	+}
	254	+
	255	+export default function createConnector(meta: ConnectorMeta) {
	256	+ return new LyonTurnbullConnector(meta);
	257	+}

added connectors/api/lyon-turnbull/meta.json +33 −0

		@@ -0,0 +1,33 @@
	1	+{
	2	+ "id": "lyon-turnbull",
	3	+ "displayName": "Lyon & Turnbull (auction results)",
	4	+ "sourceId": "lyon-turnbull",
	5	+ "sourceName": "Lyon & Turnbull",
	6	+ "sourceType": "auction_house",
	7	+ "sourceUrl": "https://www.lyonandturnbull.com",
	8	+ "module": "api/lyon-turnbull",
	9	+ "enginePriority": ["api", "firecrawl", "scrapfly"],
	10	+ "categories": ["art", "contemporary_art", "photography", "design_furniture", "antiques", "jewelry", "silver", "watches", "other_watches", "books", "whisky", "porcelain", "glass_crystal", "clocks", "scientific_instruments"],
	11	+ "regions": ["GB"],
	12	+ "languages": ["en"],
	13	+ "currency": ["GBP"],
	14	+ "supportsListings": false,
	15	+ "supportsSold": true,
	16	+ "supportsAuctions": true,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": false,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": false,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "medium",
	23	+ "trustScore": 0.9,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://www.lyonandturnbull.com/auctions/buy-sell/buy",
	26	+ "accessNotes": "Edinburgh/London/Glasgow auction house. Past auctions are listed in the public /auctions/past-auctions page (Next.js __NEXT_DATA__: title, sale number, date, link); each auction page embeds the full lot list as JSON (lot number, title, sub-title, estimate, hammer_price, status sold/unsold, images, buyer's premium tiers). Plain HTTPS with the RareIndex user agent; robots.txt disallows only /account, /auth, /search and collection_* paths. We store the HAMMER price (buyer_premium_included=false) and record the published premium tiers in metadata; sale date = auction session date. No login, no bidder data. 2 s politeness delay.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "pastAuctionsUrl": "https://www.lyonandturnbull.com/auctions/past-auctions",
	31	+ "auctionsPerRun": 3
	32	+ }
	33	+}

added connectors/api/saatchi-art/_smoke.ts +21 −0

		@@ -0,0 +1,21 @@
	1	+import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
	2	+import { saveFixture } from '@rareindex/connectors/testing';
	3	+import metaJson from './meta.json' with { type: 'json' };
	4	+import createConnector from './index.js';
	5	+
	6	+const save = process.argv.includes('--save');
	7	+const meta = ConnectorMetaSchema.parse(metaJson);
	8	+const connector = createConnector(meta);
	9	+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
	10	+const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2, categories: ['contemporary_art'] } });
	11	+let i = 0;
	12	+for await (const raw of connector.crawl(ctx)) {
	13	+ const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
	14	+ console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} listings`);
	15	+ for (const r of out.slice(0, 3)) console.log(' ', JSON.stringify(r).slice(0, 420));
	16	+ if (save && i < 2) saveFixture(meta.id, i === 0 ? 'paintings-page-1' : 'paintings-page-2', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 20, kinds: ['listing'], requiredFields: ['price', 'currency', 'attributes.brand'] }, note: 'Captured live from saatchiart.com __NEXT_DATA__' });
	17	+ i++;
	18	+}
	19	+const looked = await connector.lookup('https://www.saatchiart.com/art/Painting-Octhopuses/626451/10544327/view', ctx);
	20	+console.log('lookup →', looked.length ? JSON.stringify((await connector.normalize({ ...looked[0]!, externalId: looked[0]!.externalId ?? null, fetchedAt: new Date() }))[0]).slice(0, 300) : 'none');
	21	+console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);

added connectors/api/saatchi-art/index.test.ts +34 −0

		@@ -0,0 +1,34 @@
	1	+import { describe, expect, it } from 'vitest';
	2	+import { ConnectorMetaSchema } from '@rareindex/connectors';
	3	+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	4	+import metaJson from './meta.json' with { type: 'json' };
	5	+import createConnector, { parseBrowsePage } from './index.js';
	6	+
	7	+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
	8	+
	9	+describe('saatchi-art', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('maps artworks to USD fixed-price listings with artist, medium and size', async () => {
	13	+ const fx = loadFixture('saatchi-art', 'paintings-page-1');
	14	+ const out = await connector.normalize(fx.raw);
	15	+ expect(out.length).toBe(25);
	16	+ for (const r of out) {
	17	+ if (r.kind !== 'listing') throw new Error('expected listing');
	18	+ expect(r.currency).toBe('USD');
	19	+ expect(r.listingType).toBe('fixed_price');
	20	+ expect(['available', 'sold', 'unknown']).toContain(r.availability);
	21	+ expect(r.attributes.identifiers.saatchi_artwork_id).toMatch(/^\d+$/);
	22	+ expect(r.attributes.brand).toBeTruthy();
	23	+ expect(r.sourceUrl).toMatch(/^https:\/\/www\.saatchiart\.com\/art\//);
	24	+ }
	25	+ });
	26	+
	27	+ it('parses __NEXT_DATA__ browse pages', () => {
	28	+ const html = `<script id="__NEXT_DATA__" type="application/json">${JSON.stringify({ props: { pageProps: { initialState: { searchProvider: { serverSideData: { totalResults: 2, results: [{ artworkId: '1', artworkTitle: 'A', artworkOriginalUrl: '/art/Painting-A/1/1/view', artistFirstName: 'jane', artistLastName: 'doe', category: 'Painting', mediums: ['oil'], materials: ['canvas'], dimensions: { widthInCentimeters: 10, heightInCentimeters: 20 }, listPriceInCents: 123400, originalArtworkStatus: 'sold', imageUrl: 'https://i/x.jpg' }] } } } } } })}</script>`;
	29	+ const p = parseBrowsePage(html, 'u', { path: '/paintings', category: 'contemporary_art' }, 1)!;
	30	+ expect(p.totalResults).toBe(2);
	31	+ expect(p.works[0]).toMatchObject({ artworkId: '1', title: 'A', listPriceCents: 123400, status: 'sold', widthCm: 10 });
	32	+ expect(parseBrowsePage('<html></html>', 'u', { path: '/x', category: 'art' }, 1)).toBeNull();
	33	+ });
	34	+});

added connectors/api/saatchi-art/index.ts +202 −0

		@@ -0,0 +1,202 @@
	1	+import { z } from 'zod';
	2	+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
	4	+
	5	+/**
	6	+ * Saatchi Art — original artworks listed for sale. One raw record per browse page (25 works);
	7	+ * normalise → one listing per artwork (USD list price; sold works → availability 'sold').
	8	+ */
	9	+
	10	+const BASE = 'https://www.saatchiart.com';
	11	+const PARSER_VERSION = '1.0.0';
	12	+
	13	+const SeedSchema = z.object({ path: z.string(), category: z.string() });
	14	+type Seed = z.infer<typeof SeedSchema>;
	15	+
	16	+export const WorkSchema = z.object({
	17	+ artworkId: z.string(),
	18	+ url: z.string(),
	19	+ title: z.string(),
	20	+ artistFirstName: z.string().nullable(),
	21	+ artistLastName: z.string().nullable(),
	22	+ artistId: z.number().nullable(),
	23	+ category: z.string().nullable(),
	24	+ mediums: z.array(z.string()),
	25	+ materials: z.array(z.string()),
	26	+ styles: z.array(z.string()),
	27	+ subject: z.string().nullable(),
	28	+ widthCm: z.number().nullable(),
	29	+ heightCm: z.number().nullable(),
	30	+ depthCm: z.number().nullable(),
	31	+ listPriceCents: z.number().nullable(),
	32	+ status: z.string().nullable(),
	33	+ sku: z.string().nullable(),
	34	+ image: z.string().nullable(),
	35	+ countryName: z.string().nullable(),
	36	+ uploadedAt: z.number().nullable(),
	37	+});
	38	+export const PagePayloadSchema = z.object({ kind: z.literal('browse_page'), url: z.string(), seed: SeedSchema, page: z.number(), totalResults: z.number().nullable(), works: z.array(WorkSchema) });
	39	+export type PagePayload = z.infer<typeof PagePayloadSchema>;
	40	+
	41	+export function parseBrowsePage(htmlText: string, url: string, seed: Seed, page: number): PagePayload | null {
	42	+ const data = H.nextData(htmlText) as { props?: { pageProps?: { initialState?: { searchProvider?: { serverSideData?: { results?: Array<Record<string, unknown>>; totalResults?: number } } } } } } | null;
	43	+ const ssd = data?.props?.pageProps?.initialState?.searchProvider?.serverSideData;
	44	+ if (!ssd?.results) return null;
	45	+ const works = ssd.results
	46	+ .map((r) => {
	47	+ const dims = (r.dimensions as { widthInCentimeters?: number; heightInCentimeters?: number; depthInCentimeters?: number } | undefined) ?? {};
	48	+ const path = String(r.artworkOriginalUrl ?? '');
	49	+ return {
	50	+ artworkId: String(r.artworkId ?? ''),
	51	+ url: path.startsWith('http') ? path : BASE + path,
	52	+ title: String(r.artworkTitle ?? '').trim(),
	53	+ artistFirstName: r.artistFirstName ? String(r.artistFirstName) : null,
	54	+ artistLastName: r.artistLastName ? String(r.artistLastName) : null,
	55	+ artistId: typeof r.artistId === 'number' ? r.artistId : null,
	56	+ category: r.category ? String(r.category) : null,
	57	+ mediums: Array.isArray(r.mediums) ? (r.mediums as unknown[]).map(String) : [],
	58	+ materials: Array.isArray(r.materials) ? (r.materials as unknown[]).map(String) : [],
	59	+ styles: Array.isArray(r.styles) ? (r.styles as unknown[]).map(String) : [],
	60	+ subject: r.subject ? String(r.subject) : null,
	61	+ widthCm: typeof dims.widthInCentimeters === 'number' ? dims.widthInCentimeters : null,
	62	+ heightCm: typeof dims.heightInCentimeters === 'number' ? dims.heightInCentimeters : null,
	63	+ depthCm: typeof dims.depthInCentimeters === 'number' ? dims.depthInCentimeters : null,
	64	+ listPriceCents: typeof r.listPriceInCents === 'number' ? r.listPriceInCents : null,
	65	+ status: r.originalArtworkStatus ? String(r.originalArtworkStatus) : null,
	66	+ sku: r.sku ? String(r.sku) : null,
	67	+ image: r.imageUrl ? String(r.imageUrl) : null,
	68	+ countryName: r.countryName ? String(r.countryName) : null,
	69	+ uploadedAt: typeof r.artworkUploadedDate === 'number' ? r.artworkUploadedDate : null,
	70	+ };
	71	+ })
	72	+ .filter((w) => w.artworkId && w.title);
	73	+ return { kind: 'browse_page', url, seed, page, totalResults: typeof ssd.totalResults === 'number' ? ssd.totalResults : null, works };
	74	+}
	75	+
	76	+function titleCase(s: string): string {
	77	+ return s.replace(/\b\w/g, (c) => c.toUpperCase());
	78	+}
	79	+
	80	+export class SaatchiArtConnector extends BaseConnector {
	81	+ readonly version = '1.0.0';
	82	+ readonly parserVersion = PARSER_VERSION;
	83	+ override readonly urlPatterns = [/saatchiart\.com\/(?:en-[a-z]+\/)?art\/[^/]+\/\d+\/\d+\/view/];
	84	+ protected override minIntervalMs = 2000;
	85	+
	86	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	87	+ const seeds = z.array(SeedSchema).parse(this.meta.config.seeds ?? []);
	88	+ const pages = Number(this.meta.config.pagesPerSeed ?? 4);
	89	+ let count = 0;
	90	+ for (const seed of seeds) {
	91	+ if (ctx.options.categories?.length && !ctx.options.categories.includes(seed.category)) continue;
	92	+ for (let page = 1; page <= pages; page++) {
	93	+ if (ctx.signal?.aborted || this.reached(ctx, count)) return;
	94	+ const url = `${BASE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;
	95	+ await this.throttle();
	96	+ const res = await ctx.fetch(url, {
	97	+ engines: ['api'],
	98	+ responseType: 'text',
	99	+ expect: ['title', 'price', 'currency', 'images'],
	100	+ parse: (r) => {
	101	+ const p = r.html ? parseBrowsePage(r.html, url, seed, page) : null;
	102	+ const w = p?.works[0];
	103	+ return w ? { title: w.title, price: w.listPriceCents, currency: 'USD', images: w.image ? [w.image] : [] } : null;
	104	+ },
	105	+ });
	106	+ const payload = res.success && res.html ? parseBrowsePage(res.html, url, seed, page) : null;
	107	+ if (!payload || payload.works.length === 0) {
	108	+ ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
	109	+ break;
	110	+ }
	111	+ count++;
	112	+ yield { url, externalId: `browse:${seed.path}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	113	+ }
	114	+ }
	115	+ }
	116	+
	117	+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
	118	+ const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 });
	119	+ if (!res.success || !res.html) return [];
	120	+ const data = H.nextData(res.html) as { props?: { pageProps?: { initialState?: { page?: { data?: { pdpArtwork?: Record<string, unknown>; artwork?: Record<string, unknown> } } } } } } | null;
	121	+ const a = data?.props?.pageProps?.initialState?.page?.data?.pdpArtwork ?? data?.props?.pageProps?.initialState?.page?.data?.artwork;
	122	+ if (!a) return [];
	123	+ const dims = (a.dimensions as { widthInCentimeters?: number; heightInCentimeters?: number; depthInCentimeters?: number; width?: number; height?: number; depth?: number } | undefined) ?? {};
	124	+ const artist = (a.artist as { firstName?: string; lastName?: string; id?: number } | undefined) ?? {};
	125	+ const work: z.infer<typeof WorkSchema> = {
	126	+ artworkId: String(a.artworkId ?? a.id ?? url.match(/\/(\d+)\/view/)?.[1] ?? ''),
	127	+ url,
	128	+ title: String(a.title ?? a.artworkTitle ?? ''),
	129	+ artistFirstName: artist.firstName ?? (a.artistFirstName ? String(a.artistFirstName) : null),
	130	+ artistLastName: artist.lastName ?? (a.artistLastName ? String(a.artistLastName) : null),
	131	+ artistId: artist.id ?? null,
	132	+ category: a.category ? String(a.category) : null,
	133	+ mediums: Array.isArray(a.mediums) ? (a.mediums as unknown[]).map(String) : [],
	134	+ materials: Array.isArray(a.materials) ? (a.materials as unknown[]).map(String) : [],
	135	+ styles: Array.isArray(a.styles) ? (a.styles as unknown[]).map(String) : [],
	136	+ subject: a.subject ? String(a.subject) : null,
	137	+ widthCm: dims.widthInCentimeters ?? dims.width ?? null,
	138	+ heightCm: dims.heightInCentimeters ?? dims.height ?? null,
	139	+ depthCm: dims.depthInCentimeters ?? dims.depth ?? null,
	140	+ listPriceCents: typeof a.listPriceInCents === 'number' ? a.listPriceInCents : typeof a.price === 'number' ? Math.round(a.price * 100) : null,
	141	+ status: a.originalArtworkStatus ? String(a.originalArtworkStatus) : null,
	142	+ sku: a.sku ? String(a.sku) : null,
	143	+ image: a.imageUrl ? String(a.imageUrl) : null,
	144	+ countryName: null,
	145	+ uploadedAt: null,
	146	+ };
	147	+ if (!work.artworkId || !work.title) return [];
	148	+ const payload: PagePayload = { kind: 'browse_page', url, seed: { path: 'lookup', category: work.category?.toLowerCase() === 'photography' ? 'photography' : 'contemporary_art' }, page: 0, totalResults: null, works: [work] };
	149	+ return [{ url, externalId: `artwork:${work.artworkId}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
	150	+ }
	151	+
	152	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	153	+ const p = PagePayloadSchema.parse(raw.payload);
	154	+ const out: NormalizedRecord[] = [];
	155	+ for (const w of p.works) {
	156	+ const artist = [w.artistFirstName, w.artistLastName].filter(Boolean).map((s) => titleCase(String(s))).join(' ') || null;
	157	+ const size = w.widthCm && w.heightCm ? `${w.widthCm} × ${w.heightCm}${w.depthCm ? ` × ${w.depthCm}` : ''} cm` : null;
	158	+ const categorySlug = w.category?.toLowerCase() === 'photography' ? 'photography' : p.seed.category;
	159	+ const attributes = AssetAttributesSchema.parse({
	160	+ categorySlug,
	161	+ brand: artist,
	162	+ name: w.title,
	163	+ material: [...w.mediums, ...w.materials].join(', ') || null,
	164	+ size,
	165	+ country: w.countryName ? titleCase(w.countryName) : null,
	166	+ identifiers: { saatchi_artwork_id: w.artworkId, ...(w.sku ? { saatchi_sku: w.sku } : {}) },
	167	+ metadata: { category: w.category, styles: w.styles, subject: w.subject, artist_id: w.artistId, uploaded_at: w.uploadedAt ? new Date(w.uploadedAt * 1000).toISOString() : null, unique_original: true },
	168	+ });
	169	+ out.push(
	170	+ NormalizedListingSchema.parse({
	171	+ kind: 'listing',
	172	+ connectorId: this.meta.id,
	173	+ sourceId: this.meta.sourceId,
	174	+ sourceUrl: w.url,
	175	+ externalId: w.artworkId,
	176	+ rawTitle: artist ? `${artist} — ${w.title}` : w.title,
	177	+ description: null,
	178	+ imageUrls: w.image ? [w.image] : [],
	179	+ attributes,
	180	+ grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
	181	+ condition: { condition: null, conditionRaw: null, completeness: null },
	182	+ observedAt: raw.fetchedAt,
	183	+ confidence: 0.85,
	184	+ parserVersion: PARSER_VERSION,
	185	+ listingType: 'fixed_price',
	186	+ price: w.listPriceCents !== null ? w.listPriceCents / 100 : null,
	187	+ currency: w.listPriceCents !== null ? 'USD' : null,
	188	+ seller: artist,
	189	+ location: w.countryName ? titleCase(w.countryName) : null,
	190	+ quantity: 1,
	191	+ listedAt: w.uploadedAt ? new Date(w.uploadedAt * 1000) : null,
	192	+ availability: w.status === 'sold' ? 'sold' : w.status === 'avail' ? 'available' : 'unknown',
	193	+ }),
	194	+ );
	195	+ }
	196	+ return out;
	197	+ }
	198	+}
	199	+
	200	+export default function createConnector(meta: ConnectorMeta) {
	201	+ return new SaatchiArtConnector(meta);
	202	+}

added connectors/api/saatchi-art/meta.json +40 −0

		@@ -0,0 +1,40 @@
	1	+{
	2	+ "id": "saatchi-art",
	3	+ "displayName": "Saatchi Art (original artworks for sale)",
	4	+ "sourceId": "saatchi-art",
	5	+ "sourceName": "Saatchi Art",
	6	+ "sourceType": "marketplace",
	7	+ "sourceUrl": "https://www.saatchiart.com",
	8	+ "module": "api/saatchi-art",
	9	+ "enginePriority": ["api", "firecrawl", "scrapfly"],
	10	+ "categories": ["art", "contemporary_art", "photography"],
	11	+ "regions": ["US", "GB", "EU"],
	12	+ "languages": ["en"],
	13	+ "currency": ["USD"],
	14	+ "supportsListings": true,
	15	+ "supportsSold": false,
	16	+ "supportsAuctions": false,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": true,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": true,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "medium",
	23	+ "trustScore": 0.7,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://www.saatchiart.com/termsofservice",
	26	+ "accessNotes": "Public category browse pages (/paintings, /photography, /sculpture, /prints … with ?page=N) served as Next.js pages whose __NEXT_DATA__ carries 25 artworks per page: title, artist, category, mediums, materials, dimensions (cm), list price (USD cents), availability status (avail/sold), SKU and image. Plain HTTPS with the RareIndex user agent. robots.txt only disallows filter/sort query parameters (?category=, ?sort= …) which we do not use — pagination via ?page= is allowed. Listing asks are NOT transactions; sold works keep their last list price as a listing marked sold (no sale date is published, so no sale record is created). 2 s politeness delay.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "seeds": [
	31	+ { "path": "/paintings", "category": "contemporary_art" },
	32	+ { "path": "/photography", "category": "photography" },
	33	+ { "path": "/sculpture", "category": "contemporary_art" },
	34	+ { "path": "/prints", "category": "contemporary_art" },
	35	+ { "path": "/drawings", "category": "contemporary_art" },
	36	+ { "path": "/collage", "category": "contemporary_art" }
	37	+ ],
	38	+ "pagesPerSeed": 4
	39	+ }
	40	+}

added connectors/api/scotch-whisky-auctions/_smoke.ts +19 −0

		@@ -0,0 +1,19 @@
	1	+import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
	2	+import { saveFixture } from '@rareindex/connectors/testing';
	3	+import metaJson from './meta.json' with { type: 'json' };
	4	+import createConnector from './index.js';
	5	+
	6	+const save = process.argv.includes('--save');
	7	+const meta = ConnectorMetaSchema.parse(metaJson);
	8	+const connector = createConnector(meta);
	9	+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
	10	+const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 2 } });
	11	+let i = 0;
	12	+for await (const raw of connector.crawl(ctx)) {
	13	+ const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
	14	+ console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} sales`);
	15	+ for (const r of out.slice(0, 3)) console.log(' ', JSON.stringify(r).slice(0, 420));
	16	+ if (save && i < 2) saveFixture(meta.id, i === 0 ? 'auction-page-1' : 'auction-page-2', { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 10, kinds: ['sale'], requiredFields: ['saleDate', 'price', 'currency', 'attributes.brand'] }, note: 'Captured live from scotchwhiskyauctions.com (plain HTTPS)' });
	17	+ i++;
	18	+}
	19	+console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);

added connectors/api/scotch-whisky-auctions/index.test.ts +45 −0

		@@ -0,0 +1,45 @@
	1	+import { describe, expect, it } from 'vitest';
	2	+import { ConnectorMetaSchema } from '@rareindex/connectors';
	3	+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	4	+import metaJson from './meta.json' with { type: 'json' };
	5	+import createConnector, { parseAuctionList, parseLotPage, whiskyFacts } from './index.js';
	6	+
	7	+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
	8	+
	9	+describe('scotch-whisky-auctions', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('maps lots to GBP hammer-price auction sales dated by the auction end', async () => {
	13	+ const fx = loadFixture('scotch-whisky-auctions', 'auction-page-1');
	14	+ const out = await connector.normalize(fx.raw);
	15	+ expect(out.length).toBeGreaterThanOrEqual(10);
	16	+ for (const r of out) {
	17	+ if (r.kind !== 'sale') throw new Error('expected sale');
	18	+ expect(r.currency).toBe('GBP');
	19	+ expect(r.buyerPremiumIncluded).toBe(false);
	20	+ expect(r.saleType).toBe('auction');
	21	+ expect(['whisky', 'rum', 'cognac']).toContain(r.attributes.categorySlug);
	22	+ expect(r.attributes.identifiers.swa_item).toMatch(/^\d+$/);
	23	+ expect(r.sourceUrl).toMatch(/^https:\/\/www\.scotchwhiskyauctions\.com\/auctions\/\d+-/);
	24	+ }
	25	+ });
	26	+
	27	+ it('parses list and lot pages', () => {
	28	+ const list = `<a href="/auctions/230-the-181st-auction/" class="auction"><h4>The 181st Auction</h4><h5>Ended July 12, 2026</h5><h6>There are 4926 lots in this auction</h6></a><a href="/auctions/232-the-183rd-auction/" class="auction"><h4>The 183rd Auction</h4><h5>Ends September 13, 2026</h5><h6>There are 4624 lots in this auction</h6></a>`;
	29	+ const auctions = parseAuctionList(list);
	30	+ expect(auctions).toHaveLength(2);
	31	+ expect(auctions[0]).toMatchObject({ id: '230', slug: 'the-181st-auction', ended: true, lotCount: 4926, endedOn: '2026-07-12T00:00:00.000Z' });
	32	+ expect(auctions[1]!.ended).toBe(false);
	33	+ const page = `<select><option>Page 1 of 247</option></select><div id="lots"><a href="/auctions/230-the-181st-auction/885566-on-a-saw-mill/" class="lot"><div class="aucimg" style="background-image: url('https://x/img.jpg');"></div><h4>Macallan 1989 18 Year Old Gran Reserva 70cl</h4><h6>Lot no 181-02514</h6><div><p class="sold">Sold for £1,250 in July 2026</p></div></a></div>`;
	34	+ const p = parseLotPage(page, 'u', auctions[0]!, 1);
	35	+ expect(p.totalPages).toBe(247);
	36	+ expect(p.lots[0]).toMatchObject({ itemId: '885566', lotNo: '181-02514', priceGbp: 1250, image: 'https://x/img.jpg' });
	37	+ });
	38	+
	39	+ it('extracts whisky facts without guessing', () => {
	40	+ expect(whiskyFacts('Macallan 1989 18 Year Old Gran Reserva 70cl')).toMatchObject({ brand: 'Macallan', age: 18, vintage: 1989, size: '70cl', categorySlug: 'whisky' });
	41	+ expect(whiskyFacts('Springbank 10 Year Old 75cl')).toMatchObject({ brand: 'Springbank', age: 10, vintage: null });
	42	+ expect(whiskyFacts('Appleton Estate 21 Year Old Rum')).toMatchObject({ categorySlug: 'rum', age: 21 });
	43	+ expect(whiskyFacts('Hennessy XO Cognac 1L')).toMatchObject({ categorySlug: 'cognac', size: '1L' });
	44	+ });
	45	+});

added connectors/api/scotch-whisky-auctions/index.ts +223 −0

		@@ -0,0 +1,223 @@
	1	+import { z } from 'zod';
	2	+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { parseSourceDate, AssetAttributesSchema, NormalizedSaleSchema, type NormalizedRecord } from '@rareindex/shared';
	4	+
	5	+/**
	6	+ * Scotch Whisky Auctions — monthly online whisky auctions (Glasgow) with a public archive of
	7	+ * results. One raw record per lot-list page; normalise → one sale per lot with a "Sold for" price.
	8	+ */
	9	+
	10	+const BASE = 'https://www.scotchwhiskyauctions.com';
	11	+const PARSER_VERSION = '1.0.0';
	12	+
	13	+export const AuctionSchema = z.object({ id: z.string(), slug: z.string(), title: z.string(), endedOn: z.string().nullable(), lotCount: z.number().nullable(), ended: z.boolean() });
	14	+export type Auction = z.infer<typeof AuctionSchema>;
	15	+
	16	+export const LotSchema = z.object({ itemId: z.string(), url: z.string(), title: z.string(), lotNo: z.string().nullable(), soldText: z.string().nullable(), priceGbp: z.number().nullable(), image: z.string().nullable() });
	17	+export const PagePayloadSchema = z.object({ kind: z.literal('lot_page'), url: z.string(), auction: AuctionSchema, page: z.number(), totalPages: z.number().nullable(), lots: z.array(LotSchema) });
	18	+export type PagePayload = z.infer<typeof PagePayloadSchema>;
	19	+
	20	+/** Parse /auctions/ : cards "The 181st Auction · Ended July 12, 2026 · There are 4926 lots in this auction". */
	21	+export function parseAuctionList(htmlText: string): Auction[] {
	22	+ const $ = H.load(htmlText);
	23	+ const out: Auction[] = [];
	24	+ $('a.auction').each((_, a) => {
	25	+ const href = $(a).attr('href') ?? '';
	26	+ const m = href.match(/^\/auctions\/(\d+)-([^/]+)\/?$/);
	27	+ if (!m) return;
	28	+ const title = H.text($(a).find('h4')) ?? '';
	29	+ const status = H.text($(a).find('h5')) ?? '';
	30	+ const lots = H.text($(a).find('h6'))?.match(/(\d[\d,]*)\s+lots/i)?.[1] ?? null;
	31	+ const ended = /^Ended\b/i.test(status);
	32	+ const dateTxt = status.replace(/^(Ended|Ends)\s*/i, '');
	33	+ const d = parseSourceDate(dateTxt);
	34	+ out.push({ id: m[1]!, slug: m[2]!, title, endedOn: d ? d.toISOString() : null, lotCount: lots ? Number(lots.replace(/,/g, '')) : null, ended });
	35	+ });
	36	+ return out;
	37	+}
	38	+
	39	+/** Parse an auction lot-list page (20 lots): title, lot number, "Sold for £100 in July 2026", image. */
	40	+export function parseLotPage(htmlText: string, url: string, auction: Auction, page: number): PagePayload {
	41	+ const $ = H.load(htmlText);
	42	+ const totalPages = Number(htmlText.match(/Page \d+ of (\d+)/)?.[1]) || null;
	43	+ const lots: z.infer<typeof LotSchema>[] = [];
	44	+ $('a.lot').each((_, a) => {
	45	+ const href = $(a).attr('href') ?? '';
	46	+ const m = href.match(/\/auctions\/\d+-[^/]+\/(\d+)-[^/]*\/?$/);
	47	+ if (!m) return;
	48	+ const title = H.text($(a).find('h4'));
	49	+ if (!title) return;
	50	+ const lotNo = H.text($(a).find('h6'))?.replace(/^Lot no\s*/i, '') ?? null;
	51	+ const soldText = H.text($(a).find('p.sold')) ?? null;
	52	+ const price = soldText?.match(/Sold for £([\d,]+(?:\.\d+)?)/i)?.[1] ?? null;
	53	+ const bg = $(a).find('.aucimg').attr('style') ?? '';
	54	+ const image = bg.match(/url\('([^']+)'\)/)?.[1] ?? null;
	55	+ lots.push({ itemId: m[1]!, url: BASE + href.replace(/\/?$/, '/'), title, lotNo, soldText, priceGbp: price ? Number(price.replace(/,/g, '')) : null, image });
	56	+ });
	57	+ return { kind: 'lot_page', url, auction, page, totalPages, lots };
	58	+}
	59	+
	60	+const SIZE_RE = /(\d+(?:\.\d+)?)\s?(cl|ml|l|litre|liter)s?\b/i;
	61	+const AGE_RE = /(\d{1,2})\s*[- ]?\s*(?:year|yo\b|y\.?o\.?)/i;
	62	+const VINTAGE_RE = /\b(19[2-9]\d|20[0-2]\d)\b(?!\s*(?:year|yo))/i;
	63	+
	64	+export interface WhiskyFacts {
	65	+ brand: string | null;
	66	+ age: number | null;
	67	+ vintage: number | null;
	68	+ size: string | null;
	69	+ categorySlug: 'whisky' | 'rum' | 'cognac';
	70	+}
	71	+
	72	+/** Heuristic facts from a lot title ("Macallan 1989 18 Year Old Gran Reserva 70cl"). Unknown → null, never guessed. */
	73	+export function whiskyFacts(title: string): WhiskyFacts {
	74	+ const t = title.replace(/[‘’]/g, "'");
	75	+ const categorySlug: WhiskyFacts['categorySlug'] = /\brum\b|\brhum\b/i.test(t) ? 'rum' : /\bcognac\b|\barmagnac\b/i.test(t) ? 'cognac' : 'whisky';
	76	+ const age = t.match(AGE_RE)?.[1] ? Number(t.match(AGE_RE)![1]) : null;
	77	+ const vintage = t.match(VINTAGE_RE)?.[1] ? Number(t.match(VINTAGE_RE)![1]) : null;
	78	+ const sizeM = t.match(SIZE_RE);
	79	+ const size = sizeM ? `${sizeM[1]}${sizeM[2]!.toLowerCase().startsWith('l') ? 'L' : sizeM[2]!.toLowerCase()}` : null;
	80	+ // Brand = leading words before the first digit / age / "Year" / "-" ; keep 1–4 words
	81	+ const lead = t.split(/\s+(?=\d)|\s+-\s+|\s+\(/)[0] ?? t;
	82	+ const words = lead.replace(/^['"]|['"]$/g, '').trim().split(/\s+/).filter(Boolean);
	83	+ const brand = words.length ? words.slice(0, Math.min(words.length, 4)).join(' ') : null;
	84	+ return { brand: brand && /[a-z]/i.test(brand) ? brand : null, age, vintage, size, categorySlug };
	85	+}
	86	+
	87	+export class ScotchWhiskyAuctionsConnector extends BaseConnector {
	88	+ readonly version = '1.0.0';
	89	+ readonly parserVersion = PARSER_VERSION;
	90	+ override readonly urlPatterns = [/scotchwhiskyauctions\.com\/auctions\/\d+-[^/]+\/\d+-/];
	91	+ protected override minIntervalMs = 2000;
	92	+
	93	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	94	+ const listUrl = String(this.meta.config.auctionsUrl ?? `${BASE}/auctions/`);
	95	+ const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);
	96	+ const pagesPerRun = Number(this.meta.config.pagesPerRun ?? 12);
	97	+ const cursor = (ctx.options.cursor ?? {}) as { progress?: Record<string, number>; complete?: string[] };
	98	+ const progress: Record<string, number> = { ...(cursor.progress ?? {}) };
	99	+ const complete = new Set<string>(cursor.complete ?? []);
	100	+
	101	+ const list = await ctx.fetch(listUrl, { engines: ['api'], responseType: 'text', minQuality: 0 });
	102	+ if (!list.success || !list.html) {
	103	+ ctx.anomaly('auction_list_failed', list.error ?? String(list.httpStatus));
	104	+ return;
	105	+ }
	106	+ let auctions = parseAuctionList(list.html).filter((a) => a.ended);
	107	+ // newest first; incremental = newest auctions not yet complete, backfill = continue with older ones
	108	+ auctions.sort((a, b) => Number(b.id) - Number(a.id));
	109	+ if (ctx.options.mode !== 'backfill') auctions = auctions.slice(0, 6);
	110	+ let count = 0;
	111	+ let pagesFetched = 0;
	112	+ let auctionsTouched = 0;
	113	+ for (const auction of auctions) {
	114	+ if (complete.has(auction.id)) continue;
	115	+ if (auctionsTouched >= auctionsPerRun) break;
	116	+ auctionsTouched++;
	117	+ let page = (progress[auction.id] ?? 0) + 1;
	118	+ for (;;) {
	119	+ if (ctx.signal?.aborted || this.reached(ctx, count) || pagesFetched >= pagesPerRun) return void (await ctx.setCursor({ progress, complete: [...complete] }));
	120	+ const url = `${BASE}/auctions/${auction.id}-${auction.slug}/${page > 1 ? `?page=${page}` : ''}`;
	121	+ await this.throttle();
	122	+ const res = await ctx.fetch(url, {
	123	+ engines: ['api'],
	124	+ responseType: 'text',
	125	+ expect: ['title', 'price', 'currency', 'status'],
	126	+ parse: (r) => {
	127	+ const p = r.html ? parseLotPage(r.html, url, auction, page) : null;
	128	+ const sold = p?.lots.find((l) => l.priceGbp);
	129	+ return p && p.lots.length ? { title: p.lots[0]!.title, price: sold?.priceGbp ?? null, currency: sold ? 'GBP' : null, status: sold ? 'sold' : null } : null;
	130	+ },
	131	+ });
	132	+ pagesFetched++;
	133	+ const payload = res.success && res.html ? parseLotPage(res.html, url, auction, page) : null;
	134	+ if (!payload || payload.lots.length === 0) {
	135	+ ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
	136	+ complete.add(auction.id);
	137	+ break;
	138	+ }
	139	+ count++;
	140	+ yield { url, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	141	+ progress[auction.id] = page;
	142	+ if (payload.totalPages && page >= payload.totalPages) {
	143	+ complete.add(auction.id);
	144	+ break;
	145	+ }
	146	+ page++;
	147	+ }
	148	+ await ctx.setCursor({ progress, complete: [...complete] });
	149	+ }
	150	+ }
	151	+
	152	+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
	153	+ const m = url.match(/\/auctions\/(\d+)-([^/]+)\/(\d+)-/);
	154	+ if (!m) return [];
	155	+ const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0 });
	156	+ if (!res.success || !res.html) return [];
	157	+ const $ = H.load(res.html);
	158	+ const title = H.text($('h1').first()) ?? H.text($('h4').first());
	159	+ const soldText = res.html.match(/Sold for £[\d,]+(?:\.\d+)?[^<]*/)?.[0] ?? null;
	160	+ const lotNo = res.html.match(/Lot no\s*([\d-]+)/i)?.[1] ?? null;
	161	+ const ended = res.html.match(/Ended\s+([A-Z][a-z]+ \d{1,2}, \d{4})/)?.[1] ?? null;
	162	+ const endedOn = ended ? parseSourceDate(ended)?.toISOString() ?? null : null;
	163	+ if (!title) return [];
	164	+ const price = soldText?.match(/£([\d,]+(?:\.\d+)?)/)?.[1] ?? null;
	165	+ const image = $('meta[property="og:image"]').attr('content') ?? null;
	166	+ const auction: Auction = { id: m[1]!, slug: m[2]!, title: '', endedOn, lotCount: null, ended: Boolean(endedOn) };
	167	+ const payload: PagePayload = { kind: 'lot_page', url, auction, page: 0, totalPages: null, lots: [{ itemId: m[3]!, url, title, lotNo, soldText, priceGbp: price ? Number(price.replace(/,/g, '')) : null, image }] };
	168	+ return [{ url, externalId: `lot:${m[3]}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];
	169	+ }
	170	+
	171	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	172	+ const p = PagePayloadSchema.parse(raw.payload);
	173	+ const saleDate = p.auction.endedOn ? new Date(p.auction.endedOn) : null;
	174	+ if (!saleDate) return [];
	175	+ const out: NormalizedRecord[] = [];
	176	+ for (const lot of p.lots) {
	177	+ if (!lot.priceGbp || lot.priceGbp <= 0) continue;
	178	+ const f = whiskyFacts(lot.title);
	179	+ const attributes = AssetAttributesSchema.parse({
	180	+ categorySlug: f.categorySlug,
	181	+ brand: f.brand,
	182	+ name: lot.title,
	183	+ year: f.vintage,
	184	+ size: f.size,
	185	+ country: f.categorySlug === 'whisky' && /scotch|islay|speyside|highland|campbeltown|lowland/i.test(lot.title) ? 'GB' : null,
	186	+ identifiers: { swa_item: lot.itemId, ...(lot.lotNo ? { swa_lot: lot.lotNo } : {}) },
	187	+ metadata: { age_statement: f.age, auction_id: p.auction.id, auction_title: p.auction.title, sold_text: lot.soldText },
	188	+ });
	189	+ out.push(
	190	+ NormalizedSaleSchema.parse({
	191	+ kind: 'sale',
	192	+ connectorId: this.meta.id,
	193	+ sourceId: this.meta.sourceId,
	194	+ sourceUrl: lot.url,
	195	+ externalId: lot.itemId,
	196	+ rawTitle: lot.title,
	197	+ imageUrls: lot.image ? [lot.image] : [],
	198	+ attributes,
	199	+ grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
	200	+ condition: { condition: null, conditionRaw: null, completeness: null },
	201	+ observedAt: raw.fetchedAt,
	202	+ confidence: 0.9,
	203	+ parserVersion: PARSER_VERSION,
	204	+ saleType: 'auction',
	205	+ saleDate,
	206	+ price: lot.priceGbp,
	207	+ currency: 'GBP',
	208	+ buyerPremiumIncluded: false,
	209	+ quantity: 1,
	210	+ isBundle: /\b(x\s?\d|\d+\s?x\s|lot of|set of|\d+\s?bottles)\b/i.test(lot.title),
	211	+ location: 'Glasgow, United Kingdom',
	212	+ auctionHouse: 'Scotch Whisky Auctions',
	213	+ lotNumber: lot.lotNo,
	214	+ }),
	215	+ );
	216	+ }
	217	+ return out;
	218	+ }
	219	+}
	220	+
	221	+export default function createConnector(meta: ConnectorMeta) {
	222	+ return new ScotchWhiskyAuctionsConnector(meta);
	223	+}

added connectors/api/scotch-whisky-auctions/meta.json +34 −0

		@@ -0,0 +1,34 @@
	1	+{
	2	+ "id": "scotch-whisky-auctions",
	3	+ "displayName": "Scotch Whisky Auctions (results)",
	4	+ "sourceId": "scotch-whisky-auctions",
	5	+ "sourceName": "Scotch Whisky Auctions",
	6	+ "sourceType": "auction_house",
	7	+ "sourceUrl": "https://www.scotchwhiskyauctions.com",
	8	+ "module": "api/scotch-whisky-auctions",
	9	+ "enginePriority": ["api", "firecrawl", "scrapfly"],
	10	+ "categories": ["whisky", "rum", "cognac"],
	11	+ "regions": ["GB"],
	12	+ "languages": ["en"],
	13	+ "currency": ["GBP"],
	14	+ "supportsListings": false,
	15	+ "supportsSold": true,
	16	+ "supportsAuctions": true,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": false,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": true,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "high",
	23	+ "trustScore": 0.9,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://www.scotchwhiskyauctions.com/terms/",
	26	+ "accessNotes": "Public monthly auction archive (Glasgow; ~4,500–5,000 lots per auction, 20 lots per page, plain HTTPS with the RareIndex user agent, robots.txt disallows only /cmsplus/). Each lot card exposes title, lot number and 'Sold for £X in <month year>' — this is the hammer price; the buyer's commission (SWA charges a separate buyer's premium plus VAT) is NOT included, so buyer_premium_included=false. Sale date = the auction end date shown on the auction page. Unsold lots are skipped. 2 s politeness delay; pagesPerRun caps each incremental run (backfill mode walks older auctions).",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "auctionsUrl": "https://www.scotchwhiskyauctions.com/auctions/",
	31	+ "auctionsPerRun": 2,
	32	+ "pagesPerRun": 12
	33	+ }
	34	+}

added connectors/api/winebid/_smoke.ts +20 −0

		@@ -0,0 +1,20 @@
	1	+import { createRouter, createCrawlContext, ConnectorMetaSchema } from '@rareindex/connectors';
	2	+import { saveFixture } from '@rareindex/connectors/testing';
	3	+import metaJson from './meta.json' with { type: 'json' };
	4	+import createConnector from './index.js';
	5	+
	6	+const save = process.argv.includes('--save');
	7	+const meta = ConnectorMetaSchema.parse(metaJson);
	8	+const connector = createConnector(meta);
	9	+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
	10	+const ctx = createCrawlContext({ router, meta, options: { mode: 'probe', limit: 4 } });
	11	+let i = 0;
	12	+for await (const raw of connector.crawl(ctx)) {
	13	+ const out = await connector.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() });
	14	+ const sales = out.filter((r) => r.kind === 'sale');
	15	+ console.log(`raw ${raw.externalId} (${raw.engine}) → ${out.length} records, ${sales.length} sales`);
	16	+ for (const r of sales.slice(0, 2)) console.log(' ', JSON.stringify(r).slice(0, 420));
	17	+ if (save && i < 2 && sales.length) saveFixture(meta.id, `item-${(raw.externalId ?? '').replace('item:', '')}`, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt: raw.fetchedAt ?? new Date(), payload: raw.payload }, expect: { minCount: 2, kinds: ['sale', 'listing'], requiredFields: ['rawTitle', 'attributes.name'] }, note: 'Captured live from winebid.com item page (plain HTTPS)' });
	18	+ if (sales.length) i++;
	19	+}
	20	+console.log('engineStats', ctx.engineStats, 'anomalies', ctx.anomalies);

added connectors/api/winebid/index.test.ts +37 −0

		@@ -0,0 +1,37 @@
	1	+import { describe, expect, it } from 'vitest';
	2	+import { ConnectorMetaSchema } from '@rareindex/connectors';
	3	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	4	+import metaJson from './meta.json' with { type: 'json' };
	5	+import createConnector, { parseCategoryPage, parseItemPage, wineFacts } from './index.js';
	6	+
	7	+const connector = createConnector(ConnectorMetaSchema.parse(metaJson));
	8	+
	9	+describe('winebid', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('emits per-bottle USD sales (premium excluded) dated by the recent-sales table', async () => {
	13	+ const name = listFixtures('winebid')[0]!;
	14	+ const out = await connector.normalize(loadFixture('winebid', name).raw);
	15	+ const sales = out.filter((r) => r.kind === 'sale');
	16	+ expect(sales.length).toBeGreaterThan(0);
	17	+ for (const r of sales) {
	18	+ if (r.kind !== 'sale') continue;
	19	+ expect(r.currency).toBe('USD');
	20	+ expect(r.buyerPremiumIncluded).toBe(false);
	21	+ expect(r.attributes.metadata.buyer_premium_pct).toBe(17);
	22	+ expect(r.saleDate.getFullYear()).toBeGreaterThanOrEqual(2015);
	23	+ expect(r.sourceUrl).toMatch(/^https:\/\/www\.winebid\.com\/BuyWine\/Item\/\d+/);
	24	+ }
	25	+ });
	26	+
	27	+ it('parses item pages, category links and titles', () => {
	28	+ const html = `<h1>1983 Ch&#226;teau Margaux (1.5L)</h1><p>Light capsule condition issue</p><div id="itemDiagnostics" data-is-ended="False"></div><table class="recentSales"><thead><tr><th>Item</th><th>Sold</th><th>Amount</th><th>Date</th></tr></thead><tr><td><a href="/BuyWine/Item/10856401/1983-Chateau-Margaux">I10856401</a></td><td class="text-right">2</td><td class="text-right">$520</td><td class="text-right" title="August 23, 2026 12:00:00 AM (PDT)">Aug 23, 2026</td></tr></table>`;
	29	+ const p = parseItemPage(html, 'https://www.winebid.com/BuyWine/Item/10881873/1983-Chateau-Margaux', 'Bordeaux')!;
	30	+ expect(p.itemId).toBe('10881873');
	31	+ expect(p.title).toBe('1983 Château Margaux (1.5L)');
	32	+ expect(p.recentSales).toEqual([{ itemId: '10856401', url: 'https://www.winebid.com/BuyWine/Item/10856401/1983-Chateau-Margaux', quantity: 2, amountUsd: 520, soldOn: 'August 23, 2026 12:00:00 AM (PDT)' }]);
	33	+ expect(wineFacts('1983 Château Margaux (1.5L)')).toEqual({ vintage: 1983, name: 'Château Margaux', size: '1.5L' });
	34	+ expect(wineFacts('2019 Opus One')).toEqual({ vintage: 2019, name: 'Opus One', size: '750ml' });
	35	+ expect(parseCategoryPage('[x](https://www.winebid.com/BuyWine/Item/10887720/2019-Chateau-Haut-Brion-~) <a href="/BuyWine/Item/1/a">a</a>')).toEqual(['https://www.winebid.com/BuyWine/Item/10887720/2019-Chateau-Haut-Brion-~', 'https://www.winebid.com/BuyWine/Item/1/a']);
	36	+ });
	37	+});

added connectors/api/winebid/index.ts +215 −0

		@@ -0,0 +1,215 @@
	1	+import { z } from 'zod';
	2	+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { AssetAttributesSchema, NormalizedListingSchema, NormalizedSaleSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';
	4	+
	5	+/**
	6	+ * WineBid — item pages expose a "Recent sales" table (past winning bids for the same wine).
	7	+ * One raw record per item page; normalise → one sale per past sale row plus the current listing.
	8	+ */
	9	+
	10	+const BASE = 'https://www.winebid.com';
	11	+const PARSER_VERSION = '1.0.0';
	12	+const PREMIUM_PCT = 17;
	13	+
	14	+const CategorySchema = z.object({ path: z.string(), label: z.string() });
	15	+
	16	+export const SaleRowSchema = z.object({ itemId: z.string(), url: z.string(), quantity: z.number(), amountUsd: z.number(), soldOn: z.string() });
	17	+export const ItemPayloadSchema = z.object({
	18	+ kind: z.literal('item_page'),
	19	+ url: z.string(),
	20	+ itemId: z.string(),
	21	+ title: z.string(),
	22	+ category: z.string().nullable(),
	23	+ condition: z.string().nullable(),
	24	+ currentBid: z.number().nullable(),
	25	+ estimate: z.number().nullable(),
	26	+ ended: z.boolean().nullable(),
	27	+ image: z.string().nullable(),
	28	+ recentSales: z.array(SaleRowSchema),
	29	+});
	30	+export type ItemPayload = z.infer<typeof ItemPayloadSchema>;
	31	+
	32	+export function parseCategoryPage(text: string): string[] {
	33	+ const out = new Set<string>();
	34	+ for (const m of text.matchAll(/https:\/\/www\.winebid\.com\/BuyWine\/Item\/(\d+)\/[^)\s"'<>]*/g)) out.add(`${BASE}/BuyWine/Item/${m[1]}/${m[0].split('/').pop()!.replace(/[)"'>]+$/g, '')}`);
	35	+ for (const m of text.matchAll(/href="(\/BuyWine\/Item\/(\d+)\/[^"]+)"/g)) out.add(BASE + m[1]!);
	36	+ return [...out];
	37	+}
	38	+
	39	+export function parseItemPage(htmlText: string, url: string, category: string | null): ItemPayload | null {
	40	+ const $ = H.load(htmlText);
	41	+ const itemId = url.match(/\/BuyWine\/Item\/(\d+)/)?.[1];
	42	+ const title = H.text($('h1').first());
	43	+ if (!itemId || !title) return null;
	44	+ const condition = H.text($('h1').first().nextAll('p').first());
	45	+ const recentSales: z.infer<typeof SaleRowSchema>[] = [];
	46	+ $('table.recentSales tr').each((_, tr) => {
	47	+ const tds = $(tr).find('td');
	48	+ if (tds.length < 4) return;
	49	+ const a = $(tds[0]).find('a');
	50	+ const href = a.attr('href') ?? '';
	51	+ const id = href.match(/\/BuyWine\/Item\/(\d+)/)?.[1];
	52	+ const qty = Number(H.text($(tds[1])) ?? '');
	53	+ const amount = parsePrice(H.text($(tds[2])), 'USD');
	54	+ const soldOn = $(tds[3]).attr('title') ?? H.text($(tds[3])) ?? '';
	55	+ if (!id || !amount || !soldOn) return;
	56	+ recentSales.push({ itemId: id, url: BASE + href, quantity: Number.isFinite(qty) && qty > 0 ? qty : 1, amountUsd: amount.amount, soldOn });
	57	+ });
	58	+ const bidTxt = H.text($('.currentBid, .bidAmount, #currentBid').first()) ?? htmlText.match(/Current Bid[^$]{0,80}\$([\d,]+(?:\.\d+)?)/i)?.[1] ?? null;
	59	+ const currentBid = bidTxt ? (parsePrice(bidTxt, 'USD')?.amount ?? null) : null;
	60	+ const estimate = parsePrice(htmlText.match(/Estimate[^$]{0,120}\$([\d,]+(?:\.\d+)?)/i)?.[1] ?? null, 'USD')?.amount ?? null;
	61	+ const endedAttr = $('#itemDiagnostics').attr('data-is-ended');
	62	+ const image = $('meta[property="og:image"]').attr('content') ?? $('img[src*="/Photo/"]').first().attr('src') ?? null;
	63	+ return { kind: 'item_page', url, itemId, title, category, condition: condition ?? null, currentBid, estimate, ended: endedAttr ? endedAttr === 'True' : null, image: image ? (image.startsWith('http') ? image : BASE + image) : null, recentSales };
	64	+}
	65	+
	66	+/** "1983 Château Margaux (1.5L)" → { vintage, producer, size } — nothing guessed. */
	67	+export function wineFacts(title: string): { vintage: number | null; name: string; size: string | null } {
	68	+ const vintage = title.match(/^(19\d{2}|20\d{2})\b/)?.[1] ?? null;
	69	+ const size = title.match(/\((\d+(?:\.\d+)?\s?(?:L|ml|mL))\)/i)?.[1]?.replace(/\s/g, '') ?? null;
	70	+ const name = title
	71	+ .replace(/^(19\d{2}|20\d{2}|NV)\s+/, '')
	72	+ .replace(/\s*\((\d+(?:\.\d+)?\s?(?:L|ml|mL))\)\s*/i, ' ')
	73	+ .trim();
	74	+ return { vintage: vintage ? Number(vintage) : null, name, size: size ?? '750ml' };
	75	+}
	76	+
	77	+export class WineBidConnector extends BaseConnector {
	78	+ readonly version = '1.0.0';
	79	+ readonly parserVersion = PARSER_VERSION;
	80	+ override readonly urlPatterns = [/winebid\.com\/BuyWine\/Item\/\d+/];
	81	+ protected override minIntervalMs = 2000;
	82	+
	83	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	84	+ const categories = z.array(CategorySchema).parse(this.meta.config.categories ?? []);
	85	+ const perRun = Number(this.meta.config.itemsPerRun ?? 24);
	86	+ const cursor = (ctx.options.cursor ?? {}) as { seen?: string[] };
	87	+ const seen = new Set<string>(cursor.seen ?? []);
	88	+ let count = 0;
	89	+ let fetched = 0;
	90	+ const perCategory = Math.max(2, Math.ceil(perRun / Math.max(1, categories.length)));
	91	+ for (const cat of categories) {
	92	+ if (ctx.signal?.aborted || fetched >= perRun) break;
	93	+ const listUrl = BASE + cat.path;
	94	+ const list = await ctx.fetch(listUrl, { engines: ['firecrawl'], timeoutMs: 60_000, minQuality: 0 });
	95	+ if (!list.success) {
	96	+ ctx.anomaly('category_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);
	97	+ continue;
	98	+ }
	99	+ const urls = parseCategoryPage(`${list.markdown ?? ''}\n${list.html ?? ''}`);
	100	+ if (urls.length === 0) {
	101	+ ctx.anomaly('empty_category', listUrl);
	102	+ continue;
	103	+ }
	104	+ let done = 0;
	105	+ for (const url of urls) {
	106	+ if (ctx.signal?.aborted || this.reached(ctx, count) || fetched >= perRun || done >= perCategory) break;
	107	+ const id = url.match(/\/Item\/(\d+)/)?.[1] ?? url;
	108	+ if (seen.has(id) && ctx.options.mode !== 'backfill') continue;
	109	+ await this.throttle();
	110	+ fetched++;
	111	+ const res = await ctx.fetch(url, {
	112	+ engines: ['api', 'firecrawl'],
	113	+ responseType: 'text',
	114	+ expect: ['title', 'price', 'currency', 'date'],
	115	+ parse: (r) => {
	116	+ const p = r.html ? parseItemPage(r.html, url, cat.label) : null;
	117	+ const s = p?.recentSales[0];
	118	+ return p ? { title: p.title, price: s?.amountUsd ?? p.currentBid, currency: 'USD', date: s?.soldOn ?? null } : null;
	119	+ },
	120	+ });
	121	+ const payload = res.success && res.html ? parseItemPage(res.html, url, cat.label) : null;
	122	+ if (!payload) {
	123	+ ctx.anomaly('item_parse_failed', `${url}: ${res.error ?? res.httpStatus}`);
	124	+ continue;
	125	+ }
	126	+ seen.add(id);
	127	+ done++;
	128	+ count++;
	129	+ yield { url, externalId: `item:${payload.itemId}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	130	+ }
	131	+ await ctx.setCursor({ seen: [...seen].slice(-3000) });
	132	+ }
	133	+ }
	134	+
	135	+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
	136	+ const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0 });
	137	+ const payload = res.success && res.html ? parseItemPage(res.html, url, null) : null;
	138	+ return payload ? [{ url, externalId: `item:${payload.itemId}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];
	139	+ }
	140	+
	141	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	142	+ const p = ItemPayloadSchema.parse(raw.payload);
	143	+ const f = wineFacts(p.title);
	144	+ const categorySlug = /whisk|bourbon|scotch/i.test(p.title) ? 'whisky' : 'wine';
	145	+ const attributes = AssetAttributesSchema.parse({
	146	+ categorySlug,
	147	+ series: p.category,
	148	+ name: f.name,
	149	+ year: f.vintage,
	150	+ size: f.size,
	151	+ identifiers: { winebid_wine: `${f.vintage ?? 'NV'}|${f.name.toLowerCase()}|${f.size ?? ''}` },
	152	+ metadata: { category_page: p.category },
	153	+ });
	154	+ const common = {
	155	+ connectorId: this.meta.id,
	156	+ sourceId: this.meta.sourceId,
	157	+ attributes,
	158	+ grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },
	159	+ condition: { condition: null, conditionRaw: p.condition, completeness: null },
	160	+ observedAt: raw.fetchedAt,
	161	+ confidence: 0.85,
	162	+ parserVersion: PARSER_VERSION,
	163	+ imageUrls: p.image ? [p.image] : [],
	164	+ description: null,
	165	+ };
	166	+ const out: NormalizedRecord[] = [];
	167	+ for (const s of p.recentSales) {
	168	+ const saleDate = parseSourceDate(s.soldOn.replace(/\s+\d{1,2}:\d{2}:\d{2}\s*(AM|PM)?\s*\([A-Z]{3,4}\)$/i, ''));
	169	+ if (!saleDate) continue;
	170	+ out.push(
	171	+ NormalizedSaleSchema.parse({
	172	+ kind: 'sale',
	173	+ ...common,
	174	+ sourceUrl: s.url,
	175	+ externalId: s.itemId,
	176	+ rawTitle: p.title,
	177	+ saleType: 'auction',
	178	+ saleDate,
	179	+ price: s.amountUsd,
	180	+ currency: 'USD',
	181	+ buyerPremiumIncluded: false,
	182	+ quantity: s.quantity,
	183	+ isBundle: false,
	184	+ location: 'United States',
	185	+ auctionHouse: 'WineBid',
	186	+ lotNumber: s.itemId,
	187	+ attributes: { ...attributes, metadata: { ...attributes.metadata, price_basis: 'per_bottle', buyer_premium_pct: PREMIUM_PCT } },
	188	+ }),
	189	+ );
	190	+ }
	191	+ if (p.currentBid || p.estimate) {
	192	+ out.push(
	193	+ NormalizedListingSchema.parse({
	194	+ kind: 'listing',
	195	+ ...common,
	196	+ sourceUrl: p.url,
	197	+ externalId: p.itemId,
	198	+ rawTitle: p.title,
	199	+ listingType: 'auction',
	200	+ price: p.currentBid ?? p.estimate,
	201	+ currency: 'USD',
	202	+ seller: 'WineBid',
	203	+ location: 'United States',
	204	+ availability: p.ended === true ? 'ended' : 'available',
	205	+ attributes: { ...attributes, metadata: { ...attributes.metadata, estimate_usd: p.estimate, price_is_estimate: p.currentBid === null } },
	206	+ }),
	207	+ );
	208	+ }
	209	+ return out;
	210	+ }
	211	+}
	212	+
	213	+export default function createConnector(meta: ConnectorMeta) {
	214	+ return new WineBidConnector(meta);
	215	+}

added connectors/api/winebid/meta.json +40 −0

		@@ -0,0 +1,40 @@
	1	+{
	2	+ "id": "winebid",
	3	+ "displayName": "WineBid (recent sales per wine)",
	4	+ "sourceId": "winebid",
	5	+ "sourceName": "WineBid",
	6	+ "sourceType": "marketplace",
	7	+ "sourceUrl": "https://www.winebid.com",
	8	+ "module": "api/winebid",
	9	+ "enginePriority": ["api", "firecrawl", "scrapfly"],
	10	+ "categories": ["wine", "whisky"],
	11	+ "regions": ["US"],
	12	+ "languages": ["en"],
	13	+ "currency": ["USD"],
	14	+ "supportsListings": true,
	15	+ "supportsSold": true,
	16	+ "supportsAuctions": true,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": false,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": true,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "medium",
	23	+ "trustScore": 0.85,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://www.winebid.com/Help/TermsAndConditions",
	26	+ "accessNotes": "US online wine auction (weekly auctions). Category browse pages (/BuyWine/Items/<region>/<id>) are rendered through Firecrawl (1 credit, plain HTTP returns 403 for list pages) to discover item URLs; each public item page (/BuyWine/Item/<id>/<slug>, plain HTTPS) shows the wine, the current auction lot (estimate / current bid) and a 'Recent sales' table of past winning bids for the same wine: item id, quantity sold, per-bottle amount (USD) and sale date. robots.txt lists rules for named search engines only (no generic group); we still avoid /Sales, /AuctionClosed and account paths. Winning bids exclude WineBid's 17% buyer's premium (buyer_premium_included=false; premium % in metadata). Sale dates come from the table (PDT). 2 s politeness delay; itemsPerRun caps each run.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "categories": [
	31	+ { "path": "/BuyWine/Items/Bordeaux/18403", "label": "Bordeaux" },
	32	+ { "path": "/BuyWine/Items/Burgundy/15363", "label": "Burgundy" },
	33	+ { "path": "/BuyWine/Items/Napa-Valley/2910", "label": "Napa Valley" },
	34	+ { "path": "/BuyWine/Items/Barolo/16367", "label": "Barolo" },
	35	+ { "path": "/BuyWine/Items/Brunello-di-Montalcino/7725165", "label": "Brunello di Montalcino" },
	36	+ { "path": "/BuyWine/Items/Big-Bottles/7902856", "label": "Large formats" }
	37	+ ],
	38	+ "itemsPerRun": 24
	39	+ }
	40	+}

added connectors/firecrawl/artcurial/_smoke.ts +20 −0

Large diff (20 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/artcurial/index.test.ts +39 −0

Large diff (39 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/artcurial/index.ts +216 −0

Large diff (216 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/artcurial/meta.json +33 −0

Large diff (33 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/phillips-art/_smoke.ts +20 −0

Large diff (20 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/phillips-art/index.test.ts +31 −0

Large diff (31 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/phillips-art/index.ts +153 −0

Large diff (153 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/phillips-art/meta.json +41 −0

Large diff (41 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/artcurial/sale-6557.json +2158 −0

Large diff (2,159 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/artcurial/sale-IT6558.json +558 −0

Large diff (559 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/idealwine/bordeaux-page-1.json +718 −0

Large diff (719 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/idealwine/bordeaux-page-2.json +694 −0

Large diff (695 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/lyon-turnbull/auction-907.json +772 −0

Large diff (773 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/phillips-art/sale-NY011126.json +37 −0

Large diff (38 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/phillips-art/sale-UK010626.json +37 −0

Large diff (38 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/saatchi-art/paintings-page-1.json +801 −0

Large diff (802 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/saatchi-art/paintings-page-2.json +821 −0

Large diff (822 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/scotch-whisky-auctions/auction-page-1.json +219 −0

Large diff (220 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/scotch-whisky-auctions/auction-page-2.json +219 −0

Large diff (220 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/winebid/item-10887719.json +78 −0

Large diff (79 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/winebid/item-10887720.json +78 −0

Large diff (80 lines) is not loaded by default to keep the page fast.Load diff