import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; /** * iDealwine "cote" (Price Estimate) pages — per-vintage auction-derived reference prices for * French and Italian wines. One raw record per index page; normalise → one catalog_item per * wine-vintage plus one guide_value observation (EUR / bottle). */ const BASE = 'https://www.idealwine.com'; const PARSER_VERSION = '1.0.0'; export const VintageSchema = z.object({ vintage: z.number().nullable(), priceEur: z.number(), url: z.string(), ref: z.string() }); export const WineSchema = z.object({ wineId: z.string(), name: z.string(), color: z.string().nullable(), region: z.string().nullable(), appellation: z.string().nullable(), producerSlug: z.string().nullable(), image: z.string().nullable(), vintages: z.array(VintageSchema) }); export const PagePayloadSchema = z.object({ kind: z.literal('cote_page'), url: z.string(), region: z.string(), page: z.number(), totalPages: z.number().nullable(), wines: z.array(WineSchema) }); export type PagePayload = z.infer; /** Parse a /en/cote/ page (cards with per-vintage estimates). */ export function parseCotePage(htmlText: string, url: string, region: string, page: number): PagePayload { const $ = H.load(htmlText); const wines: z.infer[] = []; const pages = [...htmlText.matchAll(/\/en\/cote\/[a-z-]+\?page=(\d+)/g)].map((m) => Number(m[1])); const totalPages = pages.length ? Math.max(...pages) : null; $('[class*="cote_gridItem"]').each((_, el) => { const $el = $(el); const crumb = H.text($el.find('[class*="cote_breadCrumpRegion"]')) ?? ''; const [reg, app] = crumb.split('>').map((s) => s.trim()); const nameEl = $el.find('[class*="cote_nameWine"]').first(); const color = H.text(nameEl.find('[class*="cote_colorDisplay"]'))?.replace(/[()]/g, '').trim() ?? null; nameEl.find('[class*="cote_colorDisplay"]').remove(); const name = H.text(nameEl); const image = $el.find('img').attr('src') ?? null; const vintages: z.infer[] = []; let wineId: string | null = null; let producerSlug: string | null = null; $el.find('a[href*="/en/wine-prices/"]').each((__, a) => { const href = $(a).attr('href') ?? ''; const m = href.match(/\/en\/wine-prices\/(\d+)-(\d{4}|NV|[^-]+)-Bottle-(.+)$/i); const id = href.match(/\/en\/wine-prices\/(\d+)-/)?.[1] ?? null; const vTxt = H.text($(a).find('[class*="VintageRatingsCards_cote"]')) ?? ''; const vintage = vTxt.match(/^(\d{4})/)?.[1] ?? null; const priceTxt = vTxt.match(/€\s*[\d\s.,]+/)?.[0] ?? null; const price = priceTxt ? parsePrice(priceTxt.replace(/\s/g, ''), 'EUR')?.amount ?? null : null; if (!id || !price) return; wineId ??= id; if (!producerSlug && m?.[3]) producerSlug = m[3]!; vintages.push({ vintage: vintage ? Number(vintage) : null, priceEur: price, url: BASE + href, ref: href.replace(/^\/en\/wine-prices\//, '').split('-Bottle-')[0]! }); }); if (!name || !wineId || vintages.length === 0) return; wines.push({ wineId, name, color, region: reg ?? null, appellation: app ?? null, producerSlug, image: image && !image.includes('_no_picture') ? (image.startsWith('http') ? image : BASE + image) : null, vintages }); }); return { kind: 'cote_page', url, region, page, totalPages, wines }; } /** "Chateau-de-Fieuzal-red" → "Château de Fieuzal" is not recoverable exactly; keep a readable producer label without guessing accents. */ export function producerFromSlug(slug: string | null): string | null { if (!slug) return null; const s = slug.replace(/-(red|white|rose|sparkling|sweet)$/i, '').replace(/^(Second-Wine-|Cru-bourgeois-)/i, ''); const label = s.replace(/-/g, ' ').trim(); return label || null; } export class IdealwineConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; async *crawl(ctx: CrawlContext): AsyncIterable { const regions = z.array(z.string()).parse(this.meta.config.regions ?? ['bordeaux']); const pages = Number(this.meta.config.pagesPerRegion ?? 3); const cursor = (ctx.options.cursor ?? {}) as { next?: Record }; const next: Record = { ...(cursor.next ?? {}) }; let count = 0; for (const region of regions) { const start = ctx.options.mode === 'backfill' ? (next[region] ?? 1) : 1; for (let page = start; page < start + pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return void (await ctx.setCursor({ next })); const url = `${BASE}/en/cote/${region}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'currency'], parse: (r) => { const p = r.html ? parseCotePage(r.html, url, region, page) : null; const w = p?.wines[0]; return w ? { title: w.name, price: w.vintages[0]?.priceEur ?? null, currency: 'EUR' } : null; }, }); const payload = res.success && res.html ? parseCotePage(res.html, url, region, page) : null; if (!payload || payload.wines.length === 0) { ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); next[region] = 1; break; } count++; yield { url, externalId: `cote:${region}:${page}`, kind: 'price_observation', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; next[region] = payload.totalPages && page >= payload.totalPages ? 1 : page + 1; } } await ctx.setCursor({ next }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const observationDate = new Date(Date.UTC(raw.fetchedAt.getUTCFullYear(), raw.fetchedAt.getUTCMonth(), raw.fetchedAt.getUTCDate())); for (const w of p.wines) { for (const v of w.vintages) { const attributes = AssetAttributesSchema.parse({ categorySlug: 'wine', brand: producerFromSlug(w.producerSlug), series: w.region, set: w.appellation, name: w.name, year: v.vintage, color: w.color, size: '75cl', country: /italie|italy/i.test(p.region) ? 'IT' : 'FR', identifiers: { idealwine_wine_id: w.wineId, idealwine_ref: v.ref }, metadata: { region_page: p.region }, }); const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: v.url, externalId: v.ref, rawTitle: `${w.name}${v.vintage ? ` ${v.vintage}` : ''}${w.appellation ? ` · ${w.appellation}` : ''}${w.color ? ` (${w.color})` : ''}`, description: null, imageUrls: w.image ? [w.image] : [], attributes, grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: null, conditionRaw: null, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.75, parserVersion: PARSER_VERSION, }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, releaseDate: null })); out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, priceKind: 'guide_value', price: v.priceEur, currency: 'EUR', observationDate, sampleSize: null })); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new IdealwineConnector(meta); }