TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';45/**6 * iDealwine "cote" (Price Estimate) pages — per-vintage auction-derived reference prices for7 * French and Italian wines. One raw record per index page; normalise → one catalog_item per8 * wine-vintage plus one guide_value observation (EUR / bottle).9 */1011const BASE = 'https://www.idealwine.com';12const PARSER_VERSION = '1.0.0';1314export const VintageSchema = z.object({ vintage: z.number().nullable(), priceEur: z.number(), url: z.string(), ref: z.string() });15export const WineSchema = z.object({ wineId: z.string(), name: z.string(), color: z.string().nullable(), region: z.string().nullable(), appellation: z.string().nullable(), producerSlug: z.string().nullable(), image: z.string().nullable(), vintages: z.array(VintageSchema) });16export const PagePayloadSchema = z.object({ kind: z.literal('cote_page'), url: z.string(), region: z.string(), page: z.number(), totalPages: z.number().nullable(), wines: z.array(WineSchema) });17export type PagePayload = z.infer<typeof PagePayloadSchema>;1819/** Parse a /en/cote/<region> page (cards with per-vintage estimates). */20export function parseCotePage(htmlText: string, url: string, region: string, page: number): PagePayload {21 const $ = H.load(htmlText);22 const wines: z.infer<typeof WineSchema>[] = [];23 const pages = [...htmlText.matchAll(/\/en\/cote\/[a-z-]+\?page=(\d+)/g)].map((m) => Number(m[1]));24 const totalPages = pages.length ? Math.max(...pages) : null;25 $('[class*="cote_gridItem"]').each((_, el) => {26 const $el = $(el);27 const crumb = H.text($el.find('[class*="cote_breadCrumpRegion"]')) ?? '';28 const [reg, app] = crumb.split('>').map((s) => s.trim());29 const nameEl = $el.find('[class*="cote_nameWine"]').first();30 const color = H.text(nameEl.find('[class*="cote_colorDisplay"]'))?.replace(/[()]/g, '').trim() ?? null;31 nameEl.find('[class*="cote_colorDisplay"]').remove();32 const name = H.text(nameEl);33 const image = $el.find('img').attr('src') ?? null;34 const vintages: z.infer<typeof VintageSchema>[] = [];35 let wineId: string | null = null;36 let producerSlug: string | null = null;37 $el.find('a[href*="/en/wine-prices/"]').each((__, a) => {38 const href = $(a).attr('href') ?? '';39 const m = href.match(/\/en\/wine-prices\/(\d+)-(\d{4}|NV|[^-]+)-Bottle-(.+)$/i);40 const id = href.match(/\/en\/wine-prices\/(\d+)-/)?.[1] ?? null;41 const vTxt = H.text($(a).find('[class*="VintageRatingsCards_cote"]')) ?? '';42 const vintage = vTxt.match(/^(\d{4})/)?.[1] ?? null;43 const priceTxt = vTxt.match(/€\s*[\d\s.,]+/)?.[0] ?? null;44 const price = priceTxt ? parsePrice(priceTxt.replace(/\s/g, ''), 'EUR')?.amount ?? null : null;45 if (!id || !price) return;46 wineId ??= id;47 if (!producerSlug && m?.[3]) producerSlug = m[3]!;48 vintages.push({ vintage: vintage ? Number(vintage) : null, priceEur: price, url: BASE + href, ref: href.replace(/^\/en\/wine-prices\//, '').split('-Bottle-')[0]! });49 });50 if (!name || !wineId || vintages.length === 0) return;51 wines.push({ wineId, name, color, region: reg ?? null, appellation: app ?? null, producerSlug, image: image && !image.includes('_no_picture') ? (image.startsWith('http') ? image : BASE + image) : null, vintages });52 });53 return { kind: 'cote_page', url, region, page, totalPages, wines };54}5556/** "Chateau-de-Fieuzal-red" → "Château de Fieuzal" is not recoverable exactly; keep a readable producer label without guessing accents. */57export function producerFromSlug(slug: string | null): string | null {58 if (!slug) return null;59 const s = slug.replace(/-(red|white|rose|sparkling|sweet)$/i, '').replace(/^(Second-Wine-|Cru-bourgeois-)/i, '');60 const label = s.replace(/-/g, ' ').trim();61 return label || null;62}6364export class IdealwineConnector extends BaseConnector {65 readonly version = '1.0.0';66 readonly parserVersion = PARSER_VERSION;67 protected override minIntervalMs = 2000;6869 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {70 const regions = z.array(z.string()).parse(this.meta.config.regions ?? ['bordeaux']);71 const pages = Number(this.meta.config.pagesPerRegion ?? 3);72 const cursor = (ctx.options.cursor ?? {}) as { next?: Record<string, number> };73 const next: Record<string, number> = { ...(cursor.next ?? {}) };74 let count = 0;75 for (const region of regions) {76 const start = ctx.options.mode === 'backfill' ? (next[region] ?? 1) : 1;77 for (let page = start; page < start + pages; page++) {78 if (ctx.signal?.aborted || this.reached(ctx, count)) return void (await ctx.setCursor({ next }));79 const url = `${BASE}/en/cote/${region}${page > 1 ? `?page=${page}` : ''}`;80 await this.throttle();81 const res = await ctx.fetch(url, {82 engines: ['api'],83 responseType: 'text',84 expect: ['title', 'price', 'currency'],85 parse: (r) => {86 const p = r.html ? parseCotePage(r.html, url, region, page) : null;87 const w = p?.wines[0];88 return w ? { title: w.name, price: w.vintages[0]?.priceEur ?? null, currency: 'EUR' } : null;89 },90 });91 const payload = res.success && res.html ? parseCotePage(res.html, url, region, page) : null;92 if (!payload || payload.wines.length === 0) {93 ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);94 next[region] = 1;95 break;96 }97 count++;98 yield { url, externalId: `cote:${region}:${page}`, kind: 'price_observation', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };99 next[region] = payload.totalPages && page >= payload.totalPages ? 1 : page + 1;100 }101 }102 await ctx.setCursor({ next });103 }104105 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {106 const p = PagePayloadSchema.parse(raw.payload);107 const out: NormalizedRecord[] = [];108 const observationDate = new Date(Date.UTC(raw.fetchedAt.getUTCFullYear(), raw.fetchedAt.getUTCMonth(), raw.fetchedAt.getUTCDate()));109 for (const w of p.wines) {110 for (const v of w.vintages) {111 const attributes = AssetAttributesSchema.parse({112 categorySlug: 'wine',113 brand: producerFromSlug(w.producerSlug),114 series: w.region,115 set: w.appellation,116 name: w.name,117 year: v.vintage,118 color: w.color,119 size: '75cl',120 country: /italie|italy/i.test(p.region) ? 'IT' : 'FR',121 identifiers: { idealwine_wine_id: w.wineId, idealwine_ref: v.ref },122 metadata: { region_page: p.region },123 });124 const base = {125 connectorId: this.meta.id,126 sourceId: this.meta.sourceId,127 sourceUrl: v.url,128 externalId: v.ref,129 rawTitle: `${w.name}${v.vintage ? ` ${v.vintage}` : ''}${w.appellation ? ` · ${w.appellation}` : ''}${w.color ? ` (${w.color})` : ''}`,130 description: null,131 imageUrls: w.image ? [w.image] : [],132 attributes,133 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },134 condition: { condition: null, conditionRaw: null, completeness: null },135 observedAt: raw.fetchedAt,136 confidence: 0.75,137 parserVersion: PARSER_VERSION,138 };139 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, releaseDate: null }));140 out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, priceKind: 'guide_value', price: v.priceEur, currency: 'EUR', observationDate, sampleSize: null }));141 }142 }143 return out;144 }145}146147export default function createConnector(meta: ConnectorMeta) {148 return new IdealwineConnector(meta);149}150