import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { WATCH_BRANDS, caseSize, currencyOr, moneyNumber, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js'; /** European Watch Company (ex Crown & Caliber domain) — ItemList JSON-LD on brand pages. */ const BASE = 'https://www.europeanwatch.com'; const PARSER_VERSION = '1.0.0'; export const ItemSchema = z.object({ name: z.string(), sku: z.string().nullable(), url: z.string(), price: z.number(), currency: z.string(), availability: z.string().nullable(), image: z.string().nullable(), condition: z.string().nullable() }); export const PagePayloadSchema = z.object({ kind: z.literal('brand_page'), url: z.string(), brand: z.string(), page: z.number(), items: z.array(ItemSchema), details: z.record(z.string(), z.string()).optional() }); export type PagePayload = z.infer; export function parseBrandPage(htmlText: string, url: string, brand: string, page: number): PagePayload { const items: z.infer[] = []; const push = (prod: Record) => { const offers = (prod.offers as Record | undefined) ?? {}; const price = moneyNumber(offers.price as string | number | undefined); if (!price) return; const img = prod.image; items.push({ name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(), sku: prod.sku ? String(prod.sku) : null, url: String(prod.url ?? ''), price, currency: String(offers.priceCurrency ?? 'USD'), availability: offers.availability ? String(offers.availability).replace(/^.*\//, '') : null, image: Array.isArray(img) ? String(img[0] ?? '') || null : typeof img === 'string' ? img : null, condition: offers.itemCondition ? String(offers.itemCondition).replace(/^.*\//, '') : null, }); }; for (const list of H.jsonLd(htmlText, 'ItemList')) { for (const el of (list.itemListElement as Array>) ?? []) { const prod = (el.item as Record | undefined) ?? el; if (prod && prod['@type'] === 'Product') push(prod); } } for (const prod of H.jsonLd(htmlText, 'Product')) push(prod); // product pages: spec table "Condition / Box / Papers / Year / Reference" const details: Record = {}; for (const m of htmlText.matchAll(/(Condition|Box|Papers|Year|Reference|Movement|Case Size)<\/(?:p|dt|span)>\s*<(?:p|dd|span)[^>]*>([^<]{1,40}) 1 && tokens[0] === ref ? tokens.slice(1) : tokens; const modelTokens: string[] = []; for (const t of afterRef) { if (/^(SS|18k|18K|Steel|Gold|Platinum|Titanium|Ceramic|Two|Rose|Yellow|White|Black|Blue|Green|Silver|Grey|Gray|Champagne|Circa|Full|Box|Papers|Dial|Bracelet|Strap|Jubilee|Oyster|Leather|\d{2}mm|\d{4})$/i.test(t)) break; modelTokens.push(t); if (modelTokens.length >= 4) break; } return { reference: ref, model: modelTokens.length ? modelTokens.join(' ') : null }; } export class EuropeanWatchConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(www\.)?europeanwatch\.com\/watch\/[a-z0-9-]+/i, /^https?:\/\/(www\.)?crownandcaliber\.com\/products\/[a-z0-9-]+/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2); let count = 0; for (const seed of seeds) { let prevFirst = ''; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${BASE}/brand/${seed}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price'], parse: (r) => { const first = r.html ? parseBrandPage(r.html, url, seed, page).items[0] : undefined; return first ? { title: first.name, price: first.price } : null; } }); const payload = res.success && res.html ? parseBrandPage(res.html, url, seed, page) : null; if (!payload || payload.items.length === 0) { if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const firstSku = payload.items[0]!.sku ?? payload.items[0]!.url; if (firstSku === prevFirst) break; // the site serves the whole brand inventory on one page prevFirst = firstSku; count++; yield { url, externalId: `brand:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } } } async lookup(url: string, ctx: CrawlContext): Promise { await this.throttle(); const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0.2 }); if (!res.success || !res.html) return []; const payload = parseBrandPage(res.html, url, 'lookup', 1); return payload.items.length ? [{ url, externalId: `product:${payload.items[0]!.sku ?? url}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : []; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; const seen = new Set(); for (const it of p.items) { const id = it.sku ?? it.url; if (seen.has(id)) continue; seen.add(id); const brand = WATCH_BRANDS.find((b) => it.name.toLowerCase().startsWith(b.toLowerCase())) ?? it.name.split(' ')[0]!; const categorySlug = watchCategory(brand); const { reference, model } = splitName(it.name, brand); const circa = it.name.match(/Circa\.?\s*(\d{4})/i)?.[1] ?? p.details?.year?.match(/\d{4}/)?.[0]; const conditionRaw = p.details?.condition ?? watchConditionRaw(it.name) ?? (it.condition === 'UsedCondition' ? 'Pre-owned' : it.condition === 'NewCondition' ? 'Unworn' : null); const completeness = p.details ? (/yes/i.test(p.details.box ?? '') && /yes/i.test(p.details.papers ?? '') ? 'full_set' : /yes/i.test(p.details.papers ?? '') ? 'papers_only' : /yes/i.test(p.details.box ?? '') ? 'box_only' : 'watch_only') : watchCompleteness(it.name); const attributes = AssetAttributesSchema.parse({ categorySlug, brand, name: `${brand} ${model ?? ''}`.trim(), model, reference, year: circa ? Number(circa) : null, material: watchMaterial(it.name), size: caseSize(it.name), identifiers: { ...(reference ? { reference } : {}), europeanwatch_sku: id }, metadata: { brand_page: p.url }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: it.url, externalId: id, rawTitle: it.name, imageUrls: it.image ? [it.image] : [], attributes, condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness }, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: it.price, currency: currencyOr(it.currency, 'USD'), seller: 'European Watch Company', location: 'Boston, US', availability: it.availability === 'InStock' ? 'available' : it.availability ? 'sold' : 'unknown', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new EuropeanWatchConnector(meta); }