TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { WATCH_BRANDS, caseSize, currencyOr, moneyNumber, watchCategory, watchCompleteness, watchConditionRaw, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js';67/** European Watch Company (ex Crown & Caliber domain) — ItemList JSON-LD on brand pages. */8const BASE = 'https://www.europeanwatch.com';9const PARSER_VERSION = '1.0.0';1011export const ItemSchema = z.object({ name: z.string(), sku: z.string().nullable(), url: z.string(), price: z.number(), currency: z.string(), availability: z.string().nullable(), image: z.string().nullable(), condition: z.string().nullable() });12export const PagePayloadSchema = z.object({ kind: z.literal('brand_page'), url: z.string(), brand: z.string(), page: z.number(), items: z.array(ItemSchema), details: z.record(z.string(), z.string()).optional() });13export type PagePayload = z.infer<typeof PagePayloadSchema>;1415export function parseBrandPage(htmlText: string, url: string, brand: string, page: number): PagePayload {16 const items: z.infer<typeof ItemSchema>[] = [];17 const push = (prod: Record<string, unknown>) => {18 const offers = (prod.offers as Record<string, unknown> | undefined) ?? {};19 const price = moneyNumber(offers.price as string | number | undefined);20 if (!price) return;21 const img = prod.image;22 items.push({23 name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(),24 sku: prod.sku ? String(prod.sku) : null,25 url: String(prod.url ?? ''),26 price,27 currency: String(offers.priceCurrency ?? 'USD'),28 availability: offers.availability ? String(offers.availability).replace(/^.*\//, '') : null,29 image: Array.isArray(img) ? String(img[0] ?? '') || null : typeof img === 'string' ? img : null,30 condition: offers.itemCondition ? String(offers.itemCondition).replace(/^.*\//, '') : null,31 });32 };33 for (const list of H.jsonLd(htmlText, 'ItemList')) {34 for (const el of (list.itemListElement as Array<Record<string, unknown>>) ?? []) {35 const prod = (el.item as Record<string, unknown> | undefined) ?? el;36 if (prod && prod['@type'] === 'Product') push(prod);37 }38 }39 for (const prod of H.jsonLd(htmlText, 'Product')) push(prod);40 // product pages: spec table "Condition / Box / Papers / Year / Reference"41 const details: Record<string, string> = {};42 for (const m of htmlText.matchAll(/(Condition|Box|Papers|Year|Reference|Movement|Case Size)<\/(?:p|dt|span)>\s*<(?:p|dd|span)[^>]*>([^<]{1,40})</g)) details[m[1]!.toLowerCase()] = m[2]!.trim();43 return { kind: 'brand_page', url, brand, page, items, ...(Object.keys(details).length ? { details } : {}) };44}4546/** "Rolex 126234 Datejust 36 Jubilee SS Green Palm Dial" → reference 126234, model "Datejust 36" */47export function splitName(name: string, brand: string): { reference: string | null; model: string | null } {48 const rest = name.replace(new RegExp(`^${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s+`, 'i'), '');49 const tokens = rest.split(/\s+/);50 const ref = watchReferenceFromText(rest) ?? (tokens[0] && /\d/.test(tokens[0]) ? tokens[0] : null);51 const afterRef = ref && tokens.length > 1 && tokens[0] === ref ? tokens.slice(1) : tokens;52 const modelTokens: string[] = [];53 for (const t of afterRef) {54 if (/^(SS|18k|18K|Steel|Gold|Platinum|Titanium|Ceramic|Two|Rose|Yellow|White|Black|Blue|Green|Silver|Grey|Gray|Champagne|Circa|Full|Box|Papers|Dial|Bracelet|Strap|Jubilee|Oyster|Leather|\d{2}mm|\d{4})$/i.test(t)) break;55 modelTokens.push(t);56 if (modelTokens.length >= 4) break;57 }58 return { reference: ref, model: modelTokens.length ? modelTokens.join(' ') : null };59}6061export class EuropeanWatchConnector extends BaseConnector {62 readonly version = '1.0.0';63 readonly parserVersion = PARSER_VERSION;64 protected override minIntervalMs = 1500;65 override readonly urlPatterns = [/^https?:\/\/(www\.)?europeanwatch\.com\/watch\/[a-z0-9-]+/i, /^https?:\/\/(www\.)?crownandcaliber\.com\/products\/[a-z0-9-]+/i];6667 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {68 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];69 const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);70 let count = 0;71 for (const seed of seeds) {72 let prevFirst = '';73 for (let page = 1; page <= pages; page++) {74 if (ctx.signal?.aborted || this.reached(ctx, count)) return;75 const url = `${BASE}/brand/${seed}${page > 1 ? `?page=${page}` : ''}`;76 await this.throttle();77 const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price'], parse: (r) => {78 const first = r.html ? parseBrandPage(r.html, url, seed, page).items[0] : undefined;79 return first ? { title: first.name, price: first.price } : null;80 } });81 const payload = res.success && res.html ? parseBrandPage(res.html, url, seed, page) : null;82 if (!payload || payload.items.length === 0) {83 if (page === 1) ctx.anomaly(payload ? 'empty_page' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);84 break;85 }86 const firstSku = payload.items[0]!.sku ?? payload.items[0]!.url;87 if (firstSku === prevFirst) break; // the site serves the whole brand inventory on one page88 prevFirst = firstSku;89 count++;90 yield { url, externalId: `brand:${seed}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };91 }92 }93 }9495 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {96 await this.throttle();97 const res = await ctx.fetch(url, { responseType: 'text', minQuality: 0.2 });98 if (!res.success || !res.html) return [];99 const payload = parseBrandPage(res.html, url, 'lookup', 1);100 return payload.items.length ? [{ url, externalId: `product:${payload.items[0]!.sku ?? url}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }] : [];101 }102103 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {104 const p = PagePayloadSchema.parse(raw.payload);105 const out: NormalizedRecord[] = [];106 const seen = new Set<string>();107 for (const it of p.items) {108 const id = it.sku ?? it.url;109 if (seen.has(id)) continue;110 seen.add(id);111 const brand = WATCH_BRANDS.find((b) => it.name.toLowerCase().startsWith(b.toLowerCase())) ?? it.name.split(' ')[0]!;112 const categorySlug = watchCategory(brand);113 const { reference, model } = splitName(it.name, brand);114 const circa = it.name.match(/Circa\.?\s*(\d{4})/i)?.[1] ?? p.details?.year?.match(/\d{4}/)?.[0];115 const conditionRaw = p.details?.condition ?? watchConditionRaw(it.name) ?? (it.condition === 'UsedCondition' ? 'Pre-owned' : it.condition === 'NewCondition' ? 'Unworn' : null);116 const completeness = p.details ? (/yes/i.test(p.details.box ?? '') && /yes/i.test(p.details.papers ?? '') ? 'full_set' : /yes/i.test(p.details.papers ?? '') ? 'papers_only' : /yes/i.test(p.details.box ?? '') ? 'box_only' : 'watch_only') : watchCompleteness(it.name);117 const attributes = AssetAttributesSchema.parse({118 categorySlug,119 brand,120 name: `${brand} ${model ?? ''}`.trim(),121 model,122 reference,123 year: circa ? Number(circa) : null,124 material: watchMaterial(it.name),125 size: caseSize(it.name),126 identifiers: { ...(reference ? { reference } : {}), europeanwatch_sku: id },127 metadata: { brand_page: p.url },128 });129 out.push(130 NormalizedListingSchema.parse({131 kind: 'listing',132 connectorId: this.meta.id,133 sourceId: this.meta.sourceId,134 sourceUrl: it.url,135 externalId: id,136 rawTitle: it.name,137 imageUrls: it.image ? [it.image] : [],138 attributes,139 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness },140 observedAt: raw.fetchedAt,141 confidence: 0.85,142 parserVersion: PARSER_VERSION,143 listingType: 'fixed_price',144 price: it.price,145 currency: currencyOr(it.currency, 'USD'),146 seller: 'European Watch Company',147 location: 'Boston, US',148 availability: it.availability === 'InStock' ? 'available' : it.availability ? 'sold' : 'unknown',149 }),150 );151 }152 return out;153 }154}155156export default function createConnector(meta: ConnectorMeta) {157 return new EuropeanWatchConnector(meta);158}159