TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';5import { watchFromTitle } from '../_g9-asia-watch-sneaker-lib/index.js';67/**8 * European Watch Company (Boston dealer) — the server-rendered inventory grid `/all` lists every watch in stock9 * (brand + model title, USD asking price, status badge, image, product URL). Product pages carry a schema.org10 * Product (sku, mpn = reference, price, availability, condition) used for URL lookup. Asking prices → `listing`.11 */1213const SITE = 'https://www.europeanwatch.com';14const PARSER_VERSION = '1.0.0';1516export const CardSchema = z.object({ id: z.string(), href: z.string(), title: z.string(), price: z.number().nullable(), currency: z.string().nullable(), badge: z.string().nullable(), image: z.string().nullable() });17export type Card = z.infer<typeof CardSchema>;18export const ProductSchema = z.object({ id: z.string(), href: z.string(), name: z.string(), sku: z.string().nullable(), mpn: z.string().nullable(), brand: z.string().nullable(), price: z.number().nullable(), currency: z.string().nullable(), availability: z.string().nullable(), condition: z.string().nullable(), description: z.string().nullable(), images: z.array(z.string()) });19export type Product = z.infer<typeof ProductSchema>;20export const PagePayloadSchema = z.discriminatedUnion('kind', [21 z.object({ kind: z.literal('inventory_page'), url: z.string(), cards: z.array(CardSchema) }),22 z.object({ kind: z.literal('product_page'), url: z.string(), product: ProductSchema }),23]);24export type PagePayload = z.infer<typeof PagePayloadSchema>;25export type InventoryPagePayload = Extract<PagePayload, { kind: 'inventory_page' }>;2627function decodeNextImage(src: string | undefined): string | null {28 if (!src) return null;29 const m = src.match(/[?&]url=([^&]+)/);30 if (m) {31 try {32 return decodeURIComponent(m[1]!);33 } catch {34 return null;35 }36 }37 return src.startsWith('http') ? src : null;38}3940/** Inventory grid: each desktop card is a <div> holding exactly one <h3> (title) and one <p> (price); the link sits in the sibling image block. */41export function parseInventoryPage(htmlText: string, url: string): InventoryPagePayload {42 const $ = H.load(htmlText);43 const cards: Card[] = [];44 const seen = new Set<string>();45 $('div').each((_, d) => {46 const $d = $(d);47 if ($d.children('h3').length !== 1 || $d.children('p').length !== 1) return;48 const card = $d.parent();49 const href = card.find('a[href^="/watch/"]').first().attr('href') ?? null;50 if (!href) return;51 const id = href.match(/-(\d+)\/?$/)?.[1] ?? href.replace(/^\/watch\//, '');52 if (seen.has(id)) return;53 const title = H.text($d.children('h3').first());54 const priceText = H.text($d.children('p').first());55 if (!title) return;56 const price = parsePrice(priceText ?? '', 'USD');57 const badge = H.text($d.children('div').first());58 const img = card.find('img').first();59 seen.add(id);60 cards.push({ id, href: `${SITE}${href}`, title, price: price && price.amount > 0 ? price.amount : null, currency: price?.currency ?? null, badge, image: decodeNextImage(img.attr('src') ?? img.attr('srcset')?.split(' ')[0]) });61 });62 return { kind: 'inventory_page', url, cards };63}6465export function parseProductPage(htmlText: string, url: string): PagePayload | null {66 const prod = H.jsonLd(htmlText, 'Product')[0];67 if (!prod) return null;68 const offers = (Array.isArray(prod.offers) ? prod.offers[0] : prod.offers) as Record<string, unknown> | undefined;69 const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : prod.brand ? String(prod.brand) : null;70 const img = prod.image;71 const id = url.match(/-(\d+)\/?$/)?.[1] ?? String(prod.sku ?? '');72 const price = offers?.price !== undefined ? Number(offers.price) : NaN;73 return {74 kind: 'product_page',75 url,76 product: {77 id,78 href: url,79 name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(),80 sku: prod.sku ? String(prod.sku) : null,81 mpn: prod.mpn ? String(prod.mpn) : null,82 brand: brand || null,83 price: Number.isFinite(price) && price > 0 ? price : null,84 currency: offers?.priceCurrency ? String(offers.priceCurrency) : null,85 availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null,86 condition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null,87 description: prod.description ? String(prod.description).slice(0, 1200) : null,88 images: Array.isArray(img) ? img.slice(0, 4).map(String) : typeof img === 'string' ? [img] : [],89 },90 };91}9293export class EuropeanWatchCompanyConnector extends BaseConnector {94 readonly version = '1.0.0';95 readonly parserVersion = PARSER_VERSION;96 protected override minIntervalMs = 3000;97 override readonly urlPatterns = [/^https?:\/\/(?:www\.)?europeanwatch\.com\/watch\/([a-z0-9-]+)/i];9899 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {100 const pages = (this.meta.config.pages as string[] | undefined) ?? ['/all'];101 let count = 0;102 for (const path of pages) {103 if (ctx.signal?.aborted || this.reached(ctx, count)) return;104 const url = `${SITE}${path}`;105 await this.throttle(url);106 const res = await ctx.fetch(url, { engines: ['api', 'firecrawl'], responseType: 'text', timeoutMs: 90_000, expect: ['title', 'price', 'currency'], parse: (r) => {107 const p = r.html ? parseInventoryPage(r.html, url) : null;108 const c = p?.cards.find((x) => x.price);109 return c ? { title: c.title, price: c.price, currency: c.currency } : null;110 } });111 if (!res.success || !res.html) {112 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);113 continue;114 }115 const payload = parseInventoryPage(res.html, url);116 if (!payload.cards.length) {117 ctx.anomaly('selector_missing', `${url}: no inventory cards parsed`);118 continue;119 }120 count++;121 yield { url, externalId: `inventory:${path}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };122 await ctx.setCursor({ page: path, at: new Date().toISOString() });123 }124 await ctx.setCursor({ done: true, at: new Date().toISOString() });125 }126127 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {128 const slug = url.match(this.urlPatterns[0]!)?.[1];129 if (!slug) return [];130 const target = `${SITE}/watch/${slug}`;131 await this.throttle(target);132 const res = await ctx.fetch(target, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0.2 });133 const payload = res.success && res.html ? parseProductPage(res.html, target) : null;134 if (!payload) return [];135 return [{ url: target, externalId: `product:${slug}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];136 }137138 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {139 const p = PagePayloadSchema.parse(raw.payload);140 const out: NormalizedRecord[] = [];141 if (p.kind === 'product_page') {142 const pr = p.product;143 const w = watchFromTitle(pr.name, pr.brand);144 const reference = pr.mpn ?? w.reference;145 const conditionRaw = pr.condition === 'NewCondition' ? 'Unworn' : w.conditionRaw ?? (pr.condition ? 'Pre-owned' : null);146 out.push(147 NormalizedListingSchema.parse({148 kind: 'listing',149 connectorId: this.meta.id,150 sourceId: this.meta.sourceId,151 sourceUrl: pr.href,152 externalId: pr.id,153 rawTitle: pr.name,154 description: pr.description,155 imageUrls: pr.images,156 attributes: AssetAttributesSchema.parse({ categorySlug: w.categorySlug, brand: w.brand, name: pr.name, reference, year: w.year, material: w.material, size: w.size, identifiers: { ewc_sku: pr.sku ?? pr.id, ...(reference ? { reference } : {}) }, metadata: { availability_raw: pr.availability, condition_raw: pr.condition } }),157 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },158 condition: { condition: normalizeCondition(w.categorySlug, conditionRaw), conditionRaw, completeness: w.completeness },159 observedAt: raw.fetchedAt,160 confidence: 0.85,161 parserVersion: PARSER_VERSION,162 listingType: 'fixed_price',163 price: pr.price,164 currency: pr.currency === 'USD' ? 'USD' : pr.price ? 'USD' : null,165 seller: 'European Watch Company',166 location: 'Boston, MA, US',167 quantity: 1,168 availability: pr.availability === 'InStock' ? 'available' : pr.availability === 'OutOfStock' ? 'ended' : 'unknown',169 }),170 );171 return out;172 }173 for (const c of p.cards) {174 const w = watchFromTitle(c.title);175 const pending = /sale pending/i.test(c.badge ?? '');176 out.push(177 NormalizedListingSchema.parse({178 kind: 'listing',179 connectorId: this.meta.id,180 sourceId: this.meta.sourceId,181 sourceUrl: c.href,182 externalId: c.id,183 rawTitle: c.title,184 description: null,185 imageUrls: c.image ? [c.image] : [],186 attributes: AssetAttributesSchema.parse({ categorySlug: w.categorySlug, brand: w.brand, name: c.title, reference: w.reference, year: w.year, material: w.material, size: w.size, identifiers: { ewc_sku: c.id, ...(w.reference ? { reference: w.reference } : {}) }, metadata: { badge: c.badge, sale_pending: pending } }),187 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },188 condition: { condition: normalizeCondition(w.categorySlug, w.conditionRaw), conditionRaw: w.conditionRaw, completeness: w.completeness },189 observedAt: raw.fetchedAt,190 confidence: 0.8,191 parserVersion: PARSER_VERSION,192 listingType: 'fixed_price',193 price: c.price,194 currency: c.price ? ((c.currency as 'USD' | null) ?? 'USD') : null,195 seller: 'European Watch Company',196 location: 'Boston, MA, US',197 quantity: 1,198 availability: 'available',199 }),200 );201 }202 return out;203 }204}205206export default function createConnector(meta: ConnectorMeta) {207 return new EuropeanWatchCompanyConnector(meta);208}209