import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, CurrencySchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; import { watchCategory, watchMaterial, caseSize } from '../_luxury-lib/index.js'; import { extractYear } from '@rareindex/shared'; /** * CHRONEXT (Cologne, DE) — certified pre-owned/new watch retailer. Category pages (`/rolex/submariner` …) are * server-rendered with one `.product-tile` per watch (brand, model, reference, price "USD 4,380", condition label, * image, product URL `/brand/model/reference/V`); pagination links carry a stream id + `offset`. Product pages * expose a schema.org Product (sku, mpn, brand, offers) used for URL lookup. Asking prices → `listing`. */ const SITE = 'https://www.chronext.com'; const PARSER_VERSION = '1.0.0'; const PAGE_SIZE = 24; export const TileSchema = z.object({ id: z.string(), href: z.string(), brand: z.string().nullable(), model: z.string().nullable(), reference: z.string().nullable(), priceText: z.string().nullable(), condition: z.string().nullable(), image: z.string().nullable() }); export type Tile = z.infer; export const ProductSchema = z.object({ id: z.string(), href: z.string(), name: z.string(), sku: z.string().nullable(), mpn: z.string().nullable(), brand: z.string().nullable(), price: z.number().nullable(), currency: z.string().nullable(), availability: z.string().nullable(), condition: z.string().nullable(), image: z.string().nullable() }); export const PagePayloadSchema = z.discriminatedUnion('kind', [ z.object({ kind: z.literal('category_page'), url: z.string(), seed: z.string(), page: z.number(), tiles: z.array(TileSchema), nextUrl: z.string().nullable(), title: z.string().nullable() }), z.object({ kind: z.literal('product_page'), url: z.string(), product: ProductSchema }), ]); export type PagePayload = z.infer; export type CategoryPagePayload = Extract; export function parseCategoryPage(htmlText: string, url: string, seed: string, page: number): CategoryPagePayload { const $ = H.load(htmlText); const tiles: Tile[] = []; const seen = new Set(); $('.product-tile').each((_, el) => { const $t = $(el); const href = $t.find('a[href]').first().attr('href') ?? null; const id = href?.match(/\/(V\d+)\/?$/)?.[1] ?? null; if (!href || !id || seen.has(id)) return; seen.add(id); tiles.push({ id, href: href.startsWith('http') ? href : `${SITE}${href}`, brand: H.text($t.find('.product-tile__brand').first()), model: H.text($t.find('.product-tile__model').first()), reference: H.text($t.find('.product-tile__reference').first()), priceText: H.text($t.find('.product-tile__price .price').first()) ?? H.text($t.find('.price').first()), condition: H.text($t.find('.condition-with-icon__text').first()), image: $t.find('img').first().attr('src') ?? null, }); }); // Pagination links look like "?s[][offset]=24&nodeId=…"; the next page is the one whose offset = page × 24. let nextUrl: string | null = null; $('a[href*="offset"]').each((_, a) => { const href = $(a).attr('href') ?? ''; const off = decodeURIComponent(href).match(/\[offset\]=(\d+)/)?.[1]; if (off && Number(off) === page * PAGE_SIZE) nextUrl = href.startsWith('http') ? href.replace(/&/g, '&') : `${SITE}${href.replace(/&/g, '&')}`; }); const title = H.text($('h1').first()); return { kind: 'category_page', url, seed, page, tiles, nextUrl, title }; } export function parseProductPage(htmlText: string, url: string): PagePayload | null { const prod = H.jsonLd(htmlText, 'Product')[0]; if (!prod) return null; const offers = (Array.isArray(prod.offers) ? prod.offers[0] : prod.offers) as Record | undefined; const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : prod.brand ? String(prod.brand) : null; const price = offers?.price !== undefined ? Number(offers.price) : NaN; const id = url.match(/\/(V\d+)\/?$/)?.[1] ?? String(prod.sku ?? ''); return { kind: 'product_page', url, product: { id, href: url, name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(), sku: prod.sku ? String(prod.sku) : null, mpn: prod.mpn ? String(prod.mpn) : null, brand: brand || null, price: Number.isFinite(price) && price > 0 ? price : null, currency: offers?.priceCurrency ? String(offers.priceCurrency) : null, availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null, condition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null, image: typeof prod.image === 'string' ? prod.image : Array.isArray(prod.image) ? String(prod.image[0] ?? '') || null : null }, }; } function conditionFromLabel(label: string | null): string | null { if (!label) return null; const l = label.toLowerCase(); if (/like new|mint|excellent/.test(l)) return 'Excellent'; if (/unworn|\bnew\b/.test(l)) return 'Unworn'; if (/very good/.test(l)) return 'Very good'; if (/good/.test(l)) return 'Good'; if (/fair|vintage/.test(l)) return 'Fair'; return label; } export class ChronextConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 4000; override readonly urlPatterns = [/^https?:\/\/(?:www\.)?chronext\.com\/([a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9.-]+\/V\d+)/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []); const pages = ctx.options.mode === 'backfill' ? this.policy.backfillMaxPages : Number(this.meta.config.pagesPerSeed ?? 1); const cur = (ctx.options.cursor ?? {}) as { seedIndex?: number }; let count = 0; for (let si = cur.seedIndex ?? 0; si < seeds.length; si++) { const seed = seeds[si]!; let url: string | null = `${SITE}${seed.startsWith('/') ? seed : `/${seed}`}`; for (let page = 1; page <= pages && url; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const pageUrl: string = url; await this.throttle(pageUrl); const res = await ctx.fetch(pageUrl, { engines: ['api', 'firecrawl'], responseType: 'text', timeoutMs: 90_000, headers: { accept: 'text/html,application/xhtml+xml' }, expect: ['title', 'price', 'identifiers'], parse: (r) => { const p = r.html ? parseCategoryPage(r.html, pageUrl, seed, page) : null; const t = p?.tiles[0]; return t ? { title: `${t.brand} ${t.model}`, price: t.priceText, identifiers: t.reference ? { reference: t.reference } : null } : null; } }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${pageUrl}: ${res.error ?? res.httpStatus}`); break; } const payload = parseCategoryPage(res.html, pageUrl, seed, page); if (!payload.tiles.length) { if (page === 1) ctx.anomaly('selector_missing', `${pageUrl}: no product tiles`); break; } count++; yield { url: pageUrl, externalId: `${seed}#${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; await ctx.progress({ page, itemsProcessed: count }); url = payload.nextUrl; } await ctx.setCursor({ seedIndex: si + 1, at: new Date().toISOString() }); } await ctx.setCursor({ done: true, at: new Date().toISOString() }); } async lookup(url: string, ctx: CrawlContext): Promise { const path = url.match(this.urlPatterns[0]!)?.[1]; if (!path) return []; const target = `${SITE}/${path}`; await this.throttle(target); const res = await ctx.fetch(target, { engines: ['api', 'firecrawl'], responseType: 'text', headers: { accept: 'text/html,application/xhtml+xml' }, minQuality: 0.2 }); const payload = res.success && res.html ? parseProductPage(res.html, target) : null; if (!payload) return []; return [{ url: target, externalId: `product:${path.split('/').pop()}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const out: NormalizedRecord[] = []; if (p.kind === 'product_page') { const pr = p.product; const categorySlug = watchCategory(pr.brand); const conditionRaw = pr.condition === 'NewCondition' ? 'Unworn' : pr.condition === 'RefurbishedCondition' || pr.condition === 'UsedCondition' ? 'Pre-owned' : null; const cur = CurrencySchema.safeParse(pr.currency ?? ''); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: pr.href, externalId: pr.id, rawTitle: pr.name, imageUrls: pr.image ? [pr.image] : [], attributes: AssetAttributesSchema.parse({ categorySlug, brand: pr.brand, name: pr.name, model: pr.name.replace(new RegExp(`^${(pr.brand ?? '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*`, 'i'), '') || null, reference: pr.mpn, year: extractYear(pr.name), material: watchMaterial(pr.name), size: caseSize(pr.name), identifiers: { chronext_id: pr.id, ...(pr.sku ? { chronext_sku: pr.sku } : {}), ...(pr.mpn ? { reference: pr.mpn } : {}) }, metadata: { availability_raw: pr.availability, condition_raw: pr.condition } }), grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: cur.success ? pr.price : null, currency: cur.success && pr.price ? cur.data : null, seller: 'CHRONEXT', location: 'DE', quantity: 1, availability: pr.availability === 'InStock' ? 'available' : pr.availability === 'OutOfStock' ? 'ended' : 'unknown', }), ); return out; } for (const t of p.tiles) { const parsed = t.priceText ? parsePrice(t.priceText) : null; const cur = parsed?.currency ? CurrencySchema.safeParse(parsed.currency) : null; const categorySlug = watchCategory(t.brand); const name = [t.brand, t.model, t.reference].filter(Boolean).join(' '); const conditionRaw = conditionFromLabel(t.condition); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: t.href, externalId: t.id, rawTitle: name, imageUrls: t.image ? [t.image] : [], attributes: AssetAttributesSchema.parse({ categorySlug, brand: t.brand, name, model: t.model, reference: t.reference, identifiers: { chronext_id: t.id, ...(t.reference ? { reference: t.reference } : {}) }, metadata: { category_page: p.url.split('?')[0], condition_label: t.condition, price_text: t.priceText } }), grade: { grader: null, grade: null, qualifier: null, certificationNumber: null }, condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness: null }, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: cur?.success && parsed ? parsed.amount : null, currency: cur?.success ? cur.data : null, seller: 'CHRONEXT', location: 'DE', quantity: 1, availability: 'available', }), ); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new ChronextConnector(meta); }