import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { normalizeCondition } from '@rareindex/taxonomy'; import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { caseSize, currencyOr, moneyNumber, watchCategory, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js'; /** Subdial — sitemap-driven listing pages with schema.org Product + spec table (GBP). */ const BASE = 'https://subdial.com'; const PARSER_VERSION = '1.0.0'; export const ListingPayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), name: z.string(), brand: z.string().nullable(), mpn: z.string().nullable(), sku: z.string().nullable(), price: z.number(), currency: z.string(), availability: z.string().nullable(), images: z.array(z.string()), description: z.string().nullable(), specs: z.record(z.string(), z.string()), }); export type ListingPayload = z.infer; export function parseListingPage(htmlText: string, url: string): ListingPayload | null { const prod = H.jsonLd(htmlText, 'Product')[0]; if (!prod) return null; const offers = (prod.offers as Record | undefined) ?? {}; const price = moneyNumber(offers.price as string | number | undefined); if (!price) return null; const specs: Record = {}; for (const m of htmlText.matchAll(/(Reference|Year|Box|Papers|Condition|Movement|Case size|Case material|Dial|Bracelet|Diameter)[^<]{0,3}<\/[^>]+>\s*<[^>]+>([^<]{1,60}) { const max = ctx.options.limit ?? Number(this.meta.config.maxListingsPerRun ?? 250); const seeds = ctx.options.seeds?.length ? ctx.options.seeds : await this.listingUrls(ctx); let count = 0; for (const url of seeds) { if (ctx.signal?.aborted || count >= max) return; if (!(await ctx.shouldFetch(url))) continue; const rec = await this.fetchListing(url, ctx); if (rec) { count++; yield rec; } } } private async listingUrls(ctx: CrawlContext): Promise { const res = await ctx.fetch(`${BASE}/sitemap-listing.xml`, { engines: ['api'], responseType: 'text', minQuality: 0 }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `sitemap: ${res.error ?? res.httpStatus}`); return []; } const urls = [...res.html.matchAll(/\s*(https?:\/\/[^<\s]+\/listing\/[^<\s]+)\s*<\/loc>/g)].map((m) => m[1]!); // newest listings tend to have the highest SD numbers; crawl those first return urls.sort((a, b) => (b.match(/sd(\d+)$/i)?.[1] ?? '0').localeCompare(a.match(/sd(\d+)$/i)?.[1] ?? '0', undefined, { numeric: true })); } private async fetchListing(url: string, ctx: CrawlContext): Promise { await this.throttle(); const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => { const p = r.html ? parseListingPage(r.html, url) : null; return p ? { title: p.name, price: p.price, identifiers: p.mpn ? { mpn: p.mpn } : null } : null; } }); if (res.httpStatus === 404 || res.httpStatus === 410) return null; const payload = res.success && res.html ? parseListingPage(res.html, url) : null; if (!payload) { ctx.anomaly(res.success ? 'parse_failure' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return null; } return { url, externalId: payload.sku ?? url, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } async lookup(url: string, ctx: CrawlContext): Promise { const rec = await this.fetchListing(url.split('?')[0]!, ctx); return rec ? [rec] : []; } async normalize(raw: RawRecordLike): Promise { const p = ListingPayloadSchema.parse(raw.payload); const brand = p.brand ?? p.name.split(' ')[0]!; const categorySlug = watchCategory(brand); const reference = p.mpn ?? p.specs.reference ?? watchReferenceFromText(p.name); const model = p.name.replace(new RegExp(`^${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s+`, 'i'), '').replace(/\b(original\s+)?(box\s*(&|and)\s*papers|box only|papers only|full set)\b/gi, '').replace(reference ? reference.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') : /$^/, '').replace(/\b(19|20)\d{2}\b/g, '').replace(/\bRef\.?\b/gi, '').replace(/\s+/g, ' ').trim() || null; const yearStr = p.specs.year ?? p.description?.match(/·\s*(\d{4})\s*·/)?.[1]; const conditionRaw = p.specs.condition ?? null; const box = /yes|included|original/i.test(p.specs.box ?? ''); const papers = /yes|included|original|\d{4}/i.test(p.specs.papers ?? ''); const completeness = box && papers ? 'full_set' : papers ? 'papers_only' : box ? 'box_only' : p.specs.box || p.specs.papers ? 'watch_only' : null; const attributes = AssetAttributesSchema.parse({ categorySlug, brand, name: `${brand} ${model ?? ''}`.trim(), model, reference, year: yearStr ? Number(yearStr) : null, material: watchMaterial(`${p.name} ${p.specs['case material'] ?? ''}`), size: caseSize(`${p.name} ${p.specs.diameter ?? p.specs['case size'] ?? ''}`), identifiers: { ...(reference ? { reference } : {}), subdial_id: p.sku ?? p.url }, metadata: { specs: p.specs }, }); return [ NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: p.url, externalId: p.sku ?? p.url, rawTitle: `${p.name}${reference ? ` Ref. ${reference}` : ''}${yearStr ? ` (${yearStr})` : ''}`, description: p.description, imageUrls: p.images, attributes, condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness }, observedAt: raw.fetchedAt, confidence: 0.9, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: p.price, currency: currencyOr(p.currency, 'GBP'), seller: 'Subdial', location: 'London, GB', availability: p.availability === 'InStock' ? 'available' : p.availability ? 'sold' : 'unknown', }), ]; } } export default function createConnector(meta: ConnectorMeta) { return new SubdialConnector(meta); }