TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { normalizeCondition } from '@rareindex/taxonomy';4import { AssetAttributesSchema, NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';5import { caseSize, currencyOr, moneyNumber, watchCategory, watchMaterial, watchReferenceFromText } from '../_luxury-lib/index.js';67/** Subdial — sitemap-driven listing pages with schema.org Product + spec table (GBP). */8const BASE = 'https://subdial.com';9const PARSER_VERSION = '1.0.0';1011export const ListingPayloadSchema = z.object({12 kind: z.literal('listing_page'),13 url: z.string(),14 name: z.string(),15 brand: z.string().nullable(),16 mpn: z.string().nullable(),17 sku: z.string().nullable(),18 price: z.number(),19 currency: z.string(),20 availability: z.string().nullable(),21 images: z.array(z.string()),22 description: z.string().nullable(),23 specs: z.record(z.string(), z.string()),24});25export type ListingPayload = z.infer<typeof ListingPayloadSchema>;2627export function parseListingPage(htmlText: string, url: string): ListingPayload | null {28 const prod = H.jsonLd(htmlText, 'Product')[0];29 if (!prod) return null;30 const offers = (prod.offers as Record<string, unknown> | undefined) ?? {};31 const price = moneyNumber(offers.price as string | number | undefined);32 if (!price) return null;33 const specs: Record<string, string> = {};34 for (const m of htmlText.matchAll(/(Reference|Year|Box|Papers|Condition|Movement|Case size|Case material|Dial|Bracelet|Diameter)[^<]{0,3}<\/[^>]+>\s*<[^>]+>([^<]{1,60})</g)) {35 const k = m[1]!.toLowerCase();36 const v = m[2]!.replace(/\s+/g, ' ').trim();37 if (v && !(k in specs)) specs[k] = v;38 }39 const brand = prod.brand && typeof prod.brand === 'object' ? String((prod.brand as { name?: string }).name ?? '') : prod.brand ? String(prod.brand) : null;40 const img = prod.image;41 return {42 kind: 'listing_page',43 url,44 name: String(prod.name ?? '').replace(/\s+/g, ' ').trim(),45 brand: brand || null,46 mpn: prod.mpn ? String(prod.mpn) : null,47 sku: prod.sku ? String(prod.sku) : null,48 price,49 currency: String(offers.priceCurrency ?? 'GBP'),50 availability: offers.availability ? String(offers.availability).replace(/^.*\//, '') : null,51 images: Array.isArray(img) ? img.slice(0, 3).map(String) : typeof img === 'string' ? [img] : [],52 description: prod.description ? String(prod.description).slice(0, 600) : null,53 specs,54 };55}5657export class SubdialConnector extends BaseConnector {58 readonly version = '1.0.0';59 readonly parserVersion = PARSER_VERSION;60 protected override minIntervalMs = 1500;61 override readonly urlPatterns = [/^https?:\/\/(www\.)?subdial\.com\/listing\/[a-z0-9-]+/i];6263 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {64 const max = ctx.options.limit ?? Number(this.meta.config.maxListingsPerRun ?? 250);65 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : await this.listingUrls(ctx);66 let count = 0;67 for (const url of seeds) {68 if (ctx.signal?.aborted || count >= max) return;69 if (!(await ctx.shouldFetch(url))) continue;70 const rec = await this.fetchListing(url, ctx);71 if (rec) {72 count++;73 yield rec;74 }75 }76 }7778 private async listingUrls(ctx: CrawlContext): Promise<string[]> {79 const res = await ctx.fetch(`${BASE}/sitemap-listing.xml`, { engines: ['api'], responseType: 'text', minQuality: 0 });80 if (!res.success || !res.html) {81 ctx.anomaly('page_fetch_failed', `sitemap: ${res.error ?? res.httpStatus}`);82 return [];83 }84 const urls = [...res.html.matchAll(/<loc>\s*(https?:\/\/[^<\s]+\/listing\/[^<\s]+)\s*<\/loc>/g)].map((m) => m[1]!);85 // newest listings tend to have the highest SD numbers; crawl those first86 return urls.sort((a, b) => (b.match(/sd(\d+)$/i)?.[1] ?? '0').localeCompare(a.match(/sd(\d+)$/i)?.[1] ?? '0', undefined, { numeric: true }));87 }8889 private async fetchListing(url: string, ctx: CrawlContext): Promise<RawRecordInput | null> {90 await this.throttle();91 const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => {92 const p = r.html ? parseListingPage(r.html, url) : null;93 return p ? { title: p.name, price: p.price, identifiers: p.mpn ? { mpn: p.mpn } : null } : null;94 } });95 if (res.httpStatus === 404 || res.httpStatus === 410) return null;96 const payload = res.success && res.html ? parseListingPage(res.html, url) : null;97 if (!payload) {98 ctx.anomaly(res.success ? 'parse_failure' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);99 return null;100 }101 return { url, externalId: payload.sku ?? url, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };102 }103104 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {105 const rec = await this.fetchListing(url.split('?')[0]!, ctx);106 return rec ? [rec] : [];107 }108109 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {110 const p = ListingPayloadSchema.parse(raw.payload);111 const brand = p.brand ?? p.name.split(' ')[0]!;112 const categorySlug = watchCategory(brand);113 const reference = p.mpn ?? p.specs.reference ?? watchReferenceFromText(p.name);114 const model = p.name.replace(new RegExp(`^${brand.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s+`, 'i'), '').replace(/\b(original\s+)?(box\s*(&|and)\s*papers|box only|papers only|full set)\b/gi, '').replace(reference ? reference.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') : /$^/, '').replace(/\b(19|20)\d{2}\b/g, '').replace(/\bRef\.?\b/gi, '').replace(/\s+/g, ' ').trim() || null;115 const yearStr = p.specs.year ?? p.description?.match(/·\s*(\d{4})\s*·/)?.[1];116 const conditionRaw = p.specs.condition ?? null;117 const box = /yes|included|original/i.test(p.specs.box ?? '');118 const papers = /yes|included|original|\d{4}/i.test(p.specs.papers ?? '');119 const completeness = box && papers ? 'full_set' : papers ? 'papers_only' : box ? 'box_only' : p.specs.box || p.specs.papers ? 'watch_only' : null;120 const attributes = AssetAttributesSchema.parse({121 categorySlug,122 brand,123 name: `${brand} ${model ?? ''}`.trim(),124 model,125 reference,126 year: yearStr ? Number(yearStr) : null,127 material: watchMaterial(`${p.name} ${p.specs['case material'] ?? ''}`),128 size: caseSize(`${p.name} ${p.specs.diameter ?? p.specs['case size'] ?? ''}`),129 identifiers: { ...(reference ? { reference } : {}), subdial_id: p.sku ?? p.url },130 metadata: { specs: p.specs },131 });132 return [133 NormalizedListingSchema.parse({134 kind: 'listing',135 connectorId: this.meta.id,136 sourceId: this.meta.sourceId,137 sourceUrl: p.url,138 externalId: p.sku ?? p.url,139 rawTitle: `${p.name}${reference ? ` Ref. ${reference}` : ''}${yearStr ? ` (${yearStr})` : ''}`,140 description: p.description,141 imageUrls: p.images,142 attributes,143 condition: { condition: normalizeCondition(categorySlug, conditionRaw), conditionRaw, completeness },144 observedAt: raw.fetchedAt,145 confidence: 0.9,146 parserVersion: PARSER_VERSION,147 listingType: 'fixed_price',148 price: p.price,149 currency: currencyOr(p.currency, 'GBP'),150 seller: 'Subdial',151 location: 'London, GB',152 availability: p.availability === 'InStock' ? 'available' : p.availability ? 'sold' : 'unknown',153 }),154 ];155 }156}157158export default function createConnector(meta: ConnectorMeta) {159 return new SubdialConnector(meta);160}161