import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared'; import { currencyOr, moneyNumber, sneakerCategory, styleCodeFromText } from '../_luxury-lib/index.js'; /** Laced — ProductGroup JSON-LD on product pages: style code + per-size lowest asks (GBP). */ const BASE = 'https://www.laced.com'; const PARSER_VERSION = '1.0.0'; export const SizeOfferSchema = z.object({ size: z.string(), price: z.number(), currency: z.string(), available: z.boolean() }); export const ProductPayloadSchema = z.object({ kind: z.literal('product_page'), url: z.string(), slug: z.string(), name: z.string(), sku: z.string().nullable(), brand: z.string().nullable(), images: z.array(z.string()), description: z.string().nullable(), lowPrice: z.number().nullable(), highPrice: z.number().nullable(), currency: z.string(), sizes: z.array(SizeOfferSchema), }); export type ProductPayload = z.infer; export function parseProductPage(htmlText: string, url: string): ProductPayload | null { const group = H.jsonLd(htmlText, 'ProductGroup')[0] ?? H.jsonLd(htmlText, 'Product')[0]; if (!group) return null; const agg = (group.offers as Record | undefined) ?? {}; const sizes: z.infer[] = []; for (const v of (group.hasVariant as Array>) ?? []) { const off = (v.offers as Record | undefined) ?? {}; const price = moneyNumber(off.price as string | number | undefined); if (!price) continue; sizes.push({ size: String(v.size ?? '').trim(), price, currency: String(off.priceCurrency ?? agg.priceCurrency ?? 'GBP'), available: !/OutOfStock|SoldOut/.test(String(off.availability ?? '')) }); } const brand = group.brand && typeof group.brand === 'object' ? String((group.brand as { name?: string }).name ?? '') : group.brand ? String(group.brand) : null; const img = group.image; const slug = url.match(/\/products\/([a-z0-9-]+)/i)?.[1] ?? url; return { kind: 'product_page', url: `${BASE}/products/${slug}`, slug, name: String(group.name ?? '').replace(/\s+/g, ' ').trim(), sku: group.sku ? String(group.sku) : null, brand: brand || null, images: Array.isArray(img) ? img.slice(0, 3).map(String) : typeof img === 'string' ? [img] : [], description: group.description ? String(group.description).slice(0, 500) : null, lowPrice: moneyNumber(agg.lowPrice as string | number | undefined), highPrice: moneyNumber(agg.highPrice as string | number | undefined), currency: String(agg.priceCurrency ?? sizes[0]?.currency ?? 'GBP'), sizes, }; } export function parseBrandPage(htmlText: string): string[] { return [...new Set([...htmlText.matchAll(/href="\/products\/([a-z0-9-]+)"/g)].map((m) => m[1]!))]; } export class LacedConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(www\.)?laced\.com\/(?:[a-z]{2}\/)?products\/([a-z0-9-]+)/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? []; const perSeed = Number(this.meta.config.productsPerSeed ?? 24); let count = 0; for (const seed of seeds) { await this.throttle(); const list = await ctx.fetch(`${BASE}/${seed}`, { responseType: 'text', minQuality: 0.2 }); if (!list.success || !list.html) { ctx.anomaly('page_fetch_failed', `${seed}: ${list.error ?? list.httpStatus}`); continue; } const slugs = parseBrandPage(list.html).slice(0, perSeed); if (!slugs.length) ctx.anomaly('empty_page', seed); for (const slug of slugs) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${BASE}/products/${slug}`; if (!(await ctx.shouldFetch(url))) continue; const rec = await this.fetchProduct(url, ctx); if (rec) { count++; yield rec; } } } } private async fetchProduct(url: string, ctx: CrawlContext): Promise { await this.throttle(); const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => { const p = r.html ? parseProductPage(r.html, url) : null; return p ? { title: p.name, price: p.lowPrice ?? p.sizes[0]?.price, identifiers: p.sku ? { sku: p.sku } : null } : null; } }); const payload = res.success && res.html ? parseProductPage(res.html, url) : null; if (!payload) { if (res.httpStatus !== 404) ctx.anomaly(res.success ? 'parse_failure' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); return null; } return { url: payload.url, externalId: payload.slug, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } async lookup(url: string, ctx: CrawlContext): Promise { const slug = url.match(this.urlPatterns[0]!)?.[2]; if (!slug) return []; const rec = await this.fetchProduct(`${BASE}/products/${slug}`, ctx); return rec ? [rec] : []; } async normalize(raw: RawRecordLike): Promise { const p = ProductPayloadSchema.parse(raw.payload); const styleCode = p.sku ?? styleCodeFromText(p.name); const year = p.name.match(/\((\d{4})\)/)?.[1]; const attributes = AssetAttributesSchema.parse({ categorySlug: sneakerCategory(p.brand, p.name), brand: p.brand, name: p.name.replace(/\s*\(\d{4}\)\s*$/, '').trim(), year: year ? Number(year) : null, identifiers: { ...(styleCode ? { style_code: styleCode } : {}), laced_slug: p.slug }, metadata: { sizes_listed: p.sizes.length }, }); const currency = currencyOr(p.currency, 'GBP'); const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: p.url, rawTitle: p.name, description: p.description, imageUrls: p.images, attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; const cond = { condition: 'new', conditionRaw: 'Brand new (deadstock marketplace)', completeness: 'with_box' }; const out: NormalizedRecord[] = [NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${p.slug}`, confidence: 0.9 })]; const low = p.lowPrice ?? (p.sizes.length ? Math.min(...p.sizes.map((s) => s.price)) : null); if (low) out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `product:${p.slug}:low`, confidence: 0.75, condition: cond, priceKind: 'low', price: low, currency, observationDate: raw.fetchedAt, sampleSize: p.sizes.length || null })); for (const s of p.sizes) { if (!s.available) continue; const uk = s.size.match(/UK\s*([\d.]+)/i)?.[1] ?? s.size.split('|')[0]?.trim() ?? s.size; out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, attributes: { ...attributes, size: `UK ${uk}` }, externalId: `product:${p.slug}:size:${uk}`, confidence: 0.75, condition: cond, listingType: 'ask', price: s.price, currency: currencyOr(s.currency, currency), seller: 'Laced marketplace', location: 'GB', availability: 'available' })); } return out; } } export default function createConnector(meta: ConnectorMeta) { return new LacedConnector(meta); }