TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, NormalizedPriceObservationSchema, type NormalizedRecord } from '@rareindex/shared';4import { currencyOr, moneyNumber, sneakerCategory, styleCodeFromText } from '../_luxury-lib/index.js';56/** Laced — ProductGroup JSON-LD on product pages: style code + per-size lowest asks (GBP). */7const BASE = 'https://www.laced.com';8const PARSER_VERSION = '1.0.0';910export const SizeOfferSchema = z.object({ size: z.string(), price: z.number(), currency: z.string(), available: z.boolean() });11export const ProductPayloadSchema = z.object({12 kind: z.literal('product_page'),13 url: z.string(),14 slug: z.string(),15 name: z.string(),16 sku: z.string().nullable(),17 brand: z.string().nullable(),18 images: z.array(z.string()),19 description: z.string().nullable(),20 lowPrice: z.number().nullable(),21 highPrice: z.number().nullable(),22 currency: z.string(),23 sizes: z.array(SizeOfferSchema),24});25export type ProductPayload = z.infer<typeof ProductPayloadSchema>;2627export function parseProductPage(htmlText: string, url: string): ProductPayload | null {28 const group = H.jsonLd(htmlText, 'ProductGroup')[0] ?? H.jsonLd(htmlText, 'Product')[0];29 if (!group) return null;30 const agg = (group.offers as Record<string, unknown> | undefined) ?? {};31 const sizes: z.infer<typeof SizeOfferSchema>[] = [];32 for (const v of (group.hasVariant as Array<Record<string, unknown>>) ?? []) {33 const off = (v.offers as Record<string, unknown> | undefined) ?? {};34 const price = moneyNumber(off.price as string | number | undefined);35 if (!price) continue;36 sizes.push({ size: String(v.size ?? '').trim(), price, currency: String(off.priceCurrency ?? agg.priceCurrency ?? 'GBP'), available: !/OutOfStock|SoldOut/.test(String(off.availability ?? '')) });37 }38 const brand = group.brand && typeof group.brand === 'object' ? String((group.brand as { name?: string }).name ?? '') : group.brand ? String(group.brand) : null;39 const img = group.image;40 const slug = url.match(/\/products\/([a-z0-9-]+)/i)?.[1] ?? url;41 return {42 kind: 'product_page',43 url: `${BASE}/products/${slug}`,44 slug,45 name: String(group.name ?? '').replace(/\s+/g, ' ').trim(),46 sku: group.sku ? String(group.sku) : null,47 brand: brand || null,48 images: Array.isArray(img) ? img.slice(0, 3).map(String) : typeof img === 'string' ? [img] : [],49 description: group.description ? String(group.description).slice(0, 500) : null,50 lowPrice: moneyNumber(agg.lowPrice as string | number | undefined),51 highPrice: moneyNumber(agg.highPrice as string | number | undefined),52 currency: String(agg.priceCurrency ?? sizes[0]?.currency ?? 'GBP'),53 sizes,54 };55}5657export function parseBrandPage(htmlText: string): string[] {58 return [...new Set([...htmlText.matchAll(/href="\/products\/([a-z0-9-]+)"/g)].map((m) => m[1]!))];59}6061export class LacedConnector extends BaseConnector {62 readonly version = '1.0.0';63 readonly parserVersion = PARSER_VERSION;64 protected override minIntervalMs = 1500;65 override readonly urlPatterns = [/^https?:\/\/(www\.)?laced\.com\/(?:[a-z]{2}\/)?products\/([a-z0-9-]+)/i];6667 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {68 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];69 const perSeed = Number(this.meta.config.productsPerSeed ?? 24);70 let count = 0;71 for (const seed of seeds) {72 await this.throttle();73 const list = await ctx.fetch(`${BASE}/${seed}`, { responseType: 'text', minQuality: 0.2 });74 if (!list.success || !list.html) {75 ctx.anomaly('page_fetch_failed', `${seed}: ${list.error ?? list.httpStatus}`);76 continue;77 }78 const slugs = parseBrandPage(list.html).slice(0, perSeed);79 if (!slugs.length) ctx.anomaly('empty_page', seed);80 for (const slug of slugs) {81 if (ctx.signal?.aborted || this.reached(ctx, count)) return;82 const url = `${BASE}/products/${slug}`;83 if (!(await ctx.shouldFetch(url))) continue;84 const rec = await this.fetchProduct(url, ctx);85 if (rec) {86 count++;87 yield rec;88 }89 }90 }91 }9293 private async fetchProduct(url: string, ctx: CrawlContext): Promise<RawRecordInput | null> {94 await this.throttle();95 const res = await ctx.fetch(url, { responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => {96 const p = r.html ? parseProductPage(r.html, url) : null;97 return p ? { title: p.name, price: p.lowPrice ?? p.sizes[0]?.price, identifiers: p.sku ? { sku: p.sku } : null } : null;98 } });99 const payload = res.success && res.html ? parseProductPage(res.html, url) : null;100 if (!payload) {101 if (res.httpStatus !== 404) ctx.anomaly(res.success ? 'parse_failure' : 'page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);102 return null;103 }104 return { url: payload.url, externalId: payload.slug, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };105 }106107 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {108 const slug = url.match(this.urlPatterns[0]!)?.[2];109 if (!slug) return [];110 const rec = await this.fetchProduct(`${BASE}/products/${slug}`, ctx);111 return rec ? [rec] : [];112 }113114 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {115 const p = ProductPayloadSchema.parse(raw.payload);116 const styleCode = p.sku ?? styleCodeFromText(p.name);117 const year = p.name.match(/\((\d{4})\)/)?.[1];118 const attributes = AssetAttributesSchema.parse({119 categorySlug: sneakerCategory(p.brand, p.name),120 brand: p.brand,121 name: p.name.replace(/\s*\(\d{4}\)\s*$/, '').trim(),122 year: year ? Number(year) : null,123 identifiers: { ...(styleCode ? { style_code: styleCode } : {}), laced_slug: p.slug },124 metadata: { sizes_listed: p.sizes.length },125 });126 const currency = currencyOr(p.currency, 'GBP');127 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: p.url, rawTitle: p.name, description: p.description, imageUrls: p.images, attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };128 const cond = { condition: 'new', conditionRaw: 'Brand new (deadstock marketplace)', completeness: 'with_box' };129 const out: NormalizedRecord[] = [NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: `product:${p.slug}`, confidence: 0.9 })];130 const low = p.lowPrice ?? (p.sizes.length ? Math.min(...p.sizes.map((s) => s.price)) : null);131 if (low) out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `product:${p.slug}:low`, confidence: 0.75, condition: cond, priceKind: 'low', price: low, currency, observationDate: raw.fetchedAt, sampleSize: p.sizes.length || null }));132 for (const s of p.sizes) {133 if (!s.available) continue;134 const uk = s.size.match(/UK\s*([\d.]+)/i)?.[1] ?? s.size.split('|')[0]?.trim() ?? s.size;135 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, attributes: { ...attributes, size: `UK ${uk}` }, externalId: `product:${p.slug}:size:${uk}`, confidence: 0.75, condition: cond, listingType: 'ask', price: s.price, currency: currencyOr(s.currency, currency), seller: 'Laced marketplace', location: 'GB', availability: 'available' }));136 }137 return out;138 }139}140141export default function createConnector(meta: ConnectorMeta) {142 return new LacedConnector(meta);143}144