import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared'; const BASE = 'https://www.lego.com'; const PARSER_VERSION = '1.0.0'; export const LeafSchema = z.object({ code: z.string(), url: z.string(), name: z.string(), price: z.number().nullable(), badges: z.array(z.string()), image: z.string().nullable(), availability: z.string().nullable(), pieces: z.number().nullable(), ages: z.string().nullable(), }); export type Leaf = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('theme_page'), url: z.string(), theme: z.string(), products: z.array(LeafSchema) }); export type PagePayload = z.infer; const BADGE_RE = /^(New|Retiring soon|Coming Soon|Exclusive|Hard to find|Sold out|Out of stock|Backorders accepted|Pre-order|Limited edition|Insiders)$/i; export const THEME_NAMES: Record = { 'star-wars': 'Star Wars', icons: 'Icons', technic: 'Technic', ideas: 'Ideas', 'harry-potter': 'Harry Potter', marvel: 'Marvel', architecture: 'Architecture', ninjago: 'Ninjago', 'creator-expert': 'Creator Expert', 'botanical-collection': 'Botanical Collection', 'lord-of-the-rings': 'The Lord of the Rings', 'dc': 'DC', 'speed-champions': 'Speed Champions', 'super-mario': 'Super Mario', }; export function parseThemePage(htmlText: string, pageUrl: string, theme: string): PagePayload { const $ = H.load(htmlText); const products: Leaf[] = []; const seen = new Set(); $('[data-test="product-leaf"]').each((_, el) => { const e = $(el); const a = e.find('a[href*="/product/"]').first(); const url = a.attr('href'); const code = url?.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1]; if (!url || !code || seen.has(code)) return; seen.add(code); const name = (a.attr('aria-label') ?? H.text(e.find('[data-test="product-leaf-title"]').first()) ?? '').replace(/\s+/g, ' ').trim(); if (!name) return; const priceText = H.text(e.find('[data-test="product-leaf-price"]').first()) ?? H.text(e.find('[data-test="product-leaf-price-row"]').first()); const price = parsePrice(priceText, 'USD')?.amount ?? null; const badges: string[] = []; e.find('span, div').each((__, b) => { const t = $(b).children().length ? '' : $(b).text().trim(); if (t && BADGE_RE.test(t) && !badges.includes(t)) badges.push(t); }); const image = e.find('img[data-test="product-leaf-image-1"]').attr('src') ?? e.find('img').first().attr('src') ?? null; products.push({ code, url: url.startsWith('http') ? url : BASE + url, name, price, badges, image, availability: null, pieces: null, ages: null }); }); return { kind: 'theme_page', url: pageUrl, theme, products }; } export function parseProductPage(htmlText: string, url: string, theme: string): PagePayload | null { const $ = H.load(htmlText); const code = url.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1]; const name = H.text($('[data-test="product-overview-name"]').first()); if (!code || !name) return null; const price = parsePrice(H.text($('[data-test="product-price-display-price"]').first()), 'USD')?.amount ?? null; const availability = H.text($('[data-test="product-overview-availability"]').first()); const image = $('meta[property="og:image"]').attr('content') ?? null; const body = $('body').text().replace(/\s+/g, ' '); const pieces = body.match(/(\d[\d,]*)\s*Pieces/)?.[1]; const ages = body.match(/(\d{1,2}\+)\s*Ages/)?.[1] ?? null; return { kind: 'theme_page', url, theme, products: [{ code, url, name, price, badges: [], image, availability, pieces: pieces ? Number(pieces.replace(/,/g, '')) : null, ages }] }; } export class LegoShopConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 2000; override readonly urlPatterns = [/^https?:\/\/(www\.)?lego\.com\/[a-z]{2}-[a-z]{2}\/product\/[a-z0-9-]+-\d{4,6}/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const themes = (this.meta.config.themes as string[] | undefined) ?? []; const pages = Number(this.meta.config.pagesPerTheme ?? 1); const cap = ctx.options.limit; let count = 0; const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.themeIndex ?? 0) : 0; for (let i = start; i < themes.length; i++) { const theme = themes[i]!; for (let page = 1; page <= pages; page++) { if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return; const url = `${BASE}/en-us/themes/${theme}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(); const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], waitForMs: 3000, expect: ['title', 'price'], parse: (r) => { const p = r.html ? parseThemePage(r.html, url, theme) : null; return p?.products.length ? { title: 'ok', price: p.products.some((x) => x.price) ? 1 : null } : null; }, }); const payload = res.success && res.html ? parseThemePage(res.html, url, theme) : null; if (!payload?.products.length) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product leaves'}`); break; } count++; yield { url, externalId: `${theme}|p${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }; } await ctx.setCursor({ themeIndex: i + 1 >= themes.length ? 0 : i + 1, updatedAt: new Date().toISOString() }); } } async lookup(url: string, ctx: CrawlContext): Promise { if (!this.urlPatterns[0]!.test(url)) return []; const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], waitForMs: 3000, minQuality: 0 }); const payload = res.success && res.html ? parseProductPage(res.html, url, 'product') : null; if (!payload) return []; return [{ url, externalId: payload.products[0]!.code, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }]; } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const themeName = THEME_NAMES[p.theme] ?? (p.theme === 'product' ? null : p.theme.replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())); const out: NormalizedRecord[] = []; for (const l of p.products) { const name = l.name.replace(/[™®]/g, '').replace(/^LEGO\s+/i, '').trim(); const attributes = AssetAttributesSchema.parse({ categorySlug: 'lego_sets', brand: 'LEGO', franchise: themeName, set: themeName, name, number: l.code, originalMsrp: l.price, originalMsrpCurrency: l.price ? 'USD' : null, identifiers: { lego_set_number: l.code }, metadata: { badges: l.badges, pieces: l.pieces, ages: l.ages, availability: l.availability, retiring_soon: l.badges.some((b) => /retiring/i.test(b)), official_store: true }, }); const rawTitle = `LEGO ${l.code} ${name}${themeName ? ` · ${themeName}` : ''}`; const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: l.url, rawTitle, imageUrls: l.image ? [l.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION }; out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: l.code, confidence: 0.95, releaseDate: null })); if (l.price) { const soldOut = l.badges.some((b) => /sold out|out of stock/i.test(b)) || /out of stock|sold out|retired/i.test(l.availability ?? ''); const coming = l.badges.some((b) => /coming soon|pre-order/i.test(b)) || /coming soon|pre-?order/i.test(l.availability ?? ''); out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: l.code, confidence: 0.95, listingType: 'fixed_price', price: l.price, currency: 'USD', seller: 'LEGO Shop (official)', location: 'US', condition: { condition: 'sealed', conditionRaw: 'New (official retail)', completeness: 'sealed' }, availability: soldOut ? 'sold' : coming ? 'unknown' : 'available' })); } } return out; } } export default function createConnector(meta: ConnectorMeta) { return new LegoShopConnector(meta); }