TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedListingSchema, parsePrice, type NormalizedRecord } from '@rareindex/shared';45const BASE = 'https://www.lego.com';6const PARSER_VERSION = '1.0.0';78export const LeafSchema = z.object({9 code: z.string(),10 url: z.string(),11 name: z.string(),12 price: z.number().nullable(),13 badges: z.array(z.string()),14 image: z.string().nullable(),15 availability: z.string().nullable(),16 pieces: z.number().nullable(),17 ages: z.string().nullable(),18});19export type Leaf = z.infer<typeof LeafSchema>;20export const PagePayloadSchema = z.object({ kind: z.literal('theme_page'), url: z.string(), theme: z.string(), products: z.array(LeafSchema) });21export type PagePayload = z.infer<typeof PagePayloadSchema>;2223const BADGE_RE = /^(New|Retiring soon|Coming Soon|Exclusive|Hard to find|Sold out|Out of stock|Backorders accepted|Pre-order|Limited edition|Insiders)$/i;2425export const THEME_NAMES: Record<string, string> = {26 'star-wars': 'Star Wars', icons: 'Icons', technic: 'Technic', ideas: 'Ideas', 'harry-potter': 'Harry Potter', marvel: 'Marvel', architecture: 'Architecture', ninjago: 'Ninjago', 'creator-expert': 'Creator Expert', 'botanical-collection': 'Botanical Collection', 'lord-of-the-rings': 'The Lord of the Rings', 'dc': 'DC', 'speed-champions': 'Speed Champions', 'super-mario': 'Super Mario',27};2829export function parseThemePage(htmlText: string, pageUrl: string, theme: string): PagePayload {30 const $ = H.load(htmlText);31 const products: Leaf[] = [];32 const seen = new Set<string>();33 $('[data-test="product-leaf"]').each((_, el) => {34 const e = $(el);35 const a = e.find('a[href*="/product/"]').first();36 const url = a.attr('href');37 const code = url?.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1];38 if (!url || !code || seen.has(code)) return;39 seen.add(code);40 const name = (a.attr('aria-label') ?? H.text(e.find('[data-test="product-leaf-title"]').first()) ?? '').replace(/\s+/g, ' ').trim();41 if (!name) return;42 const priceText = H.text(e.find('[data-test="product-leaf-price"]').first()) ?? H.text(e.find('[data-test="product-leaf-price-row"]').first());43 const price = parsePrice(priceText, 'USD')?.amount ?? null;44 const badges: string[] = [];45 e.find('span, div').each((__, b) => {46 const t = $(b).children().length ? '' : $(b).text().trim();47 if (t && BADGE_RE.test(t) && !badges.includes(t)) badges.push(t);48 });49 const image = e.find('img[data-test="product-leaf-image-1"]').attr('src') ?? e.find('img').first().attr('src') ?? null;50 products.push({ code, url: url.startsWith('http') ? url : BASE + url, name, price, badges, image, availability: null, pieces: null, ages: null });51 });52 return { kind: 'theme_page', url: pageUrl, theme, products };53}5455export function parseProductPage(htmlText: string, url: string, theme: string): PagePayload | null {56 const $ = H.load(htmlText);57 const code = url.match(/-(\d{4,6})(?:[/?#]|$)/)?.[1];58 const name = H.text($('[data-test="product-overview-name"]').first());59 if (!code || !name) return null;60 const price = parsePrice(H.text($('[data-test="product-price-display-price"]').first()), 'USD')?.amount ?? null;61 const availability = H.text($('[data-test="product-overview-availability"]').first());62 const image = $('meta[property="og:image"]').attr('content') ?? null;63 const body = $('body').text().replace(/\s+/g, ' ');64 const pieces = body.match(/(\d[\d,]*)\s*Pieces/)?.[1];65 const ages = body.match(/(\d{1,2}\+)\s*Ages/)?.[1] ?? null;66 return { kind: 'theme_page', url, theme, products: [{ code, url, name, price, badges: [], image, availability, pieces: pieces ? Number(pieces.replace(/,/g, '')) : null, ages }] };67}6869export class LegoShopConnector extends BaseConnector {70 readonly version = '1.0.0';71 readonly parserVersion = PARSER_VERSION;72 protected override minIntervalMs = 2000;73 override readonly urlPatterns = [/^https?:\/\/(www\.)?lego\.com\/[a-z]{2}-[a-z]{2}\/product\/[a-z0-9-]+-\d{4,6}/i];7475 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {76 const themes = (this.meta.config.themes as string[] | undefined) ?? [];77 const pages = Number(this.meta.config.pagesPerTheme ?? 1);78 const cap = ctx.options.limit;79 let count = 0;80 const start = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.themeIndex ?? 0) : 0;81 for (let i = start; i < themes.length; i++) {82 const theme = themes[i]!;83 for (let page = 1; page <= pages; page++) {84 if (ctx.signal?.aborted || (cap !== undefined && count >= cap)) return;85 const url = `${BASE}/en-us/themes/${theme}${page > 1 ? `?page=${page}` : ''}`;86 await this.throttle();87 const res = await ctx.fetch(url, {88 engines: ['firecrawl', 'scrapfly'],89 waitForMs: 3000,90 expect: ['title', 'price'],91 parse: (r) => {92 const p = r.html ? parseThemePage(r.html, url, theme) : null;93 return p?.products.length ? { title: 'ok', price: p.products.some((x) => x.price) ? 1 : null } : null;94 },95 });96 const payload = res.success && res.html ? parseThemePage(res.html, url, theme) : null;97 if (!payload?.products.length) {98 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus ?? 'no product leaves'}`);99 break;100 }101 count++;102 yield { url, externalId: `${theme}|p${page}`, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };103 }104 await ctx.setCursor({ themeIndex: i + 1 >= themes.length ? 0 : i + 1, updatedAt: new Date().toISOString() });105 }106 }107108 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {109 if (!this.urlPatterns[0]!.test(url)) return [];110 const res = await ctx.fetch(url, { engines: ['firecrawl', 'scrapfly'], waitForMs: 3000, minQuality: 0 });111 const payload = res.success && res.html ? parseProductPage(res.html, url, 'product') : null;112 if (!payload) return [];113 return [{ url, externalId: payload.products[0]!.code, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt }];114 }115116 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {117 const p = PagePayloadSchema.parse(raw.payload);118 const themeName = THEME_NAMES[p.theme] ?? (p.theme === 'product' ? null : p.theme.replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()));119 const out: NormalizedRecord[] = [];120 for (const l of p.products) {121 const name = l.name.replace(/[™®]/g, '').replace(/^LEGO\s+/i, '').trim();122 const attributes = AssetAttributesSchema.parse({123 categorySlug: 'lego_sets',124 brand: 'LEGO',125 franchise: themeName,126 set: themeName,127 name,128 number: l.code,129 originalMsrp: l.price,130 originalMsrpCurrency: l.price ? 'USD' : null,131 identifiers: { lego_set_number: l.code },132 metadata: { badges: l.badges, pieces: l.pieces, ages: l.ages, availability: l.availability, retiring_soon: l.badges.some((b) => /retiring/i.test(b)), official_store: true },133 });134 const rawTitle = `LEGO ${l.code} ${name}${themeName ? ` · ${themeName}` : ''}`;135 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: l.url, rawTitle, imageUrls: l.image ? [l.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };136 out.push(NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: l.code, confidence: 0.95, releaseDate: null }));137 if (l.price) {138 const soldOut = l.badges.some((b) => /sold out|out of stock/i.test(b)) || /out of stock|sold out|retired/i.test(l.availability ?? '');139 const coming = l.badges.some((b) => /coming soon|pre-order/i.test(b)) || /coming soon|pre-?order/i.test(l.availability ?? '');140 out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: l.code, confidence: 0.95, listingType: 'fixed_price', price: l.price, currency: 'USD', seller: 'LEGO Shop (official)', location: 'US', condition: { condition: 'sealed', conditionRaw: 'New (official retail)', completeness: 'sealed' }, availability: soldOut ? 'sold' : coming ? 'unknown' : 'available' }));141 }142 }143 return out;144 }145}146147export default function createConnector(meta: ConnectorMeta) {148 return new LegoShopConnector(meta);149}150