TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { AssetAttributesSchema, NormalizedCatalogItemSchema, NormalizedPriceObservationSchema, parsePrice, parseSourceDate, type NormalizedRecord } from '@rareindex/shared';45/**6 * Brickset — LEGO set facts (pieces, minifigs, RRP, launch/exit, barcodes) + Brickset's7 * "Current value" New/Used estimates. Raw payload = the parsed definition-list of a set page.8 */910const BASE = 'https://brickset.com';11const PARSER_VERSION = '1.0.0';1213export const SetPayloadSchema = z.object({14 kind: z.literal('set_page'),15 url: z.string(),16 setNumber: z.string(),17 title: z.string(),18 image: z.string().nullable(),19 fields: z.record(z.string(), z.string()),20});21export type SetPayload = z.infer<typeof SetPayloadSchema>;2223/** Parse the <dl> facts of a set page into a flat label → text map (links flattened). */24export function parseSetPage(htmlText: string, url: string): SetPayload | null {25 const $ = H.load(htmlText);26 const fields: Record<string, string> = {};27 $('dl').each((_, dl) => {28 const dts = $(dl).find('dt');29 dts.each((__, dt) => {30 const label = H.text($(dt));31 const dd = $(dt).next('dd');32 if (!label || !dd.length) return;33 const clone = dd.clone();34 clone.find('br').replaceWith(' | ');35 const value = H.text(clone);36 if (value && !(label in fields)) fields[label] = value;37 });38 });39 const setNumber = fields['Set number'] ?? url.match(/\/sets\/(\d+-\d+)/)?.[1] ?? null;40 const title = ($('title').text().split('|')[0] ?? '').trim();41 if (!setNumber || !title) return null;42 const image = $('meta[property="og:image"]').attr('content') ?? $('img[src*="/sets/images/"]').attr('src') ?? null;43 return { kind: 'set_page', url, setNumber, title, image, fields };44}4546export function setLinks(htmlText: string): string[] {47 const out = new Set<string>();48 for (const m of htmlText.matchAll(/href="(\/sets\/\d+-\d+\/[^"#?]*)"/g)) out.add(BASE + m[1]!);49 return [...out];50}5152/** "£342.49, $499.99" → { gbp: 342.49, usd: 499.99 } */53export function parseRrp(s: string | undefined): { gbp: number | null; usd: number | null; eur: number | null } {54 const grab = (re: RegExp) => {55 const m = s?.match(re);56 return m ? Number(m[1]!.replace(/,/g, '')) : null;57 };58 return { gbp: grab(/£\s?([\d,]+(?:\.\d+)?)/), usd: grab(/\$\s?([\d,]+(?:\.\d+)?)/), eur: grab(/€\s?([\d,]+(?:\.\d+)?)/) };59}6061/** "New: ~$2858 | Used: ~$1172" → { newUsd, usedUsd } */62export function parseCurrentValue(s: string | undefined): { newUsd: number | null; usedUsd: number | null } {63 const n = s?.match(/New:\s*~?\$([\d,]+(?:\.\d+)?)/);64 const u = s?.match(/Used:\s*~?\$([\d,]+(?:\.\d+)?)/);65 return { newUsd: n ? Number(n[1]!.replace(/,/g, '')) : null, usedUsd: u ? Number(u[1]!.replace(/,/g, '')) : null };66}6768/** "01 Oct 07 - 31 Dec 09" → { launch: '2007-10-01', exit: '2009-12-31' } */69export function parseLaunchExit(s: string | undefined): { launch: Date | null; exit: Date | null } {70 if (!s) return { launch: null, exit: null };71 const [a, b] = s.split(/\s+-\s+/);72 const fix = (x?: string) => {73 if (!x) return null;74 const m = x.trim().match(/^(\d{1,2}) ([A-Za-z]{3}) (\d{2})$/);75 if (!m) return parseSourceDate(x);76 const yy = Number(m[3]);77 return parseSourceDate(`${m[1]} ${m[2]} ${yy >= 50 ? 1900 + yy : 2000 + yy}`);78 };79 return { launch: fix(a), exit: fix(b) };80}8182export class BricksetConnector extends BaseConnector {83 readonly version = '1.0.0';84 readonly parserVersion = PARSER_VERSION;85 protected override minIntervalMs = 1500;86 override readonly urlPatterns = [/^https?:\/\/(www\.)?brickset\.com\/sets\/(\d+-\d+)/i];8788 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {89 const seeds = (ctx.options.seeds?.length ? ctx.options.seeds : (this.meta.config.seeds as string[] | undefined)) ?? [];90 const pages = Number(this.meta.config.pagesPerSeed ?? 2);91 const cap = ctx.options.limit ?? Number(this.meta.config.setsPerRun ?? 120);92 const startSeed = ctx.options.mode === 'incremental' ? Number(ctx.options.cursor?.seedIndex ?? 0) : 0;93 let count = 0;94 for (let i = startSeed; i < seeds.length && count < cap; i++) {95 const seed = seeds[i]!;96 for (let page = 1; page <= pages && count < cap; page++) {97 if (ctx.signal?.aborted) return;98 const listUrl = `${BASE}/sets/${seed.replace(/^\//, '')}${page > 1 ? `/page-${page}` : ''}`;99 await this.throttle();100 const lres = await ctx.fetch(listUrl, { engines: ['api', 'firecrawl'], responseType: 'text', minQuality: 0.2 });101 const links = lres.success && lres.html ? setLinks(lres.html) : [];102 if (!links.length) {103 ctx.anomaly('discover_failed', `${listUrl}: ${lres.error ?? lres.httpStatus ?? 'no set links'}`);104 break;105 }106 for (const url of links) {107 if (ctx.signal?.aborted || count >= cap) break;108 if (!(await ctx.shouldFetch(url))) continue;109 const rec = await this.fetchSet(url, ctx);110 if (rec) {111 count++;112 yield rec;113 }114 }115 }116 await ctx.setCursor({ seedIndex: i + 1 >= seeds.length ? 0 : i + 1, updatedAt: new Date().toISOString() });117 }118 }119120 private async fetchSet(url: string, ctx: CrawlContext): Promise<RawRecordInput | null> {121 await this.throttle();122 const res = await ctx.fetch(url, {123 engines: ['api', 'firecrawl'],124 responseType: 'text',125 expect: ['title', 'identifiers', 'price'],126 parse: (r) => {127 const p = r.html ? parseSetPage(r.html, url) : null;128 return p ? { title: p.title, identifiers: { set: p.setNumber }, price: p.fields['RRP'] ?? p.fields['Current value'] ?? null } : null;129 },130 });131 const payload = res.success && res.html ? parseSetPage(res.html, url) : null;132 if (!payload) {133 ctx.anomaly('set_parse_failed', `${url}: ${res.error ?? res.httpStatus}`);134 return null;135 }136 return { url, externalId: payload.setNumber, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };137 }138139 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {140 const m = url.match(this.urlPatterns[0]!);141 if (!m) return [];142 const rec = await this.fetchSet(`${BASE}/sets/${m[2]}`, ctx);143 return rec ? [rec] : [];144 }145146 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {147 const p = SetPayloadSchema.parse(raw.payload);148 const f = p.fields;149 const number = p.setNumber.replace(/-\d+$/, '');150 const name = (f['Name'] ?? p.title.replace(/^LEGO\s+\d+(?:-\d+)?\s+/i, '')).trim();151 const year = f['Year released'] ? Number(f['Year released'].match(/\d{4}/)?.[0]) || null : null;152 const rrp = parseRrp(f['RRP']);153 const value = parseCurrentValue(f['Current value']);154 const { launch, exit } = parseLaunchExit(f['Launch/exit']);155 const pieces = f['Pieces'] ? Number(f['Pieces'].replace(/,/g, '')) || null : null;156 const minifigs = f['Minifigs'] ? Number(f['Minifigs'].match(/\d+/)?.[0]) || null : null;157 const upc = f['Barcodes']?.match(/UPC:\s*(\d+)/)?.[1] ?? null;158 const ean = f['Barcodes']?.match(/EAN:\s*(\d+)/)?.[1] ?? null;159 const theme = f['Theme'] ?? null;160 const subtheme = f['Subtheme'] ?? null;161 const rawTitle = `LEGO ${number} ${name}${theme ? ` · ${theme}` : ''}${year ? ` (${year})` : ''}`;162 const attributes = AssetAttributesSchema.parse({163 categorySlug: 'lego_sets',164 brand: 'LEGO',165 franchise: theme,166 series: subtheme,167 set: theme,168 name,169 number,170 year,171 productionQuantity: null,172 originalMsrp: rrp.usd ?? rrp.gbp ?? rrp.eur,173 originalMsrpCurrency: rrp.usd ? 'USD' : rrp.gbp ? 'GBP' : rrp.eur ? 'EUR' : null,174 identifiers: { lego_set_number: number, brickset_set: p.setNumber, ...(upc ? { upc } : {}), ...(ean ? { ean } : {}) },175 metadata: {176 pieces,177 minifigs,178 rrp_gbp: rrp.gbp,179 rrp_usd: rrp.usd,180 availability: f['Availability'] ?? null,181 packaging: f['Packaging'] ?? null,182 launch_date: launch ? launch.toISOString().slice(0, 10) : null,183 exit_date: exit ? exit.toISOString().slice(0, 10) : null,184 retired: exit ? exit.getTime() < Date.now() : null,185 age_range: f['Age range'] ?? null,186 rating: f['Rating'] ? Number(f['Rating'].match(/(\d\.\d)/)?.[1]) || null : null,187 notes: f['Notes'] ?? null,188 },189 });190 const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: p.url, rawTitle, imageUrls: p.image ? [p.image] : [], attributes, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };191 const out: NormalizedRecord[] = [NormalizedCatalogItemSchema.parse({ kind: 'catalog_item', ...base, externalId: p.setNumber, confidence: 0.95, releaseDate: launch })];192 const obsDate = raw.fetchedAt;193 if (value.newUsd) {194 out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `${p.setNumber}:new`, confidence: 0.65, condition: { condition: 'sealed', conditionRaw: 'New (Brickset current value, BrickLink-derived)', completeness: 'sealed' }, priceKind: 'guide_value', price: value.newUsd, currency: 'USD', observationDate: obsDate, sampleSize: null }));195 }196 if (value.usedUsd) {197 out.push(NormalizedPriceObservationSchema.parse({ kind: 'price_observation', ...base, externalId: `${p.setNumber}:used`, confidence: 0.65, condition: { condition: 'used_complete', conditionRaw: 'Used (Brickset current value, BrickLink-derived)', completeness: 'used_complete' }, priceKind: 'guide_value', price: value.usedUsd, currency: 'USD', observationDate: obsDate, sampleSize: null }));198 }199 return out;200 }201}202203export default function createConnector(meta: ConnectorMeta) {204 return new BricksetConnector(meta);205}206207// keep parsePrice referenced for future currency-specific fields208void parsePrice;209