TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { parsePrice, parseSourceDate, type AssetAttributes, type NormalizedCatalogItem, type NormalizedPriceObservation, type NormalizedRecord } from '@rareindex/shared';45/**6 * BrickEconomy connector — LEGO set catalog + estimated values (New/Sealed, Used) + retail price.7 * Raw payload = the markdown sections of the set page (Set Details, Set Pricing, Set Predictions);8 * normalisation re-parses that markdown so parser fixes can be replayed on stored raw records.9 */1011const BASE = 'https://www.brickeconomy.com';12const PARSER_VERSION = '1.0.0';1314export const SetPayloadSchema = z.object({15 kind: z.literal('set_page'),16 url: z.string(),17 title: z.string(),18 markdown: z.string(),19 images: z.array(z.string()),20});21export type SetPayload = z.infer<typeof SetPayloadSchema>;2223/** Extract "#### Section" blocks and return a compact markdown excerpt. */24export function excerptSections(md: string): string {25 const wanted = ['Set Details', 'Set Pricing', 'Set Predictions', 'Minifig Details', 'Minifig Pricing'];26 const parts: string[] = [];27 const re = /^####\s+(.+)$/gm;28 const heads: Array<{ name: string; start: number; end: number }> = [];29 let m: RegExpExecArray | null;30 while ((m = re.exec(md))) heads.push({ name: m[1]!.trim(), start: m.index, end: md.length });31 for (let i = 0; i < heads.length; i++) heads[i]!.end = heads[i + 1]?.start ?? md.length;32 for (const h of heads) if (wanted.includes(h.name)) parts.push(md.slice(h.start, h.end).trim());33 return parts.join('\n\n');34}3536/** Read the value that follows a label line in the markdown ("Label\n\nValue"). */37function field(md: string, label: string): string | null {38 const re = new RegExp(`(?:^|\\n)${label.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*\\n+([^\\n]+)`, 'i');39 const m = md.match(re);40 if (!m) return null;41 const v = m[1]!.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1').replace(/\*\*/g, '').trim();42 return v || null;43}4445export interface ParsedSet {46 setNumber: string | null;47 name: string | null;48 theme: string | null;49 subtheme: string | null;50 year: number | null;51 released: string | null;52 retired: string | null;53 availability: string | null;54 pieces: number | null;55 minifigs: number | null;56 retailPrice: number | null;57 newValue: number | null;58 usedValue: number | null;59 usedRangeLow: number | null;60 usedRangeHigh: number | null;61 growthPct: number | null;62}6364export function parseSetMarkdown(md: string): ParsedSet {65 const details = md.slice(md.indexOf('#### Set Details'), md.indexOf('#### Set Pricing') > 0 ? md.indexOf('#### Set Pricing') : undefined);66 const pricing = md.slice(md.indexOf('#### Set Pricing') >= 0 ? md.indexOf('#### Set Pricing') : md.length);67 const num = (s: string | null) => {68 if (!s) return null;69 const p = parsePrice(s, 'USD');70 return p && p.amount > 0 ? p.amount : null;71 };72 const int = (s: string | null) => {73 const m = s?.replace(/,/g, '').match(/\d+/);74 return m ? Number(m[0]) : null;75 };76 const yearTxt = field(details, 'Year');77 // New/Sealed value: "New/Sealed\n\nValue\n\n**$2,946.69**" ; still-available sets: "Market price\n\n$764.15-10.1%"78 const newBlock = pricing.match(/New\/Sealed\s*\n+Value\s*\n+([^\n]+)/i)?.[1] ?? null;79 const marketPrice = pricing.match(/Market price\s*\n+\$?([\d,]+\.?\d*)/i)?.[1] ?? null;80 const usedBlock = pricing.match(/Used\s*\n+Value\s*\n+([^\n]+)/i)?.[1] ?? null;81 const usedRange = pricing.match(/Used[\s\S]{0,200}?Range\s*\n+\$?([\d,]+\.?\d*)\s*-\s*\$?([\d,]+\.?\d*)/i);82 const growth = pricing.match(/(?:^|\n)Growth\s*\n+([+-]?[\d.]+)%/i)?.[1] ?? null;83 return {84 setNumber: field(details, 'Set number'),85 name: field(details, 'Name'),86 theme: field(details, 'Theme'),87 subtheme: field(details, 'Subtheme'),88 year: yearTxt ? int(yearTxt) : null,89 released: field(details, 'Released'),90 retired: field(details, 'Retired'),91 availability: field(details, 'Availability'),92 pieces: int(field(details, 'Pieces')),93 minifigs: int(field(details, 'Minifigs')),94 retailPrice: num(field(pricing, 'Retail price')),95 newValue: num(newBlock) ?? (marketPrice ? Number(marketPrice.replace(/,/g, '')) : null),96 usedValue: num(usedBlock),97 usedRangeLow: usedRange ? Number(usedRange[1]!.replace(/,/g, '')) : null,98 usedRangeHigh: usedRange ? Number(usedRange[2]!.replace(/,/g, '')) : null,99 growthPct: growth ? Number(growth) : null,100 };101}102103export class BrickEconomyConnector extends BaseConnector {104 readonly version = '1.0.0';105 readonly parserVersion = PARSER_VERSION;106 override readonly urlPatterns = [/^https?:\/\/(www\.)?brickeconomy\.com\/set\/[^/]+\/[^/?#]+/i];107 protected override minIntervalMs = 1500;108109 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {110 const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []);111 const perSeed = Number(this.meta.config.setsPerSeed ?? 60);112 const cursor = ctx.options.cursor ?? {};113 const done = new Set<string>((cursor.doneSeeds as string[] | undefined) ?? []);114 let count = 0;115 const seenSets = new Set<string>();116 for (const seed of seeds) {117 if (done.has(seed) && ctx.options.mode !== 'backfill') continue;118 if (ctx.signal?.aborted) return;119 const setUrls = seed.includes('/set/') ? [seed] : await this.discover(ctx, seed, perSeed);120 if (setUrls.length === 0) ctx.anomaly('empty_seed', seed);121 for (const url of setUrls) {122 if (this.reached(ctx, count)) return;123 if (seenSets.has(url)) continue;124 seenSets.add(url);125 if (!(await ctx.shouldFetch(url))) continue;126 await this.throttle();127 const rec = await this.fetchSet(ctx, url);128 if (rec) {129 count++;130 yield rec;131 }132 }133 done.add(seed);134 await ctx.setCursor({ doneSeeds: [...done], updatedAt: new Date().toISOString() });135 }136 await ctx.setCursor({ doneSeeds: [], updatedAt: new Date().toISOString() });137 }138139 private async discover(ctx: CrawlContext, listUrl: string, max: number): Promise<string[]> {140 await this.throttle();141 const res = await ctx.fetch(listUrl, { minQuality: 0.2 });142 const text = `${res.markdown ?? ''}\n${res.html ?? ''}`;143 if (!res.success || !text.trim()) {144 ctx.anomaly('discover_failed', `${listUrl}: ${res.error ?? res.httpStatus}`);145 return [];146 }147 const urls = new Set<string>();148 for (const m of text.matchAll(/https?:\/\/www\.brickeconomy\.com\/set\/([0-9]+-[0-9]+)\/([a-z0-9-]+)/g)) urls.add(`${BASE}/set/${m[1]}/${m[2]}`);149 for (const m of text.matchAll(/href="\/set\/([0-9]+-[0-9]+)\/([a-z0-9-]+)"/g)) urls.add(`${BASE}/set/${m[1]}/${m[2]}`);150 return [...urls].slice(0, max);151 }152153 private async fetchSet(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {154 const res = await ctx.fetch(url, {155 expect: ['title', 'price', 'identifiers'],156 parse: (r) => {157 const md = r.markdown ?? '';158 const p = parseSetMarkdown(md);159 return { title: p.name, price: p.newValue ?? p.usedValue ?? p.retailPrice, identifiers: p.setNumber ? { set: p.setNumber } : null };160 },161 });162 if (!res.success || !res.markdown) {163 ctx.anomaly('set_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);164 return null;165 }166 const md = res.markdown;167 const title = md.match(/^#\s+(.+)$/m)?.[1]?.trim() ?? '';168 const images = [...new Set([...md.matchAll(/https:\/\/www\.brickeconomy\.com\/resources\/images\/sets\/[^\s)]+_large\.jpg/g)].map((m) => m[0]))].slice(0, 3);169 const excerpt = excerptSections(md);170 if (!excerpt.includes('Set number')) {171 ctx.anomaly('parse_failure_details', url);172 return null;173 }174 const payload: SetPayload = { kind: 'set_page', url, title, markdown: excerpt, images };175 const setNumber = parseSetMarkdown(excerpt).setNumber;176 return { url, externalId: setNumber, kind: 'catalog_item', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };177 }178179 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {180 const rec = await this.fetchSet(ctx, url.split('?')[0]!);181 return rec ? [rec] : [];182 }183184 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {185 const p = SetPayloadSchema.parse(raw.payload);186 const s = parseSetMarkdown(p.markdown);187 if (!s.setNumber || !s.name) return [];188 const baseNumber = s.setNumber.replace(/-\d+$/, '');189 const attributes: AssetAttributes = {190 categorySlug: 'lego_sets',191 subcategorySlug: null,192 franchise: s.theme,193 brand: 'LEGO',194 series: s.subtheme,195 set: s.theme,196 setCode: null,197 name: s.name,198 model: null,199 reference: null,200 number: baseNumber,201 year: s.year,202 edition: null,203 variant: s.setNumber.endsWith('-1') ? null : `variant ${s.setNumber.split('-')[1]}`,204 language: null,205 region: null,206 country: null,207 material: null,208 size: null,209 color: null,210 rarity: null,211 productionQuantity: null,212 originalMsrp: s.retailPrice,213 originalMsrpCurrency: s.retailPrice !== null ? 'USD' : null,214 identifiers: { lego_set_number: baseNumber, brickeconomy_set: s.setNumber },215 metadata: { pieces: s.pieces, minifigs: s.minifigs, released: s.released, retired: s.retired, availability: s.availability, subtheme: s.subtheme, growth_pct: s.growthPct },216 };217 const observedAt = raw.fetchedAt;218 const base = {219 connectorId: this.meta.id,220 sourceId: this.meta.sourceId,221 sourceUrl: p.url,222 externalId: s.setNumber,223 rawTitle: p.title || `${baseNumber} LEGO ${s.theme ?? ''} ${s.name}`.trim(),224 description: null,225 imageUrls: p.images,226 attributes,227 grade: { grader: null, grade: null, qualifier: null, certificationNumber: null },228 observedAt,229 confidence: 0.9,230 parserVersion: PARSER_VERSION,231 };232 const out: NormalizedRecord[] = [];233 const catalog: NormalizedCatalogItem = { kind: 'catalog_item', ...base, condition: { condition: null, conditionRaw: null, completeness: null }, releaseDate: parseSourceDate(s.released) };234 out.push(catalog);235 const obsDate = new Date(Date.UTC(observedAt.getUTCFullYear(), observedAt.getUTCMonth(), observedAt.getUTCDate()));236 const obs = (price: number, completeness: 'sealed' | 'used_complete', priceKind: NormalizedPriceObservation['priceKind'], raw: string): NormalizedPriceObservation => ({237 kind: 'price_observation',238 ...base,239 condition: { condition: completeness, conditionRaw: raw, completeness },240 priceKind,241 price,242 currency: 'USD',243 observationDate: obsDate,244 sampleSize: null,245 confidence: 0.8,246 });247 if (s.newValue) out.push(obs(s.newValue, 'sealed', 'guide_value', 'New/Sealed value (BrickEconomy estimate)'));248 if (s.usedValue) out.push(obs(s.usedValue, 'used_complete', 'guide_value', 'Used value (BrickEconomy estimate)'));249 if (s.usedRangeLow) out.push(obs(s.usedRangeLow, 'used_complete', 'low', 'Used range low (BrickEconomy)'));250 if (s.usedRangeHigh) out.push(obs(s.usedRangeHigh, 'used_complete', 'high', 'Used range high (BrickEconomy)'));251 return out;252 }253}254255export default (meta: ConnectorMeta) => new BrickEconomyConnector(meta);256