TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../../api/_lib/shared.js';5import { usd } from '../../api/_g6-comics-toys-games-lib/comics.js';67/**8 * Entertainment Earth — public category listing pages (/s/<category>/p?page=N, 60 tiles per page).9 * Each tile carries structured data attributes on its add-to-cart button (data-sku, data-name, data-price,10 * data-company = manufacturer, data-theme = franchise, data-collect = category) plus the product URL,11 * image, displayed price / struck list price and the button label (Pre-Order / Add to Cart / Sold Out).12 * Cloudflare challenges non-browser clients → Scrapfly without JS rendering (~1 credit per page).13 */14const SITE = 'https://www.entertainmentearth.com';15const PARSER_VERSION = '1.0.0';1617export const TileSchema = z.object({18 sku: z.string(),19 name: z.string(),20 url: z.string(),21 image: z.string().nullable(),22 price: z.number().nullable(),23 oldPrice: z.number().nullable(),24 company: z.string().nullable(),25 theme: z.string().nullable(),26 collect: z.string().nullable(),27 buttonText: z.string().nullable(),28 ribbon: z.string().nullable(),29});30export type Tile = z.infer<typeof TileSchema>;31export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), seed: z.string(), categorySlug: z.string(), page: z.number(), totalPages: z.number().nullable(), tiles: z.array(TileSchema), snapshot: z.string().optional() });32export type PagePayload = z.infer<typeof PagePayloadSchema>;3334export function parseCategoryPage(htmlText: string, url: string, seed: string, categorySlug: string, page: number): PagePayload {35 const $ = H.load(htmlText);36 const tiles: Tile[] = [];37 $('.product-tile').each((_, el) => {38 const $el = $(el);39 const btn = $el.find('button.add-to-cart').first();40 const sku = btn.attr('data-sku') ?? $el.find('[data-sku]').first().attr('data-sku') ?? '';41 const href = $el.find('a.product-url').first().attr('href') ?? '';42 const name = btn.attr('data-name') ?? H.text($el.find('.item-name')) ?? $el.find('img').first().attr('alt') ?? '';43 if (!sku || !href || !name) return;44 const priceTxt = H.text($el.find('.item-price')) ?? btn.attr('data-price') ?? null;45 const oldTxt = H.text($el.find('.old-price')) ?? null;46 const img = $el.find('.image img').first().attr('src') ?? null;47 tiles.push({48 sku,49 name: name.replace(/\s+/g, ' ').trim(),50 url: H.absUrl(SITE, href) ?? `${SITE}${href}`,51 image: img ? H.absUrl(SITE, img) : null,52 price: usd(priceTxt) ?? usd(btn.attr('data-price')),53 oldPrice: usd(oldTxt),54 company: btn.attr('data-company')?.trim() || null,55 theme: btn.attr('data-theme')?.trim() || null,56 collect: btn.attr('data-collect')?.trim() || null,57 buttonText: H.text(btn.find('.item-buttontext')) ?? H.text(btn) ?? null,58 ribbon: H.text($el.find('.promotion .ribbon-text')) ?? null,59 });60 });61 const pages = $('a[href*="page="]')62 .map((_, a) => Number(($(a).attr('href') ?? '').match(/[?&]page=(\d+)/)?.[1] ?? 0))63 .get()64 .filter((n) => Number.isFinite(n) && n > 0);65 return { kind: 'category_page', url, seed, categorySlug, page, totalPages: pages.length ? Math.max(...pages) : null, tiles };66}6768/** "Pre-Order: Add to Cart" → available (preorder) · "Add to Cart" → available · "Sold Out"/"Alert Me" → sold. */69export function availabilityOf(buttonText: string | null): { availability: 'available' | 'sold' | 'ended' | 'unknown'; preorder: boolean } {70 const t = (buttonText ?? '').toLowerCase();71 if (!t) return { availability: 'unknown', preorder: false };72 if (/pre-?order/.test(t)) return { availability: 'available', preorder: true };73 if (/add to cart|in stock|buy/.test(t)) return { availability: 'available', preorder: false };74 if (/sold out|alert|notify|unavailable|out of stock/.test(t)) return { availability: 'sold', preorder: false };75 if (/coming soon|expected/.test(t)) return { availability: 'available', preorder: true };76 return { availability: 'unknown', preorder: false };77}7879interface Seed {80 path: string;81 categorySlug: string;82 pages?: number;83}8485export class EntertainmentEarthConnector extends BaseConnector {86 readonly version = '1.0.0';87 readonly parserVersion = PARSER_VERSION;88 protected override minIntervalMs = 3000;8990 private seeds(): Seed[] {91 return ((this.meta.config.seeds as Seed[] | undefined) ?? []).filter((s) => s.path && s.categorySlug);92 }9394 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {95 const configured = this.seeds();96 const seeds = ctx.options.seeds?.length ? configured.filter((s) => ctx.options.seeds!.includes(s.path)) : configured;97 if (!seeds.length) {98 ctx.anomaly('config_missing', 'no category seeds configured');99 return;100 }101 const backfill = ctx.options.mode === 'backfill';102 const probe = ctx.options.mode === 'probe';103 const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number };104 let resume = backfill && typeof cursor.seed === 'string';105 let count = 0;106 for (const seed of seeds) {107 if (resume && cursor.seed !== seed.path) continue;108 const maxPages = probe ? 1 : backfill ? this.policy.backfillMaxPages : (seed.pages ?? Number(this.meta.config.pagesPerSeed ?? 2));109 let page = resume && cursor.page ? cursor.page : 1;110 resume = false;111 for (; page <= maxPages; page++) {112 if (ctx.signal?.aborted || this.reached(ctx, count)) return;113 const url = `${SITE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;114 await this.throttle(url);115 const res = await ctx.fetch(url, {116 engines: ['scrapfly'],117 renderJs: false,118 country: 'us',119 timeoutMs: 90_000,120 expect: ['title', 'price', 'identifiers'],121 parse: (r) => {122 const t = r.html ? parseCategoryPage(r.html, url, seed.path, seed.categorySlug, page).tiles[0] : undefined;123 return t ? { title: t.name, price: t.price, identifiers: t.sku } : null;124 },125 });126 if (!res.success || !res.html) {127 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);128 break;129 }130 const payload = parseCategoryPage(res.html, url, seed.path, seed.categorySlug, page);131 if (!payload.tiles.length) {132 ctx.anomaly('parse_failure_page', url);133 break;134 }135 count++;136 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: 'scrapfly', httpStatus: res.httpStatus, payload, snapshot: res.html, fetchedAt: res.fetchedAt };137 await ctx.setCursor({ seed: seed.path, page: page + 1, updatedAt: new Date().toISOString() });138 if (backfill) await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count });139 if (payload.totalPages !== null && page >= payload.totalPages) break;140 }141 }142 if (backfill) await ctx.setCursor({ done: true, updatedAt: new Date().toISOString() });143 }144145 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {146 const p = PagePayloadSchema.parse(raw.payload);147 const exclude = new RegExp(String(this.meta.config.exclude ?? 'gift card|t-shirt|hoodie|mug|tumbler|poster|backpack|keychain|lanyard|blanket|socks'), 'i');148 const out: NormalizedRecord[] = [];149 for (const t of p.tiles) {150 if (t.price === null) continue;151 if (exclude.test(`${t.name} ${t.collect ?? ''}`)) continue;152 const av = availabilityOf(t.buttonText);153 const scale = t.name.match(/\b(1:\d{1,3}|1\/\d{1,3}|\d{1,2}(?:\.\d)?-Inch|\d{1,2}(?:\.\d)?")\b/i)?.[1] ?? null;154 const attributes = attrs({155 categorySlug: p.categorySlug,156 brand: t.company,157 franchise: t.theme,158 name: t.name,159 size: scale,160 identifiers: { sku: t.sku, ee_sku: t.sku },161 metadata: { ee_category: t.collect, button_text: t.buttonText, ribbon: t.ribbon, preorder: av.preorder, list_price: t.oldPrice, seed: p.seed },162 });163 out.push(164 NormalizedListingSchema.parse({165 kind: 'listing',166 connectorId: this.meta.id,167 sourceId: this.meta.sourceId,168 sourceUrl: t.url,169 externalId: t.sku,170 rawTitle: t.name,171 imageUrls: t.image ? [t.image.replace(/md\.jpg$/, 'lg.jpg')] : [],172 attributes,173 grade: {},174 condition: { condition: null, conditionRaw: 'new', completeness: 'sealed' },175 observedAt: raw.fetchedAt,176 confidence: 0.85,177 parserVersion: PARSER_VERSION,178 listingType: 'fixed_price',179 price: t.price,180 currency: 'USD',181 seller: 'Entertainment Earth',182 location: 'US',183 availability: av.availability,184 }),185 );186 }187 return out;188 }189}190191export default (meta: ConnectorMeta) => new EntertainmentEarthConnector(meta);192