import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { attrs } from '../../api/_lib/shared.js'; import { usd } from '../../api/_g6-comics-toys-games-lib/comics.js'; /** * Entertainment Earth — public category listing pages (/s//p?page=N, 60 tiles per page). * Each tile carries structured data attributes on its add-to-cart button (data-sku, data-name, data-price, * data-company = manufacturer, data-theme = franchise, data-collect = category) plus the product URL, * image, displayed price / struck list price and the button label (Pre-Order / Add to Cart / Sold Out). * Cloudflare challenges non-browser clients → Scrapfly without JS rendering (~1 credit per page). */ const SITE = 'https://www.entertainmentearth.com'; const PARSER_VERSION = '1.0.0'; export const TileSchema = z.object({ sku: z.string(), name: z.string(), url: z.string(), image: z.string().nullable(), price: z.number().nullable(), oldPrice: z.number().nullable(), company: z.string().nullable(), theme: z.string().nullable(), collect: z.string().nullable(), buttonText: z.string().nullable(), ribbon: z.string().nullable(), }); export type Tile = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('category_page'), url: z.string(), seed: z.string(), categorySlug: z.string(), page: z.number(), totalPages: z.number().nullable(), tiles: z.array(TileSchema), snapshot: z.string().optional() }); export type PagePayload = z.infer; export function parseCategoryPage(htmlText: string, url: string, seed: string, categorySlug: string, page: number): PagePayload { const $ = H.load(htmlText); const tiles: Tile[] = []; $('.product-tile').each((_, el) => { const $el = $(el); const btn = $el.find('button.add-to-cart').first(); const sku = btn.attr('data-sku') ?? $el.find('[data-sku]').first().attr('data-sku') ?? ''; const href = $el.find('a.product-url').first().attr('href') ?? ''; const name = btn.attr('data-name') ?? H.text($el.find('.item-name')) ?? $el.find('img').first().attr('alt') ?? ''; if (!sku || !href || !name) return; const priceTxt = H.text($el.find('.item-price')) ?? btn.attr('data-price') ?? null; const oldTxt = H.text($el.find('.old-price')) ?? null; const img = $el.find('.image img').first().attr('src') ?? null; tiles.push({ sku, name: name.replace(/\s+/g, ' ').trim(), url: H.absUrl(SITE, href) ?? `${SITE}${href}`, image: img ? H.absUrl(SITE, img) : null, price: usd(priceTxt) ?? usd(btn.attr('data-price')), oldPrice: usd(oldTxt), company: btn.attr('data-company')?.trim() || null, theme: btn.attr('data-theme')?.trim() || null, collect: btn.attr('data-collect')?.trim() || null, buttonText: H.text(btn.find('.item-buttontext')) ?? H.text(btn) ?? null, ribbon: H.text($el.find('.promotion .ribbon-text')) ?? null, }); }); const pages = $('a[href*="page="]') .map((_, a) => Number(($(a).attr('href') ?? '').match(/[?&]page=(\d+)/)?.[1] ?? 0)) .get() .filter((n) => Number.isFinite(n) && n > 0); return { kind: 'category_page', url, seed, categorySlug, page, totalPages: pages.length ? Math.max(...pages) : null, tiles }; } /** "Pre-Order: Add to Cart" → available (preorder) · "Add to Cart" → available · "Sold Out"/"Alert Me" → sold. */ export function availabilityOf(buttonText: string | null): { availability: 'available' | 'sold' | 'ended' | 'unknown'; preorder: boolean } { const t = (buttonText ?? '').toLowerCase(); if (!t) return { availability: 'unknown', preorder: false }; if (/pre-?order/.test(t)) return { availability: 'available', preorder: true }; if (/add to cart|in stock|buy/.test(t)) return { availability: 'available', preorder: false }; if (/sold out|alert|notify|unavailable|out of stock/.test(t)) return { availability: 'sold', preorder: false }; if (/coming soon|expected/.test(t)) return { availability: 'available', preorder: true }; return { availability: 'unknown', preorder: false }; } interface Seed { path: string; categorySlug: string; pages?: number; } export class EntertainmentEarthConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; private seeds(): Seed[] { return ((this.meta.config.seeds as Seed[] | undefined) ?? []).filter((s) => s.path && s.categorySlug); } async *crawl(ctx: CrawlContext): AsyncIterable { const configured = this.seeds(); const seeds = ctx.options.seeds?.length ? configured.filter((s) => ctx.options.seeds!.includes(s.path)) : configured; if (!seeds.length) { ctx.anomaly('config_missing', 'no category seeds configured'); return; } const backfill = ctx.options.mode === 'backfill'; const probe = ctx.options.mode === 'probe'; const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number }; let resume = backfill && typeof cursor.seed === 'string'; let count = 0; for (const seed of seeds) { if (resume && cursor.seed !== seed.path) continue; const maxPages = probe ? 1 : backfill ? this.policy.backfillMaxPages : (seed.pages ?? Number(this.meta.config.pagesPerSeed ?? 2)); let page = resume && cursor.page ? cursor.page : 1; resume = false; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${SITE}${seed.path}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(url); const res = await ctx.fetch(url, { engines: ['scrapfly'], renderJs: false, country: 'us', timeoutMs: 90_000, expect: ['title', 'price', 'identifiers'], parse: (r) => { const t = r.html ? parseCategoryPage(r.html, url, seed.path, seed.categorySlug, page).tiles[0] : undefined; return t ? { title: t.name, price: t.price, identifiers: t.sku } : null; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parseCategoryPage(res.html, url, seed.path, seed.categorySlug, page); if (!payload.tiles.length) { ctx.anomaly('parse_failure_page', url); break; } count++; yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: 'scrapfly', httpStatus: res.httpStatus, payload, snapshot: res.html, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seed: seed.path, page: page + 1, updatedAt: new Date().toISOString() }); if (backfill) await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count }); if (payload.totalPages !== null && page >= payload.totalPages) break; } } if (backfill) await ctx.setCursor({ done: true, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const exclude = new RegExp(String(this.meta.config.exclude ?? 'gift card|t-shirt|hoodie|mug|tumbler|poster|backpack|keychain|lanyard|blanket|socks'), 'i'); const out: NormalizedRecord[] = []; for (const t of p.tiles) { if (t.price === null) continue; if (exclude.test(`${t.name} ${t.collect ?? ''}`)) continue; const av = availabilityOf(t.buttonText); const scale = t.name.match(/\b(1:\d{1,3}|1\/\d{1,3}|\d{1,2}(?:\.\d)?-Inch|\d{1,2}(?:\.\d)?")\b/i)?.[1] ?? null; const attributes = attrs({ categorySlug: p.categorySlug, brand: t.company, franchise: t.theme, name: t.name, size: scale, identifiers: { sku: t.sku, ee_sku: t.sku }, metadata: { ee_category: t.collect, button_text: t.buttonText, ribbon: t.ribbon, preorder: av.preorder, list_price: t.oldPrice, seed: p.seed }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: t.url, externalId: t.sku, rawTitle: t.name, imageUrls: t.image ? [t.image.replace(/md\.jpg$/, 'lg.jpg')] : [], attributes, grade: {}, condition: { condition: null, conditionRaw: 'new', completeness: 'sealed' }, observedAt: raw.fetchedAt, confidence: 0.85, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: t.price, currency: 'USD', seller: 'Entertainment Earth', location: 'US', availability: av.availability, }), ); } return out; } } export default (meta: ConnectorMeta) => new EntertainmentEarthConnector(meta);