import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared'; import { attrs } from '../../api/_lib/shared.js'; import { usd } from '../../api/_g6-comics-toys-games-lib/comics.js'; /** * eStarland — retro/modern video game dealer (since 1991). Public platform listing pages * /platforms//?page=N (30 cards per page) give product name, platform ("Nintendo / NES"), * New/Used, price (or an "unavailable" price class) and the product URL with the store's product id. * Cloudflare challenges non-browser clients → Scrapfly without JS rendering (~1 credit per page). */ const SITE = 'https://www.estarland.com'; const PARSER_VERSION = '1.0.0'; export const CardSchema = z.object({ productId: z.string(), name: z.string(), url: z.string(), image: z.string().nullable(), platformLine: z.string().nullable(), condition: z.string().nullable(), price: z.number().nullable(), available: z.boolean() }); export type Card = z.infer; export const PagePayloadSchema = z.object({ kind: z.literal('platform_page'), url: z.string(), seed: z.string(), categorySlug: z.string(), brand: z.string().nullable(), page: z.number(), totalPages: z.number().nullable(), cards: z.array(CardSchema), snapshot: z.string().optional() }); export type PagePayload = z.infer; /** Main-grid cards only (they carry `.productConditionHolder`; carousel/featured cards use other classes and duplicate grid items). */ export function parsePlatformPage(htmlText: string, url: string, seed: string, categorySlug: string, brand: string | null, page: number): PagePayload { const $ = H.load(htmlText); const cards: Card[] = []; const seen = new Set(); $('.platform_col').each((_, el) => { const $el = $(el); if (!$el.find('.productConditionHolder').length) return; const a = $el.closest('a[href*="/product-description/"]'); const href = a.attr('href') ?? ''; const productId = href.match(/\/product-description\/[^/]+\/[^/]+\/(\d+)/)?.[1] ?? ''; const name = H.text($el.find('h4').first()) ?? ''; if (!productId || !name || seen.has(productId)) return; seen.add(productId); const priceEl = $el.find('.priceProductHolder').first(); cards.push({ productId, name, url: H.absUrl(SITE, href) ?? `${SITE}${href}`, image: $el.find('img').first().attr('src') ?? $el.find('img').first().attr('data-src') ?? null, platformLine: H.text($el.find('.platformName, .platformParagraph').first()) ?? null, condition: H.text($el.find('.productConditionHolder').first()) ?? null, price: usd(H.text(priceEl)), available: !(priceEl.attr('class') ?? '').includes('unavailPrice'), }); }); const pages = $('.commingsoon_pagig a[href*="page="], a[href*="?page="]') .map((_, a) => Number(($(a).attr('href') ?? '').match(/[?&]page=(\d+)/)?.[1] ?? 0)) .get() .filter((n) => n > 0); return { kind: 'platform_page', url, seed, categorySlug, brand, page, totalPages: pages.length ? Math.max(...pages) : null, cards }; } /** "Nintendo / NES" → { brand: 'Nintendo', platform: 'NES' }. */ export function splitPlatform(line: string | null): { brand: string | null; platform: string | null } { if (!line) return { brand: null, platform: null }; const [b, ...rest] = line.split('/').map((s) => s.trim()); return { brand: b || null, platform: rest.join(' / ') || null }; } interface Seed { path: string; categorySlug: string; brand?: string | null; pages?: number; } export class EstarlandConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 3000; async *crawl(ctx: CrawlContext): AsyncIterable { const configured = ((this.meta.config.seeds as Seed[] | undefined) ?? []).filter((s) => s.path && s.categorySlug); const seeds = ctx.options.seeds?.length ? configured.filter((s) => ctx.options.seeds!.includes(s.path)) : configured; if (!seeds.length) { ctx.anomaly('config_missing', 'no platform seeds configured'); return; } const backfill = ctx.options.mode === 'backfill'; const probe = ctx.options.mode === 'probe'; const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number }; let resume = backfill && typeof cursor.seed === 'string'; let count = 0; for (const seed of seeds) { if (resume && cursor.seed !== seed.path) continue; const maxPages = probe ? 1 : backfill ? this.policy.backfillMaxPages : (seed.pages ?? Number(this.meta.config.pagesPerSeed ?? 1)); let page = resume && cursor.page ? cursor.page : 1; resume = false; for (; page <= maxPages; page++) { if (ctx.signal?.aborted || this.reached(ctx, count)) return; const url = `${SITE}${seed.path}${page > 1 ? `?page=${page}` : ''}`; await this.throttle(url); const res = await ctx.fetch(url, { engines: ['scrapfly'], renderJs: false, country: 'us', timeoutMs: 90_000, expect: ['title', 'price', 'identifiers'], parse: (r) => { const c = r.html ? parsePlatformPage(r.html, url, seed.path, seed.categorySlug, seed.brand ?? null, page).cards[0] : undefined; return c ? { title: c.name, price: c.price, identifiers: c.productId } : null; }, }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); break; } const payload = parsePlatformPage(res.html, url, seed.path, seed.categorySlug, seed.brand ?? null, page); if (!payload.cards.length) { ctx.anomaly('parse_failure_page', url); break; } count++; yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: 'scrapfly', httpStatus: res.httpStatus, payload, snapshot: res.html, fetchedAt: res.fetchedAt }; await ctx.setCursor({ seed: seed.path, page: page + 1, updatedAt: new Date().toISOString() }); if (backfill) await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count }); if (payload.totalPages !== null && page >= payload.totalPages) break; } } if (backfill) await ctx.setCursor({ done: true, updatedAt: new Date().toISOString() }); } async normalize(raw: RawRecordLike): Promise { const p = PagePayloadSchema.parse(raw.payload); const exclude = new RegExp(String(this.meta.config.exclude ?? 'adapter|cable|cleaner|controller|memory card|charger|case\\b|stylus|replacement|repair'), 'i'); const out: NormalizedRecord[] = []; for (const c of p.cards) { if (c.price === null || exclude.test(c.name)) continue; const pl = splitPlatform(c.platformLine); const isNew = /\bnew\b/i.test(c.condition ?? ''); const attributes = attrs({ categorySlug: p.categorySlug, // seed brand only when the card really names a platform ('Nintendo / NES'); 'Multi-Platform' cards get no brand brand: pl.platform ? (p.brand ?? pl.brand) : null, set: pl.platform, name: c.name, identifiers: { estarland_product_id: c.productId }, metadata: { platform_line: c.platformLine, condition_label: c.condition, seed: p.seed }, }); out.push( NormalizedListingSchema.parse({ kind: 'listing', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: c.url, externalId: c.productId, rawTitle: `${c.name} (${c.platformLine ?? ''}${c.condition ? `, ${c.condition}` : ''})`.replace(/\(\)$/, '').trim(), imageUrls: c.image ? [c.image] : [], attributes, grade: {}, condition: { condition: null, conditionRaw: c.condition, completeness: isNew ? 'sealed' : null }, observedAt: raw.fetchedAt, confidence: 0.8, parserVersion: PARSER_VERSION, listingType: 'fixed_price', price: c.price, currency: 'USD', seller: 'eStarland', location: 'US', availability: c.available ? 'available' : 'ended', }), ); } return out; } } export default (meta: ConnectorMeta) => new EstarlandConnector(meta);