TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../../api/_lib/shared.js';5import { usd } from '../../api/_g6-comics-toys-games-lib/comics.js';67/**8 * eStarland — retro/modern video game dealer (since 1991). Public platform listing pages9 * /platforms/<Name>/<id>?page=N (30 cards per page) give product name, platform ("Nintendo / NES"),10 * New/Used, price (or an "unavailable" price class) and the product URL with the store's product id.11 * Cloudflare challenges non-browser clients → Scrapfly without JS rendering (~1 credit per page).12 */13const SITE = 'https://www.estarland.com';14const PARSER_VERSION = '1.0.0';1516export const CardSchema = z.object({ productId: z.string(), name: z.string(), url: z.string(), image: z.string().nullable(), platformLine: z.string().nullable(), condition: z.string().nullable(), price: z.number().nullable(), available: z.boolean() });17export type Card = z.infer<typeof CardSchema>;18export const PagePayloadSchema = z.object({ kind: z.literal('platform_page'), url: z.string(), seed: z.string(), categorySlug: z.string(), brand: z.string().nullable(), page: z.number(), totalPages: z.number().nullable(), cards: z.array(CardSchema), snapshot: z.string().optional() });19export type PagePayload = z.infer<typeof PagePayloadSchema>;2021/** Main-grid cards only (they carry `.productConditionHolder`; carousel/featured cards use other classes and duplicate grid items). */22export function parsePlatformPage(htmlText: string, url: string, seed: string, categorySlug: string, brand: string | null, page: number): PagePayload {23 const $ = H.load(htmlText);24 const cards: Card[] = [];25 const seen = new Set<string>();26 $('.platform_col').each((_, el) => {27 const $el = $(el);28 if (!$el.find('.productConditionHolder').length) return;29 const a = $el.closest('a[href*="/product-description/"]');30 const href = a.attr('href') ?? '';31 const productId = href.match(/\/product-description\/[^/]+\/[^/]+\/(\d+)/)?.[1] ?? '';32 const name = H.text($el.find('h4').first()) ?? '';33 if (!productId || !name || seen.has(productId)) return;34 seen.add(productId);35 const priceEl = $el.find('.priceProductHolder').first();36 cards.push({37 productId,38 name,39 url: H.absUrl(SITE, href) ?? `${SITE}${href}`,40 image: $el.find('img').first().attr('src') ?? $el.find('img').first().attr('data-src') ?? null,41 platformLine: H.text($el.find('.platformName, .platformParagraph').first()) ?? null,42 condition: H.text($el.find('.productConditionHolder').first()) ?? null,43 price: usd(H.text(priceEl)),44 available: !(priceEl.attr('class') ?? '').includes('unavailPrice'),45 });46 });47 const pages = $('.commingsoon_pagig a[href*="page="], a[href*="?page="]')48 .map((_, a) => Number(($(a).attr('href') ?? '').match(/[?&]page=(\d+)/)?.[1] ?? 0))49 .get()50 .filter((n) => n > 0);51 return { kind: 'platform_page', url, seed, categorySlug, brand, page, totalPages: pages.length ? Math.max(...pages) : null, cards };52}5354/** "Nintendo / NES" → { brand: 'Nintendo', platform: 'NES' }. */55export function splitPlatform(line: string | null): { brand: string | null; platform: string | null } {56 if (!line) return { brand: null, platform: null };57 const [b, ...rest] = line.split('/').map((s) => s.trim());58 return { brand: b || null, platform: rest.join(' / ') || null };59}6061interface Seed {62 path: string;63 categorySlug: string;64 brand?: string | null;65 pages?: number;66}6768export class EstarlandConnector extends BaseConnector {69 readonly version = '1.0.0';70 readonly parserVersion = PARSER_VERSION;71 protected override minIntervalMs = 3000;7273 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {74 const configured = ((this.meta.config.seeds as Seed[] | undefined) ?? []).filter((s) => s.path && s.categorySlug);75 const seeds = ctx.options.seeds?.length ? configured.filter((s) => ctx.options.seeds!.includes(s.path)) : configured;76 if (!seeds.length) {77 ctx.anomaly('config_missing', 'no platform seeds configured');78 return;79 }80 const backfill = ctx.options.mode === 'backfill';81 const probe = ctx.options.mode === 'probe';82 const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number };83 let resume = backfill && typeof cursor.seed === 'string';84 let count = 0;85 for (const seed of seeds) {86 if (resume && cursor.seed !== seed.path) continue;87 const maxPages = probe ? 1 : backfill ? this.policy.backfillMaxPages : (seed.pages ?? Number(this.meta.config.pagesPerSeed ?? 1));88 let page = resume && cursor.page ? cursor.page : 1;89 resume = false;90 for (; page <= maxPages; page++) {91 if (ctx.signal?.aborted || this.reached(ctx, count)) return;92 const url = `${SITE}${seed.path}${page > 1 ? `?page=${page}` : ''}`;93 await this.throttle(url);94 const res = await ctx.fetch(url, {95 engines: ['scrapfly'],96 renderJs: false,97 country: 'us',98 timeoutMs: 90_000,99 expect: ['title', 'price', 'identifiers'],100 parse: (r) => {101 const c = r.html ? parsePlatformPage(r.html, url, seed.path, seed.categorySlug, seed.brand ?? null, page).cards[0] : undefined;102 return c ? { title: c.name, price: c.price, identifiers: c.productId } : null;103 },104 });105 if (!res.success || !res.html) {106 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);107 break;108 }109 const payload = parsePlatformPage(res.html, url, seed.path, seed.categorySlug, seed.brand ?? null, page);110 if (!payload.cards.length) {111 ctx.anomaly('parse_failure_page', url);112 break;113 }114 count++;115 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: 'scrapfly', httpStatus: res.httpStatus, payload, snapshot: res.html, fetchedAt: res.fetchedAt };116 await ctx.setCursor({ seed: seed.path, page: page + 1, updatedAt: new Date().toISOString() });117 if (backfill) await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count });118 if (payload.totalPages !== null && page >= payload.totalPages) break;119 }120 }121 if (backfill) await ctx.setCursor({ done: true, updatedAt: new Date().toISOString() });122 }123124 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {125 const p = PagePayloadSchema.parse(raw.payload);126 const exclude = new RegExp(String(this.meta.config.exclude ?? 'adapter|cable|cleaner|controller|memory card|charger|case\\b|stylus|replacement|repair'), 'i');127 const out: NormalizedRecord[] = [];128 for (const c of p.cards) {129 if (c.price === null || exclude.test(c.name)) continue;130 const pl = splitPlatform(c.platformLine);131 const isNew = /\bnew\b/i.test(c.condition ?? '');132 const attributes = attrs({133 categorySlug: p.categorySlug,134 // seed brand only when the card really names a platform ('Nintendo / NES'); 'Multi-Platform' cards get no brand135 brand: pl.platform ? (p.brand ?? pl.brand) : null,136 set: pl.platform,137 name: c.name,138 identifiers: { estarland_product_id: c.productId },139 metadata: { platform_line: c.platformLine, condition_label: c.condition, seed: p.seed },140 });141 out.push(142 NormalizedListingSchema.parse({143 kind: 'listing',144 connectorId: this.meta.id,145 sourceId: this.meta.sourceId,146 sourceUrl: c.url,147 externalId: c.productId,148 rawTitle: `${c.name} (${c.platformLine ?? ''}${c.condition ? `, ${c.condition}` : ''})`.replace(/\(\)$/, '').trim(),149 imageUrls: c.image ? [c.image] : [],150 attributes,151 grade: {},152 condition: { condition: null, conditionRaw: c.condition, completeness: isNew ? 'sealed' : null },153 observedAt: raw.fetchedAt,154 confidence: 0.8,155 parserVersion: PARSER_VERSION,156 listingType: 'fixed_price',157 price: c.price,158 currency: 'USD',159 seller: 'eStarland',160 location: 'US',161 availability: c.available ? 'available' : 'ended',162 }),163 );164 }165 return out;166 }167}168169export default (meta: ConnectorMeta) => new EstarlandConnector(meta);170