TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';4import { attrs } from '../_lib/shared.js';5import { usd } from '../_g6-comics-toys-games-lib/comics.js';67/**8 * Miniature Market — Shopware 6 storefront; public category listing pages (/board-games.html?p=N, 24 product9 * boxes per page, 379 pages) served over plain HTTPS to the honest bot UA. Each box gives the product name,10 * URL, image, SKU (data-sku), Shopware product id, retail/list price and Miniature Market's price, plus the buy11 * widget (Add to cart / Preorder). robots.txt allows ?p= pagination and sets Crawl-delay: 10.12 */13const SITE = 'https://www.miniaturemarket.com';14const PARSER_VERSION = '1.0.0';1516export const BoxSchema = z.object({ name: z.string(), url: z.string(), image: z.string().nullable(), sku: z.string().nullable(), productId: z.string().nullable(), price: z.number().nullable(), listPrice: z.number().nullable(), buttonText: z.string().nullable(), badges: z.array(z.string()) });17export type Box = z.infer<typeof BoxSchema>;18export const PagePayloadSchema = z.object({ kind: z.literal('listing_page'), url: z.string(), seed: z.string(), categorySlug: z.string(), page: z.number(), totalPages: z.number().nullable(), boxes: z.array(BoxSchema), snapshot: z.string().optional() });19export type PagePayload = z.infer<typeof PagePayloadSchema>;2021export function parseListingPage(htmlText: string, url: string, seed: string, categorySlug: string, page: number): PagePayload {22 const $ = H.load(htmlText);23 const boxes: Box[] = [];24 $('.product-box').each((_, el) => {25 const $el = $(el);26 const a = $el.find('a.product-name').first();27 const name = (a.attr('title') ?? H.text(a) ?? '').replace(/\s+/g, ' ').trim();28 const href = a.attr('href') ?? '';29 if (!name || !href) return;30 const redirect = $el.find('input[name="redirectParameters"]').attr('value') ?? '';31 let productId: string | null = null;32 try {33 productId = redirect ? String((JSON.parse(redirect) as { productId?: string }).productId ?? '') || null : null;34 } catch {35 productId = redirect.match(/productId[^0-9a-f]+([0-9a-f]{32})/)?.[1] ?? null;36 }37 boxes.push({38 name,39 url: H.absUrl(SITE, href) ?? href,40 image: $el.find('img.product-image').first().attr('src') ?? null,41 sku: $el.find('[data-sku]').first().attr('data-sku') ?? null,42 productId,43 price: usd(H.text($el.find('.product-price').first())),44 listPrice: usd(H.text($el.find('.list-price-price').first())),45 buttonText: H.text($el.find('.btn-buy').first()) ?? null,46 badges: $el.find('.product-badges .badge').map((__, b) => H.text($(b)) ?? '').get().filter(Boolean),47 });48 });49 const totalTxt = $('.pagination-nav .visually-hidden').first().text();50 const totalPages = Number(totalTxt.match(/of\s+(\d+)/)?.[1] ?? 0) || null;51 return { kind: 'listing_page', url, seed, categorySlug, page, totalPages, boxes };52}5354interface Seed {55 path: string;56 categorySlug: string;57 pages?: number;58}5960export class MiniatureMarketConnector extends BaseConnector {61 readonly version = '1.0.0';62 readonly parserVersion = PARSER_VERSION;63 /** robots.txt Crawl-delay: 10 */64 protected override minIntervalMs = 10_000;6566 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {67 const configured = ((this.meta.config.seeds as Seed[] | undefined) ?? []).filter((s) => s.path && s.categorySlug);68 const seeds = ctx.options.seeds?.length ? configured.filter((s) => ctx.options.seeds!.includes(s.path)) : configured;69 if (!seeds.length) {70 ctx.anomaly('config_missing', 'no category seeds configured');71 return;72 }73 const backfill = ctx.options.mode === 'backfill';74 const probe = ctx.options.mode === 'probe';75 const cursor = (ctx.options.cursor ?? {}) as { seed?: string; page?: number };76 let resume = backfill && typeof cursor.seed === 'string';77 let count = 0;78 for (const seed of seeds) {79 if (resume && cursor.seed !== seed.path) continue;80 const maxPages = probe ? 1 : backfill ? this.policy.backfillMaxPages : (seed.pages ?? Number(this.meta.config.pagesPerSeed ?? 2));81 let page = resume && cursor.page ? cursor.page : 1;82 resume = false;83 for (; page <= maxPages; page++) {84 if (ctx.signal?.aborted || this.reached(ctx, count)) return;85 const url = `${SITE}${seed.path}${page > 1 ? `?p=${page}` : ''}`;86 await this.throttle(url);87 const res = await ctx.fetch(url, {88 engines: ['api'],89 responseType: 'text',90 timeoutMs: 60_000,91 expect: ['title', 'price', 'identifiers'],92 parse: (r) => {93 const b = r.html ? parseListingPage(r.html, url, seed.path, seed.categorySlug, page).boxes[0] : undefined;94 return b ? { title: b.name, price: b.price, identifiers: b.sku } : null;95 },96 });97 if (!res.success || !res.html) {98 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);99 break;100 }101 const payload = parseListingPage(res.html, url, seed.path, seed.categorySlug, page);102 if (!payload.boxes.length) {103 ctx.anomaly('parse_failure_page', url);104 break;105 }106 count++;107 yield { url, externalId: `${seed.path}:p${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, snapshot: res.html, fetchedAt: res.fetchedAt };108 await ctx.setCursor({ seed: seed.path, page: page + 1, updatedAt: new Date().toISOString() });109 if (backfill) await ctx.progress({ page, totalPages: payload.totalPages, itemsProcessed: count });110 if (payload.totalPages !== null && page >= payload.totalPages) break;111 }112 }113 if (backfill) await ctx.setCursor({ done: true, updatedAt: new Date().toISOString() });114 }115116 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {117 const p = PagePayloadSchema.parse(raw.payload);118 const exclude = new RegExp(String(this.meta.config.exclude ?? 'sleeves?|playmat|dice set|token set|insert|organizer|gift card'), 'i');119 const out: NormalizedRecord[] = [];120 for (const b of p.boxes) {121 if (b.price === null || exclude.test(b.name)) continue;122 const preorder = /pre-?order/i.test(b.buttonText ?? '') || b.badges.some((x) => /pre-?order/i.test(x));123 const edition = b.name.match(/\b(\d+(?:st|nd|rd|th) Edition|Deluxe Edition|Collector'?s Edition|Kickstarter Edition|Retail Edition|Big Box)\b/i)?.[1] ?? null;124 const attributes = attrs({125 categorySlug: p.categorySlug,126 name: b.name,127 edition,128 originalMsrp: b.listPrice,129 originalMsrpCurrency: b.listPrice !== null ? 'USD' : null,130 identifiers: { ...(b.sku ? { sku: b.sku } : {}), ...(b.productId ? { miniaturemarket_product_id: b.productId } : {}) },131 metadata: { list_price: b.listPrice, button_text: b.buttonText, badges: b.badges, preorder, seed: p.seed },132 });133 out.push(134 NormalizedListingSchema.parse({135 kind: 'listing',136 connectorId: this.meta.id,137 sourceId: this.meta.sourceId,138 sourceUrl: b.url,139 externalId: b.productId ?? b.sku ?? b.url,140 rawTitle: b.name,141 imageUrls: b.image ? [b.image] : [],142 attributes,143 grade: {},144 condition: { condition: null, conditionRaw: 'new', completeness: 'sealed' },145 observedAt: raw.fetchedAt,146 confidence: 0.85,147 parserVersion: PARSER_VERSION,148 listingType: 'fixed_price',149 price: b.price,150 currency: 'USD',151 seller: 'Miniature Market',152 location: 'US',153 availability: /add to cart|pre-?order/i.test(b.buttonText ?? '') ? 'available' : b.buttonText ? 'ended' : 'unknown',154 }),155 );156 }157 return out;158 }159}160161export default (meta: ConnectorMeta) => new MiniatureMarketConnector(meta);162