import { z } from 'zod'; import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors'; import type { NormalizedRecord } from '@rareindex/shared'; import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js'; const BASE = 'https://www.nobleknight.com'; const PARSER_VERSION = '1.0.0'; export const ProductSchema = z.object({ url: z.string(), nkId: z.string(), name: z.string(), sku: z.string().nullable(), mpn: z.string().nullable(), brand: z.string().nullable(), image: z.string().nullable(), description: z.string().nullable(), price: z.number().nullable(), currency: z.string().nullable(), itemCondition: z.string().nullable(), availability: z.string().nullable(), publisher: z.string().nullable(), productLine: z.string().nullable(), category: z.string().nullable(), genre: z.string().nullable(), type: z.string().nullable(), conditionText: z.string().nullable(), }); export type Product = z.infer; export const PayloadSchema = z.object({ kind: z.literal('product_page'), product: ProductSchema }); /** Product page: schema.org Product JSON-LD + the info lines (Publisher / Product Line / Category / Genre / Type). */ export function parseProduct(htmlText: string, url: string): Product | null { const ld = H.jsonLd(htmlText, 'Product')[0]; if (!ld) return null; const $ = H.load(htmlText); const offers = (Array.isArray(ld.offers) ? ld.offers[0] : ld.offers) as Record | undefined; const info: Record = {}; $('.info-line').each((_, el) => { const label = H.text($(el).find('.label')); const value = H.text($(el).find('.value')); if (label && value) info[label.toLowerCase()] = value; }); const nkId = url.match(/\/P\/(\d+)/)?.[1] ?? String(ld.sku ?? ''); const conditionText = H.text($('.conditions').first()) ?? H.text($('.condition, .item-condition').first()); const brand = ld.brand && typeof ld.brand === 'object' ? String((ld.brand as { name?: string }).name ?? '') : ld.brand ? String(ld.brand) : null; return { url, nkId, name: String(ld.name ?? ''), sku: ld.sku ? String(ld.sku) : null, mpn: ld.mpn ? String(ld.mpn) : null, brand: brand || null, image: Array.isArray(ld.image) ? (ld.image[0] as string | undefined) ?? null : ld.image ? String(ld.image) : null, description: ld.description ? String(ld.description).slice(0, 500) : null, price: offers?.price !== undefined && offers.price !== null ? Number(offers.price) : null, currency: offers?.priceCurrency ? String(offers.priceCurrency) : null, itemCondition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null, availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null, publisher: info.publisher ?? null, productLine: info['product line'] ?? null, category: info.category ?? null, genre: info.genre ?? null, type: info.type ?? null, conditionText: conditionText ?? null, }; } /** Taxonomy slug from Noble Knight's own labels; null → skip (RPG books, CCG singles, supplies…). */ export function nkCategory(p: Product): string | null { const cat = (p.category ?? '').toLowerCase(); const line = `${p.productLine ?? ''} ${p.publisher ?? ''} ${p.name}`.toLowerCase(); if (/games workshop|warhammer|citadel|forge world|age of sigmar|40k|necromunda|blood bowl/.test(line)) return 'warhammer'; if (/board ?game|war ?game|puzzle/.test(cat)) return 'board_games'; if (/miniature/.test(cat)) return /\b(gundam|gunpla|bandai)\b/.test(line) ? 'gundam' : null; if (/toys?, movies|action figure|toys/.test(cat)) { if (/\b(gundam|gunpla)\b/.test(line)) return 'gundam'; if (/\bfunko\b|\bpop!\b/.test(line)) return 'funko'; if (/\b(figure|figuarts|nendoroid|hot toys|neca|mcfarlane|hasbro|mattel)\b/.test(line)) return 'action_figures'; return null; } return null; } /** * Noble Knight Games (largest US used/new board-game & miniatures dealer): products are enumerated * from the public sitemaps (robots.txt allows /P/ product pages and sitemaps; category listings are * client-rendered and disallowed) and read from each page's schema.org Product data. */ export class NobleKnightConnector extends BaseConnector { readonly version = '1.0.0'; readonly parserVersion = PARSER_VERSION; protected override minIntervalMs = 1500; override readonly urlPatterns = [/^https?:\/\/(www\.)?nobleknight\.com\/P\/\d+/i]; async *crawl(ctx: CrawlContext): AsyncIterable { const sitemaps = Number(this.meta.config.sitemapCount ?? 5); const perRun = Number(this.meta.config.productsPerRun ?? 200); const smIndex = Number(ctx.options.cursor?.sitemapIndex ?? 1); const offset = Number(ctx.options.cursor?.offset ?? 0); const smUrl = `${BASE}/sitemapproducts${smIndex}.xml`; await this.throttle(); const sm = await ctx.fetch(smUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 90_000 }); if (!sm.success || !sm.html) { ctx.anomaly('page_fetch_failed', `${smUrl}: ${sm.error ?? sm.httpStatus}`); return; } const locs = [...sm.html.matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1]!.trim()).filter((u) => /\/P\/\d+/.test(u)); const slice = locs.slice(offset, offset + perRun); let count = 0; for (const url of slice) { if (ctx.signal?.aborted || this.reached(ctx, count)) break; if (!(await ctx.shouldFetch(url))) continue; await this.throttle(); const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => (r.html ? (() => { const p = parseProduct(r.html!, url); return p ? { title: p.name, price: p.price, identifiers: { sku: p.sku } } : null; })() : null) }); if (!res.success || !res.html) { ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`); continue; } const product = parseProduct(res.html, url); if (!product) continue; count++; yield { url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt }; } const nextOffset = offset + slice.length; const exhausted = nextOffset >= locs.length; await ctx.setCursor({ sitemapIndex: exhausted ? (smIndex % sitemaps) + 1 : smIndex, offset: exhausted ? 0 : nextOffset, updatedAt: new Date().toISOString() }); } async lookup(url: string, ctx: CrawlContext): Promise { const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text' }); if (!res.success || !res.html) return []; const product = parseProduct(res.html, url); return product ? [{ url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt }] : []; } async normalize(raw: RawRecordLike): Promise { const { product: p } = PayloadSchema.parse(raw.payload); const slug = nkCategory(p); if (!slug) return []; const condRaw = p.conditionText ?? (p.itemCondition === 'NewCondition' ? 'New' : p.itemCondition === 'UsedCondition' ? 'Used' : null); const cond = dealerCondition(condRaw); const attributes = lotAttributes({ categorySlug: slug, name: p.name, brand: p.publisher ?? p.brand, series: p.productLine, identifiers: { nobleknight_id: p.nkId, ...(p.mpn ? { mpn: p.mpn } : {}) }, metadata: { category: p.category, genre: p.genre, type: p.type, sku: p.sku } }); const common = { meta: this.meta, sourceUrl: p.url, externalId: p.nkId, rawTitle: p.name, attributes, imageUrls: p.image ? [p.image] : [], description: p.description, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness }; const out: NormalizedRecord[] = [makeCatalogItem({ ...common, confidence: 0.8 })]; if (p.price !== null && Number.isFinite(p.price) && p.price > 0) { const cur = (p.currency ?? 'USD') as 'USD'; out.push(makeListing({ ...common, price: p.price, currency: cur, listingType: 'fixed_price', seller: 'Noble Knight Games', location: 'US', availability: /InStock|PreOrder|LimitedAvailability/.test(p.availability ?? '') ? 'available' : 'sold', quantity: 1 })); } return out; } } export default (meta: ConnectorMeta) => new NobleKnightConnector(meta);