TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';3import type { NormalizedRecord } from '@rareindex/shared';4import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js';56const BASE = 'https://www.nobleknight.com';7const PARSER_VERSION = '1.0.0';89export const ProductSchema = z.object({10 url: z.string(),11 nkId: z.string(),12 name: z.string(),13 sku: z.string().nullable(),14 mpn: z.string().nullable(),15 brand: z.string().nullable(),16 image: z.string().nullable(),17 description: z.string().nullable(),18 price: z.number().nullable(),19 currency: z.string().nullable(),20 itemCondition: z.string().nullable(),21 availability: z.string().nullable(),22 publisher: z.string().nullable(),23 productLine: z.string().nullable(),24 category: z.string().nullable(),25 genre: z.string().nullable(),26 type: z.string().nullable(),27 conditionText: z.string().nullable(),28});29export type Product = z.infer<typeof ProductSchema>;30export const PayloadSchema = z.object({ kind: z.literal('product_page'), product: ProductSchema });3132/** Product page: schema.org Product JSON-LD + the info lines (Publisher / Product Line / Category / Genre / Type). */33export function parseProduct(htmlText: string, url: string): Product | null {34 const ld = H.jsonLd(htmlText, 'Product')[0];35 if (!ld) return null;36 const $ = H.load(htmlText);37 const offers = (Array.isArray(ld.offers) ? ld.offers[0] : ld.offers) as Record<string, unknown> | undefined;38 const info: Record<string, string> = {};39 $('.info-line').each((_, el) => {40 const label = H.text($(el).find('.label'));41 const value = H.text($(el).find('.value'));42 if (label && value) info[label.toLowerCase()] = value;43 });44 const nkId = url.match(/\/P\/(\d+)/)?.[1] ?? String(ld.sku ?? '');45 const conditionText = H.text($('.conditions').first()) ?? H.text($('.condition, .item-condition').first());46 const brand = ld.brand && typeof ld.brand === 'object' ? String((ld.brand as { name?: string }).name ?? '') : ld.brand ? String(ld.brand) : null;47 return {48 url,49 nkId,50 name: String(ld.name ?? ''),51 sku: ld.sku ? String(ld.sku) : null,52 mpn: ld.mpn ? String(ld.mpn) : null,53 brand: brand || null,54 image: Array.isArray(ld.image) ? (ld.image[0] as string | undefined) ?? null : ld.image ? String(ld.image) : null,55 description: ld.description ? String(ld.description).slice(0, 500) : null,56 price: offers?.price !== undefined && offers.price !== null ? Number(offers.price) : null,57 currency: offers?.priceCurrency ? String(offers.priceCurrency) : null,58 itemCondition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null,59 availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null,60 publisher: info.publisher ?? null,61 productLine: info['product line'] ?? null,62 category: info.category ?? null,63 genre: info.genre ?? null,64 type: info.type ?? null,65 conditionText: conditionText ?? null,66 };67}6869/** Taxonomy slug from Noble Knight's own labels; null → skip (RPG books, CCG singles, supplies…). */70export function nkCategory(p: Product): string | null {71 const cat = (p.category ?? '').toLowerCase();72 const line = `${p.productLine ?? ''} ${p.publisher ?? ''} ${p.name}`.toLowerCase();73 if (/games workshop|warhammer|citadel|forge world|age of sigmar|40k|necromunda|blood bowl/.test(line)) return 'warhammer';74 if (/board ?game|war ?game|puzzle/.test(cat)) return 'board_games';75 if (/miniature/.test(cat)) return /\b(gundam|gunpla|bandai)\b/.test(line) ? 'gundam' : null;76 if (/toys?, movies|action figure|toys/.test(cat)) {77 if (/\b(gundam|gunpla)\b/.test(line)) return 'gundam';78 if (/\bfunko\b|\bpop!\b/.test(line)) return 'funko';79 if (/\b(figure|figuarts|nendoroid|hot toys|neca|mcfarlane|hasbro|mattel)\b/.test(line)) return 'action_figures';80 return null;81 }82 return null;83}8485/**86 * Noble Knight Games (largest US used/new board-game & miniatures dealer): products are enumerated87 * from the public sitemaps (robots.txt allows /P/ product pages and sitemaps; category listings are88 * client-rendered and disallowed) and read from each page's schema.org Product data.89 */90export class NobleKnightConnector extends BaseConnector {91 readonly version = '1.0.0';92 readonly parserVersion = PARSER_VERSION;93 protected override minIntervalMs = 1500;94 override readonly urlPatterns = [/^https?:\/\/(www\.)?nobleknight\.com\/P\/\d+/i];9596 async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {97 const sitemaps = Number(this.meta.config.sitemapCount ?? 5);98 const perRun = Number(this.meta.config.productsPerRun ?? 200);99 const smIndex = Number(ctx.options.cursor?.sitemapIndex ?? 1);100 const offset = Number(ctx.options.cursor?.offset ?? 0);101 const smUrl = `${BASE}/sitemapproducts${smIndex}.xml`;102 await this.throttle();103 const sm = await ctx.fetch(smUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 90_000 });104 if (!sm.success || !sm.html) {105 ctx.anomaly('page_fetch_failed', `${smUrl}: ${sm.error ?? sm.httpStatus}`);106 return;107 }108 const locs = [...sm.html.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]!.trim()).filter((u) => /\/P\/\d+/.test(u));109 const slice = locs.slice(offset, offset + perRun);110 let count = 0;111 for (const url of slice) {112 if (ctx.signal?.aborted || this.reached(ctx, count)) break;113 if (!(await ctx.shouldFetch(url))) continue;114 await this.throttle();115 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => (r.html ? (() => { const p = parseProduct(r.html!, url); return p ? { title: p.name, price: p.price, identifiers: { sku: p.sku } } : null; })() : null) });116 if (!res.success || !res.html) {117 ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);118 continue;119 }120 const product = parseProduct(res.html, url);121 if (!product) continue;122 count++;123 yield { url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt };124 }125 const nextOffset = offset + slice.length;126 const exhausted = nextOffset >= locs.length;127 await ctx.setCursor({ sitemapIndex: exhausted ? (smIndex % sitemaps) + 1 : smIndex, offset: exhausted ? 0 : nextOffset, updatedAt: new Date().toISOString() });128 }129130 async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {131 const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text' });132 if (!res.success || !res.html) return [];133 const product = parseProduct(res.html, url);134 return product ? [{ url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt }] : [];135 }136137 async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {138 const { product: p } = PayloadSchema.parse(raw.payload);139 const slug = nkCategory(p);140 if (!slug) return [];141 const condRaw = p.conditionText ?? (p.itemCondition === 'NewCondition' ? 'New' : p.itemCondition === 'UsedCondition' ? 'Used' : null);142 const cond = dealerCondition(condRaw);143 const attributes = lotAttributes({ categorySlug: slug, name: p.name, brand: p.publisher ?? p.brand, series: p.productLine, identifiers: { nobleknight_id: p.nkId, ...(p.mpn ? { mpn: p.mpn } : {}) }, metadata: { category: p.category, genre: p.genre, type: p.type, sku: p.sku } });144 const common = { meta: this.meta, sourceUrl: p.url, externalId: p.nkId, rawTitle: p.name, attributes, imageUrls: p.image ? [p.image] : [], description: p.description, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness };145 const out: NormalizedRecord[] = [makeCatalogItem({ ...common, confidence: 0.8 })];146 if (p.price !== null && Number.isFinite(p.price) && p.price > 0) {147 const cur = (p.currency ?? 'USD') as 'USD';148 out.push(makeListing({ ...common, price: p.price, currency: cur, listingType: 'fixed_price', seller: 'Noble Knight Games', location: 'US', availability: /InStock|PreOrder|LimitedAvailability/.test(p.availability ?? '') ? 'available' : 'sold', quantity: 1 }));149 }150 return out;151 }152}153154export default (meta: ConnectorMeta) => new NobleKnightConnector(meta);155