SPB Git
spboucher.ai

spb/rareindex

Public

Connectors: SCP, Hake's, Morphy, Lelands, BBTS, Noble Knight, Trainz (agent U)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 7, 2026) parent f5091c0 Browse files

38 changed files +2,702 −0

38 changed files
Load all 38 files Unified Split
added connectors/api/_memorabilia-lib/_capture-nk.ts +16 −0
@@ -0,0 +1,16 @@
1+import { readFileSync } from 'node:fs';
2+import { ConnectorMetaSchema, createCrawlContext, createRouter } from '@rareindex/connectors';
3+import { saveFixture } from '@rareindex/connectors/testing';
4+import { childLogger } from '@rareindex/shared';
5+import create from '../noble-knight/index.js';
6+const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync('connectors/api/noble-knight/meta.json', 'utf8')));
7+const c = create(meta);
8+const ctx = createCrawlContext({ router: createRouter({}), meta, options: { mode: 'probe' }, log: childLogger({}) });
9+for (const [name, url] of [['board-game', 'https://www.nobleknight.com/P/2147993326/007---Spectre-Board-Game'], ['gunpla', 'https://www.nobleknight.com/P/2148351787/002-Ex-S-GUNDAM']] as const) {
10+ const raws = await c.lookup!(url, ctx);
11+ const raw = raws[0]!;
12+ const fetchedAt = raw.fetchedAt ?? new Date();
13+ const records = await c.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt });
14+ saveFixture(meta.id, name, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt, payload: raw.payload }, expect: { count: records.length, kinds: [...new Set(records.map((r) => r.kind))] }, note: `Captured live via lookup(${url}).` });
15+ console.log(name, records.length, records.map((r) => r.kind));
16+}
added connectors/api/_memorabilia-lib/index.ts +391 −0
@@ -0,0 +1,391 @@
1+import { html as H, type ConnectorMeta } from '@rareindex/connectors';
2+import { parseGradeFromTitle } from '@rareindex/taxonomy';
3+import type { AssetAttributes, CurrencyCode, NormalizedAuctionLot, NormalizedCatalogItem, NormalizedListing } from '@rareindex/shared';
4+import { lotAttributes, money } from '../../firecrawl/_carlib/index.js';
5+
6+/**
7+ * Helpers shared by the memorabilia / toys / dealer connectors (SCP, Hake's, Morphy, Lelands, BBTS,
8+ * Noble Knight, Trainz). Source vocabulary stays here; canonical models stay clean.
9+ */
10+
11+export const BOT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) RareIndexBot/0.1 (+https://www.rareindex.io/about; market-data research)';
12+
13+// ---------------------------------------------------------------------------------------------
14+// Plain HTTP helpers for sites whose paging/selection is an ASP.NET postback (what a browser does:
15+// a form POST carrying the page's own hidden fields; no authentication involved).
16+// ---------------------------------------------------------------------------------------------
17+export interface HttpResult {
18+ status: number;
19+ text: string;
20+ cookies: string[];
21+}
22+
23+export async function httpText(url: string, opts: { method?: 'GET' | 'POST'; form?: Record<string, string>; cookie?: string; referer?: string; timeoutMs?: number } = {}): Promise<HttpResult> {
24+ const ctrl = new AbortController();
25+ const timer = setTimeout(() => ctrl.abort(), opts.timeoutMs ?? 45_000);
26+ try {
27+ const headers: Record<string, string> = { 'user-agent': BOT_UA, accept: 'text/html,application/xhtml+xml,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.9' };
28+ if (opts.cookie) headers.cookie = opts.cookie;
29+ if (opts.referer) headers.referer = opts.referer;
30+ let body: string | undefined;
31+ if (opts.method === 'POST') {
32+ headers['content-type'] = 'application/x-www-form-urlencoded';
33+ body = new URLSearchParams(opts.form ?? {}).toString();
34+ }
35+ const res = await fetch(url, { method: opts.method ?? 'GET', headers, body, signal: ctrl.signal, redirect: 'follow' });
36+ const text = await res.text();
37+ const cookies: string[] = [];
38+ const anyHeaders = res.headers as Headers & { getSetCookie?: () => string[] };
39+ if (typeof anyHeaders.getSetCookie === 'function') cookies.push(...anyHeaders.getSetCookie());
40+ else {
41+ const sc = res.headers.get('set-cookie');
42+ if (sc) cookies.push(sc);
43+ }
44+ return { status: res.status, text, cookies };
45+ } finally {
46+ clearTimeout(timer);
47+ }
48+}
49+
50+/** Cookie header from Set-Cookie lines (name=value only). */
51+export function cookieHeader(setCookies: string[], previous = ''): string {
52+ const jar = new Map<string, string>();
53+ for (const part of previous.split(';')) {
54+ const [k, ...v] = part.trim().split('=');
55+ if (k) jar.set(k, v.join('='));
56+ }
57+ for (const sc of setCookies) {
58+ const first = sc.split(';')[0] ?? '';
59+ const [k, ...v] = first.trim().split('=');
60+ if (k) jar.set(k, v.join('='));
61+ }
62+ return [...jar.entries()].map(([k, v]) => `${k}=${v}`).join('; ');
63+}
64+
65+/** ASP.NET hidden inputs (__VIEWSTATE, __EVENTVALIDATION, …) plus any other hidden fields. */
66+export function hiddenFields(htmlText: string): Record<string, string> {
67+ const out: Record<string, string> = {};
68+ const re = /<input[^>]+type="hidden"[^>]*>/gi;
69+ let m: RegExpExecArray | null;
70+ while ((m = re.exec(htmlText))) {
71+ const tag = m[0];
72+ const name = tag.match(/name="([^"]+)"/)?.[1];
73+ if (!name) continue;
74+ const value = tag.match(/value="([^"]*)"/)?.[1] ?? '';
75+ out[name] = value.replace(/&quot;/g, '"').replace(/&amp;/g, '&');
76+ }
77+ return out;
78+}
79+
80+/** Selected value of a <select name="…"> (first option when none is marked). */
81+export function selectedOption(htmlText: string, name: string): string | null {
82+ const m = htmlText.match(new RegExp(`<select[^>]*name="${name.replace(/[$]/g, '\\$')}"[^>]*>([\\s\\S]*?)</select>`, 'i'));
83+ if (!m) return null;
84+ const sel = m[1]!.match(/<option[^>]*selected(?:="selected")?[^>]*value="([^"]*)"/i) ?? m[1]!.match(/<option[^>]*value="([^"]*)"[^>]*selected/i);
85+ if (sel) return sel[1]!;
86+ return m[1]!.match(/<option[^>]*value="([^"]*)"/i)?.[1] ?? null;
87+}
88+
89+export function selectOptions(htmlText: string, name: string): Array<{ value: string; label: string }> {
90+ const m = htmlText.match(new RegExp(`<select[^>]*name="${name.replace(/[$]/g, '\\$')}"[^>]*>([\\s\\S]*?)</select>`, 'i'));
91+ if (!m) return [];
92+ const out: Array<{ value: string; label: string }> = [];
93+ const re = /<option[^>]*value="([^"]*)"[^>]*>([^<]*)</gi;
94+ let o: RegExpExecArray | null;
95+ while ((o = re.exec(m[1]!))) out.push({ value: o[1]!, label: o[2]!.replace(/&amp;/g, '&').trim() });
96+ return out;
97+}
98+
99+// ---------------------------------------------------------------------------------------------
100+// Bidsquare-hosted catalog sites (SCP Auctions, Hake's) share one HTML structure.
101+// ---------------------------------------------------------------------------------------------
102+export interface BidsquareEvent {
103+ id: string;
104+ name: string;
105+ url: string;
106+ status: 'upcoming' | 'live' | 'past' | 'unknown';
107+ startDate: string | null;
108+ endDate: string | null;
109+}
110+export interface BidsquareItem {
111+ itemId: string;
112+ url: string;
113+ title: string;
114+ lotNumber: string | null;
115+ image: string | null;
116+ /** "Sold for" | "Current Bid" | "Starting Bid" | "Passed" | null */
117+ priceLabel: string | null;
118+ price: number | null;
119+ bids: number | null;
120+ estimateLow: number | null;
121+ estimateHigh: number | null;
122+ /** unix seconds from the countdown script (start, end) when present */
123+ startsAt: number | null;
124+ endsAt: number | null;
125+}
126+export interface BidsquareCatalog {
127+ event: BidsquareEvent;
128+ page: number;
129+ totalPages: number | null;
130+ items: BidsquareItem[];
131+}
132+
133+export function parseBidsquareEvents(htmlText: string): BidsquareEvent[] {
134+ const $ = H.load(htmlText);
135+ const out: BidsquareEvent[] = [];
136+ $('.gtm-visible_event').each((_, el) => {
137+ const e = $(el);
138+ const id = e.attr('data-event_id');
139+ const name = e.attr('data-event_name') ?? H.text(e.find('h1')) ?? '';
140+ const href = e.find('h1 a').attr('href') ?? e.find('a[href*="/auctions/"]').first().attr('href');
141+ if (!id || !href) return;
142+ const st = e.attr('data-event_status') ?? 'unknown';
143+ out.push({ id, name, url: href.split('#')[0]!, status: st === 'past' || st === 'upcoming' || st === 'live' ? st : 'unknown', startDate: null, endDate: null });
144+ });
145+ return out;
146+}
147+
148+export function parseBidsquareCatalog(htmlText: string, page: number): BidsquareCatalog | null {
149+ const $ = H.load(htmlText);
150+ let event: BidsquareEvent | null = null;
151+ for (const ld of H.jsonLd(htmlText, 'Event')) {
152+ const url = String(ld.url ?? '');
153+ const id = url.match(/-(\d+)$/)?.[1] ?? null;
154+ if (!id) continue;
155+ event = { id, name: String(ld.name ?? ''), url, status: 'unknown', startDate: ld.startDate ? String(ld.startDate) : null, endDate: ld.endDate ? String(ld.endDate) : null };
156+ break;
157+ }
158+ const first = $('[data-item_id][data-event_id]').first();
159+ if (!event) {
160+ const id = first.attr('data-event_id');
161+ if (!id) return null;
162+ event = { id, name: first.attr('data-event_name') ?? '', url: '', status: 'unknown', startDate: null, endDate: null };
163+ }
164+ const st = first.attr('data-event_status');
165+ if (st === 'past' || st === 'upcoming' || st === 'live') event.status = st;
166+ const items: BidsquareItem[] = [];
167+ const seen = new Set<string>();
168+ $('div[id^="stl-"][data-item_id]').each((_, el) => {
169+ const c = $(el);
170+ const itemId = c.attr('data-item_id')!;
171+ if (seen.has(itemId)) return;
172+ seen.add(itemId);
173+ const link = c.find('.lot_title a').first();
174+ const url = link.attr('href') ?? c.find('.catalog_img a').attr('href') ?? '';
175+ const title = H.text(link) ?? c.find('.catalog_img img').attr('alt') ?? '';
176+ if (!url || !title) return;
177+ const lotNumber = H.text(c.find('.lot_Num'))?.replace(/^Lot\s*/i, '') ?? null;
178+ const image = c.find('.catalog_img img').attr('src') ?? null;
179+ const label = H.text(c.find('.bid_txt span').first());
180+ const priceText = H.text(c.find('.bid_txt .price').first());
181+ const bidsText = H.text(c.find('.bidLength .num').first());
182+ const est = c.find('.estimate_amount').attr('data-exchange');
183+ let estimateLow: number | null = null;
184+ let estimateHigh: number | null = null;
185+ if (est) {
186+ try {
187+ const j = JSON.parse(est.replace(/&quot;/g, '"')) as { low_est?: string; high_est?: string };
188+ estimateLow = j.low_est ? Number(j.low_est) : null;
189+ estimateHigh = j.high_est ? Number(j.high_est) : null;
190+ } catch {
191+ /* ignore */
192+ }
193+ }
194+ const script = c.find('script').text();
195+ const cd = script.match(/countDownTimer\((\d+),\s*(\d+)/);
196+ items.push({
197+ itemId,
198+ url,
199+ title,
200+ lotNumber,
201+ image,
202+ priceLabel: label ?? null,
203+ price: money(priceText, 'USD')?.amount ?? null,
204+ bids: bidsText ? Number(bidsText.replace(/[^\d]/g, '')) || null : null,
205+ estimateLow,
206+ estimateHigh,
207+ startsAt: cd ? Number(cd[1]) : null,
208+ endsAt: cd ? Number(cd[2]) : null,
209+ });
210+ });
211+ const pages = $('ul.pagination li[data-page]')
212+ .map((_, li) => Number($(li).attr('data-page')))
213+ .get()
214+ .filter((n) => Number.isFinite(n) && n > 0);
215+ return { event, page, totalPages: pages.length ? Math.max(...pages) : null, items };
216+}
217+
218+/** "2026-07-26 22:30:00 EDT" → Date (Bidsquare prints US Eastern local time). */
219+export function bidsquareDate(s: string | null | undefined): Date | null {
220+ if (!s) return null;
221+ const m = s.match(/(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})\s*(EDT|EST|PDT|PST|CDT|CST|MDT|MST|UTC)?/);
222+ if (!m) return null;
223+ const offsets: Record<string, number> = { EDT: 4, EST: 5, CDT: 5, CST: 6, MDT: 6, MST: 7, PDT: 7, PST: 8, UTC: 0 };
224+ const off = offsets[m[7] ?? 'EDT'] ?? 4;
225+ return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]) + off, Number(m[5]), Number(m[6])));
226+}
227+
228+// ---------------------------------------------------------------------------------------------
229+// Category mappers (keyword based; null/family fallbacks, never invented attributes)
230+// ---------------------------------------------------------------------------------------------
231+const SPORT_WORDS: Array<[RegExp, string]> = [
232+ [/\b(baseball|topps|bowman|goudey|t206|bond bread|play ball|leaf 1948|mlb|world series|yankees|red sox|dodgers|jackie robinson|babe ruth|mickey mantle|lou gehrig|ty cobb|honus wagner)\b/i, 'baseball'],
233+ [/\b(basketball|nba|fleer basketball|fleer|hoops|jordan rookie|michael jordan|lebron|kobe|lakers|celtics)\b/i, 'basketball'],
234+ [/\b(football|nfl|super bowl|quarterback|packers|cowboys|steelers)\b/i, 'football'],
235+ [/\b(hockey|nhl|stanley cup|o-pee-chee|opc|canadiens|bruins|maple leafs)\b/i, 'hockey'],
236+ [/\b(soccer|fifa|world cup|premier league|messi|ronaldo|pel[eé])\b/i, 'soccer'],
237+];
238+const CARD_WORDS = /\b(card|rookie|psa|sgc|bgs|beckett|cgc|wax pack|unopened|refractor|parallel|#\s?\d+|1\/1)\b/i;
239+const MEMORABILIA_WORDS = /\b(jersey|game[- ]used|game[- ]worn|bat\b|glove|helmet|ball\b|puck|ring\b|trophy|ticket|program|contract|photo|photograph|signed|autograph|cleats|shoes|sneakers|medal|pennant|uniform)\b/i;
240+
241+/** Sports houses (SCP, Lelands, Hunt): cards by sport vs memorabilia. */
242+export function sportsCategory(title: string): string {
243+ const isCard = CARD_WORDS.test(title) && !MEMORABILIA_WORDS.test(title);
244+ if (isCard) {
245+ for (const [re, s] of SPORT_WORDS) if (re.test(title)) return `${s}_cards`;
246+ if (/\b(pok[eé]mon|charizard)\b/i.test(title)) return 'pokemon';
247+ if (/\b(magic: the gathering|mtg|black lotus)\b/i.test(title)) return 'magic_the_gathering';
248+ return 'other_sports_cards';
249+ }
250+ if (/\b(olympic|olympics)\b/i.test(title)) return 'olympic_collectibles';
251+ if (/\b(boxing|ufc|golf|tennis|nascar|wrestling|wwe|wwf|racing)\b/i.test(title)) return 'sports_memorabilia';
252+ return 'sports_memorabilia';
253+}
254+
255+/** Pop-culture houses (Hake's): toys, comics, political, pins… */
256+export function popCultureCategory(title: string): string {
257+ const t = title;
258+ if (/\bcgc\b|\bcbcs\b|comic book|#\s?\d+\s*\(?(19|20)\d\d|amazing spider-man|detective comics|action comics|x-men|batman|superman|marvel comics|dc comics/i.test(t) && !/\b(afa|figure|playset|vehicle)\b/i.test(t)) {
259+ if (/\b(marvel|spider-man|x-men|avengers|hulk|fantastic four|iron man|captain america|thor)\b/i.test(t)) return 'marvel_comics';
260+ if (/\b(dc|batman|superman|detective comics|action comics|wonder woman|flash|green lantern)\b/i.test(t)) return 'dc_comics';
261+ return 'independent_comics';
262+ }
263+ if (/\b(original art|comic art|cover art|splash page|comic book page|storyboard)\b/i.test(t)) return 'animation_art';
264+ if (/\b(campaign|president|political|button\b.*\b(19|18)\d\d|inaugur|senator|governor|election)\b/i.test(t)) return 'political_memorabilia';
265+ if (/\b(star wars|kenner|luke skywalker|darth vader|boba fett|tusken|stormtrooper|jedi|sith|millennium falcon)\b/i.test(t)) return 'star_wars';
266+ if (/\b(garbage pail kids|gpk|wacky packages|non-sport|trading card set)\b/i.test(t)) return 'non_sport_cards';
267+ if (/\b(psa|sgc|bgs)\s*\d|\btopps\b|\bbowman\b|rookie card/i.test(t)) return sportsCategory(t);
268+ if (/\b(pinback|pin-back|pin back|\bpin\b|badge)\b/i.test(t)) return 'pins';
269+ if (/\b(disney|mickey mouse|donald duck|disneyland|walt disney)\b/i.test(t)) return 'disney_collectibles';
270+ if (/\b(lunch box|lunchbox|thermos|pez|board game|premium ring|cereal|advertising|sign\b)\b/i.test(t)) return 'advertising';
271+ if (/\b(animation cel|production cel|cel\b)\b/i.test(t)) return 'animation_art';
272+ if (/\b(record|album|lp\b|concert|beatles|elvis|kiss\b)\b/i.test(t)) return 'music_memorabilia';
273+ if (/\b(movie|film|poster|lobby card|one sheet|screen[- ]used|prop\b)\b/i.test(t)) return 'movie_posters';
274+ if (/\b(signed|autograph)\b/i.test(t)) return 'autographs';
275+ if (/\b(g\.?i\.? joe|transformers|he-man|masters of the universe|tmnt|teenage mutant|action figure|afa\b|moc\b|mib\b|playset|hot wheels|matchbox|tin toy|wind-up|battery op|marx|ideal|mego|mattel|hasbro|lionel)\b/i.test(t)) return 'vintage_toys';
276+ if (/\b(jersey|game used|game worn|baseball|football|basketball|hockey|boxing)\b/i.test(t)) return 'sports_memorabilia';
277+ return 'vintage_toys';
278+}
279+
280+/** Morphy: department label + auction/lot titles → slug; firearms lots are excluded upstream. */
281+export function morphyCategory(dept: string | null, auctionTitle: string, lotTitle: string): string | null {
282+ const d = (dept ?? '').toLowerCase();
283+ const a = auctionTitle.toLowerCase();
284+ const t = lotTitle;
285+ if (/firearm|weapon|militaria|edged/.test(d) || /firearm|militaria/.test(a) || /\b(rifle|pistol|revolver|shotgun|carbine|ammunition|cartridge|bayonet)\b/i.test(t)) return null;
286+ if (/coin-op|coin op|arcade|slot machine|jukebox|vending/.test(d + a)) return /\b(slot machine|trade stimulator|casino)\b/i.test(t) ? 'casino_memorabilia' : /\b(jukebox|arcade|pinball)\b/i.test(t) ? 'arcade_pinball' : /\b(sign|tin|display|calendar|poster)\b/i.test(t) ? 'advertising' : 'vending_machines';
287+ if (/\bcoins?\b|currency|banknote|numismat/.test(d + a)) return /\b(note|bill|currency|dollar bill)\b/i.test(t) ? 'banknotes' : 'coins';
288+ if (/perfume|fragrance/.test(d + a)) return 'perfume';
289+ if (/petroliana|automobilia|gas|oil|soda|advertising/.test(d + a)) return /\b(sign|clock|thermometer|display|calendar|poster|globe)\b/i.test(t) || /advertising|petroliana|soda/.test(a) ? 'advertising' : 'automotive_memorabilia';
290+ if (/train/.test(d + a) || /\b(lionel|american flyer|marklin|märklin|ives|standard gauge|o gauge|ho scale)\b/i.test(t)) return 'model_trains';
291+ if (/doll/.test(d + a) && /\b(doll|bisque|barbie|steiff|teddy|bear)\b/i.test(t)) return 'dolls';
292+ if (/toy/.test(d + a)) return /\b(bank|cast iron)\b/i.test(t) ? 'vintage_toys' : 'vintage_toys';
293+ if (/sport/.test(d + a)) return sportsCategory(t);
294+ if (/jewel|watch/.test(d + a)) return /\b(watch|wristwatch|pocket watch)\b/i.test(t) ? 'other_watches' : 'jewelry';
295+ if (/fine|decorative|lamp|glass|art/.test(d + a)) {
296+ if (/\b(lamp|tiffany|handel|pairpoint)\b/i.test(t)) return 'antiques';
297+ if (/\b(perfume|atomizer|scent bottle)\b/i.test(t)) return 'perfume';
298+ if (/\b(vase|glass|paperweight|cameo glass|loetz|galle|gallé|daum|steuben|lalique|murano)\b/i.test(t)) return 'glass_crystal';
299+ if (/\b(porcelain|meissen|sevres|s[eè]vres|royal doulton|figurine|pottery|ceramic)\b/i.test(t)) return 'porcelain';
300+ if (/\b(sterling|silver)\b/i.test(t)) return 'silver';
301+ if (/\b(clock|regulator)\b/i.test(t)) return 'clocks';
302+ if (/\b(oil on canvas|painting|bronze|sculpture|lithograph|print)\b/i.test(t)) return 'art';
303+ return 'antiques';
304+ }
305+ if (/comic|pop culture|movie|music|entertainment/.test(d + a)) return popCultureCategory(t);
306+ return null;
307+}
308+
309+// ---------------------------------------------------------------------------------------------
310+// Builders for non-sale records
311+// ---------------------------------------------------------------------------------------------
312+export interface BaseInput {
313+ meta: ConnectorMeta;
314+ sourceUrl: string;
315+ externalId: string;
316+ rawTitle: string;
317+ attributes: AssetAttributes;
318+ imageUrls?: string[];
319+ description?: string | null;
320+ observedAt: Date;
321+ parserVersion: string;
322+ confidence?: number;
323+ grader?: string | null;
324+ grade?: string | null;
325+ condition?: string | null;
326+ conditionRaw?: string | null;
327+ completeness?: string | null;
328+}
329+
330+function base(i: BaseInput) {
331+ return {
332+ connectorId: i.meta.id,
333+ sourceId: i.meta.sourceId,
334+ sourceUrl: i.sourceUrl,
335+ externalId: i.externalId,
336+ rawTitle: i.rawTitle,
337+ description: i.description ?? null,
338+ imageUrls: i.imageUrls ?? [],
339+ attributes: i.attributes,
340+ grade: { grader: i.grader ?? null, grade: i.grade ?? null, qualifier: null, certificationNumber: null },
341+ condition: { condition: i.condition ?? null, conditionRaw: i.conditionRaw ?? null, completeness: i.completeness ?? null },
342+ observedAt: i.observedAt,
343+ confidence: i.confidence ?? 0.85,
344+ parserVersion: i.parserVersion,
345+ };
346+}
347+
348+export function makeListing(i: BaseInput & { price: number | null; currency: CurrencyCode | null; listingType?: NormalizedListing['listingType']; availability?: NormalizedListing['availability']; seller?: string | null; location?: string | null; quantity?: number | null; listedAt?: Date | null; endsAt?: Date | null; bidCount?: number | null }): NormalizedListing {
349+ return { kind: 'listing', ...base(i), listingType: i.listingType ?? 'fixed_price', price: i.price, currency: i.currency, seller: i.seller ?? null, sellerReputation: null, location: i.location ?? null, shippingCost: null, quantity: i.quantity ?? null, listedAt: i.listedAt ?? null, endsAt: i.endsAt ?? null, availability: i.availability ?? 'available', bidCount: i.bidCount ?? null };
350+}
351+
352+export function makeLot(i: BaseInput & { auctionHouse: string; auctionName?: string | null; lotNumber?: string | null; startsAt?: Date | null; endsAt?: Date | null; estimateLow?: number | null; estimateHigh?: number | null; currentBid?: number | null; currency: CurrencyCode | null; status?: NormalizedAuctionLot['status']; location?: string | null }): NormalizedAuctionLot {
353+ return { kind: 'auction_lot', ...base(i), auctionHouse: i.auctionHouse, auctionName: i.auctionName ?? null, lotNumber: i.lotNumber ?? null, startsAt: i.startsAt ?? null, endsAt: i.endsAt ?? null, estimateLow: i.estimateLow ?? null, estimateHigh: i.estimateHigh ?? null, currentBid: i.currentBid ?? null, currency: i.currency, status: i.status ?? 'unknown', location: i.location ?? null };
354+}
355+
356+export function makeCatalogItem(i: BaseInput & { releaseDate?: Date | null }): NormalizedCatalogItem {
357+ return { kind: 'catalog_item', ...base(i), releaseDate: i.releaseDate ?? null };
358+}
359+
360+/** Grade for sales/listings: only graded companies, never 'raw' as grader. */
361+export function gradeOf(title: string): { grader: string | null; grade: string | null } {
362+ const g = parseGradeFromTitle(title);
363+ if (!g.grader || g.grader === 'raw') return { grader: null, grade: null };
364+ let grade = g.grade;
365+ if (!grade) {
366+ // Houses often write the label between company and number ("SGC FR 1.5", "PSA VG-EX 4", "PSA NM-MT+ 8.5").
367+ const m = title.match(/\b(PSA|SGC|BGS|CGC|CBCS|TAG)\b[\s:-]*(?:[A-Z]{1,3}(?:[-/][A-Z]{1,3})?\+?\s*)?(\d{1,2}(?:\.\d)?)\b/i);
368+ if (m && Number(m[2]) <= 10) grade = m[2]!;
369+ }
370+ return { grader: g.grader, grade };
371+}
372+
373+/** AFA / CAS / UKG toy grades ("AFA 85", "AFA Qualified 80") kept as metadata, not graders (not in taxonomy). */
374+export function toyGrade(title: string): { company: string; grade: string } | null {
375+ const m = title.match(/\b(AFA|CAS|UKG)\s*(?:Qualified\s*)?(\d{2,3})/i);
376+ return m ? { company: m[1]!.toUpperCase(), grade: m[2]! } : null;
377+}
378+
379+/** Condition words from dealer labels → 'boxed_toys' scale (board games, trains, figures). */
380+export function dealerCondition(raw: string | null | undefined): { condition: string | null; completeness: string | null } {
381+ if (!raw) return { condition: null, completeness: null };
382+ const r = raw.toLowerCase();
383+ if (/\b(new|sealed|shrink|sw\b|mint in box|mib|misb|nib|moc)\b/.test(r)) return { condition: 'mint_in_box', completeness: 'sealed' };
384+ if (/\b(like new|ln\b|near mint|nm\b|mint\b)\b/.test(r)) return { condition: 'near_mint_box', completeness: 'boxed' };
385+ if (/\b(excellent|ex\b|very good|vg\b|vg\+|ex\+)\b/.test(r)) return { condition: 'boxed', completeness: 'boxed' };
386+ if (/\b(good|g\b|fair|played|used|restored)\b/.test(r)) return { condition: 'loose_complete', completeness: 'loose' };
387+ if (/\b(poor|damaged|parts|incomplete)\b/.test(r)) return { condition: 'loose', completeness: 'loose' };
388+ return { condition: null, completeness: null };
389+}
390+
391+export { lotAttributes, money };
added connectors/api/hakes/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1+import path from 'node:path';
2+import { fileURLToPath } from 'node:url';
3+import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4+
5+const dir = path.dirname(fileURLToPath(import.meta.url));
6+if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'catalog-page', 4);
7+else await runSmoke(dir, Number(process.argv[2] ?? 2));
added connectors/api/hakes/index.test.ts +51 −0
@@ -0,0 +1,51 @@
1+import { describe, expect, it } from 'vitest';
2+import { readFileSync } from 'node:fs';
3+import path from 'node:path';
4+import { fileURLToPath } from 'node:url';
5+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6+import { localMeta } from '../_lib/local-meta.js';
7+import { popCultureCategory, toyGrade } from '../_memorabilia-lib/index.js';
8+import createConnector from './index.js';
9+
10+const dir = path.dirname(fileURLToPath(import.meta.url));
11+const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
12+const connector = createConnector(meta);
13+
14+describe('hakes', () => {
15+ runFixtureSuite(connector, it, expect);
16+
17+ it('maps pop-culture titles to taxonomy slugs', () => {
18+ expect(popCultureCategory('STAR WARS (1978) - LUKE SKYWALKER 12 BACK-B AFA 85 Y-NM+')).toBe('star_wars');
19+ expect(popCultureCategory('AMAZING SPIDER-MAN #300 1988 CGC 9.8 NM/MINT')).toBe('marvel_comics');
20+ expect(popCultureCategory('DETECTIVE COMICS #27 1939 CGC 3.0')).toBe('dc_comics');
21+ expect(popCultureCategory('1860 ABRAHAM LINCOLN CAMPAIGN FERROTYPE PIN')).toBe('political_memorabilia');
22+ expect(popCultureCategory('1952 TOPPS #311 MICKEY MANTLE PSA 4')).toBe('baseball_cards');
23+ expect(popCultureCategory('GARBAGE PAIL KIDS 1985 SERIES 1 WAX BOX')).toBe('non_sport_cards');
24+ expect(popCultureCategory('MICKEY MOUSE 1930s LIONEL HANDCAR')).toBe('disney_collectibles');
25+ expect(toyGrade('STAR WARS DROIDS 1985 - A-WING FIGHTER VEHICLE AFA QUALIFIED 60 Q-EX')).toEqual({ company: 'AFA', grade: '60' });
26+ });
27+
28+ it('upcoming catalogs become auction lots with estimates and end times; sold lots would become sales', async () => {
29+ const fx = loadFixture('hakes', 'catalog-page');
30+ const out = await connector.normalize(fx.raw);
31+ expect(out.length).toBeGreaterThan(0);
32+ const lot = out.find((r) => r.kind === 'auction_lot');
33+ if (lot && lot.kind === 'auction_lot') {
34+ expect(lot.auctionHouse).toBe("Hake's Auctions");
35+ expect(lot.currency).toBe('USD');
36+ expect(lot.endsAt).toBeInstanceOf(Date);
37+ }
38+ // Same payload flagged as past with a sold label → sale
39+ const p = structuredClone(fx.raw.payload) as { event: { status: string }; items: Array<{ priceLabel: string | null; price: number | null }> };
40+ p.event.status = 'past';
41+ p.items[0]!.priceLabel = 'Sold for';
42+ p.items[0]!.price = 1234;
43+ const sold = await connector.normalize({ ...fx.raw, payload: p });
44+ const s = sold.find((r) => r.kind === 'sale');
45+ expect(s).toBeDefined();
46+ if (s && s.kind === 'sale') {
47+ expect(s.price).toBe(1234);
48+ expect(s.buyerPremiumIncluded).toBeNull();
49+ }
50+ });
51+});
added connectors/api/hakes/index.ts +33 −0
@@ -0,0 +1,33 @@
1+import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors';
2+import { parseBidsquareEvents, popCultureCategory } from '../_memorabilia-lib/index.js';
3+import { BidsquareHouseConnector } from '../scp-auctions/index.js';
4+
5+/**
6+ * Hake's Auctions (Bidsquare platform, same HTML as SCP). Hake's hides its past-auction list
7+ * (/auctions/past returns "No Auction Available"), so this connector works in two phases:
8+ * upcoming catalogs are crawled for auction lots and remembered in the cursor; once an event's
9+ * catalog reports status 'past' the same pages yield realized "Sold for" prices.
10+ */
11+export class HakesConnector extends BidsquareHouseConnector {
12+ protected override async listEvents(ctx: CrawlContext): Promise<Array<{ id: string; name: string; url: string; status: string }>> {
13+ const known = (Array.isArray(ctx.options.cursor?.knownEvents) ? (ctx.options.cursor!.knownEvents as Array<{ id: string; name: string; url: string }>) : []).filter((e) => e && e.id && e.url);
14+ const up = await this.fetchHtml(ctx, `${this.opts.base}/auctions`);
15+ const upcoming = up.html ? parseBidsquareEvents(up.html) : [];
16+ const upcomingIds = new Set(upcoming.map((e) => e.id));
17+ // Events seen before that are no longer upcoming are revisited as past (their catalog reports the status).
18+ const past = known.filter((e) => !upcomingIds.has(e.id)).map((e) => ({ ...e, status: 'past' }));
19+ const merged = [...known.filter((e) => upcomingIds.has(e.id)), ...upcoming.filter((e) => !known.some((k) => k.id === e.id)).map((e) => ({ id: e.id, name: e.name, url: e.url }))];
20+ await ctx.setCursor({ ...(ctx.options.cursor ?? {}), knownEvents: merged.slice(-60) });
21+ return [...past, ...upcoming.map((e) => ({ id: e.id, name: e.name, url: e.url, status: 'upcoming' }))];
22+ }
23+}
24+
25+export default (meta: ConnectorMeta) =>
26+ new HakesConnector(meta, {
27+ base: 'https://www.hakes.com',
28+ house: "Hake's Auctions",
29+ category: popCultureCategory,
30+ buyerPremiumIncluded: null,
31+ idKey: 'hakes_item_id',
32+ location: 'US',
33+ });
added connectors/api/hakes/meta.json +34 −0
@@ -0,0 +1,34 @@
1+{
2+ "id": "hakes",
3+ "displayName": "Hake's Auctions (pop culture catalogs)",
4+ "sourceId": "hakes",
5+ "sourceName": "Hake's Auctions",
6+ "sourceType": "auction_house",
7+ "sourceUrl": "https://www.hakes.com",
8+ "module": "api/hakes",
9+ "enginePriority": ["api"],
10+ "categories": ["vintage_toys", "star_wars", "marvel_comics", "dc_comics", "independent_comics", "political_memorabilia", "pins", "non_sport_cards", "disney_collectibles", "advertising", "animation_art", "movie_posters", "music_memorabilia", "sports_memorabilia", "autographs"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": false,
15+ "supportsSold": true,
16+ "supportsAuctions": true,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": false,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.85,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.hakes.com/terms",
26+ "accessNotes": "Plain HTTPS on the public Bidsquare-hosted site (robots.txt disallows /search, account pages and filtered/sorted query URLs; we only load /auctions and unfiltered catalog pages with ?page=N, 2 s politeness). Hake's hides its past-auction list, so upcoming catalogs are crawled for auction lots (estimate, current bid, countdown end) and their URLs are remembered in the connector cursor; after an event closes the same catalog pages report status 'past' with 'Sold for $X' → sales dated by the lot/event end. AFA/CAS toy grades are kept in metadata (not graders). Whether 'Sold for' includes the buyer's premium is not stated on the page → null. 0 credits.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "eventsPerRun": 2,
31+ "pagesPerEvent": 6,
32+ "includeUpcoming": true
33+ }
34+}
added connectors/api/morphy/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1+import path from 'node:path';
2+import { fileURLToPath } from 'node:url';
3+import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4+
5+const dir = path.dirname(fileURLToPath(import.meta.url));
6+if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'catalog-page', 4);
7+else await runSmoke(dir, Number(process.argv[2] ?? 2));
added connectors/api/morphy/index.test.ts +62 −0
@@ -0,0 +1,62 @@
1+import { describe, expect, it } from 'vitest';
2+import { readFileSync } from 'node:fs';
3+import path from 'node:path';
4+import { fileURLToPath } from 'node:url';
5+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6+import { localMeta } from '../_lib/local-meta.js';
7+import { morphyCategory } from '../_memorabilia-lib/index.js';
8+import createConnector, { parseCatalog, parsePastList } from './index.js';
9+
10+const dir = path.dirname(fileURLToPath(import.meta.url));
11+const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
12+const connector = createConnector(meta);
13+
14+const LIST = `<div id="auctionfeed"><article class="event auction clearfix catalog-2"><div class="event-content"><span class="category-title"><a href="/auctions/divisions/fine-decorative-art/">Fine &amp; Decorative Arts</a></span>
15+<h3><a href="https://morphyauctions.com/auctions/past-auctions/online-only-perfume-bottles/">Online Only Perfume Bottles</a></h3><div class="date">August 11, 2026</div>
16+<ul class="event-grid"><li class="pre-bid"><a href="https://auctions.morphyauctions.com/catalog.aspx?auctionid=716"><span>Online Listing</span></a></li></ul></div></article>
17+<article class="event auction"><div class="event-content"><span class="category-title"><a href="#">Firearms</a></span><h3><a href="#">Online Only Firearms &amp; Militaria</a></h3><div class="date">August 04, 2026</div><ul><li><a href="https://auctions.morphyauctions.com/catalog.aspx?auctionid=700">x</a></li></ul></div></article></div>`;
18+
19+const CATALOG = `<h2 id="ofpages"> / 19</h2><div id="galleryList"><ul><li class=""><div class="lot "><div class="lotInner"><h5><span id='LotLabel'>Lot </span><span class='LotNumberSign'>#</span><span id='LotNumber'>1001</span><span id="LotNumber Colon">: </span><span id="LotName"><a href='http://auctions.morphyauctions.com/CAMEO_GLASS_FLORAL_PERFUME_WITH_ATOMIZER-LOT662424.aspx'>CAMEO GLASS FLORAL PERFUME WITH ATOMIZER</a></span></h5>
20+<div class="imageDiv"><a href='#'><img class='lotImage' src='/ItemImages/000662/26320016_1_sm.jpeg'></a></div><div class="lotData"><h6><span># Bids: 6</span></h6><h6><span>Min Bid: $150.00</span></h6><h6><span>Final Price: $369.00</span></h6><h6><span>Estimate: $300 - $500</span></h6></div></div></div></li>
21+<li class=""><div class="lot "><div class="lotInner"><h5><span id='LotNumber'>1002</span><span id="LotName"><a href='http://auctions.morphyauctions.com/UNSOLD-LOT662425.aspx'>UNSOLD PERFUME</a></span></h5><div class="lotData"><h6><span># Bids: 0</span></h6><h6><span>Min Bid: $100.00</span></h6><h6><span>Estimate: $200 - $500</span></h6></div></div></div></li></ul></div>`;
22+
23+describe('morphy', () => {
24+ runFixtureSuite(connector, it, expect);
25+
26+ it('parses the past-auction list and a catalog page', () => {
27+ const list = parsePastList(LIST);
28+ expect(list).toEqual([
29+ { id: '716', title: 'Online Only Perfume Bottles', dept: 'Fine & Decorative Arts', dateText: 'August 11, 2026', pageUrl: 'https://morphyauctions.com/auctions/past-auctions/online-only-perfume-bottles/' },
30+ { id: '700', title: 'Online Only Firearms & Militaria', dept: 'Firearms', dateText: 'August 04, 2026', pageUrl: '#' },
31+ ]);
32+ const cat = parseCatalog(CATALOG, list[0]!, 1);
33+ expect(cat.totalPages).toBe(19);
34+ expect(cat.lots.length).toBe(2);
35+ expect(cat.lots[0]).toMatchObject({ lotNumber: '1001', finalPrice: 369, minBid: 150, bids: 6, estimateText: '$300 - $500', url: 'https://auctions.morphyauctions.com/CAMEO_GLASS_FLORAL_PERFUME_WITH_ATOMIZER-LOT662424.aspx' });
36+ expect(cat.lots[1]!.finalPrice).toBeNull();
37+ });
38+
39+ it('skips firearms and maps departments to slugs', () => {
40+ expect(morphyCategory('Firearms', 'Online Only Firearms & Militaria', 'WINCHESTER MODEL 1873 RIFLE')).toBeNull();
41+ expect(morphyCategory('Fine & Decorative Arts', 'Online Only Perfume Bottles', 'CAMEO GLASS FLORAL PERFUME WITH ATOMIZER')).toBe('perfume');
42+ expect(morphyCategory('Advertising & General Store, Automobilia & Petroliana', 'Automobilia, Petroliana, & Soda Advertising', 'TEXACO PORCELAIN SIGN')).toBe('advertising');
43+ expect(morphyCategory('Coin-Op & Advertising', 'Coin-Op & Antique Advertising', 'MILLS 5 CENT SLOT MACHINE')).toBe('casino_memorabilia');
44+ expect(morphyCategory('Toys, Dolls & Figural Cast Iron', 'Toys & Trains', 'LIONEL STANDARD GAUGE 400E LOCOMOTIVE')).toBe('model_trains');
45+ });
46+
47+ it('normalises sold lots with the auction date and USD final price', async () => {
48+ const list = parsePastList(LIST);
49+ const out = await connector.normalize({ url: 'x', externalId: 'a', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: parseCatalog(CATALOG, list[0]!, 1) });
50+ expect(out.length).toBe(1);
51+ const s = out[0]!;
52+ if (s.kind !== 'sale') throw new Error('sale');
53+ expect(s).toMatchObject({ price: 369, currency: 'USD', auctionHouse: 'Morphy Auctions', lotNumber: '1001', buyerPremiumIncluded: null });
54+ expect(s.saleDate.toISOString()).toBe('2026-08-11T00:00:00.000Z');
55+ expect(s.attributes.categorySlug).toBe('perfume');
56+ expect(s.attributes.identifiers.morphy_lot).toBe('662424');
57+ });
58+
59+ it('fixture normalises', async () => {
60+ expect((await connector.normalize(loadFixture('morphy', 'catalog-page').raw)).length).toBeGreaterThan(0);
61+ });
62+});
added connectors/api/morphy/index.ts +198 −0
@@ -0,0 +1,198 @@
1+import { z } from 'zod';
2+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import type { NormalizedRecord } from '@rareindex/shared';
4+import { isBundleTitle, safeYear } from '../_auction-lib/categories.js';
5+import { gradeOf, hiddenFields, httpText, lotAttributes, money, morphyCategory, selectedOption } from '../_memorabilia-lib/index.js';
6+import { dateWords, makeSale } from '../../firecrawl/_carlib/index.js';
7+
8+const WP = 'https://morphyauctions.com';
9+const CAT = 'https://auctions.morphyauctions.com';
10+const PARSER_VERSION = '1.0.0';
11+
12+export const AuctionSchema = z.object({ id: z.string(), title: z.string(), dept: z.string().nullable(), dateText: z.string().nullable(), pageUrl: z.string().nullable() });
13+export const LotSchema = z.object({ lotNumber: z.string().nullable(), title: z.string(), url: z.string(), image: z.string().nullable(), finalPrice: z.number().nullable(), minBid: z.number().nullable(), estimateText: z.string().nullable(), bids: z.number().nullable() });
14+export const PayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), totalPages: z.number().nullable(), lots: z.array(LotSchema) });
15+export type Payload = z.infer<typeof PayloadSchema>;
16+
17+/** WordPress past-auction list: department label, title, date and the catalog id. */
18+export function parsePastList(htmlText: string): z.infer<typeof AuctionSchema>[] {
19+ const $ = H.load(htmlText);
20+ const out: z.infer<typeof AuctionSchema>[] = [];
21+ $('article.event').each((_, el) => {
22+ const a = $(el);
23+ const link = a.find('a[href*="catalog.aspx?auctionid="]').attr('href');
24+ const id = link?.match(/auctionid=(\d+)/)?.[1];
25+ if (!id) return;
26+ out.push({ id, title: H.text(a.find('h3')) ?? '', dept: H.text(a.find('.category-title')), dateText: H.text(a.find('.date')), pageUrl: a.find('h3 a').attr('href') ?? null });
27+ });
28+ return out;
29+}
30+
31+export function parseCatalog(htmlText: string, auction: z.infer<typeof AuctionSchema>, page: number): Payload {
32+ const $ = H.load(htmlText);
33+ const lots: z.infer<typeof LotSchema>[] = [];
34+ $('#galleryList li .lot, #galleryList .lot').each((_, el) => {
35+ const l = $(el);
36+ const link = l.find('#LotName a, span[id="LotName"] a').first();
37+ const title = H.text(link);
38+ const href = link.attr('href');
39+ if (!title || !href) return;
40+ const data = H.text(l.find('.lotData')) ?? '';
41+ const fp = data.match(/Final Price:\s*\$([\d,]+(?:\.\d+)?)/i);
42+ const mb = data.match(/Min Bid:\s*\$([\d,]+(?:\.\d+)?)/i);
43+ const bids = data.match(/#\s*Bids:\s*(\d+)/i);
44+ const est = data.match(/Estimate:\s*([^\n]+?)(?:\s*$|\s*#|\s*Min|\s*Final)/i);
45+ lots.push({
46+ lotNumber: H.text(l.find('#LotNumber, span[id="LotNumber"]')),
47+ title,
48+ url: href.replace(/^http:/, 'https:'),
49+ image: (() => {
50+ const src = l.find('img.lotImage').attr('src');
51+ return src ? (src.startsWith('http') ? src : CAT + src) : null;
52+ })(),
53+ finalPrice: fp ? money(`$${fp[1]}`, 'USD')?.amount ?? null : null,
54+ minBid: mb ? money(`$${mb[1]}`, 'USD')?.amount ?? null : null,
55+ estimateText: est ? est[1]!.trim() : null,
56+ bids: bids ? Number(bids[1]) : null,
57+ });
58+ });
59+ const totalPages = Number(htmlText.match(/id\s*=\s*"ofpages">\s*\/\s*(\d+)/)?.[1] ?? '') || null;
60+ return { kind: 'catalog_page', auction, page, totalPages, lots };
61+}
62+
63+/**
64+ * Morphy Auctions: past-auction list on morphyauctions.com (WordPress, /page/N/) → auction catalogs on
65+ * auctions.morphyauctions.com (ASP.NET). Page 1 is a GET; later pages are the page's own form POST
66+ * (page number + "Go" button with the hidden __VIEWSTATE fields) — no login involved.
67+ */
68+export class MorphyConnector extends BaseConnector {
69+ readonly version = '1.0.0';
70+ readonly parserVersion = PARSER_VERSION;
71+ protected override minIntervalMs = 2000;
72+
73+ private stat(ctx: CrawlContext, ok: boolean, ms: number) {
74+ const s = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });
75+ s.attempts++;
76+ if (ok) s.success++;
77+ s.ms += ms;
78+ }
79+
80+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
81+ const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);
82+ const maxPages = Number(this.meta.config.pagesPerAuction ?? 10);
83+ const perPage = String(this.meta.config.lotsPerPage ?? 100);
84+ const backfill = ctx.options.mode === 'backfill';
85+ const listPage = backfill ? Number(ctx.options.cursor?.listPage ?? 1) : 1;
86+ const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);
87+ const listUrl = `${WP}/auctions/past-auctions/${listPage > 1 ? `page/${listPage}/` : ''}`;
88+ await this.throttle();
89+ const list = await ctx.fetch(listUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3 });
90+ if (!list.success || !list.html) {
91+ ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);
92+ return;
93+ }
94+ const auctions = parsePastList(list.html).filter((a) => !done.has(a.id) && morphyCategory(a.dept, a.title, '') !== null);
95+ let processed = 0;
96+ let count = 0;
97+ for (const auction of auctions) {
98+ if (processed >= auctionsPerRun || ctx.signal?.aborted) break;
99+ const url = `${CAT}/catalog.aspx?auctionid=${auction.id}`;
100+ await this.throttle();
101+ const t0 = Date.now();
102+ let res;
103+ try {
104+ res = await httpText(url);
105+ } catch (err) {
106+ this.stat(ctx, false, Date.now() - t0);
107+ ctx.anomaly('page_fetch_failed', `${url}: ${err instanceof Error ? err.message : String(err)}`);
108+ continue;
109+ }
110+ this.stat(ctx, res.status < 400, Date.now() - t0);
111+ if (res.status >= 400) {
112+ ctx.anomaly('page_fetch_failed', `${url}: HTTP ${res.status}`);
113+ continue;
114+ }
115+ let html = res.text;
116+ let payload = parseCatalog(html, auction, 1);
117+ let page = 1;
118+ // Switch to the larger page size through the form (what the page-size dropdown does).
119+ if (perPage !== '25' && payload.lots.length > 0) {
120+ const posted = await this.postPage(url, html, 1, perPage, ctx, 'ctl00$ContentPlaceHolder$LotsPerPageDropDownTop');
121+ if (posted) {
122+ html = posted;
123+ payload = parseCatalog(html, auction, 1);
124+ }
125+ }
126+ while (true) {
127+ if (payload.lots.length === 0) break;
128+ count++;
129+ yield { url: `${url}&page=${page}`, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };
130+ page++;
131+ if (this.reached(ctx, count) || page > maxPages || (payload.totalPages !== null && page > payload.totalPages) || ctx.signal?.aborted) break;
132+ await this.throttle();
133+ const next = await this.postPage(url, html, page, perPage, ctx);
134+ if (!next) break;
135+ html = next;
136+ payload = parseCatalog(html, auction, page);
137+ }
138+ processed++;
139+ done.add(auction.id);
140+ await ctx.setCursor({ doneAuctions: [...done].slice(-400), listPage: backfill && auctions.every((a) => done.has(a.id)) ? listPage + 1 : listPage, updatedAt: new Date().toISOString() });
141+ }
142+ }
143+
144+ /** Jump to `page` with `perPage` lots by posting the catalog form (ASP.NET postback). */
145+ private async postPage(url: string, currentHtml: string, page: number, perPage: string, ctx: CrawlContext, eventTarget?: string): Promise<string | null> {
146+ const hidden = hiddenFields(currentHtml);
147+ const form: Record<string, string> = {};
148+ for (const [k, v] of Object.entries(hidden)) if (k.startsWith('__') || k.startsWith('categoryView')) form[k] = v;
149+ const keep = ['ctl00$ContentPlaceHolder$SortByDDLTop', 'ctl00$ContentPlaceHolder$SortByDDLBot', 'ctl00$ContentPlaceHolder$displayByDropDownTop', 'ctl00$ContentPlaceHolder$displayByDropDownBot', 'ctl00$ContentPlaceHolder$searchByDropDown'];
150+ for (const k of keep) {
151+ const v = selectedOption(currentHtml, k);
152+ if (v !== null) form[k] = v;
153+ }
154+ // The auction dropdown defaults to the live sale, not the one being viewed: post the requested id explicitly.
155+ form['ctl00$ContentPlaceHolder$AuctionDDL'] = url.match(/auctionid=(\d+)/)?.[1] ?? '';
156+ form['ctl00$ContentPlaceHolder$LotsPerPageDropDownTop'] = perPage;
157+ form['ctl00$ContentPlaceHolder$LotsPerPageDropDownBot'] = perPage;
158+ form['ctl00$ContentPlaceHolder$CurrPageTopTB'] = String(page);
159+ form['ctl00$ContentPlaceHolder$CurrPageBotTB'] = String(page);
160+ form['ctl00$ContentPlaceHolder$searchTextBox'] = '';
161+ if (eventTarget) form.__EVENTTARGET = eventTarget;
162+ else form['ctl00$ContentPlaceHolder$PageJumpBtn'] = 'Go';
163+ const t0 = Date.now();
164+ try {
165+ const res = await httpText(url, { method: 'POST', form, referer: url });
166+ this.stat(ctx, res.status < 400, Date.now() - t0);
167+ if (res.status >= 400) return null;
168+ const cur = Number(res.text.match(/value="(\d+)" id="CurrPageTopTB"/)?.[1] ?? '0');
169+ if (cur !== page) {
170+ ctx.anomaly('pagination_mismatch', `${url}: asked page ${page}, got ${cur}`);
171+ return null;
172+ }
173+ return res.text;
174+ } catch (err) {
175+ this.stat(ctx, false, Date.now() - t0);
176+ ctx.anomaly('page_fetch_failed', `${url} (postback): ${err instanceof Error ? err.message : String(err)}`);
177+ return null;
178+ }
179+ }
180+
181+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
182+ const p = PayloadSchema.parse(raw.payload);
183+ const saleDate = dateWords(p.auction.dateText);
184+ const out: NormalizedRecord[] = [];
185+ if (!saleDate) return out;
186+ for (const lot of p.lots) {
187+ if (lot.finalPrice === null || lot.finalPrice <= 0) continue;
188+ const slug = morphyCategory(p.auction.dept, p.auction.title, lot.title);
189+ if (!slug) continue;
190+ const g = gradeOf(lot.title);
191+ const attributes = lotAttributes({ categorySlug: slug, name: lot.title, year: safeYear(lot.title), identifiers: { morphy_lot: lot.url.match(/LOT(\d+)\.aspx/i)?.[1] ?? `${p.auction.id}-${lot.lotNumber}` }, metadata: { auction_id: p.auction.id, auction_title: p.auction.title, department: p.auction.dept, estimate: lot.estimateText, min_bid: lot.minBid, bids: lot.bids } });
192+ out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.id}-${lot.lotNumber ?? lot.url}`, rawTitle: lot.title, attributes, price: lot.finalPrice, currency: 'USD', saleDate, buyerPremiumIncluded: null, auctionHouse: 'Morphy Auctions', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundleTitle(lot.title) }));
193+ }
194+ return out;
195+ }
196+}
197+
198+export default (meta: ConnectorMeta) => new MorphyConnector(meta);
added connectors/api/morphy/meta.json +34 −0
@@ -0,0 +1,34 @@
1+{
2+ "id": "morphy",
3+ "displayName": "Morphy Auctions (prices realized)",
4+ "sourceId": "morphy",
5+ "sourceName": "Morphy Auctions",
6+ "sourceType": "auction_house",
7+ "sourceUrl": "https://morphyauctions.com",
8+ "module": "api/morphy",
9+ "enginePriority": ["api"],
10+ "categories": ["vintage_toys", "advertising", "vending_machines", "arcade_pinball", "casino_memorabilia", "dolls", "model_trains", "coins", "banknotes", "perfume", "glass_crystal", "porcelain", "silver", "clocks", "antiques", "art", "jewelry", "other_watches", "automotive_memorabilia", "sports_memorabilia"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": false,
15+ "supportsSold": true,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": false,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.9,
24+ "attributionRequired": true,
25+ "termsUrl": "https://morphyauctions.com/bidding/terms-and-conditions/",
26+ "accessNotes": "Two public hosts, plain HTTPS, 0 credits, 2 s politeness. morphyauctions.com/auctions/past-auctions/ (WordPress; robots.txt allows) lists every past sale with department, title and date; auctions.morphyauctions.com/catalog.aspx?auctionid=N shows each lot with 'Final Price: $X', min bid, estimate and bid count. Paging on the ASP.NET catalog is the page's own form POST (page number + Go, 100 lots per page) with its hidden __VIEWSTATE fields — no account or bidding endpoint is used. Firearms & Militaria departments and weapon lots are skipped. Morphy does not state on the catalog whether 'Final Price' includes the buyer's premium → buyer_premium_included=null. Sale date = auction date from the list (first day of multi-day sales).",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "auctionsPerRun": 2,
31+ "pagesPerAuction": 10,
32+ "lotsPerPage": 100
33+ }
34+}
added connectors/api/noble-knight/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1+import path from 'node:path';
2+import { fileURLToPath } from 'node:url';
3+import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4+
5+const dir = path.dirname(fileURLToPath(import.meta.url));
6+if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'product-page', 4);
7+else await runSmoke(dir, Number(process.argv[2] ?? 3));
added connectors/api/noble-knight/index.test.ts +45 −0
@@ -0,0 +1,45 @@
1+import { describe, expect, it } from 'vitest';
2+import { readFileSync } from 'node:fs';
3+import path from 'node:path';
4+import { fileURLToPath } from 'node:url';
5+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6+import { localMeta } from '../_lib/local-meta.js';
7+import createConnector, { nkCategory, parseProduct } from './index.js';
8+
9+const dir = path.dirname(fileURLToPath(import.meta.url));
10+const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
11+const connector = createConnector(meta);
12+
13+const PAGE = `<html><head><script type="application/ld+json">{"@context":"https://schema.org/","@type":"Product","name":"Catan (5th Edition)","image":["https://image.nobleknight.com/g/jpg240/catan.jpg"],"description":"Catan - Catan Studio","sku":"2147354979","mpn":"CN3071","brand":{"@type":"Brand","name":"Catan Studio"},"offers":{"@type":"Offer","url":"https://www.nobleknight.com/P/2147354979/Catan","priceCurrency":"USD","price":"44.95","itemCondition":"https://schema.org/NewCondition","availability":"https://schema.org/InStock"}}</script></head>
14+<body><div class="info-line"><div class="label"><span>Publisher</span></div><div class="value"><a href="/publisher/catan-studio">Catan Studio</a></div></div>
15+<div class="info-line"><div class="label"><span>Product Line</span></div><div class="value"><a href="/Products/Catan">Catan</a></div></div>
16+<div class="info-line"><div class="label"><span>Category</span></div><div class="value"><a href="/main-category/boardgames">Board Games</a></div></div>
17+<div class="conditions">Condition: SW (MINT/New)</div></body></html>`;
18+
19+describe('noble-knight', () => {
20+ runFixtureSuite(connector, it, expect);
21+
22+ it('parses product JSON-LD and info lines', () => {
23+ const p = parseProduct(PAGE, 'https://www.nobleknight.com/P/2147354979/Catan')!;
24+ expect(p).toMatchObject({ nkId: '2147354979', name: 'Catan (5th Edition)', mpn: 'CN3071', brand: 'Catan Studio', price: 44.95, currency: 'USD', itemCondition: 'NewCondition', availability: 'InStock', publisher: 'Catan Studio', productLine: 'Catan', category: 'Board Games' });
25+ expect(nkCategory(p)).toBe('board_games');
26+ expect(nkCategory({ ...p, category: 'Miniatures', productLine: 'Warhammer 40,000', publisher: 'Games Workshop' })).toBe('warhammer');
27+ expect(nkCategory({ ...p, category: 'Role Playing Games', productLine: 'D&D' })).toBeNull();
28+ });
29+
30+ it('normalises to a catalog item + listing with a boxed_toys condition', async () => {
31+ const p = parseProduct(PAGE, 'https://www.nobleknight.com/P/2147354979/Catan')!;
32+ const out = await connector.normalize({ url: p.url, externalId: p.nkId, kind: 'listing', engine: 'api', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: { kind: 'product_page', product: p } });
33+ expect(out.map((r) => r.kind)).toEqual(['catalog_item', 'listing']);
34+ const l = out[1]!;
35+ if (l.kind !== 'listing') throw new Error('listing');
36+ expect(l).toMatchObject({ price: 44.95, currency: 'USD', availability: 'available' });
37+ expect(l.condition.condition).toBe('mint_in_box');
38+ expect(l.attributes.identifiers).toMatchObject({ nobleknight_id: '2147354979', mpn: 'CN3071' });
39+ });
40+
41+ it('fixtures normalise', async () => {
42+ const out = await connector.normalize(loadFixture('noble-knight', 'board-game').raw);
43+ expect(out.length).toBe(2);
44+ });
45+});
added connectors/api/noble-knight/index.ts +154 −0
@@ -0,0 +1,154 @@
1+import { z } from 'zod';
2+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import type { NormalizedRecord } from '@rareindex/shared';
4+import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js';
5+
6+const BASE = 'https://www.nobleknight.com';
7+const PARSER_VERSION = '1.0.0';
8+
9+export const ProductSchema = z.object({
10+ url: z.string(),
11+ nkId: z.string(),
12+ name: z.string(),
13+ sku: z.string().nullable(),
14+ mpn: z.string().nullable(),
15+ brand: z.string().nullable(),
16+ image: z.string().nullable(),
17+ description: z.string().nullable(),
18+ price: z.number().nullable(),
19+ currency: z.string().nullable(),
20+ itemCondition: z.string().nullable(),
21+ availability: z.string().nullable(),
22+ publisher: z.string().nullable(),
23+ productLine: z.string().nullable(),
24+ category: z.string().nullable(),
25+ genre: z.string().nullable(),
26+ type: z.string().nullable(),
27+ conditionText: z.string().nullable(),
28+});
29+export type Product = z.infer<typeof ProductSchema>;
30+export const PayloadSchema = z.object({ kind: z.literal('product_page'), product: ProductSchema });
31+
32+/** Product page: schema.org Product JSON-LD + the info lines (Publisher / Product Line / Category / Genre / Type). */
33+export function parseProduct(htmlText: string, url: string): Product | null {
34+ const ld = H.jsonLd(htmlText, 'Product')[0];
35+ if (!ld) return null;
36+ const $ = H.load(htmlText);
37+ const offers = (Array.isArray(ld.offers) ? ld.offers[0] : ld.offers) as Record<string, unknown> | undefined;
38+ const info: Record<string, string> = {};
39+ $('.info-line').each((_, el) => {
40+ const label = H.text($(el).find('.label'));
41+ const value = H.text($(el).find('.value'));
42+ if (label && value) info[label.toLowerCase()] = value;
43+ });
44+ const nkId = url.match(/\/P\/(\d+)/)?.[1] ?? String(ld.sku ?? '');
45+ const conditionText = H.text($('.conditions').first()) ?? H.text($('.condition, .item-condition').first());
46+ const brand = ld.brand && typeof ld.brand === 'object' ? String((ld.brand as { name?: string }).name ?? '') : ld.brand ? String(ld.brand) : null;
47+ return {
48+ url,
49+ nkId,
50+ name: String(ld.name ?? ''),
51+ sku: ld.sku ? String(ld.sku) : null,
52+ mpn: ld.mpn ? String(ld.mpn) : null,
53+ brand: brand || null,
54+ image: Array.isArray(ld.image) ? (ld.image[0] as string | undefined) ?? null : ld.image ? String(ld.image) : null,
55+ description: ld.description ? String(ld.description).slice(0, 500) : null,
56+ price: offers?.price !== undefined && offers.price !== null ? Number(offers.price) : null,
57+ currency: offers?.priceCurrency ? String(offers.priceCurrency) : null,
58+ itemCondition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null,
59+ availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null,
60+ publisher: info.publisher ?? null,
61+ productLine: info['product line'] ?? null,
62+ category: info.category ?? null,
63+ genre: info.genre ?? null,
64+ type: info.type ?? null,
65+ conditionText: conditionText ?? null,
66+ };
67+}
68+
69+/** Taxonomy slug from Noble Knight's own labels; null → skip (RPG books, CCG singles, supplies…). */
70+export function nkCategory(p: Product): string | null {
71+ const cat = (p.category ?? '').toLowerCase();
72+ const line = `${p.productLine ?? ''} ${p.publisher ?? ''} ${p.name}`.toLowerCase();
73+ if (/games workshop|warhammer|citadel|forge world|age of sigmar|40k|necromunda|blood bowl/.test(line)) return 'warhammer';
74+ if (/board ?game|war ?game|puzzle/.test(cat)) return 'board_games';
75+ if (/miniature/.test(cat)) return /\b(gundam|gunpla|bandai)\b/.test(line) ? 'gundam' : null;
76+ if (/toys?, movies|action figure|toys/.test(cat)) {
77+ if (/\b(gundam|gunpla)\b/.test(line)) return 'gundam';
78+ if (/\bfunko\b|\bpop!\b/.test(line)) return 'funko';
79+ if (/\b(figure|figuarts|nendoroid|hot toys|neca|mcfarlane|hasbro|mattel)\b/.test(line)) return 'action_figures';
80+ return null;
81+ }
82+ return null;
83+}
84+
85+/**
86+ * Noble Knight Games (largest US used/new board-game & miniatures dealer): products are enumerated
87+ * from the public sitemaps (robots.txt allows /P/ product pages and sitemaps; category listings are
88+ * client-rendered and disallowed) and read from each page's schema.org Product data.
89+ */
90+export class NobleKnightConnector extends BaseConnector {
91+ readonly version = '1.0.0';
92+ readonly parserVersion = PARSER_VERSION;
93+ protected override minIntervalMs = 1500;
94+ override readonly urlPatterns = [/^https?:\/\/(www\.)?nobleknight\.com\/P\/\d+/i];
95+
96+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
97+ const sitemaps = Number(this.meta.config.sitemapCount ?? 5);
98+ const perRun = Number(this.meta.config.productsPerRun ?? 200);
99+ const smIndex = Number(ctx.options.cursor?.sitemapIndex ?? 1);
100+ const offset = Number(ctx.options.cursor?.offset ?? 0);
101+ const smUrl = `${BASE}/sitemapproducts${smIndex}.xml`;
102+ await this.throttle();
103+ const sm = await ctx.fetch(smUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 90_000 });
104+ if (!sm.success || !sm.html) {
105+ ctx.anomaly('page_fetch_failed', `${smUrl}: ${sm.error ?? sm.httpStatus}`);
106+ return;
107+ }
108+ const locs = [...sm.html.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]!.trim()).filter((u) => /\/P\/\d+/.test(u));
109+ const slice = locs.slice(offset, offset + perRun);
110+ let count = 0;
111+ for (const url of slice) {
112+ if (ctx.signal?.aborted || this.reached(ctx, count)) break;
113+ if (!(await ctx.shouldFetch(url))) continue;
114+ await this.throttle();
115+ const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => (r.html ? (() => { const p = parseProduct(r.html!, url); return p ? { title: p.name, price: p.price, identifiers: { sku: p.sku } } : null; })() : null) });
116+ if (!res.success || !res.html) {
117+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
118+ continue;
119+ }
120+ const product = parseProduct(res.html, url);
121+ if (!product) continue;
122+ count++;
123+ yield { url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt };
124+ }
125+ const nextOffset = offset + slice.length;
126+ const exhausted = nextOffset >= locs.length;
127+ await ctx.setCursor({ sitemapIndex: exhausted ? (smIndex % sitemaps) + 1 : smIndex, offset: exhausted ? 0 : nextOffset, updatedAt: new Date().toISOString() });
128+ }
129+
130+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
131+ const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text' });
132+ if (!res.success || !res.html) return [];
133+ const product = parseProduct(res.html, url);
134+ return product ? [{ url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt }] : [];
135+ }
136+
137+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
138+ const { product: p } = PayloadSchema.parse(raw.payload);
139+ const slug = nkCategory(p);
140+ if (!slug) return [];
141+ const condRaw = p.conditionText ?? (p.itemCondition === 'NewCondition' ? 'New' : p.itemCondition === 'UsedCondition' ? 'Used' : null);
142+ const cond = dealerCondition(condRaw);
143+ const attributes = lotAttributes({ categorySlug: slug, name: p.name, brand: p.publisher ?? p.brand, series: p.productLine, identifiers: { nobleknight_id: p.nkId, ...(p.mpn ? { mpn: p.mpn } : {}) }, metadata: { category: p.category, genre: p.genre, type: p.type, sku: p.sku } });
144+ const common = { meta: this.meta, sourceUrl: p.url, externalId: p.nkId, rawTitle: p.name, attributes, imageUrls: p.image ? [p.image] : [], description: p.description, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness };
145+ const out: NormalizedRecord[] = [makeCatalogItem({ ...common, confidence: 0.8 })];
146+ if (p.price !== null && Number.isFinite(p.price) && p.price > 0) {
147+ const cur = (p.currency ?? 'USD') as 'USD';
148+ out.push(makeListing({ ...common, price: p.price, currency: cur, listingType: 'fixed_price', seller: 'Noble Knight Games', location: 'US', availability: /InStock|PreOrder|LimitedAvailability/.test(p.availability ?? '') ? 'available' : 'sold', quantity: 1 }));
149+ }
150+ return out;
151+ }
152+}
153+
154+export default (meta: ConnectorMeta) => new NobleKnightConnector(meta);
added connectors/api/noble-knight/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "noble-knight",
3+ "displayName": "Noble Knight Games (board games & miniatures dealer)",
4+ "sourceId": "noble-knight",
5+ "sourceName": "Noble Knight Games",
6+ "sourceType": "dealer",
7+ "sourceUrl": "https://www.nobleknight.com",
8+ "module": "api/noble-knight",
9+ "enginePriority": ["api"],
10+ "categories": ["board_games", "warhammer", "gundam", "action_figures", "funko"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": true,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": true,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 1440,
22+ "priority": "low",
23+ "trustScore": 0.7,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.nobleknight.com/Terms",
26+ "accessNotes": "Plain HTTPS. robots.txt disallows the client-rendered category/search listings (/Catalog, /category-search, /AdvancedSearch…) but allows product pages (/P/<id>/<slug>) and the sitemaps (sitemapproducts1-5.xml ≈ 120k products), so products are enumerated from the sitemaps with a rotating cursor and read from each page's schema.org Product JSON-LD (name, sku, mpn, brand, price, itemCondition, availability) plus the Publisher / Product Line / Category / Genre lines. Only board games, Warhammer/Games Workshop miniatures, Gunpla and figure products are kept (RPG books, CCG singles and supplies are skipped). Condition mapped to the boxed_toys scale. 1.5 s politeness, 0 credits, 200 products per run.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "sitemapCount": 5,
31+ "productsPerRun": 200
32+ }
33+}
added connectors/api/scp-auctions/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1+import path from 'node:path';
2+import { fileURLToPath } from 'node:url';
3+import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4+
5+const dir = path.dirname(fileURLToPath(import.meta.url));
6+if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'catalog-page', 4);
7+else await runSmoke(dir, Number(process.argv[2] ?? 2));
added connectors/api/scp-auctions/index.test.ts +63 −0
@@ -0,0 +1,63 @@
1+import { describe, expect, it } from 'vitest';
2+import { readFileSync } from 'node:fs';
3+import path from 'node:path';
4+import { fileURLToPath } from 'node:url';
5+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6+import { localMeta } from '../_lib/local-meta.js';
7+import { bidsquareDate, parseBidsquareCatalog, parseBidsquareEvents, sportsCategory } from '../_memorabilia-lib/index.js';
8+import createConnector from './index.js';
9+
10+const dir = path.dirname(fileURLToPath(import.meta.url));
11+const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
12+const connector = createConnector(meta);
13+
14+const EVENTS = `<ul><li id="st-23246"><div class="row justify-content-center gtm-visible_event" data-event_id='23246' data-event_status='past' data-event_name='2026 Summer Premier'>
15+<div class="list-top"><label>Timed Auction</label><h1><a href="https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246">2026 Summer Premier</a></h1></div></div></li></ul>`;
16+
17+const CATALOG = `<script type="application/ld+json">{"@context":"https://schema.org","@type":"Event","name":"2026 Summer Premier","url":"https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246","startDate":"2026-07-09 13:00:00 EDT","endDate":"2026-07-26 22:30:00 EDT"}</script>
18+<div class="row filter-items">
19+<div class="col-6 el_23246 gtm-visible_item" id="stl-9310176" data-item_id="9310176" data-event_id='23246' data-event_status='past' data-event_name='2026 Summer Premier'>
20+ <div class="catalog_img"><a href="https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/1947-bond-bread-jackie-robinson-9310176"><img src="https://s1.img.bidsquare.com/item/l/3726/37264383.jpeg" alt="x"/></a></div>
21+ <div class="catalog_detail"><div class="item-list-top"><div class="lot_Num">Lot 47</div><div class="lot_title"><a href="https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/1947-bond-bread-jackie-robinson-9310176">1947 Bond Bread Portrait Jackie Robinson Rookie – SGC FR 1.5</a></div></div>
22+ <div class="item-list-bottom"><div class="bidding_sec"><div class="bidPrice"><div class="bidLength bid_num"><span class="num">32 Bids</span></div><div class="bid_txt"><span>Sold for</span><div class="currency_container"><span class="price ">$14,400</span></div></div></div></div></div></div>
23+</div>
24+<div class="col-6 el_23246 gtm-visible_item" id="stl-9310177" data-item_id="9310177" data-event_id='23246' data-event_status='past' data-event_name='2026 Summer Premier'>
25+ <div class="catalog_detail"><div class="item-list-top"><div class="lot_Num">Lot 48</div><div class="lot_title"><a href="https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/passed-9310177">1986 Fleer Michael Jordan Rookie PSA 8</a></div></div>
26+ <div class="item-list-bottom"><div class="bidding_sec"><div class="bidPrice"><div class="bid_txt"><span>Passed</span></div></div></div></div></div>
27+</div></div>
28+<nav><ul class="pagination"><li data-page="1" class="page-item active"><a>1</a></li><li data-page="2" class="page-item"><a>2</a></li><li data-page="12" class="page-item"><a>12</a></li></ul></nav>`;
29+
30+describe('scp-auctions', () => {
31+ runFixtureSuite(connector, it, expect);
32+
33+ it('parses the event list and catalog cards', () => {
34+ expect(parseBidsquareEvents(EVENTS)).toEqual([{ id: '23246', name: '2026 Summer Premier', url: 'https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246', status: 'past', startDate: null, endDate: null }]);
35+ const cat = parseBidsquareCatalog(CATALOG, 1)!;
36+ expect(cat.event).toMatchObject({ id: '23246', status: 'past', endDate: '2026-07-26 22:30:00 EDT' });
37+ expect(cat.totalPages).toBe(12);
38+ expect(cat.items.length).toBe(2);
39+ expect(cat.items[0]).toMatchObject({ itemId: '9310176', lotNumber: '47', priceLabel: 'Sold for', price: 14400, bids: 32 });
40+ expect(cat.items[1]!.price).toBeNull();
41+ expect(bidsquareDate('2026-07-26 22:30:00 EDT')?.toISOString()).toBe('2026-07-27T02:30:00.000Z');
42+ });
43+
44+ it('normalises sold lots only, dated by the event end, with sport-aware categories', async () => {
45+ const cat = parseBidsquareCatalog(CATALOG, 1)!;
46+ const out = await connector.normalize({ url: 'x', externalId: 'e', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: { kind: 'catalog_page', house: 'SCP Auctions', event: cat.event, page: 1, totalPages: 12, items: cat.items } });
47+ expect(out.length).toBe(1);
48+ const s = out[0]!;
49+ if (s.kind !== 'sale') throw new Error('expected sale');
50+ expect(s).toMatchObject({ price: 14400, currency: 'USD', lotNumber: '47', auctionHouse: 'SCP Auctions', buyerPremiumIncluded: null });
51+ expect(s.saleDate.toISOString()).toBe('2026-07-27T02:30:00.000Z');
52+ expect(s.attributes.categorySlug).toBe('baseball_cards');
53+ expect(s.grade).toMatchObject({ grader: 'sgc', grade: '1.5' });
54+ expect(sportsCategory('1986 Fleer Michael Jordan Rookie PSA 8')).toBe('basketball_cards');
55+ expect(sportsCategory('1927 Babe Ruth Game Used Bat')).toBe('sports_memorabilia');
56+ });
57+
58+ it('fixture sales carry the event end date and USD prices', async () => {
59+ const out = await connector.normalize(loadFixture('scp-auctions', 'catalog-page').raw);
60+ expect(out.length).toBeGreaterThan(0);
61+ for (const r of out) if (r.kind === 'sale') expect(r.currency).toBe('USD');
62+ });
63+});
added connectors/api/scp-auctions/index.ts +165 −0
@@ -0,0 +1,165 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import type { NormalizedRecord } from '@rareindex/shared';
4+import { isBundleTitle, safeYear } from '../_auction-lib/categories.js';
5+import { bidsquareDate, gradeOf, lotAttributes, makeLot, parseBidsquareCatalog, parseBidsquareEvents, sportsCategory, toyGrade, type BidsquareCatalog } from '../_memorabilia-lib/index.js';
6+import { makeSale } from '../../firecrawl/_carlib/index.js';
7+
8+const PARSER_VERSION = '1.0.0';
9+
10+export const CatalogPayloadSchema = z.object({
11+ kind: z.literal('catalog_page'),
12+ house: z.string(),
13+ event: z.object({ id: z.string(), name: z.string(), url: z.string(), status: z.enum(['upcoming', 'live', 'past', 'unknown']), startDate: z.string().nullable(), endDate: z.string().nullable() }),
14+ page: z.number(),
15+ totalPages: z.number().nullable(),
16+ items: z.array(
17+ z.object({
18+ itemId: z.string(),
19+ url: z.string(),
20+ title: z.string(),
21+ lotNumber: z.string().nullable(),
22+ image: z.string().nullable(),
23+ priceLabel: z.string().nullable(),
24+ price: z.number().nullable(),
25+ bids: z.number().nullable(),
26+ estimateLow: z.number().nullable(),
27+ estimateHigh: z.number().nullable(),
28+ startsAt: z.number().nullable(),
29+ endsAt: z.number().nullable(),
30+ }),
31+ ),
32+});
33+export type CatalogPayload = z.infer<typeof CatalogPayloadSchema>;
34+
35+export interface BidsquareHouseOptions {
36+ base: string;
37+ house: string;
38+ /** map a lot title to a taxonomy slug */
39+ category: (title: string) => string | null;
40+ /** buyer's premium included in the displayed "Sold for" price? null when the house does not state it */
41+ buyerPremiumIncluded: boolean | null;
42+ idKey: string;
43+ location: string | null;
44+}
45+
46+/**
47+ * Bidsquare-hosted auction catalogs (SCP Auctions, Hake's). Public pages only:
48+ * /auctions (upcoming), /auctions/past?page=N, /auctions/<house>/<slug>-<id>/catalog?page=N
49+ * Past catalogs show "Sold for $X" per lot → sales dated by the event end (schema.org Event JSON-LD);
50+ * upcoming catalogs show current/starting bid + estimate → auction lots for the calendar.
51+ */
52+export class BidsquareHouseConnector extends BaseConnector {
53+ readonly version = '1.0.0';
54+ readonly parserVersion = PARSER_VERSION;
55+ protected override minIntervalMs = 2000;
56+
57+ constructor(meta: ConnectorMeta, protected readonly opts: BidsquareHouseOptions) {
58+ super(meta);
59+ }
60+
61+ protected async fetchHtml(ctx: CrawlContext, url: string): Promise<{ html: string | null; engine: 'api' | 'feed' | 'firecrawl' | 'scrapfly' | 'browser' | 'manual'; status: number | null; fetchedAt: Date }> {
62+ await this.throttle();
63+ const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 45_000 });
64+ if (!res.success || !res.html) {
65+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
66+ return { html: null, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt };
67+ }
68+ return { html: res.html, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt };
69+ }
70+
71+ /** Events to visit this run: past (sold) first, then upcoming (lots). Subclasses may override. */
72+ protected async listEvents(ctx: CrawlContext): Promise<Array<{ id: string; name: string; url: string; status: string }>> {
73+ const backfill = ctx.options.mode === 'backfill';
74+ const page = backfill ? Number(ctx.options.cursor?.pastPage ?? 1) : 1;
75+ const out: Array<{ id: string; name: string; url: string; status: string }> = [];
76+ const past = await this.fetchHtml(ctx, `${this.opts.base}/auctions/past${page > 1 ? `?page=${page}` : ''}`);
77+ if (past.html) out.push(...parseBidsquareEvents(past.html).map((e) => ({ ...e, status: 'past' })));
78+ if (this.meta.config.includeUpcoming !== false) {
79+ const up = await this.fetchHtml(ctx, `${this.opts.base}/auctions`);
80+ if (up.html) out.push(...parseBidsquareEvents(up.html).filter((e) => e.status !== 'past').map((e) => ({ ...e, status: e.status === 'past' ? 'past' : 'upcoming' })));
81+ }
82+ return out;
83+ }
84+
85+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
86+ const eventsPerRun = Number(this.meta.config.eventsPerRun ?? 2);
87+ const maxPages = Number(this.meta.config.pagesPerEvent ?? 8);
88+ const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneEvents) ? (ctx.options.cursor!.doneEvents as string[]) : []);
89+ const events = await this.listEvents(ctx);
90+ let processed = 0;
91+ let count = 0;
92+ for (const ev of events) {
93+ if (ctx.signal?.aborted || processed >= eventsPerRun) break;
94+ if (ev.status === 'past' && done.has(ev.id)) continue;
95+ const catalogBase = `${ev.url.replace(/\/$/, '')}/catalog`;
96+ let total: number | null = null;
97+ for (let page = 1; page <= maxPages; page++) {
98+ if (ctx.signal?.aborted || this.reached(ctx, count)) break;
99+ const url = page === 1 ? catalogBase : `${catalogBase}?page=${page}`;
100+ const r = await this.fetchHtml(ctx, url);
101+ if (!r.html) break;
102+ const cat = parseBidsquareCatalog(r.html, page);
103+ if (!cat || cat.items.length === 0) break;
104+ total = cat.totalPages ?? total;
105+ const payload: CatalogPayload = { kind: 'catalog_page', house: this.opts.house, event: { ...cat.event, name: cat.event.name || ev.name, url: cat.event.url || ev.url }, page, totalPages: total, items: cat.items };
106+ count++;
107+ yield { url, externalId: `event:${cat.event.id}:page:${page}`, kind: cat.event.status === 'past' ? 'sale' : 'auction_lot', engine: r.engine, httpStatus: r.status, payload, fetchedAt: r.fetchedAt };
108+ if (total !== null && page >= total) break;
109+ }
110+ processed++;
111+ if (ev.status === 'past') done.add(ev.id);
112+ }
113+ const pastPage = Number(ctx.options.cursor?.pastPage ?? 1);
114+ await ctx.setCursor({ doneEvents: [...done].slice(-500), pastPage: ctx.options.mode === 'backfill' && events.filter((e) => e.status === 'past').every((e) => done.has(e.id)) ? pastPage + 1 : pastPage, updatedAt: new Date().toISOString() });
115+ }
116+
117+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
118+ const p = CatalogPayloadSchema.parse(raw.payload);
119+ const out: NormalizedRecord[] = [];
120+ const endDate = bidsquareDate(p.event.endDate);
121+ const startDate = bidsquareDate(p.event.startDate);
122+ for (const it of p.items) {
123+ const slug = this.opts.category(it.title);
124+ if (!slug) continue;
125+ const g = gradeOf(it.title);
126+ const tg = toyGrade(it.title);
127+ const attributes = lotAttributes({
128+ categorySlug: slug,
129+ name: it.title,
130+ year: safeYear(it.title),
131+ identifiers: { [this.opts.idKey]: it.itemId },
132+ metadata: { event_id: p.event.id, event_name: p.event.name, estimate_low: it.estimateLow, estimate_high: it.estimateHigh, bids: it.bids, ...(tg ? { toy_grade: `${tg.company} ${tg.grade}` } : {}) },
133+ });
134+ const common = { meta: this.meta, sourceUrl: it.url, externalId: it.itemId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade };
135+ const sold = p.event.status === 'past' && /sold/i.test(it.priceLabel ?? '') && it.price !== null && it.price > 0;
136+ if (sold) {
137+ const saleDate = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate;
138+ if (!saleDate) continue;
139+ out.push(makeSale({ ...common, price: it.price!, currency: 'USD', saleDate, buyerPremiumIncluded: this.opts.buyerPremiumIncluded, auctionHouse: this.opts.house, lotNumber: it.lotNumber, location: this.opts.location, isBundle: isBundleTitle(it.title) }));
140+ } else if (p.event.status !== 'past') {
141+ const endsAt = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate;
142+ const status = startDate && startDate.getTime() > Date.now() ? 'upcoming' : 'live';
143+ out.push(makeLot({ ...common, auctionHouse: this.opts.house, auctionName: p.event.name, lotNumber: it.lotNumber, startsAt: startDate, endsAt, estimateLow: it.estimateLow, estimateHigh: it.estimateHigh, currentBid: /bid/i.test(it.priceLabel ?? '') ? it.price : null, currency: 'USD', status, location: this.opts.location, confidence: 0.85 }));
144+ }
145+ }
146+ return out;
147+ }
148+}
149+
150+export function scpCategory(title: string): string {
151+ return sportsCategory(title);
152+}
153+
154+export default (meta: ConnectorMeta) =>
155+ new BidsquareHouseConnector(meta, {
156+ base: 'https://catalogs.scpauctions.com',
157+ house: 'SCP Auctions',
158+ category: scpCategory,
159+ // SCP does not state on the catalog page whether "Sold for" includes the buyer's premium.
160+ buyerPremiumIncluded: null,
161+ idKey: 'scp_item_id',
162+ location: 'US',
163+ });
164+
165+export type { BidsquareCatalog };
added connectors/api/scp-auctions/meta.json +34 −0
@@ -0,0 +1,34 @@
1+{
2+ "id": "scp-auctions",
3+ "displayName": "SCP Auctions (results & catalogs)",
4+ "sourceId": "scp-auctions",
5+ "sourceName": "SCP Auctions",
6+ "sourceType": "auction_house",
7+ "sourceUrl": "https://catalogs.scpauctions.com",
8+ "module": "api/scp-auctions",
9+ "enginePriority": ["api"],
10+ "categories": ["sports_memorabilia", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "olympic_collectibles", "pokemon"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": false,
15+ "supportsSold": true,
16+ "supportsAuctions": true,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": false,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.9,
24+ "attributionRequired": true,
25+ "termsUrl": "https://catalogs.scpauctions.com/terms",
26+ "accessNotes": "Plain HTTPS on the public Bidsquare-hosted catalog (robots.txt: only /wp-admin/ disallowed, Crawl-delay 10 → 2 s politeness plus small runs). Past auctions are listed at /auctions/past?page=N; each catalog page (/auctions/scp-auctions-inc/<slug>-<id>/catalog?page=N, 48 lots) shows 'Sold for $X' and the bid count; the event start/end come from the page's schema.org Event JSON-LD (sale date = event end, or the lot's own countdown end). Upcoming catalogs yield auction lots (current/starting bid, estimate). SCP does not state on these pages whether 'Sold for' includes the 20% buyer's premium → buyer_premium_included=null. No login, no bidding endpoints. 0 credits.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "eventsPerRun": 2,
31+ "pagesPerEvent": 8,
32+ "includeUpcoming": true
33+ }
34+}
added connectors/api/trainz/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1+import path from 'node:path';
2+import { fileURLToPath } from 'node:url';
3+import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4+
5+const dir = path.dirname(fileURLToPath(import.meta.url));
6+if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'collection-page', 4);
7+else await runSmoke(dir, Number(process.argv[2] ?? 1));
added connectors/api/trainz/index.test.ts +46 −0
@@ -0,0 +1,46 @@
1+import { describe, expect, it } from 'vitest';
2+import { readFileSync } from 'node:fs';
3+import path from 'node:path';
4+import { fileURLToPath } from 'node:url';
5+import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6+import { localMeta } from '../_lib/local-meta.js';
7+import createConnector, { splitTitle, trimProduct } from './index.js';
8+
9+const dir = path.dirname(fileURLToPath(import.meta.url));
10+const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
11+const connector = createConnector(meta);
12+
13+describe('trainz', () => {
14+ runFixtureSuite(connector, it, expect);
15+
16+ it('splits Trainz condition suffixes and trims products', () => {
17+ expect(splitTitle('Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive LN/Box')).toEqual({ name: 'Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive', conditionCode: 'LN/Box' });
18+ expect(splitTitle('S. Soho & Co. HO BRASS Passenger Coach Car- Unpainted EX/Box').conditionCode).toBe('EX/Box');
19+ expect(splitTitle('Marklin 3000 HO Steam Locomotive').conditionCode).toBeNull();
20+ const p = trimProduct({ id: 1, title: 'X LN/Box', handle: 'x', vendor: 'Lionel', product_type: 'Trains', tags: ['condition:Like New', 'foo', 'scale:O Gauge'], variants: [{ id: 2, sku: 'A', price: '10.00', available: true }], images: [] });
21+ expect(p?.tags).toEqual(['condition:Like New', 'scale:O Gauge']);
22+ });
23+
24+ it('emits a catalog item and a dealer listing per product with a normalised condition', async () => {
25+ const out = await connector.normalize({
26+ url: 'https://www.trainz.com/collections/lionel-o-postwar-trains/products.json?limit=250&page=1',
27+ externalId: 'lionel-o-postwar-trains:1',
28+ kind: 'listing',
29+ engine: 'api',
30+ fetchedAt: new Date('2026-09-07T00:00:00Z'),
31+ payload: { kind: 'collection_page', collection: 'lionel-o-postwar-trains', page: 1, products: [{ id: 99, title: 'Lionel 2343 O Gauge Santa Fe F3 AA Diesel Locomotive Set EX/Box', handle: 'lionel-2343', vendor: 'Lionel', product_type: 'Trains', tags: ['condition:Excellent', 'era:Postwar', 'scale:O Gauge'], created_at: '2026-09-01T00:00:00-04:00', updated_at: null, variants: [{ id: 1, sku: 'P123', price: '499.99', compare_at_price: null, available: true }], images: [{ src: 'https://cdn.shopify.com/x.jpg' }] }] },
32+ });
33+ expect(out.map((r) => r.kind)).toEqual(['catalog_item', 'listing']);
34+ const l = out[1]!;
35+ if (l.kind !== 'listing') throw new Error('listing');
36+ expect(l).toMatchObject({ price: 499.99, currency: 'USD', availability: 'available', seller: 'Trainz' });
37+ expect(l.attributes).toMatchObject({ categorySlug: 'model_trains', brand: 'Lionel', series: 'O Gauge', name: 'Lionel 2343 O Gauge Santa Fe F3 AA Diesel Locomotive Set' });
38+ expect(l.condition).toMatchObject({ condition: 'boxed', conditionRaw: 'Excellent' });
39+ expect(l.attributes.identifiers.trainz_sku).toBe('P123');
40+ });
41+
42+ it('fixture products all carry a vendor and a USD price', async () => {
43+ const out = await connector.normalize(loadFixture('trainz', 'collection-page').raw);
44+ expect(out.filter((r) => r.kind === 'listing').length).toBeGreaterThan(0);
45+ });
46+});
added connectors/api/trainz/index.ts +108 −0
@@ -0,0 +1,108 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import type { NormalizedRecord } from '@rareindex/shared';
4+import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js';
5+
6+const BASE = 'https://www.trainz.com';
7+const PARSER_VERSION = '1.0.0';
8+
9+export const ProductSchema = z.object({
10+ id: z.number(),
11+ title: z.string(),
12+ handle: z.string(),
13+ vendor: z.string().nullable().default(null),
14+ product_type: z.string().nullable().default(null),
15+ tags: z.array(z.string()).default([]),
16+ created_at: z.string().nullable().default(null),
17+ updated_at: z.string().nullable().default(null),
18+ variants: z.array(z.object({ id: z.number(), sku: z.string().nullable().default(null), price: z.string(), compare_at_price: z.string().nullable().default(null), available: z.boolean().default(true) })).default([]),
19+ images: z.array(z.object({ src: z.string() })).default([]),
20+});
21+export type Product = z.infer<typeof ProductSchema>;
22+export const PayloadSchema = z.object({ kind: z.literal('collection_page'), collection: z.string(), page: z.number(), products: z.array(ProductSchema) });
23+export type Payload = z.infer<typeof PayloadSchema>;
24+
25+/** Keep only the fields we use so raw payloads stay small. */
26+export function trimProduct(p: Record<string, unknown>): Product | null {
27+ const parsed = ProductSchema.safeParse(p);
28+ if (!parsed.success) return null;
29+ const v = parsed.data;
30+ return { ...v, variants: v.variants.slice(0, 3), images: v.images.slice(0, 2), tags: v.tags.filter((t) => /^(condition|class|era|scale|gauge|Inventory Type2|roadname|road_name|manufacturer)[:_]/i.test(t) || /^in-stock$|^sold-out$/.test(t)).slice(0, 12) };
31+}
32+
33+function tag(p: Product, prefix: string): string | null {
34+ const t = p.tags.find((x) => x.toLowerCase().startsWith(`${prefix.toLowerCase()}:`));
35+ return t ? t.slice(prefix.length + 1).trim() : null;
36+}
37+
38+/** "Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive LN/Box" → name without the trailing condition code. */
39+export function splitTitle(title: string): { name: string; conditionCode: string | null } {
40+ const m = title.match(/\s+(LN|EX|VG|GD|G|PR|FR|NM|MT|C-?\d{1,2})(?:\/(Box|OB|Sealed|No Box))?\s*$/i);
41+ if (!m) return { name: title.trim(), conditionCode: null };
42+ return { name: title.slice(0, m.index).trim(), conditionCode: m[0].trim() };
43+}
44+
45+const CODE_WORDS: Record<string, string> = { LN: 'Like New', EX: 'Excellent', VG: 'Very Good', GD: 'Good', G: 'Good', PR: 'Poor', FR: 'Fair', NM: 'Near Mint', MT: 'Mint' };
46+
47+/**
48+ * Trainz.com (world's largest model-train dealer): public Shopify product feed per collection
49+ * (/collections/<handle>/products.json). Catalog facts + dealer asking price with graded condition.
50+ */
51+export class TrainzConnector extends BaseConnector {
52+ readonly version = '1.0.0';
53+ readonly parserVersion = PARSER_VERSION;
54+ protected override minIntervalMs = 1500;
55+
56+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
57+ const seeds = (this.meta.config.collections as string[] | undefined) ?? ['lionel-postwar-trains', 'american-flyer-postwar-trains'];
58+ const pagesPerCollection = Number(this.meta.config.pagesPerCollection ?? 2);
59+ const startIdx = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length;
60+ const perRun = Number(this.meta.config.collectionsPerRun ?? 4);
61+ let count = 0;
62+ for (let k = 0; k < Math.min(perRun, seeds.length); k++) {
63+ const handle = seeds[(startIdx + k) % seeds.length]!;
64+ for (let page = 1; page <= pagesPerCollection; page++) {
65+ if (ctx.signal?.aborted || this.reached(ctx, count)) break;
66+ const url = `${BASE}/collections/${handle}/products.json?limit=250&page=${page}`;
67+ await this.throttle();
68+ const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price'], parse: (r) => ({ title: (r.json as { products?: Array<{ title: string }> })?.products?.[0]?.title ?? null, price: (r.json as { products?: Array<{ variants?: Array<{ price: string }> }> })?.products?.[0]?.variants?.[0]?.price ?? null }) });
69+ const list = (res.json as { products?: Record<string, unknown>[] } | null)?.products;
70+ if (!res.success || !Array.isArray(list)) {
71+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
72+ break;
73+ }
74+ const products = list.map(trimProduct).filter((p): p is Product => Boolean(p));
75+ if (products.length === 0) break;
76+ count++;
77+ yield { url, externalId: `${handle}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'collection_page', collection: handle, page, products } satisfies Payload, fetchedAt: res.fetchedAt };
78+ if (products.length < 250) break;
79+ }
80+ }
81+ await ctx.setCursor({ seedIndex: (startIdx + perRun) % seeds.length, updatedAt: new Date().toISOString() });
82+ }
83+
84+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
85+ const p = PayloadSchema.parse(raw.payload);
86+ const out: NormalizedRecord[] = [];
87+ for (const pr of p.products) {
88+ const v = pr.variants[0];
89+ if (!v) continue;
90+ const { name, conditionCode } = splitTitle(pr.title);
91+ const condTag = tag(pr, 'condition');
92+ const condRaw = condTag ?? (conditionCode ? CODE_WORDS[conditionCode.split('/')[0]!.toUpperCase()] ?? conditionCode : null);
93+ const cond = dealerCondition(condRaw ?? '');
94+ const scale = tag(pr, 'scale') ?? tag(pr, 'gauge') ?? name.match(/\b(HO|N|O|S|G|Z|TT|O27|Standard)\s+(?:Scale|Gauge)\b/i)?.[0] ?? null;
95+ const url = `${BASE}/products/${pr.handle}`;
96+ const attributes = lotAttributes({ categorySlug: 'model_trains', name, brand: pr.vendor, series: scale, identifiers: { trainz_sku: v.sku ?? String(pr.id), shopify_product_id: String(pr.id) }, metadata: { product_type: pr.product_type, era: tag(pr, 'era'), class: tag(pr, 'class'), tags: pr.tags } });
97+ const price = Number(v.price);
98+ const common = { meta: this.meta, sourceUrl: url, externalId: String(pr.id), rawTitle: pr.title, attributes, imageUrls: pr.images.map((i) => i.src), observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness };
99+ out.push(makeCatalogItem({ ...common, confidence: 0.8 }));
100+ if (Number.isFinite(price) && price > 0) {
101+ out.push(makeListing({ ...common, price, currency: 'USD', listingType: 'fixed_price', seller: 'Trainz', location: 'US', availability: v.available ? 'available' : 'sold', listedAt: pr.created_at ? new Date(pr.created_at) : null, quantity: 1 }));
102+ }
103+ }
104+ return out;
105+ }
106+}
107+
108+export default (meta: ConnectorMeta) => new TrainzConnector(meta);
added connectors/api/trainz/meta.json +61 −0
Large diff (61 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/bbts/_smoke.ts +7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/bbts/index.test.ts +66 −0
Large diff (66 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/bbts/index.ts +109 −0
Large diff (109 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/bbts/meta.json +34 −0
Large diff (34 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/lelands/_smoke.ts +7 −0
Large diff (7 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/lelands/index.test.ts +53 −0
Large diff (53 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/lelands/index.ts +109 −0
Large diff (109 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/lelands/meta.json +49 −0
Large diff (49 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/bbts/search-page.json +69 −0
Large diff (70 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/hakes/catalog-page.json +89 −0
Large diff (90 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/lelands/gallery-page.json +75 −0
Large diff (76 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/morphy/catalog-page.json +71 −0
Large diff (72 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/noble-knight/board-game.json +41 −0
Large diff (42 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/noble-knight/gunpla.json +41 −0
Large diff (42 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/scp-auctions/catalog-page.json +89 −0
Large diff (90 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/trainz/collection-page.json +230 −0
Large diff (232 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere