SPB Git forge
spboucher.ai ↗

spb/rareindex

Public
54commits 1branches 0releases
7.1 MBsize
maindefault branch
10 days agolast push
TypeScript 61.9% HTML 37.2% SQL 0.7%

Connectors: SCP, Hake's, Morphy, Lelands, BBTS, Noble Knight, Trainz (agent U)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 17 days ago (Sep 7, 2026) parent f5091c0

38 changed files +2,702 −0

added connectors/api/_memorabilia-lib/_capture-nk.ts +16 −0
@@ -0,0 +1,16 @@
1 +import { readFileSync } from 'node:fs';
2 +import { ConnectorMetaSchema, createCrawlContext, createRouter } from '@rareindex/connectors';
3 +import { saveFixture } from '@rareindex/connectors/testing';
4 +import { childLogger } from '@rareindex/shared';
5 +import create from '../noble-knight/index.js';
6 +const meta = ConnectorMetaSchema.parse(JSON.parse(readFileSync('connectors/api/noble-knight/meta.json', 'utf8')));
7 +const c = create(meta);
8 +const ctx = createCrawlContext({ router: createRouter({}), meta, options: { mode: 'probe' }, log: childLogger({}) });
9 +for (const [name, url] of [['board-game', 'https://www.nobleknight.com/P/2147993326/007---Spectre-Board-Game'], ['gunpla', 'https://www.nobleknight.com/P/2148351787/002-Ex-S-GUNDAM']] as const) {
10 + const raws = await c.lookup!(url, ctx);
11 + const raw = raws[0]!;
12 + const fetchedAt = raw.fetchedAt ?? new Date();
13 + const records = await c.normalize({ ...raw, externalId: raw.externalId ?? null, fetchedAt });
14 + saveFixture(meta.id, name, { raw: { url: raw.url, externalId: raw.externalId ?? null, kind: raw.kind, engine: raw.engine, fetchedAt, payload: raw.payload }, expect: { count: records.length, kinds: [...new Set(records.map((r) => r.kind))] }, note: `Captured live via lookup(${url}).` });
15 + console.log(name, records.length, records.map((r) => r.kind));
16 +}
added connectors/api/_memorabilia-lib/index.ts +391 −0
@@ -0,0 +1,391 @@
1 +import { html as H, type ConnectorMeta } from '@rareindex/connectors';
2 +import { parseGradeFromTitle } from '@rareindex/taxonomy';
3 +import type { AssetAttributes, CurrencyCode, NormalizedAuctionLot, NormalizedCatalogItem, NormalizedListing } from '@rareindex/shared';
4 +import { lotAttributes, money } from '../../firecrawl/_carlib/index.js';
5 +
6 +/**
7 + * Helpers shared by the memorabilia / toys / dealer connectors (SCP, Hake's, Morphy, Lelands, BBTS,
8 + * Noble Knight, Trainz). Source vocabulary stays here; canonical models stay clean.
9 + */
10 +
11 +export const BOT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) RareIndexBot/0.1 (+https://www.rareindex.io/about; market-data research)';
12 +
13 +// ---------------------------------------------------------------------------------------------
14 +// Plain HTTP helpers for sites whose paging/selection is an ASP.NET postback (what a browser does:
15 +// a form POST carrying the page's own hidden fields; no authentication involved).
16 +// ---------------------------------------------------------------------------------------------
17 +export interface HttpResult {
18 + status: number;
19 + text: string;
20 + cookies: string[];
21 +}
22 +
23 +export async function httpText(url: string, opts: { method?: 'GET' | 'POST'; form?: Record<string, string>; cookie?: string; referer?: string; timeoutMs?: number } = {}): Promise<HttpResult> {
24 + const ctrl = new AbortController();
25 + const timer = setTimeout(() => ctrl.abort(), opts.timeoutMs ?? 45_000);
26 + try {
27 + const headers: Record<string, string> = { 'user-agent': BOT_UA, accept: 'text/html,application/xhtml+xml,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.9' };
28 + if (opts.cookie) headers.cookie = opts.cookie;
29 + if (opts.referer) headers.referer = opts.referer;
30 + let body: string | undefined;
31 + if (opts.method === 'POST') {
32 + headers['content-type'] = 'application/x-www-form-urlencoded';
33 + body = new URLSearchParams(opts.form ?? {}).toString();
34 + }
35 + const res = await fetch(url, { method: opts.method ?? 'GET', headers, body, signal: ctrl.signal, redirect: 'follow' });
36 + const text = await res.text();
37 + const cookies: string[] = [];
38 + const anyHeaders = res.headers as Headers & { getSetCookie?: () => string[] };
39 + if (typeof anyHeaders.getSetCookie === 'function') cookies.push(...anyHeaders.getSetCookie());
40 + else {
41 + const sc = res.headers.get('set-cookie');
42 + if (sc) cookies.push(sc);
43 + }
44 + return { status: res.status, text, cookies };
45 + } finally {
46 + clearTimeout(timer);
47 + }
48 +}
49 +
50 +/** Cookie header from Set-Cookie lines (name=value only). */
51 +export function cookieHeader(setCookies: string[], previous = ''): string {
52 + const jar = new Map<string, string>();
53 + for (const part of previous.split(';')) {
54 + const [k, ...v] = part.trim().split('=');
55 + if (k) jar.set(k, v.join('='));
56 + }
57 + for (const sc of setCookies) {
58 + const first = sc.split(';')[0] ?? '';
59 + const [k, ...v] = first.trim().split('=');
60 + if (k) jar.set(k, v.join('='));
61 + }
62 + return [...jar.entries()].map(([k, v]) => `${k}=${v}`).join('; ');
63 +}
64 +
65 +/** ASP.NET hidden inputs (__VIEWSTATE, __EVENTVALIDATION, …) plus any other hidden fields. */
66 +export function hiddenFields(htmlText: string): Record<string, string> {
67 + const out: Record<string, string> = {};
68 + const re = /<input[^>]+type="hidden"[^>]*>/gi;
69 + let m: RegExpExecArray | null;
70 + while ((m = re.exec(htmlText))) {
71 + const tag = m[0];
72 + const name = tag.match(/name="([^"]+)"/)?.[1];
73 + if (!name) continue;
74 + const value = tag.match(/value="([^"]*)"/)?.[1] ?? '';
75 + out[name] = value.replace(/&quot;/g, '"').replace(/&amp;/g, '&');
76 + }
77 + return out;
78 +}
79 +
80 +/** Selected value of a <select name="…"> (first option when none is marked). */
81 +export function selectedOption(htmlText: string, name: string): string | null {
82 + const m = htmlText.match(new RegExp(`<select[^>]*name="${name.replace(/[$]/g, '\\$')}"[^>]*>([\\s\\S]*?)</select>`, 'i'));
83 + if (!m) return null;
84 + const sel = m[1]!.match(/<option[^>]*selected(?:="selected")?[^>]*value="([^"]*)"/i) ?? m[1]!.match(/<option[^>]*value="([^"]*)"[^>]*selected/i);
85 + if (sel) return sel[1]!;
86 + return m[1]!.match(/<option[^>]*value="([^"]*)"/i)?.[1] ?? null;
87 +}
88 +
89 +export function selectOptions(htmlText: string, name: string): Array<{ value: string; label: string }> {
90 + const m = htmlText.match(new RegExp(`<select[^>]*name="${name.replace(/[$]/g, '\\$')}"[^>]*>([\\s\\S]*?)</select>`, 'i'));
91 + if (!m) return [];
92 + const out: Array<{ value: string; label: string }> = [];
93 + const re = /<option[^>]*value="([^"]*)"[^>]*>([^<]*)</gi;
94 + let o: RegExpExecArray | null;
95 + while ((o = re.exec(m[1]!))) out.push({ value: o[1]!, label: o[2]!.replace(/&amp;/g, '&').trim() });
96 + return out;
97 +}
98 +
99 +// ---------------------------------------------------------------------------------------------
100 +// Bidsquare-hosted catalog sites (SCP Auctions, Hake's) share one HTML structure.
101 +// ---------------------------------------------------------------------------------------------
102 +export interface BidsquareEvent {
103 + id: string;
104 + name: string;
105 + url: string;
106 + status: 'upcoming' | 'live' | 'past' | 'unknown';
107 + startDate: string | null;
108 + endDate: string | null;
109 +}
110 +export interface BidsquareItem {
111 + itemId: string;
112 + url: string;
113 + title: string;
114 + lotNumber: string | null;
115 + image: string | null;
116 + /** "Sold for" | "Current Bid" | "Starting Bid" | "Passed" | null */
117 + priceLabel: string | null;
118 + price: number | null;
119 + bids: number | null;
120 + estimateLow: number | null;
121 + estimateHigh: number | null;
122 + /** unix seconds from the countdown script (start, end) when present */
123 + startsAt: number | null;
124 + endsAt: number | null;
125 +}
126 +export interface BidsquareCatalog {
127 + event: BidsquareEvent;
128 + page: number;
129 + totalPages: number | null;
130 + items: BidsquareItem[];
131 +}
132 +
133 +export function parseBidsquareEvents(htmlText: string): BidsquareEvent[] {
134 + const $ = H.load(htmlText);
135 + const out: BidsquareEvent[] = [];
136 + $('.gtm-visible_event').each((_, el) => {
137 + const e = $(el);
138 + const id = e.attr('data-event_id');
139 + const name = e.attr('data-event_name') ?? H.text(e.find('h1')) ?? '';
140 + const href = e.find('h1 a').attr('href') ?? e.find('a[href*="/auctions/"]').first().attr('href');
141 + if (!id || !href) return;
142 + const st = e.attr('data-event_status') ?? 'unknown';
143 + out.push({ id, name, url: href.split('#')[0]!, status: st === 'past' || st === 'upcoming' || st === 'live' ? st : 'unknown', startDate: null, endDate: null });
144 + });
145 + return out;
146 +}
147 +
148 +export function parseBidsquareCatalog(htmlText: string, page: number): BidsquareCatalog | null {
149 + const $ = H.load(htmlText);
150 + let event: BidsquareEvent | null = null;
151 + for (const ld of H.jsonLd(htmlText, 'Event')) {
152 + const url = String(ld.url ?? '');
153 + const id = url.match(/-(\d+)$/)?.[1] ?? null;
154 + if (!id) continue;
155 + event = { id, name: String(ld.name ?? ''), url, status: 'unknown', startDate: ld.startDate ? String(ld.startDate) : null, endDate: ld.endDate ? String(ld.endDate) : null };
156 + break;
157 + }
158 + const first = $('[data-item_id][data-event_id]').first();
159 + if (!event) {
160 + const id = first.attr('data-event_id');
161 + if (!id) return null;
162 + event = { id, name: first.attr('data-event_name') ?? '', url: '', status: 'unknown', startDate: null, endDate: null };
163 + }
164 + const st = first.attr('data-event_status');
165 + if (st === 'past' || st === 'upcoming' || st === 'live') event.status = st;
166 + const items: BidsquareItem[] = [];
167 + const seen = new Set<string>();
168 + $('div[id^="stl-"][data-item_id]').each((_, el) => {
169 + const c = $(el);
170 + const itemId = c.attr('data-item_id')!;
171 + if (seen.has(itemId)) return;
172 + seen.add(itemId);
173 + const link = c.find('.lot_title a').first();
174 + const url = link.attr('href') ?? c.find('.catalog_img a').attr('href') ?? '';
175 + const title = H.text(link) ?? c.find('.catalog_img img').attr('alt') ?? '';
176 + if (!url || !title) return;
177 + const lotNumber = H.text(c.find('.lot_Num'))?.replace(/^Lot\s*/i, '') ?? null;
178 + const image = c.find('.catalog_img img').attr('src') ?? null;
179 + const label = H.text(c.find('.bid_txt span').first());
180 + const priceText = H.text(c.find('.bid_txt .price').first());
181 + const bidsText = H.text(c.find('.bidLength .num').first());
182 + const est = c.find('.estimate_amount').attr('data-exchange');
183 + let estimateLow: number | null = null;
184 + let estimateHigh: number | null = null;
185 + if (est) {
186 + try {
187 + const j = JSON.parse(est.replace(/&quot;/g, '"')) as { low_est?: string; high_est?: string };
188 + estimateLow = j.low_est ? Number(j.low_est) : null;
189 + estimateHigh = j.high_est ? Number(j.high_est) : null;
190 + } catch {
191 + /* ignore */
192 + }
193 + }
194 + const script = c.find('script').text();
195 + const cd = script.match(/countDownTimer\((\d+),\s*(\d+)/);
196 + items.push({
197 + itemId,
198 + url,
199 + title,
200 + lotNumber,
201 + image,
202 + priceLabel: label ?? null,
203 + price: money(priceText, 'USD')?.amount ?? null,
204 + bids: bidsText ? Number(bidsText.replace(/[^\d]/g, '')) || null : null,
205 + estimateLow,
206 + estimateHigh,
207 + startsAt: cd ? Number(cd[1]) : null,
208 + endsAt: cd ? Number(cd[2]) : null,
209 + });
210 + });
211 + const pages = $('ul.pagination li[data-page]')
212 + .map((_, li) => Number($(li).attr('data-page')))
213 + .get()
214 + .filter((n) => Number.isFinite(n) && n > 0);
215 + return { event, page, totalPages: pages.length ? Math.max(...pages) : null, items };
216 +}
217 +
218 +/** "2026-07-26 22:30:00 EDT" → Date (Bidsquare prints US Eastern local time). */
219 +export function bidsquareDate(s: string | null | undefined): Date | null {
220 + if (!s) return null;
221 + const m = s.match(/(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})\s*(EDT|EST|PDT|PST|CDT|CST|MDT|MST|UTC)?/);
222 + if (!m) return null;
223 + const offsets: Record<string, number> = { EDT: 4, EST: 5, CDT: 5, CST: 6, MDT: 6, MST: 7, PDT: 7, PST: 8, UTC: 0 };
224 + const off = offsets[m[7] ?? 'EDT'] ?? 4;
225 + return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]) + off, Number(m[5]), Number(m[6])));
226 +}
227 +
228 +// ---------------------------------------------------------------------------------------------
229 +// Category mappers (keyword based; null/family fallbacks, never invented attributes)
230 +// ---------------------------------------------------------------------------------------------
231 +const SPORT_WORDS: Array<[RegExp, string]> = [
232 + [/\b(baseball|topps|bowman|goudey|t206|bond bread|play ball|leaf 1948|mlb|world series|yankees|red sox|dodgers|jackie robinson|babe ruth|mickey mantle|lou gehrig|ty cobb|honus wagner)\b/i, 'baseball'],
233 + [/\b(basketball|nba|fleer basketball|fleer|hoops|jordan rookie|michael jordan|lebron|kobe|lakers|celtics)\b/i, 'basketball'],
234 + [/\b(football|nfl|super bowl|quarterback|packers|cowboys|steelers)\b/i, 'football'],
235 + [/\b(hockey|nhl|stanley cup|o-pee-chee|opc|canadiens|bruins|maple leafs)\b/i, 'hockey'],
236 + [/\b(soccer|fifa|world cup|premier league|messi|ronaldo|pel[eé])\b/i, 'soccer'],
237 +];
238 +const CARD_WORDS = /\b(card|rookie|psa|sgc|bgs|beckett|cgc|wax pack|unopened|refractor|parallel|#\s?\d+|1\/1)\b/i;
239 +const MEMORABILIA_WORDS = /\b(jersey|game[- ]used|game[- ]worn|bat\b|glove|helmet|ball\b|puck|ring\b|trophy|ticket|program|contract|photo|photograph|signed|autograph|cleats|shoes|sneakers|medal|pennant|uniform)\b/i;
240 +
241 +/** Sports houses (SCP, Lelands, Hunt): cards by sport vs memorabilia. */
242 +export function sportsCategory(title: string): string {
243 + const isCard = CARD_WORDS.test(title) && !MEMORABILIA_WORDS.test(title);
244 + if (isCard) {
245 + for (const [re, s] of SPORT_WORDS) if (re.test(title)) return `${s}_cards`;
246 + if (/\b(pok[eé]mon|charizard)\b/i.test(title)) return 'pokemon';
247 + if (/\b(magic: the gathering|mtg|black lotus)\b/i.test(title)) return 'magic_the_gathering';
248 + return 'other_sports_cards';
249 + }
250 + if (/\b(olympic|olympics)\b/i.test(title)) return 'olympic_collectibles';
251 + if (/\b(boxing|ufc|golf|tennis|nascar|wrestling|wwe|wwf|racing)\b/i.test(title)) return 'sports_memorabilia';
252 + return 'sports_memorabilia';
253 +}
254 +
255 +/** Pop-culture houses (Hake's): toys, comics, political, pins… */
256 +export function popCultureCategory(title: string): string {
257 + const t = title;
258 + if (/\bcgc\b|\bcbcs\b|comic book|#\s?\d+\s*\(?(19|20)\d\d|amazing spider-man|detective comics|action comics|x-men|batman|superman|marvel comics|dc comics/i.test(t) && !/\b(afa|figure|playset|vehicle)\b/i.test(t)) {
259 + if (/\b(marvel|spider-man|x-men|avengers|hulk|fantastic four|iron man|captain america|thor)\b/i.test(t)) return 'marvel_comics';
260 + if (/\b(dc|batman|superman|detective comics|action comics|wonder woman|flash|green lantern)\b/i.test(t)) return 'dc_comics';
261 + return 'independent_comics';
262 + }
263 + if (/\b(original art|comic art|cover art|splash page|comic book page|storyboard)\b/i.test(t)) return 'animation_art';
264 + if (/\b(campaign|president|political|button\b.*\b(19|18)\d\d|inaugur|senator|governor|election)\b/i.test(t)) return 'political_memorabilia';
265 + if (/\b(star wars|kenner|luke skywalker|darth vader|boba fett|tusken|stormtrooper|jedi|sith|millennium falcon)\b/i.test(t)) return 'star_wars';
266 + if (/\b(garbage pail kids|gpk|wacky packages|non-sport|trading card set)\b/i.test(t)) return 'non_sport_cards';
267 + if (/\b(psa|sgc|bgs)\s*\d|\btopps\b|\bbowman\b|rookie card/i.test(t)) return sportsCategory(t);
268 + if (/\b(pinback|pin-back|pin back|\bpin\b|badge)\b/i.test(t)) return 'pins';
269 + if (/\b(disney|mickey mouse|donald duck|disneyland|walt disney)\b/i.test(t)) return 'disney_collectibles';
270 + if (/\b(lunch box|lunchbox|thermos|pez|board game|premium ring|cereal|advertising|sign\b)\b/i.test(t)) return 'advertising';
271 + if (/\b(animation cel|production cel|cel\b)\b/i.test(t)) return 'animation_art';
272 + if (/\b(record|album|lp\b|concert|beatles|elvis|kiss\b)\b/i.test(t)) return 'music_memorabilia';
273 + if (/\b(movie|film|poster|lobby card|one sheet|screen[- ]used|prop\b)\b/i.test(t)) return 'movie_posters';
274 + if (/\b(signed|autograph)\b/i.test(t)) return 'autographs';
275 + if (/\b(g\.?i\.? joe|transformers|he-man|masters of the universe|tmnt|teenage mutant|action figure|afa\b|moc\b|mib\b|playset|hot wheels|matchbox|tin toy|wind-up|battery op|marx|ideal|mego|mattel|hasbro|lionel)\b/i.test(t)) return 'vintage_toys';
276 + if (/\b(jersey|game used|game worn|baseball|football|basketball|hockey|boxing)\b/i.test(t)) return 'sports_memorabilia';
277 + return 'vintage_toys';
278 +}
279 +
280 +/** Morphy: department label + auction/lot titles → slug; firearms lots are excluded upstream. */
281 +export function morphyCategory(dept: string | null, auctionTitle: string, lotTitle: string): string | null {
282 + const d = (dept ?? '').toLowerCase();
283 + const a = auctionTitle.toLowerCase();
284 + const t = lotTitle;
285 + if (/firearm|weapon|militaria|edged/.test(d) || /firearm|militaria/.test(a) || /\b(rifle|pistol|revolver|shotgun|carbine|ammunition|cartridge|bayonet)\b/i.test(t)) return null;
286 + if (/coin-op|coin op|arcade|slot machine|jukebox|vending/.test(d + a)) return /\b(slot machine|trade stimulator|casino)\b/i.test(t) ? 'casino_memorabilia' : /\b(jukebox|arcade|pinball)\b/i.test(t) ? 'arcade_pinball' : /\b(sign|tin|display|calendar|poster)\b/i.test(t) ? 'advertising' : 'vending_machines';
287 + if (/\bcoins?\b|currency|banknote|numismat/.test(d + a)) return /\b(note|bill|currency|dollar bill)\b/i.test(t) ? 'banknotes' : 'coins';
288 + if (/perfume|fragrance/.test(d + a)) return 'perfume';
289 + if (/petroliana|automobilia|gas|oil|soda|advertising/.test(d + a)) return /\b(sign|clock|thermometer|display|calendar|poster|globe)\b/i.test(t) || /advertising|petroliana|soda/.test(a) ? 'advertising' : 'automotive_memorabilia';
290 + if (/train/.test(d + a) || /\b(lionel|american flyer|marklin|märklin|ives|standard gauge|o gauge|ho scale)\b/i.test(t)) return 'model_trains';
291 + if (/doll/.test(d + a) && /\b(doll|bisque|barbie|steiff|teddy|bear)\b/i.test(t)) return 'dolls';
292 + if (/toy/.test(d + a)) return /\b(bank|cast iron)\b/i.test(t) ? 'vintage_toys' : 'vintage_toys';
293 + if (/sport/.test(d + a)) return sportsCategory(t);
294 + if (/jewel|watch/.test(d + a)) return /\b(watch|wristwatch|pocket watch)\b/i.test(t) ? 'other_watches' : 'jewelry';
295 + if (/fine|decorative|lamp|glass|art/.test(d + a)) {
296 + if (/\b(lamp|tiffany|handel|pairpoint)\b/i.test(t)) return 'antiques';
297 + if (/\b(perfume|atomizer|scent bottle)\b/i.test(t)) return 'perfume';
298 + if (/\b(vase|glass|paperweight|cameo glass|loetz|galle|gallé|daum|steuben|lalique|murano)\b/i.test(t)) return 'glass_crystal';
299 + if (/\b(porcelain|meissen|sevres|s[eè]vres|royal doulton|figurine|pottery|ceramic)\b/i.test(t)) return 'porcelain';
300 + if (/\b(sterling|silver)\b/i.test(t)) return 'silver';
301 + if (/\b(clock|regulator)\b/i.test(t)) return 'clocks';
302 + if (/\b(oil on canvas|painting|bronze|sculpture|lithograph|print)\b/i.test(t)) return 'art';
303 + return 'antiques';
304 + }
305 + if (/comic|pop culture|movie|music|entertainment/.test(d + a)) return popCultureCategory(t);
306 + return null;
307 +}
308 +
309 +// ---------------------------------------------------------------------------------------------
310 +// Builders for non-sale records
311 +// ---------------------------------------------------------------------------------------------
312 +export interface BaseInput {
313 + meta: ConnectorMeta;
314 + sourceUrl: string;
315 + externalId: string;
316 + rawTitle: string;
317 + attributes: AssetAttributes;
318 + imageUrls?: string[];
319 + description?: string | null;
320 + observedAt: Date;
321 + parserVersion: string;
322 + confidence?: number;
323 + grader?: string | null;
324 + grade?: string | null;
325 + condition?: string | null;
326 + conditionRaw?: string | null;
327 + completeness?: string | null;
328 +}
329 +
330 +function base(i: BaseInput) {
331 + return {
332 + connectorId: i.meta.id,
333 + sourceId: i.meta.sourceId,
334 + sourceUrl: i.sourceUrl,
335 + externalId: i.externalId,
336 + rawTitle: i.rawTitle,
337 + description: i.description ?? null,
338 + imageUrls: i.imageUrls ?? [],
339 + attributes: i.attributes,
340 + grade: { grader: i.grader ?? null, grade: i.grade ?? null, qualifier: null, certificationNumber: null },
341 + condition: { condition: i.condition ?? null, conditionRaw: i.conditionRaw ?? null, completeness: i.completeness ?? null },
342 + observedAt: i.observedAt,
343 + confidence: i.confidence ?? 0.85,
344 + parserVersion: i.parserVersion,
345 + };
346 +}
347 +
348 +export function makeListing(i: BaseInput & { price: number | null; currency: CurrencyCode | null; listingType?: NormalizedListing['listingType']; availability?: NormalizedListing['availability']; seller?: string | null; location?: string | null; quantity?: number | null; listedAt?: Date | null; endsAt?: Date | null; bidCount?: number | null }): NormalizedListing {
349 + return { kind: 'listing', ...base(i), listingType: i.listingType ?? 'fixed_price', price: i.price, currency: i.currency, seller: i.seller ?? null, sellerReputation: null, location: i.location ?? null, shippingCost: null, quantity: i.quantity ?? null, listedAt: i.listedAt ?? null, endsAt: i.endsAt ?? null, availability: i.availability ?? 'available', bidCount: i.bidCount ?? null };
350 +}
351 +
352 +export function makeLot(i: BaseInput & { auctionHouse: string; auctionName?: string | null; lotNumber?: string | null; startsAt?: Date | null; endsAt?: Date | null; estimateLow?: number | null; estimateHigh?: number | null; currentBid?: number | null; currency: CurrencyCode | null; status?: NormalizedAuctionLot['status']; location?: string | null }): NormalizedAuctionLot {
353 + return { kind: 'auction_lot', ...base(i), auctionHouse: i.auctionHouse, auctionName: i.auctionName ?? null, lotNumber: i.lotNumber ?? null, startsAt: i.startsAt ?? null, endsAt: i.endsAt ?? null, estimateLow: i.estimateLow ?? null, estimateHigh: i.estimateHigh ?? null, currentBid: i.currentBid ?? null, currency: i.currency, status: i.status ?? 'unknown', location: i.location ?? null };
354 +}
355 +
356 +export function makeCatalogItem(i: BaseInput & { releaseDate?: Date | null }): NormalizedCatalogItem {
357 + return { kind: 'catalog_item', ...base(i), releaseDate: i.releaseDate ?? null };
358 +}
359 +
360 +/** Grade for sales/listings: only graded companies, never 'raw' as grader. */
361 +export function gradeOf(title: string): { grader: string | null; grade: string | null } {
362 + const g = parseGradeFromTitle(title);
363 + if (!g.grader || g.grader === 'raw') return { grader: null, grade: null };
364 + let grade = g.grade;
365 + if (!grade) {
366 + // Houses often write the label between company and number ("SGC FR 1.5", "PSA VG-EX 4", "PSA NM-MT+ 8.5").
367 + const m = title.match(/\b(PSA|SGC|BGS|CGC|CBCS|TAG)\b[\s:-]*(?:[A-Z]{1,3}(?:[-/][A-Z]{1,3})?\+?\s*)?(\d{1,2}(?:\.\d)?)\b/i);
368 + if (m && Number(m[2]) <= 10) grade = m[2]!;
369 + }
370 + return { grader: g.grader, grade };
371 +}
372 +
373 +/** AFA / CAS / UKG toy grades ("AFA 85", "AFA Qualified 80") kept as metadata, not graders (not in taxonomy). */
374 +export function toyGrade(title: string): { company: string; grade: string } | null {
375 + const m = title.match(/\b(AFA|CAS|UKG)\s*(?:Qualified\s*)?(\d{2,3})/i);
376 + return m ? { company: m[1]!.toUpperCase(), grade: m[2]! } : null;
377 +}
378 +
379 +/** Condition words from dealer labels → 'boxed_toys' scale (board games, trains, figures). */
380 +export function dealerCondition(raw: string | null | undefined): { condition: string | null; completeness: string | null } {
381 + if (!raw) return { condition: null, completeness: null };
382 + const r = raw.toLowerCase();
383 + if (/\b(new|sealed|shrink|sw\b|mint in box|mib|misb|nib|moc)\b/.test(r)) return { condition: 'mint_in_box', completeness: 'sealed' };
384 + if (/\b(like new|ln\b|near mint|nm\b|mint\b)\b/.test(r)) return { condition: 'near_mint_box', completeness: 'boxed' };
385 + if (/\b(excellent|ex\b|very good|vg\b|vg\+|ex\+)\b/.test(r)) return { condition: 'boxed', completeness: 'boxed' };
386 + if (/\b(good|g\b|fair|played|used|restored)\b/.test(r)) return { condition: 'loose_complete', completeness: 'loose' };
387 + if (/\b(poor|damaged|parts|incomplete)\b/.test(r)) return { condition: 'loose', completeness: 'loose' };
388 + return { condition: null, completeness: null };
389 +}
390 +
391 +export { lotAttributes, money };
added connectors/api/hakes/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'catalog-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 2));
added connectors/api/hakes/index.test.ts +51 −0
@@ -0,0 +1,51 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../_lib/local-meta.js';
7 +import { popCultureCategory, toyGrade } from '../_memorabilia-lib/index.js';
8 +import createConnector from './index.js';
9 +
10 +const dir = path.dirname(fileURLToPath(import.meta.url));
11 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
12 +const connector = createConnector(meta);
13 +
14 +describe('hakes', () => {
15 + runFixtureSuite(connector, it, expect);
16 +
17 + it('maps pop-culture titles to taxonomy slugs', () => {
18 + expect(popCultureCategory('STAR WARS (1978) - LUKE SKYWALKER 12 BACK-B AFA 85 Y-NM+')).toBe('star_wars');
19 + expect(popCultureCategory('AMAZING SPIDER-MAN #300 1988 CGC 9.8 NM/MINT')).toBe('marvel_comics');
20 + expect(popCultureCategory('DETECTIVE COMICS #27 1939 CGC 3.0')).toBe('dc_comics');
21 + expect(popCultureCategory('1860 ABRAHAM LINCOLN CAMPAIGN FERROTYPE PIN')).toBe('political_memorabilia');
22 + expect(popCultureCategory('1952 TOPPS #311 MICKEY MANTLE PSA 4')).toBe('baseball_cards');
23 + expect(popCultureCategory('GARBAGE PAIL KIDS 1985 SERIES 1 WAX BOX')).toBe('non_sport_cards');
24 + expect(popCultureCategory('MICKEY MOUSE 1930s LIONEL HANDCAR')).toBe('disney_collectibles');
25 + expect(toyGrade('STAR WARS DROIDS 1985 - A-WING FIGHTER VEHICLE AFA QUALIFIED 60 Q-EX')).toEqual({ company: 'AFA', grade: '60' });
26 + });
27 +
28 + it('upcoming catalogs become auction lots with estimates and end times; sold lots would become sales', async () => {
29 + const fx = loadFixture('hakes', 'catalog-page');
30 + const out = await connector.normalize(fx.raw);
31 + expect(out.length).toBeGreaterThan(0);
32 + const lot = out.find((r) => r.kind === 'auction_lot');
33 + if (lot && lot.kind === 'auction_lot') {
34 + expect(lot.auctionHouse).toBe("Hake's Auctions");
35 + expect(lot.currency).toBe('USD');
36 + expect(lot.endsAt).toBeInstanceOf(Date);
37 + }
38 + // Same payload flagged as past with a sold label → sale
39 + const p = structuredClone(fx.raw.payload) as { event: { status: string }; items: Array<{ priceLabel: string | null; price: number | null }> };
40 + p.event.status = 'past';
41 + p.items[0]!.priceLabel = 'Sold for';
42 + p.items[0]!.price = 1234;
43 + const sold = await connector.normalize({ ...fx.raw, payload: p });
44 + const s = sold.find((r) => r.kind === 'sale');
45 + expect(s).toBeDefined();
46 + if (s && s.kind === 'sale') {
47 + expect(s.price).toBe(1234);
48 + expect(s.buyerPremiumIncluded).toBeNull();
49 + }
50 + });
51 +});
added connectors/api/hakes/index.ts +33 −0
@@ -0,0 +1,33 @@
1 +import type { ConnectorMeta, CrawlContext } from '@rareindex/connectors';
2 +import { parseBidsquareEvents, popCultureCategory } from '../_memorabilia-lib/index.js';
3 +import { BidsquareHouseConnector } from '../scp-auctions/index.js';
4 +
5 +/**
6 + * Hake's Auctions (Bidsquare platform, same HTML as SCP). Hake's hides its past-auction list
7 + * (/auctions/past returns "No Auction Available"), so this connector works in two phases:
8 + * upcoming catalogs are crawled for auction lots and remembered in the cursor; once an event's
9 + * catalog reports status 'past' the same pages yield realized "Sold for" prices.
10 + */
11 +export class HakesConnector extends BidsquareHouseConnector {
12 + protected override async listEvents(ctx: CrawlContext): Promise<Array<{ id: string; name: string; url: string; status: string }>> {
13 + const known = (Array.isArray(ctx.options.cursor?.knownEvents) ? (ctx.options.cursor!.knownEvents as Array<{ id: string; name: string; url: string }>) : []).filter((e) => e && e.id && e.url);
14 + const up = await this.fetchHtml(ctx, `${this.opts.base}/auctions`);
15 + const upcoming = up.html ? parseBidsquareEvents(up.html) : [];
16 + const upcomingIds = new Set(upcoming.map((e) => e.id));
17 + // Events seen before that are no longer upcoming are revisited as past (their catalog reports the status).
18 + const past = known.filter((e) => !upcomingIds.has(e.id)).map((e) => ({ ...e, status: 'past' }));
19 + const merged = [...known.filter((e) => upcomingIds.has(e.id)), ...upcoming.filter((e) => !known.some((k) => k.id === e.id)).map((e) => ({ id: e.id, name: e.name, url: e.url }))];
20 + await ctx.setCursor({ ...(ctx.options.cursor ?? {}), knownEvents: merged.slice(-60) });
21 + return [...past, ...upcoming.map((e) => ({ id: e.id, name: e.name, url: e.url, status: 'upcoming' }))];
22 + }
23 +}
24 +
25 +export default (meta: ConnectorMeta) =>
26 + new HakesConnector(meta, {
27 + base: 'https://www.hakes.com',
28 + house: "Hake's Auctions",
29 + category: popCultureCategory,
30 + buyerPremiumIncluded: null,
31 + idKey: 'hakes_item_id',
32 + location: 'US',
33 + });
added connectors/api/hakes/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "hakes",
3 + "displayName": "Hake's Auctions (pop culture catalogs)",
4 + "sourceId": "hakes",
5 + "sourceName": "Hake's Auctions",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://www.hakes.com",
8 + "module": "api/hakes",
9 + "enginePriority": ["api"],
10 + "categories": ["vintage_toys", "star_wars", "marvel_comics", "dc_comics", "independent_comics", "political_memorabilia", "pins", "non_sport_cards", "disney_collectibles", "advertising", "animation_art", "movie_posters", "music_memorabilia", "sports_memorabilia", "autographs"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": true,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.85,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.hakes.com/terms",
26 + "accessNotes": "Plain HTTPS on the public Bidsquare-hosted site (robots.txt disallows /search, account pages and filtered/sorted query URLs; we only load /auctions and unfiltered catalog pages with ?page=N, 2 s politeness). Hake's hides its past-auction list, so upcoming catalogs are crawled for auction lots (estimate, current bid, countdown end) and their URLs are remembered in the connector cursor; after an event closes the same catalog pages report status 'past' with 'Sold for $X' → sales dated by the lot/event end. AFA/CAS toy grades are kept in metadata (not graders). Whether 'Sold for' includes the buyer's premium is not stated on the page → null. 0 credits.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "eventsPerRun": 2,
31 + "pagesPerEvent": 6,
32 + "includeUpcoming": true
33 + }
34 +}
added connectors/api/morphy/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'catalog-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 2));
added connectors/api/morphy/index.test.ts +62 −0
@@ -0,0 +1,62 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../_lib/local-meta.js';
7 +import { morphyCategory } from '../_memorabilia-lib/index.js';
8 +import createConnector, { parseCatalog, parsePastList } from './index.js';
9 +
10 +const dir = path.dirname(fileURLToPath(import.meta.url));
11 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
12 +const connector = createConnector(meta);
13 +
14 +const LIST = `<div id="auctionfeed"><article class="event auction clearfix catalog-2"><div class="event-content"><span class="category-title"><a href="/auctions/divisions/fine-decorative-art/">Fine &amp; Decorative Arts</a></span>
15 +<h3><a href="https://morphyauctions.com/auctions/past-auctions/online-only-perfume-bottles/">Online Only Perfume Bottles</a></h3><div class="date">August 11, 2026</div>
16 +<ul class="event-grid"><li class="pre-bid"><a href="https://auctions.morphyauctions.com/catalog.aspx?auctionid=716"><span>Online Listing</span></a></li></ul></div></article>
17 +<article class="event auction"><div class="event-content"><span class="category-title"><a href="#">Firearms</a></span><h3><a href="#">Online Only Firearms &amp; Militaria</a></h3><div class="date">August 04, 2026</div><ul><li><a href="https://auctions.morphyauctions.com/catalog.aspx?auctionid=700">x</a></li></ul></div></article></div>`;
18 +
19 +const CATALOG = `<h2 id="ofpages"> / 19</h2><div id="galleryList"><ul><li class=""><div class="lot "><div class="lotInner"><h5><span id='LotLabel'>Lot </span><span class='LotNumberSign'>#</span><span id='LotNumber'>1001</span><span id="LotNumber Colon">: </span><span id="LotName"><a href='http://auctions.morphyauctions.com/CAMEO_GLASS_FLORAL_PERFUME_WITH_ATOMIZER-LOT662424.aspx'>CAMEO GLASS FLORAL PERFUME WITH ATOMIZER</a></span></h5>
20 +<div class="imageDiv"><a href='#'><img class='lotImage' src='/ItemImages/000662/26320016_1_sm.jpeg'></a></div><div class="lotData"><h6><span># Bids: 6</span></h6><h6><span>Min Bid: $150.00</span></h6><h6><span>Final Price: $369.00</span></h6><h6><span>Estimate: $300 - $500</span></h6></div></div></div></li>
21 +<li class=""><div class="lot "><div class="lotInner"><h5><span id='LotNumber'>1002</span><span id="LotName"><a href='http://auctions.morphyauctions.com/UNSOLD-LOT662425.aspx'>UNSOLD PERFUME</a></span></h5><div class="lotData"><h6><span># Bids: 0</span></h6><h6><span>Min Bid: $100.00</span></h6><h6><span>Estimate: $200 - $500</span></h6></div></div></div></li></ul></div>`;
22 +
23 +describe('morphy', () => {
24 + runFixtureSuite(connector, it, expect);
25 +
26 + it('parses the past-auction list and a catalog page', () => {
27 + const list = parsePastList(LIST);
28 + expect(list).toEqual([
29 + { id: '716', title: 'Online Only Perfume Bottles', dept: 'Fine & Decorative Arts', dateText: 'August 11, 2026', pageUrl: 'https://morphyauctions.com/auctions/past-auctions/online-only-perfume-bottles/' },
30 + { id: '700', title: 'Online Only Firearms & Militaria', dept: 'Firearms', dateText: 'August 04, 2026', pageUrl: '#' },
31 + ]);
32 + const cat = parseCatalog(CATALOG, list[0]!, 1);
33 + expect(cat.totalPages).toBe(19);
34 + expect(cat.lots.length).toBe(2);
35 + expect(cat.lots[0]).toMatchObject({ lotNumber: '1001', finalPrice: 369, minBid: 150, bids: 6, estimateText: '$300 - $500', url: 'https://auctions.morphyauctions.com/CAMEO_GLASS_FLORAL_PERFUME_WITH_ATOMIZER-LOT662424.aspx' });
36 + expect(cat.lots[1]!.finalPrice).toBeNull();
37 + });
38 +
39 + it('skips firearms and maps departments to slugs', () => {
40 + expect(morphyCategory('Firearms', 'Online Only Firearms & Militaria', 'WINCHESTER MODEL 1873 RIFLE')).toBeNull();
41 + expect(morphyCategory('Fine & Decorative Arts', 'Online Only Perfume Bottles', 'CAMEO GLASS FLORAL PERFUME WITH ATOMIZER')).toBe('perfume');
42 + expect(morphyCategory('Advertising & General Store, Automobilia & Petroliana', 'Automobilia, Petroliana, & Soda Advertising', 'TEXACO PORCELAIN SIGN')).toBe('advertising');
43 + expect(morphyCategory('Coin-Op & Advertising', 'Coin-Op & Antique Advertising', 'MILLS 5 CENT SLOT MACHINE')).toBe('casino_memorabilia');
44 + expect(morphyCategory('Toys, Dolls & Figural Cast Iron', 'Toys & Trains', 'LIONEL STANDARD GAUGE 400E LOCOMOTIVE')).toBe('model_trains');
45 + });
46 +
47 + it('normalises sold lots with the auction date and USD final price', async () => {
48 + const list = parsePastList(LIST);
49 + const out = await connector.normalize({ url: 'x', externalId: 'a', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: parseCatalog(CATALOG, list[0]!, 1) });
50 + expect(out.length).toBe(1);
51 + const s = out[0]!;
52 + if (s.kind !== 'sale') throw new Error('sale');
53 + expect(s).toMatchObject({ price: 369, currency: 'USD', auctionHouse: 'Morphy Auctions', lotNumber: '1001', buyerPremiumIncluded: null });
54 + expect(s.saleDate.toISOString()).toBe('2026-08-11T00:00:00.000Z');
55 + expect(s.attributes.categorySlug).toBe('perfume');
56 + expect(s.attributes.identifiers.morphy_lot).toBe('662424');
57 + });
58 +
59 + it('fixture normalises', async () => {
60 + expect((await connector.normalize(loadFixture('morphy', 'catalog-page').raw)).length).toBeGreaterThan(0);
61 + });
62 +});
added connectors/api/morphy/index.ts +198 −0
@@ -0,0 +1,198 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { isBundleTitle, safeYear } from '../_auction-lib/categories.js';
5 +import { gradeOf, hiddenFields, httpText, lotAttributes, money, morphyCategory, selectedOption } from '../_memorabilia-lib/index.js';
6 +import { dateWords, makeSale } from '../../firecrawl/_carlib/index.js';
7 +
8 +const WP = 'https://morphyauctions.com';
9 +const CAT = 'https://auctions.morphyauctions.com';
10 +const PARSER_VERSION = '1.0.0';
11 +
12 +export const AuctionSchema = z.object({ id: z.string(), title: z.string(), dept: z.string().nullable(), dateText: z.string().nullable(), pageUrl: z.string().nullable() });
13 +export const LotSchema = z.object({ lotNumber: z.string().nullable(), title: z.string(), url: z.string(), image: z.string().nullable(), finalPrice: z.number().nullable(), minBid: z.number().nullable(), estimateText: z.string().nullable(), bids: z.number().nullable() });
14 +export const PayloadSchema = z.object({ kind: z.literal('catalog_page'), auction: AuctionSchema, page: z.number(), totalPages: z.number().nullable(), lots: z.array(LotSchema) });
15 +export type Payload = z.infer<typeof PayloadSchema>;
16 +
17 +/** WordPress past-auction list: department label, title, date and the catalog id. */
18 +export function parsePastList(htmlText: string): z.infer<typeof AuctionSchema>[] {
19 + const $ = H.load(htmlText);
20 + const out: z.infer<typeof AuctionSchema>[] = [];
21 + $('article.event').each((_, el) => {
22 + const a = $(el);
23 + const link = a.find('a[href*="catalog.aspx?auctionid="]').attr('href');
24 + const id = link?.match(/auctionid=(\d+)/)?.[1];
25 + if (!id) return;
26 + out.push({ id, title: H.text(a.find('h3')) ?? '', dept: H.text(a.find('.category-title')), dateText: H.text(a.find('.date')), pageUrl: a.find('h3 a').attr('href') ?? null });
27 + });
28 + return out;
29 +}
30 +
31 +export function parseCatalog(htmlText: string, auction: z.infer<typeof AuctionSchema>, page: number): Payload {
32 + const $ = H.load(htmlText);
33 + const lots: z.infer<typeof LotSchema>[] = [];
34 + $('#galleryList li .lot, #galleryList .lot').each((_, el) => {
35 + const l = $(el);
36 + const link = l.find('#LotName a, span[id="LotName"] a').first();
37 + const title = H.text(link);
38 + const href = link.attr('href');
39 + if (!title || !href) return;
40 + const data = H.text(l.find('.lotData')) ?? '';
41 + const fp = data.match(/Final Price:\s*\$([\d,]+(?:\.\d+)?)/i);
42 + const mb = data.match(/Min Bid:\s*\$([\d,]+(?:\.\d+)?)/i);
43 + const bids = data.match(/#\s*Bids:\s*(\d+)/i);
44 + const est = data.match(/Estimate:\s*([^\n]+?)(?:\s*$|\s*#|\s*Min|\s*Final)/i);
45 + lots.push({
46 + lotNumber: H.text(l.find('#LotNumber, span[id="LotNumber"]')),
47 + title,
48 + url: href.replace(/^http:/, 'https:'),
49 + image: (() => {
50 + const src = l.find('img.lotImage').attr('src');
51 + return src ? (src.startsWith('http') ? src : CAT + src) : null;
52 + })(),
53 + finalPrice: fp ? money(`$${fp[1]}`, 'USD')?.amount ?? null : null,
54 + minBid: mb ? money(`$${mb[1]}`, 'USD')?.amount ?? null : null,
55 + estimateText: est ? est[1]!.trim() : null,
56 + bids: bids ? Number(bids[1]) : null,
57 + });
58 + });
59 + const totalPages = Number(htmlText.match(/id\s*=\s*"ofpages">\s*\/\s*(\d+)/)?.[1] ?? '') || null;
60 + return { kind: 'catalog_page', auction, page, totalPages, lots };
61 +}
62 +
63 +/**
64 + * Morphy Auctions: past-auction list on morphyauctions.com (WordPress, /page/N/) → auction catalogs on
65 + * auctions.morphyauctions.com (ASP.NET). Page 1 is a GET; later pages are the page's own form POST
66 + * (page number + "Go" button with the hidden __VIEWSTATE fields) — no login involved.
67 + */
68 +export class MorphyConnector extends BaseConnector {
69 + readonly version = '1.0.0';
70 + readonly parserVersion = PARSER_VERSION;
71 + protected override minIntervalMs = 2000;
72 +
73 + private stat(ctx: CrawlContext, ok: boolean, ms: number) {
74 + const s = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });
75 + s.attempts++;
76 + if (ok) s.success++;
77 + s.ms += ms;
78 + }
79 +
80 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
81 + const auctionsPerRun = Number(this.meta.config.auctionsPerRun ?? 2);
82 + const maxPages = Number(this.meta.config.pagesPerAuction ?? 10);
83 + const perPage = String(this.meta.config.lotsPerPage ?? 100);
84 + const backfill = ctx.options.mode === 'backfill';
85 + const listPage = backfill ? Number(ctx.options.cursor?.listPage ?? 1) : 1;
86 + const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);
87 + const listUrl = `${WP}/auctions/past-auctions/${listPage > 1 ? `page/${listPage}/` : ''}`;
88 + await this.throttle();
89 + const list = await ctx.fetch(listUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3 });
90 + if (!list.success || !list.html) {
91 + ctx.anomaly('page_fetch_failed', `${listUrl}: ${list.error ?? list.httpStatus}`);
92 + return;
93 + }
94 + const auctions = parsePastList(list.html).filter((a) => !done.has(a.id) && morphyCategory(a.dept, a.title, '') !== null);
95 + let processed = 0;
96 + let count = 0;
97 + for (const auction of auctions) {
98 + if (processed >= auctionsPerRun || ctx.signal?.aborted) break;
99 + const url = `${CAT}/catalog.aspx?auctionid=${auction.id}`;
100 + await this.throttle();
101 + const t0 = Date.now();
102 + let res;
103 + try {
104 + res = await httpText(url);
105 + } catch (err) {
106 + this.stat(ctx, false, Date.now() - t0);
107 + ctx.anomaly('page_fetch_failed', `${url}: ${err instanceof Error ? err.message : String(err)}`);
108 + continue;
109 + }
110 + this.stat(ctx, res.status < 400, Date.now() - t0);
111 + if (res.status >= 400) {
112 + ctx.anomaly('page_fetch_failed', `${url}: HTTP ${res.status}`);
113 + continue;
114 + }
115 + let html = res.text;
116 + let payload = parseCatalog(html, auction, 1);
117 + let page = 1;
118 + // Switch to the larger page size through the form (what the page-size dropdown does).
119 + if (perPage !== '25' && payload.lots.length > 0) {
120 + const posted = await this.postPage(url, html, 1, perPage, ctx, 'ctl00$ContentPlaceHolder$LotsPerPageDropDownTop');
121 + if (posted) {
122 + html = posted;
123 + payload = parseCatalog(html, auction, 1);
124 + }
125 + }
126 + while (true) {
127 + if (payload.lots.length === 0) break;
128 + count++;
129 + yield { url: `${url}&page=${page}`, externalId: `auction:${auction.id}:page:${page}`, kind: 'sale', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() };
130 + page++;
131 + if (this.reached(ctx, count) || page > maxPages || (payload.totalPages !== null && page > payload.totalPages) || ctx.signal?.aborted) break;
132 + await this.throttle();
133 + const next = await this.postPage(url, html, page, perPage, ctx);
134 + if (!next) break;
135 + html = next;
136 + payload = parseCatalog(html, auction, page);
137 + }
138 + processed++;
139 + done.add(auction.id);
140 + await ctx.setCursor({ doneAuctions: [...done].slice(-400), listPage: backfill && auctions.every((a) => done.has(a.id)) ? listPage + 1 : listPage, updatedAt: new Date().toISOString() });
141 + }
142 + }
143 +
144 + /** Jump to `page` with `perPage` lots by posting the catalog form (ASP.NET postback). */
145 + private async postPage(url: string, currentHtml: string, page: number, perPage: string, ctx: CrawlContext, eventTarget?: string): Promise<string | null> {
146 + const hidden = hiddenFields(currentHtml);
147 + const form: Record<string, string> = {};
148 + for (const [k, v] of Object.entries(hidden)) if (k.startsWith('__') || k.startsWith('categoryView')) form[k] = v;
149 + const keep = ['ctl00$ContentPlaceHolder$SortByDDLTop', 'ctl00$ContentPlaceHolder$SortByDDLBot', 'ctl00$ContentPlaceHolder$displayByDropDownTop', 'ctl00$ContentPlaceHolder$displayByDropDownBot', 'ctl00$ContentPlaceHolder$searchByDropDown'];
150 + for (const k of keep) {
151 + const v = selectedOption(currentHtml, k);
152 + if (v !== null) form[k] = v;
153 + }
154 + // The auction dropdown defaults to the live sale, not the one being viewed: post the requested id explicitly.
155 + form['ctl00$ContentPlaceHolder$AuctionDDL'] = url.match(/auctionid=(\d+)/)?.[1] ?? '';
156 + form['ctl00$ContentPlaceHolder$LotsPerPageDropDownTop'] = perPage;
157 + form['ctl00$ContentPlaceHolder$LotsPerPageDropDownBot'] = perPage;
158 + form['ctl00$ContentPlaceHolder$CurrPageTopTB'] = String(page);
159 + form['ctl00$ContentPlaceHolder$CurrPageBotTB'] = String(page);
160 + form['ctl00$ContentPlaceHolder$searchTextBox'] = '';
161 + if (eventTarget) form.__EVENTTARGET = eventTarget;
162 + else form['ctl00$ContentPlaceHolder$PageJumpBtn'] = 'Go';
163 + const t0 = Date.now();
164 + try {
165 + const res = await httpText(url, { method: 'POST', form, referer: url });
166 + this.stat(ctx, res.status < 400, Date.now() - t0);
167 + if (res.status >= 400) return null;
168 + const cur = Number(res.text.match(/value="(\d+)" id="CurrPageTopTB"/)?.[1] ?? '0');
169 + if (cur !== page) {
170 + ctx.anomaly('pagination_mismatch', `${url}: asked page ${page}, got ${cur}`);
171 + return null;
172 + }
173 + return res.text;
174 + } catch (err) {
175 + this.stat(ctx, false, Date.now() - t0);
176 + ctx.anomaly('page_fetch_failed', `${url} (postback): ${err instanceof Error ? err.message : String(err)}`);
177 + return null;
178 + }
179 + }
180 +
181 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
182 + const p = PayloadSchema.parse(raw.payload);
183 + const saleDate = dateWords(p.auction.dateText);
184 + const out: NormalizedRecord[] = [];
185 + if (!saleDate) return out;
186 + for (const lot of p.lots) {
187 + if (lot.finalPrice === null || lot.finalPrice <= 0) continue;
188 + const slug = morphyCategory(p.auction.dept, p.auction.title, lot.title);
189 + if (!slug) continue;
190 + const g = gradeOf(lot.title);
191 + const attributes = lotAttributes({ categorySlug: slug, name: lot.title, year: safeYear(lot.title), identifiers: { morphy_lot: lot.url.match(/LOT(\d+)\.aspx/i)?.[1] ?? `${p.auction.id}-${lot.lotNumber}` }, metadata: { auction_id: p.auction.id, auction_title: p.auction.title, department: p.auction.dept, estimate: lot.estimateText, min_bid: lot.minBid, bids: lot.bids } });
192 + out.push(makeSale({ meta: this.meta, sourceUrl: lot.url, externalId: `${p.auction.id}-${lot.lotNumber ?? lot.url}`, rawTitle: lot.title, attributes, price: lot.finalPrice, currency: 'USD', saleDate, buyerPremiumIncluded: null, auctionHouse: 'Morphy Auctions', lotNumber: lot.lotNumber, imageUrls: lot.image ? [lot.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundleTitle(lot.title) }));
193 + }
194 + return out;
195 + }
196 +}
197 +
198 +export default (meta: ConnectorMeta) => new MorphyConnector(meta);
added connectors/api/morphy/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "morphy",
3 + "displayName": "Morphy Auctions (prices realized)",
4 + "sourceId": "morphy",
5 + "sourceName": "Morphy Auctions",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://morphyauctions.com",
8 + "module": "api/morphy",
9 + "enginePriority": ["api"],
10 + "categories": ["vintage_toys", "advertising", "vending_machines", "arcade_pinball", "casino_memorabilia", "dolls", "model_trains", "coins", "banknotes", "perfume", "glass_crystal", "porcelain", "silver", "clocks", "antiques", "art", "jewelry", "other_watches", "automotive_memorabilia", "sports_memorabilia"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.9,
24 + "attributionRequired": true,
25 + "termsUrl": "https://morphyauctions.com/bidding/terms-and-conditions/",
26 + "accessNotes": "Two public hosts, plain HTTPS, 0 credits, 2 s politeness. morphyauctions.com/auctions/past-auctions/ (WordPress; robots.txt allows) lists every past sale with department, title and date; auctions.morphyauctions.com/catalog.aspx?auctionid=N shows each lot with 'Final Price: $X', min bid, estimate and bid count. Paging on the ASP.NET catalog is the page's own form POST (page number + Go, 100 lots per page) with its hidden __VIEWSTATE fields — no account or bidding endpoint is used. Firearms & Militaria departments and weapon lots are skipped. Morphy does not state on the catalog whether 'Final Price' includes the buyer's premium → buyer_premium_included=null. Sale date = auction date from the list (first day of multi-day sales).",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "auctionsPerRun": 2,
31 + "pagesPerAuction": 10,
32 + "lotsPerPage": 100
33 + }
34 +}
added connectors/api/noble-knight/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'product-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 3));
added connectors/api/noble-knight/index.test.ts +45 −0
@@ -0,0 +1,45 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../_lib/local-meta.js';
7 +import createConnector, { nkCategory, parseProduct } from './index.js';
8 +
9 +const dir = path.dirname(fileURLToPath(import.meta.url));
10 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
11 +const connector = createConnector(meta);
12 +
13 +const PAGE = `<html><head><script type="application/ld+json">{"@context":"https://schema.org/","@type":"Product","name":"Catan (5th Edition)","image":["https://image.nobleknight.com/g/jpg240/catan.jpg"],"description":"Catan - Catan Studio","sku":"2147354979","mpn":"CN3071","brand":{"@type":"Brand","name":"Catan Studio"},"offers":{"@type":"Offer","url":"https://www.nobleknight.com/P/2147354979/Catan","priceCurrency":"USD","price":"44.95","itemCondition":"https://schema.org/NewCondition","availability":"https://schema.org/InStock"}}</script></head>
14 +<body><div class="info-line"><div class="label"><span>Publisher</span></div><div class="value"><a href="/publisher/catan-studio">Catan Studio</a></div></div>
15 +<div class="info-line"><div class="label"><span>Product Line</span></div><div class="value"><a href="/Products/Catan">Catan</a></div></div>
16 +<div class="info-line"><div class="label"><span>Category</span></div><div class="value"><a href="/main-category/boardgames">Board Games</a></div></div>
17 +<div class="conditions">Condition: SW (MINT/New)</div></body></html>`;
18 +
19 +describe('noble-knight', () => {
20 + runFixtureSuite(connector, it, expect);
21 +
22 + it('parses product JSON-LD and info lines', () => {
23 + const p = parseProduct(PAGE, 'https://www.nobleknight.com/P/2147354979/Catan')!;
24 + expect(p).toMatchObject({ nkId: '2147354979', name: 'Catan (5th Edition)', mpn: 'CN3071', brand: 'Catan Studio', price: 44.95, currency: 'USD', itemCondition: 'NewCondition', availability: 'InStock', publisher: 'Catan Studio', productLine: 'Catan', category: 'Board Games' });
25 + expect(nkCategory(p)).toBe('board_games');
26 + expect(nkCategory({ ...p, category: 'Miniatures', productLine: 'Warhammer 40,000', publisher: 'Games Workshop' })).toBe('warhammer');
27 + expect(nkCategory({ ...p, category: 'Role Playing Games', productLine: 'D&D' })).toBeNull();
28 + });
29 +
30 + it('normalises to a catalog item + listing with a boxed_toys condition', async () => {
31 + const p = parseProduct(PAGE, 'https://www.nobleknight.com/P/2147354979/Catan')!;
32 + const out = await connector.normalize({ url: p.url, externalId: p.nkId, kind: 'listing', engine: 'api', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: { kind: 'product_page', product: p } });
33 + expect(out.map((r) => r.kind)).toEqual(['catalog_item', 'listing']);
34 + const l = out[1]!;
35 + if (l.kind !== 'listing') throw new Error('listing');
36 + expect(l).toMatchObject({ price: 44.95, currency: 'USD', availability: 'available' });
37 + expect(l.condition.condition).toBe('mint_in_box');
38 + expect(l.attributes.identifiers).toMatchObject({ nobleknight_id: '2147354979', mpn: 'CN3071' });
39 + });
40 +
41 + it('fixtures normalise', async () => {
42 + const out = await connector.normalize(loadFixture('noble-knight', 'board-game').raw);
43 + expect(out.length).toBe(2);
44 + });
45 +});
added connectors/api/noble-knight/index.ts +154 −0
@@ -0,0 +1,154 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js';
5 +
6 +const BASE = 'https://www.nobleknight.com';
7 +const PARSER_VERSION = '1.0.0';
8 +
9 +export const ProductSchema = z.object({
10 + url: z.string(),
11 + nkId: z.string(),
12 + name: z.string(),
13 + sku: z.string().nullable(),
14 + mpn: z.string().nullable(),
15 + brand: z.string().nullable(),
16 + image: z.string().nullable(),
17 + description: z.string().nullable(),
18 + price: z.number().nullable(),
19 + currency: z.string().nullable(),
20 + itemCondition: z.string().nullable(),
21 + availability: z.string().nullable(),
22 + publisher: z.string().nullable(),
23 + productLine: z.string().nullable(),
24 + category: z.string().nullable(),
25 + genre: z.string().nullable(),
26 + type: z.string().nullable(),
27 + conditionText: z.string().nullable(),
28 +});
29 +export type Product = z.infer<typeof ProductSchema>;
30 +export const PayloadSchema = z.object({ kind: z.literal('product_page'), product: ProductSchema });
31 +
32 +/** Product page: schema.org Product JSON-LD + the info lines (Publisher / Product Line / Category / Genre / Type). */
33 +export function parseProduct(htmlText: string, url: string): Product | null {
34 + const ld = H.jsonLd(htmlText, 'Product')[0];
35 + if (!ld) return null;
36 + const $ = H.load(htmlText);
37 + const offers = (Array.isArray(ld.offers) ? ld.offers[0] : ld.offers) as Record<string, unknown> | undefined;
38 + const info: Record<string, string> = {};
39 + $('.info-line').each((_, el) => {
40 + const label = H.text($(el).find('.label'));
41 + const value = H.text($(el).find('.value'));
42 + if (label && value) info[label.toLowerCase()] = value;
43 + });
44 + const nkId = url.match(/\/P\/(\d+)/)?.[1] ?? String(ld.sku ?? '');
45 + const conditionText = H.text($('.conditions').first()) ?? H.text($('.condition, .item-condition').first());
46 + const brand = ld.brand && typeof ld.brand === 'object' ? String((ld.brand as { name?: string }).name ?? '') : ld.brand ? String(ld.brand) : null;
47 + return {
48 + url,
49 + nkId,
50 + name: String(ld.name ?? ''),
51 + sku: ld.sku ? String(ld.sku) : null,
52 + mpn: ld.mpn ? String(ld.mpn) : null,
53 + brand: brand || null,
54 + image: Array.isArray(ld.image) ? (ld.image[0] as string | undefined) ?? null : ld.image ? String(ld.image) : null,
55 + description: ld.description ? String(ld.description).slice(0, 500) : null,
56 + price: offers?.price !== undefined && offers.price !== null ? Number(offers.price) : null,
57 + currency: offers?.priceCurrency ? String(offers.priceCurrency) : null,
58 + itemCondition: offers?.itemCondition ? String(offers.itemCondition).replace(/^https?:\/\/schema\.org\//, '') : null,
59 + availability: offers?.availability ? String(offers.availability).replace(/^https?:\/\/schema\.org\//, '') : null,
60 + publisher: info.publisher ?? null,
61 + productLine: info['product line'] ?? null,
62 + category: info.category ?? null,
63 + genre: info.genre ?? null,
64 + type: info.type ?? null,
65 + conditionText: conditionText ?? null,
66 + };
67 +}
68 +
69 +/** Taxonomy slug from Noble Knight's own labels; null → skip (RPG books, CCG singles, supplies…). */
70 +export function nkCategory(p: Product): string | null {
71 + const cat = (p.category ?? '').toLowerCase();
72 + const line = `${p.productLine ?? ''} ${p.publisher ?? ''} ${p.name}`.toLowerCase();
73 + if (/games workshop|warhammer|citadel|forge world|age of sigmar|40k|necromunda|blood bowl/.test(line)) return 'warhammer';
74 + if (/board ?game|war ?game|puzzle/.test(cat)) return 'board_games';
75 + if (/miniature/.test(cat)) return /\b(gundam|gunpla|bandai)\b/.test(line) ? 'gundam' : null;
76 + if (/toys?, movies|action figure|toys/.test(cat)) {
77 + if (/\b(gundam|gunpla)\b/.test(line)) return 'gundam';
78 + if (/\bfunko\b|\bpop!\b/.test(line)) return 'funko';
79 + if (/\b(figure|figuarts|nendoroid|hot toys|neca|mcfarlane|hasbro|mattel)\b/.test(line)) return 'action_figures';
80 + return null;
81 + }
82 + return null;
83 +}
84 +
85 +/**
86 + * Noble Knight Games (largest US used/new board-game & miniatures dealer): products are enumerated
87 + * from the public sitemaps (robots.txt allows /P/ product pages and sitemaps; category listings are
88 + * client-rendered and disallowed) and read from each page's schema.org Product data.
89 + */
90 +export class NobleKnightConnector extends BaseConnector {
91 + readonly version = '1.0.0';
92 + readonly parserVersion = PARSER_VERSION;
93 + protected override minIntervalMs = 1500;
94 + override readonly urlPatterns = [/^https?:\/\/(www\.)?nobleknight\.com\/P\/\d+/i];
95 +
96 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
97 + const sitemaps = Number(this.meta.config.sitemapCount ?? 5);
98 + const perRun = Number(this.meta.config.productsPerRun ?? 200);
99 + const smIndex = Number(ctx.options.cursor?.sitemapIndex ?? 1);
100 + const offset = Number(ctx.options.cursor?.offset ?? 0);
101 + const smUrl = `${BASE}/sitemapproducts${smIndex}.xml`;
102 + await this.throttle();
103 + const sm = await ctx.fetch(smUrl, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 90_000 });
104 + if (!sm.success || !sm.html) {
105 + ctx.anomaly('page_fetch_failed', `${smUrl}: ${sm.error ?? sm.httpStatus}`);
106 + return;
107 + }
108 + const locs = [...sm.html.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]!.trim()).filter((u) => /\/P\/\d+/.test(u));
109 + const slice = locs.slice(offset, offset + perRun);
110 + let count = 0;
111 + for (const url of slice) {
112 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
113 + if (!(await ctx.shouldFetch(url))) continue;
114 + await this.throttle();
115 + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', expect: ['title', 'price', 'identifiers'], parse: (r) => (r.html ? (() => { const p = parseProduct(r.html!, url); return p ? { title: p.name, price: p.price, identifiers: { sku: p.sku } } : null; })() : null) });
116 + if (!res.success || !res.html) {
117 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
118 + continue;
119 + }
120 + const product = parseProduct(res.html, url);
121 + if (!product) continue;
122 + count++;
123 + yield { url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt };
124 + }
125 + const nextOffset = offset + slice.length;
126 + const exhausted = nextOffset >= locs.length;
127 + await ctx.setCursor({ sitemapIndex: exhausted ? (smIndex % sitemaps) + 1 : smIndex, offset: exhausted ? 0 : nextOffset, updatedAt: new Date().toISOString() });
128 + }
129 +
130 + async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
131 + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text' });
132 + if (!res.success || !res.html) return [];
133 + const product = parseProduct(res.html, url);
134 + return product ? [{ url, externalId: product.nkId, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'product_page', product }, fetchedAt: res.fetchedAt }] : [];
135 + }
136 +
137 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
138 + const { product: p } = PayloadSchema.parse(raw.payload);
139 + const slug = nkCategory(p);
140 + if (!slug) return [];
141 + const condRaw = p.conditionText ?? (p.itemCondition === 'NewCondition' ? 'New' : p.itemCondition === 'UsedCondition' ? 'Used' : null);
142 + const cond = dealerCondition(condRaw);
143 + const attributes = lotAttributes({ categorySlug: slug, name: p.name, brand: p.publisher ?? p.brand, series: p.productLine, identifiers: { nobleknight_id: p.nkId, ...(p.mpn ? { mpn: p.mpn } : {}) }, metadata: { category: p.category, genre: p.genre, type: p.type, sku: p.sku } });
144 + const common = { meta: this.meta, sourceUrl: p.url, externalId: p.nkId, rawTitle: p.name, attributes, imageUrls: p.image ? [p.image] : [], description: p.description, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness };
145 + const out: NormalizedRecord[] = [makeCatalogItem({ ...common, confidence: 0.8 })];
146 + if (p.price !== null && Number.isFinite(p.price) && p.price > 0) {
147 + const cur = (p.currency ?? 'USD') as 'USD';
148 + out.push(makeListing({ ...common, price: p.price, currency: cur, listingType: 'fixed_price', seller: 'Noble Knight Games', location: 'US', availability: /InStock|PreOrder|LimitedAvailability/.test(p.availability ?? '') ? 'available' : 'sold', quantity: 1 }));
149 + }
150 + return out;
151 + }
152 +}
153 +
154 +export default (meta: ConnectorMeta) => new NobleKnightConnector(meta);
added connectors/api/noble-knight/meta.json +33 −0
@@ -0,0 +1,33 @@
1 +{
2 + "id": "noble-knight",
3 + "displayName": "Noble Knight Games (board games & miniatures dealer)",
4 + "sourceId": "noble-knight",
5 + "sourceName": "Noble Knight Games",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.nobleknight.com",
8 + "module": "api/noble-knight",
9 + "enginePriority": ["api"],
10 + "categories": ["board_games", "warhammer", "gundam", "action_figures", "funko"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": true,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.nobleknight.com/Terms",
26 + "accessNotes": "Plain HTTPS. robots.txt disallows the client-rendered category/search listings (/Catalog, /category-search, /AdvancedSearch…) but allows product pages (/P/<id>/<slug>) and the sitemaps (sitemapproducts1-5.xml ≈ 120k products), so products are enumerated from the sitemaps with a rotating cursor and read from each page's schema.org Product JSON-LD (name, sku, mpn, brand, price, itemCondition, availability) plus the Publisher / Product Line / Category / Genre lines. Only board games, Warhammer/Games Workshop miniatures, Gunpla and figure products are kept (RPG books, CCG singles and supplies are skipped). Condition mapped to the boxed_toys scale. 1.5 s politeness, 0 credits, 200 products per run.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "sitemapCount": 5,
31 + "productsPerRun": 200
32 + }
33 +}
added connectors/api/scp-auctions/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'catalog-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 2));
added connectors/api/scp-auctions/index.test.ts +63 −0
@@ -0,0 +1,63 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../_lib/local-meta.js';
7 +import { bidsquareDate, parseBidsquareCatalog, parseBidsquareEvents, sportsCategory } from '../_memorabilia-lib/index.js';
8 +import createConnector from './index.js';
9 +
10 +const dir = path.dirname(fileURLToPath(import.meta.url));
11 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
12 +const connector = createConnector(meta);
13 +
14 +const EVENTS = `<ul><li id="st-23246"><div class="row justify-content-center gtm-visible_event" data-event_id='23246' data-event_status='past' data-event_name='2026 Summer Premier'>
15 +<div class="list-top"><label>Timed Auction</label><h1><a href="https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246">2026 Summer Premier</a></h1></div></div></li></ul>`;
16 +
17 +const CATALOG = `<script type="application/ld+json">{"@context":"https://schema.org","@type":"Event","name":"2026 Summer Premier","url":"https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246","startDate":"2026-07-09 13:00:00 EDT","endDate":"2026-07-26 22:30:00 EDT"}</script>
18 +<div class="row filter-items">
19 +<div class="col-6 el_23246 gtm-visible_item" id="stl-9310176" data-item_id="9310176" data-event_id='23246' data-event_status='past' data-event_name='2026 Summer Premier'>
20 + <div class="catalog_img"><a href="https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/1947-bond-bread-jackie-robinson-9310176"><img src="https://s1.img.bidsquare.com/item/l/3726/37264383.jpeg" alt="x"/></a></div>
21 + <div class="catalog_detail"><div class="item-list-top"><div class="lot_Num">Lot 47</div><div class="lot_title"><a href="https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/1947-bond-bread-jackie-robinson-9310176">1947 Bond Bread Portrait Jackie Robinson Rookie – SGC FR 1.5</a></div></div>
22 + <div class="item-list-bottom"><div class="bidding_sec"><div class="bidPrice"><div class="bidLength bid_num"><span class="num">32 Bids</span></div><div class="bid_txt"><span>Sold for</span><div class="currency_container"><span class="price ">$14,400</span></div></div></div></div></div></div>
23 +</div>
24 +<div class="col-6 el_23246 gtm-visible_item" id="stl-9310177" data-item_id="9310177" data-event_id='23246' data-event_status='past' data-event_name='2026 Summer Premier'>
25 + <div class="catalog_detail"><div class="item-list-top"><div class="lot_Num">Lot 48</div><div class="lot_title"><a href="https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/passed-9310177">1986 Fleer Michael Jordan Rookie PSA 8</a></div></div>
26 + <div class="item-list-bottom"><div class="bidding_sec"><div class="bidPrice"><div class="bid_txt"><span>Passed</span></div></div></div></div></div>
27 +</div></div>
28 +<nav><ul class="pagination"><li data-page="1" class="page-item active"><a>1</a></li><li data-page="2" class="page-item"><a>2</a></li><li data-page="12" class="page-item"><a>12</a></li></ul></nav>`;
29 +
30 +describe('scp-auctions', () => {
31 + runFixtureSuite(connector, it, expect);
32 +
33 + it('parses the event list and catalog cards', () => {
34 + expect(parseBidsquareEvents(EVENTS)).toEqual([{ id: '23246', name: '2026 Summer Premier', url: 'https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246', status: 'past', startDate: null, endDate: null }]);
35 + const cat = parseBidsquareCatalog(CATALOG, 1)!;
36 + expect(cat.event).toMatchObject({ id: '23246', status: 'past', endDate: '2026-07-26 22:30:00 EDT' });
37 + expect(cat.totalPages).toBe(12);
38 + expect(cat.items.length).toBe(2);
39 + expect(cat.items[0]).toMatchObject({ itemId: '9310176', lotNumber: '47', priceLabel: 'Sold for', price: 14400, bids: 32 });
40 + expect(cat.items[1]!.price).toBeNull();
41 + expect(bidsquareDate('2026-07-26 22:30:00 EDT')?.toISOString()).toBe('2026-07-27T02:30:00.000Z');
42 + });
43 +
44 + it('normalises sold lots only, dated by the event end, with sport-aware categories', async () => {
45 + const cat = parseBidsquareCatalog(CATALOG, 1)!;
46 + const out = await connector.normalize({ url: 'x', externalId: 'e', kind: 'sale', engine: 'api', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: { kind: 'catalog_page', house: 'SCP Auctions', event: cat.event, page: 1, totalPages: 12, items: cat.items } });
47 + expect(out.length).toBe(1);
48 + const s = out[0]!;
49 + if (s.kind !== 'sale') throw new Error('expected sale');
50 + expect(s).toMatchObject({ price: 14400, currency: 'USD', lotNumber: '47', auctionHouse: 'SCP Auctions', buyerPremiumIncluded: null });
51 + expect(s.saleDate.toISOString()).toBe('2026-07-27T02:30:00.000Z');
52 + expect(s.attributes.categorySlug).toBe('baseball_cards');
53 + expect(s.grade).toMatchObject({ grader: 'sgc', grade: '1.5' });
54 + expect(sportsCategory('1986 Fleer Michael Jordan Rookie PSA 8')).toBe('basketball_cards');
55 + expect(sportsCategory('1927 Babe Ruth Game Used Bat')).toBe('sports_memorabilia');
56 + });
57 +
58 + it('fixture sales carry the event end date and USD prices', async () => {
59 + const out = await connector.normalize(loadFixture('scp-auctions', 'catalog-page').raw);
60 + expect(out.length).toBeGreaterThan(0);
61 + for (const r of out) if (r.kind === 'sale') expect(r.currency).toBe('USD');
62 + });
63 +});
added connectors/api/scp-auctions/index.ts +165 −0
@@ -0,0 +1,165 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { isBundleTitle, safeYear } from '../_auction-lib/categories.js';
5 +import { bidsquareDate, gradeOf, lotAttributes, makeLot, parseBidsquareCatalog, parseBidsquareEvents, sportsCategory, toyGrade, type BidsquareCatalog } from '../_memorabilia-lib/index.js';
6 +import { makeSale } from '../../firecrawl/_carlib/index.js';
7 +
8 +const PARSER_VERSION = '1.0.0';
9 +
10 +export const CatalogPayloadSchema = z.object({
11 + kind: z.literal('catalog_page'),
12 + house: z.string(),
13 + event: z.object({ id: z.string(), name: z.string(), url: z.string(), status: z.enum(['upcoming', 'live', 'past', 'unknown']), startDate: z.string().nullable(), endDate: z.string().nullable() }),
14 + page: z.number(),
15 + totalPages: z.number().nullable(),
16 + items: z.array(
17 + z.object({
18 + itemId: z.string(),
19 + url: z.string(),
20 + title: z.string(),
21 + lotNumber: z.string().nullable(),
22 + image: z.string().nullable(),
23 + priceLabel: z.string().nullable(),
24 + price: z.number().nullable(),
25 + bids: z.number().nullable(),
26 + estimateLow: z.number().nullable(),
27 + estimateHigh: z.number().nullable(),
28 + startsAt: z.number().nullable(),
29 + endsAt: z.number().nullable(),
30 + }),
31 + ),
32 +});
33 +export type CatalogPayload = z.infer<typeof CatalogPayloadSchema>;
34 +
35 +export interface BidsquareHouseOptions {
36 + base: string;
37 + house: string;
38 + /** map a lot title to a taxonomy slug */
39 + category: (title: string) => string | null;
40 + /** buyer's premium included in the displayed "Sold for" price? null when the house does not state it */
41 + buyerPremiumIncluded: boolean | null;
42 + idKey: string;
43 + location: string | null;
44 +}
45 +
46 +/**
47 + * Bidsquare-hosted auction catalogs (SCP Auctions, Hake's). Public pages only:
48 + * /auctions (upcoming), /auctions/past?page=N, /auctions/<house>/<slug>-<id>/catalog?page=N
49 + * Past catalogs show "Sold for $X" per lot → sales dated by the event end (schema.org Event JSON-LD);
50 + * upcoming catalogs show current/starting bid + estimate → auction lots for the calendar.
51 + */
52 +export class BidsquareHouseConnector extends BaseConnector {
53 + readonly version = '1.0.0';
54 + readonly parserVersion = PARSER_VERSION;
55 + protected override minIntervalMs = 2000;
56 +
57 + constructor(meta: ConnectorMeta, protected readonly opts: BidsquareHouseOptions) {
58 + super(meta);
59 + }
60 +
61 + protected async fetchHtml(ctx: CrawlContext, url: string): Promise<{ html: string | null; engine: 'api' | 'feed' | 'firecrawl' | 'scrapfly' | 'browser' | 'manual'; status: number | null; fetchedAt: Date }> {
62 + await this.throttle();
63 + const res = await ctx.fetch(url, { engines: ['api'], responseType: 'text', minQuality: 0.3, timeoutMs: 45_000 });
64 + if (!res.success || !res.html) {
65 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
66 + return { html: null, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt };
67 + }
68 + return { html: res.html, engine: res.engine, status: res.httpStatus, fetchedAt: res.fetchedAt };
69 + }
70 +
71 + /** Events to visit this run: past (sold) first, then upcoming (lots). Subclasses may override. */
72 + protected async listEvents(ctx: CrawlContext): Promise<Array<{ id: string; name: string; url: string; status: string }>> {
73 + const backfill = ctx.options.mode === 'backfill';
74 + const page = backfill ? Number(ctx.options.cursor?.pastPage ?? 1) : 1;
75 + const out: Array<{ id: string; name: string; url: string; status: string }> = [];
76 + const past = await this.fetchHtml(ctx, `${this.opts.base}/auctions/past${page > 1 ? `?page=${page}` : ''}`);
77 + if (past.html) out.push(...parseBidsquareEvents(past.html).map((e) => ({ ...e, status: 'past' })));
78 + if (this.meta.config.includeUpcoming !== false) {
79 + const up = await this.fetchHtml(ctx, `${this.opts.base}/auctions`);
80 + if (up.html) out.push(...parseBidsquareEvents(up.html).filter((e) => e.status !== 'past').map((e) => ({ ...e, status: e.status === 'past' ? 'past' : 'upcoming' })));
81 + }
82 + return out;
83 + }
84 +
85 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
86 + const eventsPerRun = Number(this.meta.config.eventsPerRun ?? 2);
87 + const maxPages = Number(this.meta.config.pagesPerEvent ?? 8);
88 + const done = new Set<string>(Array.isArray(ctx.options.cursor?.doneEvents) ? (ctx.options.cursor!.doneEvents as string[]) : []);
89 + const events = await this.listEvents(ctx);
90 + let processed = 0;
91 + let count = 0;
92 + for (const ev of events) {
93 + if (ctx.signal?.aborted || processed >= eventsPerRun) break;
94 + if (ev.status === 'past' && done.has(ev.id)) continue;
95 + const catalogBase = `${ev.url.replace(/\/$/, '')}/catalog`;
96 + let total: number | null = null;
97 + for (let page = 1; page <= maxPages; page++) {
98 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
99 + const url = page === 1 ? catalogBase : `${catalogBase}?page=${page}`;
100 + const r = await this.fetchHtml(ctx, url);
101 + if (!r.html) break;
102 + const cat = parseBidsquareCatalog(r.html, page);
103 + if (!cat || cat.items.length === 0) break;
104 + total = cat.totalPages ?? total;
105 + const payload: CatalogPayload = { kind: 'catalog_page', house: this.opts.house, event: { ...cat.event, name: cat.event.name || ev.name, url: cat.event.url || ev.url }, page, totalPages: total, items: cat.items };
106 + count++;
107 + yield { url, externalId: `event:${cat.event.id}:page:${page}`, kind: cat.event.status === 'past' ? 'sale' : 'auction_lot', engine: r.engine, httpStatus: r.status, payload, fetchedAt: r.fetchedAt };
108 + if (total !== null && page >= total) break;
109 + }
110 + processed++;
111 + if (ev.status === 'past') done.add(ev.id);
112 + }
113 + const pastPage = Number(ctx.options.cursor?.pastPage ?? 1);
114 + await ctx.setCursor({ doneEvents: [...done].slice(-500), pastPage: ctx.options.mode === 'backfill' && events.filter((e) => e.status === 'past').every((e) => done.has(e.id)) ? pastPage + 1 : pastPage, updatedAt: new Date().toISOString() });
115 + }
116 +
117 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
118 + const p = CatalogPayloadSchema.parse(raw.payload);
119 + const out: NormalizedRecord[] = [];
120 + const endDate = bidsquareDate(p.event.endDate);
121 + const startDate = bidsquareDate(p.event.startDate);
122 + for (const it of p.items) {
123 + const slug = this.opts.category(it.title);
124 + if (!slug) continue;
125 + const g = gradeOf(it.title);
126 + const tg = toyGrade(it.title);
127 + const attributes = lotAttributes({
128 + categorySlug: slug,
129 + name: it.title,
130 + year: safeYear(it.title),
131 + identifiers: { [this.opts.idKey]: it.itemId },
132 + metadata: { event_id: p.event.id, event_name: p.event.name, estimate_low: it.estimateLow, estimate_high: it.estimateHigh, bids: it.bids, ...(tg ? { toy_grade: `${tg.company} ${tg.grade}` } : {}) },
133 + });
134 + const common = { meta: this.meta, sourceUrl: it.url, externalId: it.itemId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade };
135 + const sold = p.event.status === 'past' && /sold/i.test(it.priceLabel ?? '') && it.price !== null && it.price > 0;
136 + if (sold) {
137 + const saleDate = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate;
138 + if (!saleDate) continue;
139 + out.push(makeSale({ ...common, price: it.price!, currency: 'USD', saleDate, buyerPremiumIncluded: this.opts.buyerPremiumIncluded, auctionHouse: this.opts.house, lotNumber: it.lotNumber, location: this.opts.location, isBundle: isBundleTitle(it.title) }));
140 + } else if (p.event.status !== 'past') {
141 + const endsAt = (it.endsAt ? new Date(it.endsAt * 1000) : null) ?? endDate;
142 + const status = startDate && startDate.getTime() > Date.now() ? 'upcoming' : 'live';
143 + out.push(makeLot({ ...common, auctionHouse: this.opts.house, auctionName: p.event.name, lotNumber: it.lotNumber, startsAt: startDate, endsAt, estimateLow: it.estimateLow, estimateHigh: it.estimateHigh, currentBid: /bid/i.test(it.priceLabel ?? '') ? it.price : null, currency: 'USD', status, location: this.opts.location, confidence: 0.85 }));
144 + }
145 + }
146 + return out;
147 + }
148 +}
149 +
150 +export function scpCategory(title: string): string {
151 + return sportsCategory(title);
152 +}
153 +
154 +export default (meta: ConnectorMeta) =>
155 + new BidsquareHouseConnector(meta, {
156 + base: 'https://catalogs.scpauctions.com',
157 + house: 'SCP Auctions',
158 + category: scpCategory,
159 + // SCP does not state on the catalog page whether "Sold for" includes the buyer's premium.
160 + buyerPremiumIncluded: null,
161 + idKey: 'scp_item_id',
162 + location: 'US',
163 + });
164 +
165 +export type { BidsquareCatalog };
added connectors/api/scp-auctions/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "scp-auctions",
3 + "displayName": "SCP Auctions (results & catalogs)",
4 + "sourceId": "scp-auctions",
5 + "sourceName": "SCP Auctions",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://catalogs.scpauctions.com",
8 + "module": "api/scp-auctions",
9 + "enginePriority": ["api"],
10 + "categories": ["sports_memorabilia", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "olympic_collectibles", "pokemon"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": false,
15 + "supportsSold": true,
16 + "supportsAuctions": true,
17 + "supportsImages": true,
18 + "supportsCatalog": false,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 720,
22 + "priority": "medium",
23 + "trustScore": 0.9,
24 + "attributionRequired": true,
25 + "termsUrl": "https://catalogs.scpauctions.com/terms",
26 + "accessNotes": "Plain HTTPS on the public Bidsquare-hosted catalog (robots.txt: only /wp-admin/ disallowed, Crawl-delay 10 → 2 s politeness plus small runs). Past auctions are listed at /auctions/past?page=N; each catalog page (/auctions/scp-auctions-inc/<slug>-<id>/catalog?page=N, 48 lots) shows 'Sold for $X' and the bid count; the event start/end come from the page's schema.org Event JSON-LD (sale date = event end, or the lot's own countdown end). Upcoming catalogs yield auction lots (current/starting bid, estimate). SCP does not state on these pages whether 'Sold for' includes the 20% buyer's premium → buyer_premium_included=null. No login, no bidding endpoints. 0 credits.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "eventsPerRun": 2,
31 + "pagesPerEvent": 8,
32 + "includeUpcoming": true
33 + }
34 +}
added connectors/api/trainz/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../../firecrawl/_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'collection-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 1));
added connectors/api/trainz/index.test.ts +46 −0
@@ -0,0 +1,46 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../_lib/local-meta.js';
7 +import createConnector, { splitTitle, trimProduct } from './index.js';
8 +
9 +const dir = path.dirname(fileURLToPath(import.meta.url));
10 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
11 +const connector = createConnector(meta);
12 +
13 +describe('trainz', () => {
14 + runFixtureSuite(connector, it, expect);
15 +
16 + it('splits Trainz condition suffixes and trims products', () => {
17 + expect(splitTitle('Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive LN/Box')).toEqual({ name: 'Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive', conditionCode: 'LN/Box' });
18 + expect(splitTitle('S. Soho & Co. HO BRASS Passenger Coach Car- Unpainted EX/Box').conditionCode).toBe('EX/Box');
19 + expect(splitTitle('Marklin 3000 HO Steam Locomotive').conditionCode).toBeNull();
20 + const p = trimProduct({ id: 1, title: 'X LN/Box', handle: 'x', vendor: 'Lionel', product_type: 'Trains', tags: ['condition:Like New', 'foo', 'scale:O Gauge'], variants: [{ id: 2, sku: 'A', price: '10.00', available: true }], images: [] });
21 + expect(p?.tags).toEqual(['condition:Like New', 'scale:O Gauge']);
22 + });
23 +
24 + it('emits a catalog item and a dealer listing per product with a normalised condition', async () => {
25 + const out = await connector.normalize({
26 + url: 'https://www.trainz.com/collections/lionel-o-postwar-trains/products.json?limit=250&page=1',
27 + externalId: 'lionel-o-postwar-trains:1',
28 + kind: 'listing',
29 + engine: 'api',
30 + fetchedAt: new Date('2026-09-07T00:00:00Z'),
31 + payload: { kind: 'collection_page', collection: 'lionel-o-postwar-trains', page: 1, products: [{ id: 99, title: 'Lionel 2343 O Gauge Santa Fe F3 AA Diesel Locomotive Set EX/Box', handle: 'lionel-2343', vendor: 'Lionel', product_type: 'Trains', tags: ['condition:Excellent', 'era:Postwar', 'scale:O Gauge'], created_at: '2026-09-01T00:00:00-04:00', updated_at: null, variants: [{ id: 1, sku: 'P123', price: '499.99', compare_at_price: null, available: true }], images: [{ src: 'https://cdn.shopify.com/x.jpg' }] }] },
32 + });
33 + expect(out.map((r) => r.kind)).toEqual(['catalog_item', 'listing']);
34 + const l = out[1]!;
35 + if (l.kind !== 'listing') throw new Error('listing');
36 + expect(l).toMatchObject({ price: 499.99, currency: 'USD', availability: 'available', seller: 'Trainz' });
37 + expect(l.attributes).toMatchObject({ categorySlug: 'model_trains', brand: 'Lionel', series: 'O Gauge', name: 'Lionel 2343 O Gauge Santa Fe F3 AA Diesel Locomotive Set' });
38 + expect(l.condition).toMatchObject({ condition: 'boxed', conditionRaw: 'Excellent' });
39 + expect(l.attributes.identifiers.trainz_sku).toBe('P123');
40 + });
41 +
42 + it('fixture products all carry a vendor and a USD price', async () => {
43 + const out = await connector.normalize(loadFixture('trainz', 'collection-page').raw);
44 + expect(out.filter((r) => r.kind === 'listing').length).toBeGreaterThan(0);
45 + });
46 +});
added connectors/api/trainz/index.ts +108 −0
@@ -0,0 +1,108 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { dealerCondition, lotAttributes, makeCatalogItem, makeListing } from '../_memorabilia-lib/index.js';
5 +
6 +const BASE = 'https://www.trainz.com';
7 +const PARSER_VERSION = '1.0.0';
8 +
9 +export const ProductSchema = z.object({
10 + id: z.number(),
11 + title: z.string(),
12 + handle: z.string(),
13 + vendor: z.string().nullable().default(null),
14 + product_type: z.string().nullable().default(null),
15 + tags: z.array(z.string()).default([]),
16 + created_at: z.string().nullable().default(null),
17 + updated_at: z.string().nullable().default(null),
18 + variants: z.array(z.object({ id: z.number(), sku: z.string().nullable().default(null), price: z.string(), compare_at_price: z.string().nullable().default(null), available: z.boolean().default(true) })).default([]),
19 + images: z.array(z.object({ src: z.string() })).default([]),
20 +});
21 +export type Product = z.infer<typeof ProductSchema>;
22 +export const PayloadSchema = z.object({ kind: z.literal('collection_page'), collection: z.string(), page: z.number(), products: z.array(ProductSchema) });
23 +export type Payload = z.infer<typeof PayloadSchema>;
24 +
25 +/** Keep only the fields we use so raw payloads stay small. */
26 +export function trimProduct(p: Record<string, unknown>): Product | null {
27 + const parsed = ProductSchema.safeParse(p);
28 + if (!parsed.success) return null;
29 + const v = parsed.data;
30 + return { ...v, variants: v.variants.slice(0, 3), images: v.images.slice(0, 2), tags: v.tags.filter((t) => /^(condition|class|era|scale|gauge|Inventory Type2|roadname|road_name|manufacturer)[:_]/i.test(t) || /^in-stock$|^sold-out$/.test(t)).slice(0, 12) };
31 +}
32 +
33 +function tag(p: Product, prefix: string): string | null {
34 + const t = p.tags.find((x) => x.toLowerCase().startsWith(`${prefix.toLowerCase()}:`));
35 + return t ? t.slice(prefix.length + 1).trim() : null;
36 +}
37 +
38 +/** "Lionel 6-18005 O Gauge 700E Hudson Steam Locomotive LN/Box" → name without the trailing condition code. */
39 +export function splitTitle(title: string): { name: string; conditionCode: string | null } {
40 + const m = title.match(/\s+(LN|EX|VG|GD|G|PR|FR|NM|MT|C-?\d{1,2})(?:\/(Box|OB|Sealed|No Box))?\s*$/i);
41 + if (!m) return { name: title.trim(), conditionCode: null };
42 + return { name: title.slice(0, m.index).trim(), conditionCode: m[0].trim() };
43 +}
44 +
45 +const CODE_WORDS: Record<string, string> = { LN: 'Like New', EX: 'Excellent', VG: 'Very Good', GD: 'Good', G: 'Good', PR: 'Poor', FR: 'Fair', NM: 'Near Mint', MT: 'Mint' };
46 +
47 +/**
48 + * Trainz.com (world's largest model-train dealer): public Shopify product feed per collection
49 + * (/collections/<handle>/products.json). Catalog facts + dealer asking price with graded condition.
50 + */
51 +export class TrainzConnector extends BaseConnector {
52 + readonly version = '1.0.0';
53 + readonly parserVersion = PARSER_VERSION;
54 + protected override minIntervalMs = 1500;
55 +
56 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
57 + const seeds = (this.meta.config.collections as string[] | undefined) ?? ['lionel-postwar-trains', 'american-flyer-postwar-trains'];
58 + const pagesPerCollection = Number(this.meta.config.pagesPerCollection ?? 2);
59 + const startIdx = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length;
60 + const perRun = Number(this.meta.config.collectionsPerRun ?? 4);
61 + let count = 0;
62 + for (let k = 0; k < Math.min(perRun, seeds.length); k++) {
63 + const handle = seeds[(startIdx + k) % seeds.length]!;
64 + for (let page = 1; page <= pagesPerCollection; page++) {
65 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
66 + const url = `${BASE}/collections/${handle}/products.json?limit=250&page=${page}`;
67 + await this.throttle();
68 + const res = await ctx.fetch(url, { engines: ['api'], expect: ['title', 'price'], parse: (r) => ({ title: (r.json as { products?: Array<{ title: string }> })?.products?.[0]?.title ?? null, price: (r.json as { products?: Array<{ variants?: Array<{ price: string }> }> })?.products?.[0]?.variants?.[0]?.price ?? null }) });
69 + const list = (res.json as { products?: Record<string, unknown>[] } | null)?.products;
70 + if (!res.success || !Array.isArray(list)) {
71 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
72 + break;
73 + }
74 + const products = list.map(trimProduct).filter((p): p is Product => Boolean(p));
75 + if (products.length === 0) break;
76 + count++;
77 + yield { url, externalId: `${handle}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload: { kind: 'collection_page', collection: handle, page, products } satisfies Payload, fetchedAt: res.fetchedAt };
78 + if (products.length < 250) break;
79 + }
80 + }
81 + await ctx.setCursor({ seedIndex: (startIdx + perRun) % seeds.length, updatedAt: new Date().toISOString() });
82 + }
83 +
84 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
85 + const p = PayloadSchema.parse(raw.payload);
86 + const out: NormalizedRecord[] = [];
87 + for (const pr of p.products) {
88 + const v = pr.variants[0];
89 + if (!v) continue;
90 + const { name, conditionCode } = splitTitle(pr.title);
91 + const condTag = tag(pr, 'condition');
92 + const condRaw = condTag ?? (conditionCode ? CODE_WORDS[conditionCode.split('/')[0]!.toUpperCase()] ?? conditionCode : null);
93 + const cond = dealerCondition(condRaw ?? '');
94 + const scale = tag(pr, 'scale') ?? tag(pr, 'gauge') ?? name.match(/\b(HO|N|O|S|G|Z|TT|O27|Standard)\s+(?:Scale|Gauge)\b/i)?.[0] ?? null;
95 + const url = `${BASE}/products/${pr.handle}`;
96 + const attributes = lotAttributes({ categorySlug: 'model_trains', name, brand: pr.vendor, series: scale, identifiers: { trainz_sku: v.sku ?? String(pr.id), shopify_product_id: String(pr.id) }, metadata: { product_type: pr.product_type, era: tag(pr, 'era'), class: tag(pr, 'class'), tags: pr.tags } });
97 + const price = Number(v.price);
98 + const common = { meta: this.meta, sourceUrl: url, externalId: String(pr.id), rawTitle: pr.title, attributes, imageUrls: pr.images.map((i) => i.src), observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: cond.condition, conditionRaw: condRaw, completeness: cond.completeness };
99 + out.push(makeCatalogItem({ ...common, confidence: 0.8 }));
100 + if (Number.isFinite(price) && price > 0) {
101 + out.push(makeListing({ ...common, price, currency: 'USD', listingType: 'fixed_price', seller: 'Trainz', location: 'US', availability: v.available ? 'available' : 'sold', listedAt: pr.created_at ? new Date(pr.created_at) : null, quantity: 1 }));
102 + }
103 + }
104 + return out;
105 + }
106 +}
107 +
108 +export default (meta: ConnectorMeta) => new TrainzConnector(meta);
added connectors/api/trainz/meta.json +61 −0
@@ -0,0 +1,61 @@
1 +{
2 + "id": "trainz",
3 + "displayName": "Trainz (model train dealer)",
4 + "sourceId": "trainz",
5 + "sourceName": "Trainz.com",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.trainz.com",
8 + "module": "api/trainz",
9 + "enginePriority": [
10 + "api"
11 + ],
12 + "categories": [
13 + "model_trains"
14 + ],
15 + "regions": [
16 + "US"
17 + ],
18 + "languages": [
19 + "en"
20 + ],
21 + "currency": [
22 + "USD"
23 + ],
24 + "supportsListings": true,
25 + "supportsSold": false,
26 + "supportsAuctions": false,
27 + "supportsImages": true,
28 + "supportsCatalog": true,
29 + "supportsPopulation": false,
30 + "supportsLookup": false,
31 + "refreshFrequencyMinutes": 1440,
32 + "priority": "low",
33 + "trustScore": 0.7,
34 + "attributionRequired": true,
35 + "termsUrl": "https://www.trainz.com/pages/terms-of-service",
36 + "accessNotes": "Public Shopify JSON feed (/collections/<handle>/products.json?limit=250&page=N; robots.txt allows) — the same data the storefront renders. Each product gives title (with Trainz's condition code suffix such as LN/Box, EX/Box), vendor, tags (condition:, class:, era:, Inventory Type2_), variant price/SKU/availability and images. Emits a catalog item (brand = vendor, scale/gauge when present) and a fixed-price dealer listing with the condition mapped to the boxed_toys scale. 1.5 s politeness, 0 credits; collections rotate through the configured seed list each run.",
37 + "enabled": true,
38 + "schemaVersion": "1.0",
39 + "config": {
40 + "collectionsPerRun": 4,
41 + "pagesPerCollection": 2,
42 + "collections": [
43 + "lionel-o-postwar-trains",
44 + "lionel-o-prewar-trains",
45 + "lionel-standard-gauge-trains",
46 + "american-flyer-postwar-trains",
47 + "american-flyer-s-gauge",
48 + "marklin-trains",
49 + "mth-o-gauge-trains",
50 + "brass-model-trains",
51 + "lgb-trains",
52 + "k-line",
53 + "williams",
54 + "kato-trains",
55 + "bachmann-trains",
56 + "atlas-o-gauge",
57 + "lionel-o-gauge-passenger-cars",
58 + "lionel-o-gauge-steam-locomotives"
59 + ]
60 + }
61 +}
added connectors/firecrawl/bbts/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'search-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 1));
added connectors/firecrawl/bbts/index.test.ts +66 −0
@@ -0,0 +1,66 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../../api/_lib/local-meta.js';
7 +import createConnector, { bbtsCategory, parseSearchMarkdown } from './index.js';
8 +
9 +const dir = path.dirname(fileURLToPath(import.meta.url));
10 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
11 +const connector = createConnector(meta);
12 +
13 +const MD = `Hide In Stock
14 +
15 +## Product Results
16 +
17 +- ![](https://images.bigbadtoystore.com/images/product/197167/a.png)
18 +
19 +### [Declaration of War MJZ-05 Dark Shadow 1/100 Scale Limited Edition Action Figure](https://www.bigbadtoystore.com/product/declaration-war-mjz-05-dark-shadow-1_100-scale-action-figure-197167?variation=388429)
20 +
21 +By: Hot General
22 +
23 +PRE-ORDER
24 +
25 +$118.99
26 +
27 +- ![](https://images.bigbadtoystore.com/images/product/197181/b.jpg)
28 +
29 +### [Pop! Marvel Deadpool Vinyl Figure - BBTS Exclusive](https://www.bigbadtoystore.com/product/pop-marvel-deadpool-197181?variation=388500)
30 +
31 +By: Funko
32 +
33 +SOLD OUT
34 +
35 +$14.99
36 +`;
37 +
38 +describe('bbts', () => {
39 + runFixtureSuite(connector, it, expect);
40 +
41 + it('parses product cards from the rendered search markdown', () => {
42 + const p = parseSearchMarkdown(MD, 'hot toys', 1);
43 + expect(p.items.length).toBe(2);
44 + expect(p.items[0]).toMatchObject({ productId: '197167', variation: '388429', brand: 'Hot General', status: 'PRE-ORDER', price: 118.99, url: 'https://www.bigbadtoystore.com/product/declaration-war-mjz-05-dark-shadow-1_100-scale-action-figure-197167' });
45 + expect(p.items[1]).toMatchObject({ productId: '197181', brand: 'Funko', status: 'SOLD OUT', price: 14.99 });
46 + expect(bbtsCategory(p.items[1]!.title, 'Funko')).toBe('funko');
47 + expect(bbtsCategory('MG 1/100 RX-78-2 Gundam Ver. 3.0 Model Kit', 'Bandai')).toBe('gundam');
48 + expect(bbtsCategory('BE@RBRICK KAWS 1000%', 'Medicom')).toBe('designer_toys');
49 + });
50 +
51 + it('normalises to catalog items + retailer listings with stock-aware availability', async () => {
52 + const out = await connector.normalize({ url: 'x', externalId: 'q:1', kind: 'listing', engine: 'firecrawl', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: parseSearchMarkdown(MD, 'hot toys', 1) });
53 + expect(out.map((r) => r.kind)).toEqual(['catalog_item', 'listing', 'catalog_item', 'listing']);
54 + const soldOut = out[3]!;
55 + if (soldOut.kind !== 'listing') throw new Error('listing');
56 + expect(soldOut.availability).toBe('ended');
57 + expect(soldOut.attributes.categorySlug).toBe('funko');
58 + const pre = out[1]!;
59 + if (pre.kind !== 'listing') throw new Error('listing');
60 + expect(pre).toMatchObject({ price: 118.99, currency: 'USD', availability: 'available', seller: 'BigBadToyStore' });
61 + });
62 +
63 + it('fixture normalises', async () => {
64 + expect((await connector.normalize(loadFixture('bbts', 'search-page').raw)).length).toBeGreaterThan(0);
65 + });
66 +});
added connectors/firecrawl/bbts/index.ts +109 −0
@@ -0,0 +1,109 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { lotAttributes, makeCatalogItem, makeListing, money } from '../../api/_memorabilia-lib/index.js';
5 +import { md, splitMarkdownItems } from '../_carlib/index.js';
6 +
7 +const BASE = 'https://www.bigbadtoystore.com';
8 +const PARSER_VERSION = '1.0.0';
9 +
10 +export const ItemSchema = z.object({ productId: z.string(), variation: z.string().nullable(), title: z.string(), url: z.string(), image: z.string().nullable(), brand: z.string().nullable(), status: z.string().nullable(), price: z.number().nullable(), listPrice: z.number().nullable() });
11 +export const PayloadSchema = z.object({ kind: z.literal('search_page'), query: z.string(), page: z.number(), items: z.array(ItemSchema) });
12 +export type Payload = z.infer<typeof PayloadSchema>;
13 +
14 +/** Firecrawl markdown of a search/department page → product cards. */
15 +export function parseSearchMarkdown(markdown: string, query: string, page: number): Payload {
16 + const start = markdown.indexOf('## Product Results');
17 + const body = start >= 0 ? markdown.slice(start) : markdown;
18 + const chunks = splitMarkdownItems(body, /^- !\[/m);
19 + const items: Payload['items'] = [];
20 + for (const c of chunks) {
21 + const link = c.match(/###\s*\[([^\]]+)\]\((https:\/\/www\.bigbadtoystore\.com\/product\/[^)\s]+)\)/);
22 + if (!link) continue;
23 + const url = link[2]!;
24 + const productId = url.match(/-(\d+)(?:\?|$)/)?.[1] ?? url.match(/product\/[^/]*?(\d+)/)?.[1];
25 + if (!productId) continue;
26 + const prices = [...c.matchAll(/\$([\d,]+\.\d{2})/g)].map((m) => money(`$${m[1]}`, 'USD')?.amount ?? null).filter((n): n is number => n !== null);
27 + const status = c.match(/\b(PRE-ORDER|IN STOCK|SOLD OUT|WAITLIST|BACKORDER|COMING SOON|LOW STOCK)\b/i)?.[1]?.toUpperCase() ?? null;
28 + items.push({
29 + productId,
30 + variation: url.match(/variation=(\d+)/)?.[1] ?? null,
31 + title: md.clean(link[1]!),
32 + url: url.split('?')[0]!,
33 + image: md.image(c),
34 + brand: c.match(/By:\s*([^\n]+)/)?.[1]?.trim() ?? null,
35 + status,
36 + price: prices.length ? Math.min(...prices) : null,
37 + listPrice: prices.length > 1 ? Math.max(...prices) : null,
38 + });
39 + }
40 + return { kind: 'search_page', query, page, items };
41 +}
42 +
43 +export function bbtsCategory(title: string, brand: string | null): string {
44 + const t = `${title} ${brand ?? ''}`.toLowerCase();
45 + if (/\bfunko\b|\bpop!\b|\bsoda\b.*funko/.test(t)) return 'funko';
46 + if (/\b(gundam|gunpla|zaku|master grade|real grade|high grade|perfect grade|\bhg\b|\bmg\b|\brg\b|\bpg\b)\b/.test(t)) return 'gundam';
47 + if (/\b(bearbrick|be@rbrick|medicom|kaws|pop mart|labubu|kidrobot|superplastic|mighty jaxx|designer toy|vinyl figure|art toy)\b/.test(t)) return 'designer_toys';
48 + if (/\b(lego)\b/.test(t)) return 'lego_sets';
49 + if (/\b(plush|plushie)\b/.test(t)) return 'plush';
50 + if (/\b(model kit|1\/24|1\/18 scale|diecast|die-cast)\b/.test(t)) return 'model_cars';
51 + return 'action_figures';
52 +}
53 +
54 +/**
55 + * BigBadToyStore (major US collectibles retailer). Search/department pages rendered through
56 + * Firecrawl (plain HTTP gets a bot challenge; not bypassed): title, brand line, stock status and
57 + * price per product. Emits catalog items (retail reference) and retailer listings.
58 + */
59 +export class BbtsConnector extends BaseConnector {
60 + readonly version = '1.0.0';
61 + readonly parserVersion = PARSER_VERSION;
62 + protected override minIntervalMs = 2000;
63 +
64 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
65 + const seeds = (this.meta.config.queries as string[] | undefined) ?? ['hot toys'];
66 + const pages = Number(this.meta.config.pagesPerQuery ?? 1);
67 + const perRun = Number(this.meta.config.queriesPerRun ?? 4);
68 + const start = Number(ctx.options.cursor?.seedIndex ?? 0) % seeds.length;
69 + let count = 0;
70 + for (let k = 0; k < Math.min(perRun, seeds.length); k++) {
71 + const q = seeds[(start + k) % seeds.length]!;
72 + for (let page = 1; page <= pages; page++) {
73 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
74 + const url = `${BASE}/Search?SearchText=${encodeURIComponent(q)}&PageSize=50&SortOrder=NewAndPopular${page > 1 ? `&PageIndex=${page}` : ''}`;
75 + await this.throttle();
76 + const res = await ctx.fetch(url, { engines: ['firecrawl'], expect: ['title', 'price', 'status'], parse: (r) => { const p = r.markdown ? parseSearchMarkdown(r.markdown, q, page).items[0] : null; return p ? { title: p.title, price: p.price, status: p.status } : null; } });
77 + if (!res.success || !res.markdown) {
78 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
79 + break;
80 + }
81 + const payload = parseSearchMarkdown(res.markdown, q, page);
82 + if (payload.items.length === 0) break;
83 + count++;
84 + yield { url, externalId: `${q}:${page}`, kind: 'listing', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
85 + if (payload.items.length < 50) break;
86 + }
87 + }
88 + await ctx.setCursor({ seedIndex: (start + perRun) % seeds.length, updatedAt: new Date().toISOString() });
89 + }
90 +
91 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
92 + const p = PayloadSchema.parse(raw.payload);
93 + const out: NormalizedRecord[] = [];
94 + for (const it of p.items) {
95 + const slug = bbtsCategory(it.title, it.brand);
96 + const attributes = lotAttributes({ categorySlug: slug, name: it.title, brand: it.brand, identifiers: { bbts_product_id: it.productId }, metadata: { status: it.status, list_price: it.listPrice, variation: it.variation, query: p.query } });
97 + if (it.listPrice && it.status && /PRE-ORDER|IN STOCK/.test(it.status)) attributes.originalMsrp = it.listPrice, (attributes.originalMsrpCurrency = 'USD');
98 + const common = { meta: this.meta, sourceUrl: it.url, externalId: it.productId, rawTitle: it.title, attributes, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, condition: 'mint_in_box', conditionRaw: 'New', completeness: 'sealed' };
99 + out.push(makeCatalogItem({ ...common, confidence: 0.8 }));
100 + if (it.price !== null && it.price > 0) {
101 + const availability = it.status === 'SOLD OUT' ? 'ended' : it.status === 'WAITLIST' ? 'unknown' : 'available';
102 + out.push(makeListing({ ...common, price: it.price, currency: 'USD', listingType: 'fixed_price', seller: 'BigBadToyStore', location: 'US', availability, quantity: null }));
103 + }
104 + }
105 + return out;
106 + }
107 +}
108 +
109 +export default (meta: ConnectorMeta) => new BbtsConnector(meta);
added connectors/firecrawl/bbts/meta.json +34 −0
@@ -0,0 +1,34 @@
1 +{
2 + "id": "bbts",
3 + "displayName": "BigBadToyStore (retail listings)",
4 + "sourceId": "bbts",
5 + "sourceName": "BigBadToyStore",
6 + "sourceType": "dealer",
7 + "sourceUrl": "https://www.bigbadtoystore.com",
8 + "module": "firecrawl/bbts",
9 + "enginePriority": ["firecrawl"],
10 + "categories": ["action_figures", "funko", "gundam", "designer_toys", "plush", "model_cars", "lego_sets"],
11 + "regions": ["US"],
12 + "languages": ["en"],
13 + "currency": ["USD"],
14 + "supportsListings": true,
15 + "supportsSold": false,
16 + "supportsAuctions": false,
17 + "supportsImages": true,
18 + "supportsCatalog": true,
19 + "supportsPopulation": false,
20 + "supportsLookup": false,
21 + "refreshFrequencyMinutes": 1440,
22 + "priority": "low",
23 + "trustScore": 0.7,
24 + "attributionRequired": true,
25 + "termsUrl": "https://www.bigbadtoystore.com/Help/Terms",
26 + "accessNotes": "Search pages (/Search?SearchText=…&PageSize=50) rendered through Firecrawl at 1 credit per page of 50 products; plain HTTP receives the store's bot challenge page, which is not bypassed — Firecrawl renders the public page like a browser and robots.txt is not served to non-browser agents (403), so runs stay tiny (4 queries/run, 2 s politeness). Each card gives title, 'By: brand', stock status (PRE-ORDER / IN STOCK / SOLD OUT / WAITLIST) and price. Emits a catalog item (retail price as MSRP when in stock/pre-order) and a retailer listing. Retail asks are never treated as market value.",
27 + "enabled": true,
28 + "schemaVersion": "1.0",
29 + "config": {
30 + "queriesPerRun": 4,
31 + "pagesPerQuery": 1,
32 + "queries": ["hot toys", "s.h. figuarts", "mafex", "funko pop exclusive", "gunpla master grade", "bearbrick", "hasbro black series", "neca", "mezco one:12", "transformers masterpiece", "mcfarlane", "pop mart"]
33 + }
34 +}
added connectors/firecrawl/lelands/_smoke.ts +7 −0
@@ -0,0 +1,7 @@
1 +import path from 'node:path';
2 +import { fileURLToPath } from 'node:url';
3 +import { captureFixture, runSmoke } from '../_carlib/smoke.js';
4 +
5 +const dir = path.dirname(fileURLToPath(import.meta.url));
6 +if (process.argv.includes('--save')) await captureFixture(dir, process.argv[process.argv.indexOf('--save') + 1] ?? 'gallery-page', 4);
7 +else await runSmoke(dir, Number(process.argv[2] ?? 1));
added connectors/firecrawl/lelands/index.test.ts +53 −0
@@ -0,0 +1,53 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { readFileSync } from 'node:fs';
3 +import path from 'node:path';
4 +import { fileURLToPath } from 'node:url';
5 +import { loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
6 +import { localMeta } from '../../api/_lib/local-meta.js';
7 +import createConnector, { parseGallery } from './index.js';
8 +
9 +const dir = path.dirname(fileURLToPath(import.meta.url));
10 +const meta = localMeta(JSON.parse(readFileSync(path.join(dir, 'meta.json'), 'utf8')));
11 +const connector = createConnector(meta);
12 +
13 +const PAGE = `<html><body>
14 +<div class="sidebar-widget"><h5 class="title"><span class="closed">&bull;</span> 2026 Summer Classic </h5><p> Start: 7/26/2026 12:00 PM EST <br> End: 8/15/2026 10:00 PM EST </p><p><span style="color:red;">Prices Shown Include Buyer's Premium.</span></p></div>
15 +<select name="ctl00$Auction"><option value="-1">All Auctions</option><option selected="selected" value="1008">2026 Summer Classic</option><option value="1005">2026 Spring Classic</option></select>
16 +<div class="item"><h5 class="boxed">1</h5><div class="item-details clearfix"><p class="description"><a href="https://auction.lelands.com/bids/bidplace.aspx?itemid=135734">Late 1930s Lou Gehrig Single-Signed Baseball (PSA)</a></p><div class="item-image"><a href="#"><img src="https://auction.lelands.com/images_items/thumbs/thumb_item_135734_1.jpg"/></a></div><p> Bids: <strong>49</strong> <br> Opening Bid: <strong>$5,000</strong> <br> Status: <strong>Sold</strong> </p></div><div class="item-price"><a href="#">SOLD FOR $92,050</a></div></div>
17 +<div class="item"><h5 class="boxed">2</h5><div class="item-details clearfix"><p class="description"><a href="https://auction.lelands.com/bids/bidplace.aspx?itemid=135794">Signed 1949 Bowman Baseball #82 Joe Page Card (PSA)</a></p><p> Bids: <strong>0</strong> <br> Opening Bid: <strong>$300</strong> <br> Status: <strong>Unsold</strong> </p></div><div class="item-price"></div></div>
18 +<ul class="pagination pagination-sm"><li class="active"><a href="?size=250&page=1">1</a></li><li><a href="?size=250&page=2">2</a></li><li><a href="?size=250&page=5">5</a></li></ul>
19 +</body></html>`;
20 +
21 +describe('lelands', () => {
22 + runFixtureSuite(connector, it, expect);
23 +
24 + it('parses the gallery: auction dates, premium note, items and pagination', () => {
25 + const p = parseGallery(PAGE, 1);
26 + expect(p.auction.id).toBe('1008');
27 + expect(p.auction.name).toContain('2026 Summer Classic');
28 + expect(p.auction.endText).toBe('8/15/2026 10:00 PM EST');
29 + expect(p.auction.premiumIncluded).toBe(true);
30 + expect(p.totalPages).toBe(5);
31 + expect(p.items[0]).toMatchObject({ itemId: '135734', bids: 49, openingBid: 5000, status: 'Sold', soldPrice: 92050, lotNumber: '1' });
32 + expect(p.items[1]!.soldPrice).toBeNull();
33 + });
34 +
35 + it('normalises sold lots dated by the auction end with premium included', async () => {
36 + const out = await connector.normalize({ url: 'x', externalId: 'a', kind: 'sale', engine: 'firecrawl', fetchedAt: new Date('2026-09-07T00:00:00Z'), payload: parseGallery(PAGE, 1) });
37 + expect(out.length).toBe(1);
38 + const s = out[0]!;
39 + if (s.kind !== 'sale') throw new Error('sale');
40 + expect(s).toMatchObject({ price: 92050, currency: 'USD', buyerPremiumIncluded: true, auctionHouse: 'Lelands', lotNumber: '1' });
41 + expect(s.saleDate.toISOString()).toBe('2026-08-15T00:00:00.000Z');
42 + expect(s.attributes.categorySlug).toBe('sports_memorabilia');
43 + });
44 +
45 + it('emits nothing while the displayed auction is still open', async () => {
46 + const future = PAGE.replace('End: 8/15/2026', 'End: 8/15/2099');
47 + expect(await connector.normalize({ url: 'x', externalId: 'a', kind: 'sale', engine: 'firecrawl', fetchedAt: new Date(), payload: parseGallery(future, 1) })).toEqual([]);
48 + });
49 +
50 + it('fixture normalises', async () => {
51 + expect((await connector.normalize(loadFixture('lelands', 'gallery-page').raw)).length).toBeGreaterThan(0);
52 + });
53 +});
added connectors/firecrawl/lelands/index.ts +109 −0
@@ -0,0 +1,109 @@
1 +import { z } from 'zod';
2 +import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3 +import type { NormalizedRecord } from '@rareindex/shared';
4 +import { isBundleTitle, safeYear } from '../../api/_auction-lib/categories.js';
5 +import { gradeOf, lotAttributes, money, selectedOption, sportsCategory } from '../../api/_memorabilia-lib/index.js';
6 +import { dateMDY, makeSale } from '../_carlib/index.js';
7 +
8 +const BASE = 'https://auction.lelands.com';
9 +const PARSER_VERSION = '1.1.0';
10 +
11 +export const ItemSchema = z.object({ itemId: z.string(), title: z.string(), url: z.string(), image: z.string().nullable(), bids: z.number().nullable(), openingBid: z.number().nullable(), status: z.string().nullable(), soldPrice: z.number().nullable(), lotNumber: z.string().nullable() });
12 +export const PayloadSchema = z.object({ kind: z.literal('gallery_page'), auction: z.object({ id: z.string().nullable(), name: z.string(), startText: z.string().nullable(), endText: z.string().nullable(), premiumIncluded: z.boolean() }), page: z.number(), totalPages: z.number().nullable(), items: z.array(ItemSchema) });
13 +export type Payload = z.infer<typeof PayloadSchema>;
14 +
15 +export function parseGallery(htmlText: string, page: number): Payload {
16 + const $ = H.load(htmlText);
17 + const side = $('.sidebar-widget').first();
18 + const name = H.text(side.find('h5.title')) ?? '';
19 + const sideText = H.text(side) ?? '';
20 + const startText = sideText.match(/Start:\s*(\d{1,2}\/\d{1,2}\/\d{4}[^E]*E[SD]T)/)?.[1] ?? null;
21 + const endText = sideText.match(/End:\s*(\d{1,2}\/\d{1,2}\/\d{4}[^E]*E[SD]T)/)?.[1] ?? null;
22 + const premiumIncluded = /Prices Shown Include Buyer'?s Premium/i.test(sideText);
23 + const items: Payload['items'] = [];
24 + const seen = new Set<string>();
25 + $('.item').each((_, el) => {
26 + const it = $(el);
27 + const link = it.find('p.description a').first();
28 + const href = link.attr('href');
29 + const title = H.text(link);
30 + const itemId = href?.match(/itemid=(\d+)/i)?.[1];
31 + if (!href || !title || !itemId || seen.has(itemId)) return;
32 + seen.add(itemId);
33 + const meta = H.text(it.find('.item-details > p').last()) ?? '';
34 + const bids = meta.match(/Bids:\s*(\d+)/)?.[1];
35 + const opening = meta.match(/Opening Bid:\s*\$([\d,]+)/)?.[1];
36 + const status = meta.match(/Status:\s*([A-Za-z ]+)/)?.[1]?.trim() ?? null;
37 + const priceText = H.text(it.find('.item-price')) ?? '';
38 + const sold = priceText.match(/SOLD FOR\s*\$([\d,]+(?:\.\d+)?)/i)?.[1];
39 + items.push({ itemId, title, url: `${BASE}/bids/bidplace.aspx?itemid=${itemId}`, image: it.find('.item-image img').attr('src') ?? null, bids: bids ? Number(bids) : null, openingBid: opening ? money(`$${opening}`, 'USD')?.amount ?? null : null, status, soldPrice: sold ? money(`$${sold}`, 'USD')?.amount ?? null : null, lotNumber: H.text(it.find('h5.boxed')) });
40 + });
41 + const pages = $('ul.pagination a[href*="page="]')
42 + .map((_, a) => Number(($(a).attr('href') ?? '').match(/page=(\d+)/)?.[1] ?? 0))
43 + .get()
44 + .filter((n) => n > 0);
45 + return { kind: 'gallery_page', auction: { id: selectedOption(htmlText, 'ctl00$Auction'), name, startText, endText, premiumIncluded }, page, totalPages: pages.length ? Math.max(...pages) : null, items };
46 +}
47 +
48 +/**
49 + * Lelands (sports memorabilia & cards). The public gallery auction.lelands.com/Lots/Gallery?size=250&page=N
50 + * lists the currently displayed auction's lots with "SOLD FOR $X"; the sidebar gives start/end and states that
51 + * prices include the buyer's premium. Direct requests receive a Cloudflare 403, so the pages are rendered
52 + * through Firecrawl (a browser render of the public page — no login, no bidding). Past auctions are only
53 + * reachable through a form postback, which is not attempted; each auction is captured while it is the
54 + * displayed one (lots stay marked SOLD after the close).
55 + */
56 +export class LelandsConnector extends BaseConnector {
57 + readonly version = '1.1.0';
58 + readonly parserVersion = PARSER_VERSION;
59 + protected override minIntervalMs = 2500;
60 +
61 + async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
62 + const maxPages = Number(this.meta.config.pagesPerRun ?? 5);
63 + const doneAuctions = new Set<string>(Array.isArray(ctx.options.cursor?.doneAuctions) ? (ctx.options.cursor!.doneAuctions as string[]) : []);
64 + let count = 0;
65 + let auctionId: string | null = null;
66 + for (let page = 1; page <= maxPages; page++) {
67 + if (ctx.signal?.aborted || this.reached(ctx, count)) break;
68 + const url = `${BASE}/Lots/Gallery?size=250${page > 1 ? `&page=${page}` : ''}`;
69 + await this.throttle();
70 + const res = await ctx.fetch(url, { engines: ['firecrawl'], expect: ['title', 'price', 'status', 'date'], parse: (r) => { const p = r.html ? parseGallery(r.html, page) : null; const f = p?.items.find((i) => i.soldPrice); return p ? { title: f?.title ?? p.items[0]?.title ?? null, price: f?.soldPrice ?? null, status: f?.status ?? null, date: p.auction.endText } : null; } });
71 + if (!res.success || !res.html) {
72 + ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
73 + break;
74 + }
75 + const payload = parseGallery(res.html, page);
76 + if (page === 1) {
77 + auctionId = payload.auction.id;
78 + if (!payload.auction.endText) ctx.anomaly('missing_auction_end', payload.auction.name);
79 + // Closed auctions are crawled once; the live one is refreshed each run.
80 + if (auctionId && doneAuctions.has(auctionId)) break;
81 + }
82 + if (payload.items.length === 0) break;
83 + count++;
84 + yield { url, externalId: `auction:${payload.auction.id ?? 'current'}:page:${page}`, kind: 'sale', engine: res.engine, httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
85 + if (payload.totalPages !== null && page >= payload.totalPages) {
86 + const end = dateMDY(payload.auction.endText);
87 + if (auctionId && end && end.getTime() < Date.now()) doneAuctions.add(auctionId);
88 + break;
89 + }
90 + }
91 + await ctx.setCursor({ doneAuctions: [...doneAuctions].slice(-100), updatedAt: new Date().toISOString() });
92 + }
93 +
94 + async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
95 + const p = PayloadSchema.parse(raw.payload);
96 + const saleDate = dateMDY(p.auction.endText);
97 + const out: NormalizedRecord[] = [];
98 + if (!saleDate || saleDate.getTime() > Date.now()) return out; // auction still open → no realized prices yet
99 + for (const it of p.items) {
100 + if (it.soldPrice === null || it.soldPrice <= 0 || !/sold/i.test(it.status ?? '')) continue;
101 + const g = gradeOf(it.title);
102 + const attributes = lotAttributes({ categorySlug: sportsCategory(it.title), name: it.title, year: safeYear(it.title), identifiers: { lelands_item_id: it.itemId }, metadata: { auction_id: p.auction.id, auction_name: p.auction.name, bids: it.bids, opening_bid: it.openingBid } });
103 + out.push(makeSale({ meta: this.meta, sourceUrl: it.url, externalId: it.itemId, rawTitle: it.title, attributes, price: it.soldPrice, currency: 'USD', saleDate, buyerPremiumIncluded: p.auction.premiumIncluded ? true : null, auctionHouse: 'Lelands', lotNumber: it.lotNumber, imageUrls: it.image ? [it.image] : [], observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION, grader: g.grader, grade: g.grade, location: 'US', isBundle: isBundleTitle(it.title) }));
104 + }
105 + return out;
106 + }
107 +}
108 +
109 +export default (meta: ConnectorMeta) => new LelandsConnector(meta);
added connectors/firecrawl/lelands/meta.json +49 −0
@@ -0,0 +1,49 @@
1 +{
2 + "id": "lelands",
3 + "displayName": "Lelands (auction results)",
4 + "sourceId": "lelands",
5 + "sourceName": "Lelands",
6 + "sourceType": "auction_house",
7 + "sourceUrl": "https://auction.lelands.com",
8 + "module": "firecrawl/lelands",
9 + "enginePriority": [
10 + "firecrawl"
11 + ],
12 + "categories": [
13 + "sports_memorabilia",
14 + "baseball_cards",
15 + "basketball_cards",
16 + "football_cards",
17 + "hockey_cards",
18 + "soccer_cards",
19 + "other_sports_cards",
20 + "olympic_collectibles"
21 + ],
22 + "regions": [
23 + "US"
24 + ],
25 + "languages": [
26 + "en"
27 + ],
28 + "currency": [
29 + "USD"
30 + ],
31 + "supportsListings": false,
32 + "supportsSold": true,
33 + "supportsAuctions": false,
34 + "supportsImages": true,
35 + "supportsCatalog": false,
36 + "supportsPopulation": false,
37 + "supportsLookup": false,
38 + "refreshFrequencyMinutes": 1440,
39 + "priority": "medium",
40 + "trustScore": 0.9,
41 + "attributionRequired": true,
42 + "termsUrl": "https://lelands.com/terms",
43 + "accessNotes": "The public gallery auction.lelands.com/Lots/Gallery?size=250&page=N lists the currently displayed auction's lots with title, bids, opening bid, status and 'SOLD FOR $X'; the sidebar gives the auction start/end and states 'Prices Shown Include Buyer's Premium' → buyer_premium_included=true, sale date = auction end (records are emitted only once the end has passed). Direct HTTP requests receive a Cloudflare 403 (robots.txt itself allows 'User-agent: *'), so pages are rendered through Firecrawl at 1 credit per 250 lots — a browser render of the public page, no login, no bidding, no challenge solving. Past auctions sit behind a form postback and are not fetched; each auction is captured while it is the displayed one. 2.5 s politeness, ≤5 pages per run.",
44 + "enabled": true,
45 + "schemaVersion": "1.0",
46 + "config": {
47 + "pagesPerRun": 5
48 + }
49 +}
added data/fixtures/bbts/search-page.json +69 −0
@@ -0,0 +1,69 @@
1 +{
2 + "raw": {
3 + "url": "https://www.bigbadtoystore.com/Search?SearchText=hot%20toys&PageSize=50&SortOrder=NewAndPopular",
4 + "externalId": "hot toys:1",
5 + "kind": "listing",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T07:17:05.542Z",
8 + "payload": {
9 + "kind": "search_page",
10 + "query": "hot toys",
11 + "page": 1,
12 + "items": [
13 + {
14 + "productId": "197167",
15 + "variation": "388429",
16 + "title": "Declaration of War MJZ-05 Dark Shadow 1/100 Scale Limited Edition Action Figure",
17 + "url": "https://www.bigbadtoystore.com/product/declaration-war-mjz-05-dark-shadow-1_100-scale-action-figure-197167",
18 + "image": "https://images.bigbadtoystore.com/images/product/197167/e70cc2d3-1b4c-485a-be62-cef26e2d4e10/360w.png",
19 + "brand": "Hot General",
20 + "status": "PRE-ORDER",
21 + "price": 118.99,
22 + "listPrice": null
23 + },
24 + {
25 + "productId": "197181",
26 + "variation": "388448",
27 + "title": "Shuga Chara! Chokorin Collection Box of 6 Random Figures",
28 + "url": "https://www.bigbadtoystore.com/product/shuga-chara-chokorin-collection-box-6-random-figures-197181",
29 + "image": "https://images.bigbadtoystore.com/images/product/197181/0eb86aa6-da96-4bec-baca-e44d6af76740/360w.jpg",
30 + "brand": "MegaHouse",
31 + "status": "PRE-ORDER",
32 + "price": 69.99,
33 + "listPrice": null
34 + },
35 + {
36 + "productId": "197174",
37 + "variation": "388470",
38 + "title": "Avengers: Doomsday MMS919 Thor (Deluxe Ver.) 1/6th Scale Collectible Figure",
39 + "url": "https://www.bigbadtoystore.com/product/avengers-doomsday-mms919-thor-deluxe-ver-1_6th-scale-collectible-figure-197174",
40 + "image": "https://images.bigbadtoystore.com/images/product/197174/86163320-bb24-4bec-a86f-5612e2072718/360w.png",
41 + "brand": "Hot Toys",
42 + "status": "PRE-ORDER",
43 + "price": 404.99,
44 + "listPrice": null
45 + },
46 + {
47 + "productId": "197169",
48 + "variation": "388461",
49 + "title": "Avengers: Doomsday MMS918 Thor 1/6th Scale Collectible Figure",
50 + "url": "https://www.bigbadtoystore.com/product/avengers-doomsday-mms918-thor-1_6th-scale-collectible-figure-197169",
51 + "image": "https://images.bigbadtoystore.com/images/product/197169/bb8b4a05-807d-474e-a118-e44fa71663d3/360w.png",
52 + "brand": "Hot Toys",
53 + "status": "PRE-ORDER",
54 + "price": 304.99,
55 + "listPrice": null
56 + }
57 + ]
58 + }
59 + },
60 + "expect": {
61 + "count": 8,
62 + "kinds": [
63 + "catalog_item",
64 + "listing"
65 + ]
66 + },
67 + "note": "Captured live from https://www.bigbadtoystore.com/Search?SearchText=hot%20toys&PageSize=50&SortOrder=NewAndPopular (lists trimmed to 4).",
68 + "capturedAt": "2026-09-07T07:17:05.548Z"
69 +}
\ No newline at end of file
added data/fixtures/hakes/catalog-page.json +89 −0
@@ -0,0 +1,89 @@
1 +{
2 + "raw": {
3 + "url": "https://www.hakes.com/auctions/hakes-auctions/september-2026-pop-culture-auction-24709/catalog",
4 + "externalId": "event:24709:page:1",
5 + "kind": "auction_lot",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T07:17:02.112Z",
8 + "payload": {
9 + "kind": "catalog_page",
10 + "house": "Hake's Auctions",
11 + "event": {
12 + "id": "24709",
13 + "name": "September 2026 Pop Culture Auction",
14 + "url": "https://www.hakes.com/auctions/hakes-auctions/september-2026-pop-culture-auction-24709",
15 + "status": "upcoming",
16 + "startDate": "2026-09-09 09:00:00 EDT",
17 + "endDate": "2026-09-30 21:40:00 EDT"
18 + },
19 + "page": 1,
20 + "totalPages": 14,
21 + "items": [
22 + {
23 + "itemId": "9830652",
24 + "url": "https://www.hakes.com/online-auctions/hakes-auctions/star-wars-1978---luke-skywalker-12-back-b-afa-85-y-nm-made-in-taiwan-9830652",
25 + "title": "STAR WARS (1978) - LUKE SKYWALKER 12 BACK-B AFA 85 Y-NM+ (MADE IN TAIWAN).",
26 + "lotNumber": "1",
27 + "image": "https://s1.img.bidsquare.com/item/l/3897/38972921.jpeg?t=1X2ctt",
28 + "priceLabel": "Starting Bid",
29 + "price": 10,
30 + "bids": null,
31 + "estimateLow": 1000,
32 + "estimateHigh": 3000,
33 + "startsAt": 1788958800,
34 + "endsAt": 1790818800
35 + },
36 + {
37 + "itemId": "9830653",
38 + "url": "https://www.hakes.com/online-auctions/hakes-auctions/star-wars-1978---ben-obi-wan-kenobi-12-back-a-afa-75-ex-nm-sku-on-footer-9830653",
39 + "title": "STAR WARS (1978) - BEN (OBI-WAN) KENOBI 12 BACK-A AFA 75+ EX+/NM (SKU ON FOOTER).",
40 + "lotNumber": "2",
41 + "image": "https://s1.img.bidsquare.com/item/l/3897/38972925.jpeg?t=1X2ctu",
42 + "priceLabel": "Starting Bid",
43 + "price": 10,
44 + "bids": null,
45 + "estimateLow": 1000,
46 + "estimateHigh": 2000,
47 + "startsAt": 1788958800,
48 + "endsAt": 1790818800
49 + },
50 + {
51 + "itemId": "9830654",
52 + "url": "https://www.hakes.com/online-auctions/hakes-auctions/star-wars-1978---princess-leia-organa-12-back-c-afa-70-ex-9830654",
53 + "title": "STAR WARS (1978) - PRINCESS LEIA ORGANA 12 BACK-C AFA 70+ EX+.",
54 + "lotNumber": "3",
55 + "image": "https://s1.img.bidsquare.com/item/l/3897/38972929.jpeg?t=1X2ctu",
56 + "priceLabel": "Starting Bid",
57 + "price": 10,
58 + "bids": null,
59 + "estimateLow": 500,
60 + "estimateHigh": 1000,
61 + "startsAt": 1788958800,
62 + "endsAt": 1790818800
63 + },
64 + {
65 + "itemId": "9830655",
66 + "url": "https://www.hakes.com/online-auctions/hakes-auctions/star-wars-1978---sand-people-tusken-raider-12-back-b-afa-85-nm-9830655",
67 + "title": "STAR WARS (1978) - SAND PEOPLE (TUSKEN RAIDER) 12 BACK-B AFA 85 NM+.",
68 + "lotNumber": "4",
69 + "image": "https://s1.img.bidsquare.com/item/l/3897/38972933.jpeg?t=1X2ctu",
70 + "priceLabel": "Starting Bid",
71 + "price": 10,
72 + "bids": null,
73 + "estimateLow": 2000,
74 + "estimateHigh": 4000,
75 + "startsAt": 1788958800,
76 + "endsAt": 1790818800
77 + }
78 + ]
79 + }
80 + },
81 + "expect": {
82 + "count": 4,
83 + "kinds": [
84 + "auction_lot"
85 + ]
86 + },
87 + "note": "Captured live from https://www.hakes.com/auctions/hakes-auctions/september-2026-pop-culture-auction-24709/catalog (lists trimmed to 4).",
88 + "capturedAt": "2026-09-07T07:17:02.164Z"
89 +}
\ No newline at end of file
added data/fixtures/lelands/gallery-page.json +75 −0
@@ -0,0 +1,75 @@
1 +{
2 + "raw": {
3 + "url": "https://auction.lelands.com/Lots/Gallery?size=250",
4 + "externalId": "auction:1008:page:1",
5 + "kind": "sale",
6 + "engine": "firecrawl",
7 + "fetchedAt": "2026-09-07T07:18:48.811Z",
8 + "payload": {
9 + "kind": "gallery_page",
10 + "auction": {
11 + "id": "1008",
12 + "name": "• 2026 Summer Classic",
13 + "startText": "7/26/2026 12:00 PM EST",
14 + "endText": "8/15/2026 10:00 PM EST",
15 + "premiumIncluded": true
16 + },
17 + "page": 1,
18 + "totalPages": 5,
19 + "items": [
20 + {
21 + "itemId": "135734",
22 + "title": "Late 1930s Lou Gehrig Single-Signed Baseball (PSA)",
23 + "url": "https://auction.lelands.com/bids/bidplace.aspx?itemid=135734",
24 + "image": "https://auction.lelands.com/images_items/thumbs/thumb_item_135734_1_499608.jpg",
25 + "bids": 49,
26 + "openingBid": 5000,
27 + "status": "Sold",
28 + "soldPrice": 92050,
29 + "lotNumber": "1"
30 + },
31 + {
32 + "itemId": "137205",
33 + "title": "1933 Lou Gehrig New York Yankees Player Contract (PSA)",
34 + "url": "https://auction.lelands.com/bids/bidplace.aspx?itemid=137205",
35 + "image": "https://auction.lelands.com/images_items/thumbs/thumb_item_137205_1_503191.jpg",
36 + "bids": 42,
37 + "openingBid": 10000,
38 + "status": "Sold",
39 + "soldPrice": 90190,
40 + "lotNumber": "2"
41 + },
42 + {
43 + "itemId": "136964",
44 + "title": "1996 Flair Showcase Basketball Legacy Collection Row 0 #23 Michael Jordan #24/150 PSA NM-MT+ 8.5",
45 + "url": "https://auction.lelands.com/bids/bidplace.aspx?itemid=136964",
46 + "image": "https://auction.lelands.com/images_items/thumbs/thumb_item_136964_1_494997.jpg",
47 + "bids": 19,
48 + "openingBid": 5000,
49 + "status": "Sold",
50 + "soldPrice": 75729,
51 + "lotNumber": "3"
52 + },
53 + {
54 + "itemId": "134967",
55 + "title": "1962 Wilt Chamberlain 100-Point Game Ticket Stub PSA VG-EX 4 (MK) - Highest Graded!",
56 + "url": "https://auction.lelands.com/bids/bidplace.aspx?itemid=134967",
57 + "image": "https://auction.lelands.com/images_items/thumbs/thumb_item_134967_1_489084.jpg",
58 + "bids": 36,
59 + "openingBid": 5000,
60 + "status": "Sold",
61 + "soldPrice": 48815,
62 + "lotNumber": "4"
63 + }
64 + ]
65 + }
66 + },
67 + "expect": {
68 + "count": 4,
69 + "kinds": [
70 + "sale"
71 + ]
72 + },
73 + "note": "Captured live from https://auction.lelands.com/Lots/Gallery?size=250 (lists trimmed to 4).",
74 + "capturedAt": "2026-09-07T07:18:48.903Z"
75 +}
\ No newline at end of file
added data/fixtures/morphy/catalog-page.json +71 −0
@@ -0,0 +1,71 @@
1 +{
2 + "raw": {
3 + "url": "https://auctions.morphyauctions.com/catalog.aspx?auctionid=716&page=1",
4 + "externalId": "auction:716:page:1",
5 + "kind": "sale",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T07:21:53.935Z",
8 + "payload": {
9 + "kind": "catalog_page",
10 + "auction": {
11 + "id": "716",
12 + "title": "Online Only Perfume Bottles",
13 + "dept": "Fine & Decorative Arts",
14 + "dateText": "August 11, 2026",
15 + "pageUrl": "https://morphyauctions.com/auctions/past-auctions/online-only-perfume-bottles/"
16 + },
17 + "page": 1,
18 + "totalPages": 19,
19 + "lots": [
20 + {
21 + "lotNumber": "1001",
22 + "title": "CAMEO GLASS FLORAL PERFUME WITH ATOMIZER",
23 + "url": "https://auctions.morphyauctions.com/CAMEO_GLASS_FLORAL_PERFUME_WITH_ATOMIZER-LOT662424.aspx",
24 + "image": "https://auctions.morphyauctions.com/ItemImages/000662/26320016_1_sm.jpeg",
25 + "finalPrice": 369,
26 + "minBid": 150,
27 + "estimateText": "$300 - $500",
28 + "bids": 6
29 + },
30 + {
31 + "lotNumber": "1002",
32 + "title": "RED & BLUE FOLIAGE CAMEO GLASS PERFUME WITH ATOMIZER",
33 + "url": "https://auctions.morphyauctions.com/RED___BLUE_FOLIAGE_CAMEO_GLASS_PERFUME_WITH_ATOMIZ-LOT662428.aspx",
34 + "image": "https://auctions.morphyauctions.com/ItemImages/000662/26320020_1_sm.jpeg",
35 + "finalPrice": 160,
36 + "minBid": 100,
37 + "estimateText": "$200 - $500",
38 + "bids": 3
39 + },
40 + {
41 + "lotNumber": "1003",
42 + "title": "ANGEL OLIVERAS GUART LAB TOPPED PERFUME WITH ATOMIZER",
43 + "url": "https://auctions.morphyauctions.com/ANGEL_OLIVERAS_GUART_LAB_TOPPED_PERFUME_WITH_ATOMI-LOT662431.aspx",
44 + "image": "https://auctions.morphyauctions.com/ItemImages/000662/26320023_1_sm.jpeg",
45 + "finalPrice": 128,
46 + "minBid": 100,
47 + "estimateText": "$200 - $400",
48 + "bids": 2
49 + },
50 + {
51 + "lotNumber": "1004",
52 + "title": "LOT OF 2: PERFUME ATOMIZER & POWDER BOX SETS",
53 + "url": "https://auctions.morphyauctions.com/LOT_OF_2__PERFUME_ATOMIZER___POWDER_BOX_SETS-LOT662809.aspx",
54 + "image": "https://auctions.morphyauctions.com/ItemImages/000662/26320194_1_sm.jpeg",
55 + "finalPrice": 704,
56 + "minBid": 300,
57 + "estimateText": "$600 - $700",
58 + "bids": 12
59 + }
60 + ]
61 + }
62 + },
63 + "expect": {
64 + "count": 4,
65 + "kinds": [
66 + "sale"
67 + ]
68 + },
69 + "note": "Captured live from https://auctions.morphyauctions.com/catalog.aspx?auctionid=716&page=1 (lists trimmed to 4).",
70 + "capturedAt": "2026-09-07T07:21:53.938Z"
71 +}
\ No newline at end of file
added data/fixtures/noble-knight/board-game.json +41 −0
@@ -0,0 +1,41 @@
1 +{
2 + "raw": {
3 + "url": "https://www.nobleknight.com/P/2147993326/007---Spectre-Board-Game",
4 + "externalId": "2147993326",
5 + "kind": "listing",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T07:18:47.151Z",
8 + "payload": {
9 + "kind": "product_page",
10 + "product": {
11 + "url": "https://www.nobleknight.com/P/2147993326/007---Spectre-Board-Game",
12 + "nkId": "2147993326",
13 + "name": "007 - Spectre Board Game",
14 + "sku": "2149328593",
15 + "mpn": "MUH007",
16 + "brand": "Modiphius Entertainment",
17 + "image": "https://image.nobleknight.com/m/jpg240/muh007.jpg",
18 + "description": "007 - Spectre Board Game - Board Game - Modiphius Entertainment from Modiphius Entertainment - part of our Board Games collection.",
19 + "price": 9.95,
20 + "currency": "USD",
21 + "itemCondition": "NewCondition",
22 + "availability": "InStock",
23 + "publisher": "Modiphius Entertainment",
24 + "productLine": "Board Games (Modiphius Entertainment)",
25 + "category": "Board Games",
26 + "genre": "Board Game - Family Board Game - Strategy",
27 + "type": "Boxed Game",
28 + "conditionText": "Quantity: Condition:SW (MINT/New) Our Price $9.95 Add to Cart Quantity: Condition:EX/NM Our Price $9.00 Add to Cart Quantity: Condition:VG+/NM Our Price $9.00 Add to Cart Quantity: Condition:VG+/NM Our Price $9.00 (unpunched) Add to Cart Quantity: Condition:Fair/NM Our Price $9.00 Add to Cart"
29 + }
30 + }
31 + },
32 + "expect": {
33 + "count": 2,
34 + "kinds": [
35 + "catalog_item",
36 + "listing"
37 + ]
38 + },
39 + "note": "Captured live via lookup(https://www.nobleknight.com/P/2147993326/007---Spectre-Board-Game).",
40 + "capturedAt": "2026-09-07T07:18:47.187Z"
41 +}
\ No newline at end of file
added data/fixtures/noble-knight/gunpla.json +41 −0
@@ -0,0 +1,41 @@
1 +{
2 + "raw": {
3 + "url": "https://www.nobleknight.com/P/2148351787/002-Ex-S-GUNDAM",
4 + "externalId": "2148351787",
5 + "kind": "listing",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T07:18:47.679Z",
8 + "payload": {
9 + "kind": "product_page",
10 + "product": {
11 + "url": "https://www.nobleknight.com/P/2148351787/002-Ex-S-GUNDAM",
12 + "nkId": "2148351787",
13 + "name": "002 Ex-S GUNDAM",
14 + "sku": null,
15 + "mpn": null,
16 + "brand": "Bandai",
17 + "image": "https://image.nobleknight.com/2/jpg240/25274750179nopic.jpg",
18 + "description": "002 Ex-S GUNDAM - Gundam Artifact - Bandai from Bandai - part of our Toys, Movies & More collection.",
19 + "price": 6.95,
20 + "currency": "USD",
21 + "itemCondition": "UsedCondition",
22 + "availability": "OutOfStock",
23 + "publisher": "Bandai",
24 + "productLine": "Gundam Artifact (Bandai)",
25 + "category": "Toys, Movies & More",
26 + "genre": "Model - Science Fiction",
27 + "type": "Scale Model",
28 + "conditionText": null
29 + }
30 + }
31 + },
32 + "expect": {
33 + "count": 2,
34 + "kinds": [
35 + "catalog_item",
36 + "listing"
37 + ]
38 + },
39 + "note": "Captured live via lookup(https://www.nobleknight.com/P/2148351787/002-Ex-S-GUNDAM).",
40 + "capturedAt": "2026-09-07T07:18:47.722Z"
41 +}
\ No newline at end of file
added data/fixtures/scp-auctions/catalog-page.json +89 −0
@@ -0,0 +1,89 @@
1 +{
2 + "raw": {
3 + "url": "https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246/catalog",
4 + "externalId": "event:23246:page:1",
5 + "kind": "sale",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T07:16:57.070Z",
8 + "payload": {
9 + "kind": "catalog_page",
10 + "house": "SCP Auctions",
11 + "event": {
12 + "id": "23246",
13 + "name": "2026 Summer Premier",
14 + "url": "https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246",
15 + "status": "past",
16 + "startDate": "2026-07-09 13:00:00 EDT",
17 + "endDate": "2026-07-26 22:30:00 EDT"
18 + },
19 + "page": 1,
20 + "totalPages": 12,
21 + "items": [
22 + {
23 + "itemId": "9380487",
24 + "url": "https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/september-11-2019-shohei-ohtani-game-used-asics-pro-model-bat-from-mlb-career-home-run-40-psa-dna-gu-9-5-fanatics-mlb-auth-9380487",
25 + "title": "September 11, 2019 Shohei Ohtani Game Used Asics Pro Model Bat from MLB Career Home Run #40 – PSA/DNA GU 9.5, Fanatics & MLB Auth.",
26 + "lotNumber": "1",
27 + "image": "https://s1.img.bidsquare.com/item/l/3808/38089373.jpeg?t=1WMAqa",
28 + "priceLabel": "Unsold",
29 + "price": null,
30 + "bids": 12,
31 + "estimateLow": null,
32 + "estimateHigh": null,
33 + "startsAt": null,
34 + "endsAt": null
35 + },
36 + {
37 + "itemId": "9394853",
38 + "url": "https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/2025-shohei-ohtani-los-angeles-dodgers-game-worn-home-jersey-photomatched-to-30th-hr-of-3rd-straight-mvp-season-sports-investors-loa-mlb-auth-9394853",
39 + "title": "2025 Shohei Ohtani Los Angeles Dodgers Game Worn Home Jersey Photomatched to 30th HR of 3rd Straight MVP Season – Sports Investors LOA, MLB Auth.",
40 + "lotNumber": "2",
41 + "image": "https://s1.img.bidsquare.com/item/l/3800/38007081.jpeg?t=1WKqtX",
42 + "priceLabel": "Unsold",
43 + "price": null,
44 + "bids": 14,
45 + "estimateLow": null,
46 + "estimateHigh": null,
47 + "startsAt": null,
48 + "endsAt": null
49 + },
50 + {
51 + "itemId": "9394841",
52 + "url": "https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/1986-fleer-57-michael-jordan---psa-gem-mint-10-9394841",
53 + "title": "1986 Fleer #57 Michael Jordan - PSA GEM MINT 10",
54 + "lotNumber": "3",
55 + "image": "https://s1.img.bidsquare.com/item/l/3810/38105385.jpeg?t=1WMY9G",
56 + "priceLabel": "Sold for",
57 + "price": 360000,
58 + "bids": 15,
59 + "estimateLow": null,
60 + "estimateHigh": null,
61 + "startsAt": null,
62 + "endsAt": null
63 + },
64 + {
65 + "itemId": "9434158",
66 + "url": "https://catalogs.scpauctions.com/online-auctions/scp-auctions-inc/1968-bob-gibson-autographed-st-louis-cardinals-game-worn-road-jersey-from-mvp-cy-young-record-1-12-era-season-sgc-superior-excellent-9434158",
67 + "title": "1968 Bob Gibson Autographed St. Louis Cardinals Game Worn Road Jersey from MVP, Cy Young & Record 1.12 ERA Season – SGC Superior/Excellent",
68 + "lotNumber": "4",
69 + "image": "https://s1.img.bidsquare.com/item/l/3744/37441939.jpeg?t=1Wx0je",
70 + "priceLabel": "Sold for",
71 + "price": 168000,
72 + "bids": 22,
73 + "estimateLow": null,
74 + "estimateHigh": null,
75 + "startsAt": null,
76 + "endsAt": null
77 + }
78 + ]
79 + }
80 + },
81 + "expect": {
82 + "count": 2,
83 + "kinds": [
84 + "sale"
85 + ]
86 + },
87 + "note": "Captured live from https://catalogs.scpauctions.com/auctions/scp-auctions-inc/2026-summer-premier-23246/catalog (lists trimmed to 4).",
88 + "capturedAt": "2026-09-07T07:16:57.108Z"
89 +}
\ No newline at end of file
added data/fixtures/trainz/collection-page.json +230 −0
@@ -0,0 +1,230 @@
1 +{
2 + "raw": {
3 + "url": "https://www.trainz.com/collections/lionel-o-postwar-trains/products.json?limit=250&page=1",
4 + "externalId": "lionel-o-postwar-trains:1",
5 + "kind": "listing",
6 + "engine": "api",
7 + "fetchedAt": "2026-09-07T07:18:18.883Z",
8 + "payload": {
9 + "kind": "collection_page",
10 + "collection": "lionel-o-postwar-trains",
11 + "page": 1,
12 + "products": [
13 + {
14 + "id": 8008717238394,
15 + "title": "Lionel 622-121 O Gauge 0.180\" x 0.328\" Carbon Slotted Motor Brush",
16 + "handle": "lionel-622-121-226e-92-slotted-motor-brushes",
17 + "vendor": "Lionel",
18 + "product_type": "Train Parts",
19 + "tags": [
20 + "class:Model Railroading",
21 + "Class_Model Railroading",
22 + "condition:Factory New",
23 + "Condition_Factory New",
24 + "era:Postwar",
25 + "Era_Postwar",
26 + "in-stock",
27 + "Inventory Type2_Brand New",
28 + "scale:O Gauge",
29 + "Scale_O Gauge"
30 + ],
31 + "created_at": "2023-10-05T15:08:04-04:00",
32 + "updated_at": "2026-09-07T03:18:18-04:00",
33 + "variants": [
34 + {
35 + "id": 43592139931770,
36 + "sku": "P11595961A",
37 + "price": "1.49",
38 + "compare_at_price": null,
39 + "available": true
40 + },
41 + {
42 + "id": 43592139964538,
43 + "sku": "P11595961B",
44 + "price": "12.50",
45 + "compare_at_price": "17.86",
46 + "available": true
47 + },
48 + {
49 + "id": 43592139997306,
50 + "sku": "P11595961C",
51 + "price": "44.65",
52 + "compare_at_price": "74.42",
53 + "available": true
54 + }
55 + ],
56 + "images": [
57 + {
58 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20250502-120236-C3-Trainz-11595961-STILL-00.jpg?v=1746202382"
59 + },
60 + {
61 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20250502-120236-C3-Trainz-11595961-STILL-01.jpg?v=1746202382"
62 + }
63 + ]
64 + },
65 + {
66 + "id": 8009053044858,
67 + "title": "Lionel 671-181 O Gauge Smoke Stack Gasket",
68 + "handle": "lionel-671-181-smoke-stack-gaskets",
69 + "vendor": "Lionel",
70 + "product_type": "Train Parts",
71 + "tags": [
72 + "class:Model Railroading",
73 + "Class_Model Railroading",
74 + "condition:Factory New",
75 + "Condition_Factory New",
76 + "era:Postwar",
77 + "Era_Postwar",
78 + "in-stock",
79 + "Inventory Type2_Brand New",
80 + "scale:O Gauge",
81 + "Scale_O Gauge"
82 + ],
83 + "created_at": "2023-10-05T18:05:54-04:00",
84 + "updated_at": "2026-09-07T03:18:18-04:00",
85 + "variants": [
86 + {
87 + "id": 44018885525626,
88 + "sku": "P11596070A",
89 + "price": "1.49",
90 + "compare_at_price": null,
91 + "available": true
92 + },
93 + {
94 + "id": 44018885558394,
95 + "sku": "P11596070B",
96 + "price": "12.50",
97 + "compare_at_price": "17.86",
98 + "available": true
99 + },
100 + {
101 + "id": 44018885591162,
102 + "sku": "P11596070C",
103 + "price": "44.65",
104 + "compare_at_price": "74.42",
105 + "available": true
106 + }
107 + ],
108 + "images": [
109 + {
110 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20241126-103806-C4-Trainz-11596070-STILL-00_64adefbf-314c-4c29-a389-6f879f3ab576.jpg?v=1751506145"
111 + },
112 + {
113 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20241126-103736-C4-Trainz-11596070-STILL-00.jpg?v=1751506145"
114 + }
115 + ]
116 + },
117 + {
118 + "id": 8008645017722,
119 + "title": "Lionel 480-18 O Gauge Black Horseshoe Retaining Washer 3/8\" W x 3/16\" ID",
120 + "handle": "lionel-480-18-black-horseshoe-washers",
121 + "vendor": "Lionel",
122 + "product_type": "Train Parts",
123 + "tags": [
124 + "class:Model Railroading",
125 + "Class_Model Railroading",
126 + "condition:Factory New",
127 + "Condition_Factory New",
128 + "era:Postwar",
129 + "Era_Postwar",
130 + "in-stock",
131 + "Inventory Type2_Brand New",
132 + "scale:O Gauge",
133 + "Scale_O Gauge"
134 + ],
135 + "created_at": "2023-10-05T14:30:20-04:00",
136 + "updated_at": "2026-09-07T03:18:18-04:00",
137 + "variants": [
138 + {
139 + "id": 43591991033978,
140 + "sku": "P11605867A",
141 + "price": "1.78",
142 + "compare_at_price": null,
143 + "available": true
144 + },
145 + {
146 + "id": 43591991066746,
147 + "sku": "P11605867B",
148 + "price": "15.00",
149 + "compare_at_price": null,
150 + "available": true
151 + },
152 + {
153 + "id": 43591991099514,
154 + "sku": "P11605867C",
155 + "price": "26.79",
156 + "compare_at_price": null,
157 + "available": true
158 + }
159 + ],
160 + "images": [
161 + {
162 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20231019-123813-C3-Trainz-11605867-STILL-00.jpg?v=1746045146"
163 + }
164 + ]
165 + },
166 + {
167 + "id": 8008905457786,
168 + "title": "Lionel 480-16 Coupler Spring (TC-22)",
169 + "handle": "lionel-480-16-coupler-springs-tc-22",
170 + "vendor": "Lionel",
171 + "product_type": "Train Parts",
172 + "tags": [
173 + "class:Model Railroading",
174 + "Class_Model Railroading",
175 + "condition:Factory New",
176 + "Condition_Factory New",
177 + "era:Postwar",
178 + "Era_Postwar",
179 + "in-stock",
180 + "Inventory Type2_Brand New",
181 + "scale:O Gauge",
182 + "Scale_O Gauge"
183 + ],
184 + "created_at": "2023-10-05T16:47:19-04:00",
185 + "updated_at": "2026-09-07T03:18:18-04:00",
186 + "variants": [
187 + {
188 + "id": 43592552906874,
189 + "sku": "P11602550A",
190 + "price": "1.49",
191 + "compare_at_price": null,
192 + "available": true
193 + },
194 + {
195 + "id": 43592552939642,
196 + "sku": "P11602550B",
197 + "price": "5.00",
198 + "compare_at_price": "7.15",
199 + "available": true
200 + },
201 + {
202 + "id": 43592552972410,
203 + "sku": "P11602550C",
204 + "price": "8.93",
205 + "compare_at_price": "14.88",
206 + "available": true
207 + }
208 + ],
209 + "images": [
210 + {
211 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20230331-115445-C3-Trainz-11602550-STILL-01.jpg?v=1746044687"
212 + },
213 + {
214 + "src": "https://cdn.shopify.com/s/files/1/1260/4747/files/20230331-115445-C3-Trainz-11602550-STILL-02.jpg?v=1746044687"
215 + }
216 + ]
217 + }
218 + ]
219 + }
220 + },
221 + "expect": {
222 + "count": 8,
223 + "kinds": [
224 + "catalog_item",
225 + "listing"
226 + ]
227 + },
228 + "note": "Captured live from the Shopify collection feed (products trimmed to 4).",
229 + "capturedAt": "2026-09-07T07:18:18.889Z"
230 +}
\ No newline at end of file
231