SPB Git
spboucher.ai

spb/rareindex

Public

Connectors: MTGJSON, Card Kingdom, MySlabs, TAG population, COMC, Alt (agent S)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 7, 2026) parent 45b94aa Browse files

30 changed files +4,660 −0

30 changed files
Load all 30 files Unified Split
added connectors/api/_lib/capture-s.ts +99 −0
@@ -0,0 +1,99 @@
1+/**
2+ * Live capture + smoke for the wave-4 connectors (mtgjson, cardkingdom, myslabs, tag-pop, comc, alt-xyz).
3+ * Usage: pnpm tsx connectors/api/_lib/capture-s.ts [ids…] [--no-save]
4+ */
5+import { createCrawlContext, createRouter, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors';
6+import { localMeta } from './local-meta.js';
7+import { saveFixture } from '@rareindex/connectors/testing';
8+import { childLogger } from '@rareindex/shared';
9+
10+interface Plan {
11+ dir: string;
12+ limit: number;
13+ seeds?: string[];
14+ pick: (raw: RawRecordInput, i: number) => string | null;
15+ kinds: string[];
16+ requiredFields: string[];
17+}
18+
19+const picked = new Set<string>();
20+const PLANS: Record<string, Plan> = {
21+ mtgjson: {
22+ dir: 'api',
23+ limit: 320,
24+ seeds: ['LEA'],
25+ pick: (raw, i) => {
26+ const p = raw.payload as { card: { name: string }; prices: unknown };
27+ if (p.card.name === 'Black Lotus') return 'lea-black-lotus';
28+ if (i === 0) return 'lea-first';
29+ return null;
30+ },
31+ kinds: ['catalog_item', 'price_observation'],
32+ requiredFields: ['attributes.identifiers.scryfall_id', 'attributes.setCode'],
33+ },
34+ cardkingdom: { dir: 'api', limit: 6, pick: (raw, i) => (i === 0 ? 'single-first' : i === 3 ? 'single-fourth' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.identifiers.cardkingdom_id', 'attributes.set'] },
35+ myslabs: { dir: 'api', limit: 6, pick: (raw, i) => (i === 0 ? 'slab-first' : i === 2 ? 'slab-third' : i === 5 ? 'slab-sixth' : null), kinds: ['listing'], requiredFields: ['attributes.identifiers.myslabs_id', 'price'] },
36+ 'tag-pop': { dir: 'firecrawl', limit: 1, seeds: ['https://my.taggrading.com/pop-report/Pokemon/1999/WOTC%20Pok%C3%A9mon?setName=Base%20Set'], pick: () => 'pokemon-1999-base-set', kinds: ['population_report'], requiredFields: ['attributes.set', 'attributes.number'] },
37+ comc: { dir: 'firecrawl', limit: 1, seeds: ['Cards/Basketball/1986/Fleer'], pick: () => 'basketball-1986-fleer-p1', kinds: ['listing'], requiredFields: ['attributes.set', 'price'] },
38+ 'alt-xyz': {
39+ dir: 'firecrawl',
40+ limit: 8,
41+ pick: (raw) => {
42+ const p = raw.payload as { transactions: unknown[]; altValue: number | null; listPrice: number | null };
43+ if (p.transactions.length && !picked.has('item-with-transactions')) return 'item-with-transactions';
44+ if ((p.altValue || p.listPrice) && !picked.has('item-priced')) return 'item-priced';
45+ return null;
46+ },
47+ kinds: ['sale', 'price_observation', 'listing'],
48+ requiredFields: ['attributes.identifiers.alt_item_id'],
49+ },
50+};
51+
52+const args = process.argv.slice(2);
53+const noSave = args.includes('--no-save');
54+const ids = args.filter((a) => !a.startsWith('--'));
55+const targets = ids.length ? ids : Object.keys(PLANS);
56+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
57+for (const id of targets) {
58+ const plan = PLANS[id];
59+ if (!plan) {
60+ console.log(`[${id}] no plan`);
61+ continue;
62+ }
63+ const started = Date.now();
64+ const mod = (await import(`../../${plan.dir}/${id}/index.ts`)) as { default: (m: ReturnType<typeof localMeta>) => RareIndexConnector };
65+ const metaJson = (await import(`../../${plan.dir}/${id}/meta.json`, { with: { type: 'json' } })) as { default: unknown };
66+ const connector = mod.default(localMeta(metaJson.default));
67+ const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, ...(plan.seeds ? { seeds: plan.seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) });
68+ let raws = 0;
69+ let normalized = 0;
70+ let printed = 0;
71+ let saved = 0;
72+ const kinds: Record<string, number> = {};
73+ try {
74+ for await (const raw of connector.crawl(ctx)) {
75+ const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };
76+ const out = await connector.normalize(rawLike);
77+ normalized += out.length;
78+ for (const r of out) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1;
79+ const name = plan.pick(raw, raws);
80+ raws++;
81+ if (name && !noSave) {
82+ picked.add(name);
83+ saveFixture(id, name, { raw: rawLike, expect: { minCount: 1, kinds: plan.kinds, requiredFields: plan.requiredFields }, note: `Live capture ${new Date().toISOString().slice(0, 10)} from ${raw.url}` });
84+ saved++;
85+ }
86+ if (printed < 3 && out.length) {
87+ printed++;
88+ const r = out.find((x) => x.kind === 'sale') ?? out.find((x) => x.kind === 'price_observation') ?? out.find((x) => x.kind === 'population_report') ?? out[0]!;
89+ const s = r as unknown as Record<string, unknown>;
90+ const attrs = (s.attributes ?? {}) as Record<string, unknown>;
91+ console.log(`[${id}] ${r.kind} · ${String(s.rawTitle ?? attrs.name ?? '')} · ${String(attrs.categorySlug)} · set=${String(attrs.set)} #${String(attrs.number)} · grade=${JSON.stringify(s.grade ?? null)} · price=${String(s.price ?? s.total ?? '')} ${String(s.currency ?? '')} · date=${String(s.saleDate ?? s.observationDate ?? s.reportDate ?? '')}`);
92+ }
93+ }
94+ } catch (err) {
95+ console.log(`[${id}] crawl error: ${err instanceof Error ? err.message : String(err)}`);
96+ }
97+ console.log(`[${id}] raw=${raws} normalized=${normalized} kinds=${JSON.stringify(kinds)} saved=${saved} anomalies=${ctx.anomalies.length} engines=${JSON.stringify(ctx.engineStats)} ms=${Date.now() - started}`);
98+ if (ctx.anomalies.length) console.log(`[${id}] anomalies: ${ctx.anomalies.slice(0, 5).join(' ; ')}`);
99+}
added connectors/api/_lib/wave4.ts +273 −0
@@ -0,0 +1,273 @@
1+/**
2+ * Helpers shared by the wave-4 card/comic connectors (mtgjson, cardkingdom, myslabs, tag-pop, comc,
3+ * alt-xyz). Kept inside connectors/api (not the framework).
4+ */
5+import { createGunzip, gunzipSync } from 'node:zlib';
6+import { Readable } from 'node:stream';
7+
8+export const BOT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)';
9+export const BOT_HEADERS = { 'user-agent': BOT_UA, accept: 'application/json, text/html;q=0.9, */*;q=0.8' };
10+
11+/** UTC midnight. */
12+export function dayOf(d: Date): Date {
13+ return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
14+}
15+
16+/** "2026-09-06" → UTC midnight Date. */
17+export function isoDay(s: string | null | undefined): Date | null {
18+ if (!s) return null;
19+ const m = s.match(/^(\d{4})-(\d{2})-(\d{2})/);
20+ if (!m) return null;
21+ const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])));
22+ return Number.isNaN(d.getTime()) ? null : d;
23+}
24+
25+/** "$8,830.00" | "8000.00" → number or null (never ≤ 0). */
26+export function money(s: string | number | null | undefined): number | null {
27+ if (s === null || s === undefined) return null;
28+ const n = typeof s === 'number' ? s : Number.parseFloat(String(s).replace(/[^0-9.]/g, ''));
29+ return Number.isFinite(n) && n > 0 ? n : null;
30+}
31+
32+/** Download a gzip JSON document fully (small files only, e.g. AllPricesToday ≈ 5 MB gz). */
33+/** Download a (possibly gzip) document; handles servers that already transparently decode Content-Encoding. */
34+export async function fetchMaybeGzip(url: string, signal?: AbortSignal): Promise<Buffer> {
35+ const res = await fetch(url, { headers: BOT_HEADERS, signal });
36+ if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);
37+ const buf = Buffer.from(await res.arrayBuffer());
38+ if (buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b) return gunzipSync(buf);
39+ return buf;
40+}
41+
42+export async function fetchGzipJson<T = unknown>(url: string, signal?: AbortSignal): Promise<T> {
43+ return JSON.parse((await fetchMaybeGzip(url, signal)).toString('utf8')) as T;
44+}
45+
46+/**
47+ * Stream a large gzip JSON document shaped like {"meta":{…},"data":{"<key>":<value>,…}} and invoke
48+ * `onEntry(key, value)` for each top-level entry of `data` without materialising the whole file
49+ * (MTGJSON AllPrices ≈ 1.5 GB uncompressed). Returns the number of entries seen.
50+ */
51+export async function streamGzipDataEntries(url: string, onEntry: (key: string, value: unknown) => Promise<void> | void, opts: { signal?: AbortSignal; limit?: number } = {}): Promise<number> {
52+ const res = await fetch(url, { headers: BOT_HEADERS, signal: opts.signal });
53+ if (!res.ok || !res.body) throw new Error(`${url}: HTTP ${res.status}`);
54+ const encoded = (res.headers.get('content-type') ?? '').includes('gzip') || /\.gz(\?|$)/.test(url);
55+ const raw = Readable.fromWeb(res.body as never);
56+ // Some CDNs serve .gz files with Content-Encoding: gzip, which fetch already decodes → sniff the magic bytes.
57+ const first = await new Promise<Buffer | null>((resolve, reject) => {
58+ raw.once('readable', () => resolve((raw.read(2) as Buffer | null) ?? null));
59+ raw.once('error', reject);
60+ raw.once('end', () => resolve(null));
61+ });
62+ if (first) raw.unshift(first);
63+ const isGz = first ? first[0] === 0x1f && first[1] === 0x8b : encoded;
64+ const stream = isGz ? raw.pipe(createGunzip()) : raw;
65+ let buf = '';
66+ let inData = false;
67+ let count = 0;
68+ let done = false;
69+ for await (const chunk of stream) {
70+ if (done) break;
71+ buf += (chunk as Buffer).toString('utf8');
72+ if (!inData) {
73+ const i = buf.indexOf('"data"');
74+ if (i < 0) {
75+ buf = buf.slice(-16);
76+ continue;
77+ }
78+ const brace = buf.indexOf('{', i);
79+ if (brace < 0) continue;
80+ buf = buf.slice(brace + 1);
81+ inData = true;
82+ }
83+ // Parse as many complete `"key": {…},` entries as the buffer holds.
84+ for (;;) {
85+ const k = buf.indexOf('"');
86+ if (k < 0) break;
87+ const kEnd = buf.indexOf('"', k + 1);
88+ if (kEnd < 0) break;
89+ const colon = buf.indexOf(':', kEnd);
90+ if (colon < 0) break;
91+ const start = colon + 1;
92+ const end = scanJsonValue(buf, start);
93+ if (end < 0) break; // incomplete value, wait for more data
94+ const key = buf.slice(k + 1, kEnd);
95+ const raw = buf.slice(start, end).trim();
96+ try {
97+ await onEntry(key, JSON.parse(raw));
98+ } catch (err) {
99+ if (err instanceof SyntaxError) {
100+ /* skip malformed slice */
101+ } else throw err;
102+ }
103+ count++;
104+ buf = buf.slice(end);
105+ const comma = buf.indexOf(',');
106+ const close = buf.indexOf('}');
107+ if (close >= 0 && (comma < 0 || close < comma)) {
108+ done = true;
109+ break;
110+ }
111+ buf = comma >= 0 ? buf.slice(comma + 1) : buf;
112+ if (opts.limit && count >= opts.limit) {
113+ done = true;
114+ break;
115+ }
116+ }
117+ }
118+ return count;
119+}
120+
121+/** Index just past a complete JSON value starting at `start` (whitespace allowed); -1 if incomplete. */
122+function scanJsonValue(s: string, start: number): number {
123+ let i = start;
124+ while (i < s.length && /\s/.test(s[i]!)) i++;
125+ if (i >= s.length) return -1;
126+ const c = s[i]!;
127+ if (c === '{' || c === '[') {
128+ let depth = 0;
129+ let inStr = false;
130+ for (let j = i; j < s.length; j++) {
131+ const ch = s[j]!;
132+ if (inStr) {
133+ if (ch === '\\') j++;
134+ else if (ch === '"') inStr = false;
135+ continue;
136+ }
137+ if (ch === '"') inStr = true;
138+ else if (ch === '{' || ch === '[') depth++;
139+ else if (ch === '}' || ch === ']') {
140+ depth--;
141+ if (depth === 0) return j + 1;
142+ }
143+ }
144+ return -1;
145+ }
146+ if (c === '"') {
147+ for (let j = i + 1; j < s.length; j++) {
148+ if (s[j] === '\\') j++;
149+ else if (s[j] === '"') return j + 1;
150+ }
151+ return -1;
152+ }
153+ const m = s.slice(i).match(/^(true|false|null|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/);
154+ if (!m) return -1;
155+ const end = i + m[0].length;
156+ return end < s.length ? end : -1;
157+}
158+
159+/** Parse <url><loc>…</loc><lastmod>…</lastmod></url> entries from a sitemap. */
160+export function parseSitemap(xml: string): Array<{ loc: string; lastmod: string | null }> {
161+ const out: Array<{ loc: string; lastmod: string | null }> = [];
162+ const re = /<url>([\s\S]*?)<\/url>/g;
163+ let m: RegExpExecArray | null;
164+ while ((m = re.exec(xml))) {
165+ const loc = m[1]!.match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1];
166+ if (!loc) continue;
167+ const lastmod = m[1]!.match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1] ?? null;
168+ out.push({ loc: decodeXml(loc), lastmod });
169+ }
170+ return out;
171+}
172+
173+export function decodeXml(s: string): string {
174+ return s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'");
175+}
176+
177+/** Parse GitHub-flavoured markdown tables (as emitted by Firecrawl) into rows of cells. */
178+export function markdownTables(md: string): string[][][] {
179+ const tables: string[][][] = [];
180+ let cur: string[][] | null = null;
181+ for (const line of md.split('\n')) {
182+ const t = line.trim();
183+ if (t.startsWith('|') && t.endsWith('|')) {
184+ const cells = t.slice(1, -1).split('|').map((c) => c.trim());
185+ if (cells.every((c) => /^:?-{2,}:?$/.test(c))) continue; // separator row
186+ if (!cur) cur = [];
187+ cur.push(cells);
188+ } else if (cur) {
189+ tables.push(cur);
190+ cur = null;
191+ }
192+ }
193+ if (cur) tables.push(cur);
194+ return tables;
195+}
196+
197+/** Strip markdown link syntax: "[2](https://…)" → "2"; "[Alakazam](url) <br>Holo" → "Alakazam Holo". */
198+export function mdText(cell: string): string {
199+ return cell
200+ .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
201+ .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
202+ .replace(/<br\s*\/?>/gi, ' ')
203+ .replace(/\\([\\`*_{}\[\]()#+\-.!|])/g, '$1')
204+ .replace(/\s+/g, ' ')
205+ .trim();
206+}
207+
208+export function mdLink(cell: string): string | null {
209+ return cell.match(/\]\((https?:[^)\s]+)\)/)?.[1] ?? null;
210+}
211+
212+export function toInt(s: string): number | null {
213+ const n = Number.parseInt(s.replace(/[^0-9]/g, ''), 10);
214+ return Number.isFinite(n) ? n : null;
215+}
216+
217+/** TAG / COMC / Alt category labels → taxonomy slugs (never invented; null when unknown). */
218+export function cardCategorySlug(label: string | null | undefined): string | null {
219+ if (!label) return null;
220+ const s = label.toLowerCase();
221+ if (/pok[eé]mon/.test(s)) return 'pokemon';
222+ if (/magic/.test(s)) return 'magic_the_gathering';
223+ if (/yu-?gi-?oh/.test(s)) return 'yugioh';
224+ if (/lorcana/.test(s)) return 'disney_lorcana';
225+ if (/one piece/.test(s)) return 'one_piece_card_game';
226+ if (/digimon/.test(s)) return 'digimon_tcg';
227+ if (/flesh/.test(s)) return 'flesh_and_blood';
228+ if (/star wars/.test(s)) return 'star_wars_tcg';
229+ if (/dragon ?ball/.test(s)) return 'dragon_ball_tcg';
230+ if (/baseball/.test(s)) return 'baseball_cards';
231+ if (/basketball/.test(s)) return 'basketball_cards';
232+ if (/football/.test(s)) return 'football_cards';
233+ if (/hockey/.test(s)) return 'hockey_cards';
234+ if (/soccer|futbol/.test(s)) return 'soccer_cards';
235+ if (/formula|f1\b|racing/.test(s)) return 'f1_cards';
236+ if (/boxing|golf|tennis|wrestling|wwe|ufc|mma|olympic|lacrosse|rugby|cricket|nascar|multi-?sport/.test(s)) return 'other_sports_cards';
237+ if (/comic/.test(s)) return 'comics';
238+ if (/non-?sport|entertainment|garbage pail|marvel|dc /.test(s)) return 'non_sport_cards';
239+ if (/video game|nintendo|playstation/.test(s)) return 'video_games';
240+ if (/tcg|trading card|gaming/.test(s)) return 'trading_cards';
241+ return null;
242+}
243+
244+/** "Aug 6, 2026" | "Sep 3, 2024" → UTC date. */
245+export function parseMonthDay(s: string): Date | null {
246+ const m = s.match(/([A-Za-z]{3,9})\.?\s+(\d{1,2}),?\s+(\d{4})/);
247+ if (!m) return null;
248+ const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
249+ const mo = months.indexOf(m[1]!.slice(0, 3).toLowerCase());
250+ if (mo < 0) return null;
251+ return new Date(Date.UTC(Number(m[3]), mo, Number(m[2])));
252+}
253+
254+// ---- Magic printing treatments (mirrors connectors/api/scryfall so identifier matches agree on `variant`) ----
255+const VARIANT_FRAME_EFFECTS = new Set(['showcase', 'extendedart', 'inverted', 'shatteredglass', 'etched', 'textured']);
256+const VARIANT_PROMO_TYPES = new Set(['serialized', 'prerelease', 'promopack', 'judgegift', 'buyabox', 'gameday', 'textured', 'galaxyfoil', 'surgefoil', 'stepandcompleat', 'confettifoil', 'oilslick', 'halofoil', 'neonink', 'ripplefoil', 'fracturefoil', 'rainbowfoil', 'raisedfoil', 'invisibleink', 'doublerainbow', 'manafoil', 'firstplacefoil', 'dragonscalefoil', 'silverfoil', 'gilded', 'embossed', 'startercollection', 'schinesealtart', 'datestamped', 'playerrewards', 'arenaleague', 'fnm', 'release', 'launch', 'convention', 'mediainsert', 'wizardsplaynetwork', 'thick', 'poster']);
257+const LABELS: Record<string, string> = { extendedart: 'Extended Art', promopack: 'Promo Pack', judgegift: 'Judge Promo', buyabox: 'Buy-a-Box', gameday: 'Game Day', galaxyfoil: 'Galaxy Foil', surgefoil: 'Surge Foil', stepandcompleat: 'Step-and-Compleat', confettifoil: 'Confetti Foil', oilslick: 'Oil Slick', halofoil: 'Halo Foil', neonink: 'Neon Ink', ripplefoil: 'Ripple Foil', fracturefoil: 'Fracture Foil', rainbowfoil: 'Rainbow Foil', raisedfoil: 'Raised Foil', invisibleink: 'Invisible Ink', doublerainbow: 'Double Rainbow', manafoil: 'Mana Foil', firstplacefoil: 'First Place Foil', dragonscalefoil: 'Dragon Scale Foil', silverfoil: 'Silver Foil', startercollection: 'Starter Collection', schinesealtart: 'Chinese Alt Art', datestamped: 'Date Stamped', playerrewards: 'Player Rewards', arenaleague: 'Arena League', fnm: 'FNM', mediainsert: 'Media Insert', wizardsplaynetwork: 'WPN', shatteredglass: 'Shattered Glass' };
258+
259+export function magicTreatments(card: { frameEffects?: string[]; promoTypes?: string[]; isFullArt?: boolean; borderColor?: string; frameVersion?: string; setType?: string | null; releaseDate?: string | null }): string[] {
260+ const t: string[] = [];
261+ for (const fe of card.frameEffects ?? []) if (VARIANT_FRAME_EFFECTS.has(fe)) t.push(fe);
262+ for (const pt of card.promoTypes ?? []) if (VARIANT_PROMO_TYPES.has(pt)) t.push(pt);
263+ if (card.isFullArt) t.push('fullart');
264+ if (card.borderColor === 'borderless') t.push('borderless');
265+ if (card.frameVersion === '1997' && card.setType !== 'core' && card.setType !== 'expansion' && (card.releaseDate ?? '') > '2010') t.push('retro');
266+ return [...new Set(t)].map((k) => LABELS[k] ?? (k === 'fullart' ? 'Full Art' : k.charAt(0).toUpperCase() + k.slice(1)));
267+}
268+
269+export function magicFinishVariant(finish: string): string | null {
270+ if (finish === 'foil') return 'Foil';
271+ if (finish === 'etched') return 'Etched Foil';
272+ return null;
273+}
added connectors/api/cardkingdom/index.test.ts +27 −0
@@ -0,0 +1,27 @@
1+import { describe, expect, it } from 'vitest';
2+import meta from './meta.json' with { type: 'json' };
3+import { localMeta } from '../_lib/local-meta.js';
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector, { isFoil } from './index.js';
6+
7+const connector = createConnector(localMeta(meta));
8+
9+describe('cardkingdom', () => {
10+ runFixtureSuite(connector, it, expect);
11+
12+ it('parses foil flags', () => {
13+ expect(isFoil('true')).toBe(true);
14+ expect(isFoil('false')).toBe(false);
15+ expect(isFoil(undefined)).toBe(false);
16+ });
17+
18+ it('keys singles by scryfall_id and dates observations by the price list stamp', async () => {
19+ const single = listFixtures('cardkingdom').find((n) => n.startsWith('single'))!;
20+ const out = await connector.normalize(loadFixture('cardkingdom', single).raw);
21+ const cat = out.find((r) => r.kind === 'catalog_item');
22+ const obs = out.find((r) => r.kind === 'price_observation');
23+ expect(cat && cat.kind === 'catalog_item' ? cat.attributes.identifiers.scryfall_id : null).toMatch(/^[0-9a-f-]{36}$/);
24+ expect(obs && obs.kind === 'price_observation' ? obs.currency : null).toBe('USD');
25+ expect(obs && obs.kind === 'price_observation' ? (obs.attributes.metadata as { provider: string }).provider : null).toBe('cardkingdom');
26+ });
27+});
added connectors/api/cardkingdom/index.ts +111 −0
@@ -0,0 +1,111 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { type NormalizedRecord } from '@rareindex/shared';
4+import { attrs, catalogItem, makeTitle, num, priceObservation } from '../_lib/shared.js';
5+import { BOT_HEADERS, dayOf } from '../_lib/wave4.js';
6+
7+/** Card Kingdom public price lists → Magic price observations keyed by scryfall_id. */
8+const PARSER_VERSION = '1.0.0';
9+const SINGLES = 'https://api.cardkingdom.com/api/pricelist';
10+const SEALED = 'https://api.cardkingdom.com/api/sealed_pricelist';
11+
12+const SingleSchema = z.object({
13+ id: z.number(),
14+ sku: z.string().nullable().optional(),
15+ scryfall_id: z.string().nullable().optional(),
16+ url: z.string().nullable().optional(),
17+ name: z.string(),
18+ variation: z.string().nullable().optional(),
19+ edition: z.string(),
20+ is_foil: z.union([z.string(), z.boolean()]).optional(),
21+ price_retail: z.union([z.string(), z.number()]).nullable().optional(),
22+ qty_retail: z.number().nullable().optional(),
23+ price_buy: z.union([z.string(), z.number()]).nullable().optional(),
24+ qty_buying: z.number().nullable().optional(),
25+});
26+const SealedSchema = z.object({ id: z.number(), url: z.string().nullable().optional(), name: z.string(), edition: z.string().nullable().optional(), price_retail: z.union([z.string(), z.number()]).nullable().optional(), qty_retail: z.number().nullable().optional(), price_buy: z.union([z.string(), z.number()]).nullable().optional(), qty_buying: z.number().nullable().optional() });
27+const RawPayloadSchema = z.discriminatedUnion('kind', [
28+ z.object({ kind: z.literal('single'), row: SingleSchema, createdAt: z.string().nullable(), baseUrl: z.string() }),
29+ z.object({ kind: z.literal('sealed'), row: SealedSchema, createdAt: z.string().nullable(), baseUrl: z.string() }),
30+]);
31+export type CardKingdomPayload = z.infer<typeof RawPayloadSchema>;
32+
33+export function isFoil(v: string | boolean | undefined): boolean {
34+ return v === true || String(v).toLowerCase() === 'true';
35+}
36+
37+export class CardKingdomConnector extends BaseConnector {
38+ readonly version = '1.0.0';
39+ readonly parserVersion = PARSER_VERSION;
40+
41+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
42+ const minRetail = Number(this.meta.config.minRetailUsd ?? 0);
43+ let count = 0;
44+ const singles = await ctx.fetch(SINGLES, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 180_000 });
45+ let doc = singles.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } | null;
46+ if ((!doc || !Array.isArray(doc.data)) && typeof singles.html === 'string' && singles.html.startsWith('{')) doc = JSON.parse(singles.html) as typeof doc; // served as text/html
47+ if (!singles.success || !doc || !Array.isArray(doc.data)) throw new Error(`cardkingdom pricelist failed: ${singles.error ?? singles.httpStatus}`);
48+ const createdAt = doc.meta?.created_at ?? null;
49+ const baseUrl = doc.meta?.base_url ?? 'https://www.cardkingdom.com/';
50+ for (const r of doc.data) {
51+ const parsed = SingleSchema.safeParse(r);
52+ if (!parsed.success) {
53+ ctx.anomaly('parse_failure_row', parsed.error.issues[0]?.message);
54+ continue;
55+ }
56+ const retail = num(parsed.data.price_retail);
57+ if (retail === null || retail < minRetail) continue;
58+ if (this.reached(ctx, count)) return;
59+ count++;
60+ yield { url: `${baseUrl}${parsed.data.url ?? `mtg/${parsed.data.id}`}`, externalId: `s${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: singles.httpStatus, payload: { kind: 'single', row: parsed.data, createdAt, baseUrl } satisfies CardKingdomPayload, fetchedAt: singles.fetchedAt };
61+ }
62+ if (this.meta.config.includeSealed !== false) {
63+ const sealed = await ctx.fetch(SEALED, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 120_000 });
64+ let sdoc = sealed.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } | null;
65+ if ((!sdoc || !Array.isArray(sdoc.data)) && typeof sealed.html === 'string' && sealed.html.startsWith('{')) sdoc = JSON.parse(sealed.html) as typeof sdoc;
66+ if (sealed.success && sdoc && Array.isArray(sdoc.data)) {
67+ for (const r of sdoc.data) {
68+ const parsed = SealedSchema.safeParse(r);
69+ if (!parsed.success || num(parsed.data.price_retail) === null) continue;
70+ if (this.reached(ctx, count)) return;
71+ count++;
72+ yield { url: `${baseUrl}${parsed.data.url ?? `mtg-sealed/${parsed.data.id}`}`, externalId: `x${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: sealed.httpStatus, payload: { kind: 'sealed', row: parsed.data, createdAt: sdoc.meta?.created_at ?? null, baseUrl } satisfies CardKingdomPayload, fetchedAt: sealed.fetchedAt };
73+ }
74+ } else ctx.anomaly('page_fetch_failed', `sealed_pricelist: ${sealed.error ?? sealed.httpStatus}`);
75+ }
76+ await ctx.setCursor({ completedAt: new Date().toISOString() });
77+ }
78+
79+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
80+ const p = RawPayloadSchema.parse(raw.payload);
81+ const observedAt = raw.fetchedAt;
82+ const created = p.createdAt ? new Date(p.createdAt.replace(' ', 'T') + 'Z') : null;
83+ const observationDate = created && !Number.isNaN(created.getTime()) ? dayOf(created) : dayOf(observedAt);
84+ const out: NormalizedRecord[] = [];
85+ if (p.kind === 'single') {
86+ const r = p.row;
87+ const foil = isFoil(r.is_foil);
88+ const variation = (r.variation ?? '').trim();
89+ const variant = [variation || null, foil ? 'Foil' : null].filter(Boolean).join(' ') || null;
90+ const number = r.sku?.match(/-([A-Za-z0-9]+)$/)?.[1] ?? null;
91+ const identifiers: Record<string, string> = { cardkingdom_id: String(r.id) };
92+ if (r.scryfall_id) identifiers.scryfall_id = r.scryfall_id;
93+ if (r.sku) identifiers.cardkingdom_sku = r.sku;
94+ const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition, name: r.name, number, variant, language: 'English', identifiers, metadata: { finish: foil ? 'foil' : 'nonfoil', variation: variation || null } });
95+ const rawTitle = makeTitle({ name: r.name, set: r.edition, number, variant });
96+ out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION }));
97+ const retail = num(r.price_retail);
98+ if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy), qty_retail: r.qty_retail ?? null, qty_buying: r.qty_buying ?? null } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null }));
99+ return out;
100+ }
101+ const r = p.row;
102+ const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition ?? null, name: r.name, identifiers: { cardkingdom_sealed_id: String(r.id) }, metadata: { sealed: true } });
103+ const rawTitle = makeTitle({ name: r.name, set: r.edition ?? null });
104+ out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.75, parserVersion: PARSER_VERSION }));
105+ const retail = num(r.price_retail);
106+ if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy) } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null }));
107+ return out;
108+ }
109+}
110+
111+export default (meta: ConnectorMeta) => new CardKingdomConnector(meta);
added connectors/api/cardkingdom/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "cardkingdom",
3+ "displayName": "Card Kingdom (retail & buylist prices)",
4+ "sourceId": "cardkingdom",
5+ "sourceName": "Card Kingdom",
6+ "sourceType": "dealer",
7+ "sourceUrl": "https://www.cardkingdom.com",
8+ "module": "api/cardkingdom",
9+ "enginePriority": ["api"],
10+ "categories": ["magic_the_gathering"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": false,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": false,
18+ "supportsCatalog": true,
19+ "supportsPopulation": false,
20+ "supportsLookup": false,
21+ "refreshFrequencyMinutes": 1440,
22+ "priority": "medium",
23+ "trustScore": 0.8,
24+ "attributionRequired": true,
25+ "termsUrl": "https://www.cardkingdom.com/help/terms",
26+ "accessNotes": "Card Kingdom publishes its full singles price list (https://api.cardkingdom.com/api/pricelist, ~46 MB JSON, refreshed daily) and sealed price list (/api/sealed_pricelist) publicly; each singles row carries the Scryfall id, edition, foil flag, retail price, retail stock, buylist price and buylist demand. Fetched once per run with an identifying User-Agent (two requests). Retail = dealer asking price stored as price_observation kind 'market' with metadata.buylist; not a transaction. Identifiers: cardkingdom_id, scryfall_id. Sealed products get catalog items with metadata.sealed=true.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "includeSealed": true,
31+ "minRetailUsd": 0.5
32+ }
33+}
added connectors/api/mtgjson/index.test.ts +58 −0
@@ -0,0 +1,58 @@
1+import { describe, expect, it } from 'vitest';
2+import meta from './meta.json' with { type: 'json' };
3+import { localMeta } from '../_lib/local-meta.js';
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector from './index.js';
6+import { magicTreatments, streamGzipDataEntries } from '../_lib/wave4.js';
7+import { gzipSync } from 'node:zlib';
8+
9+const connector = createConnector(localMeta(meta));
10+
11+describe('mtgjson', () => {
12+ runFixtureSuite(connector, it, expect);
13+
14+ it('emits scryfall-aligned variants and provider-dated observations', async () => {
15+ const names = listFixtures('mtgjson');
16+ const fx = loadFixture('mtgjson', names[0]!);
17+ const out = await connector.normalize(fx.raw);
18+ const cats = out.filter((r) => r.kind === 'catalog_item');
19+ const obs = out.filter((r) => r.kind === 'price_observation');
20+ expect(cats.length).toBeGreaterThanOrEqual(1);
21+ for (const c of cats) {
22+ if (c.kind !== 'catalog_item') continue;
23+ expect(c.attributes.identifiers.mtgjson_uuid).toBeTruthy();
24+ expect(c.attributes.identifiers.scryfall_id).toMatch(/^[0-9a-f-]{36}$/);
25+ expect(c.attributes.setCode).toMatch(/^[A-Z0-9]{2,6}$/);
26+ }
27+ for (const o of obs) {
28+ if (o.kind !== 'price_observation') continue;
29+ expect(['USD', 'EUR']).toContain(o.currency);
30+ expect(o.priceKind).toBe('market');
31+ expect((o.attributes.metadata as { provider: string }).provider).toBeTruthy();
32+ expect(o.observationDate.getTime()).toBeLessThanOrEqual(Date.now());
33+ }
34+ });
35+
36+ it('mirrors the scryfall treatment vocabulary', () => {
37+ expect(magicTreatments({ frameEffects: ['showcase'], promoTypes: [] })).toEqual(['Showcase']);
38+ expect(magicTreatments({ frameEffects: ['extendedart'], borderColor: 'borderless' })).toEqual(['Extended Art', 'Borderless']);
39+ expect(magicTreatments({})).toEqual([]);
40+ });
41+
42+ it('streams top-level data entries from a gzip JSON document', async () => {
43+ const doc = { meta: { date: '2026-09-06' }, data: { a: { paper: { tcgplayer: { retail: { normal: { '2026-09-06': 1.5 } } } } }, b: { paper: {} }, c: { x: [1, 2, { y: '}' }] } } };
44+ const gz = gzipSync(Buffer.from(JSON.stringify(doc)));
45+ const orig = globalThis.fetch;
46+ globalThis.fetch = (async () => new Response(gz, { status: 200 })) as typeof fetch;
47+ try {
48+ const seen: string[] = [];
49+ const n = await streamGzipDataEntries('https://example.test/x.json.gz', (k) => {
50+ seen.push(k);
51+ });
52+ expect(n).toBe(3);
53+ expect(seen).toEqual(['a', 'b', 'c']);
54+ } finally {
55+ globalThis.fetch = orig;
56+ }
57+ });
58+});
added connectors/api/mtgjson/index.ts +219 −0
@@ -0,0 +1,219 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { type NormalizedRecord } from '@rareindex/shared';
4+import { attrs, catalogItem, makeTitle, priceObservation } from '../_lib/shared.js';
5+import { BOT_HEADERS, fetchGzipJson, isoDay, magicFinishVariant, magicTreatments, streamGzipDataEntries } from '../_lib/wave4.js';
6+
7+/**
8+ * MTGJSON connector — per-set card files (identifiers across Scryfall/TCGplayer/Cardmarket/Card Kingdom)
9+ * joined with the daily price file. One raw record per printing; normalize emits catalog items per
10+ * finish and dated price observations per provider × finish.
11+ */
12+const API = 'https://mtgjson.com/api/v5';
13+const PARSER_VERSION = '1.0.0';
14+
15+const SetSchema = z.object({ code: z.string(), name: z.string(), releaseDate: z.string().nullable().optional(), type: z.string().nullable().optional(), tcgplayerGroupId: z.number().nullable().optional(), mcmId: z.number().nullable().optional(), isOnlineOnly: z.boolean().optional(), totalSetSize: z.number().optional() });
16+const CardSchema = z.object({
17+ uuid: z.string(),
18+ name: z.string(),
19+ number: z.string(),
20+ rarity: z.string().nullable().optional(),
21+ finishes: z.array(z.string()).default([]),
22+ language: z.string().nullable().optional(),
23+ identifiers: z.record(z.string(), z.string()).default({}),
24+ frameEffects: z.array(z.string()).optional(),
25+ promoTypes: z.array(z.string()).optional(),
26+ isFullArt: z.boolean().optional(),
27+ borderColor: z.string().optional(),
28+ frameVersion: z.string().optional(),
29+ isPromo: z.boolean().optional(),
30+ isReserved: z.boolean().optional(),
31+ artist: z.string().nullable().optional(),
32+ type: z.string().nullable().optional(),
33+ manaCost: z.string().nullable().optional(),
34+});
35+/** provider → finish → { date: price } */
36+const ProviderPricesSchema = z.object({ retail: z.record(z.string(), z.record(z.string(), z.number())).optional(), buylist: z.record(z.string(), z.record(z.string(), z.number())).optional(), currency: z.string().optional() });
37+const PaperPricesSchema = z.record(z.string(), ProviderPricesSchema);
38+const RawPayloadSchema = z.object({ set: SetSchema, card: CardSchema, prices: PaperPricesSchema.nullable(), priceDate: z.string().nullable() });
39+export type MtgjsonPayload = z.infer<typeof RawPayloadSchema>;
40+
41+type PriceMap = Map<string, z.infer<typeof PaperPricesSchema>>;
42+
43+const CARD_KEEP = ['uuid', 'name', 'number', 'rarity', 'finishes', 'language', 'identifiers', 'frameEffects', 'promoTypes', 'isFullArt', 'borderColor', 'frameVersion', 'isPromo', 'isReserved', 'artist', 'type', 'manaCost'] as const;
44+export function trimCard(c: Record<string, unknown>): z.infer<typeof CardSchema> {
45+ const out: Record<string, unknown> = {};
46+ for (const k of CARD_KEEP) if (c[k] !== undefined) out[k] = c[k];
47+ return CardSchema.parse(out);
48+}
49+export function trimSet(s: Record<string, unknown>): z.infer<typeof SetSchema> {
50+ const out: Record<string, unknown> = {};
51+ for (const k of ['code', 'name', 'releaseDate', 'type', 'tcgplayerGroupId', 'mcmId', 'isOnlineOnly', 'totalSetSize']) if (s[k] !== undefined) out[k] = s[k];
52+ return SetSchema.parse(out);
53+}
54+
55+/** Keep only the latest `days` dates per provider/finish (payload compactness). */
56+function trimPrices(paper: unknown, days: number): z.infer<typeof PaperPricesSchema> | null {
57+ const parsed = PaperPricesSchema.safeParse(paper);
58+ if (!parsed.success) return null;
59+ const out: z.infer<typeof PaperPricesSchema> = {};
60+ for (const [provider, p] of Object.entries(parsed.data)) {
61+ const trimmed: z.infer<typeof ProviderPricesSchema> = { currency: p.currency };
62+ for (const kind of ['retail', 'buylist'] as const) {
63+ const byFinish = p[kind];
64+ if (!byFinish) continue;
65+ const tf: Record<string, Record<string, number>> = {};
66+ for (const [finish, series] of Object.entries(byFinish)) {
67+ const dates = Object.keys(series).sort().slice(-days);
68+ tf[finish] = Object.fromEntries(dates.map((d) => [d, series[d]!]));
69+ }
70+ trimmed[kind] = tf;
71+ }
72+ out[provider] = trimmed;
73+ }
74+ return out;
75+}
76+
77+export class MtgjsonConnector extends BaseConnector {
78+ readonly version = '1.0.0';
79+ readonly parserVersion = PARSER_VERSION;
80+ protected override minIntervalMs = 250;
81+
82+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
83+ const backfill = ctx.options.mode === 'backfill';
84+ const maxSets = backfill ? Infinity : Number(this.meta.config.maxSetsPerRun ?? 60);
85+ const listRes = await ctx.fetch(`${API}/SetList.json`, { engines: ['api'], headers: BOT_HEADERS });
86+ const list = (listRes.json as { data?: unknown[] } | null)?.data;
87+ if (!listRes.success || !Array.isArray(list)) throw new Error(`mtgjson SetList failed: ${listRes.error}`);
88+ let sets = list.map((s) => trimSet(s as Record<string, unknown>)).filter((s) => !s.isOnlineOnly);
89+ if (ctx.options.seeds?.length) sets = sets.filter((s) => ctx.options.seeds!.includes(s.code));
90+ sets.sort((a, b) => (b.releaseDate ?? '').localeCompare(a.releaseDate ?? ''));
91+ sets = sets.slice(0, Number.isFinite(maxSets) ? maxSets : sets.length);
92+
93+ // Daily prices (small) — joined by uuid. Backfill streams the 90-day history instead.
94+ const prices: PriceMap = new Map();
95+ let priceDate: string | null = null;
96+ if (!backfill) {
97+ const today = await fetchGzipJson<{ meta?: { date?: string }; data?: Record<string, { paper?: unknown }> }>(`${API}/AllPricesToday.json.gz`, ctx.signal);
98+ priceDate = today.meta?.date ?? null;
99+ for (const [uuid, v] of Object.entries(today.data ?? {})) {
100+ const t = trimPrices(v.paper, 1);
101+ if (t) prices.set(uuid, t);
102+ }
103+ const s = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });
104+ s.attempts++;
105+ s.success++;
106+ }
107+
108+ let count = 0;
109+ let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);
110+ const cards = new Map<string, { set: z.infer<typeof SetSchema>; card: z.infer<typeof CardSchema>; url: string }>();
111+ for (; setIdx < sets.length; setIdx++) {
112+ if (ctx.signal?.aborted) return;
113+ const set = sets[setIdx]!;
114+ await this.throttle();
115+ const res = await ctx.fetch(`${API}/${set.code}.json`, { engines: ['api'], headers: BOT_HEADERS });
116+ const data = (res.json as { data?: { cards?: unknown[] } } | null)?.data;
117+ if (!res.success || !Array.isArray(data?.cards)) {
118+ ctx.anomaly('page_fetch_failed', `${set.code}: ${res.error ?? res.httpStatus}`);
119+ continue;
120+ }
121+ for (const raw of data.cards) {
122+ let card: z.infer<typeof CardSchema>;
123+ try {
124+ card = trimCard(raw as Record<string, unknown>);
125+ } catch (err) {
126+ ctx.anomaly('parse_failure_card', `${set.code}: ${err instanceof Error ? err.message : String(err)}`);
127+ continue;
128+ }
129+ const url = `https://mtgjson.com/api/v5/${set.code}.json#${card.uuid}`;
130+ if (backfill) {
131+ cards.set(card.uuid, { set, card, url });
132+ continue;
133+ }
134+ if (this.reached(ctx, count)) return;
135+ count++;
136+ const payload: MtgjsonPayload = { set, card, prices: prices.get(card.uuid) ?? null, priceDate };
137+ yield { url, externalId: card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
138+ }
139+ await ctx.setCursor({ setIdx: setIdx + 1, priceDate });
140+ }
141+ if (backfill) {
142+ // Stream the 90-day history and yield one record per known printing.
143+ const days = Number(this.meta.config.historyDays ?? 90);
144+ const queue: RawRecordInput[] = [];
145+ const seen = await streamGzipDataEntries(
146+ `${API}/AllPrices.json.gz`,
147+ (uuid, value) => {
148+ const c = cards.get(uuid);
149+ if (!c) return;
150+ const paper = (value as { paper?: unknown })?.paper;
151+ const trimmed = trimPrices(paper, days);
152+ const payload: MtgjsonPayload = { set: c.set, card: c.card, prices: trimmed, priceDate: null };
153+ queue.push({ url: c.url, externalId: c.card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() });
154+ },
155+ { signal: ctx.signal, limit: ctx.options.limit ? ctx.options.limit * 4 : undefined },
156+ );
157+ ctx.log.info({ seen, matched: queue.length }, 'mtgjson AllPrices streamed');
158+ for (const r of queue) {
159+ if (this.reached(ctx, count)) return;
160+ count++;
161+ yield r;
162+ }
163+ }
164+ await ctx.setCursor({ setIdx: 0, priceDate, completedAt: new Date().toISOString() });
165+ }
166+
167+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
168+ const { set, card, prices } = RawPayloadSchema.parse(raw.payload);
169+ const year = set.releaseDate ? Number(set.releaseDate.slice(0, 4)) || null : null;
170+ const ids: Record<string, string> = { mtgjson_uuid: card.uuid };
171+ const map: Array<[string, string]> = [['scryfallId', 'scryfall_id'], ['scryfallOracleId', 'oracle_id'], ['tcgplayerProductId', 'tcgplayer_id'], ['tcgplayerEtchedProductId', 'tcgplayer_etched_id'], ['mcmId', 'cardmarket_id'], ['cardKingdomId', 'cardkingdom_id'], ['cardKingdomFoilId', 'cardkingdom_foil_id'], ['cardKingdomEtchedId', 'cardkingdom_etched_id'], ['cardsphereId', 'cardsphere_id'], ['multiverseId', 'multiverse_id'], ['mtgoId', 'mtgo_id']];
172+ for (const [from, to] of map) if (card.identifiers[from]) ids[to] = card.identifiers[from]!;
173+ const treat = magicTreatments({ ...card, setType: set.type ?? null, releaseDate: set.releaseDate ?? null });
174+ const baseVariant = treat.length ? treat.join(' ') : null;
175+ const finishes = card.finishes.length ? card.finishes : ['nonfoil'];
176+ const observedAt = raw.fetchedAt;
177+ const providers = new Set((this.meta.config.providers as string[] | undefined) ?? ['tcgplayer', 'cardmarket', 'cardkingdom', 'cardsphere', 'manapool']);
178+ const out: NormalizedRecord[] = [];
179+ for (const finish of finishes) {
180+ const variant = [baseVariant, magicFinishVariant(finish)].filter(Boolean).join(' ') || null;
181+ const a = attrs({
182+ categorySlug: 'magic_the_gathering',
183+ franchise: 'Magic: The Gathering',
184+ brand: 'Wizards of the Coast',
185+ set: set.name,
186+ setCode: set.code.toUpperCase(),
187+ name: card.name,
188+ number: card.number,
189+ year,
190+ variant,
191+ language: card.language ?? 'English',
192+ rarity: card.rarity ?? null,
193+ identifiers: { ...ids, finish },
194+ metadata: { artist: card.artist ?? null, type_line: card.type ?? null, mana_cost: card.manaCost ?? null, promo: card.isPromo ?? false, reserved: card.isReserved ?? false, set_type: set.type ?? null },
195+ });
196+ const rawTitle = makeTitle({ name: card.name, set: set.name, number: card.number, year, variant });
197+ const sourceUrl = `https://scryfall.com/card/${set.code.toLowerCase()}/${encodeURIComponent(card.number)}`;
198+ out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.uuid}:${finish}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: set.releaseDate ? new Date(set.releaseDate) : null }));
199+ if (!prices) continue;
200+ for (const [provider, p] of Object.entries(prices)) {
201+ if (!providers.has(provider)) continue;
202+ // MTGJSON price files key finishes as normal|foil|etched while card.finishes uses nonfoil|foil|etched.
203+ const priceKey = finish === 'nonfoil' ? 'normal' : finish;
204+ const retail = p.retail?.[priceKey];
205+ if (!retail) continue;
206+ const currency = (p.currency ?? (provider === 'cardmarket' ? 'EUR' : 'USD')) as 'USD' | 'EUR';
207+ const buy = p.buylist?.[priceKey];
208+ for (const [date, price] of Object.entries(retail)) {
209+ const observationDate = isoDay(date);
210+ if (!observationDate || !(price > 0)) continue;
211+ out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.uuid}:${finish}:${provider}:${date}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider, buylist: buy?.[date] ?? null } }, observedAt, confidence: provider === 'tcgplayer' || provider === 'cardmarket' ? 0.75 : 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency, observationDate, sampleSize: null }));
212+ }
213+ }
214+ }
215+ return out;
216+ }
217+}
218+
219+export default (meta: ConnectorMeta) => new MtgjsonConnector(meta);
added connectors/api/mtgjson/meta.json +34 −0
@@ -0,0 +1,34 @@
1+{
2+ "id": "mtgjson",
3+ "displayName": "MTGJSON (prices & identifiers)",
4+ "sourceId": "mtgjson",
5+ "sourceName": "MTGJSON",
6+ "sourceType": "pricing_guide",
7+ "sourceUrl": "https://mtgjson.com",
8+ "module": "api/mtgjson",
9+ "enginePriority": ["api"],
10+ "categories": ["magic_the_gathering"],
11+ "regions": ["US", "EU"],
12+ "languages": ["en"],
13+ "currency": ["USD", "EUR"],
14+ "supportsListings": false,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": false,
18+ "supportsCatalog": true,
19+ "supportsPopulation": false,
20+ "supportsLookup": false,
21+ "refreshFrequencyMinutes": 1440,
22+ "priority": "high",
23+ "trustScore": 0.85,
24+ "attributionRequired": true,
25+ "termsUrl": "https://mtgjson.com/faq/",
26+ "accessNotes": "Open MTG data project (free bulk files, no key). Incremental runs download SetList.json, the newest `maxSetsPerRun` set files (LEA.json… with per-card identifiers: scryfallId, tcgplayerProductId, cardKingdomId, mcmId, cardsphereId, multiverseId) and AllPricesToday.json.gz (~5 MB gz) which carries the latest daily retail/buylist price per printing from TCGplayer, Cardmarket (EUR), Card Kingdom, Cardsphere and ManaPool. Backfill streams AllPrices.json.gz (~150 MB gz, 90 days of daily history) entry by entry. Retail prices are stored as price_observations (priceKind 'market', metadata.provider), buylist prices only in metadata; every point is dated by MTGJSON's own date keys. Variant vocabulary mirrors the scryfall connector (Foil / Etched Foil + treatments) so identifier matches agree.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "maxSetsPerRun": 60,
31+ "providers": ["tcgplayer", "cardmarket", "cardkingdom", "cardsphere", "manapool"],
32+ "historyDays": 90
33+ }
34+}
added connectors/api/myslabs/index.test.ts +32 −0
@@ -0,0 +1,32 @@
1+import { describe, expect, it } from 'vitest';
2+import meta from './meta.json' with { type: 'json' };
3+import { localMeta } from '../_lib/local-meta.js';
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector, { slabCategory } from './index.js';
6+
7+const connector = createConnector(localMeta(meta));
8+
9+describe('myslabs', () => {
10+ runFixtureSuite(connector, it, expect);
11+
12+ it('classifies slab titles', () => {
13+ expect(slabCategory('2025 Prizm FIFA Manga Kylian Mbappe BGS 9.5', null)).toBe('soccer_cards');
14+ expect(slabCategory('1999 Pokemon Base Set Charizard Holo PSA 9', null)).toBe('pokemon');
15+ expect(slabCategory('Amazing Spider-Man #300 CGC 9.8 1988', null)).toBe('marvel_comics');
16+ expect(slabCategory('2003 Topps Chrome LeBron James Rookie PSA 10', null)).toBe('basketball_cards');
17+ expect(slabCategory('Random vase', null)).toBeNull();
18+ });
19+
20+ it('records sold-out slabs as sold listings, never as sales', async () => {
21+ for (const name of listFixtures('myslabs')) {
22+ const out = await connector.normalize(loadFixture('myslabs', name).raw);
23+ for (const r of out) {
24+ expect(r.kind).toBe('listing');
25+ if (r.kind === 'listing') {
26+ expect(['available', 'sold', 'unknown']).toContain(r.availability);
27+ expect(r.currency).toBe('USD');
28+ }
29+ }
30+ }
31+ });
32+});
added connectors/api/myslabs/index.ts +164 −0
@@ -0,0 +1,164 @@
1+import { z } from 'zod';
2+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { NormalizedListingSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';
4+import { parseGradeFromTitle } from '@rareindex/taxonomy';
5+import { attrs } from '../_lib/shared.js';
6+import { BOT_HEADERS, cardCategorySlug, decodeXml, parseSitemap } from '../_lib/wave4.js';
7+
8+/** MySlabs — graded slab marketplace; public slab pages expose a Product JSON-LD. */
9+const PARSER_VERSION = '1.0.0';
10+const SITE = 'https://myslabs.com';
11+
12+const RawPayloadSchema = z.object({
13+ id: z.string(),
14+ name: z.string(),
15+ description: z.string().nullable(),
16+ images: z.array(z.string()),
17+ price: z.number().nullable(),
18+ currency: z.string().nullable(),
19+ availability: z.string().nullable(),
20+ category: z.string().nullable(),
21+ seller: z.string().nullable(),
22+});
23+export type MySlabsPayload = z.infer<typeof RawPayloadSchema>;
24+
25+const SPORTS_BRAND = /\b(topps|panini|bowman|prizm|fleer|donruss|upper deck|select|mosaic|optic|chrome|leaf|score|o-pee-chee|hoops|skybox|sp authentic|national treasures|immaculate|flawless|futera|stadium club|goudey|play ball|kellogg)\b/i;
26+const SPORT_WORDS: Array<[RegExp, string]> = [
27+ [/\b(basketball|nba|wnba|hoops|lebron|jordan|curry|wembanyama|kobe|giannis|luka|jokic)\b/i, 'basketball_cards'],
28+ [/\b(baseball|mlb|mantle|ohtani|trout|jeter|bowman|goudey|play ball)\b/i, 'baseball_cards'],
29+ [/\b(football|nfl|mahomes|brady|rookie qb|panini contenders|donruss optic football)\b/i, 'football_cards'],
30+ [/\b(hockey|nhl|gretzky|mcdavid|crosby|o-pee-chee|young guns|bedard)\b/i, 'hockey_cards'],
31+ [/\b(soccer|fifa|messi|ronaldo|mbappe|mbappé|haaland|yamal|premier league|champions league)\b/i, 'soccer_cards'],
32+ [/\b(f1|formula 1|formula one|verstappen|hamilton|leclerc|norris)\b/i, 'f1_cards'],
33+ [/\b(ufc|wwe|wrestling|boxing|golf|tennis|nascar|olympic|mma)\b/i, 'other_sports_cards'],
34+];
35+
36+/** Title → taxonomy slug for MySlabs inventory (cards/comics); null when unsure. */
37+export function slabCategory(title: string, categoryHint: string | null): string | null {
38+ const hinted = cardCategorySlug(categoryHint);
39+ if (hinted && hinted !== 'trading_cards') return hinted;
40+ const tcg = cardCategorySlug(title);
41+ if (tcg && ['pokemon', 'magic_the_gathering', 'yugioh', 'disney_lorcana', 'one_piece_card_game', 'digimon_tcg', 'flesh_and_blood', 'star_wars_tcg', 'dragon_ball_tcg'].includes(tcg)) return tcg;
42+ if ((/\b(cgc|cbcs)\b/i.test(title) && /#\s?\d+/.test(title) || /\bcomic|\b#\d+\s*\(?\d{4}\)?/i.test(title)) && !SPORTS_BRAND.test(title)) return /\bmarvel|spider-man|x-men|avengers|hulk|iron man|captain america|wolverine|daredevil|fantastic four\b/i.test(title) ? 'marvel_comics' : /\bdc\b|batman|superman|detective comics|action comics|wonder woman|flash|green lantern|joker/i.test(title) ? 'dc_comics' : 'independent_comics';
43+ for (const [re, slug] of SPORT_WORDS) if (re.test(title)) return slug;
44+ if (SPORTS_BRAND.test(title)) return 'sports_cards';
45+ if (hinted) return hinted;
46+ return null;
47+}
48+
49+function extractPayload(id: string, htmlText: string): MySlabsPayload | null {
50+ const products = H.jsonLd(htmlText, 'Product');
51+ const p = products[0];
52+ if (!p) return null;
53+ const offers = (Array.isArray(p.offers) ? p.offers[0] : p.offers) as Record<string, unknown> | undefined;
54+ const images = (Array.isArray(p.image) ? p.image : p.image ? [p.image] : []).map((u) => decodeXml(String(u)));
55+ const priceRaw = offers?.price;
56+ const price = priceRaw === undefined || priceRaw === null || priceRaw === '' ? null : Number(priceRaw);
57+ const availability = typeof offers?.availability === 'string' ? offers.availability.replace(/^https?:\/\/schema\.org\//, '') : null;
58+ const seller = (offers?.seller as { name?: string } | undefined)?.name ?? null;
59+ const $ = H.load(htmlText);
60+ const crumbs = $('nav[aria-label="breadcrumb"] a, .breadcrumb a, ol.breadcrumb li')
61+ .map((_, el) => $(el).text().trim())
62+ .get()
63+ .filter(Boolean);
64+ const category = crumbs.find((c) => /card|comic|pok|magic|yu-gi|sport/i.test(c)) ?? null;
65+ return { id, name: String(p.name ?? '').trim(), description: typeof p.description === 'string' ? p.description.trim() : null, images, price: Number.isFinite(price as number) ? (price as number) : null, currency: typeof offers?.priceCurrency === 'string' ? offers.priceCurrency : null, availability, category, seller };
66+}
67+
68+export class MySlabsConnector extends BaseConnector {
69+ readonly version = '1.0.0';
70+ readonly parserVersion = PARSER_VERSION;
71+ override readonly urlPatterns = [/^https?:\/\/(www\.)?myslabs\.com\/slab\/view\/\d+/i];
72+ protected override minIntervalMs = 1500;
73+
74+ private async fetchSlab(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {
75+ await this.throttle();
76+ const res = await ctx.fetch(url, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'text/html' }, responseType: 'text' });
77+ if (!res.success || !res.html) {
78+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
79+ return null;
80+ }
81+ const id = url.match(/slab\/view\/(\d+)/)?.[1] ?? url;
82+ const payload = extractPayload(id, res.html);
83+ if (!payload || !payload.name) {
84+ ctx.anomaly('parse_failure_slab', url);
85+ return null;
86+ }
87+ return { url, externalId: id, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
88+ }
89+
90+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
91+ const max = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillSlabs ?? 2000) : Number(this.meta.config.maxSlabsPerRun ?? 250);
92+ const files = ctx.options.mode === 'backfill' ? ['sitemap-slabs-1.xml', 'sitemap-slabs-2.xml'] : ['sitemap-slabs-1.xml'];
93+ const urls: Array<{ id: number; loc: string }> = [];
94+ for (const f of files) {
95+ const res = await ctx.fetch(`${SITE}/${f}`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml,text/xml' }, responseType: 'text', timeoutMs: 120_000 });
96+ if (!res.success || !res.html) {
97+ ctx.anomaly('page_fetch_failed', `${f}: ${res.error ?? res.httpStatus}`);
98+ continue;
99+ }
100+ for (const e of parseSitemap(res.html)) {
101+ const id = Number(e.loc.match(/slab\/view\/(\d+)/)?.[1]);
102+ if (Number.isFinite(id)) urls.push({ id, loc: e.loc });
103+ }
104+ }
105+ if (ctx.options.seeds?.length) urls.push(...ctx.options.seeds.map((s) => ({ id: Number(s.match(/\d+/)?.[0] ?? 0), loc: s.startsWith('http') ? s : `${SITE}/slab/view/${s}/` })));
106+ urls.sort((a, b) => b.id - a.id);
107+ let count = 0;
108+ for (const u of urls.slice(0, max)) {
109+ if (ctx.signal?.aborted) return;
110+ if (this.reached(ctx, count)) return;
111+ if (!(await ctx.shouldFetch(u.loc))) continue;
112+ const rec = await this.fetchSlab(ctx, u.loc);
113+ if (!rec) continue;
114+ count++;
115+ yield rec;
116+ if (count % 25 === 0) await ctx.setCursor({ lastId: u.id, at: new Date().toISOString() });
117+ }
118+ await ctx.setCursor({ completedAt: new Date().toISOString(), newestId: urls[0]?.id ?? null });
119+ }
120+
121+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
122+ const rec = await this.fetchSlab(ctx, url);
123+ return rec ? [rec] : [];
124+ }
125+
126+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
127+ const p = RawPayloadSchema.parse(raw.payload);
128+ const desc = (p.description ?? '').replace(/<[^>]+>/g, ' ').replace(/&[a-z]+;/g, ' ').slice(0, 400);
129+ const categorySlug = slabCategory(`${p.name} ${desc}`, p.category);
130+ if (!categorySlug) return [];
131+ // Slab pages carry the grade only when the seller typed it (title or description); never guessed.
132+ const g = parseGradeFromTitle(p.name).grader ? parseGradeFromTitle(p.name) : parseGradeFromTitle(desc);
133+ const cert = p.description?.match(/\b(?:cert|certification|serial)\s*(?:#|no\.?|number)?\s*[:\-]?\s*(\d{6,12})\b/i)?.[1] ?? null;
134+ const year = extractYear(p.name);
135+ const number = p.name.match(/#\s?([A-Za-z0-9-]+)/)?.[1] ?? null;
136+ const a = attrs({ categorySlug, name: p.name.replace(/\s*\b(PSA|BGS|CGC|SGC|TAG|CSG|CBCS)\b.*$/i, '').trim() || p.name, year, number, identifiers: { myslabs_id: p.id }, metadata: { seller: p.seller, category_label: p.category } });
137+ const availability = p.availability === 'OutOfStock' || p.availability === 'SoldOut' ? 'sold' : p.availability === 'InStock' || p.availability === 'PreOrder' ? 'available' : 'unknown';
138+ const currency = (p.currency ?? 'USD').toUpperCase();
139+ const listing = NormalizedListingSchema.parse({
140+ kind: 'listing',
141+ connectorId: this.meta.id,
142+ sourceId: this.meta.sourceId,
143+ sourceUrl: raw.url,
144+ externalId: p.id,
145+ rawTitle: p.name,
146+ description: p.description,
147+ imageUrls: p.images,
148+ attributes: a,
149+ grade: { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: cert },
150+ condition: {},
151+ observedAt: raw.fetchedAt,
152+ confidence: 0.75,
153+ parserVersion: PARSER_VERSION,
154+ listingType: 'fixed_price',
155+ price: p.price,
156+ currency: p.price === null ? null : currency,
157+ seller: p.seller,
158+ availability,
159+ });
160+ return [listing];
161+ }
162+}
163+
164+export default (meta: ConnectorMeta) => new MySlabsConnector(meta);
added connectors/api/myslabs/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "myslabs",
3+ "displayName": "MySlabs (graded card & comic marketplace)",
4+ "sourceId": "myslabs",
5+ "sourceName": "MySlabs",
6+ "sourceType": "marketplace",
7+ "sourceUrl": "https://myslabs.com",
8+ "module": "api/myslabs",
9+ "enginePriority": ["api"],
10+ "categories": ["sports_cards", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "pokemon", "magic_the_gathering", "yugioh", "non_sport_cards", "comics", "marvel_comics", "dc_comics", "independent_comics"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": true,
15+ "supportsSold": false,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 720,
22+ "priority": "medium",
23+ "trustScore": 0.7,
24+ "attributionRequired": true,
25+ "termsUrl": "https://myslabs.com/static/terms",
26+ "accessNotes": "Public fixed-price marketplace for graded slabs. robots.txt only disallows /account/, /api/, /admin/, /ajax_select/. Discovery uses the public sitemaps (sitemap-slabs-*.xml, newest ids first); slab pages are plain HTML with a schema.org Product JSON-LD (name, images, offers.price USD, availability InStock/OutOfStock). OutOfStock slabs are stored as listings with availability 'sold' at their last asking price — never as confirmed sales. Politeness 1.5 s between pages; ~250 slabs per incremental run (`maxSlabsPerRun`).",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "maxSlabsPerRun": 250,
31+ "backfillSlabs": 2000
32+ }
33+}
added connectors/firecrawl/alt-xyz/index.test.ts +94 −0
@@ -0,0 +1,94 @@
1+import { describe, expect, it } from 'vitest';
2+import meta from './meta.json' with { type: 'json' };
3+import { localMeta } from '../../api/_lib/local-meta.js';
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector, { parseItemMarkdown } from './index.js';
6+
7+const connector = createConnector(localMeta(meta));
8+
9+const SAMPLE = `Back
10+
11+# 2023 Bowman Chrome Prospect Autograph Purple Refractor Ludwing Espinoza \\#CPALE
12+
13+## Baseball Cards
14+
15+Serial207/250
16+
17+PSA10
18+
19+Pop24
20+
21+## List price
22+
23+### $45
24+
25+Buy nowOffer
26+
27+## LT Value
28+
29+$18
30+
31+### $12 - $29
32+
33+PSA population
34+
35+### Recent transactions
36+
37+[![eBay](https://x/e.png)\\
38+\\
39+AuctionAug 6, 2026\\
40+\\
41+$15](https://www.ebay.com/itm/278232527458)
42+
43+[![eBay](https://x/e.png)\\
44+\\
45+Best offerSep 3, 2024\\
46+\\
47+$20](https://www.ebay.com/itm/405178094963)
48+
49+View all
50+
51+Listings
52+
53+[![eBay](https://x/e.png)\\
54+\\
55+Fixed price\\
56+\\
57+$60\\
58+\\
59+live listing in eBay](https://www.ebay.com/itm/176907976889)
60+`;
61+
62+describe('alt-xyz', () => {
63+ runFixtureSuite(connector, it, expect);
64+
65+ it('parses item markdown', () => {
66+ const p = parseItemMarkdown(SAMPLE, 'abc')!;
67+ expect(p.title).toContain('Ludwing Espinoza #CPALE');
68+ expect(p.category).toBe('Baseball Cards');
69+ expect(p).toMatchObject({ grader: 'psa', grade: '10', pop: 24, listPrice: 45, listingKind: 'fixed_price', altValue: 18, altLow: 12, altHigh: 29, serial: '207/250' });
70+ expect(p.transactions).toEqual([
71+ { type: 'Auction', date: 'Aug 6, 2026', price: 15, url: 'https://www.ebay.com/itm/278232527458' },
72+ { type: 'Best offer', date: 'Sep 3, 2024', price: 20, url: 'https://www.ebay.com/itm/405178094963' },
73+ ]);
74+ });
75+
76+ it('normalises into observation + sales + listing', async () => {
77+ const raw = { url: 'https://alt.xyz/itm/abc', externalId: 'abc', kind: 'sale' as const, engine: 'firecrawl' as const, fetchedAt: new Date('2026-09-07T12:00:00Z'), payload: parseItemMarkdown(SAMPLE, 'abc') };
78+ const out = await connector.normalize(raw);
79+ const kinds = out.map((r) => r.kind);
80+ expect(kinds.filter((k) => k === 'sale')).toHaveLength(2);
81+ expect(kinds).toContain('price_observation');
82+ expect(kinds).toContain('listing');
83+ const sale = out.find((r) => r.kind === 'sale');
84+ if (sale && sale.kind === 'sale') {
85+ expect(sale.saleDate.toISOString()).toBe('2026-08-06T00:00:00.000Z');
86+ expect(sale.attributes.categorySlug).toBe('baseball_cards');
87+ expect(sale.grade.grader).toBe('psa');
88+ }
89+ for (const name of listFixtures('alt-xyz')) {
90+ const fx = await connector.normalize(loadFixture('alt-xyz', name).raw);
91+ expect(fx.length).toBeGreaterThan(0);
92+ }
93+ });
94+});
added connectors/firecrawl/alt-xyz/index.ts +168 −0
@@ -0,0 +1,168 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { NormalizedListingSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';
4+import { attrs, priceObservation } from '../../api/_lib/shared.js';
5+import { BOT_HEADERS, cardCategorySlug, dayOf, fetchMaybeGzip, money, parseMonthDay, parseSitemap } from '../../api/_lib/wave4.js';
6+
7+/** Alt item pages → Alt Value observations, recent transactions (sales) and Alt list-price listings. */
8+const PARSER_VERSION = '1.0.0';
9+const SITE = 'https://alt.xyz';
10+
11+const TxSchema = z.object({ type: z.string(), date: z.string(), price: z.number(), url: z.string().nullable() });
12+const RawPayloadSchema = z.object({
13+ id: z.string(),
14+ title: z.string(),
15+ category: z.string().nullable(),
16+ serial: z.string().nullable(),
17+ grader: z.string().nullable(),
18+ grade: z.string().nullable(),
19+ pop: z.number().nullable(),
20+ listPrice: z.number().nullable(),
21+ listingKind: z.enum(['fixed_price', 'auction']).nullable().default(null),
22+ altValue: z.number().nullable(),
23+ altLow: z.number().nullable(),
24+ altHigh: z.number().nullable(),
25+ transactions: z.array(TxSchema),
26+ images: z.array(z.string()),
27+});
28+export type AltPayload = z.infer<typeof RawPayloadSchema>;
29+
30+/** Parse the Firecrawl markdown of an item page. Exported for tests. */
31+export function parseItemMarkdown(md: string, id: string): AltPayload | null {
32+ const text = md.replace(/\\\n/g, '\n').replace(/\\([#$])/g, '$1');
33+ const title = text.match(/^#\s+(.+)$/m)?.[1]?.trim();
34+ if (!title) return null;
35+ const afterTitle = text.slice(text.indexOf(title) + title.length);
36+ const category = afterTitle.match(/^##\s+([A-Za-z' \-]+Cards?)\s*$/m)?.[1]?.trim() ?? afterTitle.match(/^##\s+(Pok[eé]mon|Magic[^\n]*|Yu-Gi-Oh!?)\s*$/m)?.[1]?.trim() ?? null;
37+ const serial = afterTitle.match(/Serial\s*(\d+\s*\/\s*\d+)/)?.[1]?.replace(/\s/g, '') ?? null;
38+ const gm = afterTitle.match(/^(PSA|BGS|SGC|CGC|CSG|TAG|HGA)\s?(\d{1,2}(?:\.\d)?)\s*$/m);
39+ const pop = afterTitle.match(/^Pop\s*([\d,]+)\s*$/m)?.[1];
40+ const fixed = afterTitle.match(/##\s+List price\s*\n+\s*###\s+\$([\d,]+(?:\.\d+)?)/)?.[1] ?? null;
41+ const bid = afterTitle.match(/(?:Starting bid|Current bid|High bid)\s*\$([\d,]+(?:\.\d+)?)/)?.[1] ?? null;
42+ const listPrice = fixed ?? bid;
43+ const listingKind: 'fixed_price' | 'auction' | null = fixed ? 'fixed_price' : bid ? 'auction' : null;
44+ const av = afterTitle.match(/##\s+(?:LT|Alt) Value\s*\n+\s*\$([\d,]+(?:\.\d+)?)\s*\n+\s*###\s+\$([\d,]+(?:\.\d+)?)\s*-\s*\$([\d,]+(?:\.\d+)?)/);
45+ const txStart = afterTitle.search(/Recent transactions/i);
46+ const txEnd = afterTitle.search(/\n##?\s+(Listings|Similar listings)|\nListings\s*\n/);
47+ const txBlock = txStart >= 0 ? afterTitle.slice(txStart, txEnd > txStart ? txEnd : undefined) : '';
48+ const transactions: z.infer<typeof TxSchema>[] = [];
49+ const re = /(Auction|Best offer|Fixed price|Buy now|Sale)\s*([A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})\s*\n+\s*\$([\d,]+(?:\.\d+)?)\]\((https?:[^)\s]+)\)/g;
50+ let m: RegExpExecArray | null;
51+ while ((m = re.exec(txBlock))) {
52+ const price = money(m[3]);
53+ if (!price) continue;
54+ transactions.push({ type: m[1]!, date: m[2]!, price, url: m[4] ?? null });
55+ }
56+ const images = [...text.matchAll(/!\[[^\]]*\]\((https:\/\/alt-images\.b-cdn\.net\/public\/[^)\s]+width=324[^)\s]*)\)/g)].map((x) => x[1]!);
57+ return {
58+ id,
59+ title,
60+ category,
61+ serial,
62+ grader: gm ? gm[1]!.toLowerCase() : null,
63+ grade: gm ? gm[2]! : null,
64+ pop: pop ? Number(pop.replace(/,/g, '')) : null,
65+ listPrice: money(listPrice),
66+ listingKind,
67+ altValue: av ? money(av[1]) : null,
68+ altLow: av ? money(av[2]) : null,
69+ altHigh: av ? money(av[3]) : null,
70+ transactions,
71+ images: [...new Set(images)],
72+ };
73+}
74+
75+async function fetchGzText(url: string, signal?: AbortSignal): Promise<string> {
76+ return (await fetchMaybeGzip(url, signal)).toString('utf8');
77+}
78+
79+export class AltConnector extends BaseConnector {
80+ readonly version = '1.0.0';
81+ readonly parserVersion = PARSER_VERSION;
82+ override readonly urlPatterns = [/^https?:\/\/(www\.)?alt\.xyz\/itm\/[0-9a-f-]{36}/i];
83+ protected override minIntervalMs = 1500;
84+
85+ private async fetchItem(ctx: CrawlContext, url: string): Promise<RawRecordInput | null> {
86+ await this.throttle();
87+ const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 9000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.match(/^#\s+.+$/m) ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) });
88+ if (!res.success || !res.markdown) {
89+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
90+ return null;
91+ }
92+ const id = url.match(/itm\/([0-9a-f-]{36})/i)?.[1] ?? url;
93+ const payload = parseItemMarkdown(res.markdown, id);
94+ if (!payload) {
95+ ctx.anomaly('parse_failure_item', url);
96+ return null;
97+ }
98+ return { url: `${SITE}/itm/${id}`, externalId: id, kind: 'sale', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
99+ }
100+
101+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
102+ const max = ctx.options.mode === 'backfill' ? Number(this.meta.config.maxItemsPerRun ?? 100) * 5 : Number(this.meta.config.maxItemsPerRun ?? 100);
103+ let urls: Array<{ loc: string; lastmod: string }> = [];
104+ if (ctx.options.seeds?.length) urls = ctx.options.seeds.map((s) => ({ loc: s.startsWith('http') ? s : `${SITE}/itm/${s}`, lastmod: '' }));
105+ else {
106+ const idx = await ctx.fetch(`${SITE}/sitemap.xml`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml' }, responseType: 'text' });
107+ if (!idx.success || !idx.html) throw new Error(`alt sitemap index failed: ${idx.error ?? idx.httpStatus}`);
108+ const wanted = (this.meta.config.sitemaps as string[] | undefined) ?? ['fixed-price', 'auctions'];
109+ const children = [...idx.html.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((x) => x[1]!).filter((u) => wanted.some((w) => u.includes(`/${w}-`)));
110+ for (const child of children.slice(0, 6)) {
111+ try {
112+ const xml = await fetchGzText(child, ctx.signal);
113+ for (const e of parseSitemap(xml)) urls.push({ loc: e.loc, lastmod: e.lastmod ?? '' });
114+ } catch (err) {
115+ ctx.anomaly('page_fetch_failed', `${child}: ${err instanceof Error ? err.message : String(err)}`);
116+ }
117+ }
118+ const since = String(ctx.options.cursor?.since ?? '');
119+ urls = urls.filter((u) => !since || u.lastmod > since).sort((a, b) => b.lastmod.localeCompare(a.lastmod));
120+ }
121+ let count = 0;
122+ let newest = String(ctx.options.cursor?.since ?? '');
123+ for (const u of urls.slice(0, max)) {
124+ if (ctx.signal?.aborted) return;
125+ if (this.reached(ctx, count)) break;
126+ const rec = await this.fetchItem(ctx, u.loc);
127+ if (!rec) continue;
128+ count++;
129+ if (u.lastmod > newest) newest = u.lastmod;
130+ yield rec;
131+ if (count % 20 === 0) await ctx.setCursor({ since: newest, at: new Date().toISOString() });
132+ }
133+ await ctx.setCursor({ since: newest, completedAt: new Date().toISOString() });
134+ }
135+
136+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
137+ const rec = await this.fetchItem(ctx, url);
138+ return rec ? [rec] : [];
139+ }
140+
141+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
142+ const p = RawPayloadSchema.parse(raw.payload);
143+ const categorySlug = cardCategorySlug(p.category) ?? cardCategorySlug(p.title);
144+ if (!categorySlug) return [];
145+ const year = extractYear(p.title);
146+ const number = p.title.match(/#\s?([A-Za-z0-9-]+)/)?.[1] ?? null;
147+ const name = p.title.replace(/^\d{4}(?:-\d{2})?\s+/, '').replace(/\s*#\S+\s*$/, '').trim() || p.title;
148+ const a = attrs({ categorySlug, name, year, number, identifiers: { alt_item_id: p.id }, metadata: { serial: p.serial, pop: p.pop } });
149+ const grade = { grader: p.grader, grade: p.grade, qualifier: null, certificationNumber: null };
150+ const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, rawTitle: p.title, imageUrls: p.images, attributes: a, grade, condition: {}, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
151+ const out: NormalizedRecord[] = [];
152+ if (p.altValue) {
153+ out.push(priceObservation({ kind: 'price_observation', ...base, externalId: `${p.id}:altvalue:${dayOf(raw.fetchedAt).toISOString().slice(0, 10)}`, confidence: 0.6, priceKind: 'guide_value', price: p.altValue, currency: 'USD', observationDate: dayOf(raw.fetchedAt), sampleSize: p.transactions.length || null, attributes: { ...a, metadata: { ...a.metadata, low: p.altLow, high: p.altHigh, model: 'Alt Value' } } }));
154+ }
155+ for (const t of p.transactions) {
156+ const saleDate = parseMonthDay(t.date);
157+ if (!saleDate) continue;
158+ const saleType = /auction/i.test(t.type) ? 'auction' : /best offer/i.test(t.type) ? 'best_offer' : 'fixed_price';
159+ out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, externalId: `${p.id}:${saleDate.toISOString().slice(0, 10)}:${t.price}`, confidence: 0.65, saleType, saleDate, price: t.price, currency: 'USD', buyerPremiumIncluded: null, quantity: 1, isBundle: false, location: null, auctionHouse: t.url?.includes('ebay.') ? 'eBay (via Alt)' : null, lotNumber: null, attributes: { ...a, metadata: { ...a.metadata, external_url: t.url, via: 'alt.xyz' } } }));
160+ }
161+ if (p.listPrice) {
162+ out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: p.id, confidence: 0.75, listingType: p.listingKind ?? 'fixed_price', price: p.listPrice, currency: 'USD', seller: 'Alt marketplace', availability: 'available' }));
163+ }
164+ return out;
165+ }
166+}
167+
168+export default (meta: ConnectorMeta) => new AltConnector(meta);
added connectors/firecrawl/alt-xyz/meta.json +33 −0
@@ -0,0 +1,33 @@
1+{
2+ "id": "alt-xyz",
3+ "displayName": "Alt (graded card values & transactions)",
4+ "sourceId": "alt",
5+ "sourceName": "Alt",
6+ "sourceType": "analytics_provider",
7+ "sourceUrl": "https://alt.xyz",
8+ "module": "firecrawl/alt-xyz",
9+ "enginePriority": ["firecrawl"],
10+ "categories": ["sports_cards", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "pokemon", "magic_the_gathering", "yugioh", "non_sport_cards"],
11+ "regions": ["US"],
12+ "languages": ["en"],
13+ "currency": ["USD"],
14+ "supportsListings": true,
15+ "supportsSold": true,
16+ "supportsAuctions": false,
17+ "supportsImages": true,
18+ "supportsCatalog": false,
19+ "supportsPopulation": false,
20+ "supportsLookup": true,
21+ "refreshFrequencyMinutes": 1440,
22+ "priority": "low",
23+ "trustScore": 0.7,
24+ "attributionRequired": true,
25+ "termsUrl": "https://alt.xyz/terms",
26+ "accessNotes": "Alt's public item pages (robots.txt allows everything; discovery through the public sitemaps with lastmod) show the card, grade, PSA/BGS population, Alt's list price, the 'Alt Value' model estimate with its range, and a 'Recent transactions' list of observed sales (mostly eBay auctions/best offers) with date and price. Pages are client-rendered → Firecrawl, 1 credit per item, capped by `maxItemsPerRun`. Alt Value is stored as a guide_value observation (confidence 0.6); recent transactions become sales attributed to Alt with the original listing URL in metadata (confidence 0.65 — second-hand aggregation); the Alt list price is a fixed-price listing.",
27+ "enabled": true,
28+ "schemaVersion": "1.0",
29+ "config": {
30+ "sitemaps": ["fixed-price", "auctions"],
31+ "maxItemsPerRun": 100
32+ }
33+}
added connectors/firecrawl/comc/index.test.ts +49 −0
@@ -0,0 +1,49 @@
1+import { describe, expect, it } from 'vitest';
2+import meta from './meta.json' with { type: 'json' };
3+import { localMeta } from '../../api/_lib/local-meta.js';
4+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
5+import createConnector, { parseBrowseMarkdown, parseComcTitle, parseSetLine } from './index.js';
6+
7+const connector = createConnector(localMeta(meta));
8+
9+const SAMPLE = `Listings **1** \\- **100** of **1,292**
10+
11+[![Steve Johnson [CSG 10 Gem Mint]](https://img.comc.com/i/Basketball/1986-87/Fleer---Base/55/Steve-Johnson.jpg?id=fc68&size=biggerthumb)](https://www.comc.com/Cards/Basketball/1986-87/Fleer_-_Base/55/Steve_Johnson/1395657/Graded/CSG/10_Gem)
12+
13+1986-87 Fleer - \\[Base\\] #55
14+
15+
16+### [Steve Johnson \\[CSG 10 Gem Mint\\]](https://www.comc.com/Cards/Basketball/1986-87/Fleer_-_Base/55/Steve_Johnson/1395657/Graded/CSG/10_Gem)
17+
18+$1,791.76
19+`;
20+
21+describe('comc', () => {
22+ runFixtureSuite(connector, it, expect);
23+
24+ it('parses browse rows, titles and set lines', () => {
25+ const rows = parseBrowseMarkdown(SAMPLE);
26+ expect(rows).toHaveLength(1);
27+ expect(rows[0]).toMatchObject({ price: 1791.76, setLine: '1986-87 Fleer - [Base] #55' });
28+ expect(parseComcTitle('Steve Johnson [CSG 10 Gem Mint]')).toMatchObject({ name: 'Steve Johnson', grader: 'csg', grade: '10' });
29+ expect(parseComcTitle('Michael Jordan [PSA 9 MINT]')).toMatchObject({ grader: 'psa', grade: '9' });
30+ expect(parseComcTitle('Larry Bird')).toMatchObject({ grader: null, grade: null });
31+ expect(parseSetLine('1986-87 Fleer - [Base] #57')).toEqual({ year: 1986, set: '1986 Fleer', variant: null, number: '57' });
32+ expect(parseSetLine('2018-19 Panini Prizm - [Base] - Silver Prizm #280')).toMatchObject({ year: 2018, number: '280' });
33+ });
34+
35+ it('emits USD listings with card attributes', async () => {
36+ for (const name of listFixtures('comc')) {
37+ const out = await connector.normalize(loadFixture('comc', name).raw);
38+ expect(out.length).toBeGreaterThan(0);
39+ for (const r of out) {
40+ expect(r.kind).toBe('listing');
41+ if (r.kind === 'listing') {
42+ expect(r.currency).toBe('USD');
43+ expect(r.price).toBeGreaterThan(0);
44+ expect(r.sourceUrl).toMatch(/comc\.com\/Cards\//);
45+ }
46+ }
47+ }
48+ });
49+});
added connectors/firecrawl/comc/index.ts +149 −0
@@ -0,0 +1,149 @@
1+import { z } from 'zod';
2+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
3+import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
4+import { attrs } from '../../api/_lib/shared.js';
5+import { cardCategorySlug, money } from '../../api/_lib/wave4.js';
6+
7+/** COMC category browse pages (Firecrawl markdown) → card listings with grades and asking prices. */
8+const PARSER_VERSION = '1.0.0';
9+const SITE = 'https://www.comc.com';
10+
11+const ListingSchema = z.object({ title: z.string(), url: z.string(), image: z.string().nullable(), setLine: z.string().nullable(), price: z.number() });
12+const RawPayloadSchema = z.object({ seed: z.string(), page: z.number(), listings: z.array(ListingSchema) });
13+export type ComcPayload = z.infer<typeof RawPayloadSchema>;
14+
15+/** Parse a rendered browse page. Exported for tests. */
16+export function parseBrowseMarkdown(md: string): z.infer<typeof ListingSchema>[] {
17+ const lines = md.split('\n').map((l) => l.trim());
18+ const out: z.infer<typeof ListingSchema>[] = [];
19+ let setLine: string | null = null;
20+ let image: string | null = null;
21+ for (let i = 0; i < lines.length; i++) {
22+ const l = lines[i]!;
23+ const img = l.match(/^\[!\[[^\]]*\]\((https?:\/\/img\.comc\.com[^)\s]+)\)\]\(https?:\/\/www\.comc\.com\/Cards\/[^)]+\)$/);
24+ if (img) {
25+ image = img[1]!;
26+ continue;
27+ }
28+ if (/^\d{4}(?:-\d{2})?\s.+#\S+$/.test(l.replace(/\\/g, ''))) {
29+ setLine = l.replace(/\\/g, '');
30+ continue;
31+ }
32+ const h = l.match(/^###\s+\[(.+?)\]\((https?:\/\/www\.comc\.com\/Cards\/[^)\s]+)\)\s*$/);
33+ if (!h) continue;
34+ let price: number | null = null;
35+ for (let j = i + 1; j < Math.min(lines.length, i + 5); j++) {
36+ const m = lines[j]!.match(/^\$([\d,]+\.\d{2})$/);
37+ if (m) {
38+ price = money(m[1]);
39+ break;
40+ }
41+ if (lines[j]!.startsWith('###')) break;
42+ }
43+ if (price === null) continue;
44+ out.push({ title: h[1]!.replace(/\\/g, ''), url: h[2]!, image, setLine, price });
45+ image = null;
46+ }
47+ return out;
48+}
49+
50+const GRADE_RE = /\[(PSA|BGS|CSG|CGC|SGC|TAG|HGA|ACE|BVG|GMA|ISA|KSA|MNT|BCCG|PSA\/DNA|BAS|JSA)\s*(\d{1,2}(?:\.\d)?)?\s*([^\]]*)\]/i;
51+
52+export function parseComcTitle(title: string): { name: string; grader: string | null; grade: string | null; qualifier: string | null; auto: boolean } {
53+ const m = title.match(GRADE_RE);
54+ const name = title.replace(/\s*\[[^\]]*\]\s*/g, ' ').replace(/\s+/g, ' ').trim();
55+ if (!m) return { name, grader: null, grade: null, qualifier: null, auto: /\bauto/i.test(title) };
56+ const grader = m[1]!.toLowerCase().replace('/', '_');
57+ const grade = m[2] ?? null;
58+ const qualifier = m[3]?.trim() || null;
59+ const isAuth = /dna|bas|jsa/.test(grader);
60+ return { name, grader: isAuth && !grade ? null : grader, grade, qualifier: isAuth ? `${m[1]} ${qualifier ?? ''}`.trim() : qualifier, auto: /\bauto/i.test(title) };
61+}
62+
63+export function parseSetLine(setLine: string | null): { year: number | null; set: string | null; variant: string | null; number: string | null } {
64+ if (!setLine) return { year: null, set: null, variant: null, number: null };
65+ const m = setLine.match(/^(\d{4})(?:-\d{2})?\s+(.+?)(?:\s+-\s+\[(.+?)\])?\s+#(\S+)$/);
66+ if (!m) return { year: null, set: setLine, variant: null, number: null };
67+ const year = Number(m[1]);
68+ const variant = m[3] && !/^base$/i.test(m[3]) ? m[3] : null;
69+ return { year, set: `${m[1]} ${m[2]}`.trim(), variant, number: m[4] ?? null };
70+}
71+
72+function seedCategory(seed: string): string | null {
73+ const sport = seed.split('/')[1]?.replace(/_/g, ' ') ?? '';
74+ return cardCategorySlug(sport) ?? (/^racing$/i.test(sport) ? 'other_sports_cards' : null);
75+}
76+
77+export class ComcConnector extends BaseConnector {
78+ readonly version = '1.0.0';
79+ readonly parserVersion = PARSER_VERSION;
80+ protected override minIntervalMs = 2000;
81+
82+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
83+ const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] | undefined) ?? []);
84+ const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);
85+ let count = 0;
86+ for (const seed of seeds) {
87+ for (let page = 1; page <= pages; page++) {
88+ if (ctx.signal?.aborted) return;
89+ if (this.reached(ctx, count)) return;
90+ const url = `${SITE}/${seed.replace(/^\/+/, '')},sh${page > 1 ? `,p${page}` : ''}`;
91+ await this.throttle();
92+ const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 3000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.includes('Listings') ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) });
93+ if (!res.success || !res.markdown) {
94+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
95+ break;
96+ }
97+ const listings = parseBrowseMarkdown(res.markdown);
98+ if (!listings.length) {
99+ ctx.anomaly('parse_failure_page', url);
100+ break;
101+ }
102+ count++;
103+ const payload: ComcPayload = { seed, page, listings };
104+ yield { url, externalId: `${seed}:p${page}`, kind: 'listing', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
105+ const total = res.markdown.match(/of\s+\*\*([\d,]+)\*\*/)?.[1];
106+ if (total && Number(total.replace(/,/g, '')) <= page * 100) break;
107+ }
108+ await ctx.setCursor({ seed, at: new Date().toISOString() });
109+ }
110+ }
111+
112+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
113+ const p = RawPayloadSchema.parse(raw.payload);
114+ const categorySlug = seedCategory(p.seed);
115+ if (!categorySlug) return [];
116+ const out: NormalizedRecord[] = [];
117+ for (const l of p.listings) {
118+ const t = parseComcTitle(l.title);
119+ const s = parseSetLine(l.setLine);
120+ const externalId = l.url.replace(`${SITE}/Cards/`, '').replace(/\/+$/, '');
121+ const a = attrs({ categorySlug, name: t.name, set: s.set, year: s.year, number: s.number, variant: s.variant, identifiers: { comc_path: externalId }, metadata: { autograph: t.auto } });
122+ out.push(
123+ NormalizedListingSchema.parse({
124+ kind: 'listing',
125+ connectorId: this.meta.id,
126+ sourceId: this.meta.sourceId,
127+ sourceUrl: l.url,
128+ externalId,
129+ rawTitle: l.setLine ? `${l.setLine} ${l.title}` : l.title,
130+ imageUrls: l.image ? [l.image.replace(/size=biggerthumb/, 'size=large')] : [],
131+ attributes: a,
132+ grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier },
133+ condition: {},
134+ observedAt: raw.fetchedAt,
135+ confidence: 0.8,
136+ parserVersion: PARSER_VERSION,
137+ listingType: 'fixed_price',
138+ price: l.price,
139+ currency: 'USD',
140+ seller: 'COMC consignment',
141+ availability: 'available',
142+ }),
143+ );
144+ }
145+ return out;
146+ }
147+}
148+
149+export default (meta: ConnectorMeta) => new ComcConnector(meta);
added connectors/firecrawl/comc/meta.json +48 −0
Large diff (48 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/tag-pop/index.test.ts +43 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/tag-pop/index.ts +148 −0
Large diff (148 lines) is not loaded by default to keep the page fast.Load diff
added connectors/firecrawl/tag-pop/meta.json +33 −0
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/alt-xyz/item-priced.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/cardkingdom/single-first.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/cardkingdom/single-fourth.json +42 −0
Large diff (43 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/comc/basketball-1986-fleer-p1.json +427 −0
Large diff (428 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/mtgjson/lea-black-lotus.json +91 −0
Large diff (92 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/mtgjson/lea-first.json +89 −0
Large diff (90 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/myslabs/slab-first.json +36 −0
Large diff (37 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/myslabs/slab-sixth.json +36 −0
Large diff (37 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/myslabs/slab-third.json +36 −0
Large diff (37 lines) is not loaded by default to keep the page fast.Load diff
added data/fixtures/tag-pop/pokemon-1999-base-set.json +1971 −0
Large diff (1,973 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere