Connectors: MTGJSON, Card Kingdom, MySlabs, TAG population, COMC, Alt (agent S)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 2 mo ago (Sep 7, 2026) parent 45b94aa Browse files

30 changed files +4,660 −0

added connectors/api/_lib/capture-s.ts +99 −0

		`@@ -0,0 +1,99 @@`
	1	+/**
	2	+ * Live capture + smoke for the wave-4 connectors (mtgjson, cardkingdom, myslabs, tag-pop, comc, alt-xyz).
	3	+ * Usage: pnpm tsx connectors/api/_lib/capture-s.ts [ids…] [--no-save]
	4	+ */
	5	+import { createCrawlContext, createRouter, type RareIndexConnector, type RawRecordInput } from '@rareindex/connectors';
	6	+import { localMeta } from './local-meta.js';
	7	+import { saveFixture } from '@rareindex/connectors/testing';
	8	+import { childLogger } from '@rareindex/shared';
	9	+
	10	+interface Plan {
	11	+ dir: string;
	12	+ limit: number;
	13	+ seeds?: string[];
	14	+ pick: (raw: RawRecordInput, i: number) => string \| null;
	15	+ kinds: string[];
	16	+ requiredFields: string[];
	17	+}
	18	+
	19	+const picked = new Set<string>();
	20	+const PLANS: Record<string, Plan> = {
	21	+ mtgjson: {
	22	+ dir: 'api',
	23	+ limit: 320,
	24	+ seeds: ['LEA'],
	25	+ pick: (raw, i) => {
	26	+ const p = raw.payload as { card: { name: string }; prices: unknown };
	27	+ if (p.card.name === 'Black Lotus') return 'lea-black-lotus';
	28	+ if (i === 0) return 'lea-first';
	29	+ return null;
	30	+ },
	31	+ kinds: ['catalog_item', 'price_observation'],
	32	+ requiredFields: ['attributes.identifiers.scryfall_id', 'attributes.setCode'],
	33	+ },
	34	+ cardkingdom: { dir: 'api', limit: 6, pick: (raw, i) => (i === 0 ? 'single-first' : i === 3 ? 'single-fourth' : null), kinds: ['catalog_item', 'price_observation'], requiredFields: ['attributes.identifiers.cardkingdom_id', 'attributes.set'] },
	35	+ myslabs: { dir: 'api', limit: 6, pick: (raw, i) => (i === 0 ? 'slab-first' : i === 2 ? 'slab-third' : i === 5 ? 'slab-sixth' : null), kinds: ['listing'], requiredFields: ['attributes.identifiers.myslabs_id', 'price'] },
	36	+ 'tag-pop': { dir: 'firecrawl', limit: 1, seeds: ['https://my.taggrading.com/pop-report/Pokemon/1999/WOTC%20Pok%C3%A9mon?setName=Base%20Set'], pick: () => 'pokemon-1999-base-set', kinds: ['population_report'], requiredFields: ['attributes.set', 'attributes.number'] },
	37	+ comc: { dir: 'firecrawl', limit: 1, seeds: ['Cards/Basketball/1986/Fleer'], pick: () => 'basketball-1986-fleer-p1', kinds: ['listing'], requiredFields: ['attributes.set', 'price'] },
	38	+ 'alt-xyz': {
	39	+ dir: 'firecrawl',
	40	+ limit: 8,
	41	+ pick: (raw) => {
	42	+ const p = raw.payload as { transactions: unknown[]; altValue: number \| null; listPrice: number \| null };
	43	+ if (p.transactions.length && !picked.has('item-with-transactions')) return 'item-with-transactions';
	44	+ if ((p.altValue \|\| p.listPrice) && !picked.has('item-priced')) return 'item-priced';
	45	+ return null;
	46	+ },
	47	+ kinds: ['sale', 'price_observation', 'listing'],
	48	+ requiredFields: ['attributes.identifiers.alt_item_id'],
	49	+ },
	50	+};
	51	+
	52	+const args = process.argv.slice(2);
	53	+const noSave = args.includes('--no-save');
	54	+const ids = args.filter((a) => !a.startsWith('--'));
	55	+const targets = ids.length ? ids : Object.keys(PLANS);
	56	+const router = createRouter({ firecrawlApiKey: process.env.FIRECRAWL_API_KEY, scrapflyApiKey: process.env.SCRAPFLY_API_KEY });
	57	+for (const id of targets) {
	58	+ const plan = PLANS[id];
	59	+ if (!plan) {
	60	+ console.log(`[${id}] no plan`);
	61	+ continue;
	62	+ }
	63	+ const started = Date.now();
	64	+ const mod = (await import(`../../${plan.dir}/${id}/index.ts`)) as { default: (m: ReturnType<typeof localMeta>) => RareIndexConnector };
	65	+ const metaJson = (await import(`../../${plan.dir}/${id}/meta.json`, { with: { type: 'json' } })) as { default: unknown };
	66	+ const connector = mod.default(localMeta(metaJson.default));
	67	+ const ctx = createCrawlContext({ router, meta: connector.meta, options: { mode: 'probe', limit: plan.limit, ...(plan.seeds ? { seeds: plan.seeds } : {}) }, log: childLogger({ connector: id, level: 'warn' }) });
	68	+ let raws = 0;
	69	+ let normalized = 0;
	70	+ let printed = 0;
	71	+ let saved = 0;
	72	+ const kinds: Record<string, number> = {};
	73	+ try {
	74	+ for await (const raw of connector.crawl(ctx)) {
	75	+ const rawLike = { ...raw, externalId: raw.externalId ?? null, fetchedAt: raw.fetchedAt ?? new Date() };
	76	+ const out = await connector.normalize(rawLike);
	77	+ normalized += out.length;
	78	+ for (const r of out) kinds[r.kind] = (kinds[r.kind] ?? 0) + 1;
	79	+ const name = plan.pick(raw, raws);
	80	+ raws++;
	81	+ if (name && !noSave) {
	82	+ picked.add(name);
	83	+ saveFixture(id, name, { raw: rawLike, expect: { minCount: 1, kinds: plan.kinds, requiredFields: plan.requiredFields }, note: `Live capture ${new Date().toISOString().slice(0, 10)} from ${raw.url}` });
	84	+ saved++;
	85	+ }
	86	+ if (printed < 3 && out.length) {
	87	+ printed++;
	88	+ const r = out.find((x) => x.kind === 'sale') ?? out.find((x) => x.kind === 'price_observation') ?? out.find((x) => x.kind === 'population_report') ?? out[0]!;
	89	+ const s = r as unknown as Record<string, unknown>;
	90	+ const attrs = (s.attributes ?? {}) as Record<string, unknown>;
	91	+ console.log(`[${id}] ${r.kind} · ${String(s.rawTitle ?? attrs.name ?? '')} · ${String(attrs.categorySlug)} · set=${String(attrs.set)} #${String(attrs.number)} · grade=${JSON.stringify(s.grade ?? null)} · price=${String(s.price ?? s.total ?? '')} ${String(s.currency ?? '')} · date=${String(s.saleDate ?? s.observationDate ?? s.reportDate ?? '')}`);
	92	+ }
	93	+ }
	94	+ } catch (err) {
	95	+ console.log(`[${id}] crawl error: ${err instanceof Error ? err.message : String(err)}`);
	96	+ }
	97	+ console.log(`[${id}] raw=${raws} normalized=${normalized} kinds=${JSON.stringify(kinds)} saved=${saved} anomalies=${ctx.anomalies.length} engines=${JSON.stringify(ctx.engineStats)} ms=${Date.now() - started}`);
	98	+ if (ctx.anomalies.length) console.log(`[${id}] anomalies: ${ctx.anomalies.slice(0, 5).join(' ; ')}`);
	99	+}

added connectors/api/_lib/wave4.ts +273 −0

		`@@ -0,0 +1,273 @@`
	1	+/**
	2	+ * Helpers shared by the wave-4 card/comic connectors (mtgjson, cardkingdom, myslabs, tag-pop, comc,
	3	+ * alt-xyz). Kept inside connectors/api (not the framework).
	4	+ */
	5	+import { createGunzip, gunzipSync } from 'node:zlib';
	6	+import { Readable } from 'node:stream';
	7	+
	8	+export const BOT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)';
	9	+export const BOT_HEADERS = { 'user-agent': BOT_UA, accept: 'application/json, text/html;q=0.9, /;q=0.8' };
	10	+
	11	+/** UTC midnight. */
	12	+export function dayOf(d: Date): Date {
	13	+ return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
	14	+}
	15	+
	16	+/** "2026-09-06" → UTC midnight Date. */
	17	+export function isoDay(s: string \| null \| undefined): Date \| null {
	18	+ if (!s) return null;
	19	+ const m = s.match(/^(\d{4})-(\d{2})-(\d{2})/);
	20	+ if (!m) return null;
	21	+ const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])));
	22	+ return Number.isNaN(d.getTime()) ? null : d;
	23	+}
	24	+
	25	+/** "$8,830.00" \| "8000.00" → number or null (never ≤ 0). */
	26	+export function money(s: string \| number \| null \| undefined): number \| null {
	27	+ if (s === null \|\| s === undefined) return null;
	28	+ const n = typeof s === 'number' ? s : Number.parseFloat(String(s).replace(/[^0-9.]/g, ''));
	29	+ return Number.isFinite(n) && n > 0 ? n : null;
	30	+}
	31	+
	32	+/** Download a gzip JSON document fully (small files only, e.g. AllPricesToday ≈ 5 MB gz). */
	33	+/** Download a (possibly gzip) document; handles servers that already transparently decode Content-Encoding. */
	34	+export async function fetchMaybeGzip(url: string, signal?: AbortSignal): Promise<Buffer> {
	35	+ const res = await fetch(url, { headers: BOT_HEADERS, signal });
	36	+ if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);
	37	+ const buf = Buffer.from(await res.arrayBuffer());
	38	+ if (buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b) return gunzipSync(buf);
	39	+ return buf;
	40	+}
	41	+
	42	+export async function fetchGzipJson<T = unknown>(url: string, signal?: AbortSignal): Promise<T> {
	43	+ return JSON.parse((await fetchMaybeGzip(url, signal)).toString('utf8')) as T;
	44	+}
	45	+
	46	+/**
	47	+ * Stream a large gzip JSON document shaped like {"meta":{…},"data":{"<key>":<value>,…}} and invoke
	48	+ * `onEntry(key, value)` for each top-level entry of `data` without materialising the whole file
	49	+ * (MTGJSON AllPrices ≈ 1.5 GB uncompressed). Returns the number of entries seen.
	50	+ */
	51	+export async function streamGzipDataEntries(url: string, onEntry: (key: string, value: unknown) => Promise<void> \| void, opts: { signal?: AbortSignal; limit?: number } = {}): Promise<number> {
	52	+ const res = await fetch(url, { headers: BOT_HEADERS, signal: opts.signal });
	53	+ if (!res.ok \|\| !res.body) throw new Error(`${url}: HTTP ${res.status}`);
	54	+ const encoded = (res.headers.get('content-type') ?? '').includes('gzip') \|\| /\.gz(\?\|$)/.test(url);
	55	+ const raw = Readable.fromWeb(res.body as never);
	56	+ // Some CDNs serve .gz files with Content-Encoding: gzip, which fetch already decodes → sniff the magic bytes.
	57	+ const first = await new Promise<Buffer \| null>((resolve, reject) => {
	58	+ raw.once('readable', () => resolve((raw.read(2) as Buffer \| null) ?? null));
	59	+ raw.once('error', reject);
	60	+ raw.once('end', () => resolve(null));
	61	+ });
	62	+ if (first) raw.unshift(first);
	63	+ const isGz = first ? first[0] === 0x1f && first[1] === 0x8b : encoded;
	64	+ const stream = isGz ? raw.pipe(createGunzip()) : raw;
	65	+ let buf = '';
	66	+ let inData = false;
	67	+ let count = 0;
	68	+ let done = false;
	69	+ for await (const chunk of stream) {
	70	+ if (done) break;
	71	+ buf += (chunk as Buffer).toString('utf8');
	72	+ if (!inData) {
	73	+ const i = buf.indexOf('"data"');
	74	+ if (i < 0) {
	75	+ buf = buf.slice(-16);
	76	+ continue;
	77	+ }
	78	+ const brace = buf.indexOf('{', i);
	79	+ if (brace < 0) continue;
	80	+ buf = buf.slice(brace + 1);
	81	+ inData = true;
	82	+ }
	83	+ // Parse as many complete `"key": {…},` entries as the buffer holds.
	84	+ for (;;) {
	85	+ const k = buf.indexOf('"');
	86	+ if (k < 0) break;
	87	+ const kEnd = buf.indexOf('"', k + 1);
	88	+ if (kEnd < 0) break;
	89	+ const colon = buf.indexOf(':', kEnd);
	90	+ if (colon < 0) break;
	91	+ const start = colon + 1;
	92	+ const end = scanJsonValue(buf, start);
	93	+ if (end < 0) break; // incomplete value, wait for more data
	94	+ const key = buf.slice(k + 1, kEnd);
	95	+ const raw = buf.slice(start, end).trim();
	96	+ try {
	97	+ await onEntry(key, JSON.parse(raw));
	98	+ } catch (err) {
	99	+ if (err instanceof SyntaxError) {
	100	+ /* skip malformed slice */
	101	+ } else throw err;
	102	+ }
	103	+ count++;
	104	+ buf = buf.slice(end);
	105	+ const comma = buf.indexOf(',');
	106	+ const close = buf.indexOf('}');
	107	+ if (close >= 0 && (comma < 0 \|\| close < comma)) {
	108	+ done = true;
	109	+ break;
	110	+ }
	111	+ buf = comma >= 0 ? buf.slice(comma + 1) : buf;
	112	+ if (opts.limit && count >= opts.limit) {
	113	+ done = true;
	114	+ break;
	115	+ }
	116	+ }
	117	+ }
	118	+ return count;
	119	+}
	120	+
	121	+/** Index just past a complete JSON value starting at `start` (whitespace allowed); -1 if incomplete. */
	122	+function scanJsonValue(s: string, start: number): number {
	123	+ let i = start;
	124	+ while (i < s.length && /\s/.test(s[i]!)) i++;
	125	+ if (i >= s.length) return -1;
	126	+ const c = s[i]!;
	127	+ if (c === '{' \|\| c === '[') {
	128	+ let depth = 0;
	129	+ let inStr = false;
	130	+ for (let j = i; j < s.length; j++) {
	131	+ const ch = s[j]!;
	132	+ if (inStr) {
	133	+ if (ch === '\\') j++;
	134	+ else if (ch === '"') inStr = false;
	135	+ continue;
	136	+ }
	137	+ if (ch === '"') inStr = true;
	138	+ else if (ch === '{' \|\| ch === '[') depth++;
	139	+ else if (ch === '}' \|\| ch === ']') {
	140	+ depth--;
	141	+ if (depth === 0) return j + 1;
	142	+ }
	143	+ }
	144	+ return -1;
	145	+ }
	146	+ if (c === '"') {
	147	+ for (let j = i + 1; j < s.length; j++) {
	148	+ if (s[j] === '\\') j++;
	149	+ else if (s[j] === '"') return j + 1;
	150	+ }
	151	+ return -1;
	152	+ }
	153	+ const m = s.slice(i).match(/^(true\|false\|null\|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/);
	154	+ if (!m) return -1;
	155	+ const end = i + m[0].length;
	156	+ return end < s.length ? end : -1;
	157	+}
	158	+
	159	+/** Parse <url><loc>…</loc><lastmod>…</lastmod></url> entries from a sitemap. */
	160	+export function parseSitemap(xml: string): Array<{ loc: string; lastmod: string \| null }> {
	161	+ const out: Array<{ loc: string; lastmod: string \| null }> = [];
	162	+ const re = /<url>([\s\S]*?)<\/url>/g;
	163	+ let m: RegExpExecArray \| null;
	164	+ while ((m = re.exec(xml))) {
	165	+ const loc = m[1]!.match(/<loc>\s([^<\s]+)\s<\/loc>/)?.[1];
	166	+ if (!loc) continue;
	167	+ const lastmod = m[1]!.match(/<lastmod>\s([^<\s]+)\s<\/lastmod>/)?.[1] ?? null;
	168	+ out.push({ loc: decodeXml(loc), lastmod });
	169	+ }
	170	+ return out;
	171	+}
	172	+
	173	+export function decodeXml(s: string): string {
	174	+ return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'");
	175	+}
	176	+
	177	+/** Parse GitHub-flavoured markdown tables (as emitted by Firecrawl) into rows of cells. */
	178	+export function markdownTables(md: string): string[][][] {
	179	+ const tables: string[][][] = [];
	180	+ let cur: string[][] \| null = null;
	181	+ for (const line of md.split('\n')) {
	182	+ const t = line.trim();
	183	+ if (t.startsWith('\|') && t.endsWith('\|')) {
	184	+ const cells = t.slice(1, -1).split('\|').map((c) => c.trim());
	185	+ if (cells.every((c) => /^:?-{2,}:?$/.test(c))) continue; // separator row
	186	+ if (!cur) cur = [];
	187	+ cur.push(cells);
	188	+ } else if (cur) {
	189	+ tables.push(cur);
	190	+ cur = null;
	191	+ }
	192	+ }
	193	+ if (cur) tables.push(cur);
	194	+ return tables;
	195	+}
	196	+
	197	+/** Strip markdown link syntax: "[2](https://…)" → "2"; "[Alakazam](url) <br>Holo" → "Alakazam Holo". */
	198	+export function mdText(cell: string): string {
	199	+ return cell
	200	+ .replace(/!\[[^\]]\]$[^)]$/g, '')
	201	+ .replace(/\[([^\]])\]$[^)]$/g, '$1')
	202	+ .replace(/<br\s*\/?>/gi, ' ')
	203	+ .replace(/\\([\\`*_{}\[\]()#+\-.!\|])/g, '$1')
	204	+ .replace(/\s+/g, ' ')
	205	+ .trim();
	206	+}
	207	+
	208	+export function mdLink(cell: string): string \| null {
	209	+ return cell.match(/\]$(https?:[^)\s]+)$/)?.[1] ?? null;
	210	+}
	211	+
	212	+export function toInt(s: string): number \| null {
	213	+ const n = Number.parseInt(s.replace(/[^0-9]/g, ''), 10);
	214	+ return Number.isFinite(n) ? n : null;
	215	+}
	216	+
	217	+/** TAG / COMC / Alt category labels → taxonomy slugs (never invented; null when unknown). */
	218	+export function cardCategorySlug(label: string \| null \| undefined): string \| null {
	219	+ if (!label) return null;
	220	+ const s = label.toLowerCase();
	221	+ if (/pok[eé]mon/.test(s)) return 'pokemon';
	222	+ if (/magic/.test(s)) return 'magic_the_gathering';
	223	+ if (/yu-?gi-?oh/.test(s)) return 'yugioh';
	224	+ if (/lorcana/.test(s)) return 'disney_lorcana';
	225	+ if (/one piece/.test(s)) return 'one_piece_card_game';
	226	+ if (/digimon/.test(s)) return 'digimon_tcg';
	227	+ if (/flesh/.test(s)) return 'flesh_and_blood';
	228	+ if (/star wars/.test(s)) return 'star_wars_tcg';
	229	+ if (/dragon ?ball/.test(s)) return 'dragon_ball_tcg';
	230	+ if (/baseball/.test(s)) return 'baseball_cards';
	231	+ if (/basketball/.test(s)) return 'basketball_cards';
	232	+ if (/football/.test(s)) return 'football_cards';
	233	+ if (/hockey/.test(s)) return 'hockey_cards';
	234	+ if (/soccer\|futbol/.test(s)) return 'soccer_cards';
	235	+ if (/formula\|f1\b\|racing/.test(s)) return 'f1_cards';
	236	+ if (/boxing\|golf\|tennis\|wrestling\|wwe\|ufc\|mma\|olympic\|lacrosse\|rugby\|cricket\|nascar\|multi-?sport/.test(s)) return 'other_sports_cards';
	237	+ if (/comic/.test(s)) return 'comics';
	238	+ if (/non-?sport\|entertainment\|garbage pail\|marvel\|dc /.test(s)) return 'non_sport_cards';
	239	+ if (/video game\|nintendo\|playstation/.test(s)) return 'video_games';
	240	+ if (/tcg\|trading card\|gaming/.test(s)) return 'trading_cards';
	241	+ return null;
	242	+}
	243	+
	244	+/** "Aug 6, 2026" \| "Sep 3, 2024" → UTC date. */
	245	+export function parseMonthDay(s: string): Date \| null {
	246	+ const m = s.match(/([A-Za-z]{3,9})\.?\s+(\d{1,2}),?\s+(\d{4})/);
	247	+ if (!m) return null;
	248	+ const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
	249	+ const mo = months.indexOf(m[1]!.slice(0, 3).toLowerCase());
	250	+ if (mo < 0) return null;
	251	+ return new Date(Date.UTC(Number(m[3]), mo, Number(m[2])));
	252	+}
	253	+
	254	+// ---- Magic printing treatments (mirrors connectors/api/scryfall so identifier matches agree on `variant`) ----
	255	+const VARIANT_FRAME_EFFECTS = new Set(['showcase', 'extendedart', 'inverted', 'shatteredglass', 'etched', 'textured']);
	256	+const VARIANT_PROMO_TYPES = new Set(['serialized', 'prerelease', 'promopack', 'judgegift', 'buyabox', 'gameday', 'textured', 'galaxyfoil', 'surgefoil', 'stepandcompleat', 'confettifoil', 'oilslick', 'halofoil', 'neonink', 'ripplefoil', 'fracturefoil', 'rainbowfoil', 'raisedfoil', 'invisibleink', 'doublerainbow', 'manafoil', 'firstplacefoil', 'dragonscalefoil', 'silverfoil', 'gilded', 'embossed', 'startercollection', 'schinesealtart', 'datestamped', 'playerrewards', 'arenaleague', 'fnm', 'release', 'launch', 'convention', 'mediainsert', 'wizardsplaynetwork', 'thick', 'poster']);
	257	+const LABELS: Record<string, string> = { extendedart: 'Extended Art', promopack: 'Promo Pack', judgegift: 'Judge Promo', buyabox: 'Buy-a-Box', gameday: 'Game Day', galaxyfoil: 'Galaxy Foil', surgefoil: 'Surge Foil', stepandcompleat: 'Step-and-Compleat', confettifoil: 'Confetti Foil', oilslick: 'Oil Slick', halofoil: 'Halo Foil', neonink: 'Neon Ink', ripplefoil: 'Ripple Foil', fracturefoil: 'Fracture Foil', rainbowfoil: 'Rainbow Foil', raisedfoil: 'Raised Foil', invisibleink: 'Invisible Ink', doublerainbow: 'Double Rainbow', manafoil: 'Mana Foil', firstplacefoil: 'First Place Foil', dragonscalefoil: 'Dragon Scale Foil', silverfoil: 'Silver Foil', startercollection: 'Starter Collection', schinesealtart: 'Chinese Alt Art', datestamped: 'Date Stamped', playerrewards: 'Player Rewards', arenaleague: 'Arena League', fnm: 'FNM', mediainsert: 'Media Insert', wizardsplaynetwork: 'WPN', shatteredglass: 'Shattered Glass' };
	258	+
	259	+export function magicTreatments(card: { frameEffects?: string[]; promoTypes?: string[]; isFullArt?: boolean; borderColor?: string; frameVersion?: string; setType?: string \| null; releaseDate?: string \| null }): string[] {
	260	+ const t: string[] = [];
	261	+ for (const fe of card.frameEffects ?? []) if (VARIANT_FRAME_EFFECTS.has(fe)) t.push(fe);
	262	+ for (const pt of card.promoTypes ?? []) if (VARIANT_PROMO_TYPES.has(pt)) t.push(pt);
	263	+ if (card.isFullArt) t.push('fullart');
	264	+ if (card.borderColor === 'borderless') t.push('borderless');
	265	+ if (card.frameVersion === '1997' && card.setType !== 'core' && card.setType !== 'expansion' && (card.releaseDate ?? '') > '2010') t.push('retro');
	266	+ return [...new Set(t)].map((k) => LABELS[k] ?? (k === 'fullart' ? 'Full Art' : k.charAt(0).toUpperCase() + k.slice(1)));
	267	+}
	268	+
	269	+export function magicFinishVariant(finish: string): string \| null {
	270	+ if (finish === 'foil') return 'Foil';
	271	+ if (finish === 'etched') return 'Etched Foil';
	272	+ return null;
	273	+}

added connectors/api/cardkingdom/index.test.ts +27 −0

		`@@ -0,0 +1,27 @@`
	1	+import { describe, expect, it } from 'vitest';
	2	+import meta from './meta.json' with { type: 'json' };
	3	+import { localMeta } from '../_lib/local-meta.js';
	4	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	5	+import createConnector, { isFoil } from './index.js';
	6	+
	7	+const connector = createConnector(localMeta(meta));
	8	+
	9	+describe('cardkingdom', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('parses foil flags', () => {
	13	+ expect(isFoil('true')).toBe(true);
	14	+ expect(isFoil('false')).toBe(false);
	15	+ expect(isFoil(undefined)).toBe(false);
	16	+ });
	17	+
	18	+ it('keys singles by scryfall_id and dates observations by the price list stamp', async () => {
	19	+ const single = listFixtures('cardkingdom').find((n) => n.startsWith('single'))!;
	20	+ const out = await connector.normalize(loadFixture('cardkingdom', single).raw);
	21	+ const cat = out.find((r) => r.kind === 'catalog_item');
	22	+ const obs = out.find((r) => r.kind === 'price_observation');
	23	+ expect(cat && cat.kind === 'catalog_item' ? cat.attributes.identifiers.scryfall_id : null).toMatch(/^[0-9a-f-]{36}$/);
	24	+ expect(obs && obs.kind === 'price_observation' ? obs.currency : null).toBe('USD');
	25	+ expect(obs && obs.kind === 'price_observation' ? (obs.attributes.metadata as { provider: string }).provider : null).toBe('cardkingdom');
	26	+ });
	27	+});

added connectors/api/cardkingdom/index.ts +111 −0

		`@@ -0,0 +1,111 @@`
	1	+import { z } from 'zod';
	2	+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { type NormalizedRecord } from '@rareindex/shared';
	4	+import { attrs, catalogItem, makeTitle, num, priceObservation } from '../_lib/shared.js';
	5	+import { BOT_HEADERS, dayOf } from '../_lib/wave4.js';
	6	+
	7	+/** Card Kingdom public price lists → Magic price observations keyed by scryfall_id. */
	8	+const PARSER_VERSION = '1.0.0';
	9	+const SINGLES = 'https://api.cardkingdom.com/api/pricelist';
	10	+const SEALED = 'https://api.cardkingdom.com/api/sealed_pricelist';
	11	+
	12	+const SingleSchema = z.object({
	13	+ id: z.number(),
	14	+ sku: z.string().nullable().optional(),
	15	+ scryfall_id: z.string().nullable().optional(),
	16	+ url: z.string().nullable().optional(),
	17	+ name: z.string(),
	18	+ variation: z.string().nullable().optional(),
	19	+ edition: z.string(),
	20	+ is_foil: z.union([z.string(), z.boolean()]).optional(),
	21	+ price_retail: z.union([z.string(), z.number()]).nullable().optional(),
	22	+ qty_retail: z.number().nullable().optional(),
	23	+ price_buy: z.union([z.string(), z.number()]).nullable().optional(),
	24	+ qty_buying: z.number().nullable().optional(),
	25	+});
	26	+const SealedSchema = z.object({ id: z.number(), url: z.string().nullable().optional(), name: z.string(), edition: z.string().nullable().optional(), price_retail: z.union([z.string(), z.number()]).nullable().optional(), qty_retail: z.number().nullable().optional(), price_buy: z.union([z.string(), z.number()]).nullable().optional(), qty_buying: z.number().nullable().optional() });
	27	+const RawPayloadSchema = z.discriminatedUnion('kind', [
	28	+ z.object({ kind: z.literal('single'), row: SingleSchema, createdAt: z.string().nullable(), baseUrl: z.string() }),
	29	+ z.object({ kind: z.literal('sealed'), row: SealedSchema, createdAt: z.string().nullable(), baseUrl: z.string() }),
	30	+]);
	31	+export type CardKingdomPayload = z.infer<typeof RawPayloadSchema>;
	32	+
	33	+export function isFoil(v: string \| boolean \| undefined): boolean {
	34	+ return v === true \|\| String(v).toLowerCase() === 'true';
	35	+}
	36	+
	37	+export class CardKingdomConnector extends BaseConnector {
	38	+ readonly version = '1.0.0';
	39	+ readonly parserVersion = PARSER_VERSION;
	40	+
	41	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	42	+ const minRetail = Number(this.meta.config.minRetailUsd ?? 0);
	43	+ let count = 0;
	44	+ const singles = await ctx.fetch(SINGLES, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 180_000 });
	45	+ let doc = singles.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } \| null;
	46	+ if ((!doc \|\| !Array.isArray(doc.data)) && typeof singles.html === 'string' && singles.html.startsWith('{')) doc = JSON.parse(singles.html) as typeof doc; // served as text/html
	47	+ if (!singles.success \|\| !doc \|\| !Array.isArray(doc.data)) throw new Error(`cardkingdom pricelist failed: ${singles.error ?? singles.httpStatus}`);
	48	+ const createdAt = doc.meta?.created_at ?? null;
	49	+ const baseUrl = doc.meta?.base_url ?? 'https://www.cardkingdom.com/';
	50	+ for (const r of doc.data) {
	51	+ const parsed = SingleSchema.safeParse(r);
	52	+ if (!parsed.success) {
	53	+ ctx.anomaly('parse_failure_row', parsed.error.issues[0]?.message);
	54	+ continue;
	55	+ }
	56	+ const retail = num(parsed.data.price_retail);
	57	+ if (retail === null \|\| retail < minRetail) continue;
	58	+ if (this.reached(ctx, count)) return;
	59	+ count++;
	60	+ yield { url: `${baseUrl}${parsed.data.url ?? `mtg/${parsed.data.id}`}`, externalId: `s${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: singles.httpStatus, payload: { kind: 'single', row: parsed.data, createdAt, baseUrl } satisfies CardKingdomPayload, fetchedAt: singles.fetchedAt };
	61	+ }
	62	+ if (this.meta.config.includeSealed !== false) {
	63	+ const sealed = await ctx.fetch(SEALED, { engines: ['api'], headers: BOT_HEADERS, timeoutMs: 120_000 });
	64	+ let sdoc = sealed.json as { meta?: { created_at?: string; base_url?: string }; data?: unknown[] } \| null;
	65	+ if ((!sdoc \|\| !Array.isArray(sdoc.data)) && typeof sealed.html === 'string' && sealed.html.startsWith('{')) sdoc = JSON.parse(sealed.html) as typeof sdoc;
	66	+ if (sealed.success && sdoc && Array.isArray(sdoc.data)) {
	67	+ for (const r of sdoc.data) {
	68	+ const parsed = SealedSchema.safeParse(r);
	69	+ if (!parsed.success \|\| num(parsed.data.price_retail) === null) continue;
	70	+ if (this.reached(ctx, count)) return;
	71	+ count++;
	72	+ yield { url: `${baseUrl}${parsed.data.url ?? `mtg-sealed/${parsed.data.id}`}`, externalId: `x${parsed.data.id}`, kind: 'catalog_item', engine: 'api', httpStatus: sealed.httpStatus, payload: { kind: 'sealed', row: parsed.data, createdAt: sdoc.meta?.created_at ?? null, baseUrl } satisfies CardKingdomPayload, fetchedAt: sealed.fetchedAt };
	73	+ }
	74	+ } else ctx.anomaly('page_fetch_failed', `sealed_pricelist: ${sealed.error ?? sealed.httpStatus}`);
	75	+ }
	76	+ await ctx.setCursor({ completedAt: new Date().toISOString() });
	77	+ }
	78	+
	79	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	80	+ const p = RawPayloadSchema.parse(raw.payload);
	81	+ const observedAt = raw.fetchedAt;
	82	+ const created = p.createdAt ? new Date(p.createdAt.replace(' ', 'T') + 'Z') : null;
	83	+ const observationDate = created && !Number.isNaN(created.getTime()) ? dayOf(created) : dayOf(observedAt);
	84	+ const out: NormalizedRecord[] = [];
	85	+ if (p.kind === 'single') {
	86	+ const r = p.row;
	87	+ const foil = isFoil(r.is_foil);
	88	+ const variation = (r.variation ?? '').trim();
	89	+ const variant = [variation \|\| null, foil ? 'Foil' : null].filter(Boolean).join(' ') \|\| null;
	90	+ const number = r.sku?.match(/-([A-Za-z0-9]+)$/)?.[1] ?? null;
	91	+ const identifiers: Record<string, string> = { cardkingdom_id: String(r.id) };
	92	+ if (r.scryfall_id) identifiers.scryfall_id = r.scryfall_id;
	93	+ if (r.sku) identifiers.cardkingdom_sku = r.sku;
	94	+ const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition, name: r.name, number, variant, language: 'English', identifiers, metadata: { finish: foil ? 'foil' : 'nonfoil', variation: variation \|\| null } });
	95	+ const rawTitle = makeTitle({ name: r.name, set: r.edition, number, variant });
	96	+ out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.8, parserVersion: PARSER_VERSION }));
	97	+ const retail = num(r.price_retail);
	98	+ if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `s${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy), qty_retail: r.qty_retail ?? null, qty_buying: r.qty_buying ?? null } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null }));
	99	+ return out;
	100	+ }
	101	+ const r = p.row;
	102	+ const a = attrs({ categorySlug: 'magic_the_gathering', franchise: 'Magic: The Gathering', brand: 'Wizards of the Coast', set: r.edition ?? null, name: r.name, identifiers: { cardkingdom_sealed_id: String(r.id) }, metadata: { sealed: true } });
	103	+ const rawTitle = makeTitle({ name: r.name, set: r.edition ?? null });
	104	+ out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.75, parserVersion: PARSER_VERSION }));
	105	+ const retail = num(r.price_retail);
	106	+ if (retail) out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, externalId: `x${r.id}:retail:${observationDate.toISOString().slice(0, 10)}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider: 'cardkingdom', buylist: num(r.price_buy) } }, observedAt, confidence: 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price: retail, currency: 'USD', observationDate, sampleSize: r.qty_retail ?? null }));
	107	+ return out;
	108	+ }
	109	+}
	110	+
	111	+export default (meta: ConnectorMeta) => new CardKingdomConnector(meta);

added connectors/api/cardkingdom/meta.json +33 −0

		`@@ -0,0 +1,33 @@`
	1	+{
	2	+ "id": "cardkingdom",
	3	+ "displayName": "Card Kingdom (retail & buylist prices)",
	4	+ "sourceId": "cardkingdom",
	5	+ "sourceName": "Card Kingdom",
	6	+ "sourceType": "dealer",
	7	+ "sourceUrl": "https://www.cardkingdom.com",
	8	+ "module": "api/cardkingdom",
	9	+ "enginePriority": ["api"],
	10	+ "categories": ["magic_the_gathering"],
	11	+ "regions": ["US"],
	12	+ "languages": ["en"],
	13	+ "currency": ["USD"],
	14	+ "supportsListings": false,
	15	+ "supportsSold": false,
	16	+ "supportsAuctions": false,
	17	+ "supportsImages": false,
	18	+ "supportsCatalog": true,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": false,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "medium",
	23	+ "trustScore": 0.8,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://www.cardkingdom.com/help/terms",
	26	+ "accessNotes": "Card Kingdom publishes its full singles price list (https://api.cardkingdom.com/api/pricelist, ~46 MB JSON, refreshed daily) and sealed price list (/api/sealed_pricelist) publicly; each singles row carries the Scryfall id, edition, foil flag, retail price, retail stock, buylist price and buylist demand. Fetched once per run with an identifying User-Agent (two requests). Retail = dealer asking price stored as price_observation kind 'market' with metadata.buylist; not a transaction. Identifiers: cardkingdom_id, scryfall_id. Sealed products get catalog items with metadata.sealed=true.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "includeSealed": true,
	31	+ "minRetailUsd": 0.5
	32	+ }
	33	+}

added connectors/api/mtgjson/index.test.ts +58 −0

		`@@ -0,0 +1,58 @@`
	1	+import { describe, expect, it } from 'vitest';
	2	+import meta from './meta.json' with { type: 'json' };
	3	+import { localMeta } from '../_lib/local-meta.js';
	4	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	5	+import createConnector from './index.js';
	6	+import { magicTreatments, streamGzipDataEntries } from '../_lib/wave4.js';
	7	+import { gzipSync } from 'node:zlib';
	8	+
	9	+const connector = createConnector(localMeta(meta));
	10	+
	11	+describe('mtgjson', () => {
	12	+ runFixtureSuite(connector, it, expect);
	13	+
	14	+ it('emits scryfall-aligned variants and provider-dated observations', async () => {
	15	+ const names = listFixtures('mtgjson');
	16	+ const fx = loadFixture('mtgjson', names[0]!);
	17	+ const out = await connector.normalize(fx.raw);
	18	+ const cats = out.filter((r) => r.kind === 'catalog_item');
	19	+ const obs = out.filter((r) => r.kind === 'price_observation');
	20	+ expect(cats.length).toBeGreaterThanOrEqual(1);
	21	+ for (const c of cats) {
	22	+ if (c.kind !== 'catalog_item') continue;
	23	+ expect(c.attributes.identifiers.mtgjson_uuid).toBeTruthy();
	24	+ expect(c.attributes.identifiers.scryfall_id).toMatch(/^[0-9a-f-]{36}$/);
	25	+ expect(c.attributes.setCode).toMatch(/^[A-Z0-9]{2,6}$/);
	26	+ }
	27	+ for (const o of obs) {
	28	+ if (o.kind !== 'price_observation') continue;
	29	+ expect(['USD', 'EUR']).toContain(o.currency);
	30	+ expect(o.priceKind).toBe('market');
	31	+ expect((o.attributes.metadata as { provider: string }).provider).toBeTruthy();
	32	+ expect(o.observationDate.getTime()).toBeLessThanOrEqual(Date.now());
	33	+ }
	34	+ });
	35	+
	36	+ it('mirrors the scryfall treatment vocabulary', () => {
	37	+ expect(magicTreatments({ frameEffects: ['showcase'], promoTypes: [] })).toEqual(['Showcase']);
	38	+ expect(magicTreatments({ frameEffects: ['extendedart'], borderColor: 'borderless' })).toEqual(['Extended Art', 'Borderless']);
	39	+ expect(magicTreatments({})).toEqual([]);
	40	+ });
	41	+
	42	+ it('streams top-level data entries from a gzip JSON document', async () => {
	43	+ const doc = { meta: { date: '2026-09-06' }, data: { a: { paper: { tcgplayer: { retail: { normal: { '2026-09-06': 1.5 } } } } }, b: { paper: {} }, c: { x: [1, 2, { y: '}' }] } } };
	44	+ const gz = gzipSync(Buffer.from(JSON.stringify(doc)));
	45	+ const orig = globalThis.fetch;
	46	+ globalThis.fetch = (async () => new Response(gz, { status: 200 })) as typeof fetch;
	47	+ try {
	48	+ const seen: string[] = [];
	49	+ const n = await streamGzipDataEntries('https://example.test/x.json.gz', (k) => {
	50	+ seen.push(k);
	51	+ });
	52	+ expect(n).toBe(3);
	53	+ expect(seen).toEqual(['a', 'b', 'c']);
	54	+ } finally {
	55	+ globalThis.fetch = orig;
	56	+ }
	57	+ });
	58	+});

added connectors/api/mtgjson/index.ts +219 −0

		`@@ -0,0 +1,219 @@`
	1	+import { z } from 'zod';
	2	+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { type NormalizedRecord } from '@rareindex/shared';
	4	+import { attrs, catalogItem, makeTitle, priceObservation } from '../_lib/shared.js';
	5	+import { BOT_HEADERS, fetchGzipJson, isoDay, magicFinishVariant, magicTreatments, streamGzipDataEntries } from '../_lib/wave4.js';
	6	+
	7	+/**
	8	+ * MTGJSON connector — per-set card files (identifiers across Scryfall/TCGplayer/Cardmarket/Card Kingdom)
	9	+ * joined with the daily price file. One raw record per printing; normalize emits catalog items per
	10	+ * finish and dated price observations per provider × finish.
	11	+ */
	12	+const API = 'https://mtgjson.com/api/v5';
	13	+const PARSER_VERSION = '1.0.0';
	14	+
	15	+const SetSchema = z.object({ code: z.string(), name: z.string(), releaseDate: z.string().nullable().optional(), type: z.string().nullable().optional(), tcgplayerGroupId: z.number().nullable().optional(), mcmId: z.number().nullable().optional(), isOnlineOnly: z.boolean().optional(), totalSetSize: z.number().optional() });
	16	+const CardSchema = z.object({
	17	+ uuid: z.string(),
	18	+ name: z.string(),
	19	+ number: z.string(),
	20	+ rarity: z.string().nullable().optional(),
	21	+ finishes: z.array(z.string()).default([]),
	22	+ language: z.string().nullable().optional(),
	23	+ identifiers: z.record(z.string(), z.string()).default({}),
	24	+ frameEffects: z.array(z.string()).optional(),
	25	+ promoTypes: z.array(z.string()).optional(),
	26	+ isFullArt: z.boolean().optional(),
	27	+ borderColor: z.string().optional(),
	28	+ frameVersion: z.string().optional(),
	29	+ isPromo: z.boolean().optional(),
	30	+ isReserved: z.boolean().optional(),
	31	+ artist: z.string().nullable().optional(),
	32	+ type: z.string().nullable().optional(),
	33	+ manaCost: z.string().nullable().optional(),
	34	+});
	35	+/** provider → finish → { date: price } */
	36	+const ProviderPricesSchema = z.object({ retail: z.record(z.string(), z.record(z.string(), z.number())).optional(), buylist: z.record(z.string(), z.record(z.string(), z.number())).optional(), currency: z.string().optional() });
	37	+const PaperPricesSchema = z.record(z.string(), ProviderPricesSchema);
	38	+const RawPayloadSchema = z.object({ set: SetSchema, card: CardSchema, prices: PaperPricesSchema.nullable(), priceDate: z.string().nullable() });
	39	+export type MtgjsonPayload = z.infer<typeof RawPayloadSchema>;
	40	+
	41	+type PriceMap = Map<string, z.infer<typeof PaperPricesSchema>>;
	42	+
	43	+const CARD_KEEP = ['uuid', 'name', 'number', 'rarity', 'finishes', 'language', 'identifiers', 'frameEffects', 'promoTypes', 'isFullArt', 'borderColor', 'frameVersion', 'isPromo', 'isReserved', 'artist', 'type', 'manaCost'] as const;
	44	+export function trimCard(c: Record<string, unknown>): z.infer<typeof CardSchema> {
	45	+ const out: Record<string, unknown> = {};
	46	+ for (const k of CARD_KEEP) if (c[k] !== undefined) out[k] = c[k];
	47	+ return CardSchema.parse(out);
	48	+}
	49	+export function trimSet(s: Record<string, unknown>): z.infer<typeof SetSchema> {
	50	+ const out: Record<string, unknown> = {};
	51	+ for (const k of ['code', 'name', 'releaseDate', 'type', 'tcgplayerGroupId', 'mcmId', 'isOnlineOnly', 'totalSetSize']) if (s[k] !== undefined) out[k] = s[k];
	52	+ return SetSchema.parse(out);
	53	+}
	54	+
	55	+/** Keep only the latest `days` dates per provider/finish (payload compactness). */
	56	+function trimPrices(paper: unknown, days: number): z.infer<typeof PaperPricesSchema> \| null {
	57	+ const parsed = PaperPricesSchema.safeParse(paper);
	58	+ if (!parsed.success) return null;
	59	+ const out: z.infer<typeof PaperPricesSchema> = {};
	60	+ for (const [provider, p] of Object.entries(parsed.data)) {
	61	+ const trimmed: z.infer<typeof ProviderPricesSchema> = { currency: p.currency };
	62	+ for (const kind of ['retail', 'buylist'] as const) {
	63	+ const byFinish = p[kind];
	64	+ if (!byFinish) continue;
	65	+ const tf: Record<string, Record<string, number>> = {};
	66	+ for (const [finish, series] of Object.entries(byFinish)) {
	67	+ const dates = Object.keys(series).sort().slice(-days);
	68	+ tf[finish] = Object.fromEntries(dates.map((d) => [d, series[d]!]));
	69	+ }
	70	+ trimmed[kind] = tf;
	71	+ }
	72	+ out[provider] = trimmed;
	73	+ }
	74	+ return out;
	75	+}
	76	+
	77	+export class MtgjsonConnector extends BaseConnector {
	78	+ readonly version = '1.0.0';
	79	+ readonly parserVersion = PARSER_VERSION;
	80	+ protected override minIntervalMs = 250;
	81	+
	82	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	83	+ const backfill = ctx.options.mode === 'backfill';
	84	+ const maxSets = backfill ? Infinity : Number(this.meta.config.maxSetsPerRun ?? 60);
	85	+ const listRes = await ctx.fetch(`${API}/SetList.json`, { engines: ['api'], headers: BOT_HEADERS });
	86	+ const list = (listRes.json as { data?: unknown[] } \| null)?.data;
	87	+ if (!listRes.success \|\| !Array.isArray(list)) throw new Error(`mtgjson SetList failed: ${listRes.error}`);
	88	+ let sets = list.map((s) => trimSet(s as Record<string, unknown>)).filter((s) => !s.isOnlineOnly);
	89	+ if (ctx.options.seeds?.length) sets = sets.filter((s) => ctx.options.seeds!.includes(s.code));
	90	+ sets.sort((a, b) => (b.releaseDate ?? '').localeCompare(a.releaseDate ?? ''));
	91	+ sets = sets.slice(0, Number.isFinite(maxSets) ? maxSets : sets.length);
	92	+
	93	+ // Daily prices (small) — joined by uuid. Backfill streams the 90-day history instead.
	94	+ const prices: PriceMap = new Map();
	95	+ let priceDate: string \| null = null;
	96	+ if (!backfill) {
	97	+ const today = await fetchGzipJson<{ meta?: { date?: string }; data?: Record<string, { paper?: unknown }> }>(`${API}/AllPricesToday.json.gz`, ctx.signal);
	98	+ priceDate = today.meta?.date ?? null;
	99	+ for (const [uuid, v] of Object.entries(today.data ?? {})) {
	100	+ const t = trimPrices(v.paper, 1);
	101	+ if (t) prices.set(uuid, t);
	102	+ }
	103	+ const s = (ctx.engineStats.api ??= { attempts: 0, success: 0, credits: 0, ms: 0 });
	104	+ s.attempts++;
	105	+ s.success++;
	106	+ }
	107	+
	108	+ let count = 0;
	109	+ let setIdx = Number(ctx.options.cursor?.setIdx ?? 0);
	110	+ const cards = new Map<string, { set: z.infer<typeof SetSchema>; card: z.infer<typeof CardSchema>; url: string }>();
	111	+ for (; setIdx < sets.length; setIdx++) {
	112	+ if (ctx.signal?.aborted) return;
	113	+ const set = sets[setIdx]!;
	114	+ await this.throttle();
	115	+ const res = await ctx.fetch(`${API}/${set.code}.json`, { engines: ['api'], headers: BOT_HEADERS });
	116	+ const data = (res.json as { data?: { cards?: unknown[] } } \| null)?.data;
	117	+ if (!res.success \|\| !Array.isArray(data?.cards)) {
	118	+ ctx.anomaly('page_fetch_failed', `${set.code}: ${res.error ?? res.httpStatus}`);
	119	+ continue;
	120	+ }
	121	+ for (const raw of data.cards) {
	122	+ let card: z.infer<typeof CardSchema>;
	123	+ try {
	124	+ card = trimCard(raw as Record<string, unknown>);
	125	+ } catch (err) {
	126	+ ctx.anomaly('parse_failure_card', `${set.code}: ${err instanceof Error ? err.message : String(err)}`);
	127	+ continue;
	128	+ }
	129	+ const url = `https://mtgjson.com/api/v5/${set.code}.json#${card.uuid}`;
	130	+ if (backfill) {
	131	+ cards.set(card.uuid, { set, card, url });
	132	+ continue;
	133	+ }
	134	+ if (this.reached(ctx, count)) return;
	135	+ count++;
	136	+ const payload: MtgjsonPayload = { set, card, prices: prices.get(card.uuid) ?? null, priceDate };
	137	+ yield { url, externalId: card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	138	+ }
	139	+ await ctx.setCursor({ setIdx: setIdx + 1, priceDate });
	140	+ }
	141	+ if (backfill) {
	142	+ // Stream the 90-day history and yield one record per known printing.
	143	+ const days = Number(this.meta.config.historyDays ?? 90);
	144	+ const queue: RawRecordInput[] = [];
	145	+ const seen = await streamGzipDataEntries(
	146	+ `${API}/AllPrices.json.gz`,
	147	+ (uuid, value) => {
	148	+ const c = cards.get(uuid);
	149	+ if (!c) return;
	150	+ const paper = (value as { paper?: unknown })?.paper;
	151	+ const trimmed = trimPrices(paper, days);
	152	+ const payload: MtgjsonPayload = { set: c.set, card: c.card, prices: trimmed, priceDate: null };
	153	+ queue.push({ url: c.url, externalId: c.card.uuid, kind: 'catalog_item', engine: 'api', httpStatus: 200, payload, fetchedAt: new Date() });
	154	+ },
	155	+ { signal: ctx.signal, limit: ctx.options.limit ? ctx.options.limit * 4 : undefined },
	156	+ );
	157	+ ctx.log.info({ seen, matched: queue.length }, 'mtgjson AllPrices streamed');
	158	+ for (const r of queue) {
	159	+ if (this.reached(ctx, count)) return;
	160	+ count++;
	161	+ yield r;
	162	+ }
	163	+ }
	164	+ await ctx.setCursor({ setIdx: 0, priceDate, completedAt: new Date().toISOString() });
	165	+ }
	166	+
	167	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	168	+ const { set, card, prices } = RawPayloadSchema.parse(raw.payload);
	169	+ const year = set.releaseDate ? Number(set.releaseDate.slice(0, 4)) \|\| null : null;
	170	+ const ids: Record<string, string> = { mtgjson_uuid: card.uuid };
	171	+ const map: Array<[string, string]> = [['scryfallId', 'scryfall_id'], ['scryfallOracleId', 'oracle_id'], ['tcgplayerProductId', 'tcgplayer_id'], ['tcgplayerEtchedProductId', 'tcgplayer_etched_id'], ['mcmId', 'cardmarket_id'], ['cardKingdomId', 'cardkingdom_id'], ['cardKingdomFoilId', 'cardkingdom_foil_id'], ['cardKingdomEtchedId', 'cardkingdom_etched_id'], ['cardsphereId', 'cardsphere_id'], ['multiverseId', 'multiverse_id'], ['mtgoId', 'mtgo_id']];
	172	+ for (const [from, to] of map) if (card.identifiers[from]) ids[to] = card.identifiers[from]!;
	173	+ const treat = magicTreatments({ ...card, setType: set.type ?? null, releaseDate: set.releaseDate ?? null });
	174	+ const baseVariant = treat.length ? treat.join(' ') : null;
	175	+ const finishes = card.finishes.length ? card.finishes : ['nonfoil'];
	176	+ const observedAt = raw.fetchedAt;
	177	+ const providers = new Set((this.meta.config.providers as string[] \| undefined) ?? ['tcgplayer', 'cardmarket', 'cardkingdom', 'cardsphere', 'manapool']);
	178	+ const out: NormalizedRecord[] = [];
	179	+ for (const finish of finishes) {
	180	+ const variant = [baseVariant, magicFinishVariant(finish)].filter(Boolean).join(' ') \|\| null;
	181	+ const a = attrs({
	182	+ categorySlug: 'magic_the_gathering',
	183	+ franchise: 'Magic: The Gathering',
	184	+ brand: 'Wizards of the Coast',
	185	+ set: set.name,
	186	+ setCode: set.code.toUpperCase(),
	187	+ name: card.name,
	188	+ number: card.number,
	189	+ year,
	190	+ variant,
	191	+ language: card.language ?? 'English',
	192	+ rarity: card.rarity ?? null,
	193	+ identifiers: { ...ids, finish },
	194	+ metadata: { artist: card.artist ?? null, type_line: card.type ?? null, mana_cost: card.manaCost ?? null, promo: card.isPromo ?? false, reserved: card.isReserved ?? false, set_type: set.type ?? null },
	195	+ });
	196	+ const rawTitle = makeTitle({ name: card.name, set: set.name, number: card.number, year, variant });
	197	+ const sourceUrl = `https://scryfall.com/card/${set.code.toLowerCase()}/${encodeURIComponent(card.number)}`;
	198	+ out.push(catalogItem({ kind: 'catalog_item', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.uuid}:${finish}`, rawTitle, imageUrls: [], attributes: a, observedAt, confidence: 0.85, parserVersion: PARSER_VERSION, releaseDate: set.releaseDate ? new Date(set.releaseDate) : null }));
	199	+ if (!prices) continue;
	200	+ for (const [provider, p] of Object.entries(prices)) {
	201	+ if (!providers.has(provider)) continue;
	202	+ // MTGJSON price files key finishes as normal\|foil\|etched while card.finishes uses nonfoil\|foil\|etched.
	203	+ const priceKey = finish === 'nonfoil' ? 'normal' : finish;
	204	+ const retail = p.retail?.[priceKey];
	205	+ if (!retail) continue;
	206	+ const currency = (p.currency ?? (provider === 'cardmarket' ? 'EUR' : 'USD')) as 'USD' \| 'EUR';
	207	+ const buy = p.buylist?.[priceKey];
	208	+ for (const [date, price] of Object.entries(retail)) {
	209	+ const observationDate = isoDay(date);
	210	+ if (!observationDate \|\| !(price > 0)) continue;
	211	+ out.push(priceObservation({ kind: 'price_observation', connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl, externalId: `${card.uuid}:${finish}:${provider}:${date}`, rawTitle, imageUrls: [], attributes: { ...a, metadata: { ...a.metadata, provider, buylist: buy?.[date] ?? null } }, observedAt, confidence: provider === 'tcgplayer' \|\| provider === 'cardmarket' ? 0.75 : 0.7, parserVersion: PARSER_VERSION, priceKind: 'market', price, currency, observationDate, sampleSize: null }));
	212	+ }
	213	+ }
	214	+ }
	215	+ return out;
	216	+ }
	217	+}
	218	+
	219	+export default (meta: ConnectorMeta) => new MtgjsonConnector(meta);

added connectors/api/mtgjson/meta.json +34 −0

		`@@ -0,0 +1,34 @@`
	1	+{
	2	+ "id": "mtgjson",
	3	+ "displayName": "MTGJSON (prices & identifiers)",
	4	+ "sourceId": "mtgjson",
	5	+ "sourceName": "MTGJSON",
	6	+ "sourceType": "pricing_guide",
	7	+ "sourceUrl": "https://mtgjson.com",
	8	+ "module": "api/mtgjson",
	9	+ "enginePriority": ["api"],
	10	+ "categories": ["magic_the_gathering"],
	11	+ "regions": ["US", "EU"],
	12	+ "languages": ["en"],
	13	+ "currency": ["USD", "EUR"],
	14	+ "supportsListings": false,
	15	+ "supportsSold": false,
	16	+ "supportsAuctions": false,
	17	+ "supportsImages": false,
	18	+ "supportsCatalog": true,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": false,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "high",
	23	+ "trustScore": 0.85,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://mtgjson.com/faq/",
	26	+ "accessNotes": "Open MTG data project (free bulk files, no key). Incremental runs download SetList.json, the newest `maxSetsPerRun` set files (LEA.json… with per-card identifiers: scryfallId, tcgplayerProductId, cardKingdomId, mcmId, cardsphereId, multiverseId) and AllPricesToday.json.gz (~5 MB gz) which carries the latest daily retail/buylist price per printing from TCGplayer, Cardmarket (EUR), Card Kingdom, Cardsphere and ManaPool. Backfill streams AllPrices.json.gz (~150 MB gz, 90 days of daily history) entry by entry. Retail prices are stored as price_observations (priceKind 'market', metadata.provider), buylist prices only in metadata; every point is dated by MTGJSON's own date keys. Variant vocabulary mirrors the scryfall connector (Foil / Etched Foil + treatments) so identifier matches agree.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "maxSetsPerRun": 60,
	31	+ "providers": ["tcgplayer", "cardmarket", "cardkingdom", "cardsphere", "manapool"],
	32	+ "historyDays": 90
	33	+ }
	34	+}

added connectors/api/myslabs/index.test.ts +32 −0

		`@@ -0,0 +1,32 @@`
	1	+import { describe, expect, it } from 'vitest';
	2	+import meta from './meta.json' with { type: 'json' };
	3	+import { localMeta } from '../_lib/local-meta.js';
	4	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	5	+import createConnector, { slabCategory } from './index.js';
	6	+
	7	+const connector = createConnector(localMeta(meta));
	8	+
	9	+describe('myslabs', () => {
	10	+ runFixtureSuite(connector, it, expect);
	11	+
	12	+ it('classifies slab titles', () => {
	13	+ expect(slabCategory('2025 Prizm FIFA Manga Kylian Mbappe BGS 9.5', null)).toBe('soccer_cards');
	14	+ expect(slabCategory('1999 Pokemon Base Set Charizard Holo PSA 9', null)).toBe('pokemon');
	15	+ expect(slabCategory('Amazing Spider-Man #300 CGC 9.8 1988', null)).toBe('marvel_comics');
	16	+ expect(slabCategory('2003 Topps Chrome LeBron James Rookie PSA 10', null)).toBe('basketball_cards');
	17	+ expect(slabCategory('Random vase', null)).toBeNull();
	18	+ });
	19	+
	20	+ it('records sold-out slabs as sold listings, never as sales', async () => {
	21	+ for (const name of listFixtures('myslabs')) {
	22	+ const out = await connector.normalize(loadFixture('myslabs', name).raw);
	23	+ for (const r of out) {
	24	+ expect(r.kind).toBe('listing');
	25	+ if (r.kind === 'listing') {
	26	+ expect(['available', 'sold', 'unknown']).toContain(r.availability);
	27	+ expect(r.currency).toBe('USD');
	28	+ }
	29	+ }
	30	+ }
	31	+ });
	32	+});

added connectors/api/myslabs/index.ts +164 −0

		`@@ -0,0 +1,164 @@`
	1	+import { z } from 'zod';
	2	+import { BaseConnector, html as H, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { NormalizedListingSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';
	4	+import { parseGradeFromTitle } from '@rareindex/taxonomy';
	5	+import { attrs } from '../_lib/shared.js';
	6	+import { BOT_HEADERS, cardCategorySlug, decodeXml, parseSitemap } from '../_lib/wave4.js';
	7	+
	8	+/** MySlabs — graded slab marketplace; public slab pages expose a Product JSON-LD. */
	9	+const PARSER_VERSION = '1.0.0';
	10	+const SITE = 'https://myslabs.com';
	11	+
	12	+const RawPayloadSchema = z.object({
	13	+ id: z.string(),
	14	+ name: z.string(),
	15	+ description: z.string().nullable(),
	16	+ images: z.array(z.string()),
	17	+ price: z.number().nullable(),
	18	+ currency: z.string().nullable(),
	19	+ availability: z.string().nullable(),
	20	+ category: z.string().nullable(),
	21	+ seller: z.string().nullable(),
	22	+});
	23	+export type MySlabsPayload = z.infer<typeof RawPayloadSchema>;
	24	+
	25	+const SPORTS_BRAND = /\b(topps\|panini\|bowman\|prizm\|fleer\|donruss\|upper deck\|select\|mosaic\|optic\|chrome\|leaf\|score\|o-pee-chee\|hoops\|skybox\|sp authentic\|national treasures\|immaculate\|flawless\|futera\|stadium club\|goudey\|play ball\|kellogg)\b/i;
	26	+const SPORT_WORDS: Array<[RegExp, string]> = [
	27	+ [/\b(basketball\|nba\|wnba\|hoops\|lebron\|jordan\|curry\|wembanyama\|kobe\|giannis\|luka\|jokic)\b/i, 'basketball_cards'],
	28	+ [/\b(baseball\|mlb\|mantle\|ohtani\|trout\|jeter\|bowman\|goudey\|play ball)\b/i, 'baseball_cards'],
	29	+ [/\b(football\|nfl\|mahomes\|brady\|rookie qb\|panini contenders\|donruss optic football)\b/i, 'football_cards'],
	30	+ [/\b(hockey\|nhl\|gretzky\|mcdavid\|crosby\|o-pee-chee\|young guns\|bedard)\b/i, 'hockey_cards'],
	31	+ [/\b(soccer\|fifa\|messi\|ronaldo\|mbappe\|mbappé\|haaland\|yamal\|premier league\|champions league)\b/i, 'soccer_cards'],
	32	+ [/\b(f1\|formula 1\|formula one\|verstappen\|hamilton\|leclerc\|norris)\b/i, 'f1_cards'],
	33	+ [/\b(ufc\|wwe\|wrestling\|boxing\|golf\|tennis\|nascar\|olympic\|mma)\b/i, 'other_sports_cards'],
	34	+];
	35	+
	36	+/** Title → taxonomy slug for MySlabs inventory (cards/comics); null when unsure. */
	37	+export function slabCategory(title: string, categoryHint: string \| null): string \| null {
	38	+ const hinted = cardCategorySlug(categoryHint);
	39	+ if (hinted && hinted !== 'trading_cards') return hinted;
	40	+ const tcg = cardCategorySlug(title);
	41	+ if (tcg && ['pokemon', 'magic_the_gathering', 'yugioh', 'disney_lorcana', 'one_piece_card_game', 'digimon_tcg', 'flesh_and_blood', 'star_wars_tcg', 'dragon_ball_tcg'].includes(tcg)) return tcg;
	42	+ if ((/\b(cgc\|cbcs)\b/i.test(title) && /#\s?\d+/.test(title) \|\| /\bcomic\|\b#\d+\s*$?\d{4}$?/i.test(title)) && !SPORTS_BRAND.test(title)) return /\bmarvel\|spider-man\|x-men\|avengers\|hulk\|iron man\|captain america\|wolverine\|daredevil\|fantastic four\b/i.test(title) ? 'marvel_comics' : /\bdc\b\|batman\|superman\|detective comics\|action comics\|wonder woman\|flash\|green lantern\|joker/i.test(title) ? 'dc_comics' : 'independent_comics';
	43	+ for (const [re, slug] of SPORT_WORDS) if (re.test(title)) return slug;
	44	+ if (SPORTS_BRAND.test(title)) return 'sports_cards';
	45	+ if (hinted) return hinted;
	46	+ return null;
	47	+}
	48	+
	49	+function extractPayload(id: string, htmlText: string): MySlabsPayload \| null {
	50	+ const products = H.jsonLd(htmlText, 'Product');
	51	+ const p = products[0];
	52	+ if (!p) return null;
	53	+ const offers = (Array.isArray(p.offers) ? p.offers[0] : p.offers) as Record<string, unknown> \| undefined;
	54	+ const images = (Array.isArray(p.image) ? p.image : p.image ? [p.image] : []).map((u) => decodeXml(String(u)));
	55	+ const priceRaw = offers?.price;
	56	+ const price = priceRaw === undefined \|\| priceRaw === null \|\| priceRaw === '' ? null : Number(priceRaw);
	57	+ const availability = typeof offers?.availability === 'string' ? offers.availability.replace(/^https?:\/\/schema\.org\//, '') : null;
	58	+ const seller = (offers?.seller as { name?: string } \| undefined)?.name ?? null;
	59	+ const $ = H.load(htmlText);
	60	+ const crumbs = $('nav[aria-label="breadcrumb"] a, .breadcrumb a, ol.breadcrumb li')
	61	+ .map((_, el) => $(el).text().trim())
	62	+ .get()
	63	+ .filter(Boolean);
	64	+ const category = crumbs.find((c) => /card\|comic\|pok\|magic\|yu-gi\|sport/i.test(c)) ?? null;
	65	+ return { id, name: String(p.name ?? '').trim(), description: typeof p.description === 'string' ? p.description.trim() : null, images, price: Number.isFinite(price as number) ? (price as number) : null, currency: typeof offers?.priceCurrency === 'string' ? offers.priceCurrency : null, availability, category, seller };
	66	+}
	67	+
	68	+export class MySlabsConnector extends BaseConnector {
	69	+ readonly version = '1.0.0';
	70	+ readonly parserVersion = PARSER_VERSION;
	71	+ override readonly urlPatterns = [/^https?:\/\/(www\.)?myslabs\.com\/slab\/view\/\d+/i];
	72	+ protected override minIntervalMs = 1500;
	73	+
	74	+ private async fetchSlab(ctx: CrawlContext, url: string): Promise<RawRecordInput \| null> {
	75	+ await this.throttle();
	76	+ const res = await ctx.fetch(url, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'text/html' }, responseType: 'text' });
	77	+ if (!res.success \|\| !res.html) {
	78	+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
	79	+ return null;
	80	+ }
	81	+ const id = url.match(/slab\/view\/(\d+)/)?.[1] ?? url;
	82	+ const payload = extractPayload(id, res.html);
	83	+ if (!payload \|\| !payload.name) {
	84	+ ctx.anomaly('parse_failure_slab', url);
	85	+ return null;
	86	+ }
	87	+ return { url, externalId: id, kind: 'listing', engine: 'api', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	88	+ }
	89	+
	90	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	91	+ const max = ctx.options.mode === 'backfill' ? Number(this.meta.config.backfillSlabs ?? 2000) : Number(this.meta.config.maxSlabsPerRun ?? 250);
	92	+ const files = ctx.options.mode === 'backfill' ? ['sitemap-slabs-1.xml', 'sitemap-slabs-2.xml'] : ['sitemap-slabs-1.xml'];
	93	+ const urls: Array<{ id: number; loc: string }> = [];
	94	+ for (const f of files) {
	95	+ const res = await ctx.fetch(`${SITE}/${f}`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml,text/xml' }, responseType: 'text', timeoutMs: 120_000 });
	96	+ if (!res.success \|\| !res.html) {
	97	+ ctx.anomaly('page_fetch_failed', `${f}: ${res.error ?? res.httpStatus}`);
	98	+ continue;
	99	+ }
	100	+ for (const e of parseSitemap(res.html)) {
	101	+ const id = Number(e.loc.match(/slab\/view\/(\d+)/)?.[1]);
	102	+ if (Number.isFinite(id)) urls.push({ id, loc: e.loc });
	103	+ }
	104	+ }
	105	+ if (ctx.options.seeds?.length) urls.push(...ctx.options.seeds.map((s) => ({ id: Number(s.match(/\d+/)?.[0] ?? 0), loc: s.startsWith('http') ? s : `${SITE}/slab/view/${s}/` })));
	106	+ urls.sort((a, b) => b.id - a.id);
	107	+ let count = 0;
	108	+ for (const u of urls.slice(0, max)) {
	109	+ if (ctx.signal?.aborted) return;
	110	+ if (this.reached(ctx, count)) return;
	111	+ if (!(await ctx.shouldFetch(u.loc))) continue;
	112	+ const rec = await this.fetchSlab(ctx, u.loc);
	113	+ if (!rec) continue;
	114	+ count++;
	115	+ yield rec;
	116	+ if (count % 25 === 0) await ctx.setCursor({ lastId: u.id, at: new Date().toISOString() });
	117	+ }
	118	+ await ctx.setCursor({ completedAt: new Date().toISOString(), newestId: urls[0]?.id ?? null });
	119	+ }
	120	+
	121	+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
	122	+ const rec = await this.fetchSlab(ctx, url);
	123	+ return rec ? [rec] : [];
	124	+ }
	125	+
	126	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	127	+ const p = RawPayloadSchema.parse(raw.payload);
	128	+ const desc = (p.description ?? '').replace(/<[^>]+>/g, ' ').replace(/&[a-z]+;/g, ' ').slice(0, 400);
	129	+ const categorySlug = slabCategory(`${p.name} ${desc}`, p.category);
	130	+ if (!categorySlug) return [];
	131	+ // Slab pages carry the grade only when the seller typed it (title or description); never guessed.
	132	+ const g = parseGradeFromTitle(p.name).grader ? parseGradeFromTitle(p.name) : parseGradeFromTitle(desc);
	133	+ const cert = p.description?.match(/\b(?:cert\|certification\|serial)\s(?:#\|no\.?\|number)?\s[:\-]?\s*(\d{6,12})\b/i)?.[1] ?? null;
	134	+ const year = extractYear(p.name);
	135	+ const number = p.name.match(/#\s?([A-Za-z0-9-]+)/)?.[1] ?? null;
	136	+ const a = attrs({ categorySlug, name: p.name.replace(/\s\b(PSA\|BGS\|CGC\|SGC\|TAG\|CSG\|CBCS)\b.$/i, '').trim() \|\| p.name, year, number, identifiers: { myslabs_id: p.id }, metadata: { seller: p.seller, category_label: p.category } });
	137	+ const availability = p.availability === 'OutOfStock' \|\| p.availability === 'SoldOut' ? 'sold' : p.availability === 'InStock' \|\| p.availability === 'PreOrder' ? 'available' : 'unknown';
	138	+ const currency = (p.currency ?? 'USD').toUpperCase();
	139	+ const listing = NormalizedListingSchema.parse({
	140	+ kind: 'listing',
	141	+ connectorId: this.meta.id,
	142	+ sourceId: this.meta.sourceId,
	143	+ sourceUrl: raw.url,
	144	+ externalId: p.id,
	145	+ rawTitle: p.name,
	146	+ description: p.description,
	147	+ imageUrls: p.images,
	148	+ attributes: a,
	149	+ grade: { grader: g.grader, grade: g.grade, qualifier: g.qualifier, certificationNumber: cert },
	150	+ condition: {},
	151	+ observedAt: raw.fetchedAt,
	152	+ confidence: 0.75,
	153	+ parserVersion: PARSER_VERSION,
	154	+ listingType: 'fixed_price',
	155	+ price: p.price,
	156	+ currency: p.price === null ? null : currency,
	157	+ seller: p.seller,
	158	+ availability,
	159	+ });
	160	+ return [listing];
	161	+ }
	162	+}
	163	+
	164	+export default (meta: ConnectorMeta) => new MySlabsConnector(meta);

added connectors/api/myslabs/meta.json +33 −0

		`@@ -0,0 +1,33 @@`
	1	+{
	2	+ "id": "myslabs",
	3	+ "displayName": "MySlabs (graded card & comic marketplace)",
	4	+ "sourceId": "myslabs",
	5	+ "sourceName": "MySlabs",
	6	+ "sourceType": "marketplace",
	7	+ "sourceUrl": "https://myslabs.com",
	8	+ "module": "api/myslabs",
	9	+ "enginePriority": ["api"],
	10	+ "categories": ["sports_cards", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "pokemon", "magic_the_gathering", "yugioh", "non_sport_cards", "comics", "marvel_comics", "dc_comics", "independent_comics"],
	11	+ "regions": ["US"],
	12	+ "languages": ["en"],
	13	+ "currency": ["USD"],
	14	+ "supportsListings": true,
	15	+ "supportsSold": false,
	16	+ "supportsAuctions": false,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": false,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": true,
	21	+ "refreshFrequencyMinutes": 720,
	22	+ "priority": "medium",
	23	+ "trustScore": 0.7,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://myslabs.com/static/terms",
	26	+ "accessNotes": "Public fixed-price marketplace for graded slabs. robots.txt only disallows /account/, /api/, /admin/, /ajax_select/. Discovery uses the public sitemaps (sitemap-slabs-*.xml, newest ids first); slab pages are plain HTML with a schema.org Product JSON-LD (name, images, offers.price USD, availability InStock/OutOfStock). OutOfStock slabs are stored as listings with availability 'sold' at their last asking price — never as confirmed sales. Politeness 1.5 s between pages; ~250 slabs per incremental run (`maxSlabsPerRun`).",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "maxSlabsPerRun": 250,
	31	+ "backfillSlabs": 2000
	32	+ }
	33	+}

added connectors/firecrawl/alt-xyz/index.test.ts +94 −0

		`@@ -0,0 +1,94 @@`
	1	+import { describe, expect, it } from 'vitest';
	2	+import meta from './meta.json' with { type: 'json' };
	3	+import { localMeta } from '../../api/_lib/local-meta.js';
	4	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	5	+import createConnector, { parseItemMarkdown } from './index.js';
	6	+
	7	+const connector = createConnector(localMeta(meta));
	8	+
	9	+const SAMPLE = `Back
	10	+
	11	+# 2023 Bowman Chrome Prospect Autograph Purple Refractor Ludwing Espinoza \\#CPALE
	12	+
	13	+## Baseball Cards
	14	+
	15	+Serial207/250
	16	+
	17	+PSA10
	18	+
	19	+Pop24
	20	+
	21	+## List price
	22	+
	23	+### $45
	24	+
	25	+Buy nowOffer
	26	+
	27	+## LT Value
	28	+
	29	+$18
	30	+
	31	+### $12 - $29
	32	+
	33	+PSA population
	34	+
	35	+### Recent transactions
	36	+
	37	+[![eBay](https://x/e.png)\\
	38	+\\
	39	+AuctionAug 6, 2026\\
	40	+\\
	41	+$15](https://www.ebay.com/itm/278232527458)
	42	+
	43	+[![eBay](https://x/e.png)\\
	44	+\\
	45	+Best offerSep 3, 2024\\
	46	+\\
	47	+$20](https://www.ebay.com/itm/405178094963)
	48	+
	49	+View all
	50	+
	51	+Listings
	52	+
	53	+[![eBay](https://x/e.png)\\
	54	+\\
	55	+Fixed price\\
	56	+\\
	57	+$60\\
	58	+\\
	59	+live listing in eBay](https://www.ebay.com/itm/176907976889)
	60	+`;
	61	+
	62	+describe('alt-xyz', () => {
	63	+ runFixtureSuite(connector, it, expect);
	64	+
	65	+ it('parses item markdown', () => {
	66	+ const p = parseItemMarkdown(SAMPLE, 'abc')!;
	67	+ expect(p.title).toContain('Ludwing Espinoza #CPALE');
	68	+ expect(p.category).toBe('Baseball Cards');
	69	+ expect(p).toMatchObject({ grader: 'psa', grade: '10', pop: 24, listPrice: 45, listingKind: 'fixed_price', altValue: 18, altLow: 12, altHigh: 29, serial: '207/250' });
	70	+ expect(p.transactions).toEqual([
	71	+ { type: 'Auction', date: 'Aug 6, 2026', price: 15, url: 'https://www.ebay.com/itm/278232527458' },
	72	+ { type: 'Best offer', date: 'Sep 3, 2024', price: 20, url: 'https://www.ebay.com/itm/405178094963' },
	73	+ ]);
	74	+ });
	75	+
	76	+ it('normalises into observation + sales + listing', async () => {
	77	+ const raw = { url: 'https://alt.xyz/itm/abc', externalId: 'abc', kind: 'sale' as const, engine: 'firecrawl' as const, fetchedAt: new Date('2026-09-07T12:00:00Z'), payload: parseItemMarkdown(SAMPLE, 'abc') };
	78	+ const out = await connector.normalize(raw);
	79	+ const kinds = out.map((r) => r.kind);
	80	+ expect(kinds.filter((k) => k === 'sale')).toHaveLength(2);
	81	+ expect(kinds).toContain('price_observation');
	82	+ expect(kinds).toContain('listing');
	83	+ const sale = out.find((r) => r.kind === 'sale');
	84	+ if (sale && sale.kind === 'sale') {
	85	+ expect(sale.saleDate.toISOString()).toBe('2026-08-06T00:00:00.000Z');
	86	+ expect(sale.attributes.categorySlug).toBe('baseball_cards');
	87	+ expect(sale.grade.grader).toBe('psa');
	88	+ }
	89	+ for (const name of listFixtures('alt-xyz')) {
	90	+ const fx = await connector.normalize(loadFixture('alt-xyz', name).raw);
	91	+ expect(fx.length).toBeGreaterThan(0);
	92	+ }
	93	+ });
	94	+});

added connectors/firecrawl/alt-xyz/index.ts +168 −0

		`@@ -0,0 +1,168 @@`
	1	+import { z } from 'zod';
	2	+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { NormalizedListingSchema, NormalizedSaleSchema, extractYear, type NormalizedRecord } from '@rareindex/shared';
	4	+import { attrs, priceObservation } from '../../api/_lib/shared.js';
	5	+import { BOT_HEADERS, cardCategorySlug, dayOf, fetchMaybeGzip, money, parseMonthDay, parseSitemap } from '../../api/_lib/wave4.js';
	6	+
	7	+/** Alt item pages → Alt Value observations, recent transactions (sales) and Alt list-price listings. */
	8	+const PARSER_VERSION = '1.0.0';
	9	+const SITE = 'https://alt.xyz';
	10	+
	11	+const TxSchema = z.object({ type: z.string(), date: z.string(), price: z.number(), url: z.string().nullable() });
	12	+const RawPayloadSchema = z.object({
	13	+ id: z.string(),
	14	+ title: z.string(),
	15	+ category: z.string().nullable(),
	16	+ serial: z.string().nullable(),
	17	+ grader: z.string().nullable(),
	18	+ grade: z.string().nullable(),
	19	+ pop: z.number().nullable(),
	20	+ listPrice: z.number().nullable(),
	21	+ listingKind: z.enum(['fixed_price', 'auction']).nullable().default(null),
	22	+ altValue: z.number().nullable(),
	23	+ altLow: z.number().nullable(),
	24	+ altHigh: z.number().nullable(),
	25	+ transactions: z.array(TxSchema),
	26	+ images: z.array(z.string()),
	27	+});
	28	+export type AltPayload = z.infer<typeof RawPayloadSchema>;
	29	+
	30	+/** Parse the Firecrawl markdown of an item page. Exported for tests. */
	31	+export function parseItemMarkdown(md: string, id: string): AltPayload \| null {
	32	+ const text = md.replace(/\\\n/g, '\n').replace(/\\([#$])/g, '$1');
	33	+ const title = text.match(/^#\s+(.+)$/m)?.[1]?.trim();
	34	+ if (!title) return null;
	35	+ const afterTitle = text.slice(text.indexOf(title) + title.length);
	36	+ const category = afterTitle.match(/^##\s+([A-Za-z' \-]+Cards?)\s$/m)?.[1]?.trim() ?? afterTitle.match(/^##\s+(Pok[eé]mon\|Magic[^\n]\|Yu-Gi-Oh!?)\s*$/m)?.[1]?.trim() ?? null;
	37	+ const serial = afterTitle.match(/Serial\s(\d+\s\/\s*\d+)/)?.[1]?.replace(/\s/g, '') ?? null;
	38	+ const gm = afterTitle.match(/^(PSA\|BGS\|SGC\|CGC\|CSG\|TAG\|HGA)\s?(\d{1,2}(?:\.\d)?)\s*$/m);
	39	+ const pop = afterTitle.match(/^Pop\s([\d,]+)\s$/m)?.[1];
	40	+ const fixed = afterTitle.match(/##\s+List price\s\n+\s###\s+\$([\d,]+(?:\.\d+)?)/)?.[1] ?? null;
	41	+ const bid = afterTitle.match(/(?:Starting bid\|Current bid\|High bid)\s*\$([\d,]+(?:\.\d+)?)/)?.[1] ?? null;
	42	+ const listPrice = fixed ?? bid;
	43	+ const listingKind: 'fixed_price' \| 'auction' \| null = fixed ? 'fixed_price' : bid ? 'auction' : null;
	44	+ const av = afterTitle.match(/##\s+(?:LT\|Alt) Value\s\n+\s\$([\d,]+(?:\.\d+)?)\s\n+\s###\s+\$([\d,]+(?:\.\d+)?)\s-\s\$([\d,]+(?:\.\d+)?)/);
	45	+ const txStart = afterTitle.search(/Recent transactions/i);
	46	+ const txEnd = afterTitle.search(/\n##?\s+(Listings\|Similar listings)\|\nListings\s*\n/);
	47	+ const txBlock = txStart >= 0 ? afterTitle.slice(txStart, txEnd > txStart ? txEnd : undefined) : '';
	48	+ const transactions: z.infer<typeof TxSchema>[] = [];
	49	+ const re = /(Auction\|Best offer\|Fixed price\|Buy now\|Sale)\s([A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})\s\n+\s*\$([\d,]+(?:\.\d+)?)\]$(https?:[^)\s]+)$/g;
	50	+ let m: RegExpExecArray \| null;
	51	+ while ((m = re.exec(txBlock))) {
	52	+ const price = money(m[3]);
	53	+ if (!price) continue;
	54	+ transactions.push({ type: m[1]!, date: m[2]!, price, url: m[4] ?? null });
	55	+ }
	56	+ const images = [...text.matchAll(/!\[[^\]]\]$(https:\/\/alt-images\.b-cdn\.net\/public\/[^)\s]+width=324[^)\s])$/g)].map((x) => x[1]!);
	57	+ return {
	58	+ id,
	59	+ title,
	60	+ category,
	61	+ serial,
	62	+ grader: gm ? gm[1]!.toLowerCase() : null,
	63	+ grade: gm ? gm[2]! : null,
	64	+ pop: pop ? Number(pop.replace(/,/g, '')) : null,
	65	+ listPrice: money(listPrice),
	66	+ listingKind,
	67	+ altValue: av ? money(av[1]) : null,
	68	+ altLow: av ? money(av[2]) : null,
	69	+ altHigh: av ? money(av[3]) : null,
	70	+ transactions,
	71	+ images: [...new Set(images)],
	72	+ };
	73	+}
	74	+
	75	+async function fetchGzText(url: string, signal?: AbortSignal): Promise<string> {
	76	+ return (await fetchMaybeGzip(url, signal)).toString('utf8');
	77	+}
	78	+
	79	+export class AltConnector extends BaseConnector {
	80	+ readonly version = '1.0.0';
	81	+ readonly parserVersion = PARSER_VERSION;
	82	+ override readonly urlPatterns = [/^https?:\/\/(www\.)?alt\.xyz\/itm\/[0-9a-f-]{36}/i];
	83	+ protected override minIntervalMs = 1500;
	84	+
	85	+ private async fetchItem(ctx: CrawlContext, url: string): Promise<RawRecordInput \| null> {
	86	+ await this.throttle();
	87	+ const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 9000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.match(/^#\s+.+$/m) ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) });
	88	+ if (!res.success \|\| !res.markdown) {
	89	+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
	90	+ return null;
	91	+ }
	92	+ const id = url.match(/itm\/([0-9a-f-]{36})/i)?.[1] ?? url;
	93	+ const payload = parseItemMarkdown(res.markdown, id);
	94	+ if (!payload) {
	95	+ ctx.anomaly('parse_failure_item', url);
	96	+ return null;
	97	+ }
	98	+ return { url: `${SITE}/itm/${id}`, externalId: id, kind: 'sale', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	99	+ }
	100	+
	101	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	102	+ const max = ctx.options.mode === 'backfill' ? Number(this.meta.config.maxItemsPerRun ?? 100) * 5 : Number(this.meta.config.maxItemsPerRun ?? 100);
	103	+ let urls: Array<{ loc: string; lastmod: string }> = [];
	104	+ if (ctx.options.seeds?.length) urls = ctx.options.seeds.map((s) => ({ loc: s.startsWith('http') ? s : `${SITE}/itm/${s}`, lastmod: '' }));
	105	+ else {
	106	+ const idx = await ctx.fetch(`${SITE}/sitemap.xml`, { engines: ['api'], headers: { ...BOT_HEADERS, accept: 'application/xml' }, responseType: 'text' });
	107	+ if (!idx.success \|\| !idx.html) throw new Error(`alt sitemap index failed: ${idx.error ?? idx.httpStatus}`);
	108	+ const wanted = (this.meta.config.sitemaps as string[] \| undefined) ?? ['fixed-price', 'auctions'];
	109	+ const children = [...idx.html.matchAll(/<loc>\s([^<\s]+)\s<\/loc>/g)].map((x) => x[1]!).filter((u) => wanted.some((w) => u.includes(`/${w}-`)));
	110	+ for (const child of children.slice(0, 6)) {
	111	+ try {
	112	+ const xml = await fetchGzText(child, ctx.signal);
	113	+ for (const e of parseSitemap(xml)) urls.push({ loc: e.loc, lastmod: e.lastmod ?? '' });
	114	+ } catch (err) {
	115	+ ctx.anomaly('page_fetch_failed', `${child}: ${err instanceof Error ? err.message : String(err)}`);
	116	+ }
	117	+ }
	118	+ const since = String(ctx.options.cursor?.since ?? '');
	119	+ urls = urls.filter((u) => !since \|\| u.lastmod > since).sort((a, b) => b.lastmod.localeCompare(a.lastmod));
	120	+ }
	121	+ let count = 0;
	122	+ let newest = String(ctx.options.cursor?.since ?? '');
	123	+ for (const u of urls.slice(0, max)) {
	124	+ if (ctx.signal?.aborted) return;
	125	+ if (this.reached(ctx, count)) break;
	126	+ const rec = await this.fetchItem(ctx, u.loc);
	127	+ if (!rec) continue;
	128	+ count++;
	129	+ if (u.lastmod > newest) newest = u.lastmod;
	130	+ yield rec;
	131	+ if (count % 20 === 0) await ctx.setCursor({ since: newest, at: new Date().toISOString() });
	132	+ }
	133	+ await ctx.setCursor({ since: newest, completedAt: new Date().toISOString() });
	134	+ }
	135	+
	136	+ async lookup(url: string, ctx: CrawlContext): Promise<RawRecordInput[]> {
	137	+ const rec = await this.fetchItem(ctx, url);
	138	+ return rec ? [rec] : [];
	139	+ }
	140	+
	141	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	142	+ const p = RawPayloadSchema.parse(raw.payload);
	143	+ const categorySlug = cardCategorySlug(p.category) ?? cardCategorySlug(p.title);
	144	+ if (!categorySlug) return [];
	145	+ const year = extractYear(p.title);
	146	+ const number = p.title.match(/#\s?([A-Za-z0-9-]+)/)?.[1] ?? null;
	147	+ const name = p.title.replace(/^\d{4}(?:-\d{2})?\s+/, '').replace(/\s#\S+\s$/, '').trim() \|\| p.title;
	148	+ const a = attrs({ categorySlug, name, year, number, identifiers: { alt_item_id: p.id }, metadata: { serial: p.serial, pop: p.pop } });
	149	+ const grade = { grader: p.grader, grade: p.grade, qualifier: null, certificationNumber: null };
	150	+ const base = { connectorId: this.meta.id, sourceId: this.meta.sourceId, sourceUrl: raw.url, rawTitle: p.title, imageUrls: p.images, attributes: a, grade, condition: {}, observedAt: raw.fetchedAt, parserVersion: PARSER_VERSION };
	151	+ const out: NormalizedRecord[] = [];
	152	+ if (p.altValue) {
	153	+ out.push(priceObservation({ kind: 'price_observation', ...base, externalId: `${p.id}:altvalue:${dayOf(raw.fetchedAt).toISOString().slice(0, 10)}`, confidence: 0.6, priceKind: 'guide_value', price: p.altValue, currency: 'USD', observationDate: dayOf(raw.fetchedAt), sampleSize: p.transactions.length \|\| null, attributes: { ...a, metadata: { ...a.metadata, low: p.altLow, high: p.altHigh, model: 'Alt Value' } } }));
	154	+ }
	155	+ for (const t of p.transactions) {
	156	+ const saleDate = parseMonthDay(t.date);
	157	+ if (!saleDate) continue;
	158	+ const saleType = /auction/i.test(t.type) ? 'auction' : /best offer/i.test(t.type) ? 'best_offer' : 'fixed_price';
	159	+ out.push(NormalizedSaleSchema.parse({ kind: 'sale', ...base, externalId: `${p.id}:${saleDate.toISOString().slice(0, 10)}:${t.price}`, confidence: 0.65, saleType, saleDate, price: t.price, currency: 'USD', buyerPremiumIncluded: null, quantity: 1, isBundle: false, location: null, auctionHouse: t.url?.includes('ebay.') ? 'eBay (via Alt)' : null, lotNumber: null, attributes: { ...a, metadata: { ...a.metadata, external_url: t.url, via: 'alt.xyz' } } }));
	160	+ }
	161	+ if (p.listPrice) {
	162	+ out.push(NormalizedListingSchema.parse({ kind: 'listing', ...base, externalId: p.id, confidence: 0.75, listingType: p.listingKind ?? 'fixed_price', price: p.listPrice, currency: 'USD', seller: 'Alt marketplace', availability: 'available' }));
	163	+ }
	164	+ return out;
	165	+ }
	166	+}
	167	+
	168	+export default (meta: ConnectorMeta) => new AltConnector(meta);

added connectors/firecrawl/alt-xyz/meta.json +33 −0

		`@@ -0,0 +1,33 @@`
	1	+{
	2	+ "id": "alt-xyz",
	3	+ "displayName": "Alt (graded card values & transactions)",
	4	+ "sourceId": "alt",
	5	+ "sourceName": "Alt",
	6	+ "sourceType": "analytics_provider",
	7	+ "sourceUrl": "https://alt.xyz",
	8	+ "module": "firecrawl/alt-xyz",
	9	+ "enginePriority": ["firecrawl"],
	10	+ "categories": ["sports_cards", "baseball_cards", "basketball_cards", "football_cards", "hockey_cards", "soccer_cards", "other_sports_cards", "pokemon", "magic_the_gathering", "yugioh", "non_sport_cards"],
	11	+ "regions": ["US"],
	12	+ "languages": ["en"],
	13	+ "currency": ["USD"],
	14	+ "supportsListings": true,
	15	+ "supportsSold": true,
	16	+ "supportsAuctions": false,
	17	+ "supportsImages": true,
	18	+ "supportsCatalog": false,
	19	+ "supportsPopulation": false,
	20	+ "supportsLookup": true,
	21	+ "refreshFrequencyMinutes": 1440,
	22	+ "priority": "low",
	23	+ "trustScore": 0.7,
	24	+ "attributionRequired": true,
	25	+ "termsUrl": "https://alt.xyz/terms",
	26	+ "accessNotes": "Alt's public item pages (robots.txt allows everything; discovery through the public sitemaps with lastmod) show the card, grade, PSA/BGS population, Alt's list price, the 'Alt Value' model estimate with its range, and a 'Recent transactions' list of observed sales (mostly eBay auctions/best offers) with date and price. Pages are client-rendered → Firecrawl, 1 credit per item, capped by `maxItemsPerRun`. Alt Value is stored as a guide_value observation (confidence 0.6); recent transactions become sales attributed to Alt with the original listing URL in metadata (confidence 0.65 — second-hand aggregation); the Alt list price is a fixed-price listing.",
	27	+ "enabled": true,
	28	+ "schemaVersion": "1.0",
	29	+ "config": {
	30	+ "sitemaps": ["fixed-price", "auctions"],
	31	+ "maxItemsPerRun": 100
	32	+ }
	33	+}

added connectors/firecrawl/comc/index.test.ts +49 −0

		`@@ -0,0 +1,49 @@`
	1	+import { describe, expect, it } from 'vitest';
	2	+import meta from './meta.json' with { type: 'json' };
	3	+import { localMeta } from '../../api/_lib/local-meta.js';
	4	+import { listFixtures, loadFixture, runFixtureSuite } from '@rareindex/connectors/testing';
	5	+import createConnector, { parseBrowseMarkdown, parseComcTitle, parseSetLine } from './index.js';
	6	+
	7	+const connector = createConnector(localMeta(meta));
	8	+
	9	+const SAMPLE = `Listings 1 \\- 100 of 1,292
	10	+
	11	+[![Steve Johnson [CSG 10 Gem Mint]](https://img.comc.com/i/Basketball/1986-87/Fleer---Base/55/Steve-Johnson.jpg?id=fc68&size=biggerthumb)](https://www.comc.com/Cards/Basketball/1986-87/Fleer_-_Base/55/Steve_Johnson/1395657/Graded/CSG/10_Gem)
	12	+
	13	+1986-87 Fleer - \\[Base\\] #55
	14	+
	15	+
	16	+### [Steve Johnson \\[CSG 10 Gem Mint\\]](https://www.comc.com/Cards/Basketball/1986-87/Fleer_-_Base/55/Steve_Johnson/1395657/Graded/CSG/10_Gem)
	17	+
	18	+$1,791.76
	19	+`;
	20	+
	21	+describe('comc', () => {
	22	+ runFixtureSuite(connector, it, expect);
	23	+
	24	+ it('parses browse rows, titles and set lines', () => {
	25	+ const rows = parseBrowseMarkdown(SAMPLE);
	26	+ expect(rows).toHaveLength(1);
	27	+ expect(rows[0]).toMatchObject({ price: 1791.76, setLine: '1986-87 Fleer - [Base] #55' });
	28	+ expect(parseComcTitle('Steve Johnson [CSG 10 Gem Mint]')).toMatchObject({ name: 'Steve Johnson', grader: 'csg', grade: '10' });
	29	+ expect(parseComcTitle('Michael Jordan [PSA 9 MINT]')).toMatchObject({ grader: 'psa', grade: '9' });
	30	+ expect(parseComcTitle('Larry Bird')).toMatchObject({ grader: null, grade: null });
	31	+ expect(parseSetLine('1986-87 Fleer - [Base] #57')).toEqual({ year: 1986, set: '1986 Fleer', variant: null, number: '57' });
	32	+ expect(parseSetLine('2018-19 Panini Prizm - [Base] - Silver Prizm #280')).toMatchObject({ year: 2018, number: '280' });
	33	+ });
	34	+
	35	+ it('emits USD listings with card attributes', async () => {
	36	+ for (const name of listFixtures('comc')) {
	37	+ const out = await connector.normalize(loadFixture('comc', name).raw);
	38	+ expect(out.length).toBeGreaterThan(0);
	39	+ for (const r of out) {
	40	+ expect(r.kind).toBe('listing');
	41	+ if (r.kind === 'listing') {
	42	+ expect(r.currency).toBe('USD');
	43	+ expect(r.price).toBeGreaterThan(0);
	44	+ expect(r.sourceUrl).toMatch(/comc\.com\/Cards\//);
	45	+ }
	46	+ }
	47	+ }
	48	+ });
	49	+});

added connectors/firecrawl/comc/index.ts +149 −0

		`@@ -0,0 +1,149 @@`
	1	+import { z } from 'zod';
	2	+import { BaseConnector, type ConnectorMeta, type CrawlContext, type RawRecordInput, type RawRecordLike } from '@rareindex/connectors';
	3	+import { NormalizedListingSchema, type NormalizedRecord } from '@rareindex/shared';
	4	+import { attrs } from '../../api/_lib/shared.js';
	5	+import { cardCategorySlug, money } from '../../api/_lib/wave4.js';
	6	+
	7	+/** COMC category browse pages (Firecrawl markdown) → card listings with grades and asking prices. */
	8	+const PARSER_VERSION = '1.0.0';
	9	+const SITE = 'https://www.comc.com';
	10	+
	11	+const ListingSchema = z.object({ title: z.string(), url: z.string(), image: z.string().nullable(), setLine: z.string().nullable(), price: z.number() });
	12	+const RawPayloadSchema = z.object({ seed: z.string(), page: z.number(), listings: z.array(ListingSchema) });
	13	+export type ComcPayload = z.infer<typeof RawPayloadSchema>;
	14	+
	15	+/** Parse a rendered browse page. Exported for tests. */
	16	+export function parseBrowseMarkdown(md: string): z.infer<typeof ListingSchema>[] {
	17	+ const lines = md.split('\n').map((l) => l.trim());
	18	+ const out: z.infer<typeof ListingSchema>[] = [];
	19	+ let setLine: string \| null = null;
	20	+ let image: string \| null = null;
	21	+ for (let i = 0; i < lines.length; i++) {
	22	+ const l = lines[i]!;
	23	+ const img = l.match(/^\[!\[[^\]]*\]$(https?:\/\/img\.comc\.com[^)\s]+)$\]$https?:\/\/www\.comc\.com\/Cards\/[^)]+$$/);
	24	+ if (img) {
	25	+ image = img[1]!;
	26	+ continue;
	27	+ }
	28	+ if (/^\d{4}(?:-\d{2})?\s.+#\S+$/.test(l.replace(/\\/g, ''))) {
	29	+ setLine = l.replace(/\\/g, '');
	30	+ continue;
	31	+ }
	32	+ const h = l.match(/^###\s+\[(.+?)\]$(https?:\/\/www\.comc\.com\/Cards\/[^)\s]+)$\s*$/);
	33	+ if (!h) continue;
	34	+ let price: number \| null = null;
	35	+ for (let j = i + 1; j < Math.min(lines.length, i + 5); j++) {
	36	+ const m = lines[j]!.match(/^\$([\d,]+\.\d{2})$/);
	37	+ if (m) {
	38	+ price = money(m[1]);
	39	+ break;
	40	+ }
	41	+ if (lines[j]!.startsWith('###')) break;
	42	+ }
	43	+ if (price === null) continue;
	44	+ out.push({ title: h[1]!.replace(/\\/g, ''), url: h[2]!, image, setLine, price });
	45	+ image = null;
	46	+ }
	47	+ return out;
	48	+}
	49	+
	50	+const GRADE_RE = /\[(PSA\|BGS\|CSG\|CGC\|SGC\|TAG\|HGA\|ACE\|BVG\|GMA\|ISA\|KSA\|MNT\|BCCG\|PSA\/DNA\|BAS\|JSA)\s(\d{1,2}(?:\.\d)?)?\s([^\]]*)\]/i;
	51	+
	52	+export function parseComcTitle(title: string): { name: string; grader: string \| null; grade: string \| null; qualifier: string \| null; auto: boolean } {
	53	+ const m = title.match(GRADE_RE);
	54	+ const name = title.replace(/\s\[[^\]]\]\s*/g, ' ').replace(/\s+/g, ' ').trim();
	55	+ if (!m) return { name, grader: null, grade: null, qualifier: null, auto: /\bauto/i.test(title) };
	56	+ const grader = m[1]!.toLowerCase().replace('/', '_');
	57	+ const grade = m[2] ?? null;
	58	+ const qualifier = m[3]?.trim() \|\| null;
	59	+ const isAuth = /dna\|bas\|jsa/.test(grader);
	60	+ return { name, grader: isAuth && !grade ? null : grader, grade, qualifier: isAuth ? `${m[1]} ${qualifier ?? ''}`.trim() : qualifier, auto: /\bauto/i.test(title) };
	61	+}
	62	+
	63	+export function parseSetLine(setLine: string \| null): { year: number \| null; set: string \| null; variant: string \| null; number: string \| null } {
	64	+ if (!setLine) return { year: null, set: null, variant: null, number: null };
	65	+ const m = setLine.match(/^(\d{4})(?:-\d{2})?\s+(.+?)(?:\s+-\s+\[(.+?)\])?\s+#(\S+)$/);
	66	+ if (!m) return { year: null, set: setLine, variant: null, number: null };
	67	+ const year = Number(m[1]);
	68	+ const variant = m[3] && !/^base$/i.test(m[3]) ? m[3] : null;
	69	+ return { year, set: `${m[1]} ${m[2]}`.trim(), variant, number: m[4] ?? null };
	70	+}
	71	+
	72	+function seedCategory(seed: string): string \| null {
	73	+ const sport = seed.split('/')[1]?.replace(/_/g, ' ') ?? '';
	74	+ return cardCategorySlug(sport) ?? (/^racing$/i.test(sport) ? 'other_sports_cards' : null);
	75	+}
	76	+
	77	+export class ComcConnector extends BaseConnector {
	78	+ readonly version = '1.0.0';
	79	+ readonly parserVersion = PARSER_VERSION;
	80	+ protected override minIntervalMs = 2000;
	81	+
	82	+ async *crawl(ctx: CrawlContext): AsyncIterable<RawRecordInput> {
	83	+ const seeds = ctx.options.seeds?.length ? ctx.options.seeds : ((this.meta.config.seeds as string[] \| undefined) ?? []);
	84	+ const pages = ctx.options.mode === 'backfill' ? 10 : Number(this.meta.config.pagesPerSeed ?? 2);
	85	+ let count = 0;
	86	+ for (const seed of seeds) {
	87	+ for (let page = 1; page <= pages; page++) {
	88	+ if (ctx.signal?.aborted) return;
	89	+ if (this.reached(ctx, count)) return;
	90	+ const url = `${SITE}/${seed.replace(/^\/+/, '')},sh${page > 1 ? `,p${page}` : ''}`;
	91	+ await this.throttle();
	92	+ const res = await ctx.fetch(url, { engines: ['firecrawl'], waitForMs: 3000, timeoutMs: 90_000, expect: ['title', 'price'], parse: (r) => ({ title: r.markdown?.includes('Listings') ? 'ok' : null, price: r.markdown && /\$\d/.test(r.markdown) ? 1 : null }) });
	93	+ if (!res.success \|\| !res.markdown) {
	94	+ ctx.anomaly('page_fetch_failed', `${url}: ${res.error ?? res.httpStatus}`);
	95	+ break;
	96	+ }
	97	+ const listings = parseBrowseMarkdown(res.markdown);
	98	+ if (!listings.length) {
	99	+ ctx.anomaly('parse_failure_page', url);
	100	+ break;
	101	+ }
	102	+ count++;
	103	+ const payload: ComcPayload = { seed, page, listings };
	104	+ yield { url, externalId: `${seed}:p${page}`, kind: 'listing', engine: 'firecrawl', httpStatus: res.httpStatus, payload, fetchedAt: res.fetchedAt };
	105	+ const total = res.markdown.match(/of\s+\\([\d,]+)\\/)?.[1];
	106	+ if (total && Number(total.replace(/,/g, '')) <= page * 100) break;
	107	+ }
	108	+ await ctx.setCursor({ seed, at: new Date().toISOString() });
	109	+ }
	110	+ }
	111	+
	112	+ async normalize(raw: RawRecordLike): Promise<NormalizedRecord[]> {
	113	+ const p = RawPayloadSchema.parse(raw.payload);
	114	+ const categorySlug = seedCategory(p.seed);
	115	+ if (!categorySlug) return [];
	116	+ const out: NormalizedRecord[] = [];
	117	+ for (const l of p.listings) {
	118	+ const t = parseComcTitle(l.title);
	119	+ const s = parseSetLine(l.setLine);
	120	+ const externalId = l.url.replace(`${SITE}/Cards/`, '').replace(/\/+$/, '');
	121	+ const a = attrs({ categorySlug, name: t.name, set: s.set, year: s.year, number: s.number, variant: s.variant, identifiers: { comc_path: externalId }, metadata: { autograph: t.auto } });
	122	+ out.push(
	123	+ NormalizedListingSchema.parse({
	124	+ kind: 'listing',
	125	+ connectorId: this.meta.id,
	126	+ sourceId: this.meta.sourceId,
	127	+ sourceUrl: l.url,
	128	+ externalId,
	129	+ rawTitle: l.setLine ? `${l.setLine} ${l.title}` : l.title,
	130	+ imageUrls: l.image ? [l.image.replace(/size=biggerthumb/, 'size=large')] : [],
	131	+ attributes: a,
	132	+ grade: { grader: t.grader, grade: t.grade, qualifier: t.qualifier },
	133	+ condition: {},
	134	+ observedAt: raw.fetchedAt,
	135	+ confidence: 0.8,
	136	+ parserVersion: PARSER_VERSION,
	137	+ listingType: 'fixed_price',
	138	+ price: l.price,
	139	+ currency: 'USD',
	140	+ seller: 'COMC consignment',
	141	+ availability: 'available',
	142	+ }),
	143	+ );
	144	+ }
	145	+ return out;
	146	+ }
	147	+}
	148	+
	149	+export default (meta: ConnectorMeta) => new ComcConnector(meta);

added connectors/firecrawl/comc/meta.json +48 −0

Large diff (48 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/tag-pop/index.test.ts +43 −0

Large diff (43 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/tag-pop/index.ts +148 −0

Large diff (148 lines) is not loaded by default to keep the page fast.Load diff

added connectors/firecrawl/tag-pop/meta.json +33 −0

Large diff (33 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/alt-xyz/item-priced.json +42 −0

Large diff (43 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/cardkingdom/single-first.json +42 −0

Large diff (43 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/cardkingdom/single-fourth.json +42 −0

Large diff (43 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/comc/basketball-1986-fleer-p1.json +427 −0

Large diff (428 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/mtgjson/lea-black-lotus.json +91 −0

Large diff (92 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/mtgjson/lea-first.json +89 −0

Large diff (90 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/myslabs/slab-first.json +36 −0

Large diff (37 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/myslabs/slab-sixth.json +36 −0

Large diff (37 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/myslabs/slab-third.json +36 −0

Large diff (37 lines) is not loaded by default to keep the page fast.Load diff

added data/fixtures/tag-pop/pokemon-1999-base-set.json +1971 −0

Large diff (1,973 lines) is not loaded by default to keep the page fast.Load diff

spb/rareindex

Connectors: MTGJSON, Card Kingdom, MySlabs, TAG population, COMC, Alt (agent S)

Theme