TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Helpers shared by the wave-4 card/comic connectors (mtgjson, cardkingdom, myslabs, tag-pop, comc,3 * alt-xyz). Kept inside connectors/api (not the framework).4 */5import { createGunzip, gunzipSync } from 'node:zlib';6import { Readable } from 'node:stream';78export const BOT_UA = 'RareIndexBot/0.1 (+https://www.rareindex.io/about/data; contact data@rareindex.io)';9export const BOT_HEADERS = { 'user-agent': BOT_UA, accept: 'application/json, text/html;q=0.9, */*;q=0.8' };1011/** UTC midnight. */12export function dayOf(d: Date): Date {13 return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));14}1516/** "2026-09-06" → UTC midnight Date. */17export function isoDay(s: string | null | undefined): Date | null {18 if (!s) return null;19 const m = s.match(/^(\d{4})-(\d{2})-(\d{2})/);20 if (!m) return null;21 const d = new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])));22 return Number.isNaN(d.getTime()) ? null : d;23}2425/** "$8,830.00" | "8000.00" → number or null (never ≤ 0). */26export function money(s: string | number | null | undefined): number | null {27 if (s === null || s === undefined) return null;28 const n = typeof s === 'number' ? s : Number.parseFloat(String(s).replace(/[^0-9.]/g, ''));29 return Number.isFinite(n) && n > 0 ? n : null;30}3132/** Download a gzip JSON document fully (small files only, e.g. AllPricesToday ≈ 5 MB gz). */33/** Download a (possibly gzip) document; handles servers that already transparently decode Content-Encoding. */34export async function fetchMaybeGzip(url: string, signal?: AbortSignal): Promise<Buffer> {35 const res = await fetch(url, { headers: BOT_HEADERS, signal });36 if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);37 const buf = Buffer.from(await res.arrayBuffer());38 if (buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b) return gunzipSync(buf);39 return buf;40}4142export async function fetchGzipJson<T = unknown>(url: string, signal?: AbortSignal): Promise<T> {43 return JSON.parse((await fetchMaybeGzip(url, signal)).toString('utf8')) as T;44}4546/**47 * Stream a large gzip JSON document shaped like {"meta":{…},"data":{"<key>":<value>,…}} and invoke48 * `onEntry(key, value)` for each top-level entry of `data` without materialising the whole file49 * (MTGJSON AllPrices ≈ 1.5 GB uncompressed). Returns the number of entries seen.50 */51export async function streamGzipDataEntries(url: string, onEntry: (key: string, value: unknown) => Promise<void> | void, opts: { signal?: AbortSignal; limit?: number } = {}): Promise<number> {52 const res = await fetch(url, { headers: BOT_HEADERS, signal: opts.signal });53 if (!res.ok || !res.body) throw new Error(`${url}: HTTP ${res.status}`);54 const encoded = (res.headers.get('content-type') ?? '').includes('gzip') || /\.gz(\?|$)/.test(url);55 const raw = Readable.fromWeb(res.body as never);56 // Some CDNs serve .gz files with Content-Encoding: gzip, which fetch already decodes → sniff the magic bytes.57 const first = await new Promise<Buffer | null>((resolve, reject) => {58 raw.once('readable', () => resolve((raw.read(2) as Buffer | null) ?? null));59 raw.once('error', reject);60 raw.once('end', () => resolve(null));61 });62 if (first) raw.unshift(first);63 const isGz = first ? first[0] === 0x1f && first[1] === 0x8b : encoded;64 const stream = isGz ? raw.pipe(createGunzip()) : raw;65 let buf = '';66 let inData = false;67 let count = 0;68 let done = false;69 for await (const chunk of stream) {70 if (done) break;71 buf += (chunk as Buffer).toString('utf8');72 if (!inData) {73 const i = buf.indexOf('"data"');74 if (i < 0) {75 buf = buf.slice(-16);76 continue;77 }78 const brace = buf.indexOf('{', i);79 if (brace < 0) continue;80 buf = buf.slice(brace + 1);81 inData = true;82 }83 // Parse as many complete `"key": {…},` entries as the buffer holds.84 for (;;) {85 const k = buf.indexOf('"');86 if (k < 0) break;87 const kEnd = buf.indexOf('"', k + 1);88 if (kEnd < 0) break;89 const colon = buf.indexOf(':', kEnd);90 if (colon < 0) break;91 const start = colon + 1;92 const end = scanJsonValue(buf, start);93 if (end < 0) break; // incomplete value, wait for more data94 const key = buf.slice(k + 1, kEnd);95 const raw = buf.slice(start, end).trim();96 try {97 await onEntry(key, JSON.parse(raw));98 } catch (err) {99 if (err instanceof SyntaxError) {100 /* skip malformed slice */101 } else throw err;102 }103 count++;104 buf = buf.slice(end);105 const comma = buf.indexOf(',');106 const close = buf.indexOf('}');107 if (close >= 0 && (comma < 0 || close < comma)) {108 done = true;109 break;110 }111 buf = comma >= 0 ? buf.slice(comma + 1) : buf;112 if (opts.limit && count >= opts.limit) {113 done = true;114 break;115 }116 }117 }118 return count;119}120121/** Index just past a complete JSON value starting at `start` (whitespace allowed); -1 if incomplete. */122function scanJsonValue(s: string, start: number): number {123 let i = start;124 while (i < s.length && /\s/.test(s[i]!)) i++;125 if (i >= s.length) return -1;126 const c = s[i]!;127 if (c === '{' || c === '[') {128 let depth = 0;129 let inStr = false;130 for (let j = i; j < s.length; j++) {131 const ch = s[j]!;132 if (inStr) {133 if (ch === '\\') j++;134 else if (ch === '"') inStr = false;135 continue;136 }137 if (ch === '"') inStr = true;138 else if (ch === '{' || ch === '[') depth++;139 else if (ch === '}' || ch === ']') {140 depth--;141 if (depth === 0) return j + 1;142 }143 }144 return -1;145 }146 if (c === '"') {147 for (let j = i + 1; j < s.length; j++) {148 if (s[j] === '\\') j++;149 else if (s[j] === '"') return j + 1;150 }151 return -1;152 }153 const m = s.slice(i).match(/^(true|false|null|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/);154 if (!m) return -1;155 const end = i + m[0].length;156 return end < s.length ? end : -1;157}158159/** Parse <url><loc>…</loc><lastmod>…</lastmod></url> entries from a sitemap. */160export function parseSitemap(xml: string): Array<{ loc: string; lastmod: string | null }> {161 const out: Array<{ loc: string; lastmod: string | null }> = [];162 const re = /<url>([\s\S]*?)<\/url>/g;163 let m: RegExpExecArray | null;164 while ((m = re.exec(xml))) {165 const loc = m[1]!.match(/<loc>\s*([^<\s]+)\s*<\/loc>/)?.[1];166 if (!loc) continue;167 const lastmod = m[1]!.match(/<lastmod>\s*([^<\s]+)\s*<\/lastmod>/)?.[1] ?? null;168 out.push({ loc: decodeXml(loc), lastmod });169 }170 return out;171}172173export function decodeXml(s: string): string {174 return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'");175}176177/** Parse GitHub-flavoured markdown tables (as emitted by Firecrawl) into rows of cells. */178export function markdownTables(md: string): string[][][] {179 const tables: string[][][] = [];180 let cur: string[][] | null = null;181 for (const line of md.split('\n')) {182 const t = line.trim();183 if (t.startsWith('|') && t.endsWith('|')) {184 const cells = t.slice(1, -1).split('|').map((c) => c.trim());185 if (cells.every((c) => /^:?-{2,}:?$/.test(c))) continue; // separator row186 if (!cur) cur = [];187 cur.push(cells);188 } else if (cur) {189 tables.push(cur);190 cur = null;191 }192 }193 if (cur) tables.push(cur);194 return tables;195}196197/** Strip markdown link syntax: "[2](https://…)" → "2"; "[Alakazam](url) <br>Holo" → "Alakazam Holo". */198export function mdText(cell: string): string {199 return cell200 .replace(/!\[[^\]]*\]\([^)]*\)/g, '')201 .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')202 .replace(/<br\s*\/?>/gi, ' ')203 .replace(/\\([\\`*_{}\[\]()#+\-.!|])/g, '$1')204 .replace(/\s+/g, ' ')205 .trim();206}207208export function mdLink(cell: string): string | null {209 return cell.match(/\]\((https?:[^)\s]+)\)/)?.[1] ?? null;210}211212export function toInt(s: string): number | null {213 const n = Number.parseInt(s.replace(/[^0-9]/g, ''), 10);214 return Number.isFinite(n) ? n : null;215}216217/** TAG / COMC / Alt category labels → taxonomy slugs (never invented; null when unknown). */218export function cardCategorySlug(label: string | null | undefined): string | null {219 if (!label) return null;220 const s = label.toLowerCase();221 if (/pok[eé]mon/.test(s)) return 'pokemon';222 if (/magic/.test(s)) return 'magic_the_gathering';223 if (/yu-?gi-?oh/.test(s)) return 'yugioh';224 if (/lorcana/.test(s)) return 'disney_lorcana';225 if (/one piece/.test(s)) return 'one_piece_card_game';226 if (/digimon/.test(s)) return 'digimon_tcg';227 if (/flesh/.test(s)) return 'flesh_and_blood';228 if (/star wars/.test(s)) return 'star_wars_tcg';229 if (/dragon ?ball/.test(s)) return 'dragon_ball_tcg';230 if (/baseball/.test(s)) return 'baseball_cards';231 if (/basketball/.test(s)) return 'basketball_cards';232 if (/football/.test(s)) return 'football_cards';233 if (/hockey/.test(s)) return 'hockey_cards';234 if (/soccer|futbol/.test(s)) return 'soccer_cards';235 if (/formula|f1\b|racing/.test(s)) return 'f1_cards';236 if (/boxing|golf|tennis|wrestling|wwe|ufc|mma|olympic|lacrosse|rugby|cricket|nascar|multi-?sport/.test(s)) return 'other_sports_cards';237 if (/comic/.test(s)) return 'comics';238 if (/non-?sport|entertainment|garbage pail|marvel|dc /.test(s)) return 'non_sport_cards';239 if (/video game|nintendo|playstation/.test(s)) return 'video_games';240 if (/tcg|trading card|gaming/.test(s)) return 'trading_cards';241 return null;242}243244/** "Aug 6, 2026" | "Sep 3, 2024" → UTC date. */245export function parseMonthDay(s: string): Date | null {246 const m = s.match(/([A-Za-z]{3,9})\.?\s+(\d{1,2}),?\s+(\d{4})/);247 if (!m) return null;248 const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];249 const mo = months.indexOf(m[1]!.slice(0, 3).toLowerCase());250 if (mo < 0) return null;251 return new Date(Date.UTC(Number(m[3]), mo, Number(m[2])));252}253254// ---- Magic printing treatments (mirrors connectors/api/scryfall so identifier matches agree on `variant`) ----255const VARIANT_FRAME_EFFECTS = new Set(['showcase', 'extendedart', 'inverted', 'shatteredglass', 'etched', 'textured']);256const VARIANT_PROMO_TYPES = new Set(['serialized', 'prerelease', 'promopack', 'judgegift', 'buyabox', 'gameday', 'textured', 'galaxyfoil', 'surgefoil', 'stepandcompleat', 'confettifoil', 'oilslick', 'halofoil', 'neonink', 'ripplefoil', 'fracturefoil', 'rainbowfoil', 'raisedfoil', 'invisibleink', 'doublerainbow', 'manafoil', 'firstplacefoil', 'dragonscalefoil', 'silverfoil', 'gilded', 'embossed', 'startercollection', 'schinesealtart', 'datestamped', 'playerrewards', 'arenaleague', 'fnm', 'release', 'launch', 'convention', 'mediainsert', 'wizardsplaynetwork', 'thick', 'poster']);257const LABELS: Record<string, string> = { extendedart: 'Extended Art', promopack: 'Promo Pack', judgegift: 'Judge Promo', buyabox: 'Buy-a-Box', gameday: 'Game Day', galaxyfoil: 'Galaxy Foil', surgefoil: 'Surge Foil', stepandcompleat: 'Step-and-Compleat', confettifoil: 'Confetti Foil', oilslick: 'Oil Slick', halofoil: 'Halo Foil', neonink: 'Neon Ink', ripplefoil: 'Ripple Foil', fracturefoil: 'Fracture Foil', rainbowfoil: 'Rainbow Foil', raisedfoil: 'Raised Foil', invisibleink: 'Invisible Ink', doublerainbow: 'Double Rainbow', manafoil: 'Mana Foil', firstplacefoil: 'First Place Foil', dragonscalefoil: 'Dragon Scale Foil', silverfoil: 'Silver Foil', startercollection: 'Starter Collection', schinesealtart: 'Chinese Alt Art', datestamped: 'Date Stamped', playerrewards: 'Player Rewards', arenaleague: 'Arena League', fnm: 'FNM', mediainsert: 'Media Insert', wizardsplaynetwork: 'WPN', shatteredglass: 'Shattered Glass' };258259export function magicTreatments(card: { frameEffects?: string[]; promoTypes?: string[]; isFullArt?: boolean; borderColor?: string; frameVersion?: string; setType?: string | null; releaseDate?: string | null }): string[] {260 const t: string[] = [];261 for (const fe of card.frameEffects ?? []) if (VARIANT_FRAME_EFFECTS.has(fe)) t.push(fe);262 for (const pt of card.promoTypes ?? []) if (VARIANT_PROMO_TYPES.has(pt)) t.push(pt);263 if (card.isFullArt) t.push('fullart');264 if (card.borderColor === 'borderless') t.push('borderless');265 if (card.frameVersion === '1997' && card.setType !== 'core' && card.setType !== 'expansion' && (card.releaseDate ?? '') > '2010') t.push('retro');266 return [...new Set(t)].map((k) => LABELS[k] ?? (k === 'fullart' ? 'Full Art' : k.charAt(0).toUpperCase() + k.slice(1)));267}268269export function magicFinishVariant(finish: string): string | null {270 if (finish === 'foil') return 'Foil';271 if (finish === 'etched') return 'Etched Foil';272 return null;273}274