TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1/**2 * Image cache core (§113 basics, §166 delivery). Node-only: shared by the `/img` proxy route3 * (apps/web, via src/lib/images-core.ts re-export + experimental.externalDir) and the4 * image-processing worker. Lives in workers/ so it is an ES module for tsc. No Next.js imports here.5 *6 * Why a self-hosted cache: Next's optimizer fetches with Node's default `User-Agent: node`, which7 * Scryfall (62k Magic images) rejects with 400; other hosts hot-link-protect or rate-limit. We fetch8 * once with honest, host-appropriate headers, keep the original on disk and serve resized WebP.9 */10import { createHash } from 'node:crypto';11import { existsSync, mkdirSync } from 'node:fs';12import { readFile, rename, stat, writeFile } from 'node:fs/promises';13import path from 'node:path';14import sharp, { type Metadata } from 'sharp';1516export const IMAGE_WIDTHS = [96, 192, 384, 768, 1200] as const;17export type ImageWidth = (typeof IMAGE_WIDTHS)[number];18export const MAX_BYTES = 8 * 1024 * 1024;19export const FETCH_TIMEOUT_MS = 20_000;20const NEGATIVE_TTL_MS = 24 * 3600_000;21const USER_AGENT = 'RareIndexImageCache/1.0 (+https://www.rareindex.io/about; caches product images with attribution; contact data@rareindex.io)';22const ACCEPT = 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8';2324/** Per-host request rules. `referer: 'self'` sends the host's own origin (hot-link protection). */25interface HostRule {26 referer?: 'self' | string;27 headers?: Record<string, string>;28 maxConcurrent?: number;29 /** minimum ms between requests to the host */30 minIntervalMs?: number;31}32const HOST_RULES: Array<[RegExp, HostRule]> = [33 [/(^|\.)scryfall\.(io|com)$/, { maxConcurrent: 4, minIntervalMs: 60 }],34 [/^storage\.googleapis\.com$/, { referer: 'https://www.pricecharting.com/', maxConcurrent: 6 }],35 [/(^|\.)pricecharting\.com$/, { referer: 'self', maxConcurrent: 4 }],36 [/(^|\.)chrono24\.(com|de|fr|ch)$/, { referer: 'https://www.chrono24.com/', maxConcurrent: 3, minIntervalMs: 200 }],37 [/(^|\.)goldin\.co$/, { referer: 'https://goldin.co/', maxConcurrent: 3 }],38 [/(^|\.)comicconnect\.com$/, { referer: 'self', maxConcurrent: 3 }],39 [/(^|\.)phillips\.com$/, { referer: 'https://www.phillips.com/', maxConcurrent: 4 }],40 [/(^|\.)brickeconomy\.com$/, { referer: 'self', maxConcurrent: 2, minIntervalMs: 300 }],41 [/(^|\.)ygoprodeck\.com$/, { maxConcurrent: 4, minIntervalMs: 50 }],42 [/(^|\.)pokemontcg\.io$/, { maxConcurrent: 4 }],43 [/(^|\.)tcgdex\.net$/, { maxConcurrent: 4 }],44 [/(^|\.)lorcast\.io$/, { maxConcurrent: 4 }],45 [/(^|\.)cloudfront\.net$/, { maxConcurrent: 6 }],46 [/(^|\.)catawiki\.(com|nl)$|(^|\.)assets\.catawiki\.nl$/, { referer: 'https://www.catawiki.com/', maxConcurrent: 3 }],47 [/(^|\.)sothebys\.com$|(^|\.)christies\.com$|(^|\.)bonhams\.com$/, { referer: 'self', maxConcurrent: 3 }],48 [/(^|\.)bringatrailer\.com$/, { referer: 'self', maxConcurrent: 3 }],49 [/(^|\.)ebayimg\.com$/, { maxConcurrent: 6 }],50];5152export function hostRule(host: string): HostRule {53 const h = host.toLowerCase();54 for (const [re, rule] of HOST_RULES) if (re.test(h)) return rule;55 return { maxConcurrent: 3 };56}5758export function requestHeadersFor(url: URL): Record<string, string> {59 const rule = hostRule(url.hostname);60 const headers: Record<string, string> = { 'user-agent': USER_AGENT, accept: ACCEPT, 'accept-language': 'en-US,en;q=0.8', ...(rule.headers ?? {}) };61 if (rule.referer === 'self') headers.referer = `${url.protocol}//${url.hostname}/`;62 else if (rule.referer) headers.referer = rule.referer;63 return headers;64}6566export function imageKey(url: string): string {67 return createHash('sha1').update(url).digest('hex');68}6970/** Resolve RI_DATA_DIR; a relative value is anchored at the monorepo root so web and workers share one cache. */71export function dataDir(): string {72 const raw = process.env.RI_DATA_DIR ?? './data';73 if (path.isAbsolute(raw)) return raw;74 let dir = process.cwd();75 for (let i = 0; i < 5; i++) {76 if (existsSync(path.join(dir, 'pnpm-workspace.yaml'))) return path.resolve(dir, raw);77 const parent = path.dirname(dir);78 if (parent === dir) break;79 dir = parent;80 }81 return path.resolve(process.cwd(), raw);82}8384export function cacheRoot(): string {85 return path.join(dataDir(), 'images');86}87export function origPath(key: string, ext: string): string {88 return path.join(cacheRoot(), 'orig', key.slice(0, 2), `${key}.${ext}`);89}90export function variantPath(key: string, width: number, fmt: 'webp' | 'avif'): string {91 return path.join(cacheRoot(), `w${width}`, key.slice(0, 2), `${key}.${fmt}`);92}9394export function nearestWidth(w: unknown): ImageWidth {95 const n = Number(w);96 if (!Number.isFinite(n) || n <= 0) return 384;97 for (const cand of IMAGE_WIDTHS) if (n <= cand) return cand;98 return 1200;99}100101// ---------- host politeness ----------102const inflight = new Map<string, number>();103const lastAt = new Map<string, number>();104async function acquire(host: string): Promise<() => void> {105 const rule = hostRule(host);106 const max = rule.maxConcurrent ?? 3;107 while ((inflight.get(host) ?? 0) >= max) await new Promise((r) => setTimeout(r, 25));108 inflight.set(host, (inflight.get(host) ?? 0) + 1);109 const wait = (lastAt.get(host) ?? 0) + (rule.minIntervalMs ?? 0) - Date.now();110 if (wait > 0) await new Promise((r) => setTimeout(r, wait));111 lastAt.set(host, Date.now());112 return () => inflight.set(host, Math.max(0, (inflight.get(host) ?? 1) - 1));113}114115// ---------- negative cache (in-process; the worker also persists status in the DB) ----------116const negative = new Map<string, { until: number; reason: string }>();117export function negativeFor(url: string): string | null {118 const n = negative.get(url);119 if (!n) return null;120 if (n.until < Date.now()) {121 negative.delete(url);122 return null;123 }124 return n.reason;125}126export function markNegative(url: string, reason: string, ttlMs = NEGATIVE_TTL_MS): void {127 negative.set(url, { until: Date.now() + ttlMs, reason });128 if (negative.size > 50_000) negative.delete(negative.keys().next().value!);129}130131export interface OriginalInfo {132 key: string;133 path: string;134 ext: string;135 contentType: string;136 bytes: number;137 width: number | null;138 height: number | null;139 fromCache: boolean;140}141export type FetchOutcome = { ok: true; info: OriginalInfo } | { ok: false; status: 'dead' | 'blocked' | 'error'; reason: string; httpStatus: number | null };142143const EXT_BY_FORMAT: Record<string, string> = { jpeg: 'jpg', jpg: 'jpg', png: 'png', webp: 'webp', avif: 'avif', gif: 'gif', tiff: 'tif', svg: 'svg', heif: 'heic' };144const CT_BY_EXT: Record<string, string> = { jpg: 'image/jpeg', png: 'image/png', webp: 'image/webp', avif: 'image/avif', gif: 'image/gif', tif: 'image/tiff', svg: 'image/svg+xml', heic: 'image/heic' };145146const originalInflight = new Map<string, Promise<FetchOutcome>>();147148/** Find an already cached original for a key (any extension). */149export async function findOriginal(key: string): Promise<OriginalInfo | null> {150 for (const ext of Object.keys(CT_BY_EXT)) {151 const p = origPath(key, ext);152 try {153 const s = await stat(p);154 return { key, path: p, ext, contentType: CT_BY_EXT[ext]!, bytes: s.size, width: null, height: null, fromCache: true };155 } catch {156 /* next */157 }158 }159 return null;160}161162/** Fetch + validate + store the original image; idempotent and de-duplicated per URL. */163export async function ensureOriginal(url: string): Promise<FetchOutcome> {164 const key = imageKey(url);165 const cached = await findOriginal(key);166 if (cached) return { ok: true, info: cached };167 const neg = negativeFor(url);168 if (neg) return { ok: false, status: neg.startsWith('blocked') ? 'blocked' : 'dead', reason: neg, httpStatus: null };169 const existing = originalInflight.get(url);170 if (existing) return existing;171 const p = fetchOriginal(url, key).finally(() => originalInflight.delete(url));172 originalInflight.set(url, p);173 return p;174}175176async function fetchOriginal(url: string, key: string): Promise<FetchOutcome> {177 let u: URL;178 try {179 u = new URL(url);180 if (u.protocol !== 'https:' && u.protocol !== 'http:') throw new Error('protocol');181 } catch {182 markNegative(url, 'dead: invalid url');183 return { ok: false, status: 'dead', reason: 'invalid url', httpStatus: null };184 }185 const release = await acquire(u.hostname);186 const ctrl = new AbortController();187 const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT_MS);188 try {189 const res = await fetch(u, { headers: requestHeadersFor(u), signal: ctrl.signal, redirect: 'follow' });190 if (res.status === 404 || res.status === 410) {191 markNegative(url, `dead: http ${res.status}`);192 return { ok: false, status: 'dead', reason: `http ${res.status}`, httpStatus: res.status };193 }194 if (res.status === 401 || res.status === 403 || res.status === 429 || res.status === 400) {195 markNegative(url, `blocked: http ${res.status}`, res.status === 429 ? 3600_000 : NEGATIVE_TTL_MS);196 return { ok: false, status: 'blocked', reason: `http ${res.status}`, httpStatus: res.status };197 }198 if (!res.ok) {199 markNegative(url, `error: http ${res.status}`, 3600_000);200 return { ok: false, status: 'error', reason: `http ${res.status}`, httpStatus: res.status };201 }202 const len = Number(res.headers.get('content-length') ?? 0);203 if (len > MAX_BYTES) {204 markNegative(url, 'dead: too large');205 return { ok: false, status: 'dead', reason: `too large (${len} bytes)`, httpStatus: res.status };206 }207 const buf = Buffer.from(await res.arrayBuffer());208 if (buf.byteLength === 0 || buf.byteLength > MAX_BYTES) {209 markNegative(url, 'dead: empty or too large');210 return { ok: false, status: 'dead', reason: `size ${buf.byteLength}`, httpStatus: res.status };211 }212 // Validate by decoding metadata (also guards against HTML error pages served as 200).213 let meta: Metadata;214 try {215 meta = await sharp(buf, { limitInputPixels: 80_000_000 }).metadata();216 } catch {217 markNegative(url, 'dead: not an image');218 return { ok: false, status: 'dead', reason: `not an image (${res.headers.get('content-type') ?? 'unknown type'})`, httpStatus: res.status };219 }220 const ext = EXT_BY_FORMAT[meta.format ?? ''] ?? 'jpg';221 const dest = origPath(key, ext);222 mkdirSync(path.dirname(dest), { recursive: true });223 const tmp = `${dest}.${process.pid}.tmp`;224 await writeFile(tmp, buf);225 await rename(tmp, dest);226 return { ok: true, info: { key, path: dest, ext, contentType: CT_BY_EXT[ext] ?? 'application/octet-stream', bytes: buf.byteLength, width: meta.width ?? null, height: meta.height ?? null, fromCache: false } };227 } catch (err) {228 const reason = err instanceof Error ? (err.name === 'AbortError' ? 'timeout' : err.message) : String(err);229 markNegative(url, `error: ${reason}`, 3600_000);230 return { ok: false, status: 'error', reason, httpStatus: null };231 } finally {232 clearTimeout(timer);233 release();234 }235}236237const variantInflight = new Map<string, Promise<string>>();238239/** Produce (or reuse) a resized WebP/AVIF variant; returns its path. */240export async function ensureVariant(info: OriginalInfo, width: ImageWidth, fmt: 'webp' | 'avif' = 'webp'): Promise<string> {241 const dest = variantPath(info.key, width, fmt);242 try {243 await stat(dest);244 return dest;245 } catch {246 /* build */247 }248 const k = `${dest}`;249 const existing = variantInflight.get(k);250 if (existing) return existing;251 const p = (async () => {252 mkdirSync(path.dirname(dest), { recursive: true });253 const tmp = `${dest}.${process.pid}.tmp`;254 let pipeline = sharp(info.path, { limitInputPixels: 80_000_000, animated: false }).rotate().resize({ width, withoutEnlargement: true, fit: 'inside' });255 pipeline = fmt === 'avif' ? pipeline.avif({ quality: 55, effort: 3 }) : pipeline.webp({ quality: 80, effort: 4 });256 await pipeline.toFile(tmp);257 await rename(tmp, dest);258 return dest;259 })().finally(() => variantInflight.delete(k));260 variantInflight.set(k, p);261 return p;262}263264export async function readVariant(p: string): Promise<Buffer> {265 return readFile(p);266}267268/** 64-bit difference hash (dHash) over a 9×8 grayscale thumbnail; robust to resize/recompression. */269export function dhashFromGray(pixels: Uint8Array | number[], width = 9, height = 8): string {270 let bits = '';271 for (let y = 0; y < height; y++) {272 for (let x = 0; x < width - 1; x++) {273 const a = pixels[y * width + x]!;274 const b = pixels[y * width + x + 1]!;275 bits += a > b ? '1' : '0';276 }277 }278 return BigInt(`0b${bits}`).toString(16).padStart(16, '0');279}280281export function hamming(a: string, b: string): number {282 const x = BigInt(`0x${a}`) ^ BigInt(`0x${b}`);283 let n = 0;284 let v = x;285 while (v) {286 n += Number(v & 1n);287 v >>= 1n;288 }289 return n;290}291292export async function perceptualHash(filePath: string): Promise<string | null> {293 try {294 const { data } = await sharp(filePath, { limitInputPixels: 80_000_000 }).rotate().grayscale().resize(9, 8, { fit: 'fill' }).raw().toBuffer({ resolveWithObject: true });295 return dhashFromGray(data, 9, 8);296 } catch {297 return null;298 }299}300