TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';2import path from 'node:path';3import { fileURLToPath } from 'node:url';4import { NormalizedRecordSchema, type NormalizedRecord } from '@rareindex/shared';5import type { RareIndexConnector, RawRecordLike } from './types.js';67const here = path.dirname(fileURLToPath(import.meta.url));8export const FIXTURES_DIR = path.resolve(here, '../../../data/fixtures');910/**11 * Fixture conventions (§146): data/fixtures/<connectorId>/<name>.json holds a captured RawRecordLike12 * (`{ url, externalId, kind, engine, fetchedAt, payload }`) plus optional `expect` block used by13 * the standard test-suite. Large HTML captures live next to it as <name>.html and are referenced14 * by payload.snapshot = "<name>.html".15 */16export interface Fixture {17 raw: RawRecordLike;18 expect?: {19 count?: number;20 minCount?: number;21 kinds?: string[];22 first?: Partial<Record<string, unknown>>;23 /** every record must have these fields non-null */24 requiredFields?: string[];25 };26 capturedAt?: string;27 note?: string;28}2930export function fixtureDir(connectorId: string): string {31 return path.join(FIXTURES_DIR, connectorId);32}3334export function listFixtures(connectorId: string): string[] {35 const dir = fixtureDir(connectorId);36 if (!existsSync(dir)) return [];37 return readdirSync(dir).filter((f) => f.endsWith('.json')).map((f) => f.replace(/\.json$/, ''));38}3940export function loadFixture(connectorId: string, name: string): Fixture {41 const dir = fixtureDir(connectorId);42 const f = JSON.parse(readFileSync(path.join(dir, `${name}.json`), 'utf8')) as Fixture;43 f.raw.fetchedAt = new Date(f.raw.fetchedAt);44 const p = f.raw.payload as { snapshot?: string } | null;45 if (p && typeof p === 'object' && typeof p.snapshot === 'string' && p.snapshot.endsWith('.html')) {46 p.snapshot = readFileSync(path.join(dir, p.snapshot), 'utf8');47 }48 return f;49}5051export function saveFixture(connectorId: string, name: string, fixture: Fixture): void {52 const dir = fixtureDir(connectorId);53 mkdirSync(dir, { recursive: true });54 const p = fixture.raw.payload as { snapshot?: string } | null;55 if (p && typeof p === 'object' && typeof p.snapshot === 'string' && p.snapshot.length > 20_000) {56 writeFileSync(path.join(dir, `${name}.html`), p.snapshot);57 p.snapshot = `${name}.html`;58 }59 writeFileSync(path.join(dir, `${name}.json`), JSON.stringify({ ...fixture, capturedAt: fixture.capturedAt ?? new Date().toISOString() }, null, 2));60}6162/** Validate normalised output against the canonical schemas. Throws with a readable message. */63export function assertNormalized(records: NormalizedRecord[]): NormalizedRecord[] {64 return records.map((r, i) => {65 const parsed = NormalizedRecordSchema.safeParse(r);66 if (!parsed.success) throw new Error(`record[${i}] invalid: ${parsed.error.issues.map((x) => `${x.path.join('.')}: ${x.message}`).join('; ')}`);67 return parsed.data;68 });69}7071/**72 * Standard fixture suite: run `normalize` on every fixture of a connector and check the common73 * invariants (schema validity, no crawl-time-as-sale-date, positive prices, currency present,74 * canonical URL, duplicate handling). Use from vitest:75 *76 * describe('scryfall', () => runFixtureSuite(connector, it, expect))77 */78export function runFixtureSuite(connector: RareIndexConnector, it: (name: string, fn: () => Promise<void>) => void, expectFn: (v: unknown) => any): void {79 const names = listFixtures(connector.meta.id);80 it('has at least one fixture', async () => {81 expectFn(names.length).toBeGreaterThan(0);82 });83 for (const name of names) {84 it(`fixture ${name} normalises`, async () => {85 const fx = loadFixture(connector.meta.id, name);86 const out = assertNormalized(await connector.normalize(fx.raw));87 if (fx.expect?.count !== undefined) expectFn(out.length).toBe(fx.expect.count);88 if (fx.expect?.minCount !== undefined) expectFn(out.length).toBeGreaterThanOrEqual(fx.expect.minCount);89 if (fx.expect?.kinds) for (const r of out) expectFn(fx.expect.kinds).toContain(r.kind);90 const seen = new Set<string>();91 for (const r of out) {92 expectFn(r.sourceUrl.startsWith('http')).toBe(true);93 expectFn(r.connectorId).toBe(connector.meta.id);94 if (r.kind === 'sale') {95 expectFn(r.price).toBeGreaterThan(0);96 expectFn(r.currency).toBeTruthy();97 expectFn(r.saleDate.getTime()).toBeLessThanOrEqual(Date.now() + 86_400_000);98 // sale date must not simply equal the fetch time (§172)99 expectFn(Math.abs(r.saleDate.getTime() - fx.raw.fetchedAt.getTime()) > 1000 || r.saleDate.getUTCHours() === 0).toBe(true);100 }101 if (r.kind === 'listing' && r.price !== null) {102 expectFn(r.price).toBeGreaterThanOrEqual(0);103 expectFn(r.currency).toBeTruthy();104 }105 if (r.kind === 'price_observation') {106 expectFn(r.price).toBeGreaterThan(0);107 expectFn(r.currency).toBeTruthy();108 }109 if ('rawTitle' in r) expectFn(r.rawTitle.length).toBeGreaterThan(0);110 if ('attributes' in r) expectFn(r.attributes.categorySlug.length).toBeGreaterThan(0);111 const key = `${r.kind}|${'externalId' in r ? r.externalId : ''}|${r.sourceUrl}|${'price' in r ? r.price : ''}|${'saleDate' in r ? r.saleDate.toISOString() : ''}|${'priceKind' in r ? r.priceKind : ''}`;112 expectFn(seen.has(key)).toBe(false);113 seen.add(key);114 }115 if (fx.expect?.first) {116 const first = out[0] as unknown as Record<string, unknown>;117 for (const [k, v] of Object.entries(fx.expect.first)) {118 const actual = k.includes('.') ? k.split('.').reduce<unknown>((acc, p) => (acc && typeof acc === 'object' ? (acc as Record<string, unknown>)[p] : undefined), first) : first[k];119 expectFn(actual).toEqual(v);120 }121 }122 if (fx.expect?.requiredFields) {123 for (const r of out) {124 const rec = r as unknown as Record<string, unknown>;125 for (const f of fx.expect.requiredFields) {126 const val = f.includes('.') ? f.split('.').reduce<unknown>((acc, p) => (acc && typeof acc === 'object' ? (acc as Record<string, unknown>)[p] : undefined), rec) : rec[f];127 expectFn(val === null || val === undefined || val === '').toBe(false);128 }129 }130 }131 });132 }133}134