import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { NormalizedRecordSchema, type NormalizedRecord } from '@rareindex/shared'; import type { RareIndexConnector, RawRecordLike } from './types.js'; const here = path.dirname(fileURLToPath(import.meta.url)); export const FIXTURES_DIR = path.resolve(here, '../../../data/fixtures'); /** * Fixture conventions (§146): data/fixtures//.json holds a captured RawRecordLike * (`{ url, externalId, kind, engine, fetchedAt, payload }`) plus optional `expect` block used by * the standard test-suite. Large HTML captures live next to it as .html and are referenced * by payload.snapshot = ".html". */ export interface Fixture { raw: RawRecordLike; expect?: { count?: number; minCount?: number; kinds?: string[]; first?: Partial>; /** every record must have these fields non-null */ requiredFields?: string[]; }; capturedAt?: string; note?: string; } export function fixtureDir(connectorId: string): string { return path.join(FIXTURES_DIR, connectorId); } export function listFixtures(connectorId: string): string[] { const dir = fixtureDir(connectorId); if (!existsSync(dir)) return []; return readdirSync(dir).filter((f) => f.endsWith('.json')).map((f) => f.replace(/\.json$/, '')); } export function loadFixture(connectorId: string, name: string): Fixture { const dir = fixtureDir(connectorId); const f = JSON.parse(readFileSync(path.join(dir, `${name}.json`), 'utf8')) as Fixture; f.raw.fetchedAt = new Date(f.raw.fetchedAt); const p = f.raw.payload as { snapshot?: string } | null; if (p && typeof p === 'object' && typeof p.snapshot === 'string' && p.snapshot.endsWith('.html')) { p.snapshot = readFileSync(path.join(dir, p.snapshot), 'utf8'); } return f; } export function saveFixture(connectorId: string, name: string, fixture: Fixture): void { const dir = fixtureDir(connectorId); mkdirSync(dir, { recursive: true }); const p = fixture.raw.payload as { snapshot?: string } | null; if (p && typeof p === 'object' && typeof p.snapshot === 'string' && p.snapshot.length > 20_000) { writeFileSync(path.join(dir, `${name}.html`), p.snapshot); p.snapshot = `${name}.html`; } writeFileSync(path.join(dir, `${name}.json`), JSON.stringify({ ...fixture, capturedAt: fixture.capturedAt ?? new Date().toISOString() }, null, 2)); } /** Validate normalised output against the canonical schemas. Throws with a readable message. */ export function assertNormalized(records: NormalizedRecord[]): NormalizedRecord[] { return records.map((r, i) => { const parsed = NormalizedRecordSchema.safeParse(r); if (!parsed.success) throw new Error(`record[${i}] invalid: ${parsed.error.issues.map((x) => `${x.path.join('.')}: ${x.message}`).join('; ')}`); return parsed.data; }); } /** * Standard fixture suite: run `normalize` on every fixture of a connector and check the common * invariants (schema validity, no crawl-time-as-sale-date, positive prices, currency present, * canonical URL, duplicate handling). Use from vitest: * * describe('scryfall', () => runFixtureSuite(connector, it, expect)) */ export function runFixtureSuite(connector: RareIndexConnector, it: (name: string, fn: () => Promise) => void, expectFn: (v: unknown) => any): void { const names = listFixtures(connector.meta.id); it('has at least one fixture', async () => { expectFn(names.length).toBeGreaterThan(0); }); for (const name of names) { it(`fixture ${name} normalises`, async () => { const fx = loadFixture(connector.meta.id, name); const out = assertNormalized(await connector.normalize(fx.raw)); if (fx.expect?.count !== undefined) expectFn(out.length).toBe(fx.expect.count); if (fx.expect?.minCount !== undefined) expectFn(out.length).toBeGreaterThanOrEqual(fx.expect.minCount); if (fx.expect?.kinds) for (const r of out) expectFn(fx.expect.kinds).toContain(r.kind); const seen = new Set(); for (const r of out) { expectFn(r.sourceUrl.startsWith('http')).toBe(true); expectFn(r.connectorId).toBe(connector.meta.id); if (r.kind === 'sale') { expectFn(r.price).toBeGreaterThan(0); expectFn(r.currency).toBeTruthy(); expectFn(r.saleDate.getTime()).toBeLessThanOrEqual(Date.now() + 86_400_000); // sale date must not simply equal the fetch time (§172) expectFn(Math.abs(r.saleDate.getTime() - fx.raw.fetchedAt.getTime()) > 1000 || r.saleDate.getUTCHours() === 0).toBe(true); } if (r.kind === 'listing' && r.price !== null) { expectFn(r.price).toBeGreaterThanOrEqual(0); expectFn(r.currency).toBeTruthy(); } if (r.kind === 'price_observation') { expectFn(r.price).toBeGreaterThan(0); expectFn(r.currency).toBeTruthy(); } if ('rawTitle' in r) expectFn(r.rawTitle.length).toBeGreaterThan(0); if ('attributes' in r) expectFn(r.attributes.categorySlug.length).toBeGreaterThan(0); const key = `${r.kind}|${'externalId' in r ? r.externalId : ''}|${r.sourceUrl}|${'price' in r ? r.price : ''}|${'saleDate' in r ? r.saleDate.toISOString() : ''}|${'priceKind' in r ? r.priceKind : ''}`; expectFn(seen.has(key)).toBe(false); seen.add(key); } if (fx.expect?.first) { const first = out[0] as unknown as Record; for (const [k, v] of Object.entries(fx.expect.first)) { const actual = k.includes('.') ? k.split('.').reduce((acc, p) => (acc && typeof acc === 'object' ? (acc as Record)[p] : undefined), first) : first[k]; expectFn(actual).toEqual(v); } } if (fx.expect?.requiredFields) { for (const r of out) { const rec = r as unknown as Record; for (const f of fx.expect.requiredFields) { const val = f.includes('.') ? f.split('.').reduce((acc, p) => (acc && typeof acc === 'object' ? (acc as Record)[p] : undefined), rec) : rec[f]; expectFn(val === null || val === undefined || val === '').toBe(false); } } } }); } }