TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { z } from 'zod';2import { SUPPORTED_CURRENCIES } from './money.js';34/**5 * Canonical Zod schemas. These are the contracts between connectors, the ingestion pipeline,6 * the database layer, and the apps. Source-specific quirks must be resolved BEFORE data7 * reaches these shapes (see CLAUDE.md §108, §196.6).8 */910export const CurrencySchema = z.enum(SUPPORTED_CURRENCIES);1112export const SourceTypeSchema = z.enum([13 'marketplace', 'auction_house', 'dealer', 'pricing_guide', 'grading_company', 'manufacturer', 'museum', 'archive',14 'catalog', 'collector_database', 'community', 'forum', 'news', 'analytics_provider', 'registry', 'official_source',15]);16export type SourceType = z.infer<typeof SourceTypeSchema>;1718export const EngineSchema = z.enum(['api', 'feed', 'firecrawl', 'scrapfly', 'browser', 'manual']);19export type Engine = z.infer<typeof EngineSchema>;2021export const RecordKindSchema = z.enum(['sale', 'listing', 'auction_lot', 'price_observation', 'catalog_item', 'population_report', 'news_item']);22export type RecordKind = z.infer<typeof RecordKindSchema>;2324/** Immutable raw payload captured from a source. Never mutated after insert (§108). */25export const RawRecordSchema = z.object({26 connectorId: z.string(),27 sourceId: z.string(),28 engine: EngineSchema,29 url: z.string().url().or(z.string().min(1)),30 externalId: z.string().nullable().default(null),31 kind: RecordKindSchema,32 fetchedAt: z.coerce.date(),33 contentHash: z.string(),34 httpStatus: z.number().int().nullable().default(null),35 payload: z.unknown(),36 /** raw HTML/markdown snapshot key in object storage or inline text (bounded) */37 snapshotRef: z.string().nullable().default(null),38 parserVersion: z.string(),39 connectorVersion: z.string(),40});41export type RawRecord = z.infer<typeof RawRecordSchema>;4243/** Attributes used for canonical asset identity; anything unknown must be null, never guessed (§192). */44export const AssetAttributesSchema = z.object({45 categorySlug: z.string(),46 subcategorySlug: z.string().nullable().default(null),47 franchise: z.string().nullable().default(null),48 brand: z.string().nullable().default(null),49 series: z.string().nullable().default(null),50 set: z.string().nullable().default(null),51 setCode: z.string().nullable().default(null),52 name: z.string(),53 model: z.string().nullable().default(null),54 reference: z.string().nullable().default(null),55 number: z.string().nullable().default(null),56 year: z.number().int().nullable().default(null),57 edition: z.string().nullable().default(null),58 variant: z.string().nullable().default(null),59 language: z.string().nullable().default(null),60 region: z.string().nullable().default(null),61 country: z.string().nullable().default(null),62 material: z.string().nullable().default(null),63 size: z.string().nullable().default(null),64 color: z.string().nullable().default(null),65 rarity: z.string().nullable().default(null),66 productionQuantity: z.number().int().nullable().default(null),67 originalMsrp: z.number().nullable().default(null),68 originalMsrpCurrency: CurrencySchema.nullable().default(null),69 /** Deterministic identifiers: upc, isbn, sku, scryfall_id, tcgplayer_id, bricklink_id, style_code, ref, cert… */70 identifiers: z.record(z.string(), z.string()).default({}),71 metadata: z.record(z.string(), z.unknown()).default({}),72});73export type AssetAttributes = z.infer<typeof AssetAttributesSchema>;7475export const GradeSchema = z.object({76 grader: z.string().nullable().default(null), // PSA, BGS, CGC, SGC, TAG, WATA, VGA, PCGS, NGC, CBCS, raw77 grade: z.string().nullable().default(null), // "10", "9.5", "9.8", "MS65"78 qualifier: z.string().nullable().default(null), // "OC", "Signature Series", "Black Label"79 certificationNumber: z.string().nullable().default(null),80});81export type Grade = z.infer<typeof GradeSchema>;8283export const ConditionSchema = z.object({84 /** category-specific normalised condition slug (see taxonomy condition scales) */85 condition: z.string().nullable().default(null),86 conditionRaw: z.string().nullable().default(null),87 completeness: z.string().nullable().default(null), // sealed, cib, loose, box_papers, papers_only…88});8990const BaseNormalized = z.object({91 connectorId: z.string(),92 sourceId: z.string(),93 sourceUrl: z.string(),94 externalId: z.string().nullable().default(null),95 rawTitle: z.string(),96 description: z.string().nullable().default(null),97 imageUrls: z.array(z.string()).default([]),98 attributes: AssetAttributesSchema,99 grade: GradeSchema.prefault({}),100 condition: ConditionSchema.prefault({}),101 observedAt: z.coerce.date(),102 /** 0–1 confidence in the extraction & identification */103 confidence: z.number().min(0).max(1).default(0.8),104 parserVersion: z.string(),105});106107export const NormalizedSaleSchema = BaseNormalized.extend({108 kind: z.literal('sale'),109 saleType: z.enum(['auction', 'fixed_price', 'best_offer', 'private', 'dealer', 'unknown']).default('unknown'),110 saleDate: z.coerce.date(),111 price: z.number().nonnegative(),112 currency: CurrencySchema,113 buyerPremiumIncluded: z.boolean().nullable().default(null),114 quantity: z.number().int().positive().default(1),115 isBundle: z.boolean().default(false),116 location: z.string().nullable().default(null),117 auctionHouse: z.string().nullable().default(null),118 lotNumber: z.string().nullable().default(null),119});120export type NormalizedSale = z.infer<typeof NormalizedSaleSchema>;121122export const NormalizedListingSchema = BaseNormalized.extend({123 kind: z.literal('listing'),124 listingType: z.enum(['fixed_price', 'auction', 'best_offer', 'ask', 'unknown']).default('unknown'),125 price: z.number().nonnegative().nullable(),126 currency: CurrencySchema.nullable(),127 seller: z.string().nullable().default(null),128 sellerReputation: z.string().nullable().default(null),129 location: z.string().nullable().default(null),130 shippingCost: z.number().nullable().default(null),131 quantity: z.number().int().nullable().default(null),132 listedAt: z.coerce.date().nullable().default(null),133 endsAt: z.coerce.date().nullable().default(null),134 availability: z.enum(['available', 'sold', 'ended', 'removed', 'unknown']).default('available'),135 bidCount: z.number().int().nullable().default(null),136});137export type NormalizedListing = z.infer<typeof NormalizedListingSchema>;138139export const NormalizedAuctionLotSchema = BaseNormalized.extend({140 kind: z.literal('auction_lot'),141 auctionHouse: z.string(),142 auctionName: z.string().nullable().default(null),143 lotNumber: z.string().nullable().default(null),144 startsAt: z.coerce.date().nullable().default(null),145 endsAt: z.coerce.date().nullable().default(null),146 estimateLow: z.number().nullable().default(null),147 estimateHigh: z.number().nullable().default(null),148 currentBid: z.number().nullable().default(null),149 currency: CurrencySchema.nullable(),150 status: z.enum(['upcoming', 'live', 'ended', 'unknown']).default('unknown'),151 location: z.string().nullable().default(null),152});153export type NormalizedAuctionLot = z.infer<typeof NormalizedAuctionLotSchema>;154155/** Price-guide style observation (e.g. Scryfall market price). Not a transaction (§111 spirit). */156export const NormalizedPriceObservationSchema = BaseNormalized.extend({157 kind: z.literal('price_observation'),158 priceKind: z.enum(['market', 'low', 'mid', 'high', 'trend', 'average_7d', 'average_30d', 'last_sale_reported', 'guide_value']),159 price: z.number().nonnegative(),160 currency: CurrencySchema,161 observationDate: z.coerce.date(),162 sampleSize: z.number().int().nullable().default(null),163});164export type NormalizedPriceObservation = z.infer<typeof NormalizedPriceObservationSchema>;165166/** Catalog entry from a manufacturer/reference database: creates or enriches canonical assets. */167export const NormalizedCatalogItemSchema = BaseNormalized.extend({168 kind: z.literal('catalog_item'),169 releaseDate: z.coerce.date().nullable().default(null),170});171export type NormalizedCatalogItem = z.infer<typeof NormalizedCatalogItemSchema>;172173export const NormalizedPopulationReportSchema = z.object({174 kind: z.literal('population_report'),175 connectorId: z.string(),176 sourceId: z.string(),177 sourceUrl: z.string(),178 grader: z.string(),179 attributes: AssetAttributesSchema,180 reportDate: z.coerce.date(),181 total: z.number().int().nonnegative(),182 byGrade: z.record(z.string(), z.number().int().nonnegative()),183 parserVersion: z.string(),184 confidence: z.number().min(0).max(1).default(0.9),185});186export type NormalizedPopulationReport = z.infer<typeof NormalizedPopulationReportSchema>;187188export const NormalizedNewsItemSchema = z.object({189 kind: z.literal('news_item'),190 connectorId: z.string(),191 sourceId: z.string(),192 sourceUrl: z.string(),193 title: z.string(),194 summary: z.string().nullable().default(null),195 publishedAt: z.coerce.date().nullable().default(null),196 categorySlugs: z.array(z.string()).default([]),197 imageUrl: z.string().nullable().default(null),198 parserVersion: z.string(),199});200export type NormalizedNewsItem = z.infer<typeof NormalizedNewsItemSchema>;201202export const NormalizedRecordSchema = z.discriminatedUnion('kind', [203 NormalizedSaleSchema,204 NormalizedListingSchema,205 NormalizedAuctionLotSchema,206 NormalizedPriceObservationSchema,207 NormalizedCatalogItemSchema,208 NormalizedPopulationReportSchema,209 NormalizedNewsItemSchema,210]);211export type NormalizedRecord = z.infer<typeof NormalizedRecordSchema>;212213export const ConnectorStatusSchema = z.enum(['healthy', 'degraded', 'failing', 'paused', 'disabled', 'maintenance', 'unknown']);214export type ConnectorStatus = z.infer<typeof ConnectorStatusSchema>;215216/** Operator-facing labels (SPEC §12): UP · DEGRADED · BROKEN · DISABLED · MAINTENANCE. */217export const HEALTH_LABELS: Record<ConnectorStatus, 'UP' | 'DEGRADED' | 'BROKEN' | 'DISABLED' | 'MAINTENANCE' | 'UNKNOWN'> = {218 healthy: 'UP',219 degraded: 'DEGRADED',220 failing: 'BROKEN',221 paused: 'DISABLED',222 disabled: 'DISABLED',223 maintenance: 'MAINTENANCE',224 unknown: 'UNKNOWN',225};226227export const ConnectorHealthSchema = z.object({228 connector: z.string(),229 status: ConnectorStatusSchema,230 success_rate_24h: z.number().min(0).max(1).nullable(),231 pages_attempted: z.number().int().nonnegative(),232 pages_success: z.number().int().nonnegative(),233 firecrawl_success_rate: z.number().min(0).max(1).nullable(),234 scrapfly_fallback_rate: z.number().min(0).max(1).nullable(),235 parse_failure_rate: z.number().min(0).max(1).nullable(),236 last_success: z.string().nullable(),237 last_error: z.string().nullable(),238 schema_version: z.string(),239 records_24h: z.number().int().nonnegative(),240 duplicates_24h: z.number().int().nonnegative(),241 anomalies: z.array(z.string()).default([]),242 /** SPEC §12 additions (optional so older snapshots still validate) */243 requests_24h: z.number().int().nonnegative().optional(),244 latency_ms_avg: z.number().nullable().optional(),245 http_errors: z.record(z.string(), z.number()).optional(),246 challenges_24h: z.number().int().nonnegative().optional(),247 circuit_refusals_24h: z.number().int().nonnegative().optional(),248 rate_limited_24h: z.number().int().nonnegative().optional(),249 /** average raw records/day over the trailing 7 days (baseline for result-count collapse) */250 records_7d_daily_avg: z.number().nullable().optional(),251 schema_drift: z.array(z.string()).optional(),252 /** ISO of the newest source-side date ingested (sale/listing date), i.e. data freshness */253 data_freshness: z.string().nullable().optional(),254 /** environment variables missing for a gated connector */255 missing_requirements: z.array(z.string()).optional(),256});257export type ConnectorHealth = z.infer<typeof ConnectorHealthSchema>;258259export const ExtractionResultSchema = z.object({260 success: z.boolean(),261 engine: EngineSchema,262 url: z.string(),263 finalUrl: z.string().nullable().default(null),264 httpStatus: z.number().int().nullable().default(null),265 html: z.string().nullable().default(null),266 markdown: z.string().nullable().default(null),267 json: z.unknown().nullable().default(null),268 /** raw bytes for binary documents (PDF catalogues, gzip feeds) — never persisted in the DB */269 buffer: z.custom<Uint8Array>((v) => v instanceof Uint8Array).nullable().optional(),270 qualityScore: z.number().min(0).max(1).default(0),271 requiresReview: z.boolean().default(false),272 error: z.string().nullable().default(null),273 costCredits: z.number().nullable().default(null),274 durationMs: z.number().nullable().default(null),275 fetchedAt: z.coerce.date(),276});277export type ExtractionResult = z.infer<typeof ExtractionResultSchema>;278