TypeScript 61.9%
HTML 37.2%
SQL 0.7%
1import { fileURLToPath } from 'node:url';2import path from 'node:path';3import { readFileSync, existsSync } from 'node:fs';4import { sql } from 'drizzle-orm';5import { CATEGORIES, GRADERS, INDICES, INDEX_BASE_DATE, INDEX_BASE_VALUE } from '@rareindex/taxonomy';6import { getDb, closeDb } from './client.js';7import { categories, graders, indices, sources, connectors } from './schema/index.js';89/** Upsert taxonomy nodes + graders from data/taxonomy (idempotent). */10export async function seedTaxonomy(): Promise<void> {11 const db = getDb();12 for (const c of CATEGORIES) {13 await db14 .insert(categories)15 .values({16 slug: c.slug,17 parentSlug: c.parent ?? null,18 familySlug: c.familySlug,19 name: c.name,20 shortName: c.short ?? null,21 description: c.description ?? null,22 level: c.level,23 phase: c.phase,24 conditionScale: c.conditionScale ?? 'general',25 graders: c.graders ?? [],26 indexTicker: c.index ?? null,27 sortOrder: c.sortOrder,28 complianceFlags: c.compliance ?? [],29 icon: c.icon ?? null,30 })31 .onConflictDoUpdate({32 target: categories.slug,33 set: {34 parentSlug: c.parent ?? null,35 familySlug: c.familySlug,36 name: c.name,37 shortName: c.short ?? null,38 description: c.description ?? null,39 level: c.level,40 phase: c.phase,41 conditionScale: c.conditionScale ?? 'general',42 graders: c.graders ?? [],43 indexTicker: c.index ?? null,44 sortOrder: c.sortOrder,45 complianceFlags: c.compliance ?? [],46 icon: c.icon ?? null,47 updatedAt: new Date(),48 },49 });50 }51 for (const g of GRADERS) {52 await db53 .insert(graders)54 .values({ slug: g.slug, name: g.name, categorySlugs: g.categories, scale: g.scale, populationUrl: g.populationUrl ?? null, verifyUrl: g.verifyUrl ?? null })55 .onConflictDoUpdate({ target: graders.slug, set: { name: g.name, categorySlugs: g.categories, scale: g.scale, populationUrl: g.populationUrl ?? null, verifyUrl: g.verifyUrl ?? null } });56 }57}5859/** Upsert index definitions (§3). */60export async function seedIndices(): Promise<void> {61 const db = getDb();62 for (const i of INDICES) {63 const familySlugs = i.families;64 const categorySlugs = CATEGORIES.filter((c) => familySlugs.includes(c.familySlug)).map((c) => c.slug);65 await db66 .insert(indices)67 .values({68 id: `idx_${i.ticker.toLowerCase().replace(/[^a-z0-9]+/g, '_')}`,69 ticker: i.ticker,70 name: i.name,71 description: i.description ?? null,72 categorySlugs,73 familySlugs,74 parentTicker: i.flagship ? null : 'RARE',75 baseDate: INDEX_BASE_DATE,76 baseValue: INDEX_BASE_VALUE,77 minConstituents: i.minConstituents,78 isFlagship: Boolean(i.flagship),79 color: i.color,80 })81 .onConflictDoUpdate({ target: indices.ticker, set: { name: i.name, description: i.description ?? null, categorySlugs, familySlugs, minConstituents: i.minConstituents, color: i.color } });82 }83}8485/** Mirror connectors/registry.json into sources + connectors tables (state columns are preserved). */86export async function seedRegistry(): Promise<void> {87 const here = path.dirname(fileURLToPath(import.meta.url));88 const registryPath = path.resolve(here, '../../../connectors/registry.json');89 if (!existsSync(registryPath)) return;90 const reg = JSON.parse(readFileSync(registryPath, 'utf8')) as { connectors: Array<Record<string, any>> };91 const db = getDb();92 let i = 0;93 for (const c of reg.connectors) {94 i++;95 // Stagger first runs of newly registered connectors over 6 hours so a wave of new sources does not hit every96 // host at once (existing rows keep their own next_run_at — the conflict clause does not touch it).97 const firstRunAt = new Date(Date.now() + (i % 24) * 15 * 60_000);98 await db99 .insert(sources)100 .values({ id: c.sourceId, name: c.sourceName, sourceType: c.sourceType, url: c.sourceUrl, countries: c.regions ?? [], languages: c.languages ?? ['en'], currencies: c.currency ?? ['USD'], trustScore: c.trustScore ?? 0.6, attributionRequired: c.attributionRequired ?? true, termsUrl: c.termsUrl ?? null })101 .onConflictDoUpdate({ target: sources.id, set: { name: c.sourceName, sourceType: c.sourceType, url: c.sourceUrl, trustScore: c.trustScore ?? 0.6, termsUrl: c.termsUrl ?? null, updatedAt: new Date() } });102 await db103 .insert(connectors)104 .values({105 id: c.id,106 sourceId: c.sourceId,107 displayName: c.displayName,108 enginePriority: c.enginePriority,109 categories: c.categories,110 regions: c.regions ?? [],111 languages: c.languages ?? ['en'],112 currency: c.currency ?? ['USD'],113 supportsListings: c.supportsListings ?? false,114 supportsSold: c.supportsSold ?? false,115 supportsAuctions: c.supportsAuctions ?? false,116 supportsImages: c.supportsImages ?? true,117 supportsCatalog: c.supportsCatalog ?? false,118 supportsPopulation: c.supportsPopulation ?? false,119 refreshFrequencyMinutes: c.refreshFrequencyMinutes ?? 1440,120 priority: c.priority ?? 'medium',121 status: c.enabled === false ? 'disabled' : 'active',122 schemaVersion: c.schemaVersion ?? '1.0',123 config: c.config ?? {},124 meta: c,125 nextRunAt: firstRunAt,126 })127 .onConflictDoUpdate({128 target: connectors.id,129 set: {130 sourceId: c.sourceId,131 displayName: c.displayName,132 enginePriority: c.enginePriority,133 categories: c.categories,134 regions: c.regions ?? [],135 languages: c.languages ?? ['en'],136 currency: c.currency ?? ['USD'],137 supportsListings: c.supportsListings ?? false,138 supportsSold: c.supportsSold ?? false,139 supportsAuctions: c.supportsAuctions ?? false,140 supportsImages: c.supportsImages ?? true,141 supportsCatalog: c.supportsCatalog ?? false,142 supportsPopulation: c.supportsPopulation ?? false,143 refreshFrequencyMinutes: c.refreshFrequencyMinutes ?? 1440,144 priority: c.priority ?? 'medium',145 schemaVersion: c.schemaVersion ?? '1.0',146 // registry config wins on shared keys, but runtime state stored in config (cursor) survives reseeds147 config: sql`coalesce(${connectors.config}, '{}'::jsonb) || ${JSON.stringify(c.config ?? {})}::jsonb`,148 meta: c,149 updatedAt: new Date(),150 // status is intentionally NOT overwritten (admin pause/maintenance survives reseeds) except when the registry151 // disables the connector, or re-enables one that was only disabled by the registry.152 status: c.enabled === false ? 'disabled' : sql`case when ${connectors.status} = 'disabled' then 'active' else ${connectors.status} end`,153 },154 });155 }156}157158export async function seedAll(): Promise<void> {159 await seedTaxonomy();160 await seedIndices();161 await seedRegistry();162 const db = getDb();163 const [{ n }] = (await db.execute(sql`select count(*)::int as n from categories`)) as unknown as [{ n: number }];164 console.log(`[seed] categories=${n} indices=${INDICES.length}`);165}166167const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);168if (isMain) {169 seedAll()170 .then(() => closeDb())171 .catch(async (e) => {172 console.error(e);173 await closeDb();174 process.exit(1);175 });176}177