spb/rareindex

1/**2 * Assemble connectors/registry.json from every connectors/<engine>/<id>/meta.json.3 * Per-connector meta files avoid merge conflicts; registry.json stays the machine-readable4 * registry required by CLAUDE.md §106. Derived fields (domain, country, capabilities, refreshClass)5 * are filled in here so meta.json stays short. Run: pnpm tsx scripts/build-registry.ts [--check]6 */7import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';8import path from 'node:path';9import { fileURLToPath } from 'node:url';10import { RegistrySchema, ConnectorMetaSchema, capabilitiesOf, domainOf, effectiveRefreshClass, type ConnectorMeta } from '../packages/connectors/src/types.ts';11import { SourcesFileSchema } from '../packages/connectors/src/sources.ts';12import { DomainsFileSchema, DOMAINS_DIR } from '../packages/connectors/src/domains.ts';1314const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');15const dir = path.join(root, 'connectors');16const check = process.argv.includes('--check');17const metas: ConnectorMeta[] = [];18const problems: string[] = [];19for (const engine of readdirSync(dir)) {20  const p = path.join(dir, engine);21  if (!statSync(p).isDirectory() || engine === 'node_modules' || engine.startsWith('_')) continue;22  for (const id of readdirSync(p)) {23    if (!statSync(path.join(p, id)).isDirectory() || id.startsWith('_')) continue;24    const metaPath = path.join(p, id, 'meta.json');25    try {26      const meta = JSON.parse(readFileSync(metaPath, 'utf8'));27      const parsed = ConnectorMetaSchema.parse({ ...meta, module: meta.module ?? `${engine}/${id}` });28      if (parsed.id !== id) throw new Error(`meta.id ${parsed.id} ≠ folder ${id}`);29      parsed.domain ??= domainOf(parsed.sourceUrl);30      parsed.country ??= parsed.regions[0] ?? 'global';31      parsed.capabilities = capabilitiesOf(parsed);32      parsed.refreshClass = effectiveRefreshClass(parsed);33      if (!existsSync(path.join(p, id, 'index.ts'))) problems.push(`${engine}/${id}: index.ts missing`);34      if (!existsSync(path.join(p, id, 'index.test.ts'))) problems.push(`${engine}/${id}: index.test.ts missing (SPEC §14)`);35      const fx = path.join(root, 'data/fixtures', id);36      if (!existsSync(fx) || !readdirSync(fx).some((f) => f.endsWith('.json'))) problems.push(`${engine}/${id}: no fixture in data/fixtures/${id} (SPEC §14)`);37      if (!parsed.accessNotes) problems.push(`${engine}/${id}: accessNotes missing (legal/compliance notes, SPEC §36)`);38      metas.push(parsed);39    } catch (err) {40      if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue;41      throw new Error(`${metaPath}: ${err instanceof Error ? err.message : String(err)}`);42    }43  }44}45metas.sort((a, b) => (a.id < b.id ? -1 : 1));46const registry = RegistrySchema.parse({ version: '1.1', connectors: metas });4748// Cross-check with the source registry (SPEC §4): every connector's sourceId should be a documented source.49const sourcesPath = path.join(root, 'data/sources/sources.json');50if (existsSync(sourcesPath)) {51  const sources = SourcesFileSchema.parse(JSON.parse(readFileSync(sourcesPath, 'utf8')));52  const ids = new Set(sources.sources.map((s) => s.id));53  for (const m of metas) if (!ids.has(m.sourceId) && !ids.has(m.id)) problems.push(`${m.id}: sourceId "${m.sourceId}" has no entry in data/sources/sources.json`);54}5556// Strict validation of domain-policy fragments (runtime skips invalid ones with a warning).57if (existsSync(DOMAINS_DIR)) {58  for (const f of readdirSync(DOMAINS_DIR).filter((x) => x.endsWith('.json'))) {59    const r = DomainsFileSchema.safeParse({ ...JSON.parse(readFileSync(path.join(DOMAINS_DIR, f), 'utf8')), version: '1.0' });60    if (!r.success) problems.push(`connectors/domains.d/${f}: ${r.error.issues[0]?.path.join('.')} ${r.error.issues[0]?.message}`);61  }62}6364if (!check) writeFileSync(path.join(dir, 'registry.json'), JSON.stringify(registry, null, 2) + '\n');65console.log(`[registry] ${registry.connectors.length} connectors ${check ? 'validated' : 'written to connectors/registry.json'}`);66if (problems.length) {67  console.log(`[registry] ${problems.length} completeness warning(s):`);68  for (const p of problems) console.log(`  - ${p}`);69  if (check && process.argv.includes('--strict')) process.exit(1);70}71