/**
 * Validate data/sources/sources.json (SPEC §4) and generate docs/connectors/SOURCES.md.
 * Run: pnpm tsx scripts/build-sources.ts
 */
import { readFileSync, writeFileSync, existsSync, readdirSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { SourcesFileSchema, type SourceEntry } from '../packages/connectors/src/sources.ts';

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
// Base catalogue + fragments: data/sources/entries/<group>.json = SourceEntry[] (fragments override the base by id).
const basePath = path.join(root, 'data/sources/sources.json');
const file = SourcesFileSchema.parse(existsSync(basePath) ? JSON.parse(readFileSync(basePath, 'utf8')) : { version: '1.0', sources: [] });
const entriesDir = path.join(root, 'data/sources/entries');
if (existsSync(entriesDir)) {
  const byId = new Map(file.sources.map((s) => [s.id, s] as const));
  for (const f of readdirSync(entriesDir).filter((x) => x.endsWith('.json')).sort()) {
    const arr = JSON.parse(readFileSync(path.join(entriesDir, f), 'utf8')) as unknown[];
    if (!Array.isArray(arr)) throw new Error(`${f}: expected an array of SourceEntry`);
    for (const raw of arr) {
      const parsed = SourcesFileSchema.shape.sources.element.safeParse(raw);
      if (!parsed.success) throw new Error(`${f}: invalid entry ${JSON.stringify((raw as { id?: string })?.id)}: ${parsed.error.issues[0]?.path.join('.')} ${parsed.error.issues[0]?.message}`);
      byId.set(parsed.data.id, { ...(byId.get(parsed.data.id) ?? {}), ...parsed.data });
    }
  }
  file.sources = [...byId.values()];
}
const registryPath = path.join(root, 'connectors/registry.json');
const registry = existsSync(registryPath) ? (JSON.parse(readFileSync(registryPath, 'utf8')) as { connectors: Array<{ id: string; sourceId: string; enabled: boolean }> }) : { connectors: [] };
const implemented = new Set(registry.connectors.map((c) => c.id));
const bySource = new Map<string, string[]>();
for (const c of registry.connectors) bySource.set(c.sourceId, [...(bySource.get(c.sourceId) ?? []), c.id]);

// Synthesize a catalogue entry for every registered connector whose source has no hand-written entry,
// so SOURCES.md always lists 100% of the connectors (facts come from meta.json; marked verified because
// the connector was built against the live source).
const fullRegistry = existsSync(registryPath) ? (JSON.parse(readFileSync(registryPath, 'utf8')) as { connectors: Array<Record<string, any>> }) : { connectors: [] };
const known = new Set(file.sources.map((s) => s.id));
for (const c of fullRegistry.connectors) {
  if (known.has(c.sourceId) || known.has(c.id)) continue;
  const engine = String(c.enginePriority?.[0] ?? 'api');
  const entry = SourcesFileSchema.shape.sources.element.parse({
    id: c.sourceId,
    name: c.sourceName ?? c.displayName,
    domain: c.domain ?? new URL(c.sourceUrl).hostname.replace(/^www\./, ''),
    country: c.country ?? c.regions?.[0] ?? 'global',
    supportedCountries: c.regions ?? [],
    languages: c.languages ?? ['en'],
    sourceType: c.sourceType,
    categories: c.categories,
    api: { available: engine === 'api' && /api|json|feed|bulk/i.test(String(c.acquisitionMethod ?? c.accessNotes ?? '')), docsUrl: null, auth: c.requires?.length ? 'api_key' : 'none', notes: c.acquisitionMethod ?? null },
    urls: { robots: `${c.sourceUrl.replace(/\/$/, '')}/robots.txt` },
    data: { liveListings: Boolean(c.supportsListings), soldResults: Boolean(c.supportsSold), auctionResults: Boolean(c.supportsAuctions), historical: c.historicalDepth ?? (c.supportsSold ? 'years' : 'none'), images: c.supportsImages !== false, saleDate: Boolean(c.supportsSold), realizedPrice: Boolean(c.supportsSold), population: Boolean(c.supportsPopulation), catalog: Boolean(c.supportsCatalog), grading: /grade|psa|bgs|cgc|slab/i.test(String(c.accessNotes ?? '')), certNumber: /cert/i.test(String(c.accessNotes ?? '')) },
    access: { pagination: 'page_number', jsRendering: engine !== 'api', antiBot: engine === 'scrapfly' ? 'high' : engine === 'firecrawl' ? 'low' : 'none', cloudflare: /cloudflare/i.test(String(c.accessNotes ?? '')), robots: null, rateLimit: null, preferredEngine: engine },
    currency: c.currency ?? ['USD'],
    freshness: (c.refreshFrequencyMinutes ?? 1440) <= 60 ? 'hourly' : (c.refreshFrequencyMinutes ?? 1440) <= 1440 ? 'daily' : 'weekly',
    priority: c.priority === 'high' ? 'wave1' : c.priority === 'low' ? 'wave3' : 'wave2',
    difficulty: engine === 'scrapfly' ? 'hard' : engine === 'firecrawl' ? 'medium' : 'easy',
    reliability: c.trustScore ?? 0.6,
    legal: c.accessNotes ? String(c.accessNotes).slice(0, 400) : null,
    status: c.enabled === false ? 'partial' : c.requires?.length ? 'gated' : 'implemented',
    connectors: [c.id],
    statusReason: c.enabled === false ? 'connector disabled in registry' : null,
    verified: true,
    verifiedAt: null,
    notes: 'entry synthesized from connector meta.json',
  });
  file.sources.push(entry);
  known.add(entry.id);
}

// Reconcile status with the registry: a source with a registered connector is at least "partial".
const ids = new Set<string>();
for (const s of file.sources) {
  if (ids.has(s.id)) throw new Error(`duplicate source id ${s.id}`);
  ids.add(s.id);
  const cons = [...new Set([...(bySource.get(s.id) ?? []), ...s.connectors.filter((c) => implemented.has(c))])];
  s.connectors = cons;
  if (cons.length && (s.status === 'planned' || s.status === 'researching')) s.status = 'partial';
  if (!cons.length && s.status === 'implemented') s.status = 'planned';
}
file.updatedAt = new Date().toISOString();
file.sources.sort((a, b) => a.id.localeCompare(b.id));
writeFileSync(path.join(root, 'data/sources/sources.json'), JSON.stringify(file, null, 2) + '\n');

const count = (f: (s: SourceEntry) => boolean) => file.sources.filter(f).length;
const statuses = ['implemented', 'partial', 'gated', 'planned', 'researching', 'blocked', 'rejected'] as const;
const groups = new Map<string, SourceEntry[]>();
for (const s of file.sources) {
  const key = s.categories[0] ?? 'other';
  groups.set(key, [...(groups.get(key) ?? []), s]);
}
const flag = (b: boolean) => (b ? '✓' : '·');
const esc = (s: string | null | undefined) => (s ?? '').replace(/\|/g, '\\|').replace(/\n/g, ' ');

let md = `# Source catalog (SPEC §4)\n\n_Generated from \`data/sources/sources.json\` by \`scripts/build-sources.ts\` on ${file.updatedAt.slice(0, 10)}. Edit the JSON, not this file._\n\n`;
md += `**${file.sources.length} sources researched** · ${statuses.map((st) => `${st}: ${count((s) => s.status === st)}`).join(' · ')}\n\n`;
md += `Status legend: **implemented** connector in production · **partial** connector exists but incomplete (see reason) · **gated** needs an API key/partner access we do not hold · **planned** researched, connector to build · **researching** facts not yet verified · **blocked** login/anti-bot/robots prevents lawful public access · **rejected** not worth building (see reason).\n\n`;

md += `## By country\n\n| Country | Sources | Implemented |\n|---|---:|---:|\n`;
const countries = [...new Set(file.sources.map((s) => s.country))].sort();
for (const c of countries) md += `| ${c} | ${count((s) => s.country === c)} | ${count((s) => s.country === c && (s.status === 'implemented' || s.status === 'partial'))} |\n`;

md += `\n## By category\n\n`;
for (const [cat, list] of [...groups.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
  md += `### ${cat} (${list.length})\n\n| Source | Domain | Country | Type | Status | Connectors | API | Live | Sold | Auction | Hist. | Pop. | Cert | GTIN | JS | Anti-bot | Pagination | Engine | Currency | Priority | Difficulty | Notes |\n|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n`;
  for (const s of list.sort((a, b) => a.name.localeCompare(b.name))) {
    md += `| ${esc(s.name)} | ${s.domain} | ${s.country} | ${s.sourceType} | **${s.status}** | ${s.connectors.join(', ') || '—'} | ${s.api.available ? `✓ (${s.api.auth})` : '·'} | ${flag(s.data.liveListings)} | ${flag(s.data.soldResults)} | ${flag(s.data.auctionResults)} | ${s.data.historical} | ${flag(s.data.population)} | ${flag(s.data.certNumber)} | ${flag(s.data.gtin)} | ${flag(s.access.jsRendering)} | ${s.access.antiBot}${s.access.cloudflare ? ' (CF)' : ''} | ${s.access.pagination} | ${s.access.preferredEngine} | ${s.currency.join('/')} | ${s.priority} | ${s.difficulty} | ${esc(s.statusReason ?? s.notes)} |\n`;
  }
  md += '\n';
}

md += `## Rejected / blocked sources and why\n\n| Source | Status | Reason |\n|---|---|---|\n`;
for (const s of file.sources.filter((x) => x.status === 'rejected' || x.status === 'blocked').sort((a, b) => a.name.localeCompare(b.name))) md += `| ${esc(s.name)} | ${s.status} | ${esc(s.statusReason)} |\n`;

md += `\n## Legal / compliance notes\n\n| Source | Notes |\n|---|---|\n`;
for (const s of file.sources.filter((x) => x.legal).sort((a, b) => a.name.localeCompare(b.name))) md += `| ${esc(s.name)} | ${esc(s.legal)} |\n`;

writeFileSync(path.join(root, 'docs/connectors/SOURCES.md'), md);
console.log(`[sources] ${file.sources.length} sources → docs/connectors/SOURCES.md (${statuses.map((st) => `${st}=${count((s) => s.status === st)}`).join(' ')})`);