import { newId, slugify, sourceImportanceFromTier } from "@websensor/core";
import { db, discoveryCandidates, entities, entityAliases, eq, sensors, sourceEntities, sources, sql, textArray } from "@websensor/db";
import { discoverDomain } from "@websensor/connectors";
import { config, log } from "./config";

/**
 * Source registry: `config/sources.yaml` is the seed (organizations + curated sensors).
 * Booleans under `discover:` only allow discovery; every candidate is validated by a real
 * fetch+parse before becoming a sensor.
 */
export type { SourceSeed } from "./seeds";
import { loadSeedsDetailed, type SourceSeed } from "./seeds";

export function loadSeeds(file = config.sourcesFile, dir = config.sourcesDir): SourceSeed[] {
  const { seeds, issues, files } = loadSeedsDetailed(file, dir);
  for (const i of issues) log.error({ file: i.file, source: i.source, issue: i.message }, "invalid source seed");
  log.info({ files: files.length, sources: seeds.length, sensors: seeds.reduce((n, s) => n + s.sensors.length, 0) }, "registry seeds loaded");
  return seeds;
}

/**
 * Upsert one organization (source row + organization entity + aliases + product entities). Shared by the YAML
 * registry sync (`origin = seed`) and the Source Factory (`origin = factory`). Sensors are handled by the caller.
 */
export async function upsertSourceRecord(s: SourceSeed, opts: { origin?: "seed" | "import" | "factory"; sector?: string | null } = {}): Promise<void> {
  // Provenance: media/aggregators are third-party; everything else is the organization's own channel.
  const firstParty = s.first_party ?? !s.categories.some((c) => c === "news" || c === "media");
  const country = s.country ?? inferCountry(s.domain, s.categories);
  const language = s.language ?? inferLanguage(s.domain);
  const origin = opts.origin ?? "seed";
  const base = { name: s.name, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, categories: s.categories, tier: s.tier, importanceWeight: s.weight, discover: s.discover, fallback: s.fallback, notes: s.notes, enabled: s.enabled, llmEnabled: s.llm, firstParty, country, language };
  await db
    .insert(sources)
    .values({ id: s.id, ...base, origin, sector: opts.sector ?? null })
    .onConflictDoUpdate({ target: sources.id, set: origin === "seed" ? { ...base, origin, updatedAt: new Date() } : { ...base, updatedAt: new Date(), ...(opts.sector ? { sector: opts.sector } : {}) } });

  // Organization entity + aliases
  const entId = `org_${s.id}`;
  await db
    .insert(entities)
    .values({ id: entId, name: s.name, type: s.entity_type, domain: s.domain, homepage: s.homepage ?? `https://${s.domain}`, description: s.description, importance: sourceImportanceFromTier(s.tier, s.weight), categories: s.categories })
    .onConflictDoUpdate({ target: entities.id, set: { name: s.name, type: s.entity_type, domain: s.domain, description: s.description, importance: sourceImportanceFromTier(s.tier, s.weight), categories: s.categories } });
  await db.insert(sourceEntities).values({ sourceId: s.id, entityId: entId }).onConflictDoNothing();
  for (const alias of new Set([s.name, s.domain, s.domain.replace(/^www\./, ""), ...s.aliases])) {
    await db.insert(entityAliases).values({ alias: alias.toLowerCase(), entityId: entId }).onConflictDoNothing();
  }
  for (const p of s.products) {
    const pid = p.id ?? `prd_${s.id}_${slugify(p.name)}`;
    await db
      .insert(entities)
      .values({ id: pid, name: p.name, type: p.type, parentId: entId, importance: Math.max(30, sourceImportanceFromTier(s.tier, s.weight) - 10), categories: s.categories, domain: s.domain })
      .onConflictDoUpdate({ target: entities.id, set: { name: p.name, type: p.type, parentId: entId } });
    await db.insert(sourceEntities).values({ sourceId: s.id, entityId: pid }).onConflictDoNothing();
    for (const alias of new Set([p.name, ...p.aliases])) await db.insert(entityAliases).values({ alias: alias.toLowerCase(), entityId: pid }).onConflictDoNothing();
    await db.execute(sql`insert into entity_relations (from_id, relation, to_id) values (${entId}, 'owns', ${pid}) on conflict do nothing`);
  }
}

export async function syncRegistry(seeds = loadSeeds()): Promise<{ sources: number; sensors: number }> {
  let nSensors = 0;
  for (const s of seeds) {
    await upsertSourceRecord(s, { origin: "seed" });

    const seedIds: string[] = [];
    for (const sen of s.sensors) {
      const id = sen.id ?? `${s.id}_${slugify(sen.name)}`;
      seedIds.push(id);
      const cfg = { ...sen.config, seed: true };
      const tier = sen.tier ?? s.tier;
      const priority = priorityFor(tier, s.categories);
      await db
        .insert(sensors)
        .values({ id, sourceId: s.id, name: sen.name, url: sen.url, type: sen.type, connector: sen.connector, tier, importanceWeight: sen.weight ?? 1, config: cfg, baseIntervalSeconds: sen.interval ?? null, priority, status: "VALIDATED", validatedAt: new Date() })
        .onConflictDoUpdate({ target: sensors.id, set: { name: sen.name, url: sen.url, type: sen.type, connector: sen.connector, tier, importanceWeight: sen.weight ?? 1, config: cfg, baseIntervalSeconds: sen.interval ?? null, priority, enabled: true, updatedAt: new Date() } });
      nSensors++;
    }
    // Seed sensors removed from the YAML are disabled (history kept), discovery-created ones are untouched.
    await db.execute(sql`update sensors set enabled = false, status = 'DISABLED', updated_at = now() where source_id = ${s.id} and enabled and (config->>'seed') = 'true' and not (id = any(${textArray(seedIds)}))`);
  }
  log.info({ sources: seeds.length, sensors: nSensors }, "registry synced");
  return { sources: seeds.length, sensors: nSensors };
}

/**
 * Discovery: for each enabled source whose `discover` flags allow it, probe the domain,
 * store candidates and promote validated feeds / sitemaps / statuspages to sensors that do
 * not already exist for that URL.
 */
export async function runDiscovery(opts: { onlyMissing?: boolean; sourceIds?: string[] } = {}): Promise<number> {
  const rows = await db.select().from(sources).where(eq(sources.enabled, true));
  let promoted = 0;
  const targets = rows.filter((r) => !opts.sourceIds || opts.sourceIds.includes(r.id));
  let i = 0;
  const workers = Array.from({ length: 4 }, async () => {
    while (i < targets.length) {
      const src = targets[i++]!;
      const d = src.discover as { rss?: boolean; sitemap?: boolean; status?: boolean; pages?: boolean };
      if (!d.rss && !d.sitemap && !d.status && !d.pages) continue;
      const existing = await db.select({ url: sensors.url, type: sensors.type }).from(sensors).where(eq(sensors.sourceId, src.id));
      const lastRun = (src.notes ?? "").match(/discovered_at=(\S+)/)?.[1];
      if (opts.onlyMissing && lastRun && Date.now() - new Date(lastRun).getTime() < config.discovery.intervalDays * 86400e3) continue;
      const started = Date.now();
      let found;
      try {
        found = await discoverDomain(src.domain, { probePages: Boolean(d.pages), sensorIdForLogs: `discover_${src.id}` });
      } catch (e) {
        log.warn({ source: src.id, err: (e as Error).message }, "discovery failed");
        continue;
      }
      const existingUrls = new Set(existing.map((e) => e.url.replace(/\/$/, "")));
      const hasFeed = existing.some((e) => e.type === "RSS" || e.type === "ATOM");
      let feedsAdded = 0;
      for (const f of found) {
        await db
          .insert(discoveryCandidates)
          .values({ id: newId("cand"), sourceId: src.id, url: f.url, kind: f.type, evidence: f.evidence, score: { value: f.value, itemCount: f.itemCount ?? null, title: f.title ?? null } })
          .onConflictDoNothing();
        if (existingUrls.has(f.url.replace(/\/$/, ""))) continue;
        const allowed = (f.connector === "rss" && d.rss) || (f.connector === "sitemap" && d.sitemap) || (f.connector === "statuspage" && d.status) || (f.connector === "http" && d.pages);
        if (!allowed) continue;
        // Promotion policy: feeds (max 3 per source, best first), one sitemap, one statuspage, pricing/changelog/security pages.
        if (f.connector === "rss" && feedsAdded + (hasFeed ? 1 : 0) >= 3) continue;
        if (f.connector === "sitemap" && existing.some((e) => e.type === "SITEMAP")) continue;
        if (f.connector === "statuspage" && existing.some((e) => e.type === "STATUSPAGE")) continue;
        if (f.connector === "http" && !/pricing|changelog|security|releases/i.test(f.url)) continue;
        const name = f.connector === "rss" ? feedName(f.url, f.title) : f.connector === "sitemap" ? "sitemap" : f.connector === "statuspage" ? "status" : new URL(f.url).pathname.replace(/\W+/g, " ").trim() || "page";
        const id = `${src.id}_${slugify(name)}`.slice(0, 80);
        const tier = f.connector === "statuspage" ? "S" : f.connector === "sitemap" ? (src.tier === "S" ? "A" : src.tier) : src.tier;
        await db
          .insert(sensors)
          .values({ id, sourceId: src.id, name, url: f.url, type: f.type, connector: f.connector, tier, config: f.connector === "sitemap" ? { maxChildren: 4, maxUrls: 3000 } : {}, status: "VALIDATED", validatedAt: new Date(), priority: priorityFor(tier, src.categories) })
          .onConflictDoNothing();
        await db.update(discoveryCandidates).set({ status: "promoted" }).where(sql`source_id = ${src.id} and url = ${f.url}`);
        existingUrls.add(f.url.replace(/\/$/, ""));
        if (f.connector === "rss") feedsAdded++;
        promoted++;
      }
      const notes = ((src.notes ?? "").replace(/\s*discovered_at=\S+/, "") + ` discovered_at=${new Date().toISOString()}`).trim();
      await db.update(sources).set({ notes, robotsCheckedAt: new Date(), updatedAt: new Date() }).where(eq(sources.id, src.id));
      log.info({ source: src.id, candidates: found.length, ms: Date.now() - started }, "discovery done");
    }
  });
  await Promise.all(workers);
  log.info({ promoted }, "discovery promoted sensors");
  return promoted;
}

/** Priority tier (spec §76): 0 critical infrastructure/government/major AI/cyber · 1 major companies/markets/science/health · 2 specialized · 3 low. */
export function priorityFor(tier: string, categories: string[]): number {
  if (tier === "S") return 0;
  if (tier === "A" && categories.some((c) => ["cyber", "government", "ai", "cloud", "internet", "infrastructure", "finance", "health"].includes(c))) return 0;
  if (tier === "A" || (tier === "B" && categories.some((c) => ["finance", "science", "health", "pharma", "ai", "cyber", "government"].includes(c)))) return 1;
  if (tier === "B" || tier === "C") return 2;
  return 3;
}

const TLD_COUNTRY: Record<string, string> = { ca: "CA", "gc.ca": "CA", "gouv.qc.ca": "CA", uk: "GB", "gov.uk": "GB", fr: "FR", "gouv.fr": "FR", de: "DE", it: "IT", es: "ES", jp: "JP", "go.jp": "JP", kr: "KR", "go.kr": "KR", au: "AU", "gov.au": "AU", in: "IN", "gov.in": "IN", br: "BR", "gov.br": "BR", mx: "MX", "gob.mx": "MX", ch: "CH", nl: "NL", se: "SE", no: "NO", fi: "FI", dk: "DK", ie: "IE", be: "BE", at: "AT", pt: "PT", pl: "PL", cn: "CN", hk: "HK", tw: "TW", sg: "SG", il: "IL", ae: "AE", sa: "SA", qa: "QA", za: "ZA", ng: "NG", ke: "KE", ar: "AR", cl: "CL", co: "CO", nz: "NZ", ru: "RU", tr: "TR", id: "ID", th: "TH", vn: "VN", my: "MY", ph: "PH", eg: "EG", ua: "UA", cz: "CZ", gr: "GR", lu: "LU", eu: "EU", "europa.eu": "EU", gov: "US", mil: "US", "us.com": "US" };

/** Country inference for seeds without an explicit `country:` — only from unambiguous TLDs; `.com/.org/.io` stay null. */
export function inferCountry(domain: string, categories: string[]): string | null {
  const d = domain.toLowerCase();
  for (const [suffix, code] of Object.entries(TLD_COUNTRY).sort((a, b) => b[0].length - a[0].length)) if (d === suffix || d.endsWith("." + suffix)) return code;
  if (/\.int$|^un\.org$|\.who\.int$|\.imf\.org$|\.worldbank\.org$|\.oecd\.org$|\.bis\.org$|\.wto\.org$|\.iso\.org$|\.ietf\.org$|\.w3\.org$|\.icann\.org$/.test(d) || categories.includes("international")) return "INT";
  return null;
}

export function inferLanguage(domain: string): string | null {
  const d = domain.toLowerCase();
  if (/\.qc\.ca$|quebec\.ca$|gouv\.fr$|\.fr$|lapresse\.ca|ledevoir\.com|journaldemontreal\.com|tvanouvelles\.ca|radio-canada\.ca|lemonde\.fr|lefigaro\.fr|24heures\.ca|noovo\.ca|lesoleil\.com|ledroit\.com|lapresse\.ca/.test(d)) return "fr";
  if (/\.de$|spiegel\.de|faz\.net|sueddeutsche\.de|handelsblatt\.com|\.at$|swissinfo\.ch/.test(d)) return "de";
  if (/\.es$|elpais\.com|elmundo\.es|\.mx$|eluniversal\.com\.mx|\.ar$|\.cl$|\.co$/.test(d)) return "es";
  if (/\.it$|repubblica\.it|corriere\.it/.test(d)) return "it";
  if (/\.br$|\.pt$|globo\.com|folha\.uol\.com\.br/.test(d)) return "pt";
  if (/\.jp$|nhk\.or\.jp|nikkei\.com$/.test(d)) return "ja";
  if (/\.kr$/.test(d)) return "ko";
  if (/\.cn$|xinhuanet\.com/.test(d)) return "zh";
  if (/\.nl$/.test(d)) return "nl";
  if (/\.ru$|tass\.(ru|com)/.test(d)) return "ru";
  return null;
}

function feedName(url: string, title?: string): string {
  const p = new URL(url).pathname.toLowerCase();
  if (/blog/.test(p)) return "blog feed";
  if (/news|press/.test(p)) return "news feed";
  if (/release|changelog/.test(p)) return "changelog feed";
  if (/security|advisor/.test(p)) return "security feed";
  if (title) return slugify(title).replace(/-/g, " ").slice(0, 40) + " feed";
  return "feed";
}