/**
 * Golden tests: archived production bodies (apps/worker/fixtures/<connectorId>/<name>.{html,json,xml,pdf}) run through
 * extract → normalize → validate with the real YAML config and the real registered parsers — no network.
 * Expectations live next to each body in `<name>.expected.json` (schema: docs/CONNECTORS.md § Fixtures). Adding a
 * fixture = adding two files; nothing here needs editing.
 *
 * `_todo` blocks inside an expected entity (or the announcement) hold the CORRECT value the parser does not produce
 * yet: they run as `it.fails`, so fixing the parser flips them red until the field is moved out of `_todo`.
 */
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import type { NormalizedEntity } from "@dci/core";
import { GenericConnector, THIRD_PARTY_SUMMARY_MAX, createTestContext, getImplementation, parseConnectorConfig, validateEntities, type Connector, type ExtractedRecord, type RawDocument } from "@dci/connectors";
import { register as registerOperators1 } from "./operators1/index.js";
import { register as registerOperators2 } from "./operators2/index.js";
import { register as registerCloud } from "./cloud/index.js";
import { register as registerDatasets } from "./datasets/index.js";
import { register as registerNews } from "./news/index.js";
import { articleContent } from "./news/article-parser.js";
import { extractAnnouncement, type Announcement } from "./news/extract-project.js";

const HERE = dirname(fileURLToPath(import.meta.url));
const FIXTURES_DIR = resolve(HERE, "../../fixtures");
const CONFIG_DIR = resolve(HERE, "../../../../config/connectors");
const BODY_EXT = [".html", ".htm", ".json", ".xml", ".pdf", ".txt", ".md"];

type Todo = Record<string, unknown> & { _reason?: string };
interface ExpectedEntity { key: string; _todo?: Todo; [field: string]: unknown }
interface ExpectedAnnouncement { _todo?: Todo; [field: string]: unknown }
interface Expected {
  url: string; documentId: string; fetchedAt: string; contentType: string; note?: string;
  /** asserted count per entityType (absent key = not asserted) */
  counts?: Partial<Record<NormalizedEntity["entityType"], number>>;
  facilities?: ExpectedEntity[]; projects?: ExpectedEntity[]; newsEvents?: ExpectedEntity[]; cloudRegions?: ExpectedEntity[]; operators?: ExpectedEntity[];
  /** kind = news publisher: summaries ≤ THIRD_PARTY_SUMMARY_MAX and no article text in the records */
  thirdParty?: boolean;
  /** `extractAnnouncement(title, text)` facts (news / government fixtures) */
  announcement?: ExpectedAnnouncement;
}
interface Fixture { connectorId: string; name: string; bodyPath: string; expected: Expected }

let registered = false;
function registerParsers(): void {
  if (registered) return;
  registered = true;
  for (const reg of [registerOperators1, registerOperators2, registerCloud, registerDatasets, registerNews]) {
    try { reg(); } catch (e) { if (!/already registered/i.test((e as Error).message)) throw e; }
  }
}

function listFixtures(): Fixture[] {
  if (!existsSync(FIXTURES_DIR)) return [];
  const out: Fixture[] = [];
  for (const dir of readdirSync(FIXTURES_DIR, { withFileTypes: true })) {
    if (!dir.isDirectory()) continue;
    const files = readdirSync(join(FIXTURES_DIR, dir.name));
    for (const f of files.filter((x) => x.endsWith(".expected.json")).sort()) {
      const name = f.slice(0, -".expected.json".length);
      const body = files.find((x) => x !== f && x.startsWith(`${name}.`) && BODY_EXT.some((ext) => x.endsWith(ext)));
      if (!body) throw new Error(`fixture ${dir.name}/${name}: expected.json without a body file (${BODY_EXT.join("|")})`);
      out.push({ connectorId: dir.name, name, bodyPath: join(FIXTURES_DIR, dir.name, body), expected: JSON.parse(readFileSync(join(FIXTURES_DIR, dir.name, f), "utf8")) as Expected });
    }
  }
  return out;
}

function loadConnector(connectorId: string): Connector {
  const file = join(CONFIG_DIR, `${connectorId}.yaml`);
  if (!existsSync(file)) throw new Error(`no config/connectors/${connectorId}.yaml for fixture dir ${connectorId}`);
  const cfg = parseConnectorConfig(readFileSync(file, "utf8"), file);
  const impl = cfg.implementation ? getImplementation(cfg.implementation) : undefined;
  if (cfg.implementation && !impl) throw new Error(`implementation "${cfg.implementation}" not registered`);
  return impl ? impl(cfg) : new GenericConnector(cfg);
}

function rawDocument(fx: Fixture): RawDocument {
  const body = readFileSync(fx.bodyPath);
  const { url, contentType, fetchedAt } = fx.expected;
  return { url, finalUrl: url, fetchedAt, status: 200, contentType, body, text: /pdf/i.test(contentType) ? "" : body.toString("utf8"), headers: {}, etag: null, lastModified: null, notModified: false, fetcher: "cache", level: 1, durationMs: 0, credits: 0 };
}

/** "mentions.countriesIso2" → value; missing path → undefined */
function get(obj: unknown, path: string): unknown {
  return path.split(".").reduce<unknown>((o, k) => (o != null && typeof o === "object" ? (o as Record<string, unknown>)[k] : undefined), obj);
}

/** `null` in expected.json means "absent or null" (normalized entities use null for unknown fields). */
function normalize(v: unknown): unknown { return v === undefined ? null : v; }

interface Run { records: ExtractedRecord[]; entities: NormalizedEntity[]; errors: Array<{ key: string; field: string | undefined; message: string }>; warnings: number; announcement: Announcement | null; published: string | null }
async function runFixture(fx: Fixture): Promise<Run> {
  registerParsers();
  const connector = loadConnector(fx.connectorId);
  const cfg = (connector as GenericConnector).cfg ?? parseConnectorConfig(readFileSync(join(CONFIG_DIR, `${fx.connectorId}.yaml`), "utf8"));
  const ctx = createTestContext(cfg);
  ctx.log = () => {}; // keep vitest output clean; the connector never fetches during extract/normalize
  const doc = rawDocument(fx);
  const records = await connector.extract(ctx, doc);
  const entities = await connector.normalize(ctx, records);
  const report = validateEntities(entities);
  const errors = report.issues.filter((i) => i.level === "error").map(({ key, field, message }) => ({ key, field, message }));
  let announcement: Announcement | null = null, published: string | null = null;
  if (fx.expected.announcement) {
    const content = await articleContent(rawDocument(fx));
    published = content.published;
    announcement = extractAnnouncement(content.title, content.text, { publishedAt: content.published });
  }
  return { records, entities, errors, warnings: report.issues.length - errors.length, announcement, published };
}

/** Flattened view of an Announcement for expected.json (`announcement.<key>`). */
function announcementFacts(a: Announcement, published: string | null): Record<string, unknown> {
  return {
    title: a.title, published, class: a.classification.class, mayCreateProject: a.classification.mayCreateProject, physical: a.classification.physical,
    headlineMw: a.headlineMw, mwAll: a.mwAll, status: a.status, operator: a.operator?.name ?? null, operators: a.operators.map((o) => o.name),
    country: a.location?.country ?? null, city: a.location?.city ?? null, region: a.location?.region ?? null,
    money: a.money, investmentUsd: a.investmentUsd, acreage: a.acreage, phaseCount: a.phaseCount, expectedOpening: a.expectedOpening, projectName: a.projectName, explicitName: a.explicitName,
    relevance: a.relevance, leadRelevance: a.leadRelevance, titleSignal: a.titleSignal, nonProjectTitle: a.nonProjectTitle, pageType: a.pageType, eventType: a.eventType ?? null,
  };
}

const ENTITY_LISTS: Array<[keyof Expected, NormalizedEntity["entityType"]]> = [["facilities", "facility"], ["projects", "project"], ["newsEvents", "news_event"], ["cloudRegions", "cloud_region"], ["operators", "operator"]];

const fixtures = listFixtures();

describe("connector fixtures (golden)", () => {
  it("every fixture directory has a connector config and complete metadata", () => {
    expect(fixtures.length).toBeGreaterThan(0);
    for (const fx of fixtures) {
      expect(existsSync(join(CONFIG_DIR, `${fx.connectorId}.yaml`)), `${fx.connectorId}: config/connectors/${fx.connectorId}.yaml`).toBe(true);
      for (const k of ["url", "documentId", "fetchedAt", "contentType"] as const) expect(typeof fx.expected[k], `${fx.connectorId}/${fx.name}: ${k}`).toBe("string");
      expect(fx.expected.url, `${fx.connectorId}/${fx.name}: url`).toMatch(/^https?:\/\//);
      expect(fx.expected.documentId, `${fx.connectorId}/${fx.name}: documentId`).toMatch(/^doc_[0-9a-f]{16}$/);
      expect(Number.isFinite(Date.parse(fx.expected.fetchedAt)), `${fx.connectorId}/${fx.name}: fetchedAt`).toBe(true);
    }
  });

  for (const fx of fixtures) {
    const label = `${fx.connectorId}/${fx.name}`;
    const exp = fx.expected;
    let cached: Promise<Run> | null = null;
    const run = () => (cached ??= runFixture(fx));

    describe(label, () => {
      it("extracts, normalizes and validates as expected.json says", async () => {
        const r = await run();
        expect(r.errors, `validation errors for ${label}`).toEqual([]);
        for (const [type, n] of Object.entries(exp.counts ?? {})) expect(r.entities.filter((e) => e.entityType === type).length, `${type} count`).toBe(n);
        for (const [listKey, entityType] of ENTITY_LISTS) {
          for (const want of (exp[listKey] as ExpectedEntity[] | undefined) ?? []) {
            const got = r.entities.find((e) => e.entityType === entityType && e.key === want.key);
            expect(got, `${entityType} ${want.key} present (have: ${r.entities.filter((e) => e.entityType === entityType).map((e) => e.key).join(", ")})`).toBeDefined();
            for (const [field, value] of Object.entries(want)) {
              if (field === "key" || field === "_todo") continue;
              expect(normalize(get(got, field)), `${want.key}.${field}`).toEqual(normalize(value));
            }
          }
        }
        if (exp.thirdParty) {
          for (const e of r.entities) if (e.entityType === "news_event") expect((e.summary ?? "").length, `${e.key} summary ≤ ${THIRD_PARTY_SUMMARY_MAX}`).toBeLessThanOrEqual(THIRD_PARTY_SUMMARY_MAX);
          for (const rec of r.records) if (rec.kind === "news_event") expect(rec.data.text ?? null, `${rec.key} must not store third-party article text`).toBeNull();
        }
        if (exp.announcement) {
          expect(r.announcement).not.toBeNull();
          const facts = announcementFacts(r.announcement!, r.published);
          for (const [field, value] of Object.entries(exp.announcement)) {
            if (field === "_todo") continue;
            expect(field in facts, `announcement.${field} is not a known fact (${Object.keys(facts).join(", ")})`).toBe(true);
            expect(normalize(facts[field]), `announcement.${field}`).toEqual(normalize(value));
          }
        }
      });

      // Known parser gaps: the CORRECT value is asserted and the test is expected to fail until the parser is fixed.
      for (const [listKey, entityType] of ENTITY_LISTS) {
        for (const want of (exp[listKey] as ExpectedEntity[] | undefined) ?? []) {
          for (const [field, value] of Object.entries(want._todo ?? {})) {
            if (field === "_reason") continue;
            it.fails(`known gap — ${want.key}.${field} should be ${JSON.stringify(value)} (${want._todo?._reason ?? "no reason given"})`, async () => {
              const r = await run();
              const got = r.entities.find((e) => e.entityType === entityType && e.key === want.key);
              expect(normalize(get(got, field))).toEqual(normalize(value));
            });
          }
        }
      }
      for (const [field, value] of Object.entries(exp.announcement?._todo ?? {})) {
        if (field === "_reason") continue;
        it.fails(`known gap — announcement.${field} should be ${JSON.stringify(value)} (${exp.announcement?._todo?._reason ?? "no reason given"})`, async () => {
          const r = await run();
          expect(normalize(announcementFacts(r.announcement!, r.published)[field])).toEqual(normalize(value));
        });
      }
    });
  }
});