import { sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";
import { httpFetchWithRetry } from "./fetcher";
import { getPath } from "./http";
import { NormalizeError, type WebSensorConnector } from "./types";

/**
 * Generic JSON API list connector — covers official public APIs that return a list of
 * records: CISA KEV, Federal Register, ClinicalTrials.gov v2, USGS GeoJSON, Have I Been
 * Pwned breaches, NVD CVE 2.0, JSON changelogs…
 *
 * Config:
 *   itemsPath: "vulnerabilities" | "results" | "studies" | "features" | ""   (dot path to the array)
 *   keyField:  "cveID" | "document_number" | "protocolSection.identificationModule.nctId" | "id"
 *   titleField, urlField, summaryField, dateField: dot paths inside each item
 *   compareFields: string[]  (fields whose change counts as an update)
 *   urlTemplate: "https://nvd.nist.gov/vuln/detail/{key}"
 *   headers: {}  (e.g. User-Agent contact required by SEC)
 *   maxItems: 200
 */
export class JsonListConnector implements WebSensorConnector {
  mode = "list" as const;
  metadata(): ConnectorMetadata {
    return { key: "jsonlist", name: "JSON API list", sensorTypes: ["REST_API", "JSON"], description: "Keyed records from an official JSON API (KEV, Federal Register, ClinicalTrials, USGS, HIBP, NVD…)", version: "1.0.0" };
  }
  async fetch(endpoint: SensorEndpoint): Promise<Observation> {
    const cfg = endpoint.config as { headers?: Record<string, string>; url?: string; timeoutMs?: number; noConditional?: boolean };
    // Support {date} placeholders for APIs that require a window (NVD).
    const url = expandUrl(cfg.url ?? endpoint.url);
    const obs = await httpFetchWithRetry(endpoint.id, url, { etag: cfg.noConditional ? null : endpoint.etag, lastModified: cfg.noConditional ? null : endpoint.lastModified, headers: cfg.headers, accept: "application/json, */*;q=0.5", timeoutMs: cfg.timeoutMs ?? 40_000, maxBytes: 40 * 1024 * 1024 });
    return { ...obs, url: endpoint.url };
  }
  async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {
    if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");
    const text = obs.body.toString("utf8");
    let root: unknown;
    try {
      root = JSON.parse(text);
    } catch {
      throw new NormalizeError("bad_json", "API response is not JSON");
    }
    const cfg = endpoint.config as { itemsPath?: string; keyField?: string; titleField?: string; urlField?: string; summaryField?: string; dateField?: string; compareFields?: string[]; urlTemplate?: string; maxItems?: number; titleTemplate?: string };
    const arr = cfg.itemsPath ? getPath(root, cfg.itemsPath) : root;
    if (!Array.isArray(arr)) throw new NormalizeError("bad_shape", `itemsPath "${cfg.itemsPath ?? ""}" is not an array`);
    const items = (arr as Record<string, unknown>[]).slice(0, cfg.maxItems ?? 300).map((it) => {
      const key = str(getPath(it, cfg.keyField ?? "id"));
      const title = cfg.titleTemplate ? fill(cfg.titleTemplate, it) : str(getPath(it, cfg.titleField ?? "title"));
      // `{key}` keeps path separators (Hugging Face `owner/model` ids); `{field.path}` fills from the record.
      const url = cfg.urlTemplate ? fill(cfg.urlTemplate.replace("{key}", key.split("/").map(encodeURIComponent).join("/")), it) : str(getPath(it, cfg.urlField ?? "url"));
      const summary = str(getPath(it, cfg.summaryField ?? "summary")).slice(0, 1200);
      const dateRaw = str(getPath(it, cfg.dateField ?? "date"));
      const d = dateRaw ? new Date(dateRaw) : null;
      const out: Record<string, unknown> & { key: string } = { key, title, url, summary, publishedAt: d && !Number.isNaN(d.getTime()) ? d.toISOString() : null };
      for (const f of cfg.compareFields ?? []) out[f] = getPath(it, f);
      return out;
    });
    const compareFields = cfg.compareFields ?? ["title"];
    const canonical = items.map((i) => `${i.key}\t${compareFields.map((f) => JSON.stringify(i[f] ?? "")).join("\t")}`).join("\n");
    const newest = items.map((i) => i.publishedAt as string | null).filter((x): x is string => Boolean(x)).sort().at(-1);
    return { mode: "list", items, compareFields, rawHash: sha256(text), canonicalHash: sha256(canonical), semanticHash: simhash(items.map((i) => String(i.title)).join("\n")), publishedAt: newest ? new Date(newest) : null, extractionConfidence: 1, extra: { count: items.length } };
  }
}

function str(v: unknown): string {
  if (v === null || v === undefined) return "";
  if (typeof v === "string") return v;
  if (typeof v === "number" || typeof v === "boolean") return String(v);
  if (Array.isArray(v)) return v.map(str).filter(Boolean).join(", ");
  if (typeof v === "object") {
    const o = v as Record<string, unknown>;
    if (typeof o.value === "string") return o.value;
    if (Array.isArray(o.descriptions)) return str((o.descriptions as Record<string, unknown>[]).find((d) => d.lang === "en")?.value ?? o.descriptions[0]);
  }
  return JSON.stringify(v).slice(0, 400);
}

function fill(tpl: string, it: Record<string, unknown>): string {
  return tpl.replace(/\{([^}]+)\}/g, (_, p: string) => str(getPath(it, p)));
}

/** `{now-2h}` / `{now}` placeholders → ISO-8601 (no millis, NVD-compatible). */
export function expandUrl(url: string): string {
  return url.replace(/\{now(?:-(\d+)([hmd]))?\}/g, (_, n: string | undefined, u: string | undefined) => {
    const d = new Date();
    if (n && u) {
      const ms = Number(n) * (u === "h" ? 3600e3 : u === "m" ? 60e3 : 86400e3);
      d.setTime(d.getTime() - ms);
    }
    return encodeURIComponent(d.toISOString().replace(/\.\d{3}Z$/, ".000"));
  });
}