import { canonicalizeHtml, canonicalizeText, canonicalJson, sha256, simhash, type ConnectorMetadata, type NormalizedContent, type Observation, type SensorEndpoint } from "@websensor/core";
import { httpFetchWithRetry } from "./fetcher";
import { NormalizeError, type WebSensorConnector } from "./types";

/**
 * Generic HTTP connector: HTML pages (canonical text), plain text and JSON documents.
 * Config: { keepChrome?: boolean, selector?: string, accept?: string, headers?: {} }
 */
export class HttpConnector implements WebSensorConnector {
  metadata(): ConnectorMetadata {
    return { key: "http", name: "Generic HTTP", sensorTypes: ["HTML", "JSON", "XML", "HTTP_HEADERS", "FILE"], description: "Conditional GET + canonical extraction for HTML/JSON/text", version: "1.0.0" };
  }

  async fetch(endpoint: SensorEndpoint): Promise<Observation> {
    const cfg = endpoint.config as { accept?: string; headers?: Record<string, string>; method?: "GET" | "HEAD"; timeoutMs?: number };
    return httpFetchWithRetry(endpoint.id, endpoint.url, {
      method: cfg.method ?? "GET",
      etag: endpoint.etag,
      lastModified: endpoint.lastModified,
      accept: cfg.accept ?? (endpoint.type === "JSON" ? "application/json, */*;q=0.5" : undefined),
      headers: cfg.headers,
      timeoutMs: cfg.timeoutMs,
    });
  }

  async normalize(endpoint: SensorEndpoint, obs: Observation): Promise<NormalizedContent> {
    if (!obs.body) throw new NormalizeError("no_body", "Observation has no body");
    const ct = (obs.meta.contentType ?? "").toLowerCase();
    const text = obs.body.toString("utf8");
    const cfg = endpoint.config as { keepChrome?: boolean; jsonPath?: string; ignoreKeys?: string[] };

    if (endpoint.type === "HTTP_HEADERS" || obs.meta.method === "HEAD") {
      const h = { ...obs.meta.headers };
      delete h.date;
      delete h.age;
      delete h["cf-ray"];
      const j = canonicalJson(h);
      return { mode: "json", json: h, rawHash: sha256(j), canonicalHash: sha256(j), semanticHash: simhash(j), extractionConfidence: 1 };
    }

    if (endpoint.type === "JSON" || ct.includes("json") || /^\s*[[{]/.test(text.slice(0, 50))) {
      let parsed: unknown;
      try {
        parsed = JSON.parse(text);
      } catch {
        if (endpoint.type === "JSON") throw new NormalizeError("bad_json", "Response is not valid JSON");
        parsed = undefined;
      }
      if (parsed !== undefined) {
        let node: unknown = parsed;
        if (cfg.jsonPath) node = getPath(node, cfg.jsonPath);
        if (cfg.ignoreKeys?.length) node = stripKeys(node, new Set(cfg.ignoreKeys));
        const j = canonicalJson(node);
        return { mode: "json", json: node as object | null, rawHash: sha256(text), canonicalHash: sha256(j), semanticHash: simhash(j), extractionConfidence: 1 };
      }
    }

    if (ct.includes("html") || /<\s*(!doctype|html|body|div|p)\b/i.test(text.slice(0, 4000))) {
      const c = canonicalizeHtml(text, obs.meta.finalUrl, { keepChrome: cfg.keepChrome });
      if (c.text.length < 40 && c.headings.length === 0) {
        // Probably a JS shell, a challenge page, or an interstitial: low extraction confidence.
        return { mode: "text", text: c.text, title: c.title, headings: c.headings, links: c.links, rawHash: c.rawHash, canonicalHash: c.canonicalHash, semanticHash: c.semanticHash, extractionConfidence: 0.3, publishedAt: parseMetaDate(c.meta), extra: { structure: c.structure, meta: c.meta, thin: true } };
      }
      return { mode: "text", text: c.text, title: c.title, headings: c.headings, links: c.links, rawHash: c.rawHash, canonicalHash: c.canonicalHash, semanticHash: c.semanticHash, extractionConfidence: 0.75, publishedAt: parseMetaDate(c.meta), extra: { structure: c.structure, meta: c.meta } };
    }

    const t = canonicalizeText(text);
    return { mode: "text", text: t.text, rawHash: t.rawHash, canonicalHash: t.canonicalHash, semanticHash: t.semanticHash, extractionConfidence: 0.9 };
  }
}

function parseMetaDate(meta: Record<string, string>): Date | null {
  const s = meta["article:modified_time"] ?? meta["article:published_time"] ?? meta["last-modified"];
  if (!s) return null;
  const d = new Date(s);
  return Number.isNaN(d.getTime()) ? null : d;
}

export function getPath(obj: unknown, path: string): unknown {
  let cur = obj;
  for (const part of path.split(".").filter(Boolean)) {
    if (cur === null || cur === undefined) return undefined;
    const m = part.match(/^(\w+)?\[(\d+)\]$/);
    if (m) {
      if (m[1]) cur = (cur as Record<string, unknown>)[m[1]];
      cur = Array.isArray(cur) ? cur[Number(m[2])] : undefined;
    } else cur = (cur as Record<string, unknown>)[part];
  }
  return cur;
}

function stripKeys(v: unknown, keys: Set<string>): unknown {
  if (Array.isArray(v)) return v.map((x) => stripKeys(x, keys));
  if (v && typeof v === "object") {
    const out: Record<string, unknown> = {};
    for (const [k, val] of Object.entries(v as Record<string, unknown>)) if (!keys.has(k)) out[k] = stripKeys(val, keys);
    return out;
  }
  return v;
}