// author: simon-pierre boucher <contact@spboucher.ai>
import { performance } from "node:perf_hooks";
import { err, ok, tendrilError, type FetchTimings, type Result, type Tier } from "@tendril/shared";
import { shouldEscalate } from "@tendril/router";
import { getRobots, isAllowed } from "@tendril/frontier";
import { httpFetch } from "@tendril/fetcher-http";
import { extract, PIPELINE_VERSION, type ExtractResult } from "@tendril/extract";
import type { ScrapeRequest } from "./schemas.js";

export interface ScrapeData {
  markdown?: string;
  html?: string;
  rawHtml?: string;
  links?: ExtractResult["links"];
  structured?: ExtractResult["structured"];
  metadata: ExtractResult["metadata"] & { statusCode: number; sourceURL: string; pipelineVersion: string };
  tierUsed: Tier;
  cached: boolean;
  timings: FetchTimings;
}

function nonHtmlMarkdown(contentType: string, body: string): Result<string> {
  const ct = contentType.toLowerCase();
  if (ct.includes("application/json")) {
    try {
      return ok("```json\n" + JSON.stringify(JSON.parse(body), null, 2) + "\n```");
    } catch {
      return ok(body);
    }
  }
  if (ct.includes("text/plain") || ct.includes("text/markdown") || ct.includes("text/csv") || ct.includes("xml")) {
    return ok(body);
  }
  return err(tendrilError("ERR_UNSUPPORTED_TYPE", { details: { contentType } }));
}

/**
 * Orchestrate a single scrape (§14.1): Tier 0 fetch → escalation decision →
 * deterministic extraction. Tiers 1/2 are not yet built, so a page that demands
 * escalation under `tier: "auto"` fails closed with ERR_TARGET_BLOCKED and the
 * decisive reason, rather than returning a challenge page as if it were content.
 */
export async function runScrape(req: ScrapeRequest): Promise<Result<ScrapeData>> {
  const started = performance.now();
  const timings: FetchTimings = { total: 0, escalations: [] };

  if (req.respectRobots) {
    let target: URL;
    try {
      target = new URL(req.url);
    } catch {
      return err(tendrilError("ERR_INVALID_URL", { details: { url: req.url } }));
    }
    const robots = await getRobots(target.origin);
    if (!isAllowed(robots, target.pathname + target.search)) {
      return err(tendrilError("ERR_ROBOTS_DENIED", { details: { url: req.url } }));
    }
  }

  const fetched = await httpFetch(req.url, {
    timeout: req.timeout,
    ...(req.headers !== undefined ? { headers: req.headers } : {}),
    ...(req.maxBytes !== undefined ? { maxBytes: req.maxBytes } : {}),
  });
  if (!fetched.ok) return fetched;
  const page = fetched.value;

  const decision = shouldEscalate({ status: page.status, contentType: page.contentType, body: page.body });

  if (decision.action === "non-html") {
    const md = nonHtmlMarkdown(page.contentType, page.body);
    if (!md.ok) return md;
    timings.total = performance.now() - started;
    return ok({
      ...(req.formats.includes("markdown") ? { markdown: md.value } : {}),
      ...(req.formats.includes("rawHtml") ? { rawHtml: page.body } : {}),
      metadata: {
        statusCode: page.status,
        sourceURL: page.finalUrl,
        pipelineVersion: PIPELINE_VERSION,
      },
      tierUsed: "http",
      cached: false,
      timings,
    });
  }

  if (decision.action === "escalate" && req.tier === "auto") {
    timings.escalations.push({ from: "http", to: "webkit", reason: decision.reason });
    return err(
      tendrilError("ERR_TARGET_BLOCKED", {
        message: "Tier 0 insufficient and higher tiers are not yet available",
        details: { reason: decision.reason, status: page.status },
      }),
    );
  }

  const extractStart = performance.now();
  const extracted = extract(page.body, {
    url: page.finalUrl,
    onlyMainContent: req.onlyMainContent,
    ...(req.includeTags !== undefined ? { includeTags: req.includeTags } : {}),
    ...(req.excludeTags !== undefined ? { excludeTags: req.excludeTags } : {}),
  });
  if (!extracted.ok) return extracted;
  const e = extracted.value;
  timings.extract = performance.now() - extractStart;
  timings.total = performance.now() - started;

  return ok({
    ...(req.formats.includes("markdown") ? { markdown: e.markdown } : {}),
    ...(req.formats.includes("html") ? { html: e.html } : {}),
    ...(req.formats.includes("rawHtml") ? { rawHtml: page.body } : {}),
    ...(req.formats.includes("links") ? { links: e.links } : {}),
    ...(req.formats.includes("structured") ? { structured: e.structured } : {}),
    metadata: {
      ...e.metadata,
      statusCode: page.status,
      sourceURL: page.finalUrl,
      pipelineVersion: PIPELINE_VERSION,
    },
    tierUsed: "http",
    cached: false,
    timings,
  });
}