// author: simon-pierre boucher <contact@spboucher.ai>
import { httpFetch } from "@tendril/fetcher-http";
import { parseRobots, type Robots } from "./robots.js";

const ALLOW_ALL: Robots = { groups: [], sitemaps: [] };
const DISALLOW_ALL: Robots = { groups: [{ agents: ["*"], rules: [{ allow: false, pattern: "/" }] }], sitemaps: [] };

const OK_TTL_MS = 24 * 60 * 60 * 1000;
const DISALLOW_TTL_MS = 60 * 60 * 1000;

interface Entry {
  robots: Robots;
  expiresAt: number;
}

const cache = new Map<string, Entry>();

/**
 * Fetch and cache robots.txt for an origin (§10.1). Cached 24h. A 5xx is treated
 * as disallow-all for 1h (per the spec's intent); a 4xx or network failure is
 * treated as allow-all. `origin` must be a scheme://host[:port] string.
 */
export async function getRobots(origin: string): Promise<Robots> {
  const now = Date.now();
  const cached = cache.get(origin);
  if (cached !== undefined && cached.expiresAt > now) return cached.robots;

  const res = await httpFetch(`${origin}/robots.txt`, { timeout: 8_000 });
  let robots: Robots;
  let ttl = OK_TTL_MS;

  if (!res.ok) {
    robots = ALLOW_ALL;
  } else if (res.value.status >= 500) {
    robots = DISALLOW_ALL;
    ttl = DISALLOW_TTL_MS;
  } else if (res.value.status >= 400) {
    robots = ALLOW_ALL;
  } else {
    robots = parseRobots(res.value.body);
  }

  cache.set(origin, { robots, expiresAt: now + ttl });
  return robots;
}

export function clearRobotsCache(): void {
  cache.clear();
}