import { existsSync, readFileSync, readdirSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { z } from 'zod';
import { EngineSchema } from '@rareindex/shared';
import { domainOf, RefreshClassSchema } from './types.js';

/**
 * Central per-domain acquisition policy (SPEC §16). One file — connectors/domains.json — holds
 * every rate limit, concurrency cap, timeout, retry count, engine policy, locale, currency and
 * crawl-depth setting. Connector code must not hard-code these; it asks `policyFor(url)`.
 *
 * Resolution order: exact host → parent domain (sub.example.com → example.com) → `*` defaults.
 */
export const DomainPolicySchema = z.object({
  /** minimum spacing between two requests to this host */
  minIntervalMs: z.number().int().nonnegative().default(1000),
  /** maximum simultaneous in-flight requests to this host (all connectors together) */
  concurrency: z.number().int().positive().default(2),
  timeoutMs: z.number().int().positive().default(45_000),
  maxRetries: z.number().int().nonnegative().default(2),
  /** allowed engines in order; connectors may narrow but not widen */
  engines: z.array(EngineSchema).optional(),
  /** Firecrawl policy */
  firecrawl: z.object({ waitForMs: z.number().int().nonnegative().optional(), onlyMainContent: z.boolean().optional(), maxCreditsPerRun: z.number().int().positive().optional() }).default({}),
  /** Scrapfly policy */
  scrapfly: z.object({ renderJs: z.boolean().optional(), asp: z.boolean().optional(), country: z.string().optional(), proxyPool: z.enum(['datacenter', 'residential']).optional(), maxCreditsPerRun: z.number().int().positive().optional() }).default({}),
  /** ISO country used for geo routing (Firecrawl location / Scrapfly country) */
  country: z.string().optional(),
  locale: z.string().optional(),
  currency: z.string().optional(),
  /** default page depth for incremental crawls */
  crawlDepth: z.number().int().positive().default(3),
  /** hard cap of pages a single backfill run may fetch */
  backfillMaxPages: z.number().int().positive().default(200),
  /** honest bot UA (default) or a browser-like UA (only where the site serves bots a degraded page and terms allow) */
  userAgent: z.enum(['bot', 'browser']).default('bot'),
  refreshClass: RefreshClassSchema.optional(),
  /** consecutive failures before the circuit opens */
  circuitFailures: z.number().int().positive().default(6),
  /** how long the circuit stays open */
  circuitCooldownMs: z.number().int().positive().default(10 * 60_000),
  /** robots.txt / terms observations kept next to the policy for auditability */
  notes: z.string().optional(),
});
export type DomainPolicy = z.infer<typeof DomainPolicySchema>;

/** Patch form (no defaults applied — Zod's .partial() would still fill defaults, breaking layered merges). */
export const DomainPolicyPatchSchema = z.object({
  minIntervalMs: z.number().int().nonnegative().optional(),
  concurrency: z.number().int().positive().optional(),
  timeoutMs: z.number().int().positive().optional(),
  maxRetries: z.number().int().nonnegative().optional(),
  engines: z.array(EngineSchema).optional(),
  firecrawl: z.object({ waitForMs: z.number().int().nonnegative().optional(), onlyMainContent: z.boolean().optional(), maxCreditsPerRun: z.number().int().positive().optional() }).optional(),
  scrapfly: z.object({ renderJs: z.boolean().optional(), asp: z.boolean().optional(), country: z.string().optional(), proxyPool: z.enum(['datacenter', 'residential']).optional(), maxCreditsPerRun: z.number().int().positive().optional() }).optional(),
  country: z.string().optional(),
  locale: z.string().optional(),
  currency: z.string().optional(),
  crawlDepth: z.number().int().positive().optional(),
  backfillMaxPages: z.number().int().positive().optional(),
  userAgent: z.enum(['bot', 'browser']).optional(),
  refreshClass: RefreshClassSchema.optional(),
  circuitFailures: z.number().int().positive().optional(),
  circuitCooldownMs: z.number().int().positive().optional(),
  notes: z.string().optional(),
});
export type DomainPolicyPatch = z.infer<typeof DomainPolicyPatchSchema>;

export const DomainsFileSchema = z.object({
  version: z.string().default('1.0'),
  defaults: DomainPolicyPatchSchema.default({}),
  domains: z.record(z.string(), DomainPolicyPatchSchema).default({}),
});
export type DomainsFile = z.infer<typeof DomainsFileSchema>;

const here = path.dirname(fileURLToPath(import.meta.url));
export const DOMAINS_PATH = path.resolve(here, '../../../connectors/domains.json');
/** Fragment directory: connectors/domains.d/<group>.json = { domains: {...} } merged over domains.json (lets parallel contributors add hosts without conflicts). */
export const DOMAINS_DIR = path.resolve(here, '../../../connectors/domains.d');

let cached: DomainsFile | null = null;
const policyCache = new Map<string, DomainPolicy>();

export function loadDomains(force = false): DomainsFile {
  if (cached && !force) return cached;
  if (!existsSync(DOMAINS_PATH)) {
    cached = DomainsFileSchema.parse({});
    return cached;
  }
  const base = DomainsFileSchema.parse(JSON.parse(readFileSync(DOMAINS_PATH, 'utf8')));
  if (existsSync(DOMAINS_DIR)) {
    for (const f of readdirSync(DOMAINS_DIR).filter((x) => x.endsWith('.json')).sort()) {
      // A malformed fragment must never take the whole pipeline down: skip it loudly (pnpm registry validates strictly).
      try {
        const frag = DomainsFileSchema.parse({ ...JSON.parse(readFileSync(path.join(DOMAINS_DIR, f), 'utf8')), version: base.version });
        for (const [host, patch] of Object.entries(frag.domains)) base.domains[host] = { ...(base.domains[host] ?? {}), ...patch };
      } catch (err) {
        console.warn(`[domains] ignoring invalid fragment ${f}: ${err instanceof Error ? err.message.slice(0, 300) : String(err)}`);
      }
    }
  }
  cached = base;
  return cached;
}

/** Testing/embedding hook: replace the loaded file. */
export function setDomains(file: DomainsFile): void {
  cached = DomainsFileSchema.parse(file);
  policyCache.clear();
}

/** Resolve the effective policy for a URL or host. */
export function policyFor(urlOrHost: string): DomainPolicy {
  const host = urlOrHost.includes('://') ? domainOf(urlOrHost) : urlOrHost.replace(/^www\./, '').toLowerCase();
  const hit = policyCache.get(host);
  if (hit) return hit;
  const file = loadDomains();
  const layers: DomainPolicyPatch[] = [file.defaults];
  const parts = host.split('.');
  // parent domains first, exact host last (exact wins)
  for (let i = parts.length - 2; i >= 0; i--) {
    const candidate = parts.slice(i).join('.');
    const layer = file.domains[candidate];
    if (layer) layers.push(layer);
  }
  const merged = layers.reduce<Record<string, unknown>>((acc, l) => ({ ...acc, ...stripUndefined(l), firecrawl: { ...((acc.firecrawl as object) ?? {}), ...(l.firecrawl ?? {}) }, scrapfly: { ...((acc.scrapfly as object) ?? {}), ...(l.scrapfly ?? {}) } }), {});
  const policy = DomainPolicySchema.parse(merged);
  policyCache.set(host, policy);
  return policy;
}

export function clearPolicyCache(): void {
  policyCache.clear();
}

function stripUndefined<T extends object>(o: T): Partial<T> {
  return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined)) as Partial<T>;
}