import fs from "node:fs";
import path from "node:path";
import {
  createLogger,
  nowIso,
  truncate,
  type AgentDecision,
  type AppConfig,
  type CrawlJob,
  type ObservedEntity,
  type PageState,
  type Platform,
} from "@src/shared";
import { EventBus, JsonlEventLog } from "@src/events";
import { SocialBrowserSession } from "@src/browser";
import { NetworkObserver, DomObserver, snapshotDom, classifyPage, type ClassifiedResponse } from "@src/observers";
import { mergeSurfaces } from "@src/entities";
import { detectVideos, sampleVideoFrames } from "@src/media";
import { WorldModel, LoopDetector, HeuristicPlanner, LlmPlanner, createLlmClient, actionKey, type Planner, type GainContext } from "@src/agent";
import { PlatformModel, compileConnector } from "@src/platform-model";
import { openStore, type PostgresStore } from "@src/storage";
import { getAdapter, entitiesFromDom, buildActions, pageFingerprint, type SocialPlatformAdapter } from "@src/connectors";
import { ActionExecutor } from "./executor.ts";

const log = createLogger("engine");

export interface CrawlSummary {
  session_id: string;
  platform: Platform;
  steps: number;
  entities: number;
  videos: number;
  network_responses: number;
  schemas: number;
  patterns_learned: number;
  ended_because: string;
  world: ReturnType<WorldModel["stats"]>;
  connector: ReturnType<PlatformModel["summary"]>;
  session_dir: string;
}

/**
 * The crawl loop (§2): observe → summarize → plan → act → learn, until the budget or the goal ends it.
 * Every subsystem is a separate package; this file only orchestrates.
 */
export class CrawlEngine {
  private readonly bus = new EventBus();
  private readonly world = new WorldModel();
  private readonly loops = new LoopDetector();
  private readonly adapter: SocialPlatformAdapter;
  private readonly model: PlatformModel;
  private session!: SocialBrowserSession;
  private net!: NetworkObserver;
  private dom!: DomObserver;
  private store?: PostgresStore;
  private eventLog!: JsonlEventLog;
  private planner!: Planner;
  private executor!: ActionExecutor;
  private sessionDir!: string;
  private step = 0;
  private stepsWithoutNew = 0;
  private recentFailures = new Map<string, number>(); // action key → step of last failure
  private counts = { entities: 0, videos: 0, profiles: 0, posts: 0, videos_seen: 0, profiles_seen: 0, posts_seen: 0, patterns: 0, schemas: 0 };
  private degraded = false;

  constructor(private readonly cfg: AppConfig, private readonly job: CrawlJob) {
    this.adapter = getAdapter(job.platform);
    this.model = new PlatformModel(job.platform, cfg.platformModelDir);
  }

  async run(): Promise<CrawlSummary> {
    const { job, cfg } = this;
    this.session = new SocialBrowserSession({ platform: job.platform, accountAlias: job.account_alias, profilesDir: cfg.profilesDir, headless: cfg.headless, channel: cfg.browserChannel, bus: this.bus });
    this.sessionDir = path.join(cfg.sessionsDir, this.session.id);
    this.eventLog = new JsonlEventLog(this.sessionDir);
    this.eventLog.attach(this.bus);
    this.store = await openStore(cfg.databaseUrl);
    this.store?.attach(this.bus);
    fs.writeFileSync(path.join(this.sessionDir, "job.json"), JSON.stringify({ ...job, session_id: this.session.id }, null, 2));
    await this.store?.linkJob(job, this.session.id).catch(() => {});

    if (!this.session.hasProfile()) log.warn(`No saved browser profile for ${job.platform}/${job.account_alias}. Run: pnpm login ${job.platform} --account ${job.account_alias}`);

    const llm = createLlmClient(cfg);
    this.planner = llm ? new LlmPlanner(llm, new HeuristicPlanner(), job.budget.max_llm_tokens) : new HeuristicPlanner();
    log.info("planner", { planner: this.planner.name, mode: job.mode, goal: job.goal });

    await this.session.start();
    // SESSION_STARTED is emitted by the session; enrich the DB row with mode/goal.
    this.bus.emit({ event_type: "PAGE_OPENED", platform: job.platform, session_id: this.session.id, step: 0, payload: { url: "about:blank", mode: job.mode, goal: job.goal, budget: job.budget } });
    const page = this.session.getPage();
    this.net = new NetworkObserver({ page, bus: this.bus, sessionId: this.session.id, platform: job.platform, hints: this.adapter.classifierHints });
    this.net.attach();
    this.dom = new DomObserver({ page, bus: this.bus, sessionId: this.session.id, platform: job.platform });
    await this.dom.attach();
    this.executor = new ActionExecutor(page, this.adapter, { stayOnPlatform: job.stay_on_platform });

    const started = Date.now();
    let endedBecause = "unknown";
    try {
      // Seed navigation
      this.net.beginStep(0, "seed");
      this.dom.beginStep(0);
      const seed = job.seed_url ?? (job.query && job.mode !== "observe" ? this.adapter.searchUrl(job.query) : this.adapter.homeUrl);
      await this.session.navigate(seed);
      await page.waitForLoadState("networkidle", { timeout: 6000 }).catch(() => {});
      await this.adapter.dismissOverlays?.(page).catch(() => {});
      let state = await this.observe(await this.net.collectStep(), "seed");
      if (state.classification.page_type === "LOGIN") {
        this.session.setHealth("auth_required");
        this.bus.emit({ event_type: "AUTH_REQUIRED", platform: job.platform, session_id: this.session.id, step: 0, payload: { url: state.url, hint: `pnpm login ${job.platform} --account ${job.account_alias}` } });
        endedBecause = "auth_required";
        return this.finish(endedBecause, started);
      }

      while (true) {
        const elapsedMin = (Date.now() - started) / 60_000;
        const b = job.budget;
        if (this.step >= b.max_actions) { endedBecause = "max_actions"; break; }
        if (elapsedMin >= b.max_minutes) { endedBecause = "max_minutes"; break; }
        if (this.counts.profiles >= b.max_profiles) { endedBecause = "max_profiles"; break; }
        if (this.counts.posts >= b.max_posts) { endedBecause = "max_posts"; break; }
        if (this.counts.videos >= b.max_videos) { endedBecause = "max_videos"; break; }
        if (this.session.getInfo().health === "crashed") { endedBecause = "browser_crashed"; break; }

        this.step++;
        const decision = await this.plan(state);
        this.bus.emit({ event_type: "ACTION_PLANNED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { action: decision.chosen_action, expected_information_gain: decision.expected_information_gain, novelty: decision.novelty, relevance: decision.relevance, reason: decision.reason, planner: decision.planner, top_scores: decision.scores.slice(0, 5) } });
        log.info(`step ${this.step}: ${decision.chosen_action.type} — ${truncate(decision.chosen_action.label, 80)}`, { gain: decision.expected_information_gain.toFixed(3), planner: decision.planner, reason: truncate(decision.reason, 100) });
        if (decision.chosen_action.type === "END_SESSION") { endedBecause = "agent_ended"; break; }

        const before = { url: state.url, page_type: state.classification.page_type, visible_entities: state.entities.length, world_nodes: this.world.nodes.size };
        this.net.beginStep(this.step, decision.chosen_action.id);
        this.dom.beginStep(this.step);
        const target = decision.chosen_action.target_ref ? state.entities.find((e) => e.ref === decision.chosen_action.target_ref) : undefined;
        const result = await this.executor.execute(decision.chosen_action, { entityUrl: target?.url });
        if (!result.ok) this.recentFailures.set(actionKey(decision.chosen_action), this.step);
        for (const [k, s] of this.recentFailures) if (this.step - s > 8) this.recentFailures.delete(k);
        this.session.recordAction(decision.chosen_action.type, result.navigated && /^OPEN_/.test(decision.chosen_action.type) ? 1 : decision.chosen_action.type === "BACK" ? -1 : 0);
        if (target) this.world.markVisited(target.fingerprint, target.url);
        if (result.navigated) this.bus.emit({ event_type: "NAVIGATION_COMPLETED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { from: result.url_before, to: result.url_after, action: decision.chosen_action.type, duration_ms: result.duration_ms } });

        const responses = await this.net.collectStep();
        const domDeltas = this.dom.collectStep();
        const prevState = state;
        // Observation is bounded too (§66): a page that never settles must not stall the loop.
        state = await Promise.race([
          this.observe(responses, decision.chosen_action.id, target?.fingerprint),
          new Promise<PageState>((_, rej) => setTimeout(() => rej(new Error("observe timeout")), 45_000)),
        ]).catch((err: Error) => {
          log.warn("observation failed", { err: err.message });
          return prevState;
        });
        const newEntities = this.world.observe(state.entities, this.step, target?.fingerprint);
        this.stepsWithoutNew = newEntities.length ? 0 : this.stepsWithoutNew + 1;
        this.tally(newEntities);
        if (result.ok && result.navigated) this.tallyVisit(decision.chosen_action.type);

        const after = { url: state.url, page_type: state.classification.page_type, visible_entities: state.entities.length, new_entities: newEntities.length, network_responses: responses.length, dom_nodes_added: domDeltas.reduce((s, d) => s + d.added, 0), world_nodes: this.world.nodes.size };
        this.bus.emit({ event_type: result.ok ? "ACTION_EXECUTED" : "ACTION_FAILED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { action: decision.chosen_action, ok: result.ok, error: result.error, before, after, duration_ms: result.duration_ms }, discovered_via: { action_id: decision.chosen_action.id } });
        await this.store?.recordAction(this.session.id, decision, before, after, result.ok, result.error, result.duration_ms).catch((e) => log.debug("recordAction failed", { err: (e as Error).message }));
        await this.persistStep(state, newEntities, responses);

        // Learning (§30, §71)
        const outcome = this.model.learnStep({ step: this.step, action_type: decision.chosen_action.type, action_id: decision.chosen_action.id, from_page_type: prevState.classification.page_type, to_page_type: state.classification.page_type, to_url: state.url, page_confidence: state.classification.confidence, entities_total: state.entities.length, new_entities: newEntities.length, entity_types: countTypes(state.entities), responses: responses.map((r) => ({ fingerprint: r.fingerprint, entity_count: r.entities.length, kind: r.kind, url: r.response.url, fields: r.schema?.fields.filter((f) => f.semantic[0]?.kind !== "unknown").slice(0, 60).map((f) => ({ path: f.path, semantic: f.semantic[0], example: f.examples[0] })) })), failed: !result.ok }, this.session.id);
        if (outcome.newPatterns.length) {
          this.counts.patterns += outcome.newPatterns.length;
          this.model.announce(this.bus, this.session.id, this.step, outcome.newPatterns);
        }
        this.model.save();

        // Degradation / self-healing signal (§32)
        const degradedNow = this.model.isDegraded(state.classification.page_type, state.entities.length, this.adapter.expectedEntities(state.classification.page_type));
        if (degradedNow && !this.degraded) {
          this.degraded = true;
          this.bus.emit({ event_type: "CONNECTOR_DEGRADED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { page_type: state.classification.page_type, expected: this.adapter.expectedEntities(state.classification.page_type), observed: 0, url: state.url, action: "switching to exploration heuristics" } });
        } else if (!degradedNow && this.degraded && state.entities.length > 0) {
          this.degraded = false;
          this.bus.emit({ event_type: "CONNECTOR_REPAIRED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { page_type: state.classification.page_type, observed: state.entities.length } });
        }

        // Loop safety (§41)
        this.loops.record(state.fingerprint, `${decision.chosen_action.type}:${decision.chosen_action.target_url ?? ""}`);
        const loop = this.loops.detect();
        if (loop.loop) {
          this.bus.emit({ event_type: "LOOP_DETECTED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { pattern: loop.pattern, url: state.url } });
          log.warn("navigation loop detected — breaking out", { pattern: loop.pattern });
          await this.executor.execute({ id: "loopbreak", type: "RETURN_TO_FEED", label: "loop break", cost: 1, target_url: job.query ? this.adapter.searchUrl(job.query) : this.adapter.homeUrl }, {});
          state = await this.observe(await this.net.collectStep(), "loopbreak");
        }
        if (state.classification.page_type === "LOGIN") {
          this.session.setHealth("auth_required");
          this.bus.emit({ event_type: "AUTH_REQUIRED", platform: job.platform, session_id: this.session.id, step: this.step, payload: { url: state.url } });
          endedBecause = "auth_required";
          break;
        }
        if (this.stepsWithoutNew >= 10) { endedBecause = "no_new_information"; break; }
      }
    } catch (err) {
      endedBecause = `error: ${(err as Error).message.split("\n")[0]}`;
      log.error("engine error", { err: (err as Error).stack?.split("\n").slice(0, 3).join(" | ") });
      this.bus.emit({ event_type: "WORKER_ERROR", platform: job.platform, session_id: this.session.id, step: this.step, payload: { error: (err as Error).message } });
    }
    return this.finish(endedBecause, started);
  }

  /** Observe the current page across surfaces and build the compact PageState (§9, §14). */
  private async observe(responses: ClassifiedResponse[], via: string, discoveredFrom?: string): Promise<PageState> {
    const page = this.session.getPage();
    const snapshot = await snapshotDom(page).catch((err: Error) => {
      log.warn("dom snapshot failed", { err: err.message.split("\n")[0] });
      return undefined;
    });
    if (!snapshot) {
      return { url: page.url(), title: "", platform: this.job.platform, classification: { page_type: "UNKNOWN", confidence: 0.1, signals: ["snapshot failed"] }, entities: [], actions: [{ id: "A1", type: "WAIT_FOR_CONTENT", label: "Wait for content", cost: 1 }, { id: "A2", type: "RETURN_TO_FEED", label: "Return to feed", cost: 2, target_url: this.adapter.homeUrl }, { id: "A3", type: "END_SESSION", label: "End", cost: 0.5 }], media: [], summary_text: "", fingerprint: pageFingerprint(page.url(), []), captured_at: nowIso() };
    }
    const classification = classifyPage(snapshot, this.adapter.pageTypeHints);
    const domEntities = entitiesFromDom(snapshot, this.adapter);
    const netEntities = responses.flatMap((r) => r.entities);
    const merged = mergeSurfaces(netEntities, domEntities);
    // Keep the planner's list focused: DOM-visible entities first, then network-only ones that have a page URL.
    const entities = merged.merged.filter((e) => e.provenance.some((p) => p.surface === "dom") || e.url).slice(0, 80);
    const media = detectVideos({ platform: this.job.platform, snapshot, entities, responses, pageUrl: snapshot.url });
    const visited = new Set([...this.world.nodes.values()].filter((n) => n.visited).map((n) => n.fingerprint));
    const actions = buildActions(entities, snapshot, this.adapter, { query: this.job.query, visited });
    const summary = renderSummary(snapshot.title, classification.page_type, entities, actions);
    const state: PageState = { url: snapshot.url, title: snapshot.title, platform: this.job.platform, classification, entities, actions, media, summary_text: summary, fingerprint: pageFingerprint(snapshot.url, entities), captured_at: nowIso() };

    this.bus.emit({ event_type: "PAGE_OPENED", platform: this.job.platform, session_id: this.session.id, step: this.step, payload: { url: snapshot.url, title: snapshot.title, page_type: classification.page_type, confidence: classification.confidence, signals: classification.signals, entities: entities.length, dom_entities: domEntities.length, network_entities: netEntities.length, both_surfaces: merged.both, field_agreements: merged.field_agreements, field_conflicts: merged.field_conflicts.slice(0, 5), videos: media.length, actions: actions.length, summary }, provenance: [{ surface: "dom", confidence: classification.confidence }], discovered_via: { action_id: via } });
    this.bus.emit({ event_type: "PAGE_CLASSIFIED", platform: this.job.platform, session_id: this.session.id, step: this.step, payload: { url: snapshot.url, ...classification } });
    for (const e of entities) {
      const type = e.type === "video" ? "VIDEO_DISCOVERED" : e.type === "post" ? "POST_DISCOVERED" : e.type === "comment" ? "COMMENT_DISCOVERED" : e.type === "profile" || e.type === "channel" || e.type === "person" ? "PROFILE_DISCOVERED" : "ENTITY_OBSERVED";
      const known = this.world.nodes.has(e.fingerprint);
      this.bus.emit({ event_type: known ? "ENTITY_OBSERVED" : type, platform: this.job.platform, session_id: this.session.id, step: this.step, payload: { entity: { type: e.type, platform_id: e.platform_id, url: e.url, name: e.name, text: e.text, author: e.author, metrics: e.metrics, media: e.media, context: e.context, fingerprint: e.fingerprint, canonical_id: null }, fields: e.fields }, provenance: e.provenance, discovered_via: { action_id: via, url: snapshot.url } });
    }
    for (const m of media) if (m.width || m.duration_s || m.delivery) this.bus.emit({ event_type: "MEDIA_DISCOVERED", platform: this.job.platform, session_id: this.session.id, step: this.step, payload: { ...m } as Record<string, unknown>, provenance: m.provenance, discovered_via: { action_id: via } });
    if (this.step === 0) this.world.observe(entities, 0, discoveredFrom);
    return state;
  }

  private async plan(state: PageState): Promise<AgentDecision> {
    // Relevance is measured against the goal *and* the topic query (the query carries the domain words).
    const goal = this.job.query ? `${this.job.goal} ${this.job.query}` : this.job.goal;
    const ctx: GainContext = { goal, mode: this.job.mode, world: this.world, state, recentActionTypes: this.loops.recentActionTypes(), recentUrls: [...this.world.visitedUrls].slice(-10), stepsWithoutNewEntities: this.stepsWithoutNew, learnedYield: this.model.learnedYield(), recentFailures: new Set(this.recentFailures.keys()) };
    return this.planner.plan(ctx, this.step);
  }

  /** Discovery counters (what we have seen). */
  private tally(fresh: ObservedEntity[]): void {
    for (const e of fresh) {
      this.counts.entities++;
      if (e.type === "video") this.counts.videos_seen++;
      else if (e.type === "post" || e.type === "comment") this.counts.posts_seen++;
      else if (e.type === "profile" || e.type === "channel" || e.type === "person" || e.type === "page") this.counts.profiles_seen++;
    }
  }

  /** Budget counters (§40) count pages we actually opened, not entities merely seen in a feed. */
  private tallyVisit(actionType: string): void {
    if (/^OPEN_(CHANNEL|PROFILE|PAGE)$/.test(actionType)) this.counts.profiles++;
    else if (/^OPEN_(POST|COMMENTS|ENTITY)$/.test(actionType)) this.counts.posts++;
    else if (actionType === "OPEN_VIDEO") this.counts.videos++;
  }

  private async persistStep(state: PageState, fresh: ObservedEntity[], responses: ClassifiedResponse[]): Promise<void> {
    const ts = nowIso();
    const frames: Record<string, string[]> = {};
    if (this.job.media_level >= 2) {
      const current = state.media.find((m) => m.width || m.delivery?.hostnames.length);
      if (current) {
        const files = await sampleVideoFrames(this.session.getPage(), current, this.cfg.mediaDir, this.job.media_level).catch(() => []);
        if (files.length) {
          frames[current.fingerprint] = files;
          this.bus.emit({ event_type: "VIDEO_DISCOVERED", platform: this.job.platform, session_id: this.session.id, step: this.step, payload: { fingerprint: current.fingerprint, title: current.title, frames: files, level: this.job.media_level }, provenance: [{ surface: "visual", confidence: 0.8 }] });
        }
      }
    }
    if (!this.store) return;
    try {
      await this.store.upsertEntities(state.entities, this.session.id, this.step, ts);
      await this.store.upsertMedia(state.media, frames, ts);
      await this.store.recordFeedItems(this.session.id, this.step, state, ts);
      const recentEdges = this.world.edges.filter((e) => e.step === this.step);
      await this.store.recordRelationships(this.session.id, recentEdges);
    } catch (err) {
      log.debug("persist failed", { err: (err as Error).message, fresh: fresh.length, responses: responses.length });
    }
  }

  private async finish(endedBecause: string, started: number): Promise<CrawlSummary> {
    this.model.save();
    const connector = this.model.summary();
    if (connector.confidence >= 40) {
      const manifest = compileConnector(this.model, path.join(process.cwd(), "connectors", this.job.platform));
      await this.store?.recordConnectorVersion(this.job.platform, connector, manifest).catch(() => {});
    }
    const world = this.world.stats();
    fs.writeFileSync(path.join(this.sessionDir, "world_model.json"), JSON.stringify(this.world.toJSON(), null, 2));
    const summary: CrawlSummary = { session_id: this.session.id, platform: this.job.platform, steps: this.step, entities: this.world.nodes.size, videos: this.counts.videos_seen, network_responses: this.net?.totalResponses ?? 0, schemas: this.net?.seenShapeHashes.size ?? 0, patterns_learned: this.counts.patterns, ended_because: endedBecause, world, connector, session_dir: this.sessionDir };
    this.bus.emit({ event_type: "BUDGET_EXHAUSTED", platform: this.job.platform, session_id: this.session.id, step: this.step, payload: { reason: endedBecause, elapsed_ms: Date.now() - started } });
    await this.session.stop();
    await this.bus.flush();
    fs.writeFileSync(path.join(this.sessionDir, "summary.json"), JSON.stringify(summary, null, 2));
    await this.eventLog.close();
    await this.store?.close();
    return summary;
  }
}

function countTypes(entities: ObservedEntity[]): Record<string, number> {
  const out: Record<string, number> = {};
  for (const e of entities) out[e.type] = (out[e.type] ?? 0) + 1;
  return out;
}

/** Textual semantic page representation (§14) — what a planner (or a human in the dashboard) reads. */
export function renderSummary(title: string, pageType: string, entities: ObservedEntity[], actions: PageState["actions"]): string {
  const lines = [`PAGE: ${pageType} — ${truncate(title, 90)}`, "", "VISIBLE ENTITIES"];
  for (const e of entities.slice(0, 25)) {
    lines.push(`[${e.ref}] ${e.type}${e.context ? ` (${e.context})` : ""}`);
    if (e.name) lines.push(`    Name: ${truncate(e.name, 100)}`);
    if (e.author) lines.push(`    Author: ${truncate(e.author, 60)}`);
    if (e.media?.has_video) lines.push(`    Video: yes${e.media.duration_s ? ` (${Math.round(e.media.duration_s)}s)` : ""}`);
    if (e.metrics && Object.keys(e.metrics).length) lines.push(`    Metrics: ${Object.entries(e.metrics).map(([k, v]) => `${k}=${v}`).join(", ")}`);
    lines.push(`    Sources: ${[...new Set(e.provenance.map((p) => p.surface))].join("+")}`);
  }
  if (entities.length > 25) lines.push(`… ${entities.length - 25} more`);
  lines.push("", "ACTIONS");
  for (const a of actions) lines.push(`[${a.id}] ${a.label}`);
  return lines.join("\n");
}