// Pipeline d'ingestion : scan des dossiers de cours → classification → parsing structurel →
// fragments + métadonnées → FTS5 + embeddings. Incrémental par somme de contrôle SHA-256.

import { createHash } from "node:crypto";
import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";
import { basename, join, relative, resolve } from "node:path";
import { all, db, get, run, transaction } from "../db/index.ts";
import { embedPassages, vecToBlob } from "./embeddings.ts";
import { parseArticleSections, parseBeamerFrames, splitLong } from "./latex.ts";

export type IngestResult = {
  runId: number;
  scanned: number;
  ingested: number;
  skipped: number;
  chunks: number;
  errors: { path: string; error: string }[];
  report: string;
};

type FileClass = {
  docType: string;
  title: string;
  week: number | null;
  category: string;
  space: string;
  visible: boolean;
};

const EXCLUDED_DIRS = new Set([".git", ".claude", "node_modules", "__pycache__", "plateforme_ateliers", "plateforme_data", "images", "static", "assets"]);
const EXCLUDED_EXT = new Set([".aux", ".log", ".out", ".toc", ".nav", ".snm", ".bbl", ".bcf", ".blg", ".fls", ".xml", ".sty", ".bib", ".png", ".jpg", ".jpeg", ".webp", ".db", ".py", ".js", ".html", ".json", ".gitignore", ".fdb_latexmk", ".xlsx", ".csv"]);

function classify(path: string, courseCode: string): FileClass | null {
  const name = basename(path).toLowerCase();
  const officialSpace = `official-${courseCode.toLowerCase()}`;

  if (name.startsWith("examen") || name.includes("_exam") || name.includes("solutionnaire") || name.includes("blueprint") || name.includes("grille_correction") || name.includes("-analysis")) {
    return { docType: "exam", title: prettyTitle(name), week: null, category: "examen", space: "instructor-private", visible: false };
  }
  const seance = name.match(/seance(\d+)/);
  if (seance && name.endsWith(".tex")) {
    return { docType: "slides", title: prettyTitle(name), week: parseInt(seance[1], 10), category: "seance", space: officialSpace, visible: true };
  }
  if (name === "plan_de_cours.tex") {
    return { docType: "plan", title: "Plan de cours", week: null, category: "plan", space: officialSpace, visible: true };
  }
  const atelier = name.match(/atelier(\d)/);
  if (name.endsWith(".tex") && atelier) {
    const isSolution = name.startsWith("solution");
    return {
      docType: isSolution ? "solution" : "exercise",
      title: `${isSolution ? "Solution" : "Énoncé"} — Atelier ${atelier[1]}`,
      week: null,
      category: `atelier${atelier[1]}`,
      space: officialSpace,
      visible: true,
    };
  }
  if (name === "aide_memoire.tex") {
    return { docType: "aide-memoire", title: "Aide-mémoire", week: null, category: "reference", space: officialSpace, visible: true };
  }
  if (name === "glossaire.tex") {
    return { docType: "glossary", title: "Glossaire bilingue", week: null, category: "reference", space: officialSpace, visible: true };
  }
  if (name === "description_moodle.md" || name === "readme.md") {
    return { docType: "markdown", title: prettyTitle(name), week: null, category: "info", space: officialSpace, visible: true };
  }
  return null;
}

function prettyTitle(name: string): string {
  const seance = name.match(/seance(\d+)_?([a-z_]*)/);
  if (seance) {
    const suffix = (seance[2] || "").replace(/_/g, " ").trim();
    return `Séance ${parseInt(seance[1], 10)}${suffix ? " — " + capitalize(suffix) : ""}`;
  }
  return capitalize(name.replace(/\.(tex|md|pdf)$/, "").replace(/[_-]/g, " "));
}
function capitalize(s: string): string {
  return s ? s[0].toUpperCase() + s.slice(1) : s;
}

function* walk(dir: string): Generator<string> {
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    if (entry.isDirectory()) {
      // Dossiers d'archives (_archive*, archive*) : matériel retiré du cours — jamais indexé
      if (entry.name.startsWith("_") || /^archives?/i.test(entry.name)) continue;
      if (!EXCLUDED_DIRS.has(entry.name)) yield* walk(join(dir, entry.name));
    } else {
      const ext = entry.name.slice(entry.name.lastIndexOf("."));
      if (!EXCLUDED_EXT.has(ext.toLowerCase()) && !entry.name.startsWith(".")) yield join(dir, entry.name);
    }
  }
}

type PreparedChunk = {
  refType: string;
  refNumber: number | null;
  refLabel: string;
  sectionTitle: string;
  title: string;
  content: string;
  boxTypes: string;
  week: number | null;
};

function chunksForFile(path: string, cls: FileClass, courseCode: string): PreparedChunk[] {
  const out: PreparedChunk[] = [];
  const pushSections = (raw: string, refType: string) => {
    for (const s of parseArticleSections(raw)) {
      const parts = splitLong(s.content);
      parts.forEach((part, i) => {
        out.push({
          refType,
          refNumber: null,
          refLabel: s.path + (parts.length > 1 ? ` (${i + 1}/${parts.length})` : ""),
          sectionTitle: s.path,
          title: s.title,
          content: part,
          boxTypes: "",
          week: cls.week,
        });
      });
    }
  };

  if (cls.docType === "slides") {
    const raw = readFileSync(path, "utf8");
    for (const f of parseBeamerFrames(raw)) {
      const content = f.content.length > 3200 ? splitLong(f.content, 3200)[0] : f.content;
      out.push({
        refType: "slide",
        refNumber: f.slideNumber,
        refLabel: `Séance ${cls.week} — Diapositive ${f.slideNumber}`,
        sectionTitle: f.sectionTitle,
        title: f.title,
        content,
        boxTypes: f.boxTypes.join(","),
        week: cls.week,
      });
    }
  } else if (path.endsWith(".tex")) {
    pushSections(readFileSync(path, "utf8"), cls.docType === "exercise" || cls.docType === "solution" ? "exercise" : "section");
  } else if (path.endsWith(".md")) {
    const raw = readFileSync(path, "utf8");
    const blocks = raw.split(/\n(?=#{1,3} )/);
    blocks.forEach((b) => {
      const title = (b.match(/^#{1,3} (.*)/) || [])[1] || cls.title;
      for (const part of splitLong(b.trim())) {
        if (part.length < 40) continue;
        out.push({ refType: "section", refNumber: null, refLabel: title, sectionTitle: title, title, content: part, boxTypes: "", week: cls.week });
      }
    });
  }
  // Contexte minimal garanti : préfixe cours/document dans le contenu indexé
  return out
    .filter((c) => c.content.trim().length >= 25)
    .map((c) => ({ ...c, content: c.content.trim() }));
}

export async function ingestCourses(opts: { force?: boolean; triggeredBy?: string } = {}): Promise<IngestResult> {
  db();
  const courses = all<{ code: string; source_path: string; full_code: string }>(
    "SELECT code, source_path, full_code FROM courses WHERE active = 1"
  );
  const runRow = run("INSERT INTO ingestion_runs (triggered_by) VALUES (?)", opts.triggeredBy ?? "script");
  const runId = Number(runRow.lastInsertRowid);
  const errors: { path: string; error: string }[] = [];
  let scanned = 0, ingested = 0, skipped = 0, chunksTotal = 0;
  const reportLines: string[] = [];

  // Sources supplémentaires réservées au professeur (analyses, examens générés)
  const extraPrivate: { path: string; course: string }[] = [];
  const outputDir = resolve(process.cwd(), "..", "output");
  if (existsSync(outputDir)) {
    for (const c of courses) {
      const analysis = join(outputDir, "course-analysis", `${c.full_code}-analysis.md`);
      if (existsSync(analysis)) extraPrivate.push({ path: analysis, course: c.code });
    }
  }

  for (const course of courses) {
    const root = resolve(process.cwd(), course.source_path);
    if (!existsSync(root)) {
      errors.push({ path: root, error: "Dossier de cours introuvable" });
      continue;
    }
    const files = [...walk(root)].sort();
    for (const file of files) {
      const cls = classify(file, course.code);
      if (!cls) continue;
      scanned++;
      try {
        const raw = readFileSync(file);
        const checksum = createHash("sha256").update(raw).digest("hex");
        const relPath = relative(resolve(process.cwd(), ".."), file);
        const existing = get<{ id: number; checksum: string }>("SELECT id, checksum FROM documents WHERE path = ?", relPath);
        if (existing && existing.checksum === checksum && !opts.force) {
          skipped++;
          continue;
        }
        const chunks = chunksForFile(file, cls, course.code);
        if (chunks.length === 0) {
          reportLines.push(`⚠ ${relPath} : aucun fragment extrait`);
        }
        const embeddings = await embedPassages(
          chunks.map((c) => `${course.code} ${c.title}. ${c.content}`)
        );
        transaction(() => {
          let docId: number;
          if (existing) {
            const old = all<{ id: number }>("SELECT id FROM chunks WHERE document_id = ?", existing.id);
            for (const o of old) run("DELETE FROM chunks_fts WHERE rowid = ?", o.id);
            run("DELETE FROM chunks WHERE document_id = ?", existing.id);
            run(
              "UPDATE documents SET checksum = ?, status = 'ok', error = NULL, ingested_at = datetime('now'), chunk_count = ?, space = ?, visible_to_students = ?, week = ?, title = ?, doc_type = ?, category = ? WHERE id = ?",
              checksum, chunks.length, cls.space, cls.visible ? 1 : 0, cls.week, cls.title, cls.docType, cls.category, existing.id
            );
            docId = existing.id;
          } else {
            const r = run(
              `INSERT INTO documents (course_code, space, path, filename, doc_type, title, category, week, checksum, status, visible_to_students, ingested_at, chunk_count)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'ok', ?, datetime('now'), ?)`,
              course.code, cls.space, relPath, basename(file), cls.docType, cls.title, cls.category, cls.week, checksum, cls.visible ? 1 : 0, chunks.length
            );
            docId = Number(r.lastInsertRowid);
          }
          chunks.forEach((c, i) => {
            const cr = run(
              `INSERT INTO chunks (document_id, course_code, space, seq, ref_type, ref_number, ref_label, section_title, title, content, display_content, box_types, week, embedding)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
              docId, course.code, cls.space, i, c.refType, c.refNumber, c.refLabel, c.sectionTitle, c.title, c.content, c.content, c.boxTypes, c.week,
              vecToBlob(embeddings[i])
            );
            run("INSERT INTO chunks_fts (rowid, title, content) VALUES (?, ?, ?)", Number(cr.lastInsertRowid), c.title, c.content);
          });
        });
        ingested++;
        chunksTotal += chunks.length;
        reportLines.push(`✓ ${relPath} — ${chunks.length} fragments (${cls.space})`);
      } catch (e) {
        const msg = e instanceof Error ? e.message : String(e);
        errors.push({ path: file, error: msg });
        reportLines.push(`✗ ${file} — ${msg}`);
      }
    }
  }

  // Documents privés du professeur
  for (const extra of extraPrivate) {
    scanned++;
    try {
      const raw = readFileSync(extra.path);
      const checksum = createHash("sha256").update(raw).digest("hex");
      const relPath = relative(resolve(process.cwd(), ".."), extra.path);
      const existing = get<{ id: number; checksum: string }>("SELECT id, checksum FROM documents WHERE path = ?", relPath);
      if (existing && existing.checksum === checksum && !opts.force) {
        skipped++;
        continue;
      }
      const cls: FileClass = { docType: "exam", title: prettyTitle(basename(extra.path)), week: null, category: "analyse", space: "instructor-private", visible: false };
      const chunks = chunksForFile(extra.path, cls, extra.course);
      const embeddings = await embedPassages(chunks.map((c) => `${extra.course} ${c.title}. ${c.content}`));
      transaction(() => {
        if (existing) {
          const old = all<{ id: number }>("SELECT id FROM chunks WHERE document_id = ?", existing.id);
          for (const o of old) run("DELETE FROM chunks_fts WHERE rowid = ?", o.id);
          run("DELETE FROM chunks WHERE document_id = ?", existing.id);
          run("UPDATE documents SET checksum = ?, ingested_at = datetime('now'), chunk_count = ? WHERE id = ?", checksum, chunks.length, existing.id);
          insertChunks(existing.id, extra.course, cls, chunks, embeddings);
        } else {
          const r = run(
            `INSERT INTO documents (course_code, space, path, filename, doc_type, title, category, week, checksum, status, visible_to_students, ingested_at, chunk_count)
             VALUES (?, 'instructor-private', ?, ?, 'exam', ?, 'analyse', NULL, ?, 'ok', 0, datetime('now'), ?)`,
            extra.course, relPath, basename(extra.path), cls.title, checksum, chunks.length
          );
          insertChunks(Number(r.lastInsertRowid), extra.course, cls, chunks, embeddings);
        }
      });
      ingested++;
      chunksTotal += chunks.length;
      reportLines.push(`✓ ${relPath} — ${chunks.length} fragments (instructor-private)`);
    } catch (e) {
      errors.push({ path: extra.path, error: e instanceof Error ? e.message : String(e) });
    }
  }

  // Purge : documents officiels dont le fichier source a disparu ou est archivé
  const officialDocs = all<{ id: number; path: string }>(
    "SELECT id, path FROM documents WHERE space LIKE 'official-%' OR space = 'instructor-private'"
  );
  let purged = 0;
  for (const doc of officialDocs) {
    const abs = resolve(process.cwd(), "..", doc.path);
    const archived = /(^|\/)_|(^|\/)archives?\//i.test(doc.path);
    if (archived || !existsSync(abs)) {
      transaction(() => {
        const old = all<{ id: number }>("SELECT id FROM chunks WHERE document_id = ?", doc.id);
        for (const o of old) run("DELETE FROM chunks_fts WHERE rowid = ?", o.id);
        run("DELETE FROM chunks WHERE document_id = ?", doc.id);
        run("DELETE FROM documents WHERE id = ?", doc.id);
      });
      purged++;
      reportLines.push(`− ${doc.path} — retiré (fichier supprimé ou archivé)`);
    }
  }
  if (purged) reportLines.push(`Purge : ${purged} document(s) retiré(s) de l'index.`);

  const report = reportLines.join("\n");
  run(
    "UPDATE ingestion_runs SET finished_at = datetime('now'), files_scanned = ?, files_ingested = ?, files_skipped = ?, chunks_created = ?, status = ?, report = ? WHERE id = ?",
    scanned, ingested, skipped, chunksTotal, errors.length ? "completed-with-errors" : "completed", report + (errors.length ? "\n\nErreurs:\n" + errors.map((e) => `${e.path}: ${e.error}`).join("\n") : ""), runId
  );
  return { runId, scanned, ingested, skipped, chunks: chunksTotal, errors, report };
}

function insertChunks(docId: number, courseCode: string, cls: FileClass, chunks: PreparedChunk[], embeddings: Float32Array[]) {
  chunks.forEach((c, i) => {
    const cr = run(
      `INSERT INTO chunks (document_id, course_code, space, seq, ref_type, ref_number, ref_label, section_title, title, content, display_content, box_types, week, embedding)
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      docId, courseCode, cls.space, i, c.refType, c.refNumber, c.refLabel, c.sectionTitle, c.title, c.content, c.content, c.boxTypes, c.week,
      vecToBlob(embeddings[i])
    );
    run("INSERT INTO chunks_fts (rowid, title, content) VALUES (?, ?, ?)", Number(cr.lastInsertRowid), c.title, c.content);
  });
}