TypeScript 98.3%
CSS 0.9%
Shell 0.7%
1// Pipeline d'ingestion : scan des dossiers de cours → classification → parsing structurel →2// fragments + métadonnées → FTS5 + embeddings. Incrémental par somme de contrôle SHA-256.34import { createHash } from "node:crypto";5import { readFileSync, readdirSync, statSync, existsSync } from "node:fs";6import { basename, join, relative, resolve } from "node:path";7import { all, db, get, run, transaction } from "../db/index.ts";8import { embedPassages, vecToBlob } from "./embeddings.ts";9import { parseArticleSections, parseBeamerFrames, splitLong } from "./latex.ts";1011export type IngestResult = {12 runId: number;13 scanned: number;14 ingested: number;15 skipped: number;16 chunks: number;17 errors: { path: string; error: string }[];18 report: string;19};2021type FileClass = {22 docType: string;23 title: string;24 week: number | null;25 category: string;26 space: string;27 visible: boolean;28};2930const EXCLUDED_DIRS = new Set([".git", ".claude", "node_modules", "__pycache__", "plateforme_ateliers", "plateforme_data", "images", "static", "assets"]);31const EXCLUDED_EXT = new Set([".aux", ".log", ".out", ".toc", ".nav", ".snm", ".bbl", ".bcf", ".blg", ".fls", ".xml", ".sty", ".bib", ".png", ".jpg", ".jpeg", ".webp", ".db", ".py", ".js", ".html", ".json", ".gitignore", ".fdb_latexmk", ".xlsx", ".csv"]);3233function classify(path: string, courseCode: string): FileClass | null {34 const name = basename(path).toLowerCase();35 const officialSpace = `official-${courseCode.toLowerCase()}`;3637 if (name.startsWith("examen") || name.includes("_exam") || name.includes("solutionnaire") || name.includes("blueprint") || name.includes("grille_correction") || name.includes("-analysis")) {38 return { docType: "exam", title: prettyTitle(name), week: null, category: "examen", space: "instructor-private", visible: false };39 }40 const seance = name.match(/seance(\d+)/);41 if (seance && name.endsWith(".tex")) {42 return { docType: "slides", title: prettyTitle(name), week: parseInt(seance[1], 10), category: "seance", space: officialSpace, visible: true };43 }44 if (name === "plan_de_cours.tex") {45 return { docType: "plan", title: "Plan de cours", week: null, category: "plan", space: officialSpace, visible: true };46 }47 const atelier = name.match(/atelier(\d)/);48 if (name.endsWith(".tex") && atelier) {49 const isSolution = name.startsWith("solution");50 return {51 docType: isSolution ? "solution" : "exercise",52 title: `${isSolution ? "Solution" : "Énoncé"} — Atelier ${atelier[1]}`,53 week: null,54 category: `atelier${atelier[1]}`,55 space: officialSpace,56 visible: true,57 };58 }59 if (name === "aide_memoire.tex") {60 return { docType: "aide-memoire", title: "Aide-mémoire", week: null, category: "reference", space: officialSpace, visible: true };61 }62 if (name === "glossaire.tex") {63 return { docType: "glossary", title: "Glossaire bilingue", week: null, category: "reference", space: officialSpace, visible: true };64 }65 if (name === "description_moodle.md" || name === "readme.md") {66 return { docType: "markdown", title: prettyTitle(name), week: null, category: "info", space: officialSpace, visible: true };67 }68 return null;69}7071function prettyTitle(name: string): string {72 const seance = name.match(/seance(\d+)_?([a-z_]*)/);73 if (seance) {74 const suffix = (seance[2] || "").replace(/_/g, " ").trim();75 return `Séance ${parseInt(seance[1], 10)}${suffix ? " — " + capitalize(suffix) : ""}`;76 }77 return capitalize(name.replace(/\.(tex|md|pdf)$/, "").replace(/[_-]/g, " "));78}79function capitalize(s: string): string {80 return s ? s[0].toUpperCase() + s.slice(1) : s;81}8283function* walk(dir: string): Generator<string> {84 for (const entry of readdirSync(dir, { withFileTypes: true })) {85 if (entry.isDirectory()) {86 // Dossiers d'archives (_archive*, archive*) : matériel retiré du cours — jamais indexé87 if (entry.name.startsWith("_") || /^archives?/i.test(entry.name)) continue;88 if (!EXCLUDED_DIRS.has(entry.name)) yield* walk(join(dir, entry.name));89 } else {90 const ext = entry.name.slice(entry.name.lastIndexOf("."));91 if (!EXCLUDED_EXT.has(ext.toLowerCase()) && !entry.name.startsWith(".")) yield join(dir, entry.name);92 }93 }94}9596type PreparedChunk = {97 refType: string;98 refNumber: number | null;99 refLabel: string;100 sectionTitle: string;101 title: string;102 content: string;103 boxTypes: string;104 week: number | null;105};106107function chunksForFile(path: string, cls: FileClass, courseCode: string): PreparedChunk[] {108 const out: PreparedChunk[] = [];109 const pushSections = (raw: string, refType: string) => {110 for (const s of parseArticleSections(raw)) {111 const parts = splitLong(s.content);112 parts.forEach((part, i) => {113 out.push({114 refType,115 refNumber: null,116 refLabel: s.path + (parts.length > 1 ? ` (${i + 1}/${parts.length})` : ""),117 sectionTitle: s.path,118 title: s.title,119 content: part,120 boxTypes: "",121 week: cls.week,122 });123 });124 }125 };126127 if (cls.docType === "slides") {128 const raw = readFileSync(path, "utf8");129 for (const f of parseBeamerFrames(raw)) {130 const content = f.content.length > 3200 ? splitLong(f.content, 3200)[0] : f.content;131 out.push({132 refType: "slide",133 refNumber: f.slideNumber,134 refLabel: `Séance ${cls.week} — Diapositive ${f.slideNumber}`,135 sectionTitle: f.sectionTitle,136 title: f.title,137 content,138 boxTypes: f.boxTypes.join(","),139 week: cls.week,140 });141 }142 } else if (path.endsWith(".tex")) {143 pushSections(readFileSync(path, "utf8"), cls.docType === "exercise" || cls.docType === "solution" ? "exercise" : "section");144 } else if (path.endsWith(".md")) {145 const raw = readFileSync(path, "utf8");146 const blocks = raw.split(/\n(?=#{1,3} )/);147 blocks.forEach((b) => {148 const title = (b.match(/^#{1,3} (.*)/) || [])[1] || cls.title;149 for (const part of splitLong(b.trim())) {150 if (part.length < 40) continue;151 out.push({ refType: "section", refNumber: null, refLabel: title, sectionTitle: title, title, content: part, boxTypes: "", week: cls.week });152 }153 });154 }155 // Contexte minimal garanti : préfixe cours/document dans le contenu indexé156 return out157 .filter((c) => c.content.trim().length >= 25)158 .map((c) => ({ ...c, content: c.content.trim() }));159}160161export async function ingestCourses(opts: { force?: boolean; triggeredBy?: string } = {}): Promise<IngestResult> {162 db();163 const courses = all<{ code: string; source_path: string; full_code: string }>(164 "SELECT code, source_path, full_code FROM courses WHERE active = 1"165 );166 const runRow = run("INSERT INTO ingestion_runs (triggered_by) VALUES (?)", opts.triggeredBy ?? "script");167 const runId = Number(runRow.lastInsertRowid);168 const errors: { path: string; error: string }[] = [];169 let scanned = 0, ingested = 0, skipped = 0, chunksTotal = 0;170 const reportLines: string[] = [];171172 // Sources supplémentaires réservées au professeur (analyses, examens générés)173 const extraPrivate: { path: string; course: string }[] = [];174 const outputDir = resolve(process.cwd(), "..", "output");175 if (existsSync(outputDir)) {176 for (const c of courses) {177 const analysis = join(outputDir, "course-analysis", `${c.full_code}-analysis.md`);178 if (existsSync(analysis)) extraPrivate.push({ path: analysis, course: c.code });179 }180 }181182 for (const course of courses) {183 const root = resolve(process.cwd(), course.source_path);184 if (!existsSync(root)) {185 errors.push({ path: root, error: "Dossier de cours introuvable" });186 continue;187 }188 const files = [...walk(root)].sort();189 for (const file of files) {190 const cls = classify(file, course.code);191 if (!cls) continue;192 scanned++;193 try {194 const raw = readFileSync(file);195 const checksum = createHash("sha256").update(raw).digest("hex");196 const relPath = relative(resolve(process.cwd(), ".."), file);197 const existing = get<{ id: number; checksum: string }>("SELECT id, checksum FROM documents WHERE path = ?", relPath);198 if (existing && existing.checksum === checksum && !opts.force) {199 skipped++;200 continue;201 }202 const chunks = chunksForFile(file, cls, course.code);203 if (chunks.length === 0) {204 reportLines.push(`⚠ ${relPath} : aucun fragment extrait`);205 }206 const embeddings = await embedPassages(207 chunks.map((c) => `${course.code} ${c.title}. ${c.content}`)208 );209 transaction(() => {210 let docId: number;211 if (existing) {212 const old = all<{ id: number }>("SELECT id FROM chunks WHERE document_id = ?", existing.id);213 for (const o of old) run("DELETE FROM chunks_fts WHERE rowid = ?", o.id);214 run("DELETE FROM chunks WHERE document_id = ?", existing.id);215 run(216 "UPDATE documents SET checksum = ?, status = 'ok', error = NULL, ingested_at = datetime('now'), chunk_count = ?, space = ?, visible_to_students = ?, week = ?, title = ?, doc_type = ?, category = ? WHERE id = ?",217 checksum, chunks.length, cls.space, cls.visible ? 1 : 0, cls.week, cls.title, cls.docType, cls.category, existing.id218 );219 docId = existing.id;220 } else {221 const r = run(222 `INSERT INTO documents (course_code, space, path, filename, doc_type, title, category, week, checksum, status, visible_to_students, ingested_at, chunk_count)223 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'ok', ?, datetime('now'), ?)`,224 course.code, cls.space, relPath, basename(file), cls.docType, cls.title, cls.category, cls.week, checksum, cls.visible ? 1 : 0, chunks.length225 );226 docId = Number(r.lastInsertRowid);227 }228 chunks.forEach((c, i) => {229 const cr = run(230 `INSERT INTO chunks (document_id, course_code, space, seq, ref_type, ref_number, ref_label, section_title, title, content, display_content, box_types, week, embedding)231 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,232 docId, course.code, cls.space, i, c.refType, c.refNumber, c.refLabel, c.sectionTitle, c.title, c.content, c.content, c.boxTypes, c.week,233 vecToBlob(embeddings[i])234 );235 run("INSERT INTO chunks_fts (rowid, title, content) VALUES (?, ?, ?)", Number(cr.lastInsertRowid), c.title, c.content);236 });237 });238 ingested++;239 chunksTotal += chunks.length;240 reportLines.push(`✓ ${relPath} — ${chunks.length} fragments (${cls.space})`);241 } catch (e) {242 const msg = e instanceof Error ? e.message : String(e);243 errors.push({ path: file, error: msg });244 reportLines.push(`✗ ${file} — ${msg}`);245 }246 }247 }248249 // Documents privés du professeur250 for (const extra of extraPrivate) {251 scanned++;252 try {253 const raw = readFileSync(extra.path);254 const checksum = createHash("sha256").update(raw).digest("hex");255 const relPath = relative(resolve(process.cwd(), ".."), extra.path);256 const existing = get<{ id: number; checksum: string }>("SELECT id, checksum FROM documents WHERE path = ?", relPath);257 if (existing && existing.checksum === checksum && !opts.force) {258 skipped++;259 continue;260 }261 const cls: FileClass = { docType: "exam", title: prettyTitle(basename(extra.path)), week: null, category: "analyse", space: "instructor-private", visible: false };262 const chunks = chunksForFile(extra.path, cls, extra.course);263 const embeddings = await embedPassages(chunks.map((c) => `${extra.course} ${c.title}. ${c.content}`));264 transaction(() => {265 if (existing) {266 const old = all<{ id: number }>("SELECT id FROM chunks WHERE document_id = ?", existing.id);267 for (const o of old) run("DELETE FROM chunks_fts WHERE rowid = ?", o.id);268 run("DELETE FROM chunks WHERE document_id = ?", existing.id);269 run("UPDATE documents SET checksum = ?, ingested_at = datetime('now'), chunk_count = ? WHERE id = ?", checksum, chunks.length, existing.id);270 insertChunks(existing.id, extra.course, cls, chunks, embeddings);271 } else {272 const r = run(273 `INSERT INTO documents (course_code, space, path, filename, doc_type, title, category, week, checksum, status, visible_to_students, ingested_at, chunk_count)274 VALUES (?, 'instructor-private', ?, ?, 'exam', ?, 'analyse', NULL, ?, 'ok', 0, datetime('now'), ?)`,275 extra.course, relPath, basename(extra.path), cls.title, checksum, chunks.length276 );277 insertChunks(Number(r.lastInsertRowid), extra.course, cls, chunks, embeddings);278 }279 });280 ingested++;281 chunksTotal += chunks.length;282 reportLines.push(`✓ ${relPath} — ${chunks.length} fragments (instructor-private)`);283 } catch (e) {284 errors.push({ path: extra.path, error: e instanceof Error ? e.message : String(e) });285 }286 }287288 // Purge : documents officiels dont le fichier source a disparu ou est archivé289 const officialDocs = all<{ id: number; path: string }>(290 "SELECT id, path FROM documents WHERE space LIKE 'official-%' OR space = 'instructor-private'"291 );292 let purged = 0;293 for (const doc of officialDocs) {294 const abs = resolve(process.cwd(), "..", doc.path);295 const archived = /(^|\/)_|(^|\/)archives?\//i.test(doc.path);296 if (archived || !existsSync(abs)) {297 transaction(() => {298 const old = all<{ id: number }>("SELECT id FROM chunks WHERE document_id = ?", doc.id);299 for (const o of old) run("DELETE FROM chunks_fts WHERE rowid = ?", o.id);300 run("DELETE FROM chunks WHERE document_id = ?", doc.id);301 run("DELETE FROM documents WHERE id = ?", doc.id);302 });303 purged++;304 reportLines.push(`− ${doc.path} — retiré (fichier supprimé ou archivé)`);305 }306 }307 if (purged) reportLines.push(`Purge : ${purged} document(s) retiré(s) de l'index.`);308309 const report = reportLines.join("\n");310 run(311 "UPDATE ingestion_runs SET finished_at = datetime('now'), files_scanned = ?, files_ingested = ?, files_skipped = ?, chunks_created = ?, status = ?, report = ? WHERE id = ?",312 scanned, ingested, skipped, chunksTotal, errors.length ? "completed-with-errors" : "completed", report + (errors.length ? "\n\nErreurs:\n" + errors.map((e) => `${e.path}: ${e.error}`).join("\n") : ""), runId313 );314 return { runId, scanned, ingested, skipped, chunks: chunksTotal, errors, report };315}316317function insertChunks(docId: number, courseCode: string, cls: FileClass, chunks: PreparedChunk[], embeddings: Float32Array[]) {318 chunks.forEach((c, i) => {319 const cr = run(320 `INSERT INTO chunks (document_id, course_code, space, seq, ref_type, ref_number, ref_label, section_title, title, content, display_content, box_types, week, embedding)321 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,322 docId, courseCode, cls.space, i, c.refType, c.refNumber, c.refLabel, c.sectionTitle, c.title, c.content, c.content, c.boxTypes, c.week,323 vecToBlob(embeddings[i])324 );325 run("INSERT INTO chunks_fts (rowid, title, content) VALUES (?, ?, ?)", Number(cr.lastInsertRowid), c.title, c.content);326 });327}328