spb/modelmap Public License

1// ============================================================================2//  Project   : modelmap3//  File      : site/lib/content.js4//  Purpose   : Content index over the synced research-repo snapshot (content/)5//  Author    : Simon-Pierre Boucher6//  Contact   : contact@spboucher.ai7//  Website   : https://modelmap.io8//  Created   : 2026-08-129//  Modified  : 2026-08-1210//  Platform  : macOS / Apple Silicon (arm64) — Node.js (deployed on MacLustr)11//  License   : All rights reserved (research code)12// ============================================================================13"use strict";1415const fs = require("fs");16const path = require("path");17const matter = require("gray-matter");1819const CONTENT_ROOT = path.join(__dirname, "..", "content");2021const TEXT_EXT = new Set([22  ".md", ".py", ".sh", ".toml", ".yaml", ".yml", ".cff", ".json",23  ".js", ".css", ".metal", ".cpp", ".h", ".hpp", ".swift", ".m", ".txt", ".log",24]);2526/** Resolve a repo-relative path safely inside content/, or return null. */27function safeResolve(rel) {28  const abs = path.resolve(CONTENT_ROOT, rel);29  if (!abs.startsWith(CONTENT_ROOT + path.sep) && abs !== CONTENT_ROOT) return null;30  return abs;31}3233function exists(rel) {34  const abs = safeResolve(rel);35  return abs !== null && fs.existsSync(abs);36}3738function readText(rel) {39  const abs = safeResolve(rel);40  if (!abs || !fs.existsSync(abs) || !fs.statSync(abs).isFile()) return null;41  return fs.readFileSync(abs, "utf8");42}4344/** Read a markdown file, returning { data (front matter), content }. */45function readMarkdown(rel) {46  const raw = readText(rel);47  if (raw === null) return null;48  try {49    return matter(raw);50  } catch {51    return { data: {}, content: raw };52  }53}5455/** List a directory inside content/: [{name, rel, dir, size}] */56function listDir(rel) {57  const abs = safeResolve(rel || ".");58  if (!abs || !fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) return null;59  return fs60    .readdirSync(abs)61    .filter((n) => !n.startsWith(".") && n !== "node_modules")62    .map((n) => {63      const st = fs.statSync(path.join(abs, n));64      return {65        name: n,66        rel: path.posix.join(rel || "", n),67        dir: st.isDirectory(),68        size: st.size,69      };70    })71    .sort((a, b) => (a.dir !== b.dir ? (a.dir ? -1 : 1) : a.name.localeCompare(b.name)));72}7374/** Recursively collect files under rel matching a predicate. */75function walk(rel, predicate, out = []) {76  const entries = listDir(rel);77  if (!entries) return out;78  for (const e of entries) {79    if (e.dir) walk(e.rel, predicate, out);80    else if (predicate(e)) out.push(e);81  }82  return out;83}8485function isTextFile(rel) {86  return TEXT_EXT.has(path.extname(rel).toLowerCase()) || path.basename(rel) === "Makefile";87}8889/** Parse research/LOG.md into entries [{title, body}] (## headings). */90function parseLogEntries() {91  const md = readMarkdown("research/LOG.md");92  if (!md) return [];93  const parts = md.content.split(/^## /m).slice(1);94  return parts.map((p) => {95    const nl = p.indexOf("\n");96    return { title: p.slice(0, nl).trim(), body: p.slice(nl + 1).replace(/^---\s*$/m, "").trim() };97  });98}99100/** Experiment cards: micro experiments + candidates. */101function listExperiments() {102  const out = [];103  for (const base of ["experiments/micro", "experiments"]) {104    const entries = listDir(base) || [];105    for (const e of entries) {106      if (!e.dir || e.name === "micro") continue;107      const readme = readMarkdown(path.posix.join(e.rel, "README.md"));108      const hasAnalysis = (readText(path.posix.join(e.rel, "analysis.md")) || "").length > 400;109      const runs = listResultRuns().filter((r) => r.experiment === e.name);110      let purpose = "";111      if (readme) {112        const line = readme.content.split("\n").find((l) => l.trim() && !l.startsWith("#"));113        purpose = line ? line.trim() : "";114      }115      out.push({116        id: e.name,117        rel: e.rel,118        purpose,119        status: runs.length > 0 && hasAnalysis ? "completed" : runs.length > 0 ? "has results" : "scaffolded",120        runs: runs.length,121      });122    }123  }124  return out;125}126127/** Result runs: results/<experiment>/<timestamp>/*.json */128function listResultRuns() {129  const out = [];130  for (const exp of listDir("results") || []) {131    if (!exp.dir) continue;132    for (const run of listDir(exp.rel) || []) {133      if (!run.dir) continue;134      const files = (listDir(run.rel) || []).filter((f) => !f.dir);135      out.push({ experiment: exp.name, timestamp: run.name, rel: run.rel, files });136    }137  }138  return out.sort((a, b) => b.timestamp.localeCompare(a.timestamp));139}140141/**142 * Atlas entries: atlas/<model_id>/<map_type>/<version>/ with provenance.json143 * and confidence.md. The confidence level is parsed from confidence.md; an144 * entry with no committed level is reported as "unrated" and the platform145 * never presents it above Level 0.146 */147function listAtlasEntries() {148  const out = [];149  for (const model of listDir("atlas") || []) {150    if (!model.dir) continue;151    for (const mapType of listDir(model.rel) || []) {152      if (!mapType.dir) continue;153      for (const version of listDir(mapType.rel) || []) {154        if (!version.dir) continue;155        const provenance = readJson(path.posix.join(version.rel, "provenance.json"));156        const confidence = readText(path.posix.join(version.rel, "confidence.md")) || "";157        const m = confidence.match(/^Level\s*:\s*([0-3])\b/m);158        out.push({159          model: model.name,160          mapType: mapType.name,161          version: version.name,162          rel: version.rel,163          provenance,164          level: m ? Number(m[1]) : null,165          files: (listDir(version.rel) || []).filter((f) => !f.dir),166        });167      }168    }169  }170  return out;171}172173function readJson(rel) {174  const raw = readText(rel);175  if (raw === null) return null;176  try {177    return JSON.parse(raw);178  } catch {179    return null;180  }181}182183/** Build info written by sync-content.sh (commit, counts, date). */184function buildInfo() {185  return readJson("build-info.json") || {};186}187188/** expA probe map from the first atlas entry (null before it exists). */189function probeMapV1() {190  return readJson("atlas/qwen3-0.6b-4bit/probes/v1/map.json");191}192193/** Parsed expH runs for charts: warm/cold storage rows + compute rows. */194function expHData() {195  const out = { warm: null, cold: null, compute: [] };196  for (const r of listResultRuns().filter((x) => x.experiment === "expH_capture_cost_frontier")) {197    const d = readJson(path.posix.join(r.rel, "results.json"));198    if (!d) continue;199    if (d.run === 1) {200      out.warm = d.storage;201      out.compute.push(...(d.compute || []).map((c) => ({ ...c, source: "synthetic" })));202    } else if (d.run === 2) {203      out.cold = d.storage;204    } else if (d.run === 3) {205      out.compute.push(...(d.compute || []).map((c) => ({ ...c, source: "qwen3-0.6b-4bit" })));206    }207  }208  return out;209}210211/** Site-wide stats for the home page. */212function stats() {213  const bib = readText("research/bibliography.md") || "";214  const sources = bib.split("\n").filter((l) => l.trim().startsWith("- ")).length;215  const gapsDoc = readText("research/research_gaps.md") || "";216  const gaps = (gapsDoc.match(/^#{2,3}\s+G\d{2}/gm) || []).length;217  const experiments = listExperiments();218  const notes = (listDir("research/notes") || []).filter((f) => f.name.endsWith(".md") && f.name !== "README.md").length;219  return {220    sources,221    gaps,222    notes,223    experiments: experiments.length,224    experimentsDone: experiments.filter((e) => e.status === "completed").length,225    resultRuns: listResultRuns().length,226    logEntries: parseLogEntries().length,227    atlasEntries: listAtlasEntries().length,228    build: buildInfo(),229  };230}231232module.exports = {233  CONTENT_ROOT,234  safeResolve,235  exists,236  readText,237  readMarkdown,238  readJson,239  listDir,240  walk,241  isTextFile,242  parseLogEntries,243  listExperiments,244  listResultRuns,245  listAtlasEntries,246  buildInfo,247  stats,248  probeMapV1,249  expHData,250};251