spb/note-spboucher-ai

1/**2 * Research AI features. Every feature is an async generator of ResearchAIEvent (SSE-ready) built on route() → streamAI/callAI.3 * Workspace content is always wrapped in <workspace> tags and the INJECTION_GUARD is appended to every system prompt.4 * Outputs are AI-generated suggestions: nothing here writes into pages, references or tasks.5 */6import { and, eq, inArray, isNull } from "drizzle-orm";7import { db, schema } from "@/db";8import { route } from "@/lib/ai/router";9import { streamAI, callAI } from "@/lib/ai/providers";10import { INJECTION_GUARD } from "@/lib/ai/prompts";11import { recordUsage } from "@/lib/ai/usage";12import { costUsd } from "@/lib/ai/catalog";13import { hybridSearch } from "@/lib/search";14import { approxTokens } from "@/lib/search/chunk";15import { toMarkdown, type PMNode } from "@/lib/editor/text";16import { HttpError } from "@/lib/auth/guard";17import type { TaskClass, AIUsage } from "@/lib/ai/types";18import { listReferences, type ReferenceRow } from "./references";19import { shortCite, lastName } from "./bibtex";2021export const RESEARCH_FEATURES = ["synthesis", "compare", "question", "equation", "draft", "refcheck", "plan"] as const;22export type ResearchFeature = (typeof RESEARCH_FEATURES)[number];23export interface ResearchAIRequest { feature: ResearchFeature; pageId: string; referenceIds?: string[]; attachmentIds?: string[]; text?: string; instruction?: string; question?: string; latex?: string; language?: "fr" | "en" | null; model?: string | null }24export type ResearchAIEvent =25  | { type: "meta"; feature: ResearchFeature; model: string; provider: string; reason: string; context: { references: number; attachments: number; sources: number; pageChars: number } }26  | { type: "text"; delta: string }27  | { type: "result"; data: unknown }28  | { type: "done"; usage: AIUsage | null; costUsd: number | null; latencyMs: number }29  | { type: "error"; message: string };3031const BASE = `You are the research assistant inside Note SPBoucher AI, the private workspace of Simon-Pierre Boucher (professor of real-estate finance and AI, UQO). You help write academic working papers in finance, economics and AI. Be precise, concise, technical; prefer Markdown with LaTeX ($…$ inline, $$…$$ display) for mathematics and tables for comparisons. Never invent references, data, results or quotations: when something is not in the provided material, say so explicitly. Everything you produce is a suggestion that the author will review.`;32const lang = (l?: string | null) => (l === "fr" ? "Answer in French (Québec academic usage)." : l === "en" ? "Answer in English." : "Answer in the language of the page/material (French or English).");33const sys = (extra: string, l?: string | null) => `${BASE}\n${extra}\n${lang(l)}\n${INJECTION_GUARD}`;34const ws = (kind: string, body: string, attrs = "") => `<workspace kind="${kind}"${attrs ? " " + attrs : ""}>\n${body}\n</workspace>`;35const esc = (s: string) => s.replace(/"/g, "'");3637// ---------- context builders38async function pageOf(wsId: string, pageId: string) {39  const [p] = await db.select().from(schema.pages).where(and(eq(schema.pages.id, pageId), eq(schema.pages.workspaceId, wsId), isNull(schema.pages.deletedAt)));40  if (!p) throw new HttpError(404, "Page not found"); return p;41}42export function refLine(r: ReferenceRow, withAbstract = true) {43  const f = (r.fields ?? {}) as Record<string, string>;44  const head = `[${r.citeKey}] ${(r.authors ?? []).join(", ") || "Unknown authors"} (${r.year ?? "n.d."}). ${r.title}.${r.journal ? ` ${r.journal}.` : ""}${f.volume ? ` ${f.volume}${f.number ? `(${f.number})` : ""}${f.pages ? `, ${f.pages}` : ""}.` : ""}${r.doi ? ` doi:${r.doi}` : ""} — verification: ${r.verified}`;45  return withAbstract && f.abstract ? `${head}\n  Abstract: ${f.abstract.slice(0, 1500)}` : head;46}47async function attachmentTexts(wsId: string, ids: string[], capEach: number) {48  if (!ids.length) return [] as { id: string; filename: string; text: string; pages: number | null }[];49  const rows = await db.select({ id: schema.attachments.id, filename: schema.attachments.filename, text: schema.attachments.extractedText, pages: schema.attachments.pageCount }).from(schema.attachments).where(and(eq(schema.attachments.workspaceId, wsId), isNull(schema.attachments.deletedAt), inArray(schema.attachments.id, ids)));50  return rows.map((a) => ({ id: a.id, filename: a.filename, pages: a.pages, text: (a.text ?? "").replace(/\s+\n/g, "\n").slice(0, capEach) }));51}52async function pageAttachmentIds(pageId: string) {53  return (await db.select({ id: schema.attachments.id }).from(schema.attachments).where(and(eq(schema.attachments.pageId, pageId), isNull(schema.attachments.deletedAt)))).map((a) => a.id);54}55async function retrieved(wsId: string, query: string, pageIds: string[], limit = 12) {56  try { const hits = await hybridSearch(wsId, query.slice(0, 1000), { limit, pageIds }); return hits.map((h, i) => `[S${i + 1}] ${h.title}${h.heading ? ` › ${h.heading}` : ""} (${h.sourceType})\n${h.content.slice(0, 900)}`); } catch (e) { console.error("research retrieval failed", e); return []; }57}5859// ---------- generic runners60interface Prepared { system: string; user: string; task: TaskClass; context: { references: number; attachments: number; sources: number; pageChars: number }; maxTokens?: number }61async function* streamText(wsId: string, feature: ResearchFeature, p: Prepared, preferred: string | null | undefined, signal?: AbortSignal): AsyncGenerator<ResearchAIEvent, string> {62  const dec = await route({ task: p.task, preferred: preferred ?? null, estTokens: approxTokens(p.system + p.user) + (p.maxTokens ?? 6000) });63  yield { type: "meta", feature, model: dec.model.id, provider: dec.provider, reason: dec.reason, context: p.context };64  let text = ""; let usage: AIUsage | null = null; let ms = 0; let ttft: number | null = null; let err: string | null = null;65  for await (const ev of streamAI({ model: dec.model.id, provider: dec.provider, system: p.system, messages: [{ role: "user", content: p.user }], effort: dec.effort, maxTokens: p.maxTokens ?? 6000, signal })) {66    if (ev.type === "text") { text += ev.delta; yield { type: "text", delta: ev.delta }; } else if (ev.type === "done") { usage = ev.usage; ms = ev.totalMs; ttft = ev.ttftMs; } else if (ev.type === "error") err = ev.message;67  }68  const cost = usage ? costUsd(dec.model, usage) : null;69  await recordUsage({ workspaceId: wsId, provider: dec.provider, model: dec.model.id, task: `research:${feature}`, usage, costUsd: cost, latencyMs: ms, ttftMs: ttft, ok: !err, error: err });70  if (err && !text) { yield { type: "error", message: err }; return ""; }71  yield { type: "done", usage, costUsd: cost, latencyMs: Math.round(ms) };72  return text;73}74async function jsonCall<T>(wsId: string, feature: ResearchFeature, p: Prepared, schemaName: string, jsonSchema: Record<string, unknown>, preferred: string | null | undefined, signal?: AbortSignal): Promise<{ data: T; meta: ResearchAIEvent & { type: "meta" }; done: ResearchAIEvent & { type: "done" } }> {75  const dec = await route({ task: p.task, schema: true, preferred: preferred ?? null, estTokens: approxTokens(p.system + p.user) + 4000 });76  const meta: ResearchAIEvent & { type: "meta" } = { type: "meta", feature, model: dec.model.id, provider: dec.provider, reason: dec.reason, context: p.context };77  let res; try { res = await callAI({ model: dec.model.id, provider: dec.provider, system: p.system, messages: [{ role: "user", content: p.user }], effort: dec.effort, maxTokens: p.maxTokens ?? 6000, jsonSchema: { name: schemaName, schema: jsonSchema }, signal }); }78  catch (e: any) { await recordUsage({ workspaceId: wsId, provider: dec.provider, model: dec.model.id, task: `research:${feature}`, ok: false, error: String(e?.message ?? e) }); throw e; }79  await recordUsage({ workspaceId: wsId, provider: dec.provider, model: dec.model.id, task: `research:${feature}`, usage: res.usage, costUsd: res.costUsd, latencyMs: res.totalMs, ttftMs: res.ttftMs, ok: true });80  let data: T; try { data = JSON.parse(res.text.trim().replace(/^```(?:json)?\s*|\s*```$/g, "")) as T; } catch { throw new Error("The model did not return valid JSON"); }81  return { data, meta, done: { type: "done", usage: res.usage, costUsd: res.costUsd, latencyMs: Math.round(res.totalMs) } };82}8384// ---------- features85async function prepSynthesis(wsId: string, r: ResearchAIRequest): Promise<Prepared> {86  const page = await pageOf(wsId, r.pageId); const refs = await listReferences(wsId, { pageId: r.pageId });87  const chosen = r.referenceIds?.length ? refs.filter((x) => r.referenceIds!.includes(x.id)) : refs;88  const attIds = [...new Set([...chosen.map((x) => x.attachmentId).filter(Boolean) as string[], ...(r.attachmentIds ?? [])])];89  const atts = await attachmentTexts(wsId, attIds, 12_000);90  const query = r.question?.trim() || `${page.title} ${(page.properties as any)?.tags?.join?.(" ") ?? ""} literature review methods results`;91  const hits = await retrieved(wsId, query, [r.pageId], 12);92  const md = toMarkdown(page.content as PMNode).slice(0, 12_000);93  const user = [ws("current-page", md, `title="${esc(page.title)}"`), ws("references", chosen.map((x) => refLine(x)).join("\n") || "(no references stored for this page)"), atts.length ? ws("attached-papers", atts.map((a) => `### ${a.filename}${a.pages ? ` (${a.pages} p.)` : ""}\n${a.text}`).join("\n\n")) : "", hits.length ? ws("retrieved-passages", hits.join("\n\n")) : "", `Write a literature synthesis for this working paper${r.question ? ` focused on: ${r.question}` : ""}. Structure: (1) the central questions the literature addresses, (2) main strands and findings with inline citations using the stored cite keys in the form \\cite{key}, (3) methodological patterns and disagreements, (4) gaps this paper can fill, (5) a table of the references (key, question, data, method, key finding). Cite only the references listed above; if a claim has no supporting reference in the material, mark it "[no stored reference]".`].filter(Boolean).join("\n\n");94  return { system: sys("Task: literature synthesis grounded strictly in the stored references, attached papers and retrieved passages.", r.language), user, task: "deep", maxTokens: 7000, context: { references: chosen.length, attachments: atts.length, sources: hits.length, pageChars: md.length } };95}96async function prepCompare(wsId: string, r: ResearchAIRequest): Promise<Prepared> {97  const refs = r.referenceIds?.length ? await listReferences(wsId, { ids: r.referenceIds }) : [];98  const attIds = [...new Set([...refs.map((x) => x.attachmentId).filter(Boolean) as string[], ...(r.attachmentIds ?? [])])];99  if (refs.length + attIds.length < 2) throw new HttpError(400, "Select at least two references or files to compare");100  const atts = await attachmentTexts(wsId, attIds, 18_000);101  const user = [ws("references", refs.map((x) => refLine(x)).join("\n") || "(none)"), atts.length ? ws("papers-full-text", atts.map((a) => `### ${a.filename}\n${a.text}`).join("\n\n")) : "", `Compare the methodology of these works: research question, theoretical framework, data (sample, period, frequency), identification/estimation strategy, inference, main results, limitations. Produce a side-by-side Markdown table followed by a critical discussion (strengths, weaknesses, what each approach can and cannot identify) and recommendations for the author's own design. Refer to works by \\cite{key} or filename. Where the full text is not available, say that the comparison relies on the abstract/metadata only.${r.instruction ? `\n\nAdditional focus: ${r.instruction}` : ""}`].filter(Boolean).join("\n\n");102  return { system: sys("Task: methodology comparison between academic works.", r.language), user, task: "reason", maxTokens: 6000, context: { references: refs.length, attachments: atts.length, sources: 0, pageChars: 0 } };103}104async function prepQuestion(wsId: string, r: ResearchAIRequest): Promise<Prepared> {105  const page = await pageOf(wsId, r.pageId); const refs = await listReferences(wsId, { pageId: r.pageId });106  const md = toMarkdown(page.content as PMNode).slice(0, 15_000); const q = (r.question ?? r.text ?? "").trim();107  const hits = q ? await retrieved(wsId, q, [r.pageId], 6) : [];108  const user = [ws("current-page", md, `title="${esc(page.title)}"`), refs.length ? ws("references", refs.map((x) => refLine(x, false)).join("\n")) : "", hits.length ? ws("retrieved-passages", hits.join("\n\n")) : "", q ? `Research question to refine:\n"${q}"` : "Derive the research question from the page content and refine it.", `Refine the research question: (1) restate it precisely (population, variables, mechanism, period); (2) list 3–5 sharper alternative formulations ranked by feasibility and contribution; (3) for the best one, state testable hypotheses with the expected sign and the identifying assumption; (4) what data and method would answer it; (5) what would falsify it; (6) which stored references it builds on (\\cite{key}) and where the novelty lies.`].filter(Boolean).join("\n\n");109  return { system: sys("Task: research-question refinement for an academic working paper.", r.language), user, task: "reason", context: { references: refs.length, attachments: 0, sources: hits.length, pageChars: md.length } };110}111async function prepEquation(wsId: string, r: ResearchAIRequest): Promise<Prepared> {112  const latex = (r.latex ?? r.text ?? "").trim(); if (!latex) throw new HttpError(400, "No equation provided");113  const page = await pageOf(wsId, r.pageId); const md = toMarkdown(page.content as PMNode).slice(0, 8000);114  const user = [ws("current-page-excerpt", md, `title="${esc(page.title)}"`), `Equation to explain:\n$$\n${latex}\n$$`, `Explain this equation for a graduate reader: define every symbol (use the page's notation when it appears there), give the intuition, derive or justify each term step by step, state the assumptions it relies on, note special cases and common pitfalls, and suggest how it could be estimated or computed. Use LaTeX.${r.instruction ? `\n\nFocus: ${r.instruction}` : ""}`].join("\n\n");115  return { system: sys("Task: mathematical explanation.", r.language), user, task: "reason", context: { references: 0, attachments: 0, sources: 0, pageChars: md.length } };116}117async function prepDraft(wsId: string, r: ResearchAIRequest): Promise<Prepared> {118  const text = (r.text ?? "").trim(); if (!text) throw new HttpError(400, "Select or paste the section to edit");119  const page = await pageOf(wsId, r.pageId); const refs = await listReferences(wsId, { pageId: r.pageId });120  const user = [ws("section-to-edit", text, `page="${esc(page.title)}"`), refs.length ? ws("available-references", refs.map((x) => `\\cite{${x.citeKey}} = ${shortCite(x)}, ${x.title}`).join("\n")) : "", `Instruction: ${r.instruction?.trim() || "Edit this section to journal standard: tighten the argument, improve flow and precision, keep the author's voice, meaning and language."}\n\nReturn the revised section first (ready to paste, same Markdown/LaTeX conventions, cite only with the listed \\cite{keys}), then a short "Changes" list explaining what you changed and any claims that still need a reference or evidence.`].filter(Boolean).join("\n\n");121  return { system: sys("Task: academic draft editing.", r.language), user, task: "write", maxTokens: 7000, context: { references: refs.length, attachments: 0, sources: 0, pageChars: text.length } };122}123124// ---------- reference checking125export interface CitationHit { citation: string; kind: "cite" | "author-year"; context: string; status: "matched" | "unverified" | "ambiguous"; citeKey: string | null; referenceId: string | null; note: string; candidates?: string[] }126const deaccent = (s: string) => s.normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase();127export function extractCitations(text: string): { citation: string; kind: "cite" | "author-year"; context: string }[] {128  const out: { citation: string; kind: "cite" | "author-year"; context: string }[] = []; const seen = new Set<string>();129  const ctx = (i: number) => text.slice(Math.max(0, i - 120), i + 160).replace(/\s+/g, " ").trim();130  for (const m of text.matchAll(/\\(?:cite|citep|citet|citeauthor|citeyear|parencite|textcite|autocite|nocite)\*?(?:\[[^\]]*\]){0,2}\{([^}]+)\}/g)) for (const k of m[1].split(",").map((x) => x.trim()).filter(Boolean)) { const id = "k:" + k.toLowerCase(); if (!seen.has(id)) { seen.add(id); out.push({ citation: `\\cite{${k}}`, kind: "cite", context: ctx(m.index ?? 0) }); } }131  const name = "[A-ZÀ-Ý][\\p{L}'’\\-]+(?:\\s+(?:de|van|von|der|du|le|la|di)\\s+[A-ZÀ-Ý][\\p{L}'’\\-]+)?";132  const re = new RegExp(`\\b(${name}(?:\\s*(?:,|and|&|et)\\s*${name})?(?:\\s+et\\s+al\\.?)?)\\s*[,(]?\\s*((?:19|20)\\d{2}[a-z]?)\\)?`, "gu");133  for (const m of text.matchAll(re)) { const c = `${m[1].replace(/\s+/g, " ")} (${m[2]})`; const id = "a:" + deaccent(c); if (seen.has(id)) continue; if (/^(Table|Figure|Section|Equation|Panel|Appendix|Working|Paper|Journal|Review|In|On|The|See|Since|During|From|Of|And|For)\b/.test(m[1])) continue; seen.add(id); out.push({ citation: c, kind: "author-year", context: ctx(m.index ?? 0) }); }134  return out.slice(0, 80);135}136function matchDeterministic(c: { citation: string; kind: "cite" | "author-year" }, refs: ReferenceRow[]): { status: CitationHit["status"]; ref: ReferenceRow | null; candidates: ReferenceRow[] } {137  if (c.kind === "cite") { const key = c.citation.slice(6, -1).toLowerCase(); const ref = refs.find((r) => r.citeKey.toLowerCase() === key) ?? null; return { status: ref ? "matched" : "unverified", ref, candidates: [] }; }138  const m = c.citation.match(/^(.*)\s\((\d{4})[a-z]?\)$/); if (!m) return { status: "unverified", ref: null, candidates: [] };139  const year = Number(m[2]); const names = m[1].replace(/\bet al\.?/i, "").split(/\s*(?:,|and|&|et)\s*/).map(deaccent).filter(Boolean);140  const cands = refs.filter((r) => r.year === year && names.every((n) => (r.authors ?? []).some((a) => deaccent(lastName(a)) === n || deaccent(a).includes(n))));141  if (cands.length === 1) return { status: "matched", ref: cands[0], candidates: cands }; if (cands.length > 1) return { status: "ambiguous", ref: null, candidates: cands };142  const loose = refs.filter((r) => Math.abs((r.year ?? 0) - year) <= 1 && names.some((n) => (r.authors ?? []).some((a) => deaccent(lastName(a)) === n)));143  return { status: "unverified", ref: null, candidates: loose };144}145async function* runRefcheck(wsId: string, r: ResearchAIRequest, signal?: AbortSignal): AsyncGenerator<ResearchAIEvent> {146  const page = await pageOf(wsId, r.pageId); const refs = await listReferences(wsId, { pageId: r.pageId });147  const allRefs = refs.length ? refs : await listReferences(wsId); // fall back to the whole library when the page has none148  const text = r.text?.trim() || page.plainText || toMarkdown(page.content as PMNode);149  const found = extractCitations(text);150  const items: CitationHit[] = found.map((c) => { const d = matchDeterministic(c, allRefs); return { ...c, status: d.status, citeKey: d.ref?.citeKey ?? null, referenceId: d.ref?.id ?? null, note: d.status === "matched" ? "exact match" : d.status === "ambiguous" ? "several stored references match" : "no stored reference matches", candidates: d.candidates.map((x) => x.citeKey) }; });151  const toJudge = items.filter((i) => i.status !== "matched" || i.kind === "author-year");152  let meta: ResearchAIEvent & { type: "meta" } = { type: "meta", feature: "refcheck", model: "deterministic", provider: "local", reason: "no model call needed", context: { references: allRefs.length, attachments: 0, sources: 0, pageChars: text.length } };153  let done: ResearchAIEvent & { type: "done" } = { type: "done", usage: null, costUsd: null, latencyMs: 0 };154  if (toJudge.length && allRefs.length) {155    const keys = new Set(allRefs.map((x) => x.citeKey));156    const p: Prepared = { task: "reason", context: meta.context, system: sys("Task: reference checking. You decide, for each citation found in the author's text, whether it refers to one of the STORED references. Rules: a citation is matched only when author surnames and year agree with a stored reference (a one-year difference is acceptable only if the surnames match exactly and you say so in the note). Never propose a cite key that is not in the stored list. When unsure, answer unverified.", r.language), user: [ws("stored-references", allRefs.map((x) => refLine(x, false)).join("\n")), ws("citations-found", toJudge.map((c, i) => `#${i} ${c.citation} [${c.kind}; pre-check: ${c.status}${c.candidates?.length ? `; candidates: ${c.candidates.join(", ")}` : ""}]\n  context: …${c.context}…`).join("\n")), `Return JSON {"items":[{"index":number,"status":"matched"|"unverified","citeKey":string|null,"note":string}]} with one item per citation (#index).`].join("\n\n") };157    try {158      const res = await jsonCall<{ items: { index: number; status: "matched" | "unverified"; citeKey: string | null; note: string }[] }>(wsId, "refcheck", p, "refcheck", { type: "object", additionalProperties: false, required: ["items"], properties: { items: { type: "array", items: { type: "object", additionalProperties: false, required: ["index", "status", "citeKey", "note"], properties: { index: { type: "integer" }, status: { type: "string", enum: ["matched", "unverified"] }, citeKey: { type: ["string", "null"] }, note: { type: "string" } } } } } }, r.model, signal);159      meta = res.meta; done = res.done;160      for (const it of res.data.items ?? []) {161        const target = toJudge[it.index]; if (!target) continue;162        const ok = it.status === "matched" && it.citeKey && keys.has(it.citeKey); const ref = ok ? allRefs.find((x) => x.citeKey === it.citeKey)! : null;163        if (target.kind === "cite" && target.status === "matched") { target.note = it.note || target.note; continue; } // exact key match always wins164        target.status = ok ? "matched" : "unverified"; target.citeKey = ref?.citeKey ?? null; target.referenceId = ref?.id ?? null;165        target.note = (ok ? "" : "unverified — ") + (it.note || (ok ? "matched by the model" : "no stored reference matches")) + (it.status === "matched" && !ok ? " (model proposed an unknown key; rejected)" : "");166      }167    } catch (e: any) { meta = { ...meta, reason: `model unavailable, deterministic check only (${String(e?.message ?? e).slice(0, 120)})` }; }168  }169  for (const i of items) if (i.status === "ambiguous") { i.status = "unverified"; i.note = `unverified — ambiguous between ${i.candidates?.join(", ")}`; }170  yield meta;171  const matched = items.filter((i) => i.status === "matched"); const unverified = items.filter((i) => i.status !== "matched");172  const md = [`**Reference check** — ${found.length} citation${found.length === 1 ? "" : "s"} found in the page, ${matched.length} matched to stored references, ${unverified.length} unverified. Checked against ${allRefs.length} stored reference${allRefs.length === 1 ? "" : "s"}${refs.length ? " of this page" : " of the whole library (this page has none)"}.`,173    unverified.length ? `### Unverified (${unverified.length})\n| Citation | Note | Context |\n|---|---|---|\n${unverified.map((i) => `| ${i.citation.replace(/\|/g, "\\|")} | ${i.note.replace(/\|/g, "\\|")} | …${i.context.replace(/\|/g, "\\|").slice(0, 140)}… |`).join("\n")}` : "No unverified citations.",174    matched.length ? `### Matched (${matched.length})\n| Citation | Stored reference | Note |\n|---|---|---|\n${matched.map((i) => `| ${i.citation.replace(/\|/g, "\\|")} | \\cite{${i.citeKey}} | ${i.note.replace(/\|/g, "\\|")} |`).join("\n")}` : "",175    found.length ? "" : "_No citation-like strings (\\cite{…} or Author (Year)) were found in the page text._"].filter(Boolean).join("\n\n");176  yield { type: "text", delta: md }; yield { type: "result", data: { items, matched: matched.length, unverified: unverified.length, found: found.length } }; yield done;177}178179// ---------- experimental planning180export interface PlanData { title: string; objective: string; hypotheses: string[]; design: string; data: string[]; steps: { title: string; description: string; deliverable: string }[]; tasks: { title: string; priority: "low" | "medium" | "high"; dueInDays: number | null }[]; risks: string[] }181async function* runPlan(wsId: string, r: ResearchAIRequest, signal?: AbortSignal): AsyncGenerator<ResearchAIEvent> {182  const page = await pageOf(wsId, r.pageId); const refs = await listReferences(wsId, { pageId: r.pageId });183  const exps = await db.select().from(schema.experiments).where(eq(schema.experiments.pageId, r.pageId));184  const md = toMarkdown(page.content as PMNode).slice(0, 15_000);185  const p: Prepared = { task: "deep", maxTokens: 6000, context: { references: refs.length, attachments: 0, sources: 0, pageChars: md.length }, system: sys("Task: experimental / empirical planning for a working paper. Produce a concrete, sequenced plan whose tasks can be executed by the author; tasks must be specific (what, with which data/tool, expected output).", r.language), user: [ws("current-page", md, `title="${esc(page.title)}"`), refs.length ? ws("references", refs.map((x) => refLine(x, false)).join("\n")) : "", exps.length ? ws("existing-experiments", exps.map((e) => `- ${e.name} [${e.status}] params=${JSON.stringify(e.params)} metrics=${JSON.stringify(e.metrics)}`).join("\n")) : "", `Plan the experiments/empirical work for this paper${r.instruction ? `: ${r.instruction}` : ""}. Return JSON {"title","objective","hypotheses":[…],"design":"…","data":[…],"steps":[{"title","description","deliverable"}],"tasks":[{"title","priority":"low|medium|high","dueInDays":number|null}],"risks":[…]}; 5–12 steps, 6–15 tasks ordered by execution, Markdown/LaTeX allowed inside strings.`].filter(Boolean).join("\n\n") };186  const schemaDef = { type: "object", additionalProperties: false, required: ["title", "objective", "hypotheses", "design", "data", "steps", "tasks", "risks"], properties: { title: { type: "string" }, objective: { type: "string" }, hypotheses: { type: "array", items: { type: "string" } }, design: { type: "string" }, data: { type: "array", items: { type: "string" } }, steps: { type: "array", items: { type: "object", additionalProperties: false, required: ["title", "description", "deliverable"], properties: { title: { type: "string" }, description: { type: "string" }, deliverable: { type: "string" } } } }, tasks: { type: "array", items: { type: "object", additionalProperties: false, required: ["title", "priority", "dueInDays"], properties: { title: { type: "string" }, priority: { type: "string", enum: ["low", "medium", "high"] }, dueInDays: { type: ["integer", "null"] } } } }, risks: { type: "array", items: { type: "string" } } } };187  const res = await jsonCall<PlanData>(wsId, "plan", p, "experimental_plan", schemaDef, r.model, signal);188  yield res.meta;189  const d = res.data;190  const md2 = [`## ${d.title}`, `**Objective.** ${d.objective}`, d.hypotheses?.length ? `### Hypotheses\n${d.hypotheses.map((h, i) => `- **H${i + 1}.** ${h}`).join("\n")}` : "", `### Design\n${d.design}`, d.data?.length ? `### Data\n${d.data.map((x) => `- ${x}`).join("\n")}` : "", `### Steps\n${(d.steps ?? []).map((s, i) => `${i + 1}. **${s.title}** — ${s.description} _Deliverable: ${s.deliverable}_`).join("\n")}`, `### Proposed tasks\n${(d.tasks ?? []).map((t) => `- [ ] ${t.title} (${t.priority}${t.dueInDays != null ? `, ~${t.dueInDays} d` : ""})`).join("\n")}`, d.risks?.length ? `### Risks\n${d.risks.map((x) => `- ${x}`).join("\n")}` : ""].filter(Boolean).join("\n\n");191  yield { type: "text", delta: md2 }; yield { type: "result", data: d }; yield res.done;192}193194/** Entry point used by POST /api/research/ai. */195export async function* researchAI(wsId: string, r: ResearchAIRequest, signal?: AbortSignal): AsyncGenerator<ResearchAIEvent> {196  try {197    switch (r.feature) {198      case "synthesis": yield* streamText(wsId, r.feature, await prepSynthesis(wsId, r), r.model, signal); return;199      case "compare": yield* streamText(wsId, r.feature, await prepCompare(wsId, r), r.model, signal); return;200      case "question": yield* streamText(wsId, r.feature, await prepQuestion(wsId, r), r.model, signal); return;201      case "equation": yield* streamText(wsId, r.feature, await prepEquation(wsId, r), r.model, signal); return;202      case "draft": yield* streamText(wsId, r.feature, await prepDraft(wsId, r), r.model, signal); return;203      case "refcheck": yield* runRefcheck(wsId, r, signal); return;204      case "plan": yield* runPlan(wsId, r, signal); return;205      default: throw new HttpError(400, "Unknown feature");206    }207  } catch (e: any) { if (signal?.aborted) return; yield { type: "error", message: String(e?.message ?? e).slice(0, 500) }; }208}209