Repository Public Research preview
1// Combined audit, referee-report scaffold and version comparison.2import { DISCLAIMER, ROLES, auditCitations, auditClaims, auditCrossReferences, auditNumbers, auditReplication, auditStatistics, auditStyle, auditTables, extractNumbers, outline } from "./audits.mjs";3import { lineAt } from "./parse.mjs";45export const ENGINE_VERSION = "1.0.0";67const ORDER = { error: 0, warning: 1, info: 2 };89export function fullAudit(doc, opts = {}) {10 const parts = [outline(doc, opts), auditCitations(doc), auditCrossReferences(doc), auditNumbers(doc, opts), auditStatistics(doc), auditTables(doc), auditClaims(doc), auditStyle(doc, opts), auditReplication(doc)];11 const findings = parts.flatMap((p) => p.findings.map((f) => ({ ...f, audit: p.tool })));12 findings.sort((a, b) => ORDER[a.severity] - ORDER[b.severity] || a.location.file.localeCompare(b.location.file) || a.location.line - b.location.line || a.code.localeCompare(b.code));13 const count = (list, s) => list.filter((f) => f.severity === s).length;14 const by_role = {};15 for (const role of Object.keys(ROLES)) {16 const list = findings.filter((f) => f.role === role);17 by_role[role] = { description: ROLES[role], errors: count(list, "error"), warnings: count(list, "warning"), infos: count(list, "info") };18 }19 const by_code = {};20 for (const f of findings) by_code[f.code] = (by_code[f.code] ?? 0) + 1;21 const weight = { error: 6, warning: 2, info: 0.5 };22 const penalty = findings.reduce((a, f) => a + weight[f.severity] * (f.heuristic ? 0.5 : 1), 0);23 const o = parts[0];24 return {25 tool: "full_audit", file: doc.file, format: doc.format, title: doc.title,26 summary: {27 total: findings.length, errors: count(findings, "error"), warnings: count(findings, "warning"), infos: count(findings, "info"),28 heuristic: findings.filter((f) => f.heuristic).length,29 hygiene_score: Math.max(0, Math.round(100 - penalty)),30 hygiene_score_meaning: "100 minus 6 per error, 2 per warning and 0.5 per note (heuristic findings count half), floored at 0. It measures how clean the text is on these mechanical checks. It is not a quality, validity or publishability score.",31 },32 by_role, by_code,33 by_audit: Object.fromEntries(parts.map((p) => [p.tool, p.summary])),34 manuscript: { abstract: o.abstract, totals: o.totals, sections: o.sections, unresolved_includes: o.unresolved_includes },35 findings,36 disclaimer: DISCLAIMER,37 };38}3940const QUESTIONS = {41 "num-abstract-not-in-body": (f) => `Where in the paper is the value ${f.value} (abstract, line ${f.location.line}) reported or derived?`,42 "num-inconsistent-count": (f) => `Which sample size is correct, and what explains the difference (${(f.values ?? []).map((v) => `${v.value} at line ${v.line}`).join("; ")})?`,43 "num-inconsistent-statistic": (f) => `The same statistic is reported with different values (${(f.values ?? []).map((v) => `${v.value} at line ${v.line}`).join("; ")}). Which one is correct?`,44 "num-percent-count-mismatch": (f) => `Line ${f.location.line}: ${f.numerator} of ${f.denominator} is ${f.computed.toFixed(1)}%, yet ${f.stated}% is stated. Which figure is wrong?`,45 "stat-p-without-test": (f) => `Line ${f.location.line}: which test produced this p-value, with what statistic and degrees of freedom?`,46 "stat-n-not-reported": () => "What is the sample size behind each reported test?",47 "stat-no-uncertainty": (f) => `Table at line ${f.location.line}: please report standard errors or confidence intervals for each estimate.`,48 "stat-causal-language": (f) => `Line ${f.location.line}: what design supports the causal reading, given that the data are described as observational?`,49 "claim-priority-unsupported": (f) => `Line ${f.location.line}: which literature search supports the claim "${f.term}"? Please cite the closest prior work.`,50 "claim-robustness-unsupported": (f) => `Line ${f.location.line}: which table or figure shows the robustness that is asserted?`,51 "repl-no-data-statement": () => "How can a reader obtain the data, and under what restrictions?",52 "repl-no-code-statement": () => "Will the analysis code or a replication package be made available, and where?",53 "repl-no-seed": () => "Which random seeds were used, so that the stochastic results can be reproduced exactly?",54};5556export function reviewScaffold(doc, opts = {}) {57 const cap = opts.max_items_per_section ?? 15;58 const withInfo = opts.include_info ?? false;59 const a = fullAudit(doc, opts);60 const line = (f, k) => `${k}. **${f.code}**${f.heuristic ? " (heuristic)" : ""}, ${f.location.file}:${f.location.line}. ${f.message}${f.evidence ? `\n Evidence: \`${f.evidence.replace(/`/g, "'")}\`` : ""}`;61 const block = (list) => {62 if (!list.length) return "_No deterministic finding in this category. That is not evidence that nothing is wrong._";63 const shown = list.slice(0, cap).map((f, k) => line(f, k + 1)).join("\n");64 return list.length > cap ? `${shown}\n\n_(${list.length - cap} more not shown: run full_audit for the complete list.)_` : shown;65 };66 const errors = a.findings.filter((f) => f.severity === "error");67 const rest = a.findings.filter((f) => f.severity === "warning" || (withInfo && f.severity === "info"));68 const role = (r) => rest.filter((f) => f.role === r);69 const questions = [];70 const asked = new Set();71 for (const f of a.findings) {72 const q = QUESTIONS[f.code];73 if (!q) continue;74 const text = q(f);75 if (asked.has(text)) continue;76 asked.add(text);77 questions.push(text);78 }79 const t = a.manuscript.totals;80 const md = [81 `# Referee report: ${doc.title ?? doc.file}`,82 "",83 `_Scaffold produced by manuscript-audit ${ENGINE_VERSION} from deterministic text checks. Every part marked [REVIEWER] must be written by a person or an agent who has read the manuscript. No item below has been checked against the science, and each one must be confirmed in the source before it is sent to authors._`,84 "",85 "## Manuscript facts (computed)",86 "",87 `- Source: ${doc.file} (${doc.format})`,88 `- Length: ${t.words} words in the body, ${a.manuscript.abstract.words} in the abstract, ${t.sections} sections`,89 `- Exhibits: ${t.tables} tables, ${t.figures} figures, ${t.numbered_equations} numbered equations`,90 `- Citations: ${t.citation_occurrences} occurrences, ${t.distinct_cited_keys} distinct keys, ${t.bibliography_entries} bibliography entries`,91 `- Mechanical findings: ${a.summary.errors} errors, ${a.summary.warnings} warnings, ${a.summary.infos} notes (hygiene score ${a.summary.hygiene_score}/100, not a quality score)`,92 "",93 "## 1. Summary of the submission",94 "",95 "[REVIEWER] Three to five sentences in your own words: the question, the data and method, the main result, the contribution the authors claim.",96 "",97 "## 2. Assessment",98 "",99 "[REVIEWER] Contribution and novelty relative to the closest prior work.",100 "",101 "[REVIEWER] Soundness of the method or identification, and whether the evidence supports each main claim.",102 "",103 "[REVIEWER] Clarity of exposition.",104 "",105 "## 3. Major comments",106 "",107 "[REVIEWER] Number each comment. State what is wrong or not shown, where (section, table, line), why it matters for the conclusions, and what would resolve it.",108 "",109 "Candidates from the deterministic audit (errors). Keep only those you confirm matter:",110 "",111 block(errors),112 "",113 "## 4. Minor comments",114 "",115 "### Method, statistics and claims",116 "",117 block(role("referee")),118 "",119 "### Tables, figures and cross-references",120 "",121 block(role("table_figure_editor")),122 "",123 "### Citations and internal consistency",124 "",125 block(role("integrity_auditor")),126 "",127 "### Replication and data",128 "",129 block(role("replication_auditor")),130 "",131 "### Structure and style",132 "",133 block(role("editor")),134 "",135 "## 5. Questions for the authors",136 "",137 questions.length ? questions.slice(0, cap * 2).map((q, k) => `${k + 1}. ${q}`).join("\n") : "[REVIEWER] List the questions whose answers would change your assessment.",138 "",139 "## 6. Recommendation",140 "",141 "[REVIEWER] Accept, minor revision, major revision or reject, with the two or three reasons that drive the recommendation. The mechanical findings above do not justify a recommendation on their own.",142 "",143 "## What this scaffold did not check",144 "",145 "- Whether the method is valid, the identification credible or the conclusions warranted.",146 "- Whether any cited work exists or says what the manuscript claims.",147 "- Figures, images, supplementary files, code and data.",148 "- Whether the numbers in the tables are correct: only their consistency within the text.",149 "",150 ].join("\n");151 return { tool: "review_scaffold", file: doc.file, format: doc.format, summary: a.summary, by_role: a.by_role, questions, markdown: md, disclaimer: DISCLAIMER };152}153154// ------------------------------------------------------------------ compare155156const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();157const loc = (doc, i) => ({ file: doc.L[i].file, line: doc.L[i].line });158const clip = (s) => (s.length > 220 ? `${s.slice(0, 217)}...` : s);159160function sentenceTemplates(doc) {161 const map = new Map();162 for (const s of doc.sentences) {163 const nums = extractNumbers(s.text);164 if (!nums.length) continue;165 let tpl = "", last = 0;166 for (const n of nums) { tpl += `${s.text.slice(last, n.pos)}#`; last = n.pos + n.len; }167 tpl = norm(tpl + s.text.slice(last)).replace(/ +/g, " ");168 if (tpl.split(" ").length < 5) continue;169 if (!map.has(tpl)) map.set(tpl, []);170 map.get(tpl).push({ s, nums });171 }172 return map;173}174175export function compareVersions(oldDoc, newDoc) {176 const secs = (d) => d.sections.filter((s) => !s.isTitle);177 const oldS = new Map(), newS = new Map();178 for (const s of secs(oldDoc)) if (!oldS.has(norm(s.title))) oldS.set(norm(s.title), s);179 for (const s of secs(newDoc)) if (!newS.has(norm(s.title))) newS.set(norm(s.title), s);180 const added = [], removed = [], kept = [];181 for (const [k, s] of newS) {182 if (!oldS.has(k)) added.push({ title: s.title, level: s.level, ...loc(newDoc, s.i), words: s.wordsTotal });183 else { const o = oldS.get(k); kept.push({ title: s.title, level: s.level, words_old: o.wordsTotal, words_new: s.wordsTotal, delta: s.wordsTotal - o.wordsTotal, old: loc(oldDoc, o.i), new: loc(newDoc, s.i) }); }184 }185 for (const [k, s] of oldS) if (!newS.has(k)) removed.push({ title: s.title, level: s.level, ...loc(oldDoc, s.i), words: s.wordsTotal });186187 const ta = sentenceTemplates(oldDoc), tb = sentenceTemplates(newDoc);188 const changed = [];189 for (const [tpl, bs] of tb) {190 const as = ta.get(tpl);191 if (!as || as.length !== 1 || bs.length !== 1) continue;192 const a = as[0], b = bs[0];193 if (a.nums.length !== b.nums.length) continue;194 const pairs = [];195 for (let k = 0; k < a.nums.length; k++) if (a.nums[k].raw !== b.nums[k].raw) pairs.push({ old: a.nums[k].raw, new: b.nums[k].raw, k });196 for (const p of pairs) {197 changed.push({198 old: p.old, new: p.new,199 old_location: loc(oldDoc, lineAt(a.s, a.nums[p.k].pos)), new_location: loc(newDoc, lineAt(b.s, b.nums[p.k].pos)),200 in_abstract: !!b.s.para.abs, old_sentence: clip(a.s.text), new_sentence: clip(b.s.text),201 });202 }203 }204 const absNums = (d) => { const m = new Map(); for (const s of d.sentences) if (s.para.abs) for (const n of extractNumbers(s.text)) if (!n.year) m.set(n.raw, loc(d, lineAt(s, n.pos))); return m; };205 const an = absNums(oldDoc), bn = absNums(newDoc);206 const absText = (d) => d.sentences.filter((s) => s.para.abs).map((s) => s.text).join(" ");207208 const keys = (d) => { const m = new Map(); for (const c of d.cites) if (c.key && !m.has(c.key)) m.set(c.key, c.i); for (const c of d.numCites) if (!m.has(`[${c.n}]`)) m.set(`[${c.n}]`, c.i); return m; };209 const ka = keys(oldDoc), kb = keys(newDoc);210 const labs = (d) => { const m = new Map(); for (const l of d.labels) if (!m.has(l.name)) m.set(l.name, l); return m; };211 const la = labs(oldDoc), lb = labs(newDoc);212 const diff = (x, y, d, f) => [...x].filter(([k]) => !y.has(k)).map(([k, v]) => f(k, v, d));213 const out = {214 tool: "compare_versions",215 old: { file: oldDoc.file, format: oldDoc.format, words: oldDoc.words, abstract_words: oldDoc.abstract?.words ?? 0, sections: secs(oldDoc).length },216 new: { file: newDoc.file, format: newDoc.format, words: newDoc.words, abstract_words: newDoc.abstract?.words ?? 0, sections: secs(newDoc).length },217 sections: { added, removed, changed_length: kept.filter((k) => k.delta !== 0), unchanged: kept.filter((k) => k.delta === 0).length },218 abstract: {219 changed: absText(oldDoc) !== absText(newDoc),220 numbers_added: [...bn].filter(([k]) => !an.has(k)).map(([value, l]) => ({ value, ...l })),221 numbers_removed: [...an].filter(([k]) => !bn.has(k)).map(([value, l]) => ({ value, ...l })),222 },223 numbers: { changed },224 citations: { added: diff(kb, ka, newDoc, (key, i, d) => ({ key, ...loc(d, i) })), removed: diff(ka, kb, oldDoc, (key, i, d) => ({ key, ...loc(d, i) })) },225 labels: { added: diff(lb, la, newDoc, (name, l, d) => ({ label: name, kind: l.kind, ...loc(d, l.i) })), removed: diff(la, lb, oldDoc, (name, l, d) => ({ label: name, kind: l.kind, ...loc(d, l.i) })) },226 note: "A changed number is detected when a sentence is word-for-word identical in both versions once numbers are masked, and occurs once in each. Numbers in reworded sentences and in tables are not paired: compare those by hand. Sections are matched by title.",227 disclaimer: DISCLAIMER,228 };229 out.summary = {230 sections_added: added.length, sections_removed: removed.length, sections_changed_length: out.sections.changed_length.length,231 numbers_changed: changed.length, abstract_changed: out.abstract.changed,232 citations_added: out.citations.added.length, citations_removed: out.citations.removed.length,233 labels_added: out.labels.added.length, labels_removed: out.labels.removed.length,234 word_delta: newDoc.words - oldDoc.words,235 };236 return out;237}238