1"""Deterministic manuscript audits: LaTeX/Markdown loading, tables/equations/citations checks, numeric preservation, corpus overlap,2unsupported-claim flags, evidence lookup. No LLM calls here."""3from __future__ import annotations4import re, json, hashlib5from pathlib import Path6import numpy as np7from . import config, segmentation8from .provenance import stamp, now_iso910NUM_RE = re.compile(r"(?<![A-Za-z\\])[-−]?\d{1,3}(?:[,\d]{0,12})?(?:\.\d+)?%?")11CITE_RE = re.compile(r"\\(?:cite|citep|citet|citealp|citealt|citeauthor|citeyear|parencite|textcite|autocite)\*?(?:\[[^\]]*\])*\{([^}]+)\}")12CLAIM_VERBS = re.compile(r"\b(causes?|caused|leads? to|led to|results? in|drives?|increases?|decreases?|reduces?|raises?|lowers?|improves?|shows? that|demonstrates? that|proves? that|establishes? that|confirms? that|we find that|significantly)\b", re.I)13HEDGES = re.compile(r"\b(may|might|could|suggests?|appears?|consistent with|likely|possibly|tend to|we conjecture|we hypothesize)\b", re.I)1415# ---------------- manuscript loading ----------------16def _strip_comments(t: str) -> str: return re.sub(r"(?<!\\)%.*", "", t)1718def _resolve_inputs(text: str, base: Path, depth=0) -> str:19 def rep(m):20 name = m.group(2).strip(); p = base / name21 for cand in (p, p.with_suffix(".tex"), base / "paper" / name, (base / "paper" / name).with_suffix(".tex")):22 if cand.exists() and depth < 6: return "\n" + _resolve_inputs(_strip_comments(cand.read_text(errors="ignore")), cand.parent, depth + 1) + "\n"23 return f"\n% [missing input: {name}]\n"24 return re.sub(r"\\(input|include)\{([^}]+)\}", rep, text)2526def load_manuscript(path: Path) -> dict:27 """Load a paper from a directory (main.tex / paper/main.tex / *.md) or a single .tex/.md file. Returns text + extracted structure."""28 path = Path(path).expanduser().resolve(); root = path if path.is_dir() else path.parent29 main = None30 if path.is_file(): main = path31 else:32 for cand in ["paper/main.tex", "main.tex", "manuscript/main.tex", "paper/paper.tex"]:33 if (path / cand).exists(): main = path / cand; break34 if main is None:35 texs = sorted(path.rglob("*.tex"), key=lambda p: -p.stat().st_size); mds = sorted(path.glob("*.md"))36 main = texs[0] if texs else (mds[0] if mds else None)37 if main is None: raise FileNotFoundError(f"no .tex/.md manuscript found under {path}")38 raw = main.read_text(errors="ignore"); kind = "latex" if main.suffix == ".tex" else "markdown"39 text = _resolve_inputs(_strip_comments(raw), main.parent) if kind == "latex" else raw40 # macros defined in main (UQO template: \WPtitle etc.)41 macros = {m.group(1): m.group(2) for m in re.finditer(r"\\newcommand\{\\(WP\w+)\}\{((?:[^{}]|\{[^{}]*\})*)\}", text)}42 title = macros.get("WPtitle") or (re.search(r"\\title\{([^}]+)\}", text) or re.search(r"^#\s+(.+)$", text, re.M) or [None, None])[1]43 abstract = macros.get("WPabstract") or (re.search(r"\\begin\{abstract\}(.*?)\\end\{abstract\}", text, re.S) or [None, None])[1]44 if abstract: abstract = re.sub(r"\s+", " ", abstract.replace("%", "")).strip()45 sections = []46 if kind == "latex":47 parts = re.split(r"(\\(?:section|subsection)\*?\{[^}]+\})", text)48 cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}49 for i in range(1, len(parts), 2):50 m = re.match(r"\\(section|subsection)\*?\{([^}]+)\}", parts[i]); sections.append(cur)51 cur = {"heading": m.group(2), "level": 1 if m.group(1) == "section" else 2, "text": parts[i + 1] if i + 1 < len(parts) else ""}52 sections.append(cur)53 else:54 parts = re.split(r"^(#{1,3}\s+.+)$", text, flags=re.M); cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}55 for i in range(1, len(parts), 2):56 lvl = max(len(parts[i]) - len(parts[i].lstrip("#")) - 1, 0) # '#' = document title (level 0), '##' = section (1), '###' = subsection (2)57 sections.append(cur); cur = {"heading": parts[i].lstrip("# ").strip(), "level": lvl if lvl > 0 else 0, "text": parts[i + 1] if i + 1 < len(parts) else ""}58 sections.append(cur)59 # level-1 sections own the text of their subsections60 for i, s in enumerate(sections):61 if s["level"] == 1:62 j = i + 1; full = [s["text"]]63 while j < len(sections) and sections[j]["level"] > 1: full.append(f"\\subsection{{{sections[j]['heading']}}}\n" + sections[j]["text"]); j += 164 s["full_text"] = "\n".join(full)65 else: s["full_text"] = s["text"]66 if kind == "markdown" and not abstract:67 ab = next((s for s in sections if s["heading"].strip().lower() == "abstract"), None)68 if ab: abstract = re.sub(r"\s+", " ", ab["full_text"]).strip(); ab["role_override"] = "abstract"69 for s in sections:70 s["role"] = s.get("role_override") or segmentation.section_role(s["heading"], is_appendix=bool(re.search(r"appendix", s["heading"], re.I)), is_intro=bool(re.search(r"^introduction", s["heading"], re.I)))71 if kind == "markdown" and s["heading"].strip().lower() in ("tables", "figures", "references"): s["role"] = s["heading"].strip().lower()72 s["n_words"] = len(re.sub(r"\\[a-zA-Z]+\*?(\[[^\]]*\])?(\{[^}]*\})?", " ", s["full_text"]).split())73 tables = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "has_notes": bool(re.search(r"tablenotes|\\footnotesize|\\note", b)), "n_numbers": len(NUM_RE.findall(b)), "body": b} for b in re.findall(r"\\begin\{table\*?\}(.*?)\\end\{table\*?\}", text, re.S)]74 figures = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1]} for b in re.findall(r"\\begin\{figure\*?\}(.*?)\\end\{figure\*?\}", text, re.S)]75 equations = [{"label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "body": b.strip()[:200]} for b in re.findall(r"\\begin\{(?:equation|align|gather|multline)\*?\}(.*?)\\end\{(?:equation|align|gather|multline)\*?\}", text, re.S)]76 if kind == "markdown":77 for m in re.finditer(r"\*\*(Table [^*]+)\*\*\s*[—-]+\s*(.*?)\n\s*\nNotes:\s*(.*?)(?=\n\s*\n\*\*|\n## |\Z)", text, re.S):78 tables.append({"caption": f"{m.group(1)}: {m.group(2).strip()}", "label": m.group(1).strip(), "has_notes": True, "n_numbers": len(NUM_RE.findall(m.group(3))), "body": m.group(0)})79 for m in re.finditer(r"\*\*(Figure [^*]+)\*\*\s*[—-]+\s*(.*?)(?=\n\s*\n|\Z)", text, re.S):80 figures.append({"caption": f"{m.group(1)}: {m.group(2).strip()[:300]}", "label": m.group(1).strip()})81 labels = set(re.findall(r"\\label\{([^}]+)\}", text)); refs = set(re.findall(r"\\(?:ref|eqref|autoref|cref|Cref)\{([^}]+)\}", text))82 refs = {x.strip() for r in refs for x in r.split(",")}83 cites = [k.strip() for m in CITE_RE.findall(text) for k in m.split(",")]84 bib_keys, bib_dois = set(), {}85 for bibp in list(root.rglob("*.bib"))[:5]:86 bt = bibp.read_text(errors="ignore")87 for m in re.finditer(r"@\w+\{([^,\s]+),(.*?)(?=\n@|\Z)", bt, re.S):88 bib_keys.add(m.group(1)); d = re.search(r"doi\s*=\s*[{\"]([^}\"]+)", m.group(2), re.I)89 if d: bib_dois[m.group(1)] = d.group(1).strip()90 body_text = "\n".join(s["text"] for s in sections if s["level"] > 0 and s["role"] != "appendix")91 return {"path": str(main), "root": str(root), "kind": kind, "title": title, "abstract": abstract, "macros": {k: v for k, v in macros.items() if k in ("WPnumber", "WPkeywords", "WPjel", "WPdate", "WPversion")},92 "sections": sections, "tables": tables, "figures": figures, "equations": equations, "labels": sorted(labels), "refs": sorted(refs), "cites": cites, "bib_keys": sorted(bib_keys), "bib_dois": bib_dois,93 "n_words_body": len(body_text.split()), "text": text, "sha256": hashlib.sha256(text.encode()).hexdigest()}9495def section_text(ms: dict, name: str) -> str | None:96 for s in ms["sections"]:97 if s["level"] == 1 and (s["role"] == name or name.lower() in s["heading"].lower()): return s["full_text"]98 return None99100# ---------------- audits ----------------101def numbers_in(text: str) -> set[str]:102 text = CITE_RE.sub(" ", text); text = re.sub(r"\\(?:label|ref|eqref|cref)\{[^}]*\}", " ", text)103 out = set()104 for n in NUM_RE.findall(text):105 n = n.replace(",", "").replace("−", "-").rstrip(".")106 if not re.search(r"\d", n) or re.fullmatch(r"-?(19|20)\d{2}", n) or re.fullmatch(r"-?0?[0-9]", n): continue # drop years and single digits (enumerations)107 out.add(n)108 return out109110def numeric_preservation(source: str, output: str) -> dict:111 a, b = numbers_in(source), numbers_in(output)112 return {"n_source": len(a), "n_output": len(b), "missing_in_output": sorted(a - b)[:50], "new_in_output": sorted(b - a)[:50], "preserved_rate": round(len(a & b) / max(len(a), 1), 3), "new_rate": round(len(b - a) / max(len(b), 1), 3)}113114def citation_check(output: str, bib_keys: set[str] | list[str]) -> dict:115 keys = [k.strip() for m in CITE_RE.findall(output) for k in m.split(",")]116 unknown = sorted({k for k in keys if k not in set(bib_keys)})117 # author-year patterns not backed by \cite are also flagged in LaTeX output118 loose = re.findall(r"\b([A-Z][A-Za-z\-]+(?: (?:and|&) [A-Z][A-Za-z\-]+)?(?: et al\.)?) \((19|20)\d{2}[a-z]?\)", output)119 return {"n_cite_commands": len(keys), "unknown_keys": unknown, "n_author_year_without_cite": len(loose) if "\\cite" in output or keys else len(loose), "loose_examples": [f"{a} ({y}…)" for a, y in loose[:5]]}120121def unsupported_claims(text: str) -> list[dict]:122 """Sentences with strong claim verbs that carry no citation, cross-reference or number and no hedge."""123 sents = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", text)); out = []124 for s in sents:125 if len(s) < 40 or not CLAIM_VERBS.search(s): continue126 supported = bool(CITE_RE.search(s) or re.search(r"\\(?:ref|eqref|cref|autoref)\{|Table|Figure|Section|Appendix|\d", s)) or bool(HEDGES.search(s))127 if not supported: out.append({"sentence": s[:300], "verb": CLAIM_VERBS.search(s).group(0)})128 return out129130@np.vectorize131def _h(s): return int(hashlib.blake2b(s.encode(), digest_size=8).hexdigest(), 16)132133def _shingles(text: str, n: int = 8) -> set[int]:134 w = re.findall(r"[a-z0-9]+", text.lower())135 return {int(hashlib.blake2b(" ".join(w[i:i + n]).encode(), digest_size=8).hexdigest(), 16) for i in range(max(len(w) - n + 1, 0))}136137def corpus_shingles() -> dict:138 """8-gram hash sets of every corpus document (built once, cached under data/derived/index/)."""139 p = config.ROOT / "data" / "derived" / "index" / "shingles.json"140 if p.exists(): return {k: set(v) for k, v in json.loads(p.read_text()).items()}141 out = {}142 for f in sorted((config.derived_dir() / "parsed").glob("*.json")):143 d = json.loads(f.read_text()); txt = (d["abstract"] or "") + "\n" + "\n".join(pp["text"] for s in d["sections"] for pp in s["paragraphs"])144 out[d["doc_id"]] = _shingles(txt)145 p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps({k: sorted(v) for k, v in out.items()}))146 return out147148def overlap_with_corpus(text: str, n: int = 8) -> dict:149 sh = _shingles(text, n)150 if not sh: return {"n_shingles": 0, "max_overlap_share": 0.0, "top_docs": []}151 cs = corpus_shingles(); hits = sorted(((len(sh & v) / len(sh), k) for k, v in cs.items()), reverse=True)[:3]152 return {"n_shingles": len(sh), "max_overlap_share": round(hits[0][0], 4) if hits else 0.0, "top_docs": [{"doc_id": k, "share": round(s, 4)} for s, k in hits if s > 0]}153154def audit_manuscript(path: Path, tables=True, equations=True, citations=True, overlap=True) -> dict:155 ms = load_manuscript(path); rep = {"manuscript": ms["path"], "title": ms["title"], "generated": stamp(), "n_words_body": ms["n_words_body"], "sections": [{"heading": s["heading"], "role": s["role"], "n_words": s["n_words"]} for s in ms["sections"] if s["level"] == 1], "issues": []}156 if tables:157 t_issues = []158 for i, t in enumerate(ms["tables"]):159 if not t["caption"]: t_issues.append(f"table {i+1}: no caption")160 if t["label"] and t["label"] not in ms["refs"]: t_issues.append(f"table {i+1} ({t['label']}): never referenced in text")161 if not t["has_notes"]: t_issues.append(f"table {i+1} ({t['label'] or 'no label'}): no notes block (JoF convention: notes state sample, units, standard errors)")162 rep["tables"] = {"n": len(ms["tables"]), "issues": t_issues}; rep["issues"] += t_issues163 f_issues = [f"figure {i+1} ({f['label']}): never referenced" for i, f in enumerate(ms["figures"]) if f["label"] and f["label"] not in ms["refs"]]164 rep["figures"] = {"n": len(ms["figures"]), "issues": f_issues}; rep["issues"] += f_issues165 if equations:166 e_issues = [f"equation {i+1} ({e['label']}): labelled but never referenced" for i, e in enumerate(ms["equations"]) if e["label"] and e["label"] not in ms["refs"]]167 missing = sorted(r for r in ms["refs"] if r not in set(ms["labels"])); e_issues += [f"\\ref to undefined label: {r}" for r in missing[:20]]168 rep["equations"] = {"n": len(ms["equations"]), "n_labelled": sum(1 for e in ms["equations"] if e["label"]), "issues": e_issues}; rep["issues"] += e_issues169 if citations:170 cc = citation_check(ms["text"], ms["bib_keys"]); cc["n_bib_entries"] = len(ms["bib_keys"]); cc["n_bib_with_doi"] = len(ms["bib_dois"])171 cc["doi_verification"] = verify_dois(ms["bib_dois"]) if ms["bib_dois"] else {"checked": 0}172 rep["citations"] = cc; rep["issues"] += [f"unknown citation key: {k}" for k in cc["unknown_keys"]]173 rep["unsupported_claims"] = unsupported_claims("\n".join(s["text"] for s in ms["sections"] if s["level"] > 0))[:30]174 if overlap: rep["corpus_overlap"] = overlap_with_corpus(ms["text"])175 return rep176177def verify_dois(bib_dois: dict, limit: int = 60) -> dict:178 """DOI resolution via Crossref (network; cached in data/manifests/crossref_cache.json)."""179 import httpx, time180 cache_p = config.manifests_dir() / "crossref_cache.json"; cache = json.loads(cache_p.read_text()) if cache_p.exists() else {}181 mailto = config.load("models")["crossref_mailto"]; res = {"checked": 0, "resolved": 0, "unresolved": []}182 with httpx.Client(timeout=20, headers={"User-Agent": f"jof250/0.1 (mailto:{mailto})"}) as cl:183 for key, doi in list(bib_dois.items())[:limit]:184 doi = doi.strip().lower().replace("https://doi.org/", "")185 if doi not in cache:186 try:187 r = cl.get(f"https://api.crossref.org/works/{doi}", params={"mailto": mailto}); cache[doi] = {"status": r.status_code, "title": (r.json()["message"].get("title") or [None])[0] if r.status_code == 200 else None, "fetched": now_iso()}188 except Exception as e: cache[doi] = {"status": "error", "error": str(e)[:100], "fetched": now_iso()}189 time.sleep(0.1)190 res["checked"] += 1191 if cache[doi].get("status") == 200: res["resolved"] += 1192 else: res["unresolved"].append({"key": key, "doi": doi, "status": cache[doi].get("status")})193 cache_p.write_text(json.dumps(cache, indent=1, ensure_ascii=False)); return res194195def evidence_for_claim(path: Path, claim: str) -> dict:196 ms = load_manuscript(path); nums = numbers_in(claim); words = [w for w in re.findall(r"[a-z]{4,}", claim.lower())][:12]197 hits = []198 for i, t in enumerate(ms["tables"]):199 body = t["body"]; found = [n for n in nums if n in body.replace(",", "")]; kw = sum(1 for w in words if w in body.lower())200 if found or kw >= 2: hits.append({"where": f"table {i+1} ({t['label'] or 'no label'})", "caption": (t["caption"] or "")[:120], "numbers_found": found, "keyword_hits": kw})201 for s in ms["sections"]:202 if s["level"] == 0: continue203 for sent in re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", s["text"])):204 found = [n for n in nums if n in sent.replace(",", "")]; kw = sum(1 for w in words if w in sent.lower())205 if (found and kw >= 1) or kw >= 4: hits.append({"where": f"section '{s['heading']}'", "sentence": sent[:240], "numbers_found": found, "keyword_hits": kw})206 status = "supported_by_table" if any(h["where"].startswith("table") and h["numbers_found"] for h in hits) else ("mentioned_in_text" if hits else "no_evidence_found")207 return {"claim": claim, "numbers_in_claim": sorted(nums), "status": status, "hits": hits[:12], "generated": stamp()}208209def render_report(rep: dict) -> str:210 L = [f"# Manuscript audit — {rep.get('title') or rep['manuscript']}", "", f"Generated {rep['generated']['timestamp']} (jof250 {rep['generated']['code_version']}). Body words: {rep['n_words_body']}.", "", "## Sections", ""]211 L += [f"- {s['heading']} — role `{s['role']}`, {s['n_words']} words" for s in rep["sections"]]212 if "tables" in rep: L += ["", f"## Tables ({rep['tables']['n']}) and figures ({rep['figures']['n']})", ""] + ([f"- {x}" for x in rep["tables"]["issues"] + rep["figures"]["issues"]] or ["- no issues"])213 if "equations" in rep: L += ["", f"## Equations ({rep['equations']['n']}, {rep['equations']['n_labelled']} labelled)", ""] + ([f"- {x}" for x in rep["equations"]["issues"]] or ["- no issues"])214 if "citations" in rep:215 c = rep["citations"]; L += ["", "## Citations", "", f"- \\cite commands: {c['n_cite_commands']}; bib entries: {c['n_bib_entries']} ({c['n_bib_with_doi']} with DOI)", f"- unknown keys: {', '.join(c['unknown_keys']) or 'none'}", f"- DOI check: {c['doi_verification']}"]216 L += ["", f"## Possibly unsupported claims ({len(rep['unsupported_claims'])})", ""] + ([f"- [{x['verb']}] {x['sentence']}" for x in rep["unsupported_claims"]] or ["- none flagged"])217 if "corpus_overlap" in rep: L += ["", "## Overlap with the JoF corpus (8-gram shingles)", "", f"- max share of manuscript shingles found in one corpus document: {rep['corpus_overlap']['max_overlap_share']} {rep['corpus_overlap']['top_docs']}"]218 return "\n".join(L) + "\n"219