1"""Deterministic manuscript audits: LaTeX/Markdown loading, tables/equations/citations checks, numeric preservation, corpus overlap,2unsupported-claim flags, evidence lookup. No LLM calls here."""3from __future__ import annotations4import re, json, hashlib5from pathlib import Path6import numpy as np7from . import config, segmentation8from .provenance import stamp, now_iso910NUM_RE = re.compile(r"(?<![A-Za-z\\])[-−]?\d{1,3}(?:[,\d]{0,12})?(?:\.\d+)?%?")11CITE_RE = re.compile(r"\\(?:cite|citep|citet|citealp|citealt|citeauthor|citeyear|parencite|textcite|autocite)\*?(?:\[[^\]]*\])*\{([^}]+)\}")12CLAIM_VERBS = re.compile(r"\b(causes?|caused|leads? to|led to|results? in|drives?|increases?|decreases?|reduces?|raises?|lowers?|improves?|shows? that|demonstrates? that|proves? that|establishes? that|confirms? that|we find that|significantly)\b", re.I)13HEDGES = re.compile(r"\b(may|might|could|suggests?|appears?|consistent with|likely|possibly|tend to|we conjecture|we hypothesize)\b", re.I)1415# ---------------- manuscript loading ----------------16def _strip_comments(t: str) -> str: return re.sub(r"(?<!\\)%.*", "", t)1718def _resolve_inputs(text: str, base: Path, depth=0) -> str:19 def rep(m):20 name = m.group(2).strip(); p = base / name21 for cand in (p, p.with_suffix(".tex"), base / "paper" / name, (base / "paper" / name).with_suffix(".tex")):22 if cand.exists() and depth < 6: return "\n" + _resolve_inputs(_strip_comments(cand.read_text(errors="ignore")), cand.parent, depth + 1) + "\n"23 return f"\n% [missing input: {name}]\n"24 return re.sub(r"\\(input|include)\{([^}]+)\}", rep, text)2526def load_manuscript(path: Path) -> dict:27 """Load a paper from a directory (main.tex / paper/main.tex / *.md) or a single .tex/.md file. Returns text + extracted structure."""28 path = Path(path).expanduser().resolve(); root = path if path.is_dir() else path.parent29 main = None30 if path.is_file(): main = path31 else:32 for cand in ["paper/main.tex", "main.tex", "manuscript/main.tex", "paper/paper.tex"]:33 if (path / cand).exists(): main = path / cand; break34 if main is None:35 texs = sorted(path.rglob("*.tex"), key=lambda p: -p.stat().st_size); mds = sorted(path.glob("*.md"))36 main = texs[0] if texs else (mds[0] if mds else None)37 if main is None: raise FileNotFoundError(f"no .tex/.md manuscript found under {path}")38 raw = main.read_text(errors="ignore"); kind = "latex" if main.suffix == ".tex" else "markdown"39 text = _resolve_inputs(_strip_comments(raw), main.parent) if kind == "latex" else raw40 # macros defined in main (UQO template: \WPtitle etc.)41 macros = {m.group(1): m.group(2) for m in re.finditer(r"\\newcommand\{\\(WP\w+)\}\{((?:[^{}]|\{[^{}]*\})*)\}", text)}42 title = macros.get("WPtitle") or (re.search(r"\\title\{([^}]+)\}", text) or re.search(r"^#\s+(.+)$", text, re.M) or [None, None])[1]43 abstract = macros.get("WPabstract") or (re.search(r"\\begin\{abstract\}(.*?)\\end\{abstract\}", text, re.S) or [None, None])[1]44 if abstract: abstract = re.sub(r"\s+", " ", abstract.replace("%", "")).strip()45 sections = []46 if kind == "latex":47 parts = re.split(r"(\\(?:section|subsection)\*?\{[^}]+\})", text)48 cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}49 for i in range(1, len(parts), 2):50 m = re.match(r"\\(section|subsection)\*?\{([^}]+)\}", parts[i]); sections.append(cur)51 cur = {"heading": m.group(2), "level": 1 if m.group(1) == "section" else 2, "text": parts[i + 1] if i + 1 < len(parts) else ""}52 sections.append(cur)53 else:54 parts = re.split(r"^(#{1,3}\s+.+)$", text, flags=re.M); cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}55 for i in range(1, len(parts), 2):56 sections.append(cur); cur = {"heading": parts[i].lstrip("# ").strip(), "level": parts[i].count("#"), "text": parts[i + 1] if i + 1 < len(parts) else ""}57 sections.append(cur)58 # level-1 sections own the text of their subsections59 for i, s in enumerate(sections):60 if s["level"] == 1:61 j = i + 1; full = [s["text"]]62 while j < len(sections) and sections[j]["level"] > 1: full.append(f"\\subsection{{{sections[j]['heading']}}}\n" + sections[j]["text"]); j += 163 s["full_text"] = "\n".join(full)64 else: s["full_text"] = s["text"]65 for s in sections:66 s["role"] = segmentation.section_role(s["heading"], is_appendix=bool(re.search(r"appendix", s["heading"], re.I)), is_intro=bool(re.search(r"^introduction", s["heading"], re.I)))67 s["n_words"] = len(re.sub(r"\\[a-zA-Z]+\*?(\[[^\]]*\])?(\{[^}]*\})?", " ", s["full_text"]).split())68 tables = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "has_notes": bool(re.search(r"tablenotes|\\footnotesize|\\note", b)), "n_numbers": len(NUM_RE.findall(b)), "body": b} for b in re.findall(r"\\begin\{table\*?\}(.*?)\\end\{table\*?\}", text, re.S)]69 figures = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1]} for b in re.findall(r"\\begin\{figure\*?\}(.*?)\\end\{figure\*?\}", text, re.S)]70 equations = [{"label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "body": b.strip()[:200]} for b in re.findall(r"\\begin\{(?:equation|align|gather|multline)\*?\}(.*?)\\end\{(?:equation|align|gather|multline)\*?\}", text, re.S)]71 labels = set(re.findall(r"\\label\{([^}]+)\}", text)); refs = set(re.findall(r"\\(?:ref|eqref|autoref|cref|Cref)\{([^}]+)\}", text))72 refs = {x.strip() for r in refs for x in r.split(",")}73 cites = [k.strip() for m in CITE_RE.findall(text) for k in m.split(",")]74 bib_keys, bib_dois = set(), {}75 for bibp in list(root.rglob("*.bib"))[:5]:76 bt = bibp.read_text(errors="ignore")77 for m in re.finditer(r"@\w+\{([^,\s]+),(.*?)(?=\n@|\Z)", bt, re.S):78 bib_keys.add(m.group(1)); d = re.search(r"doi\s*=\s*[{\"]([^}\"]+)", m.group(2), re.I)79 if d: bib_dois[m.group(1)] = d.group(1).strip()80 body_text = "\n".join(s["text"] for s in sections if s["level"] > 0 and s["role"] != "appendix")81 return {"path": str(main), "root": str(root), "kind": kind, "title": title, "abstract": abstract, "macros": {k: v for k, v in macros.items() if k in ("WPnumber", "WPkeywords", "WPjel", "WPdate", "WPversion")},82 "sections": sections, "tables": tables, "figures": figures, "equations": equations, "labels": sorted(labels), "refs": sorted(refs), "cites": cites, "bib_keys": sorted(bib_keys), "bib_dois": bib_dois,83 "n_words_body": len(body_text.split()), "text": text, "sha256": hashlib.sha256(text.encode()).hexdigest()}8485def section_text(ms: dict, name: str) -> str | None:86 for s in ms["sections"]:87 if s["level"] == 1 and (s["role"] == name or name.lower() in s["heading"].lower()): return s["full_text"]88 return None8990# ---------------- audits ----------------91def numbers_in(text: str) -> set[str]:92 text = CITE_RE.sub(" ", text); text = re.sub(r"\\(?:label|ref|eqref|cref)\{[^}]*\}", " ", text)93 out = set()94 for n in NUM_RE.findall(text):95 n = n.replace(",", "").replace("−", "-").rstrip(".")96 if not re.search(r"\d", n) or re.fullmatch(r"-?(19|20)\d{2}", n) or re.fullmatch(r"-?0?[0-9]", n): continue # drop years and single digits (enumerations)97 out.add(n)98 return out99100def numeric_preservation(source: str, output: str) -> dict:101 a, b = numbers_in(source), numbers_in(output)102 return {"n_source": len(a), "n_output": len(b), "missing_in_output": sorted(a - b)[:50], "new_in_output": sorted(b - a)[:50], "preserved_rate": round(len(a & b) / max(len(a), 1), 3), "new_rate": round(len(b - a) / max(len(b), 1), 3)}103104def citation_check(output: str, bib_keys: set[str] | list[str]) -> dict:105 keys = [k.strip() for m in CITE_RE.findall(output) for k in m.split(",")]106 unknown = sorted({k for k in keys if k not in set(bib_keys)})107 # author-year patterns not backed by \cite are also flagged in LaTeX output108 loose = re.findall(r"\b([A-Z][A-Za-z\-]+(?: (?:and|&) [A-Z][A-Za-z\-]+)?(?: et al\.)?) \((19|20)\d{2}[a-z]?\)", output)109 return {"n_cite_commands": len(keys), "unknown_keys": unknown, "n_author_year_without_cite": len(loose) if "\\cite" in output or keys else len(loose), "loose_examples": [f"{a} ({y}…)" for a, y in loose[:5]]}110111def unsupported_claims(text: str) -> list[dict]:112 """Sentences with strong claim verbs that carry no citation, cross-reference or number and no hedge."""113 sents = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", text)); out = []114 for s in sents:115 if len(s) < 40 or not CLAIM_VERBS.search(s): continue116 supported = bool(CITE_RE.search(s) or re.search(r"\\(?:ref|eqref|cref|autoref)\{|Table|Figure|Section|Appendix|\d", s)) or bool(HEDGES.search(s))117 if not supported: out.append({"sentence": s[:300], "verb": CLAIM_VERBS.search(s).group(0)})118 return out119120@np.vectorize121def _h(s): return int(hashlib.blake2b(s.encode(), digest_size=8).hexdigest(), 16)122123def _shingles(text: str, n: int = 8) -> set[int]:124 w = re.findall(r"[a-z0-9]+", text.lower())125 return {int(hashlib.blake2b(" ".join(w[i:i + n]).encode(), digest_size=8).hexdigest(), 16) for i in range(max(len(w) - n + 1, 0))}126127def corpus_shingles() -> dict:128 """8-gram hash sets of every corpus document (built once, cached under data/derived/index/)."""129 p = config.ROOT / "data" / "derived" / "index" / "shingles.json"130 if p.exists(): return {k: set(v) for k, v in json.loads(p.read_text()).items()}131 out = {}132 for f in sorted((config.derived_dir() / "parsed").glob("*.json")):133 d = json.loads(f.read_text()); txt = (d["abstract"] or "") + "\n" + "\n".join(pp["text"] for s in d["sections"] for pp in s["paragraphs"])134 out[d["doc_id"]] = _shingles(txt)135 p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps({k: sorted(v) for k, v in out.items()}))136 return out137138def overlap_with_corpus(text: str, n: int = 8) -> dict:139 sh = _shingles(text, n)140 if not sh: return {"n_shingles": 0, "max_overlap_share": 0.0, "top_docs": []}141 cs = corpus_shingles(); hits = sorted(((len(sh & v) / len(sh), k) for k, v in cs.items()), reverse=True)[:3]142 return {"n_shingles": len(sh), "max_overlap_share": round(hits[0][0], 4) if hits else 0.0, "top_docs": [{"doc_id": k, "share": round(s, 4)} for s, k in hits if s > 0]}143144def audit_manuscript(path: Path, tables=True, equations=True, citations=True, overlap=True) -> dict:145 ms = load_manuscript(path); rep = {"manuscript": ms["path"], "title": ms["title"], "generated": stamp(), "n_words_body": ms["n_words_body"], "sections": [{"heading": s["heading"], "role": s["role"], "n_words": s["n_words"]} for s in ms["sections"] if s["level"] == 1], "issues": []}146 if tables:147 t_issues = []148 for i, t in enumerate(ms["tables"]):149 if not t["caption"]: t_issues.append(f"table {i+1}: no caption")150 if t["label"] and t["label"] not in ms["refs"]: t_issues.append(f"table {i+1} ({t['label']}): never referenced in text")151 if not t["has_notes"]: t_issues.append(f"table {i+1} ({t['label'] or 'no label'}): no notes block (JoF convention: notes state sample, units, standard errors)")152 rep["tables"] = {"n": len(ms["tables"]), "issues": t_issues}; rep["issues"] += t_issues153 f_issues = [f"figure {i+1} ({f['label']}): never referenced" for i, f in enumerate(ms["figures"]) if f["label"] and f["label"] not in ms["refs"]]154 rep["figures"] = {"n": len(ms["figures"]), "issues": f_issues}; rep["issues"] += f_issues155 if equations:156 e_issues = [f"equation {i+1} ({e['label']}): labelled but never referenced" for i, e in enumerate(ms["equations"]) if e["label"] and e["label"] not in ms["refs"]]157 missing = sorted(r for r in ms["refs"] if r not in set(ms["labels"])); e_issues += [f"\\ref to undefined label: {r}" for r in missing[:20]]158 rep["equations"] = {"n": len(ms["equations"]), "n_labelled": sum(1 for e in ms["equations"] if e["label"]), "issues": e_issues}; rep["issues"] += e_issues159 if citations:160 cc = citation_check(ms["text"], ms["bib_keys"]); cc["n_bib_entries"] = len(ms["bib_keys"]); cc["n_bib_with_doi"] = len(ms["bib_dois"])161 cc["doi_verification"] = verify_dois(ms["bib_dois"]) if ms["bib_dois"] else {"checked": 0}162 rep["citations"] = cc; rep["issues"] += [f"unknown citation key: {k}" for k in cc["unknown_keys"]]163 rep["unsupported_claims"] = unsupported_claims("\n".join(s["text"] for s in ms["sections"] if s["level"] > 0))[:30]164 if overlap: rep["corpus_overlap"] = overlap_with_corpus(ms["text"])165 return rep166167def verify_dois(bib_dois: dict, limit: int = 60) -> dict:168 """DOI resolution via Crossref (network; cached in data/manifests/crossref_cache.json)."""169 import httpx, time170 cache_p = config.manifests_dir() / "crossref_cache.json"; cache = json.loads(cache_p.read_text()) if cache_p.exists() else {}171 mailto = config.load("models")["crossref_mailto"]; res = {"checked": 0, "resolved": 0, "unresolved": []}172 with httpx.Client(timeout=20, headers={"User-Agent": f"jof250/0.1 (mailto:{mailto})"}) as cl:173 for key, doi in list(bib_dois.items())[:limit]:174 doi = doi.strip().lower().replace("https://doi.org/", "")175 if doi not in cache:176 try:177 r = cl.get(f"https://api.crossref.org/works/{doi}", params={"mailto": mailto}); cache[doi] = {"status": r.status_code, "title": (r.json()["message"].get("title") or [None])[0] if r.status_code == 200 else None, "fetched": now_iso()}178 except Exception as e: cache[doi] = {"status": "error", "error": str(e)[:100], "fetched": now_iso()}179 time.sleep(0.1)180 res["checked"] += 1181 if cache[doi].get("status") == 200: res["resolved"] += 1182 else: res["unresolved"].append({"key": key, "doi": doi, "status": cache[doi].get("status")})183 cache_p.write_text(json.dumps(cache, indent=1, ensure_ascii=False)); return res184185def evidence_for_claim(path: Path, claim: str) -> dict:186 ms = load_manuscript(path); nums = numbers_in(claim); words = [w for w in re.findall(r"[a-z]{4,}", claim.lower())][:12]187 hits = []188 for i, t in enumerate(ms["tables"]):189 body = t["body"]; found = [n for n in nums if n in body.replace(",", "")]; kw = sum(1 for w in words if w in body.lower())190 if found or kw >= 2: hits.append({"where": f"table {i+1} ({t['label'] or 'no label'})", "caption": (t["caption"] or "")[:120], "numbers_found": found, "keyword_hits": kw})191 for s in ms["sections"]:192 if s["level"] == 0: continue193 for sent in re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", s["text"])):194 found = [n for n in nums if n in sent.replace(",", "")]; kw = sum(1 for w in words if w in sent.lower())195 if (found and kw >= 1) or kw >= 4: hits.append({"where": f"section '{s['heading']}'", "sentence": sent[:240], "numbers_found": found, "keyword_hits": kw})196 status = "supported_by_table" if any(h["where"].startswith("table") and h["numbers_found"] for h in hits) else ("mentioned_in_text" if hits else "no_evidence_found")197 return {"claim": claim, "numbers_in_claim": sorted(nums), "status": status, "hits": hits[:12], "generated": stamp()}198199def render_report(rep: dict) -> str:200 L = [f"# Manuscript audit — {rep.get('title') or rep['manuscript']}", "", f"Generated {rep['generated']['timestamp']} (jof250 {rep['generated']['code_version']}). Body words: {rep['n_words_body']}.", "", "## Sections", ""]201 L += [f"- {s['heading']} — role `{s['role']}`, {s['n_words']} words" for s in rep["sections"]]202 if "tables" in rep: L += ["", f"## Tables ({rep['tables']['n']}) and figures ({rep['figures']['n']})", ""] + ([f"- {x}" for x in rep["tables"]["issues"] + rep["figures"]["issues"]] or ["- no issues"])203 if "equations" in rep: L += ["", f"## Equations ({rep['equations']['n']}, {rep['equations']['n_labelled']} labelled)", ""] + ([f"- {x}" for x in rep["equations"]["issues"]] or ["- no issues"])204 if "citations" in rep:205 c = rep["citations"]; L += ["", "## Citations", "", f"- \\cite commands: {c['n_cite_commands']}; bib entries: {c['n_bib_entries']} ({c['n_bib_with_doi']} with DOI)", f"- unknown keys: {', '.join(c['unknown_keys']) or 'none'}", f"- DOI check: {c['doi_verification']}"]206 L += ["", f"## Possibly unsupported claims ({len(rep['unsupported_claims'])})", ""] + ([f"- [{x['verb']}] {x['sentence']}" for x in rep["unsupported_claims"]] or ["- none flagged"])207 if "corpus_overlap" in rep: L += ["", "## Overlap with the JoF corpus (8-gram shingles)", "", f"- max share of manuscript shingles found in one corpus document: {rep['corpus_overlap']['max_overlap_share']} {rep['corpus_overlap']['top_docs']}"]208 return "\n".join(L) + "\n"209