1"""Deterministic manuscript audits: LaTeX/Markdown loading, tables/equations/citations checks, numeric preservation, corpus overlap,2unsupported-claim flags, evidence lookup. No LLM calls here."""3from __future__ import annotations4import re, json, hashlib5from pathlib import Path6import numpy as np7from . import config, segmentation8from .provenance import stamp, now_iso910NUM_RE = re.compile(r"(?<![A-Za-z\\])[-−]?\d{1,3}(?:[,\d]{0,12})?(?:\.\d+)?%?")11CITE_RE = re.compile(r"\\(?:cite|citep|citet|citealp|citealt|citeauthor|citeyear|parencite|textcite|autocite)\*?(?:\[[^\]]*\])*\{([^}]+)\}")12CLAIM_VERBS = re.compile(r"\b(causes?|caused|leads? to|led to|results? in|drives?|increases?|decreases?|reduces?|raises?|lowers?|improves?|shows? that|demonstrates? that|proves? that|establishes? that|confirms? that|we find that|significantly)\b", re.I)13HEDGES = re.compile(r"\b(may|might|could|suggests?|appears?|consistent with|likely|possibly|tend to|we conjecture|we hypothesize)\b", re.I)1415# ---------------- manuscript loading ----------------16def _strip_comments(t: str) -> str: return re.sub(r"(?<!\\)%.*", "", t)1718def _resolve_inputs(text: str, base: Path, depth=0) -> str:19 def rep(m):20 name = m.group(2).strip(); p = base / name21 for cand in (p, p.with_suffix(".tex"), base / "paper" / name, (base / "paper" / name).with_suffix(".tex")):22 if cand.exists() and depth < 6: return "\n" + _resolve_inputs(_strip_comments(cand.read_text(errors="ignore")), cand.parent, depth + 1) + "\n"23 return f"\n% [missing input: {name}]\n"24 return re.sub(r"\\(input|include)\{([^}]+)\}", rep, text)2526def load_manuscript(path: Path) -> dict:27 """Load a paper from a directory (main.tex / paper/main.tex / *.md) or a single .tex/.md file. Returns text + extracted structure."""28 path = Path(path).expanduser().resolve(); root = path if path.is_dir() else path.parent29 main = None30 if path.is_file(): main = path31 else:32 for cand in ["paper/main.tex", "main.tex", "manuscript/main.tex", "paper/paper.tex"]:33 if (path / cand).exists(): main = path / cand; break34 if main is None:35 texs = sorted(path.rglob("*.tex"), key=lambda p: -p.stat().st_size); mds = sorted(path.glob("*.md"))36 main = texs[0] if texs else (mds[0] if mds else None)37 if main is None: raise FileNotFoundError(f"no .tex/.md manuscript found under {path}")38 raw = main.read_text(errors="ignore"); kind = "latex" if main.suffix == ".tex" else "markdown"39 text = _resolve_inputs(_strip_comments(raw), main.parent) if kind == "latex" else raw40 # macros defined in main (UQO template: \WPtitle etc.)41 macros = {m.group(1): m.group(2) for m in re.finditer(r"\\newcommand\{\\(WP\w+)\}\{((?:[^{}]|\{[^{}]*\})*)\}", text)}42 title = macros.get("WPtitle") or (re.search(r"\\title\{([^}]+)\}", text) or re.search(r"^#\s+(.+)$", text, re.M) or [None, None])[1]43 abstract = macros.get("WPabstract") or (re.search(r"\\begin\{abstract\}(.*?)\\end\{abstract\}", text, re.S) or [None, None])[1]44 if abstract: abstract = re.sub(r"\s+", " ", abstract.replace("%", "")).strip()45 sections = []46 if kind == "latex":47 parts = re.split(r"(\\(?:section|subsection)\*?\{[^}]+\})", text)48 cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}49 for i in range(1, len(parts), 2):50 m = re.match(r"\\(section|subsection)\*?\{([^}]+)\}", parts[i]); sections.append(cur)51 cur = {"heading": m.group(2), "level": 1 if m.group(1) == "section" else 2, "text": parts[i + 1] if i + 1 < len(parts) else ""}52 sections.append(cur)53 else:54 parts = re.split(r"^(#{1,3}\s+.+)$", text, flags=re.M); cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}55 for i in range(1, len(parts), 2):56 sections.append(cur); cur = {"heading": parts[i].lstrip("# ").strip(), "level": parts[i].count("#"), "text": parts[i + 1] if i + 1 < len(parts) else ""}57 sections.append(cur)58 # level-1 sections own the text of their subsections59 for i, s in enumerate(sections):60 if s["level"] == 1:61 j = i + 1; full = [s["text"]]62 while j < len(sections) and sections[j]["level"] > 1: full.append(f"\\subsection{{{sections[j]['heading']}}}\n" + sections[j]["text"]); j += 163 s["full_text"] = "\n".join(full)64 else: s["full_text"] = s["text"]65 for s in sections:66 s["role"] = segmentation.section_role(s["heading"], is_appendix=bool(re.search(r"appendix", s["heading"], re.I)), is_intro=bool(re.search(r"^introduction", s["heading"], re.I)))67 s["n_words"] = len(re.sub(r"\\[a-zA-Z]+\*?(\[[^\]]*\])?(\{[^}]*\})?", " ", s["full_text"]).split())68 tables = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "has_notes": bool(re.search(r"tablenotes|\\footnotesize|\\note", b)), "n_numbers": len(NUM_RE.findall(b)), "body": b} for b in re.findall(r"\\begin\{table\*?\}(.*?)\\end\{table\*?\}", text, re.S)]69 figures = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1]} for b in re.findall(r"\\begin\{figure\*?\}(.*?)\\end\{figure\*?\}", text, re.S)]70 equations = [{"label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "body": b.strip()[:200]} for b in re.findall(r"\\begin\{(?:equation|align|gather|multline)\*?\}(.*?)\\end\{(?:equation|align|gather|multline)\*?\}", text, re.S)]71 labels = set(re.findall(r"\\label\{([^}]+)\}", text)); refs = set(re.findall(r"\\(?:ref|eqref|autoref|cref|Cref)\{([^}]+)\}", text))72 refs = {x.strip() for r in refs for x in r.split(",")}73 cites = [k.strip() for m in CITE_RE.findall(text) for k in m.split(",")]74 bib_keys, bib_dois = set(), {}75 for bibp in list(root.rglob("*.bib"))[:5]:76 bt = bibp.read_text(errors="ignore")77 for m in re.finditer(r"@\w+\{([^,\s]+),(.*?)(?=\n@|\Z)", bt, re.S):78 bib_keys.add(m.group(1)); d = re.search(r"doi\s*=\s*[{\"]([^}\"]+)", m.group(2), re.I)79 if d: bib_dois[m.group(1)] = d.group(1).strip()80 body_text = "\n".join(s["text"] for s in sections if s["level"] > 0 and s["role"] != "appendix")81 return {"path": str(main), "root": str(root), "kind": kind, "title": title, "abstract": abstract, "macros": {k: v for k, v in macros.items() if k in ("WPnumber", "WPkeywords", "WPjel", "WPdate", "WPversion")},82 "sections": sections, "tables": tables, "figures": figures, "equations": equations, "labels": sorted(labels), "refs": sorted(refs), "cites": cites, "bib_keys": sorted(bib_keys), "bib_dois": bib_dois,83 "n_words_body": len(body_text.split()), "text": text, "sha256": hashlib.sha256(text.encode()).hexdigest()}8485def section_text(ms: dict, name: str) -> str | None:86 for s in ms["sections"]:87 if s["level"] == 1 and (s["role"] == name or name.lower() in s["heading"].lower()): return s["full_text"]88 return None8990# ---------------- audits ----------------91def numbers_in(text: str) -> set[str]:92 return {n.replace(",", "").replace("−", "-") for n in NUM_RE.findall(text) if re.search(r"\d", n) and not re.fullmatch(r"(19|20)\d{2}", n)}9394def numeric_preservation(source: str, output: str) -> dict:95 a, b = numbers_in(source), numbers_in(output)96 return {"n_source": len(a), "n_output": len(b), "missing_in_output": sorted(a - b)[:50], "new_in_output": sorted(b - a)[:50], "preserved_rate": round(len(a & b) / max(len(a), 1), 3), "new_rate": round(len(b - a) / max(len(b), 1), 3)}9798def citation_check(output: str, bib_keys: set[str] | list[str]) -> dict:99 keys = [k.strip() for m in CITE_RE.findall(output) for k in m.split(",")]100 unknown = sorted({k for k in keys if k not in set(bib_keys)})101 # author-year patterns not backed by \cite are also flagged in LaTeX output102 loose = re.findall(r"\b([A-Z][A-Za-z\-]+(?: (?:and|&) [A-Z][A-Za-z\-]+)?(?: et al\.)?) \((19|20)\d{2}[a-z]?\)", output)103 return {"n_cite_commands": len(keys), "unknown_keys": unknown, "n_author_year_without_cite": len(loose) if "\\cite" in output or keys else len(loose), "loose_examples": [f"{a} ({y}…)" for a, y in loose[:5]]}104105def unsupported_claims(text: str) -> list[dict]:106 """Sentences with strong claim verbs that carry no citation, cross-reference or number and no hedge."""107 sents = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", text)); out = []108 for s in sents:109 if len(s) < 40 or not CLAIM_VERBS.search(s): continue110 supported = bool(CITE_RE.search(s) or re.search(r"\\(?:ref|eqref|cref|autoref)\{|Table|Figure|Section|Appendix|\d", s)) or bool(HEDGES.search(s))111 if not supported: out.append({"sentence": s[:300], "verb": CLAIM_VERBS.search(s).group(0)})112 return out113114@np.vectorize115def _h(s): return int(hashlib.blake2b(s.encode(), digest_size=8).hexdigest(), 16)116117def _shingles(text: str, n: int = 8) -> set[int]:118 w = re.findall(r"[a-z0-9]+", text.lower())119 return {int(hashlib.blake2b(" ".join(w[i:i + n]).encode(), digest_size=8).hexdigest(), 16) for i in range(max(len(w) - n + 1, 0))}120121def corpus_shingles() -> dict:122 """8-gram hash sets of every corpus document (built once, cached under data/derived/index/)."""123 p = config.ROOT / "data" / "derived" / "index" / "shingles.json"124 if p.exists(): return {k: set(v) for k, v in json.loads(p.read_text()).items()}125 out = {}126 for f in sorted((config.derived_dir() / "parsed").glob("*.json")):127 d = json.loads(f.read_text()); txt = (d["abstract"] or "") + "\n" + "\n".join(pp["text"] for s in d["sections"] for pp in s["paragraphs"])128 out[d["doc_id"]] = _shingles(txt)129 p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps({k: sorted(v) for k, v in out.items()}))130 return out131132def overlap_with_corpus(text: str, n: int = 8) -> dict:133 sh = _shingles(text, n)134 if not sh: return {"n_shingles": 0, "max_overlap_share": 0.0, "top_docs": []}135 cs = corpus_shingles(); hits = sorted(((len(sh & v) / len(sh), k) for k, v in cs.items()), reverse=True)[:3]136 return {"n_shingles": len(sh), "max_overlap_share": round(hits[0][0], 4) if hits else 0.0, "top_docs": [{"doc_id": k, "share": round(s, 4)} for s, k in hits if s > 0]}137138def audit_manuscript(path: Path, tables=True, equations=True, citations=True, overlap=True) -> dict:139 ms = load_manuscript(path); rep = {"manuscript": ms["path"], "title": ms["title"], "generated": stamp(), "n_words_body": ms["n_words_body"], "sections": [{"heading": s["heading"], "role": s["role"], "n_words": s["n_words"]} for s in ms["sections"] if s["level"] == 1], "issues": []}140 if tables:141 t_issues = []142 for i, t in enumerate(ms["tables"]):143 if not t["caption"]: t_issues.append(f"table {i+1}: no caption")144 if t["label"] and t["label"] not in ms["refs"]: t_issues.append(f"table {i+1} ({t['label']}): never referenced in text")145 if not t["has_notes"]: t_issues.append(f"table {i+1} ({t['label'] or 'no label'}): no notes block (JoF convention: notes state sample, units, standard errors)")146 rep["tables"] = {"n": len(ms["tables"]), "issues": t_issues}; rep["issues"] += t_issues147 f_issues = [f"figure {i+1} ({f['label']}): never referenced" for i, f in enumerate(ms["figures"]) if f["label"] and f["label"] not in ms["refs"]]148 rep["figures"] = {"n": len(ms["figures"]), "issues": f_issues}; rep["issues"] += f_issues149 if equations:150 e_issues = [f"equation {i+1} ({e['label']}): labelled but never referenced" for i, e in enumerate(ms["equations"]) if e["label"] and e["label"] not in ms["refs"]]151 missing = sorted(r for r in ms["refs"] if r not in set(ms["labels"])); e_issues += [f"\\ref to undefined label: {r}" for r in missing[:20]]152 rep["equations"] = {"n": len(ms["equations"]), "n_labelled": sum(1 for e in ms["equations"] if e["label"]), "issues": e_issues}; rep["issues"] += e_issues153 if citations:154 cc = citation_check(ms["text"], ms["bib_keys"]); cc["n_bib_entries"] = len(ms["bib_keys"]); cc["n_bib_with_doi"] = len(ms["bib_dois"])155 cc["doi_verification"] = verify_dois(ms["bib_dois"]) if ms["bib_dois"] else {"checked": 0}156 rep["citations"] = cc; rep["issues"] += [f"unknown citation key: {k}" for k in cc["unknown_keys"]]157 rep["unsupported_claims"] = unsupported_claims("\n".join(s["text"] for s in ms["sections"] if s["level"] > 0))[:30]158 if overlap: rep["corpus_overlap"] = overlap_with_corpus(ms["text"])159 return rep160161def verify_dois(bib_dois: dict, limit: int = 60) -> dict:162 """DOI resolution via Crossref (network; cached in data/manifests/crossref_cache.json)."""163 import httpx, time164 cache_p = config.manifests_dir() / "crossref_cache.json"; cache = json.loads(cache_p.read_text()) if cache_p.exists() else {}165 mailto = config.load("models")["crossref_mailto"]; res = {"checked": 0, "resolved": 0, "unresolved": []}166 with httpx.Client(timeout=20, headers={"User-Agent": f"jof250/0.1 (mailto:{mailto})"}) as cl:167 for key, doi in list(bib_dois.items())[:limit]:168 doi = doi.strip().lower().replace("https://doi.org/", "")169 if doi not in cache:170 try:171 r = cl.get(f"https://api.crossref.org/works/{doi}", params={"mailto": mailto}); cache[doi] = {"status": r.status_code, "title": (r.json()["message"].get("title") or [None])[0] if r.status_code == 200 else None, "fetched": now_iso()}172 except Exception as e: cache[doi] = {"status": "error", "error": str(e)[:100], "fetched": now_iso()}173 time.sleep(0.1)174 res["checked"] += 1175 if cache[doi].get("status") == 200: res["resolved"] += 1176 else: res["unresolved"].append({"key": key, "doi": doi, "status": cache[doi].get("status")})177 cache_p.write_text(json.dumps(cache, indent=1, ensure_ascii=False)); return res178179def evidence_for_claim(path: Path, claim: str) -> dict:180 ms = load_manuscript(path); nums = numbers_in(claim); words = [w for w in re.findall(r"[a-z]{4,}", claim.lower())][:12]181 hits = []182 for i, t in enumerate(ms["tables"]):183 body = t["body"]; found = [n for n in nums if n in body.replace(",", "")]; kw = sum(1 for w in words if w in body.lower())184 if found or kw >= 2: hits.append({"where": f"table {i+1} ({t['label'] or 'no label'})", "caption": (t["caption"] or "")[:120], "numbers_found": found, "keyword_hits": kw})185 for s in ms["sections"]:186 if s["level"] == 0: continue187 for sent in re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", s["text"])):188 found = [n for n in nums if n in sent.replace(",", "")]; kw = sum(1 for w in words if w in sent.lower())189 if (found and kw >= 1) or kw >= 4: hits.append({"where": f"section '{s['heading']}'", "sentence": sent[:240], "numbers_found": found, "keyword_hits": kw})190 status = "supported_by_table" if any(h["where"].startswith("table") and h["numbers_found"] for h in hits) else ("mentioned_in_text" if hits else "no_evidence_found")191 return {"claim": claim, "numbers_in_claim": sorted(nums), "status": status, "hits": hits[:12], "generated": stamp()}192193def render_report(rep: dict) -> str:194 L = [f"# Manuscript audit — {rep.get('title') or rep['manuscript']}", "", f"Generated {rep['generated']['timestamp']} (jof250 {rep['generated']['code_version']}). Body words: {rep['n_words_body']}.", "", "## Sections", ""]195 L += [f"- {s['heading']} — role `{s['role']}`, {s['n_words']} words" for s in rep["sections"]]196 if "tables" in rep: L += ["", f"## Tables ({rep['tables']['n']}) and figures ({rep['figures']['n']})", ""] + ([f"- {x}" for x in rep["tables"]["issues"] + rep["figures"]["issues"]] or ["- no issues"])197 if "equations" in rep: L += ["", f"## Equations ({rep['equations']['n']}, {rep['equations']['n_labelled']} labelled)", ""] + ([f"- {x}" for x in rep["equations"]["issues"]] or ["- no issues"])198 if "citations" in rep:199 c = rep["citations"]; L += ["", "## Citations", "", f"- \\cite commands: {c['n_cite_commands']}; bib entries: {c['n_bib_entries']} ({c['n_bib_with_doi']} with DOI)", f"- unknown keys: {', '.join(c['unknown_keys']) or 'none'}", f"- DOI check: {c['doi_verification']}"]200 L += ["", f"## Possibly unsupported claims ({len(rep['unsupported_claims'])})", ""] + ([f"- [{x['verb']}] {x['sentence']}" for x in rep["unsupported_claims"]] or ["- none flagged"])201 if "corpus_overlap" in rep: L += ["", "## Overlap with the JoF corpus (8-gram shingles)", "", f"- max share of manuscript shingles found in one corpus document: {rep['corpus_overlap']['max_overlap_share']} {rep['corpus_overlap']['top_docs']}"]202 return "\n".join(L) + "\n"203