"""Deterministic manuscript audits: LaTeX/Markdown loading, tables/equations/citations checks, numeric preservation, corpus overlap,
unsupported-claim flags, evidence lookup. No LLM calls here."""
from __future__ import annotations
import re, json, hashlib
from pathlib import Path
import numpy as np
from . import config, segmentation
from .provenance import stamp, now_iso

NUM_RE = re.compile(r"(?<![A-Za-z\\])[-−]?\d{1,3}(?:[,\d]{0,12})?(?:\.\d+)?%?")
CITE_RE = re.compile(r"\\(?:cite|citep|citet|citealp|citealt|citeauthor|citeyear|parencite|textcite|autocite)\*?(?:\[[^\]]*\])*\{([^}]+)\}")
CLAIM_VERBS = re.compile(r"\b(causes?|caused|leads? to|led to|results? in|drives?|increases?|decreases?|reduces?|raises?|lowers?|improves?|shows? that|demonstrates? that|proves? that|establishes? that|confirms? that|we find that|significantly)\b", re.I)
HEDGES = re.compile(r"\b(may|might|could|suggests?|appears?|consistent with|likely|possibly|tend to|we conjecture|we hypothesize)\b", re.I)

# ---------------- manuscript loading ----------------
def _strip_comments(t: str) -> str: return re.sub(r"(?<!\\)%.*", "", t)

def _resolve_inputs(text: str, base: Path, depth=0) -> str:
    def rep(m):
        name = m.group(2).strip(); p = base / name
        for cand in (p, p.with_suffix(".tex"), base / "paper" / name, (base / "paper" / name).with_suffix(".tex")):
            if cand.exists() and depth < 6: return "\n" + _resolve_inputs(_strip_comments(cand.read_text(errors="ignore")), cand.parent, depth + 1) + "\n"
        return f"\n% [missing input: {name}]\n"
    return re.sub(r"\\(input|include)\{([^}]+)\}", rep, text)

def load_manuscript(path: Path) -> dict:
    """Load a paper from a directory (main.tex / paper/main.tex / *.md) or a single .tex/.md file. Returns text + extracted structure."""
    path = Path(path).expanduser().resolve(); root = path if path.is_dir() else path.parent
    main = None
    if path.is_file(): main = path
    else:
        for cand in ["paper/main.tex", "main.tex", "manuscript/main.tex", "paper/paper.tex"]:
            if (path / cand).exists(): main = path / cand; break
        if main is None:
            texs = sorted(path.rglob("*.tex"), key=lambda p: -p.stat().st_size); mds = sorted(path.glob("*.md"))
            main = texs[0] if texs else (mds[0] if mds else None)
    if main is None: raise FileNotFoundError(f"no .tex/.md manuscript found under {path}")
    raw = main.read_text(errors="ignore"); kind = "latex" if main.suffix == ".tex" else "markdown"
    text = _resolve_inputs(_strip_comments(raw), main.parent) if kind == "latex" else raw
    # macros defined in main (UQO template: \WPtitle etc.)
    macros = {m.group(1): m.group(2) for m in re.finditer(r"\\newcommand\{\\(WP\w+)\}\{((?:[^{}]|\{[^{}]*\})*)\}", text)}
    title = macros.get("WPtitle") or (re.search(r"\\title\{([^}]+)\}", text) or re.search(r"^#\s+(.+)$", text, re.M) or [None, None])[1]
    abstract = macros.get("WPabstract") or (re.search(r"\\begin\{abstract\}(.*?)\\end\{abstract\}", text, re.S) or [None, None])[1]
    if abstract: abstract = re.sub(r"\s+", " ", abstract.replace("%", "")).strip()
    sections = []
    if kind == "latex":
        parts = re.split(r"(\\(?:section|subsection)\*?\{[^}]+\})", text)
        cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}
        for i in range(1, len(parts), 2):
            m = re.match(r"\\(section|subsection)\*?\{([^}]+)\}", parts[i]); sections.append(cur)
            cur = {"heading": m.group(2), "level": 1 if m.group(1) == "section" else 2, "text": parts[i + 1] if i + 1 < len(parts) else ""}
        sections.append(cur)
    else:
        parts = re.split(r"^(#{1,3}\s+.+)$", text, flags=re.M); cur = {"heading": "(preamble)", "level": 0, "text": parts[0]}
        for i in range(1, len(parts), 2):
            sections.append(cur); cur = {"heading": parts[i].lstrip("# ").strip(), "level": parts[i].count("#"), "text": parts[i + 1] if i + 1 < len(parts) else ""}
        sections.append(cur)
    # level-1 sections own the text of their subsections
    for i, s in enumerate(sections):
        if s["level"] == 1:
            j = i + 1; full = [s["text"]]
            while j < len(sections) and sections[j]["level"] > 1: full.append(f"\\subsection{{{sections[j]['heading']}}}\n" + sections[j]["text"]); j += 1
            s["full_text"] = "\n".join(full)
        else: s["full_text"] = s["text"]
    for s in sections:
        s["role"] = segmentation.section_role(s["heading"], is_appendix=bool(re.search(r"appendix", s["heading"], re.I)), is_intro=bool(re.search(r"^introduction", s["heading"], re.I)))
        s["n_words"] = len(re.sub(r"\\[a-zA-Z]+\*?(\[[^\]]*\])?(\{[^}]*\})?", " ", s["full_text"]).split())
    tables = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "has_notes": bool(re.search(r"tablenotes|\\footnotesize|\\note", b)), "n_numbers": len(NUM_RE.findall(b)), "body": b} for b in re.findall(r"\\begin\{table\*?\}(.*?)\\end\{table\*?\}", text, re.S)]
    figures = [{"caption": (re.search(r"\\caption\{((?:[^{}]|\{[^{}]*\})*)\}", b) or [None, None])[1], "label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1]} for b in re.findall(r"\\begin\{figure\*?\}(.*?)\\end\{figure\*?\}", text, re.S)]
    equations = [{"label": (re.search(r"\\label\{([^}]+)\}", b) or [None, None])[1], "body": b.strip()[:200]} for b in re.findall(r"\\begin\{(?:equation|align|gather|multline)\*?\}(.*?)\\end\{(?:equation|align|gather|multline)\*?\}", text, re.S)]
    labels = set(re.findall(r"\\label\{([^}]+)\}", text)); refs = set(re.findall(r"\\(?:ref|eqref|autoref|cref|Cref)\{([^}]+)\}", text))
    refs = {x.strip() for r in refs for x in r.split(",")}
    cites = [k.strip() for m in CITE_RE.findall(text) for k in m.split(",")]
    bib_keys, bib_dois = set(), {}
    for bibp in list(root.rglob("*.bib"))[:5]:
        bt = bibp.read_text(errors="ignore")
        for m in re.finditer(r"@\w+\{([^,\s]+),(.*?)(?=\n@|\Z)", bt, re.S):
            bib_keys.add(m.group(1)); d = re.search(r"doi\s*=\s*[{\"]([^}\"]+)", m.group(2), re.I)
            if d: bib_dois[m.group(1)] = d.group(1).strip()
    body_text = "\n".join(s["text"] for s in sections if s["level"] > 0 and s["role"] != "appendix")
    return {"path": str(main), "root": str(root), "kind": kind, "title": title, "abstract": abstract, "macros": {k: v for k, v in macros.items() if k in ("WPnumber", "WPkeywords", "WPjel", "WPdate", "WPversion")},
            "sections": sections, "tables": tables, "figures": figures, "equations": equations, "labels": sorted(labels), "refs": sorted(refs), "cites": cites, "bib_keys": sorted(bib_keys), "bib_dois": bib_dois,
            "n_words_body": len(body_text.split()), "text": text, "sha256": hashlib.sha256(text.encode()).hexdigest()}

def section_text(ms: dict, name: str) -> str | None:
    for s in ms["sections"]:
        if s["level"] == 1 and (s["role"] == name or name.lower() in s["heading"].lower()): return s["full_text"]
    return None

# ---------------- audits ----------------
def numbers_in(text: str) -> set[str]:
    text = CITE_RE.sub(" ", text); text = re.sub(r"\\(?:label|ref|eqref|cref)\{[^}]*\}", " ", text)
    out = set()
    for n in NUM_RE.findall(text):
        n = n.replace(",", "").replace("−", "-").rstrip(".")
        if not re.search(r"\d", n) or re.fullmatch(r"-?(19|20)\d{2}", n) or re.fullmatch(r"-?0?[0-9]", n): continue  # drop years and single digits (enumerations)
        out.add(n)
    return out

def numeric_preservation(source: str, output: str) -> dict:
    a, b = numbers_in(source), numbers_in(output)
    return {"n_source": len(a), "n_output": len(b), "missing_in_output": sorted(a - b)[:50], "new_in_output": sorted(b - a)[:50], "preserved_rate": round(len(a & b) / max(len(a), 1), 3), "new_rate": round(len(b - a) / max(len(b), 1), 3)}

def citation_check(output: str, bib_keys: set[str] | list[str]) -> dict:
    keys = [k.strip() for m in CITE_RE.findall(output) for k in m.split(",")]
    unknown = sorted({k for k in keys if k not in set(bib_keys)})
    # author-year patterns not backed by \cite are also flagged in LaTeX output
    loose = re.findall(r"\b([A-Z][A-Za-z\-]+(?: (?:and|&) [A-Z][A-Za-z\-]+)?(?: et al\.)?) \((19|20)\d{2}[a-z]?\)", output)
    return {"n_cite_commands": len(keys), "unknown_keys": unknown, "n_author_year_without_cite": len(loose) if "\\cite" in output or keys else len(loose), "loose_examples": [f"{a} ({y}…)" for a, y in loose[:5]]}

def unsupported_claims(text: str) -> list[dict]:
    """Sentences with strong claim verbs that carry no citation, cross-reference or number and no hedge."""
    sents = re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", text)); out = []
    for s in sents:
        if len(s) < 40 or not CLAIM_VERBS.search(s): continue
        supported = bool(CITE_RE.search(s) or re.search(r"\\(?:ref|eqref|cref|autoref)\{|Table|Figure|Section|Appendix|\d", s)) or bool(HEDGES.search(s))
        if not supported: out.append({"sentence": s[:300], "verb": CLAIM_VERBS.search(s).group(0)})
    return out

@np.vectorize
def _h(s): return int(hashlib.blake2b(s.encode(), digest_size=8).hexdigest(), 16)

def _shingles(text: str, n: int = 8) -> set[int]:
    w = re.findall(r"[a-z0-9]+", text.lower())
    return {int(hashlib.blake2b(" ".join(w[i:i + n]).encode(), digest_size=8).hexdigest(), 16) for i in range(max(len(w) - n + 1, 0))}

def corpus_shingles() -> dict:
    """8-gram hash sets of every corpus document (built once, cached under data/derived/index/)."""
    p = config.ROOT / "data" / "derived" / "index" / "shingles.json"
    if p.exists(): return {k: set(v) for k, v in json.loads(p.read_text()).items()}
    out = {}
    for f in sorted((config.derived_dir() / "parsed").glob("*.json")):
        d = json.loads(f.read_text()); txt = (d["abstract"] or "") + "\n" + "\n".join(pp["text"] for s in d["sections"] for pp in s["paragraphs"])
        out[d["doc_id"]] = _shingles(txt)
    p.parent.mkdir(parents=True, exist_ok=True); p.write_text(json.dumps({k: sorted(v) for k, v in out.items()}))
    return out

def overlap_with_corpus(text: str, n: int = 8) -> dict:
    sh = _shingles(text, n)
    if not sh: return {"n_shingles": 0, "max_overlap_share": 0.0, "top_docs": []}
    cs = corpus_shingles(); hits = sorted(((len(sh & v) / len(sh), k) for k, v in cs.items()), reverse=True)[:3]
    return {"n_shingles": len(sh), "max_overlap_share": round(hits[0][0], 4) if hits else 0.0, "top_docs": [{"doc_id": k, "share": round(s, 4)} for s, k in hits if s > 0]}

def audit_manuscript(path: Path, tables=True, equations=True, citations=True, overlap=True) -> dict:
    ms = load_manuscript(path); rep = {"manuscript": ms["path"], "title": ms["title"], "generated": stamp(), "n_words_body": ms["n_words_body"], "sections": [{"heading": s["heading"], "role": s["role"], "n_words": s["n_words"]} for s in ms["sections"] if s["level"] == 1], "issues": []}
    if tables:
        t_issues = []
        for i, t in enumerate(ms["tables"]):
            if not t["caption"]: t_issues.append(f"table {i+1}: no caption")
            if t["label"] and t["label"] not in ms["refs"]: t_issues.append(f"table {i+1} ({t['label']}): never referenced in text")
            if not t["has_notes"]: t_issues.append(f"table {i+1} ({t['label'] or 'no label'}): no notes block (JoF convention: notes state sample, units, standard errors)")
        rep["tables"] = {"n": len(ms["tables"]), "issues": t_issues}; rep["issues"] += t_issues
        f_issues = [f"figure {i+1} ({f['label']}): never referenced" for i, f in enumerate(ms["figures"]) if f["label"] and f["label"] not in ms["refs"]]
        rep["figures"] = {"n": len(ms["figures"]), "issues": f_issues}; rep["issues"] += f_issues
    if equations:
        e_issues = [f"equation {i+1} ({e['label']}): labelled but never referenced" for i, e in enumerate(ms["equations"]) if e["label"] and e["label"] not in ms["refs"]]
        missing = sorted(r for r in ms["refs"] if r not in set(ms["labels"])); e_issues += [f"\\ref to undefined label: {r}" for r in missing[:20]]
        rep["equations"] = {"n": len(ms["equations"]), "n_labelled": sum(1 for e in ms["equations"] if e["label"]), "issues": e_issues}; rep["issues"] += e_issues
    if citations:
        cc = citation_check(ms["text"], ms["bib_keys"]); cc["n_bib_entries"] = len(ms["bib_keys"]); cc["n_bib_with_doi"] = len(ms["bib_dois"])
        cc["doi_verification"] = verify_dois(ms["bib_dois"]) if ms["bib_dois"] else {"checked": 0}
        rep["citations"] = cc; rep["issues"] += [f"unknown citation key: {k}" for k in cc["unknown_keys"]]
    rep["unsupported_claims"] = unsupported_claims("\n".join(s["text"] for s in ms["sections"] if s["level"] > 0))[:30]
    if overlap: rep["corpus_overlap"] = overlap_with_corpus(ms["text"])
    return rep

def verify_dois(bib_dois: dict, limit: int = 60) -> dict:
    """DOI resolution via Crossref (network; cached in data/manifests/crossref_cache.json)."""
    import httpx, time
    cache_p = config.manifests_dir() / "crossref_cache.json"; cache = json.loads(cache_p.read_text()) if cache_p.exists() else {}
    mailto = config.load("models")["crossref_mailto"]; res = {"checked": 0, "resolved": 0, "unresolved": []}
    with httpx.Client(timeout=20, headers={"User-Agent": f"jof250/0.1 (mailto:{mailto})"}) as cl:
        for key, doi in list(bib_dois.items())[:limit]:
            doi = doi.strip().lower().replace("https://doi.org/", "")
            if doi not in cache:
                try:
                    r = cl.get(f"https://api.crossref.org/works/{doi}", params={"mailto": mailto}); cache[doi] = {"status": r.status_code, "title": (r.json()["message"].get("title") or [None])[0] if r.status_code == 200 else None, "fetched": now_iso()}
                except Exception as e: cache[doi] = {"status": "error", "error": str(e)[:100], "fetched": now_iso()}
                time.sleep(0.1)
            res["checked"] += 1
            if cache[doi].get("status") == 200: res["resolved"] += 1
            else: res["unresolved"].append({"key": key, "doi": doi, "status": cache[doi].get("status")})
    cache_p.write_text(json.dumps(cache, indent=1, ensure_ascii=False)); return res

def evidence_for_claim(path: Path, claim: str) -> dict:
    ms = load_manuscript(path); nums = numbers_in(claim); words = [w for w in re.findall(r"[a-z]{4,}", claim.lower())][:12]
    hits = []
    for i, t in enumerate(ms["tables"]):
        body = t["body"]; found = [n for n in nums if n in body.replace(",", "")]; kw = sum(1 for w in words if w in body.lower())
        if found or kw >= 2: hits.append({"where": f"table {i+1} ({t['label'] or 'no label'})", "caption": (t["caption"] or "")[:120], "numbers_found": found, "keyword_hits": kw})
    for s in ms["sections"]:
        if s["level"] == 0: continue
        for sent in re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", s["text"])):
            found = [n for n in nums if n in sent.replace(",", "")]; kw = sum(1 for w in words if w in sent.lower())
            if (found and kw >= 1) or kw >= 4: hits.append({"where": f"section '{s['heading']}'", "sentence": sent[:240], "numbers_found": found, "keyword_hits": kw})
    status = "supported_by_table" if any(h["where"].startswith("table") and h["numbers_found"] for h in hits) else ("mentioned_in_text" if hits else "no_evidence_found")
    return {"claim": claim, "numbers_in_claim": sorted(nums), "status": status, "hits": hits[:12], "generated": stamp()}

def render_report(rep: dict) -> str:
    L = [f"# Manuscript audit — {rep.get('title') or rep['manuscript']}", "", f"Generated {rep['generated']['timestamp']} (jof250 {rep['generated']['code_version']}). Body words: {rep['n_words_body']}.", "", "## Sections", ""]
    L += [f"- {s['heading']} — role `{s['role']}`, {s['n_words']} words" for s in rep["sections"]]
    if "tables" in rep: L += ["", f"## Tables ({rep['tables']['n']}) and figures ({rep['figures']['n']})", ""] + ([f"- {x}" for x in rep["tables"]["issues"] + rep["figures"]["issues"]] or ["- no issues"])
    if "equations" in rep: L += ["", f"## Equations ({rep['equations']['n']}, {rep['equations']['n_labelled']} labelled)", ""] + ([f"- {x}" for x in rep["equations"]["issues"]] or ["- no issues"])
    if "citations" in rep:
        c = rep["citations"]; L += ["", "## Citations", "", f"- \\cite commands: {c['n_cite_commands']}; bib entries: {c['n_bib_entries']} ({c['n_bib_with_doi']} with DOI)", f"- unknown keys: {', '.join(c['unknown_keys']) or 'none'}", f"- DOI check: {c['doi_verification']}"]
    L += ["", f"## Possibly unsupported claims ({len(rep['unsupported_claims'])})", ""] + ([f"- [{x['verb']}] {x['sentence']}" for x in rep["unsupported_claims"]] or ["- none flagged"])
    if "corpus_overlap" in rep: L += ["", "## Overlap with the JoF corpus (8-gram shingles)", "", f"- max share of manuscript shingles found in one corpus document: {rep['corpus_overlap']['max_overlap_share']} {rep['corpus_overlap']['top_docs']}"]
    return "\n".join(L) + "\n"