spb/traite-evaluation-qc

1#!/usr/bin/env python32"""Fusionne sources/registry-*.csv → sources/registry.csv, sources/normes-*.csv → sources/normes-matrice.csv,3references/lib-*.bib → references/library.bib (clés dédoublonnées, la première occurrence gagne ; library.bib hérité d'abord).4Idempotent : les fichiers fusionnés sont reconstruits à partir des fichiers sources + de la partie « héritée » de library.bib."""5import csv, re, pathlib, sys6ROOT = pathlib.Path(__file__).resolve().parents[1]7# --- registre8hdr = "id,type,titre,organisme,version_ou_date,url,date_consultation,statut,utilise_dans,notes".split(",")9rows = []10for f in sorted((ROOT / "sources").glob("registry-*.csv")):11    with f.open(encoding="utf-8") as fh:12        r = csv.DictReader(fh)13        for row in r:14            rows.append({k: (row.get(k) or "").strip() for k in hdr})15seen = set(); out = []16for row in rows:17    if row["id"] in seen or not row["id"]: continue18    seen.add(row["id"]); out.append(row)19with (ROOT / "sources/registry.csv").open("w", encoding="utf-8", newline="") as fh:20    w = csv.DictWriter(fh, fieldnames=hdr); w.writeheader(); w.writerows(out)21print(f"registry.csv : {len(out)} sources ({sum(1 for r in out if r['statut']=='verifie')} vérifiées)")22# --- normes23hdrn = "id,domaine,disposition,texte_source,reference_precise,effet_sur_evaluation,version_verifiee,date_consultation,chapitres".split(",")24nrows = []25for f in sorted((ROOT / "sources").glob("normes-*.csv")):26    if f.name == "normes-matrice.csv": continue27    with f.open(encoding="utf-8") as fh:28        for row in csv.DictReader(fh):29            nrows.append({k: (row.get(k) or "").strip() for k in hdrn})30with (ROOT / "sources/normes-matrice.csv").open("w", encoding="utf-8", newline="") as fh:31    w = csv.DictWriter(fh, fieldnames=hdrn); w.writeheader(); w.writerows(nrows)32print(f"normes-matrice.csv : {len(nrows)} dispositions")33# --- bib34lib = ROOT / "references/library.bib"35txt = lib.read_text(encoding="utf-8")36marker = "\n% ===== ENTRÉES VÉRIFIÉES (phase 1, fusion automatique de lib-*.bib) =====\n"37base = txt.split(marker)[0].rstrip() + "\n"38def keys(s): return re.findall(r"^@\w+\s*\{\s*([^,\s]+)\s*,", s, re.M)39have = set(keys(base)); added = []; dup = []40for f in sorted((ROOT / "references").glob("lib-*.bib")):41    s = f.read_text(encoding="utf-8")42    # découpe en entrées43    parts = re.split(r"(?m)^(?=@\w+\s*\{)", s)44    for p in parts:45        m = re.match(r"@\w+\s*\{\s*([^,\s]+)\s*,", p)46        if not m: continue47        k = m.group(1)48        if k in have: dup.append(f"{f.name}:{k}"); continue49        have.add(k); added.append(p.strip() + "\n")50lib.write_text(base + marker + "\n".join(added), encoding="utf-8")51print(f"library.bib : {len(have)} clés ({len(added)} ajoutées ; doublons ignorés : {len(dup)})")52if dup: print("  doublons :", ", ".join(dup[:20]))5354# --- fichiers par chapitre (verification/chapters/chNN-*.csv, references/lib-chNN.bib) → fichiers globaux55def merge_csv(pattern, target, keycol):56    tgt = ROOT / target57    with tgt.open(encoding="utf-8") as fh:58        r = csv.DictReader(fh); cols = r.fieldnames; rows = list(r)59    keys = {row[keycol] for row in rows}60    n = 061    for f in sorted((ROOT / "verification/chapters").glob(pattern)):62        with f.open(encoding="utf-8") as fh:63            for row in csv.DictReader(fh):64                row = {c: (row.get(c) or "").strip() for c in cols}65                if row[keycol] and row[keycol] not in keys:66                    rows.append(row); keys.add(row[keycol]); n += 167    with tgt.open("w", encoding="utf-8", newline="") as fh:68        w = csv.DictWriter(fh, fieldnames=cols); w.writeheader(); w.writerows(rows)69    print(f"{target} : {len(rows)} lignes (+{n})")70merge_csv("*-figures.csv", "verification/figures.csv", "fichier")71merge_csv("*-calculations.csv", "verification/calculations.csv", "id")72merge_csv("*-claims.csv", "verification/claims.csv", "id")73# registre par chapitre → registry.csv (append sans doublon d'id)74reg = ROOT / "sources/registry.csv"75with reg.open(encoding="utf-8") as fh:76    r = csv.DictReader(fh); rrows = list(r)77ids = {x["id"] for x in rrows}; n = 078for f in sorted((ROOT / "verification/chapters").glob("*-registry.csv")):79    with f.open(encoding="utf-8") as fh:80        for row in csv.DictReader(fh):81            row = {c: (row.get(c) or "").strip() for c in hdr}82            if row["id"] and row["id"] not in ids: rrows.append(row); ids.add(row["id"]); n += 183with reg.open("w", encoding="utf-8", newline="") as fh:84    w = csv.DictWriter(fh, fieldnames=hdr); w.writeheader(); w.writerows(rrows)85print(f"registry.csv (+chapitres) : {len(rrows)} (+{n})")86# bib par chapitre87txt = lib.read_text(encoding="utf-8"); have = set(keys(txt)); added = []88for f in sorted((ROOT / "references").glob("lib-ch*.bib")) + sorted((ROOT / "references").glob("lib-app*.bib")):89    for p in re.split(r"(?m)^(?=@\w+\s*\{)", f.read_text(encoding="utf-8")):90        m = re.match(r"@\w+\s*\{\s*([^,\s]+)\s*,", p)91        if m and m.group(1) not in have: have.add(m.group(1)); added.append(p.strip() + "\n")92if added:93    lib.write_text(txt.rstrip() + "\n\n% ===== ENTRÉES AJOUTÉES PAR LES CHAPITRES =====\n" + "\n".join(added), encoding="utf-8")94print(f"library.bib (+chapitres) : {len(have)} clés (+{len(added)})")95