Repository Public
1"""Téléchargement épinglé et vérification d'intégrité des poids d'un modèle du catalogue.23 python scripts/model.py fetch maclustr-qwen3 télécharge la révision exacte de config/models.json4 python scripts/model.py verify maclustr-qwen3 compare taille + SHA-256 de chaque fichier à ceux de5 Hugging Face pour cette révision, écrit un manifeste67La vérification recalcule les condensats localement : elle détecte un téléchargement tronqué ou altéré.8"""910from __future__ import annotations1112import hashlib13import json14import os15import subprocess16import sys17import time18import urllib.request19from pathlib import Path2021ROOT = Path(__file__).resolve().parents[1]222324def spec(model_id: str) -> dict:25 catalog = Path(os.environ.get("MLLM_CATALOG", ROOT / "config" / "models.json"))26 for m in json.loads(catalog.read_text())["models"]:27 if m["id"] == model_id or m["repo"] == model_id:28 m["dir"] = Path(m["engine_model"].replace("~", str(Path.home()), 1))29 return m30 raise SystemExit(f"modèle inconnu dans le catalogue : {model_id}")313233def remote_files(repo: str, revision: str) -> dict[str, dict]:34 url = f"https://huggingface.co/api/models/{repo}/revision/{revision}?blobs=true"35 with urllib.request.urlopen(url, timeout=30) as r: # noqa: S310 - URL https fixe36 info = json.load(r)37 if info["sha"] != revision:38 raise SystemExit(f"révision inattendue : {info['sha']}")39 return {s["rfilename"]: s for s in info["siblings"]}404142def git_blob_sha1(path: Path) -> str:43 h = hashlib.sha1(f"blob {path.stat().st_size}\0".encode()) # noqa: S324 - identifiant git, pas un usage crypto44 with path.open("rb") as fh:45 for chunk in iter(lambda: fh.read(1 << 22), b""):46 h.update(chunk)47 return h.hexdigest()484950def sha256(path: Path) -> str:51 h = hashlib.sha256()52 with path.open("rb") as fh:53 for chunk in iter(lambda: fh.read(1 << 22), b""):54 h.update(chunk)55 return h.hexdigest()565758def fetch(m: dict) -> None:59 hf = Path(sys.executable).parent / "hf"60 env = {**os.environ, "HF_HUB_DISABLE_XET": "1"} # le client Xet se fige sur ce réseau ; HTTP direct61 subprocess.run(62 [str(hf), "download", m["repo"], "--revision", m["revision"], "--local-dir", str(m["dir"])], check=True, env=env63 )646566def verify(m: dict) -> bool:67 files = remote_files(m["repo"], m["revision"])68 manifest, ok = [], True69 for name, meta in sorted(files.items()):70 if name in (".gitattributes", "README.md"):71 continue72 path = m["dir"] / name73 if not path.exists():74 print(f"MANQUANT {name}")75 ok = False76 continue77 lfs = meta.get("lfs")78 if lfs:79 digest, expected, kind = sha256(path), lfs["sha256"], "sha256"80 else:81 digest, expected, kind = git_blob_sha1(path), meta["blobId"], "git-sha1"82 good = digest == expected and path.stat().st_size == (lfs or meta)["size"]83 ok &= good84 print(f"{'ok ' if good else 'DIFFÉRENT'} {name} {kind}:{digest[:16]}… {path.stat().st_size} octets")85 manifest.append({"file": name, "size": path.stat().st_size, kind: digest, "verified": good})86 out = m["dir"] / "INTEGRITY.json"87 out.write_text(88 json.dumps(89 {90 "repo": m["repo"],91 "revision": m["revision"],92 "license": m["license"],93 "verified_at": time.strftime("%Y-%m-%d %H:%M:%S"),94 "ok": ok,95 "files": manifest,96 },97 indent=1,98 )99 )100 print(f"\n{'INTÉGRITÉ VÉRIFIÉE' if ok else 'ÉCHEC DE VÉRIFICATION'} — {m['repo']}@{m['revision'][:12]} → {out}")101 return ok102103104def main() -> None:105 if len(sys.argv) != 3 or sys.argv[1] not in ("fetch", "verify"):106 raise SystemExit(__doc__)107 m = spec(sys.argv[2])108 if sys.argv[1] == "fetch":109 fetch(m)110 raise SystemExit(0 if verify(m) else 1)111112113if __name__ == "__main__":114 main()115