SPB Git

Repository Public

model.py 4.0 KB 115 lines python

1"""Téléchargement épinglé et vérification d'intégrité des poids d'un modèle du catalogue.23    python scripts/model.py fetch  maclustr-qwen3     télécharge la révision exacte de config/models.json4    python scripts/model.py verify maclustr-qwen3     compare taille + SHA-256 de chaque fichier à ceux de5                                                      Hugging Face pour cette révision, écrit un manifeste67La vérification recalcule les condensats localement : elle détecte un téléchargement tronqué ou altéré.8"""910from __future__ import annotations1112import hashlib13import json14import os15import subprocess16import sys17import time18import urllib.request19from pathlib import Path2021ROOT = Path(__file__).resolve().parents[1]222324def spec(model_id: str) -> dict:25    catalog = Path(os.environ.get("MLLM_CATALOG", ROOT / "config" / "models.json"))26    for m in json.loads(catalog.read_text())["models"]:27        if m["id"] == model_id or m["repo"] == model_id:28            m["dir"] = Path(m["engine_model"].replace("~", str(Path.home()), 1))29            return m30    raise SystemExit(f"modèle inconnu dans le catalogue : {model_id}")313233def remote_files(repo: str, revision: str) -> dict[str, dict]:34    url = f"https://huggingface.co/api/models/{repo}/revision/{revision}?blobs=true"35    with urllib.request.urlopen(url, timeout=30) as r:  # noqa: S310 - URL https fixe36        info = json.load(r)37    if info["sha"] != revision:38        raise SystemExit(f"révision inattendue : {info['sha']}")39    return {s["rfilename"]: s for s in info["siblings"]}404142def git_blob_sha1(path: Path) -> str:43    h = hashlib.sha1(f"blob {path.stat().st_size}\0".encode())  # noqa: S324 - identifiant git, pas un usage crypto44    with path.open("rb") as fh:45        for chunk in iter(lambda: fh.read(1 << 22), b""):46            h.update(chunk)47    return h.hexdigest()484950def sha256(path: Path) -> str:51    h = hashlib.sha256()52    with path.open("rb") as fh:53        for chunk in iter(lambda: fh.read(1 << 22), b""):54            h.update(chunk)55    return h.hexdigest()565758def fetch(m: dict) -> None:59    hf = Path(sys.executable).parent / "hf"60    env = {**os.environ, "HF_HUB_DISABLE_XET": "1"}  # le client Xet se fige sur ce réseau ; HTTP direct61    subprocess.run(62        [str(hf), "download", m["repo"], "--revision", m["revision"], "--local-dir", str(m["dir"])], check=True, env=env63    )646566def verify(m: dict) -> bool:67    files = remote_files(m["repo"], m["revision"])68    manifest, ok = [], True69    for name, meta in sorted(files.items()):70        if name in (".gitattributes", "README.md"):71            continue72        path = m["dir"] / name73        if not path.exists():74            print(f"MANQUANT  {name}")75            ok = False76            continue77        lfs = meta.get("lfs")78        if lfs:79            digest, expected, kind = sha256(path), lfs["sha256"], "sha256"80        else:81            digest, expected, kind = git_blob_sha1(path), meta["blobId"], "git-sha1"82        good = digest == expected and path.stat().st_size == (lfs or meta)["size"]83        ok &= good84        print(f"{'ok       ' if good else 'DIFFÉRENT'} {name}  {kind}:{digest[:16]}…  {path.stat().st_size} octets")85        manifest.append({"file": name, "size": path.stat().st_size, kind: digest, "verified": good})86    out = m["dir"] / "INTEGRITY.json"87    out.write_text(88        json.dumps(89            {90                "repo": m["repo"],91                "revision": m["revision"],92                "license": m["license"],93                "verified_at": time.strftime("%Y-%m-%d %H:%M:%S"),94                "ok": ok,95                "files": manifest,96            },97            indent=1,98        )99    )100    print(f"\n{'INTÉGRITÉ VÉRIFIÉE' if ok else 'ÉCHEC DE VÉRIFICATION'} — {m['repo']}@{m['revision'][:12]} → {out}")101    return ok102103104def main() -> None:105    if len(sys.argv) != 3 or sys.argv[1] not in ("fetch", "verify"):106        raise SystemExit(__doc__)107    m = spec(sys.argv[2])108    if sys.argv[1] == "fetch":109        fetch(m)110    raise SystemExit(0 if verify(m) else 1)111112113if __name__ == "__main__":114    main()115
Files main
Menu

Theme