SPB Git

Repository Public MIT

verify-models.sh 5.3 KB 105 lines shellscript

1#!/usr/bin/env bash2# =============================================================================3#                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4#             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5# =============================================================================6#7# Project          : ORIX Astra8# File             : Scripts/verify-models.sh9# Module           : Scripts10# Repository       : https://git.spboucher.ai/orix-astra11# Created          : 2026-11-2112# Last Modified    : 2026-11-2213#14# Author           : Simon-Pierre Boucher15# Position         : Professor of Finance16# Institution      : Université du Québec en Outaouais (UQO)17# Organization     : SP Boucher18# Website          : https://spboucher.ai19# Git              : https://git.spboucher.ai20#21# Preferred Contact: support@spboucher.ai22# Technical Contact: dev@spboucher.ai23# Security/Admin   : admin@spboucher.ai24#25# Copyright (c) 202626# Simon-Pierre Boucher27# All rights reserved where applicable. This file is governed by the28# applicable project license, copyright notices and relevant law.29# Existing third-party rights and notices remain unaffected.30#31# License          : Proprietary, all rights reserved (see license file)32# License File     : LICENSE33#34# Do not embed credentials, private keys, access tokens,35# certificates or confidential personal information.36#37# =============================================================================38# END OF SP BOUCHER SOURCE HEADER39# =============================================================================4041# Usage: Scripts/verify-models.sh [repo ...]        verify candidates (default: the full list below), one after the other42#        Scripts/verify-models.sh --report           only regenerate docs/catalog/catalog.json and docs/MODEL_COMPATIBILITY.md43#44# For each Hugging Face repository this resolves the current commit and file list from the public Hub API, downloads it through45# AstraLocal's own DownloadCoordinator into build/models/<name>/ (git-ignored), re-hashes every file, loads it with the MLX runtime46# ON THIS MAC, runs a two-turn chat, a mid-stream cancellation, a reload from disk and the fixed benchmark, and writes the measured47# record to build/verify/<name>.json. The catalog and the compatibility table are then generated from those records only.48# Numbers are measurements of this Mac. They say nothing about an iPhone.49#50# Environment: ASTRA_VERIFY_PACKAGE_DIR  package directory to build from (default: the repository root). While other modules of the51#              package do not compile, point it at build/local-pkg (an isolated view: AstraCore + AstraLocal + these tests).52set -uo pipefail53ROOT="$(cd "$(dirname "$0")/.." && pwd)"54PKG="${ASTRA_VERIFY_PACKAGE_DIR:-$ROOT}"55SCRATCH="${ASTRA_VERIFY_SCRATCH:-$ROOT/build/spm-local}"56CANDIDATES=(57  mlx-community/Qwen3-0.6B-4bit58  mlx-community/Qwen2.5-0.5B-Instruct-4bit59  mlx-community/SmolLM2-135M-Instruct60  mlx-community/SmolLM2-135M-Instruct-8bit61  mlx-community/SmolLM2-360M-Instruct62  mlx-community/SmolLM2-360M-Instruct-6bit63  mlx-community/SmolLM2-1.7B-Instruct64  mlx-community/Llama-3.2-1B-Instruct-4bit65  mlx-community/gemma-3-1b-it-qat-4bit66  mlx-community/Qwen3-1.7B-4bit67  mlx-community/Llama-3.2-3B-Instruct-4bit68  mlx-community/Phi-4-mini-instruct-4bit69  mlx-community/Qwen3-4B-4bit70  mlx-community/gemma-3-4b-it-qat-4bit71  # Named in the original candidate list; the Hub has no such repositories (recorded as failures at "resolve").72  mlx-community/SmolLM2-1.7B-Instruct-4bit73  mlx-community/SmolLM2-360M-Instruct-4bit74)75# Release build: a debug build of MLX Swift decodes about 2.5 times slower, which would understate every model.76CONFIG=(-c release -Xswiftc -enable-testing)77REPORT_ONLY=078if [ "${1:-}" = "--report" ]; then REPORT_ONLY=1; shift; fi79if [ "$#" -gt 0 ]; then CANDIDATES=("$@"); fi8081export ASTRA_PROJECT_ROOT="$ROOT"82mkdir -p "$ROOT/build/verify" "$ROOT/build/models"83cd "$PKG" || exit 184echo "== building the test bundle ($PKG)"85swift build "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --build-tests 2>&1 | tail -3 || exit 18687FAILED=088if [ "$REPORT_ONLY" -eq 0 ]; then89  # Fetch everything first, several repositories at once: the Hub limits each connection to about 2 MB/s after an initial burst.90  # Still AstraLocal's DownloadCoordinator; the sequential pass below then finds every file present and verified.91  echo "== prefetching ${#CANDIDATES[@]} repositories"92  ASTRA_PREFETCH_REPOS="$(IFS=,; echo "${CANDIDATES[*]}")" swift test "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --skip-build --filter prefetchModels 2>&1 \93    | grep -E "^\[prefetch" | cut -c1-40094  for repo in "${CANDIDATES[@]}"; do95    echo "== $repo"96    ASTRA_VERIFY_REPO="$repo" swift test "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --skip-build --filter verifyOneModel 2>&1 \97      | grep -E "^\[verify|✔ Test verifyOneModel|✘ Test verifyOneModel|error:" | cut -c1-40098    [ "${PIPESTATUS[0]}" -eq 0 ] || FAILED=$((FAILED + 1))99  done100fi101echo "== generating docs/catalog/catalog.json and docs/MODEL_COMPATIBILITY.md"102ASTRA_VERIFY_REPORT=1 swift test "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --skip-build --filter generateCatalogAndCompatibilityReport 2>&1 \103  | grep -E "^\[report|✔ Test generate|✘ Test generate|error:" | cut -c1-400104echo "== done ($FAILED candidate(s) did not pass)"105
Files main
Menu

Theme