Repository Public MIT
1#!/usr/bin/env bash2# =============================================================================3# SP BOUCHER | INFRASTRUCTURE & AUTOMATION4# ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5# =============================================================================6#7# Project : ORIX Astra8# File : Scripts/verify-models.sh9# Module : Scripts10# Repository : https://git.spboucher.ai/orix-astra11# Created : 2026-11-2112# Last Modified : 2026-11-2213#14# Author : Simon-Pierre Boucher15# Position : Professor of Finance16# Institution : Université du Québec en Outaouais (UQO)17# Organization : SP Boucher18# Website : https://spboucher.ai19# Git : https://git.spboucher.ai20#21# Preferred Contact: support@spboucher.ai22# Technical Contact: dev@spboucher.ai23# Security/Admin : admin@spboucher.ai24#25# Copyright (c) 202626# Simon-Pierre Boucher27# All rights reserved where applicable. This file is governed by the28# applicable project license, copyright notices and relevant law.29# Existing third-party rights and notices remain unaffected.30#31# License : Proprietary, all rights reserved (see license file)32# License File : LICENSE33#34# Do not embed credentials, private keys, access tokens,35# certificates or confidential personal information.36#37# =============================================================================38# END OF SP BOUCHER SOURCE HEADER39# =============================================================================4041# Usage: Scripts/verify-models.sh [repo ...] verify candidates (default: the full list below), one after the other42# Scripts/verify-models.sh --report only regenerate docs/catalog/catalog.json and docs/MODEL_COMPATIBILITY.md43#44# For each Hugging Face repository this resolves the current commit and file list from the public Hub API, downloads it through45# AstraLocal's own DownloadCoordinator into build/models/<name>/ (git-ignored), re-hashes every file, loads it with the MLX runtime46# ON THIS MAC, runs a two-turn chat, a mid-stream cancellation, a reload from disk and the fixed benchmark, and writes the measured47# record to build/verify/<name>.json. The catalog and the compatibility table are then generated from those records only.48# Numbers are measurements of this Mac. They say nothing about an iPhone.49#50# Environment: ASTRA_VERIFY_PACKAGE_DIR package directory to build from (default: the repository root). While other modules of the51# package do not compile, point it at build/local-pkg (an isolated view: AstraCore + AstraLocal + these tests).52set -uo pipefail53ROOT="$(cd "$(dirname "$0")/.." && pwd)"54PKG="${ASTRA_VERIFY_PACKAGE_DIR:-$ROOT}"55SCRATCH="${ASTRA_VERIFY_SCRATCH:-$ROOT/build/spm-local}"56CANDIDATES=(57 mlx-community/Qwen3-0.6B-4bit58 mlx-community/Qwen2.5-0.5B-Instruct-4bit59 mlx-community/SmolLM2-135M-Instruct60 mlx-community/SmolLM2-135M-Instruct-8bit61 mlx-community/SmolLM2-360M-Instruct62 mlx-community/SmolLM2-360M-Instruct-6bit63 mlx-community/SmolLM2-1.7B-Instruct64 mlx-community/Llama-3.2-1B-Instruct-4bit65 mlx-community/gemma-3-1b-it-qat-4bit66 mlx-community/Qwen3-1.7B-4bit67 mlx-community/Llama-3.2-3B-Instruct-4bit68 mlx-community/Phi-4-mini-instruct-4bit69 mlx-community/Qwen3-4B-4bit70 mlx-community/gemma-3-4b-it-qat-4bit71 # Named in the original candidate list; the Hub has no such repositories (recorded as failures at "resolve").72 mlx-community/SmolLM2-1.7B-Instruct-4bit73 mlx-community/SmolLM2-360M-Instruct-4bit74)75# Release build: a debug build of MLX Swift decodes about 2.5 times slower, which would understate every model.76CONFIG=(-c release -Xswiftc -enable-testing)77REPORT_ONLY=078if [ "${1:-}" = "--report" ]; then REPORT_ONLY=1; shift; fi79if [ "$#" -gt 0 ]; then CANDIDATES=("$@"); fi8081export ASTRA_PROJECT_ROOT="$ROOT"82mkdir -p "$ROOT/build/verify" "$ROOT/build/models"83cd "$PKG" || exit 184echo "== building the test bundle ($PKG)"85swift build "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --build-tests 2>&1 | tail -3 || exit 18687FAILED=088if [ "$REPORT_ONLY" -eq 0 ]; then89 # Fetch everything first, several repositories at once: the Hub limits each connection to about 2 MB/s after an initial burst.90 # Still AstraLocal's DownloadCoordinator; the sequential pass below then finds every file present and verified.91 echo "== prefetching ${#CANDIDATES[@]} repositories"92 ASTRA_PREFETCH_REPOS="$(IFS=,; echo "${CANDIDATES[*]}")" swift test "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --skip-build --filter prefetchModels 2>&1 \93 | grep -E "^\[prefetch" | cut -c1-40094 for repo in "${CANDIDATES[@]}"; do95 echo "== $repo"96 ASTRA_VERIFY_REPO="$repo" swift test "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --skip-build --filter verifyOneModel 2>&1 \97 | grep -E "^\[verify|✔ Test verifyOneModel|✘ Test verifyOneModel|error:" | cut -c1-40098 [ "${PIPESTATUS[0]}" -eq 0 ] || FAILED=$((FAILED + 1))99 done100fi101echo "== generating docs/catalog/catalog.json and docs/MODEL_COMPATIBILITY.md"102ASTRA_VERIFY_REPORT=1 swift test "${CONFIG[@]}" --disable-keychain --scratch-path "$SCRATCH" --skip-build --filter generateCatalogAndCompatibilityReport 2>&1 \103 | grep -E "^\[report|✔ Test generate|✘ Test generate|error:" | cut -c1-400104echo "== done ($FAILED candidate(s) did not pass)"105