SPB Git

Repository Public MIT

publish-models.sh 3.5 KB 73 lines shellscript

1#!/usr/bin/env bash2# =============================================================================3#                  SP BOUCHER | INFRASTRUCTURE & AUTOMATION4#             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5# =============================================================================6#7# Project          : ORIX Astra8# File             : Scripts/publish-models.sh9# Module           : Scripts10# Repository       : https://git.spboucher.ai/orix-astra11# Created          : 2026-11-2212# Last Modified    : 2026-11-2213#14# Author           : Simon-Pierre Boucher15# Position         : Professor of Finance16# Institution      : Université du Québec en Outaouais (UQO)17# Organization     : SP Boucher18# Website          : https://spboucher.ai19# Git              : https://git.spboucher.ai20#21# Preferred Contact: support@spboucher.ai22# Technical Contact: dev@spboucher.ai23# Security/Admin   : admin@spboucher.ai24#25# Copyright (c) 202626# Simon-Pierre Boucher27# All rights reserved where applicable. This file is governed by the28# applicable project license, copyright notices and relevant law.29# Existing third-party rights and notices remain unaffected.30#31# License          : Proprietary, all rights reserved (see license file)32# License File     : LICENSE33#34# Do not embed credentials, private keys, access tokens,35# certificates or confidential personal information.36#37# =============================================================================38# END OF SP BOUCHER SOURCE HEADER39# =============================================================================4041# Usage: Scripts/publish-models.sh [catalog.json]42# Publishes the model library to the node that serves https://www.orixlab.dev/astra/models/v1/ (m4mm, internal SSD,43# ~/apps-data/orix-models/v1). For every model the catalogue marks as redistributable it copies exactly the files the44# catalogue lists, re-computes their SHA-256 on the node, and only then installs the catalogue itself. Models whose45# licence does not allow re-hosting are never copied: the app downloads those from Hugging Face.46set -euo pipefail47cd "$(dirname "$0")/.."48CATALOG="${1:-docs/catalog/catalog.json}"49NODE="${ASTRA_MODELS_NODE:-m4mm}"50ROOT="apps-data/orix-models/v1"51ssh "$NODE" "mkdir -p ~/$ROOT/files"52python3 - "$CATALOG" > build/publish-plan.tsv <<'PY'53import json,sys54c=json.load(open(sys.argv[1]))55for m in c["models"]:56    if not m.get("redistributable") or not m.get("baseURL"): continue57    for f in m["files"]: print(m["id"], m["repo"].split("/")[-1], f["path"], f["size"], f["sha256"], sep="\t")58PY59for id in $(cut -f1 build/publish-plan.tsv | sort -u); do60  dir="build/models/$(awk -F'\t' -v i="$id" '$1==i{print $2; exit}' build/publish-plan.tsv)"61  awk -F'\t' -v i="$id" '$1==i{print $3}' build/publish-plan.tsv > build/publish-files.txt62  echo "== $id ($(wc -l < build/publish-files.txt | tr -d ' ') files)"63  ssh "$NODE" "mkdir -p ~/$ROOT/files/$id"64  rsync -a --partial --files-from=build/publish-files.txt "$dir/" "$NODE:$ROOT/files/$id/"65  awk -F'\t' -v i="$id" '$1==i{print $5"  "$3}' build/publish-plan.tsv > build/publish-sums.txt66  scp -q build/publish-sums.txt "$NODE:$ROOT/files/$id/.sha256sums"67  ssh "$NODE" "cd ~/$ROOT/files/$id && shasum -a 256 -c .sha256sums --quiet && rm .sha256sums && echo '   checksums verified on the node'"68done69scp -q "$CATALOG" "$NODE:$ROOT/catalog.json.new"70ssh "$NODE" "cd ~/$ROOT && mv catalog.json.new catalog.json && du -sh . | cut -f1"71rm -f build/publish-plan.tsv build/publish-files.txt build/publish-sums.txt72echo "published: https://www.orixlab.dev/astra/models/v1/catalog.json"73
Files main
Menu

Theme