Repository Public License
1#!/bin/bash2# ORIX Convert — Scripts/bench.sh3# Description: Compact, repeatable end-to-end benchmark through the `orix-convert` tool (release build): every case is4# one `convert` invocation — detector, planner, queue, engine, validation, safe writer — measured with5# `/usr/bin/time -l` (wall time, CPU time, peak resident size). Writes docs/reports/benchmarks.{md,json}6# with the machine description and the load average at the time of each case.7# Author: Simon-Pierre Boucher — contact@spboucher.ai8#9# Usage: Scripts/bench.sh [case-name-substring] ORIX_BENCH_WORK=<dir> keeps the generated inputs between runs.10# ORIX_BENCH_IMAGES=100 ORIX_BENCH_CSV_BYTES=1000000000 ORIX_BENCH_ARCHIVE_BYTES=500000000 scale the inputs.11# A case is run a second time when the 1-minute load average exceeded twice the number of cores when it started;12# both runs are kept.13set -uo pipefail14export LC_ALL=C15ROOT="$(cd "$(dirname "$0")/.." && pwd)"16ORIX_CONFIGURATION=release17source "$ROOT/Scripts/cli-build.sh"18orix_build_cli || exit 11920FILTER="${1:-}"21WORK="${ORIX_BENCH_WORK:-${TMPDIR:-/tmp}/orix-convert-bench}"22IN="$WORK/in"; OUT="$WORK/out"23IMAGES="${ORIX_BENCH_IMAGES:-100}"24CSV_BYTES="${ORIX_BENCH_CSV_BYTES:-1000000000}"25ARCHIVE_BYTES="${ORIX_BENCH_ARCHIVE_BYTES:-500000000}"26CORES=$(sysctl -n hw.ncpu)27mkdir -p "$IN" "$ROOT/docs/reports"28rm -rf "$OUT"; mkdir -p "$OUT"29CLEAN=(env -i "PATH=/usr/bin:/bin" "HOME=$HOME" "TMPDIR=${TMPDIR:-/tmp}")30cli() { "${CLEAN[@]}" "$ORIX_CLI" "$@"; }31RESULTS="$WORK/results.tsv"32: > "$RESULTS"3334echo "bench.sh: preparing inputs in $IN (generated in code; reused when present)" >&235[ -d "$IN/png" ] && [ "$(ls "$IN/png" | wc -l | tr -d ' ')" -eq "$IMAGES" ] || { rm -rf "$IN/png"; cli dev seed images "$IN/png" --count "$IMAGES" --width 4000 --height 3000 >&2; }36if [ ! -d "$IN/heic" ] || [ "$(ls "$IN/heic" | wc -l | tr -d ' ')" -ne "$IMAGES" ]; then37 rm -rf "$IN/heic"; cli convert "$IN/png" --recursive --to heic --output-dir "$IN/heic" --no-history --quiet >&238fi39[ -f "$IN/video-4k.mp4" ] || cli dev seed video "$IN/video-4k.mp4" --seconds 10 --width 3840 --height 2160 --codec h264_videotoolbox >&240[ -f "$IN/tone-5min.wav" ] || cli dev seed wav "$IN/tone-5min.wav" --seconds 300 >&241[ -f "$IN/text-100p.pdf" ] || cli dev seed pdf "$IN/text-100p.pdf" --pages 100 >&242[ -f "$IN/scan-20p.pdf" ] || cli dev seed pdf "$IN/scan-20p.pdf" --pages 20 --scanned >&243[ -f "$IN/table.csv" ] && [ "$(stat -f %z "$IN/table.csv")" -ge "$CSV_BYTES" ] || cli dev seed csv "$IN/table.csv" --bytes "$CSV_BYTES" >&244if [ ! -f "$IN/report-50p.docx" ]; then45 cli dev seed markdown "$IN/report-50p.md" --pages 50 >&246 cli convert "$IN/report-50p.md" --to docx --output "$IN/report-50p.docx" --no-history --quiet >&247fi48HAVE_ARCHIVE=049if cli paths --from folder --to zip 2>/dev/null | grep -q 'archive.create'; then50 HAVE_ARCHIVE=151 [ -d "$IN/tree" ] || cli dev seed folder "$IN/tree" --bytes "$ARCHIVE_BYTES" >&252fi53size_mb() { du -sk "$1" | awk '{printf "%.1f", $1/1024}'; }54size_h() { du -sk "$1" | awk '{ if ($1 < 1024) printf "%d KB", $1; else printf "%.1f MB", $1/1024 }'; }55NOTES="$WORK/notes.txt"56: > "$NOTES"5758# bench <name> <units> <unit label> <input description> -- <orix-convert arguments…>59bench() {60 local name="$1" units="$2" label="$3" input="$4"; shift 561 if [ -n "$FILTER" ] && [[ "$name" != *"$FILTER"* ]]; then return 0; fi62 local run=163 while :; do64 local load log status real user sys rss65 load=$(sysctl -n vm.loadavg | awk '{print $2}')66 log="$WORK/time.log"67 rm -rf "$OUT/case"; mkdir -p "$OUT/case"68 /usr/bin/time -l "${CLEAN[@]}" "$ORIX_CLI" "$@" --no-history --quiet > "$WORK/case.out" 2> "$log"69 status=$?70 real=$(awk '/ real / {print $1}' "$log" | tail -1)71 user=$(awk '/ real / {print $3}' "$log" | tail -1)72 sys=$(awk '/ real / {print $5}' "$log" | tail -1)73 rss=$(awk '/maximum resident set size/ {print $1}' "$log" | tail -1)74 local outputs output_mb described75 outputs=$(find "$OUT/case" -type f | wc -l | tr -d ' ')76 output_mb=$(size_mb "$OUT/case")77 described="$outputs file(s), $output_mb MB"78 if [ "$label" = "pages (of the PDF produced)" ]; then79 # The page count is read from the PDF that was produced, after the measurement.80 local pages81 pages=$(cli inspect "$(find "$OUT/case" -name '*.pdf' | head -1)" --json 2>/dev/null | awk -F: '/"pageCount"/ {gsub(/[ ,]/, "", $2); print $2; exit}')82 units="${pages:-0}"; described="$described, ${pages:-?} pages"83 fi84 printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$name" "$run" "$status" "$real" "$user" "$sys" "${rss:-0}" "$load" "$units" "$label" "$input" "$described" >> "$RESULTS"85 if [ "$status" -ne 0 ]; then86 { echo "- **$name** (run $run, exit $status):"; grep -h -E '^(FAILED|cancelled|Summary|orix-convert:)' "$WORK/case.out" "$log" | cut -c1-300 | head -8 | sed 's/^/ - `/; s/$/`/'; } >> "$NOTES"87 fi88 echo "bench.sh: $name (run $run): exit $status, ${real}s wall, load $load" >&289 # Extreme load: run once more and keep both.90 if [ "$run" -eq 1 ] && awk -v l="$load" -v c="$CORES" 'BEGIN { exit !(l > 2 * c) }'; then run=2; else break; fi91 done92}9394PNG_MB=$(size_mb "$IN/png"); HEIC_MB=$(size_mb "$IN/heic")95bench "PNG → JPEG" "$IMAGES" "images" "$IMAGES PNG, 4000 × 3000 (12 MP), $PNG_MB MB" -- convert "$IN/png" --recursive --to jpeg --output-dir "$OUT/case"96bench "PNG → WebP" "$IMAGES" "images" "$IMAGES PNG, 4000 × 3000 (12 MP), $PNG_MB MB" -- convert "$IN/png" --recursive --to webp --output-dir "$OUT/case"97bench "PNG → SVG (image.vectorize)" "$IMAGES" "images" "$IMAGES PNG, 4000 × 3000 (12 MP), $PNG_MB MB" -- convert "$IN/png" --recursive --to svg --operation image.vectorize --output-dir "$OUT/case"98bench "HEIC → JPEG" "$IMAGES" "images" "$IMAGES HEIC, 4000 × 3000 (12 MP), $HEIC_MB MB" -- convert "$IN/heic" --recursive --to jpeg --output-dir "$OUT/case"99bench "4K H.264 → HEVC (hardware encoder)" "10" "s of video" "3840 × 2160, 30 fps, 10 s, $(size_mb "$IN/video-4k.mp4") MB" -- convert "$IN/video-4k.mp4" --to mp4 --option video.codec=hevc --option video.encoder=hardware --output-dir "$OUT/case"100bench "WAV → MP3" "300" "s of audio" "5 min, 44.1 kHz stereo 16-bit, $(size_mb "$IN/tone-5min.wav") MB" -- convert "$IN/tone-5min.wav" --to mp3 --output-dir "$OUT/case"101bench "WAV → AAC (M4A)" "300" "s of audio" "5 min, 44.1 kHz stereo 16-bit, $(size_mb "$IN/tone-5min.wav") MB" -- convert "$IN/tone-5min.wav" --to m4a --output-dir "$OUT/case"102bench "WAV → FLAC" "300" "s of audio" "5 min, 44.1 kHz stereo 16-bit, $(size_mb "$IN/tone-5min.wav") MB" -- convert "$IN/tone-5min.wav" --to flac --output-dir "$OUT/case"103bench "PDF → PNG, 150 dpi" "100" "pages" "100 text pages, US Letter, $(size_mb "$IN/text-100p.pdf") MB" -- convert "$IN/text-100p.pdf" --to png --option resolution.dpi=150 --output-dir "$OUT/case"104bench "PDF OCR (pdf.ocr)" "20" "pages" "20 page images at 150 dpi without text layer, $(size_mb "$IN/scan-20p.pdf") MB" -- convert "$IN/scan-20p.pdf" --to pdf --operation pdf.ocr --output-dir "$OUT/case"105CSV_MB=$(size_mb "$IN/table.csv")106bench "CSV → Parquet" "$CSV_MB" "MB of CSV" "$CSV_MB MB, 9 columns" -- convert "$IN/table.csv" --to parquet --output-dir "$OUT/case"107bench "DOCX → PDF" "0" "pages (of the PDF produced)" "$(size_h "$IN/report-50p.docx") DOCX made by the documents engine from generated Markdown (250 paragraphs, 10 tables)" -- convert "$IN/report-50p.docx" --to pdf --output-dir "$OUT/case"108if [ "$HAVE_ARCHIVE" -eq 1 ]; then109 TREE_MB=$(size_mb "$IN/tree")110 bench "Folder → ZIP (archive.create)" "$TREE_MB" "MB" "$TREE_MB MB in $(find "$IN/tree" -type f | wc -l | tr -d ' ') files: half repetitive text (very compressible), half pseudo-random bytes (incompressible)" -- convert "$IN/tree" --to zip --operation archive.create --output-dir "$OUT/case"111 if [ -z "$FILTER" ] || [[ "ZIP → folder (archive.extract)" == *"$FILTER"* ]]; then112 rm -rf "$WORK/tree.zip"; cli convert "$IN/tree" --to zip --operation archive.create --output "$WORK/tree.zip" --no-history --quiet >&2113 bench "ZIP → folder (archive.extract)" "$TREE_MB" "MB" "ZIP of the same tree, $(size_mb "$WORK/tree.zip") MB" -- convert "$WORK/tree.zip" --to folder --operation archive.extract --output-dir "$OUT/case"114 fi115fi116rm -rf "$OUT"117118# Reports.119MD="$ROOT/docs/reports/benchmarks.md"; JSON="$ROOT/docs/reports/benchmarks.json"120[ -n "$FILTER" ] && { MD="$WORK/benchmarks-partial.md"; JSON="$WORK/benchmarks-partial.json"; }121MACHINE="$(sysctl -n machdep.cpu.brand_string), $CORES cores, $(( $(sysctl -n hw.memsize) / 1073741824 )) GB, macOS $(sw_vers -productVersion)"122{123 echo "# ORIX Convert — end-to-end benchmarks"124 echo125 echo "> Generated file — do not edit. Written by \`Scripts/bench.sh\` on $(date -u +%Y-%m-%dT%H:%M:%SZ)."126 echo127 echo "Every row is one \`orix-convert convert\` invocation of the release build, measured with \`/usr/bin/time -l\`: the whole pipeline"128 echo "(detection, planning, queue, engine, validation of every output, safe writer), process start-up included. Inputs are generated in code"129 echo "(\`orix-convert dev seed\`) and are not timed. Default settings of each path unless the case names one; default concurrency of the queue"130 echo "(one job per performance core, at most 8; video and LibreOffice jobs limited as in the app)."131 echo132 echo "- Machine: $MACHINE"133 echo "- **This Mac is shared and was under load during the run.** The 1-minute load average at the start of each case is in the table; with $CORES cores,"134 echo " a load above $CORES means the cases competed for processors. A case that started with a load above $((2 * CORES)) was run a second time and both runs are listed."135 echo " These are observations on a loaded machine, not best cases; nothing was repeated to compute a variance."136 echo "- Wall = elapsed time. CPU % = (user + system time) / wall, as reported by \`time\` for the tool and the helper processes it waited for."137 echo " Peak RSS = maximum resident set size reported by \`time -l\` for the \`orix-convert\` process (memory of helper processes such as ffmpeg"138 echo " or LibreOffice is not included in that figure)."139 echo140 echo "| Case | Run | Input | Exit | Wall (s) | Throughput | CPU % | Peak RSS (MB) | Output | Load avg (1 min) |"141 echo "|---|---:|---|---:|---:|---|---:|---:|---|---:|"142 awk -F'\t' '{143 u = $10; sub(/ \(of the PDF produced\)/, "", u)144 thr = ($4 > 0 && $9 > 0) ? sprintf("%.1f %s/s", $9 / $4, u) : "—"145 cpu = ($4 > 0) ? sprintf("%.0f", ($5 + $6) / $4 * 100) : "—"146 printf "| %s | %s | %s | %s | %.2f | %s | %s | %.0f | %s | %s |\n", $1, $2, $11, $3, $4, thr, cpu, $7 / 1048576, $12, $8147 }' "$RESULTS"148 echo149 FAILED=$(awk -F'\t' '$3 != 0' "$RESULTS" | wc -l | tr -d ' ')150 echo "$(wc -l < "$RESULTS" | tr -d ' ') measurement(s), $FAILED with a non-zero exit status."151 if [ -s "$NOTES" ]; then152 echo153 echo "## Cases that did not end with status 0"154 echo155 echo "The time of such a row covers the whole batch, including the files that failed; what the tool printed:"156 echo157 cat "$NOTES"158 fi159} > "$MD"160{161 echo "{"162 echo " \"date\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\","163 echo " \"machine\": \"$MACHINE\","164 echo " \"cores\": $CORES,"165 echo " \"build\": \"release\","166 echo " \"results\": ["167 awk -F'\t' 'BEGIN { first = 1 } {168 gsub(/"/, "\\\"", $11)169 if (!first) printf ",\n"; first = 0170 printf " {\"case\": \"%s\", \"run\": %s, \"exitStatus\": %s, \"wallSeconds\": %s, \"userSeconds\": %s, \"systemSeconds\": %s, \"peakRSSBytes\": %s, \"loadAverage1m\": %s, \"units\": %s, \"unit\": \"%s\", \"input\": \"%s\", \"output\": \"%s\"}", $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12171 } END { printf "\n" }' "$RESULTS"172 echo " ]"173 echo "}"174} > "$JSON"175echo "bench.sh: wrote $MD" >&2176