Repository Public License
1#!/usr/bin/env bash2# ORIX Audio3# File: Scripts/bench-transcription.sh4# Description: Synthesises the reference passages with macOS voices and measures on-device WER / CER (results in Benchmarks/results/).5#6# Author: Simon-Pierre Boucher7# Contact: contact@spboucher.ai8#9# The reference audio is synthetic speech (macOS `say`), so the passages and audio are free to use. Synthetic speech is10# cleaner than real meeting audio: these numbers are an upper bound on accuracy, not a prediction for real rooms.11set -euo pipefail12cd "$(dirname "$0")/.."13OUT=Benchmarks/results; TMP=build/bench; mkdir -p "$OUT" "$TMP"14(cd Core && swift build -c release >/dev/null)15BIN="$(cd Core && swift build -c release --show-bin-path)/orix-bench"16run() { # locale voice17 say -v "$2" -r 170 -f "Benchmarks/reference/$1.txt" -o "$TMP/$1-$2.aiff"18 "$BIN" transcribe "$TMP/$1-$2.aiff" "$1" "Benchmarks/reference/$1.txt" "$TMP/$1-$2.transcript.json" 2>/dev/null > "$OUT/$1-$2.json" || echo "{\"error\": \"failed\", \"locale\": \"$1\"}" > "$OUT/$1-$2.json"19 python3 -c "import json,sys; d=json.load(open('$OUT/$1-$2.json')); print('$1 $2', {k:(round(v,4) if isinstance(v,float) else v) for k,v in d.items() if k in ('wer','cer','engine','audioSeconds','realTimeFactor','referenceWords','timedWords','error')})"20}21run en-US "${EN_VOICE_1:-Samantha}"22run en-US "${EN_VOICE_2:-Daniel}"23run fr-CA "${FR_VOICE_1:-Amélie}"24run fr-CA "${FR_VOICE_2:-Thomas}"25