spb/wpai0002_uqo

1#!/usr/bin/env bash2# Assemble the Medium-stage corpus from every raw source directory, with splits that preserve the pilot assignments3# (documents already rewritten keep their split). Usage: scripts/build_medium_corpus.sh [out_name] [heldout_disciplines] [heldout_before_year]4set -euo pipefail5cd "$(dirname "$0")/.."6PY=.venv/bin/python7NAME=${1:-medium_v1}8HELD=${2:-physics,earth_sciences}9BEFORE=${3:-1900}10FIXED=${4:-data/splits/pilot_pmc/splits.jsonl}   # previous assignments to preserve (pilot, or an earlier medium version)11RAW=(data/raw/pmc data/raw/arxiv data/raw/crossref data/raw/books/gutenberg data/raw/books/bookshelf data/raw/books/openstax)12EXISTING=()13for d in "${RAW[@]}"; do [ -d "$d" ] && EXISTING+=("$d"); done14echo "raw dirs: ${EXISTING[*]}"15$PY scripts/build_corpus.py --raw "${EXISTING[@]}" --out "data/parsed/$NAME" --tag "$NAME"16$PY scripts/relabel_disciplines.py --corpus "data/parsed/$NAME"17$PY scripts/make_splits.py --corpus "data/parsed/$NAME" --heldout-disciplines "$HELD" --heldout-before-year "$BEFORE" --fixed "$FIXED"18echo "MEDIUM CORPUS BUILT: data/parsed/$NAME ; splits data/splits/$NAME/splits.jsonl"19