SPB Git

Repository Public

trim-fixtures.py 2.5 KB 59 lines python

1#!/usr/bin/env python32"""Trims recorded backend responses into test fixtures: structure kept, long arrays shortened.34Rules (deterministic):5  - an array of scalars or of arrays longer than MAX_POINTS is decimated with a uniform stride that keeps the6    first and last element; arrays of equal length get the same indices, so parallel arrays (x / y / lower /7    upper of a chart series, bars) stay aligned;8  - an array of objects longer than MAX_OBJECTS keeps its first MAX_OBJECTS elements;9  - nothing else is touched (no key is removed, no value rewritten).10Usage: trim-fixtures.py <raw-dir> <fixtures-dir>11"""12import json, os, sys1314MAX_POINTS = 16015MAX_OBJECTS = 6016SKIP = {"sessions_list.json"}                      # real conversation titles: not a fixture17KEEP_WHOLE = {"quant_tools.json", "quant_tools_legacy_client.json"}1819def pick(n):20    if n <= MAX_POINTS: return list(range(n))21    step = (n - 1) / (MAX_POINTS - 1)22    return sorted({round(i * step) for i in range(MAX_POINTS)})2324def trim(v):25    if isinstance(v, dict):26        return {k: trim(x) for k, x in v.items()}27    if isinstance(v, list):28        if v and all(isinstance(x, dict) for x in v):29            return [trim(x) for x in v[:MAX_OBJECTS]]30        if len(v) > MAX_POINTS:31            return [trim(v[i]) for i in pick(len(v))]32        return [trim(x) for x in v]33    return v3435def main(raw, out):36    os.makedirs(out, exist_ok=True)37    for name in sorted(os.listdir(raw)):38        if name in SKIP or name.startswith("job_start_"): continue39        if name.startswith("job_running_") and name != "job_running_volatility_lab.json": continue40        src = os.path.join(raw, name)41        if name.endswith(".txt"):42            text = open(src, encoding="utf-8").read()43            assert "vqn_" not in text44            open(os.path.join(out, name), "w", encoding="utf-8").write(text)45            continue46        if not name.endswith(".json"): continue47        text = open(src, encoding="utf-8").read()48        assert "vqn_" not in text, f"token-looking string in {name}"49        data = json.loads(text)50        if name not in KEEP_WHOLE: data = trim(data)51        with open(os.path.join(out, name), "w", encoding="utf-8") as f:52            json.dump(data, f, ensure_ascii=False, separators=(",", ":"))53            f.write("\n")54    total = sum(os.path.getsize(os.path.join(out, n)) for n in os.listdir(out))55    print(f"{len(os.listdir(out))} fixtures, {total // 1024} KB")5657if __name__ == "__main__":58    main(sys.argv[1], sys.argv[2])59
Files main
Menu

Theme