# -*- coding: utf-8 -*-
"""04b — Rassemble les fragments (shards) de la rétro-évaluation calculés sur plusieurs nœuds, dans l'ordre des cibles."""
import glob
from common import *
sh = sorted(glob.glob(str(OUT / "targets.shard*.parquet")))
log(f"{len(sh)} fragments")
Ts = [pd.read_parquet(p) for p in sh]
parts = {}
for name in ["est_E1", "est_E2", "sd_E2", "n_avail"]:
    parts[name] = [np.load(p.replace("targets.", f"{name}.").replace(".parquet", ".npy")) for p in sh]
T = pd.concat(Ts, ignore_index=True); order = np.argsort(T["idx"].to_numpy(), kind="stable")
T = T.iloc[order].reset_index(drop=True); save_parquet(T, "targets.parquet")
for name, arrs in parts.items():
    A = np.concatenate(arrs, axis=0)[order]; np.save(OUT / f"{name}.npy", A); log(f"  {name}: {A.shape}")
log("rassemblement terminé")