1"""Download Statistics Canada benchmarks via the WDS API (public, no key).23- 18-10-0004-01 CPI monthly NSA: Quebec (geo 11), Montréal (13), Canada (2) × food aggregates (members 3,4,5,21,22,28,29,30,34,39,49,58)4- 18-10-0245-01 Monthly average retail prices for selected products: Quebec, all 110 products5Raw JSON kept in data/raw/statcan/ with download timestamp; tidy CSVs in data/processed/.6"""7from __future__ import annotations8import json, time, requests, pandas as pd9from src.common import ROOT, path, get_logger, ledger1011log = get_logger("statcan")12WDS = "https://www150.statcan.gc.ca/t1/wds/rest/"13RAW = ROOT / "data" / "raw" / "statcan"; RAW.mkdir(parents=True, exist_ok=True)1415def coords(product_id: int, pairs: list[tuple[int, int]], periods: int) -> pd.DataFrame:16 body = [{"productId": product_id, "coordinate": f"{g}.{p}.0.0.0.0.0.0.0.0", "latestN": periods} for g, p in pairs]17 r = requests.post(WDS + "getDataFromCubePidCoordAndLatestNPeriods", json=body, timeout=120); r.raise_for_status()18 (RAW / f"{product_id}_{int(time.time())}.json").write_text(r.text)19 rows = []20 for item in r.json():21 if item["status"] != "SUCCESS":22 log.warning("WDS item failed: %s", item); continue23 o = item["object"]24 for dp in o["vectorDataPoint"]:25 rows.append(dict(productId=product_id, coordinate=o["coordinate"], vectorId=o["vectorId"],26 ref_date=dp["refPer"], value=dp["value"], scalar=dp["scalarFactorCode"], status=dp.get("statusCode")))27 return pd.DataFrame(rows)2829def main() -> None:30 meta = {}31 for pid in (18100004, 18100245):32 r = requests.post(WDS + "getCubeMetadata", json=[{"productId": pid}], timeout=120); r.raise_for_status()33 meta[pid] = r.json()[0]["object"]; (RAW / f"meta_{pid}.json").write_text(r.text)34 # CPI35 m = meta[18100004]; geo = {x["memberId"]: x["memberNameEn"] for x in m["dimension"][0]["member"]}36 prodn = {x["memberId"]: x["memberNameEn"] for x in m["dimension"][1]["member"]}37 food_members = [2, 3, 4, 5, 16, 21, 22, 28, 29, 30, 34, 39, 49, 58]38 cpi = coords(18100004, [(g, p) for g in (2, 11, 13) for p in food_members], periods=60)39 cpi["geo"] = cpi.coordinate.str.split(".").str[0].astype(int).map(geo)40 cpi["product"] = cpi.coordinate.str.split(".").str[1].astype(int).map(prodn)41 cpi.to_csv(path("processed") / "statcan_cpi_food.csv", index=False)42 log.info("CPI: %d points, %s → %s, latest release %s", len(cpi), cpi.ref_date.min(), cpi.ref_date.max(), m.get("releaseTime"))43 # Average retail prices44 m2 = meta[18100245]; geo2 = {x["memberId"]: x["memberNameEn"] for x in m2["dimension"][0]["member"]}45 prod2 = {x["memberId"]: x["memberNameEn"] for x in m2["dimension"][1]["member"]}46 qc = [k for k, v in geo2.items() if v == "Quebec"][0]47 arp = coords(18100245, [(qc, p) for p in prod2], periods=36)48 arp["geo"] = "Quebec"; arp["product"] = arp.coordinate.str.split(".").str[1].astype(int).map(prod2)49 arp.to_csv(path("processed") / "statcan_avg_retail_prices_qc.csv", index=False)50 log.info("Average retail prices QC: %d points, %d products, %s → %s", len(arp), arp["product"].nunique(), arp.ref_date.min(), arp.ref_date.max())51 ledger("1", "statcan", f"CPI {len(cpi)} pts to {cpi.ref_date.max()}; ARP {len(arp)} pts to {arp.ref_date.max()}")5253if __name__ == "__main__":54 main()55