TeX 55.1%
Python 44.2%
1import csv, sys, re, collections2src, pid, downloaded_at, outpath, zipurl = sys.argv[1:6]3rows = list(csv.DictReader(open(src, encoding="utf-8-sig")))4hdr = list(rows[0].keys()); print(pid, "columns:", hdr, "n", len(rows))5dimcols = [c for c in hdr if c not in ("REF_DATE","GEO","DGUID","UOM","UOM_ID","SCALAR_FACTOR","SCALAR_ID","VECTOR","COORDINATE","VALUE","STATUS","SYMBOL","TERMINATED","DECIMALS")]6print("extra dims:", dimcols)7for c in ["GEO"] + dimcols:8 print(c, "->", sorted(set(r[c] for r in rows))[:60])9geo_keep = {"Canada","Quebec","Québec, Quebec","Sherbrooke, Quebec","Trois-Rivières, Quebec","Montréal, Quebec","Ottawa-Gatineau, Quebec part, Ontario/Quebec",10 "Montréal, Quebec, census metropolitan area","Québec, Quebec, census metropolitan area","Ottawa-Gatineau, Quebec part, Ontario/Quebec, census metropolitan area"}11geo_found = sorted(set(r["GEO"] for r in rows if (any(r["GEO"].startswith(g.split(",")[0]) for g in geo_keep) and "Ontario part" not in r["GEO"]) or r["GEO"].startswith("Six census")))12print("geo matched:", geo_found)13if pid == "18100205":14 srcname = "Statistics Canada, Table 18-10-0205-01 New housing price index, monthly"15 freq = "monthly"; scope_base = "new houses sold by builders (contractors' selling prices), "16 base = "2016-12=100"17else:18 srcname = "Statistics Canada, Table 18-10-0169-01 Residential property price index, quarterly (inactive)"19 freq = "quarterly"; scope_base = ""20 base = "2017=100"21out = []; cov = collections.defaultdict(lambda: [None,None,0])22def period(ref):23 if freq == "monthly": return ref[:7]24 y, m = ref[:4], int(ref[5:7]); return f"{y}-Q{(m-1)//3+1}"25for r in rows:26 if r["GEO"] not in geo_found: continue27 p = period(r["REF_DATE"])28 if p < "2021-01" or p > "2026-10": continue29 if r["VALUE"] in ("", None): continue30 dimdesc = " | ".join(r[c] for c in dimcols)31 uom = r["UOM"]32 sname = f"{dimdesc} [{uom}; vector {r['VECTOR']}]"33 scope = scope_base + dimdesc34 out.append({"source": srcname, "series_name": sname, "geography": r["GEO"], "property_scope": scope,35 "frequency": freq, "period": p, "value": r["VALUE"], "base_period": base if "index" in uom.lower() else uom,36 "download_url": zipurl, "downloaded_at": downloaded_at})37 k = (r["GEO"], dimdesc); c = cov[k]38 c[0] = c[0] or p; c[1] = p; c[2] += 139cols = ["source","series_name","geography","property_scope","frequency","period","value","base_period","download_url","downloaded_at"]40with open(outpath, "w", newline="") as f:41 w = csv.DictWriter(f, fieldnames=cols); w.writeheader(); w.writerows(out)42for k, v in sorted(cov.items()): print("coverage", k, v)43print("rows written", len(out))44