spb/vrai-prix

1// Auteur : Simon-Pierre Boucher — contact@spboucher.ai2/**3 * Connecteur Statistique Canada — Indices des prix de la construction de4 * bâtiments (tableau 18-10-0289-01, trimestriel, licence ouverte) via l'API5 * WDS. Séries : Québec (province), RMR de Québec, Montréal, Ottawa–Gatineau ;6 * bâtiments résidentiels, appartements, maison individuelle, maison en rangée ;7 * agrégat + divisions (Montréal et Québec résidentiel).8 * Code d'indice : statcan:18100289:<geo>:<type>:<division>.9 */10import type { CanonicalIndexObservation, CanonicalObservation, ConnectorConfig, CostConnector, RawDocument, RawObservation, SourceDocument, ValidationResult } from "./types";11import { contentHash } from "./firecrawl";12import { validateObservations } from "./validate";13import { recentPricesFor, referencePriceFor } from "./store";1415export const STATCAN_PARSER_VERSION = "statcan-1.0";16const PID = 18100289;17export const WDS = "https://www150.statcan.gc.ca/t1/wds/rest/getDataFromCubePidCoordAndLatestNPeriods";1819export const STATCAN_CONFIG: ConnectorConfig = {20  key: "statcan", name: "Statistique Canada — IPCB 18-10-0289-01", domain: "www150.statcan.gc.ca", maxPages: 5,21  allowedPaths: [/^https:\/\/www150\.statcan\.gc\.ca\/t1\/wds\/rest\//], excludedPaths: [], refreshDays: 30, timeoutMs: 60_000, retries: 2, rateLimitMs: 1000, concurrency: 1,22};2324export const GEOS: Record<number, string> = { 8: "Québec (province)", 9: "Québec (RMR)", 10: "Montréal", 12: "Ottawa–Gatineau (partie ontarienne)" };25export const TYPES: Record<number, string> = { 1: "Bâtiments résidentiels", 2: "Immeubles d'appartements", 5: "Maison individuelle", 6: "Maison en rangée" };26export const DIVISIONS: Record<number, string> = {27  1: "Agrégat des divisions", 4: "Béton", 5: "Maçonnerie", 8: "Bois, plastiques et composites", 9: "Isolation thermique et étanchéité", 10: "Ouvertures et fermetures", 11: "Revêtements de finition",28  16: "Plomberie", 17: "Chauffage, ventilation et conditionnement d'air", 19: "Électricité", 22: "Terrassements", 23: "Aménagements extérieurs",29};3031export interface SeriesSpec { geo: number; type: number; division: number }3233export function seriesList(): SeriesSpec[] {34  const out: SeriesSpec[] = [];35  for (const geo of [8, 9, 10, 12]) for (const type of [1, 2, 5, 6]) out.push({ geo, type, division: 1 });36  for (const geo of [9, 10]) for (const division of Object.keys(DIVISIONS).map(Number)) if (division !== 1) out.push({ geo, type: 1, division });37  return out;38}3940export const coordinate = (s: SeriesSpec) => `${s.geo}.${s.type}.${s.division}.0.0.0.0.0.0.0`;41export const indexCode = (s: SeriesSpec) => `statcan:${PID}:${s.geo}:${s.type}:${s.division}`;4243interface WdsPoint { refPer: string; value: number | null; statusCode?: number }44interface WdsItem { status: string; object: { coordinate: string; vectorId?: number; vectorDataPoint: WdsPoint[] } }4546export async function fetchWds(specs: SeriesSpec[], latestN = 40, timeoutMs = 60_000): Promise<WdsItem[]> {47  const out: WdsItem[] = [];48  for (let i = 0; i < specs.length; i += 20) {49    const batch = specs.slice(i, i + 20).map((s) => ({ productId: PID, coordinate: coordinate(s), latestN }));50    const ctrl = new AbortController();51    const t = setTimeout(() => ctrl.abort(), timeoutMs);52    try {53      const res = await fetch(WDS, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(batch), signal: ctrl.signal });54      if (!res.ok) throw new Error(`StatCan WDS HTTP ${res.status}`);55      out.push(...((await res.json()) as WdsItem[]));56    } finally {57      clearTimeout(t);58    }59    if (i + 20 < specs.length) await new Promise((r) => setTimeout(r, 1000));60  }61  return out;62}6364/** Points → observations d'indice avec variations trimestrielle et annuelle. */65export function pointsToObservations(spec: SeriesSpec, points: WdsPoint[], retrievedAt: string): CanonicalIndexObservation[] {66  const pts = points.filter((p) => p.value != null && Number.isFinite(p.value)).sort((a, b) => a.refPer.localeCompare(b.refPer));67  return pts.map((p, i) => {68    const prev = i > 0 ? pts[i - 1] : null;69    const yearAgo = pts.find((q) => q.refPer === shiftYear(p.refPer, -1)) ?? null;70    return {71      kind: "index", indexCode: indexCode(spec), geography: GEOS[spec.geo] ?? String(spec.geo), buildingType: TYPES[spec.type] ?? String(spec.type), division: DIVISIONS[spec.division] ?? String(spec.division),72      period: p.refPer, value: p.value as number, pctQoq: prev?.value ? r1(((p.value as number) / prev.value - 1) * 100) : null, pctYoy: yearAgo?.value ? r1(((p.value as number) / yearAgo.value - 1) * 100) : null, retrievedAt,73    };74  });75}7677const r1 = (x: number) => Math.round(x * 10) / 10;78function shiftYear(iso: string, dy: number): string {79  return `${Number(iso.slice(0, 4)) + dy}${iso.slice(4)}`;80}8182export const statcanConnector: CostConnector = {83  config: STATCAN_CONFIG,84  async discover() {85    return [{ url: WDS, title: "StatCan 18-10-0289-01 (WDS)", meta: { series: seriesList().length } }];86  },87  async fetch(doc) {88    const items = await fetchWds(seriesList(), 40, STATCAN_CONFIG.timeoutMs);89    const md = JSON.stringify(items.map((it) => ({ c: it.object?.coordinate, s: it.status, p: it.object?.vectorDataPoint?.map((p) => [p.refPer, p.value]) })));90    // les indices changent à chaque diffusion : le hash porte sur les données, pas sur la date de récupération91    return { url: doc.url, fetchedAt: new Date().toISOString(), markdown: md, metadata: { items: items.length }, statusCode: 200, contentHash: contentHash(md), unchanged: false, meta: doc.meta };92  },93  async extract(raw) {94    const items = JSON.parse(raw.markdown) as { c: string; s: string; p: [string, number | null][] }[];95    return items.filter((it) => it.s === "SUCCESS" && it.c).map((it) => ({96      externalId: it.c, sourceUrl: `https://www150.statcan.gc.ca/t1/tbl1/fr/tv.action?pid=${PID}01`, retrievedAt: raw.fetchedAt, effectiveDate: it.p.at(-1)?.[0] ?? null, title: `IPCB ${it.c}`, description: null, unit: "index", price: it.p.at(-1)?.[1] ?? null,97      regularPrice: null, salePrice: null, currency: "CAD", location: it.c.split(".")[0], category: "index", payload: { points: it.p },98    } satisfies RawObservation));99  },100  async normalize(obs, raw) {101    const out: CanonicalObservation[] = [];102    for (const o of obs) {103      const [geo, type, division] = (o.externalId ?? "").split(".").map(Number);104      const pts = (o.payload.points as [string, number | null][]).map(([refPer, value]) => ({ refPer, value }));105      out.push(...pointsToObservations({ geo, type, division }, pts, raw.fetchedAt));106    }107    return out;108  },109  validate(obs, opts): ValidationResult {110    return validateObservations(obs, { recentPrices: (c) => recentPricesFor(c, 120, opts.db), referencePrice: (c) => referencePriceFor(c, opts.db) });111  },112};113114export type { RawDocument, SourceDocument };115