spb/vrai-prix

1// Auteur : Simon-Pierre Boucher — contact@spboucher.ai2// Tests des connecteurs sur FIXTURES enregistrées (jamais le Web live).3import fs from "fs";4import path from "path";5import { describe, expect, it } from "vitest";6import { dateFromUrl, discoverFromLinks, parseApchqMarkdown, rawDocFromMarkdown, sectorFromUrl, tradeCodeFor, apchqConnector } from "./apchq";7import { reconcile } from "./apchq-ai";8import { pointsToObservations, seriesList, indexCode, statcanConnector } from "./statcan";9import { detectIncident, hasRateTable } from "./ccq";10import { bmrAdapter, canacAdapter, detectPackQty, detectPromotion, patrickMorinAdapter, unitTextToSourceUnit, makeRetailConnector } from "./retail";11import { deterministicMatch, scoreCandidate } from "./matching";12import { validateObservations } from "./validate";13import { contentHash, parseMoney } from "./firecrawl";14import { normalizeBenchmarkRows, parseCsv } from "./imports";15import { openCostDbAt } from "../db";16import { ensureConnectorSchema, saveRawObservation, lastHashFor } from "./store";17import type { CanonicalPriceObservation } from "./types";1819const FX = path.join(__dirname, "__fixtures__");20const readMd = (f: string) => fs.readFileSync(path.join(FX, f), "utf8");21const readJson = (f: string) => JSON.parse(fs.readFileSync(path.join(FX, f), "utf8")) as { data: { markdown: string; metadata: Record<string, unknown>; links?: string[] } };2223const noCtx = { recentPrices: () => [] as number[], referencePrice: () => null };2425/* ------------------------------------------------------------------ APCHQ */2627describe("APCHQ", () => {28  it("lit la date et le secteur depuis l'URL (coquille « arvil » tolérée)", () => {29    expect(dateFromUrl("https://media.apchq.com/x/cout-main-d-oeuvre-residentiel-leger-26-arvil-2026-temps-simple.pdf")).toBe("2026-04-26");30    expect(dateFromUrl("https://media.apchq.com/x/cout-main-d-oeuvre-residentiel-lourd-1er-janvier-2024.pdf")).toBe("2024-01-01");31    expect(dateFromUrl("https://media.apchq.com/x/cout-main-d-oeuvre-commercial-28-decembre-2025.pdf")).toBe("2025-12-28");32    expect(sectorFromUrl("…/cout-main-d-oeuvre-commercial-26-avril-2026.pdf")).toBe("ic");33    expect(sectorFromUrl("…/cout-main-d-oeuvre-residentiel-lourd-x.pdf")).toBe("residentiel_lourd");34  });35  it("découvre les grilles temps simple léger/lourd/IC et exclut temps demi, exemples de paie, chantiers isolés", () => {36    const j = readJson("apchq-landing.json");37    const docs = discoverFromLinks(j.data.links ?? [], 100);38    expect(docs.length).toBeGreaterThanOrEqual(12);39    expect(docs.every((d) => !/temps-demi|temps-double|exemple|isoles|baie-james/i.test(d.url))).toBe(true);40    expect(docs.some((d) => d.url.includes("residentiel-leger-26-arvil-2026-temps-simple"))).toBe(true);41    expect(docs.some((d) => d.url.includes("commercial-26-avril-2026"))).toBe(true);42    expect(docs.map((d) => String(d.meta?.effectiveDate))).toEqual([...docs.map((d) => String(d.meta?.effectiveDate))].sort());43  });44  it("parse une grille propre (2025) : compagnons plausibles, total > taux, apprentis ordonnés", () => {45    const rows = parseApchqMarkdown(readJson("apchq-leger-2025-04-27.json").data.markdown);46    expect(rows.length).toBeGreaterThan(100);47    const charp = rows.filter((r) => /charpentier/i.test(r.trade));48    const comp = charp.find((r) => r.classification === "compagnon")!;49    expect(comp.baseWage).toBeCloseTo(40.16, 2);50    expect(comp.total).toBeCloseTo(62.54, 2);51    expect(comp.vacation).toBeCloseTo(comp.baseWage * 0.13, 1);52    for (const r of rows) { expect(r.total).toBeGreaterThan(r.baseWage * 1.25); expect(r.total).toBeLessThan(r.baseWage * 2.1); }53    const appr = charp.filter((r) => r.classification.startsWith("apprenti")).sort((a, b) => a.classification.localeCompare(b.classification));54    for (let i = 1; i < appr.length; i++) expect(appr[i].baseWage).toBeGreaterThan(appr[i - 1].baseWage);55  });56  it("parse une grille aux rangées fusionnées/décalées (2026) : lignes aplaties et compagnon = taux le plus élevé", () => {57    const rows = parseApchqMarkdown(readMd("apchq-residentiel-leger-2026-04-26.md"));58    const rev = rows.find((r) => /poseur de rev/i.test(r.trade) && r.classification === "compagnon")!; // rangée « valeurs puis libellé » dans une seule cellule59    expect(rev.baseWage).toBeCloseTo(41.71, 2);60    expect(rev.total).toBeCloseTo(64.87, 2);61    const briq = rows.find((r) => /briqueteur/i.test(r.trade) && r.classification === "compagnon")!; // cellule « 46,59 $ 27,96 $ » = compagnon + apprenti 162    expect(briq.baseWage).toBeCloseTo(46.59, 2);63    expect(briq.total).toBeCloseTo(72.11, 2);64    expect(rows.find((r) => /briqueteur/i.test(r.trade) && r.classification === "apprenti-1")!.baseWage).toBeCloseTo(27.96, 2);65    expect(rows.find((r) => /manœuvre \(journalier\)/i.test(r.trade))!.total).toBeCloseTo(58.55, 2);66  });67  it("associe les libellés APCHQ aux codes de métier (œ, abréviations)", () => {68    expect(tradeCodeFor("Charpentier-menuisier").code).toBe("charpentier");69    expect(tradeCodeFor("Tuyauteur").code).toBe("plombier");70    expect(tradeCodeFor("Manœuvre (journalier)").code).toBe("manoeuvre");71    expect(tradeCodeFor("Poseur d&#x27;armature du béton".replace("&#x27;", "'")).code).toBe("ferrailleur");72    expect(tradeCodeFor("Poseur de rev. Souples").code).toBe("poseur_revetements");73    expect(tradeCodeFor("Op. de p. méc. - Cl. A").code).toBe("operateur");74    expect(tradeCodeFor("Métier imaginaire").matched).toBe(false);75  });76  it("normalise en observations de main-d'œuvre valides (secteur, dérivés, validation)", async () => {77    const raw = rawDocFromMarkdown("https://media.apchq.com/x/cout-main-d-oeuvre-residentiel-leger-27-avril-2025-temps-simple.pdf", readJson("apchq-leger-2025-04-27.json").data.markdown);78    const obs = await apchqConnector.extract(raw, { onlyItems: ["no-ai"] });79    expect(obs.length).toBeGreaterThan(100);80    const canon = await apchqConnector.normalize(obs, raw, {});81    const lab = canon.filter((c) => c.kind === "labour");82    expect(lab.some((l) => l.kind === "labour" && l.tradeCode === "poseur_systemes")).toBe(true); // spécialité dérivée du charpentier83    expect(lab.every((l) => l.kind === "labour" && l.sector === "residentiel_leger" && l.effectiveFrom === "2025-04-27")).toBe(true);84    const v = validateObservations(canon, noCtx);85    expect(v.rejected.length).toBe(0);86  });87  it("réconciliation IA : accepte les lignes dont (taux, total) existent dans le parseur ou dont la somme boucle ; rejette le reste", () => {88    const regex = parseApchqMarkdown(readJson("apchq-leger-2025-04-27.json").data.markdown);89    const ok = { trade: "Charpentier-menuisier", classification: "compagnon", base_wage: 40.16, vacation: 5.22, benefits: 7.89, total: 62.54, contributions_sum: 9.27 };90    const sumOnly = { trade: "Métier X", classification: "compagnon", base_wage: 50, vacation: 6.5, benefits: 8, total: 74.5, contributions_sum: 10 };91    const bad = { trade: "Inventé", classification: "compagnon", base_wage: 44.44, vacation: 5.78, benefits: 8, total: 70.1, contributions_sum: 3 };92    const r = reconcile([ok, sumOnly, bad], regex, 1.153);93    expect(r.rows.map((x) => x.trade)).toEqual(["Charpentier-menuisier", "Métier X"]);94    expect(r.dropped).toBe(1);95  });96});9798/* ---------------------------------------------------------------- StatCan */99100describe("StatCan", () => {101  it("construit les 40 séries attendues avec les codes lus par le catalogue", () => {102    const s = seriesList();103    expect(s.length).toBe(16 + 22);104    expect(s.map(indexCode)).toContain("statcan:18100289:10:1:1");105    expect(s.map(indexCode)).toContain("statcan:18100289:10:1:8");106  });107  it("calcule les variations trimestrielle et annuelle", () => {108    const pts = [{ refPer: "2025-01-01", value: 100 }, { refPer: "2025-04-01", value: 102 }, { refPer: "2025-07-01", value: 104 }, { refPer: "2025-10-01", value: 105 }, { refPer: "2026-01-01", value: 110 }];109    const obs = pointsToObservations({ geo: 10, type: 1, division: 1 }, pts, "2026-09-06T00:00:00Z");110    expect(obs[0].pctQoq).toBeNull();111    expect(obs[1].pctQoq).toBeCloseTo(2, 1);112    expect(obs[4].pctYoy).toBeCloseTo(10, 1);113    expect(obs[4].geography).toBe("Montréal");114    expect(validateObservations(obs, noCtx).rejected.length).toBe(0);115  });116  it("normalise une réponse WDS enregistrée", async () => {117    const wds = [{ status: "SUCCESS", object: { coordinate: "10.1.1.0.0.0.0.0.0.0", vectorDataPoint: [{ refPer: "2026-01-01", value: 112.4 }, { refPer: "2026-04-01", value: 114.8 }] } }, { status: "FAILED", object: { coordinate: "9.9.9.0.0.0.0.0.0.0", vectorDataPoint: [] } }];118    const md = JSON.stringify(wds.map((it) => ({ c: it.object.coordinate, s: it.status, p: it.object.vectorDataPoint.map((p) => [p.refPer, p.value]) })));119    const raw = { url: "wds", fetchedAt: "2026-09-06T00:00:00Z", markdown: md, metadata: {}, statusCode: 200, contentHash: contentHash(md), unchanged: false };120    const obs = await statcanConnector.extract(raw, {});121    expect(obs.length).toBe(1);122    const canon = await statcanConnector.normalize(obs, raw, {});123    expect(canon.length).toBe(2);124    expect(canon[1]).toMatchObject({ kind: "index", indexCode: "statcan:18100289:10:1:1", period: "2026-04-01", value: 114.8 });125  });126});127128/* -------------------------------------------------------------------- CCQ */129130describe("CCQ", () => {131  it("détecte l'incident et l'absence de grille (source indisponible, jamais de valeur inventée)", () => {132    const md = "Salaire et taux\n\n**Incident de sécurité informatique en cours – Les services en ligne demeurent fermés. La reprise est prévue le 8 septembre.**\n| Date | Modifications |\n| --- | --- |\n| À compter du 26 avril 2026 | Taux |";133    expect(detectIncident(md)).toMatch(/incident de sécurité/i);134    expect(hasRateTable(md)).toBe(false);135  });136});137138/* ------------------------------------------------------------- détaillants */139140describe("détaillants", () => {141  it("Canac : prix et unité depuis la fiche (métadonnée product:price:amount)", () => {142    const j = readJson("canac-2x6x8.json");143    const p = canacAdapter.parse({ markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, links: [] });144    expect(p.price).toBeCloseTo(5.98, 2);145    expect(p.unitText?.toLowerCase()).toContain("chaque");146    expect(p.title).toMatch(/2 po x 6 po x 8 pi/);147    expect(p.currency).toBe("CAD");148  });149  it("BMR : prix depuis les métadonnées malgré les variantes « à partir de »", () => {150    const j = readJson("bmr-2x6x8.json");151    const p = bmrAdapter.parse({ markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, links: [] });152    expect(p.price).toBeCloseTo(6.98, 2);153  });154  it("Patrick Morin : prix depuis les métadonnées", () => {155    const j = readJson("pm-2x6x8.json");156    const p = patrickMorinAdapter.parse({ markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, links: [] });157    expect(p.price).toBeCloseTo(5.98, 2);158  });159  it("promotion : prix régulier préféré, promo non interprétée comme régulier", () => {160    expect(detectPromotion("Prix régulier : 7,49 $  Spécial 5,98 $", 5.98)).toEqual({ regular: 7.49, sale: 5.98 });161    expect(detectPromotion("Économisez 20 % — 5,98 $ / Chaque", 5.98)).toEqual({ regular: null, sale: 5.98 });162    expect(detectPromotion("Rabais de la semaine sur la peinture !\n\n\n\n\n\n\n\n\n\n" + " ".repeat(300) + "# Tuyau\n5,98 $ / Chaque", 5.98)).toEqual({ regular: 5.98, sale: null });163    expect(detectPromotion("Prix 5,98 $ / Chaque", 5.98)).toEqual({ regular: 5.98, sale: null });164  });165  it("emballage : couverture pi², rouleau 100 pi, boîte de 50 lb, paquet de 12", () => {166    expect(detectPackQty("Isolant R24 — couvre 39,8 pi²", "pi2")).toBeCloseTo(39.8, 1);167    expect(detectPackQty("Panneau de gypse 4 pi x 8 pi", "pi2")).toBe(32);168    expect(detectPackQty("Tuyau PEX 1/2 po 100 pi", "pi_lin")).toBe(100);169    expect(detectPackQty("Clous 3 1/4 po 50 lb", "lb")).toBe(50);170    expect(detectPackQty("Attaches de brique pqt/100", "unit")).toBe(100);171    expect(unitTextToSourceUnit("Chaque", "pi2", 32, "feuille")).toBe("feuille");172    expect(unitTextToSourceUnit("Chaque", "unit", null, null)).toBe("chaque");173  });174  it("page sans prix → aucune observation ; conversion par emballage → prix canonique ; unité inconnue → rejet", async () => {175    const dbPath = path.join(FX, ".test-retail.db");176    fs.rmSync(dbPath, { force: true });177    const d = openCostDbAt(dbPath);178    ensureConnectorSchema(d);179    const c = makeRetailConnector(canacAdapter);180    const j = readJson("canac-2x6x8.json");181    const noPrice = { url: "https://www.canac.ca/canac/fr/2/p/x/1", fetchedAt: "2026-09-06T10:00:00Z", markdown: "# Produit\nDisponibilité", metadata: { title: "Produit" }, statusCode: 200, contentHash: "h1", unchanged: false, meta: { itemCode: "LUM-2X6-8", packQty: null, packUnit: null } };182    expect((await c.extract(noPrice, { db: d })).length).toBe(0);183    const gyp = { url: "https://www.canac.ca/canac/fr/2/p/gypse/2", fetchedAt: "2026-09-06T10:00:00Z", markdown: "# Panneau de gypse 1/2 po 4 pi x 8 pi\n17,60 $  / Chaque", metadata: { "product:price:amount": "17,60 $", "product:price:currency": "CAD", ogTitle: "Panneau de gypse 1/2 po 4 pi x 8 pi - Canac" }, statusCode: 200, contentHash: "h2", unchanged: false, meta: { itemCode: "GYP-1/2-4X8", packQty: 32, packUnit: "feuille" } };184    const obs = await c.extract(gyp, { db: d });185    expect(obs.length).toBe(1);186    const canon = await c.normalize(obs, gyp, { db: d });187    const p = canon[0] as CanonicalPriceObservation;188    expect(p.totalCost).toBeCloseTo(0.55, 3);189    expect(p.conversionFactor).toBeCloseTo(1 / 32, 6);190    expect(p.sourceUnit).toBe("feuille");191    expect(c.validate(canon, { db: d }).rejected.length).toBe(0);192    const lum = { url: j.data.metadata.url as string, fetchedAt: "2026-09-06T10:00:00Z", markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, contentHash: "h3", unchanged: false, meta: { itemCode: "LUM-2X6-8", packQty: null, packUnit: null } };193    const lc = await c.normalize(await c.extract(lum, { db: d }), lum, { db: d });194    expect((lc[0] as CanonicalPriceObservation).totalCost).toBeCloseTo(5.98, 2);195    // unité inconnue (aucun emballage pour un pi²) → rejet explicite196    const weird = { ...gyp, url: "https://www.canac.ca/canac/fr/2/p/gypse/3", metadata: { ...gyp.metadata, ogTitle: "Gypse" }, markdown: "# Gypse\n17,60 $ / Palette", meta: { itemCode: "GYP-1/2-4X8", packQty: null, packUnit: null } };197    const wc = await c.normalize(await c.extract(weird, { db: d }), weird, { db: d });198    const v = c.validate(wc, { db: d });199    expect(v.accepted.length + v.rejected.length).toBe(1);200    expect(v.rejected.length).toBe(1);201    d.close();202    fs.rmSync(dbPath, { force: true });203  });204  it("erreur 403 / contenu vide → aucune observation, statut d'erreur propre", async () => {205    const c = makeRetailConnector(bmrAdapter);206    const forbidden = { url: "https://www.bmr.ca/fr/x.html", fetchedAt: "2026-09-06T10:00:00Z", markdown: "Just a moment...", metadata: {}, statusCode: 403, contentHash: "h", unchanged: false, meta: { itemCode: "LUM-2X6-8" } };207    expect((await c.extract(forbidden, {})).length).toBe(0);208    const empty = { ...forbidden, statusCode: 200, markdown: "" };209    expect((await c.extract(empty, {})).length).toBe(0);210  });211});212213/* -------------------------------------------------------------- matching */214215describe("matching déterministe", () => {216  it("exige les dimensions et privilégie le recouvrement des mots", () => {217    expect(scoreCandidate("épinette de construction 2 po x 6 po x 8 pi sec", "Épinette de construction 2 po x 6 po x 8 pi sec - Canac")).toBeGreaterThan(0.85);218    expect(scoreCandidate("épinette de construction 2 po x 6 po x 8 pi sec", "Épinette de construction 2 po x 4 po x 8 pi sec")).toBeLessThan(0.4);219    const dec = deterministicMatch("panneau de gypse 1/2 po 4 x 8", [{ url: "a", title: "Panneau de gypse 1/2 po 4 pi x 8 pi", description: "" }, { url: "b", title: "Panneau de gypse 5/8 po 4 x 8", description: "" }, { url: "c", title: "Ensemble de vis à gypse", description: "" }]);220    expect(dec.candidate?.url).toBe("a");221    expect(dec.confidence).toBeGreaterThan(0.8);222  });223});224225/* ------------------------------------------------------------- validation */226227describe("validation", () => {228  const base = (over: Partial<CanonicalPriceObservation>): CanonicalPriceObservation => ({ kind: "price", itemCode: "LUM-2X6-8", sourceUrl: "u", observationDate: "2026-09-06", sourceUnit: "chaque", canonicalUnit: "unit", conversionFactor: 1, totalCost: 6, isRegularPrice: true, rawPrice: 6, rawRegularPrice: null, rawSalePrice: null, currency: "CAD", locationCode: "QC", title: "t", raw: { externalId: null, sourceUrl: "u", retrievedAt: "2026-09-06T00:00:00Z", effectiveDate: null, title: "t", description: null, unit: "chaque", price: 6, regularPrice: null, salePrice: null, currency: "CAD", location: null, category: null, payload: {} }, ...over });229  it("rejette prix ≤ 0, devise ≠ CAD, unité inconnue, hors bornes métier ; marque les aberrations sans effacer", () => {230    const ctx = { recentPrices: () => [5.9, 6.1, 6.0], referencePrice: () => 5.98 };231    const v = validateObservations([base({}), base({ rawPrice: 0, totalCost: 0 }), base({ currency: "USD" }), base({ sourceUnit: "palette" }), base({ totalCost: 40, rawPrice: 40 }), base({ totalCost: 15, rawPrice: 15 })], ctx);232    expect(v.accepted.length).toBe(1);233    expect(v.rejected.map((r) => r.reason)).toEqual(expect.arrayContaining([expect.stringMatching(/≤ 0/), expect.stringMatching(/devise/), expect.stringMatching(/unité inconnue/), expect.stringMatching(/bornes métier/), expect.stringMatching(/aberration/)]));234  });235});236237/* ---------------------------------------------------------- hash / imports */238239describe("hash et imports", () => {240  it("un document inchangé est reconnu par son hash (statut unchanged)", () => {241    const dbPath = path.join(FX, ".test-hash.db");242    fs.rmSync(dbPath, { force: true });243    const d = openCostDbAt(dbPath);244    ensureConnectorSchema(d);245    const md = "| Charpentier | 60,24 $ | 89,14 $ |";246    const raw = rawDocFromMarkdown("https://media.apchq.com/x/y.pdf", md);247    expect(lastHashFor(raw.url, d)).toBeNull();248    saveRawObservation("apchq", { externalId: "x", sourceUrl: raw.url, retrievedAt: raw.fetchedAt, effectiveDate: "2026-04-26", title: "t", description: null, unit: "h", price: 89.14, regularPrice: null, salePrice: null, currency: "CAD", location: "QC", category: "labour", payload: {} }, raw, "accepted", null, "test", d);249    expect(lastHashFor(raw.url, d)).toBe(contentHash(md));250    expect(contentHash(md + "\n")).toBe(contentHash(md)); // espaces normalisés251    expect(contentHash("| Charpentier | 61,00 $ | 90,00 $ |")).not.toBe(contentHash(md));252    d.close();253    fs.rmSync(dbPath, { force: true });254  });255  it("parseMoney et import de benchmarks (CSV)", () => {256    expect(parseMoney("5,98 $")).toBeCloseTo(5.98);257    expect(parseMoney("$1 234,50")).toBeCloseTo(1234.5);258    expect(parseMoney("abc")).toBeNull();259    const csv = "source,building_type,market,unit,low,high,year,notes\nAltus Group — Canadian Cost Guide 2026,single_family,Montréal,$/pi2,255,395,2026,custom\nAltus,castle,Montréal,$/pi2,1,2,2026,\n";260    expect(parseCsv(csv).length).toBe(2);261    const r = normalizeBenchmarkRows(csv);262    expect(r.rows.length).toBe(1);263    expect(r.errors.length).toBe(1);264  });265});266