// Auteur : Simon-Pierre Boucher — contact@spboucher.ai
// Tests des connecteurs sur FIXTURES enregistrées (jamais le Web live).
import fs from "fs";
import path from "path";
import { describe, expect, it } from "vitest";
import { dateFromUrl, discoverFromLinks, parseApchqMarkdown, rawDocFromMarkdown, sectorFromUrl, tradeCodeFor, apchqConnector } from "./apchq";
import { reconcile } from "./apchq-ai";
import { pointsToObservations, seriesList, indexCode, statcanConnector } from "./statcan";
import { detectIncident, hasRateTable } from "./ccq";
import { bmrAdapter, canacAdapter, detectPackQty, detectPromotion, patrickMorinAdapter, unitTextToSourceUnit, makeRetailConnector } from "./retail";
import { deterministicMatch, scoreCandidate } from "./matching";
import { validateObservations } from "./validate";
import { contentHash, parseMoney } from "./firecrawl";
import { normalizeBenchmarkRows, parseCsv } from "./imports";
import { openCostDbAt } from "../db";
import { ensureConnectorSchema, saveRawObservation, lastHashFor } from "./store";
import type { CanonicalPriceObservation } from "./types";

const FX = path.join(__dirname, "__fixtures__");
const readMd = (f: string) => fs.readFileSync(path.join(FX, f), "utf8");
const readJson = (f: string) => JSON.parse(fs.readFileSync(path.join(FX, f), "utf8")) as { data: { markdown: string; metadata: Record<string, unknown>; links?: string[] } };

const noCtx = { recentPrices: () => [] as number[], referencePrice: () => null };

/* ------------------------------------------------------------------ APCHQ */

describe("APCHQ", () => {
  it("lit la date et le secteur depuis l'URL (coquille « arvil » tolérée)", () => {
    expect(dateFromUrl("https://media.apchq.com/x/cout-main-d-oeuvre-residentiel-leger-26-arvil-2026-temps-simple.pdf")).toBe("2026-04-26");
    expect(dateFromUrl("https://media.apchq.com/x/cout-main-d-oeuvre-residentiel-lourd-1er-janvier-2024.pdf")).toBe("2024-01-01");
    expect(dateFromUrl("https://media.apchq.com/x/cout-main-d-oeuvre-commercial-28-decembre-2025.pdf")).toBe("2025-12-28");
    expect(sectorFromUrl("…/cout-main-d-oeuvre-commercial-26-avril-2026.pdf")).toBe("ic");
    expect(sectorFromUrl("…/cout-main-d-oeuvre-residentiel-lourd-x.pdf")).toBe("residentiel_lourd");
  });
  it("découvre les grilles temps simple léger/lourd/IC et exclut temps demi, exemples de paie, chantiers isolés", () => {
    const j = readJson("apchq-landing.json");
    const docs = discoverFromLinks(j.data.links ?? [], 100);
    expect(docs.length).toBeGreaterThanOrEqual(12);
    expect(docs.every((d) => !/temps-demi|temps-double|exemple|isoles|baie-james/i.test(d.url))).toBe(true);
    expect(docs.some((d) => d.url.includes("residentiel-leger-26-arvil-2026-temps-simple"))).toBe(true);
    expect(docs.some((d) => d.url.includes("commercial-26-avril-2026"))).toBe(true);
    expect(docs.map((d) => String(d.meta?.effectiveDate))).toEqual([...docs.map((d) => String(d.meta?.effectiveDate))].sort());
  });
  it("parse une grille propre (2025) : compagnons plausibles, total > taux, apprentis ordonnés", () => {
    const rows = parseApchqMarkdown(readJson("apchq-leger-2025-04-27.json").data.markdown);
    expect(rows.length).toBeGreaterThan(100);
    const charp = rows.filter((r) => /charpentier/i.test(r.trade));
    const comp = charp.find((r) => r.classification === "compagnon")!;
    expect(comp.baseWage).toBeCloseTo(40.16, 2);
    expect(comp.total).toBeCloseTo(62.54, 2);
    expect(comp.vacation).toBeCloseTo(comp.baseWage * 0.13, 1);
    for (const r of rows) { expect(r.total).toBeGreaterThan(r.baseWage * 1.25); expect(r.total).toBeLessThan(r.baseWage * 2.1); }
    const appr = charp.filter((r) => r.classification.startsWith("apprenti")).sort((a, b) => a.classification.localeCompare(b.classification));
    for (let i = 1; i < appr.length; i++) expect(appr[i].baseWage).toBeGreaterThan(appr[i - 1].baseWage);
  });
  it("parse une grille aux rangées fusionnées/décalées (2026) : lignes aplaties et compagnon = taux le plus élevé", () => {
    const rows = parseApchqMarkdown(readMd("apchq-residentiel-leger-2026-04-26.md"));
    const rev = rows.find((r) => /poseur de rev/i.test(r.trade) && r.classification === "compagnon")!; // rangée « valeurs puis libellé » dans une seule cellule
    expect(rev.baseWage).toBeCloseTo(41.71, 2);
    expect(rev.total).toBeCloseTo(64.87, 2);
    const briq = rows.find((r) => /briqueteur/i.test(r.trade) && r.classification === "compagnon")!; // cellule « 46,59 $ 27,96 $ » = compagnon + apprenti 1
    expect(briq.baseWage).toBeCloseTo(46.59, 2);
    expect(briq.total).toBeCloseTo(72.11, 2);
    expect(rows.find((r) => /briqueteur/i.test(r.trade) && r.classification === "apprenti-1")!.baseWage).toBeCloseTo(27.96, 2);
    expect(rows.find((r) => /manœuvre \(journalier\)/i.test(r.trade))!.total).toBeCloseTo(58.55, 2);
  });
  it("associe les libellés APCHQ aux codes de métier (œ, abréviations)", () => {
    expect(tradeCodeFor("Charpentier-menuisier").code).toBe("charpentier");
    expect(tradeCodeFor("Tuyauteur").code).toBe("plombier");
    expect(tradeCodeFor("Manœuvre (journalier)").code).toBe("manoeuvre");
    expect(tradeCodeFor("Poseur d&#x27;armature du béton".replace("&#x27;", "'")).code).toBe("ferrailleur");
    expect(tradeCodeFor("Poseur de rev. Souples").code).toBe("poseur_revetements");
    expect(tradeCodeFor("Op. de p. méc. - Cl. A").code).toBe("operateur");
    expect(tradeCodeFor("Métier imaginaire").matched).toBe(false);
  });
  it("normalise en observations de main-d'œuvre valides (secteur, dérivés, validation)", async () => {
    const raw = rawDocFromMarkdown("https://media.apchq.com/x/cout-main-d-oeuvre-residentiel-leger-27-avril-2025-temps-simple.pdf", readJson("apchq-leger-2025-04-27.json").data.markdown);
    const obs = await apchqConnector.extract(raw, { onlyItems: ["no-ai"] });
    expect(obs.length).toBeGreaterThan(100);
    const canon = await apchqConnector.normalize(obs, raw, {});
    const lab = canon.filter((c) => c.kind === "labour");
    expect(lab.some((l) => l.kind === "labour" && l.tradeCode === "poseur_systemes")).toBe(true); // spécialité dérivée du charpentier
    expect(lab.every((l) => l.kind === "labour" && l.sector === "residentiel_leger" && l.effectiveFrom === "2025-04-27")).toBe(true);
    const v = validateObservations(canon, noCtx);
    expect(v.rejected.length).toBe(0);
  });
  it("réconciliation IA : accepte les lignes dont (taux, total) existent dans le parseur ou dont la somme boucle ; rejette le reste", () => {
    const regex = parseApchqMarkdown(readJson("apchq-leger-2025-04-27.json").data.markdown);
    const ok = { trade: "Charpentier-menuisier", classification: "compagnon", base_wage: 40.16, vacation: 5.22, benefits: 7.89, total: 62.54, contributions_sum: 9.27 };
    const sumOnly = { trade: "Métier X", classification: "compagnon", base_wage: 50, vacation: 6.5, benefits: 8, total: 74.5, contributions_sum: 10 };
    const bad = { trade: "Inventé", classification: "compagnon", base_wage: 44.44, vacation: 5.78, benefits: 8, total: 70.1, contributions_sum: 3 };
    const r = reconcile([ok, sumOnly, bad], regex, 1.153);
    expect(r.rows.map((x) => x.trade)).toEqual(["Charpentier-menuisier", "Métier X"]);
    expect(r.dropped).toBe(1);
  });
});

/* ---------------------------------------------------------------- StatCan */

describe("StatCan", () => {
  it("construit les 40 séries attendues avec les codes lus par le catalogue", () => {
    const s = seriesList();
    expect(s.length).toBe(16 + 22);
    expect(s.map(indexCode)).toContain("statcan:18100289:10:1:1");
    expect(s.map(indexCode)).toContain("statcan:18100289:10:1:8");
  });
  it("calcule les variations trimestrielle et annuelle", () => {
    const pts = [{ refPer: "2025-01-01", value: 100 }, { refPer: "2025-04-01", value: 102 }, { refPer: "2025-07-01", value: 104 }, { refPer: "2025-10-01", value: 105 }, { refPer: "2026-01-01", value: 110 }];
    const obs = pointsToObservations({ geo: 10, type: 1, division: 1 }, pts, "2026-09-06T00:00:00Z");
    expect(obs[0].pctQoq).toBeNull();
    expect(obs[1].pctQoq).toBeCloseTo(2, 1);
    expect(obs[4].pctYoy).toBeCloseTo(10, 1);
    expect(obs[4].geography).toBe("Montréal");
    expect(validateObservations(obs, noCtx).rejected.length).toBe(0);
  });
  it("normalise une réponse WDS enregistrée", async () => {
    const wds = [{ status: "SUCCESS", object: { coordinate: "10.1.1.0.0.0.0.0.0.0", vectorDataPoint: [{ refPer: "2026-01-01", value: 112.4 }, { refPer: "2026-04-01", value: 114.8 }] } }, { status: "FAILED", object: { coordinate: "9.9.9.0.0.0.0.0.0.0", vectorDataPoint: [] } }];
    const md = JSON.stringify(wds.map((it) => ({ c: it.object.coordinate, s: it.status, p: it.object.vectorDataPoint.map((p) => [p.refPer, p.value]) })));
    const raw = { url: "wds", fetchedAt: "2026-09-06T00:00:00Z", markdown: md, metadata: {}, statusCode: 200, contentHash: contentHash(md), unchanged: false };
    const obs = await statcanConnector.extract(raw, {});
    expect(obs.length).toBe(1);
    const canon = await statcanConnector.normalize(obs, raw, {});
    expect(canon.length).toBe(2);
    expect(canon[1]).toMatchObject({ kind: "index", indexCode: "statcan:18100289:10:1:1", period: "2026-04-01", value: 114.8 });
  });
});

/* -------------------------------------------------------------------- CCQ */

describe("CCQ", () => {
  it("détecte l'incident et l'absence de grille (source indisponible, jamais de valeur inventée)", () => {
    const md = "Salaire et taux\n\n**Incident de sécurité informatique en cours – Les services en ligne demeurent fermés. La reprise est prévue le 8 septembre.**\n| Date | Modifications |\n| --- | --- |\n| À compter du 26 avril 2026 | Taux |";
    expect(detectIncident(md)).toMatch(/incident de sécurité/i);
    expect(hasRateTable(md)).toBe(false);
  });
});

/* ------------------------------------------------------------- détaillants */

describe("détaillants", () => {
  it("Canac : prix et unité depuis la fiche (métadonnée product:price:amount)", () => {
    const j = readJson("canac-2x6x8.json");
    const p = canacAdapter.parse({ markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, links: [] });
    expect(p.price).toBeCloseTo(5.98, 2);
    expect(p.unitText?.toLowerCase()).toContain("chaque");
    expect(p.title).toMatch(/2 po x 6 po x 8 pi/);
    expect(p.currency).toBe("CAD");
  });
  it("BMR : prix depuis les métadonnées malgré les variantes « à partir de »", () => {
    const j = readJson("bmr-2x6x8.json");
    const p = bmrAdapter.parse({ markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, links: [] });
    expect(p.price).toBeCloseTo(6.98, 2);
  });
  it("Patrick Morin : prix depuis les métadonnées", () => {
    const j = readJson("pm-2x6x8.json");
    const p = patrickMorinAdapter.parse({ markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, links: [] });
    expect(p.price).toBeCloseTo(5.98, 2);
  });
  it("promotion : prix régulier préféré, promo non interprétée comme régulier", () => {
    expect(detectPromotion("Prix régulier : 7,49 $  Spécial 5,98 $", 5.98)).toEqual({ regular: 7.49, sale: 5.98 });
    expect(detectPromotion("Économisez 20 % — 5,98 $ / Chaque", 5.98)).toEqual({ regular: null, sale: 5.98 });
    expect(detectPromotion("Rabais de la semaine sur la peinture !\n\n\n\n\n\n\n\n\n\n" + " ".repeat(300) + "# Tuyau\n5,98 $ / Chaque", 5.98)).toEqual({ regular: 5.98, sale: null });
    expect(detectPromotion("Prix 5,98 $ / Chaque", 5.98)).toEqual({ regular: 5.98, sale: null });
  });
  it("emballage : couverture pi², rouleau 100 pi, boîte de 50 lb, paquet de 12", () => {
    expect(detectPackQty("Isolant R24 — couvre 39,8 pi²", "pi2")).toBeCloseTo(39.8, 1);
    expect(detectPackQty("Panneau de gypse 4 pi x 8 pi", "pi2")).toBe(32);
    expect(detectPackQty("Tuyau PEX 1/2 po 100 pi", "pi_lin")).toBe(100);
    expect(detectPackQty("Clous 3 1/4 po 50 lb", "lb")).toBe(50);
    expect(detectPackQty("Attaches de brique pqt/100", "unit")).toBe(100);
    expect(unitTextToSourceUnit("Chaque", "pi2", 32, "feuille")).toBe("feuille");
    expect(unitTextToSourceUnit("Chaque", "unit", null, null)).toBe("chaque");
  });
  it("page sans prix → aucune observation ; conversion par emballage → prix canonique ; unité inconnue → rejet", async () => {
    const dbPath = path.join(FX, ".test-retail.db");
    fs.rmSync(dbPath, { force: true });
    const d = openCostDbAt(dbPath);
    ensureConnectorSchema(d);
    const c = makeRetailConnector(canacAdapter);
    const j = readJson("canac-2x6x8.json");
    const noPrice = { url: "https://www.canac.ca/canac/fr/2/p/x/1", fetchedAt: "2026-09-06T10:00:00Z", markdown: "# Produit\nDisponibilité", metadata: { title: "Produit" }, statusCode: 200, contentHash: "h1", unchanged: false, meta: { itemCode: "LUM-2X6-8", packQty: null, packUnit: null } };
    expect((await c.extract(noPrice, { db: d })).length).toBe(0);
    const gyp = { url: "https://www.canac.ca/canac/fr/2/p/gypse/2", fetchedAt: "2026-09-06T10:00:00Z", markdown: "# Panneau de gypse 1/2 po 4 pi x 8 pi\n17,60 $  / Chaque", metadata: { "product:price:amount": "17,60 $", "product:price:currency": "CAD", ogTitle: "Panneau de gypse 1/2 po 4 pi x 8 pi - Canac" }, statusCode: 200, contentHash: "h2", unchanged: false, meta: { itemCode: "GYP-1/2-4X8", packQty: 32, packUnit: "feuille" } };
    const obs = await c.extract(gyp, { db: d });
    expect(obs.length).toBe(1);
    const canon = await c.normalize(obs, gyp, { db: d });
    const p = canon[0] as CanonicalPriceObservation;
    expect(p.totalCost).toBeCloseTo(0.55, 3);
    expect(p.conversionFactor).toBeCloseTo(1 / 32, 6);
    expect(p.sourceUnit).toBe("feuille");
    expect(c.validate(canon, { db: d }).rejected.length).toBe(0);
    const lum = { url: j.data.metadata.url as string, fetchedAt: "2026-09-06T10:00:00Z", markdown: j.data.markdown, metadata: j.data.metadata, statusCode: 200, contentHash: "h3", unchanged: false, meta: { itemCode: "LUM-2X6-8", packQty: null, packUnit: null } };
    const lc = await c.normalize(await c.extract(lum, { db: d }), lum, { db: d });
    expect((lc[0] as CanonicalPriceObservation).totalCost).toBeCloseTo(5.98, 2);
    // unité inconnue (aucun emballage pour un pi²) → rejet explicite
    const weird = { ...gyp, url: "https://www.canac.ca/canac/fr/2/p/gypse/3", metadata: { ...gyp.metadata, ogTitle: "Gypse" }, markdown: "# Gypse\n17,60 $ / Palette", meta: { itemCode: "GYP-1/2-4X8", packQty: null, packUnit: null } };
    const wc = await c.normalize(await c.extract(weird, { db: d }), weird, { db: d });
    const v = c.validate(wc, { db: d });
    expect(v.accepted.length + v.rejected.length).toBe(1);
    expect(v.rejected.length).toBe(1);
    d.close();
    fs.rmSync(dbPath, { force: true });
  });
  it("erreur 403 / contenu vide → aucune observation, statut d'erreur propre", async () => {
    const c = makeRetailConnector(bmrAdapter);
    const forbidden = { url: "https://www.bmr.ca/fr/x.html", fetchedAt: "2026-09-06T10:00:00Z", markdown: "Just a moment...", metadata: {}, statusCode: 403, contentHash: "h", unchanged: false, meta: { itemCode: "LUM-2X6-8" } };
    expect((await c.extract(forbidden, {})).length).toBe(0);
    const empty = { ...forbidden, statusCode: 200, markdown: "" };
    expect((await c.extract(empty, {})).length).toBe(0);
  });
});

/* -------------------------------------------------------------- matching */

describe("matching déterministe", () => {
  it("exige les dimensions et privilégie le recouvrement des mots", () => {
    expect(scoreCandidate("épinette de construction 2 po x 6 po x 8 pi sec", "Épinette de construction 2 po x 6 po x 8 pi sec - Canac")).toBeGreaterThan(0.85);
    expect(scoreCandidate("épinette de construction 2 po x 6 po x 8 pi sec", "Épinette de construction 2 po x 4 po x 8 pi sec")).toBeLessThan(0.4);
    const dec = deterministicMatch("panneau de gypse 1/2 po 4 x 8", [{ url: "a", title: "Panneau de gypse 1/2 po 4 pi x 8 pi", description: "" }, { url: "b", title: "Panneau de gypse 5/8 po 4 x 8", description: "" }, { url: "c", title: "Ensemble de vis à gypse", description: "" }]);
    expect(dec.candidate?.url).toBe("a");
    expect(dec.confidence).toBeGreaterThan(0.8);
  });
});

/* ------------------------------------------------------------- validation */

describe("validation", () => {
  const base = (over: Partial<CanonicalPriceObservation>): CanonicalPriceObservation => ({ kind: "price", itemCode: "LUM-2X6-8", sourceUrl: "u", observationDate: "2026-09-06", sourceUnit: "chaque", canonicalUnit: "unit", conversionFactor: 1, totalCost: 6, isRegularPrice: true, rawPrice: 6, rawRegularPrice: null, rawSalePrice: null, currency: "CAD", locationCode: "QC", title: "t", raw: { externalId: null, sourceUrl: "u", retrievedAt: "2026-09-06T00:00:00Z", effectiveDate: null, title: "t", description: null, unit: "chaque", price: 6, regularPrice: null, salePrice: null, currency: "CAD", location: null, category: null, payload: {} }, ...over });
  it("rejette prix ≤ 0, devise ≠ CAD, unité inconnue, hors bornes métier ; marque les aberrations sans effacer", () => {
    const ctx = { recentPrices: () => [5.9, 6.1, 6.0], referencePrice: () => 5.98 };
    const v = validateObservations([base({}), base({ rawPrice: 0, totalCost: 0 }), base({ currency: "USD" }), base({ sourceUnit: "palette" }), base({ totalCost: 40, rawPrice: 40 }), base({ totalCost: 15, rawPrice: 15 })], ctx);
    expect(v.accepted.length).toBe(1);
    expect(v.rejected.map((r) => r.reason)).toEqual(expect.arrayContaining([expect.stringMatching(/≤ 0/), expect.stringMatching(/devise/), expect.stringMatching(/unité inconnue/), expect.stringMatching(/bornes métier/), expect.stringMatching(/aberration/)]));
  });
});

/* ---------------------------------------------------------- hash / imports */

describe("hash et imports", () => {
  it("un document inchangé est reconnu par son hash (statut unchanged)", () => {
    const dbPath = path.join(FX, ".test-hash.db");
    fs.rmSync(dbPath, { force: true });
    const d = openCostDbAt(dbPath);
    ensureConnectorSchema(d);
    const md = "| Charpentier | 60,24 $ | 89,14 $ |";
    const raw = rawDocFromMarkdown("https://media.apchq.com/x/y.pdf", md);
    expect(lastHashFor(raw.url, d)).toBeNull();
    saveRawObservation("apchq", { externalId: "x", sourceUrl: raw.url, retrievedAt: raw.fetchedAt, effectiveDate: "2026-04-26", title: "t", description: null, unit: "h", price: 89.14, regularPrice: null, salePrice: null, currency: "CAD", location: "QC", category: "labour", payload: {} }, raw, "accepted", null, "test", d);
    expect(lastHashFor(raw.url, d)).toBe(contentHash(md));
    expect(contentHash(md + "\n")).toBe(contentHash(md)); // espaces normalisés
    expect(contentHash("| Charpentier | 61,00 $ | 90,00 $ |")).not.toBe(contentHash(md));
    d.close();
    fs.rmSync(dbPath, { force: true });
  });
  it("parseMoney et import de benchmarks (CSV)", () => {
    expect(parseMoney("5,98 $")).toBeCloseTo(5.98);
    expect(parseMoney("$1 234,50")).toBeCloseTo(1234.5);
    expect(parseMoney("abc")).toBeNull();
    const csv = "source,building_type,market,unit,low,high,year,notes\nAltus Group — Canadian Cost Guide 2026,single_family,Montréal,$/pi2,255,395,2026,custom\nAltus,castle,Montréal,$/pi2,1,2,2026,\n";
    expect(parseCsv(csv).length).toBe(2);
    const r = normalizeBenchmarkRows(csv);
    expect(r.rows.length).toBe(1);
    expect(r.errors.length).toBe(1);
  });
});