spb/vrai-prix

1// Auteur : Simon-Pierre Boucher — contact@spboucher.ai2/**3 * Connecteur APCHQ — grilles publiques « Coût horaire de la main-d'œuvre »4 * (PDF, media.apchq.com) : taux horaire conventionné (CCQ), indemnité de5 * vacances (13 %), avantages sociaux, cotisations (AE, RQAP, RRQ, FSS, CNESST),6 * prélèvements (CCQ, AECQ, fonds), TOTAL = coût employeur complet.7 *8 * Firecrawl rend le PDF en tableau markdown ; la mise en page fusionne parfois9 * deux lignes (compagnon + apprenti 1, ou apprenti 2 + apprenti 3) dans une10 * même rangée : le parseur lit les montants cellule par cellule et reconstitue11 * un enregistrement par valeur de « Taux horaire ». Les champs exacts sont le12 * taux horaire et le total ; la ventilation intermédiaire est lue quand la13 * cellule est nette, sinon reconstituée (vacances = 13 % réglementaire,14 * cotisations employeur = total − autres postes) et la confiance est réduite.15 */16import { TRADES } from "../taxonomy";17import { contentHash, parseMoney, scrape } from "./firecrawl";18import { lastHashFor } from "./store";19import { downloadPdf, extractApchqWithClaude } from "./apchq-ai";20import type { CanonicalObservation, ConnectorConfig, CostConnector, RawDocument, RawObservation, RunOptions, SourceDocument, ValidationResult } from "./types";21import { validateObservations } from "./validate";22import { recentPricesFor, referencePriceFor } from "./store";2324export const APCHQ_PARSER_VERSION = "apchq-1.0";25export const APCHQ_LANDING = "https://www.apchq.com/nos-services/relations-du-travail/paie/couts-horaires-et-paie/";2627export const APCHQ_CONFIG: ConnectorConfig = {28  key: "apchq", name: "APCHQ — Coûts horaires de la main-d'œuvre", domain: "apchq.com", maxPages: 40,29  allowedPaths: [/^https:\/\/media\.apchq\.com\/.+\.pdf$/i, /^https:\/\/www\.apchq\.com\/nos-services\/relations-du-travail\/paie\//],30  excludedPaths: [/temps-(demi|et-demi|double)/i, /exemple-de-paie/i, /avantages-(sociaux|imposable)/i, /residentiel-r-2/i, /chantiers?-isoles/i, /baie-james/i, /special-paie/i, /ic_special/i, /ic-speciale/i],31  refreshDays: 7, timeoutMs: 150_000, retries: 2, rateLimitMs: 1500, concurrency: 2,32};3334const MONTHS: [RegExp, string][] = [35  [/^jan/, "01"], [/^f[eé]v/, "02"], [/^mar/, "03"], [/^a[vr]r?[iv]?/, "04"], [/^mai/, "05"], [/^juin/, "06"], [/^juil/, "07"], [/^ao[uû]/, "08"], [/^sep/, "09"], [/^oct/, "10"], [/^nov/, "11"], [/^d[eé]c/, "12"],36];3738/** Date d'entrée en vigueur depuis le nom du PDF (« 26-arvil-2026 », « 1er-janvier-2024 »). */39export function dateFromUrl(url: string): string | null {40  const m = url.toLowerCase().match(/(\d{1,2})(?:er)?-([a-zéû]+)-(\d{4})/);41  if (!m) return null;42  const month = MONTHS.find(([re]) => re.test(m[2]))?.[1];43  if (!month) return null;44  return `${m[3]}-${month}-${m[1].padStart(2, "0")}`;45}4647export function sectorFromUrl(url: string): "residentiel_leger" | "residentiel_lourd" | "ic" | null {48  const u = url.toLowerCase();49  if (/residentiel-leger/.test(u)) return "residentiel_leger";50  if (/residentiel-lourd/.test(u)) return "residentiel_lourd";51  if (/commercial/.test(u)) return "ic";52  return null;53}5455const fold = (s: string) => s.replace(/œ/g, "oe").replace(/Œ/g, "oe").normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();5657/** Libellés APCHQ (abrégés dans les PDF) → codes de métier internes. */58const ALIASES: Record<string, string> = {59  "charpentier menuisier": "charpentier",60  "briqueteur macon": "briqueteur",61  carreleur: "carreleur",62  "cimentier applicateur": "cimentier",63  "coffreur a beton": "coffreur",64  couvreur: "couvreur",65  electricien: "electricien",66  tuyauteur: "plombier",67  plombier: "plombier",68  ferblantier: "ferblantier",69  frigoriste: "frigoriste",70  peintre: "peintre",71  platrier: "platrier",72  "tir joints peint platr": "tireur_joints",73  "poseur de systemes interieurs": "poseur_systemes",74  "poseur de rev souples": "poseur_revetements",75  "poseur de revetements souples": "poseur_revetements",76  calorifugeur: "calorifugeur",77  "poseur d armature du beton": "ferrailleur",78  ferrailleur: "ferrailleur",79  "monteur mecanicien vitrier": "vitrier",80  "manoeuvre journalier": "manoeuvre",81  manoeuvre: "manoeuvre",82  "manoeuvre specialise": "manoeuvre_specialise",83  "op de p mec cl a": "operateur",84  "ope de p mec cl b": "operateur_b",85  "op d equip l classe a": "operateur_equipement_lourd",86  "op d equip l classe b": "operateur_equipement_lourd_b",87  "parqueteur sableur": "parqueteur",88  "grutier classe a": "grutier",89  "mecanicien de chantier": "mecanicien_chantier",90  "monteur assembleur": "monteur_assembleur",91  "poseur de fondation profonde": "poseur_fondation",92  "install de sys de securite": "installateur_securite",93  "mec protection incendie": "mecanicien_protection_incendie",94  soudeur: "soudeur",95  boutefeu: "boutefeu",96  foreur: "foreur",97};9899/** Spécialités de la CCQ rattachées à un métier : le poseur de systèmes intérieurs est une spécialité du charpentier-menuisier. */100const DERIVED: { from: string; to: string; label: string }[] = [101  { from: "charpentier", to: "poseur_systemes", label: "Charpentier-menuisier — spécialité poseur de systèmes intérieurs" },102  { from: "couvreur", to: "poseur_bardeaux", label: "Couvreur — pose de bardeaux" },103];104105export function tradeCodeFor(label: string): { code: string; matched: boolean } {106  const f = fold(label);107  if (ALIASES[f]) return { code: ALIASES[f], matched: true };108  for (const t of TRADES) if (t.apchq.some((a) => fold(a) === f)) return { code: t.code, matched: true };109  for (const [k, v] of Object.entries(ALIASES)) if (f.startsWith(k) || k.startsWith(f)) return { code: v, matched: true };110  return { code: f.replace(/ /g, "_").slice(0, 40) || "inconnu", matched: false };111}112113/* ---------------------------------------------------------------- parseur */114115export interface ApchqRow {116  trade: string; // libellé APCHQ117  classification: string; // compagnon | apprenti-1…118  baseWage: number;119  vacation: number;120  benefits: number;121  otherFixed: number;122  employer: number;123  total: number;124  exactBreakdown: boolean;125}126127const FIXED_CANDIDATES = [0.26, 0.6, 0.65, 0.03, 0.02, 0.2, 0.043];128const MONEY_RE = /-?\d[\d\s]*(?:[.,]\d+)?\s*\$/g;129const APPR_RE = /apprenti[- ]p[eé]riode\s*(\d)/gi;130131function cellValues(cell: string): number[] {132  return [...cell.matchAll(MONEY_RE)].map((m) => parseMoney(m[0])).filter((v): v is number => v != null);133}134135function cellText(cell: string): string {136  return cell.replace(MONEY_RE, " ").replace(/&#x27;|&#39;|’/g, "'").replace(/\s+/g, " ").trim();137}138139/** Parse le markdown d'une grille APCHQ en enregistrements (compagnon + apprentis). */140export function parseApchqMarkdown(md: string): ApchqRow[] {141  const rows: ApchqRow[] = [];142  let currentTrade: string | null = null;143  for (const line of md.split("\n")) {144    if (!line.startsWith("|")) continue;145    const cells = line.trim().replace(/^\|/, "").replace(/\|$/, "").split("|");146    const text = cells.map(cellText).filter(Boolean).join(" ").trim();147    if (!text || /^-+$/.test(text) || /après avoir|valeur unité|métiers et spécialités|occupations|^total$|horaire/i.test(text)) continue;148    let valueCells = cells.map(cellValues).filter((v) => v.length > 0);149    const apprs = [...text.matchAll(APPR_RE)].map((m) => Number(m[1]));150    const nameOnly = text.replace(APPR_RE, " ").replace(/\s+/g, " ").trim();151    const looksLikeName = nameOnly.length >= 3 && nameOnly.length <= 60 && !/%|\$/.test(nameOnly);152    if (!valueCells.length) {153      if (looksLikeName && !apprs.length) currentTrade = nameOnly;154      continue;155    }156    if (looksLikeName) currentTrade = nameOnly;157    if (!currentTrade) continue;158    // rangée aplatie : toutes les valeurs dans une seule cellule → un seul enregistrement159    if (valueCells.length === 1 && valueCells[0].length >= 12) valueCells = valueCells[0].map((v) => [v]);160    const first = valueCells[0];161    const last = valueCells[valueCells.length - 1];162    const k = Math.max(1, first.length);163    const classes: string[] = apprs.length ? apprs.map((n) => `apprenti-${n}`) : ["compagnon"];164    while (classes.length < k) {165      const prev = classes[classes.length - 1];166      classes.push(`apprenti-${prev === "compagnon" ? 1 : Number(prev.split("-")[1]) + 1}`);167    }168    const allValues = valueCells.flat();169    const otherFixed = FIXED_CANDIDATES.filter((c) => allValues.some((v) => Math.abs(v - c) < 0.0005)).reduce((s, v) => s + v, 0);170    for (let i = 0; i < k; i++) {171      const base = first[i];172      const total = last.length > i ? last[i] : last.length === 1 && i === 0 ? last[0] : null;173      if (base == null || total == null) continue;174      if (!(base > 10 && base < 200) || !(total > base * 1.25 && total < base * 2.1)) continue;175      const pick = (idx: number, lo: number, hi: number): number | null => {176        const c = valueCells[idx];177        if (!c) return null;178        const v = c.length > i ? c[i] : c.length === 1 ? c[0] : null;179        return v != null && v >= lo && v <= hi ? v : null;180      };181      const vac = pick(1, base * 0.1, base * 0.16);182      const ben = pick(2, 3, 14);183      const vacation = vac ?? Math.round(base * 0.13 * 100) / 100;184      const benefits = ben ?? 0;185      const employer = Math.max(0, Math.round((total - base - vacation - benefits - otherFixed) * 100) / 100);186      rows.push({ trade: currentTrade, classification: classes[i], baseWage: base, vacation, benefits, otherFixed: Math.round(otherFixed * 1000) / 1000, employer, total, exactBreakdown: vac != null && ben != null });187    }188  }189  return reclassify(rows);190}191192/**193 * La mise en page du PDF décale parfois les libellés : on rétablit la194 * classification à partir des salaires. Le compagnon est le taux le plus élevé195 * du métier ; chaque apprenti reçoit la période dont le ratio conventionnel196 * (≈ 50 / 60 / 70 / 85 % du compagnon) est le plus proche, dans un régime à197 * 3, 4 ou 5 périodes selon le métier.198 */199function reclassify(rows: ApchqRow[]): ApchqRow[] {200  const groups = new Map<string, ApchqRow[]>();201  for (const r of rows) groups.set(fold(r.trade), [...(groups.get(fold(r.trade)) ?? []), r]);202  const out: ApchqRow[] = [];203  for (const g of groups.values()) {204    const uniq = [...new Map(g.map((r) => [r.baseWage.toFixed(2), r])).values()].sort((a, b) => b.baseWage - a.baseWage);205    const comp = uniq[0];206    out.push({ ...comp, classification: "compagnon" });207    const appr = uniq.slice(1).sort((a, b) => a.baseWage - b.baseWage);208    if (!appr.length) continue;209    const parsedMax = Math.max(...g.map((r) => Number(r.classification.split("-")[1]) || 0));210    const periods = Math.max(appr.length, Math.min(5, parsedMax || 0), 3);211    const scheme = periods >= 5 ? [0.5, 0.6, 0.7, 0.85, 0.85] : periods === 4 ? [0.5, 0.6, 0.7, 0.85] : [0.6, 0.7, 0.85];212    const used = new Set<number>();213    for (const r of appr) {214      const ratio = r.baseWage / comp.baseWage;215      let best = 1;216      let bestD = Infinity;217      scheme.forEach((x, idx) => { const d = Math.abs(x - ratio); if (d < bestD && !used.has(idx + 1)) { bestD = d; best = idx + 1; } });218      used.add(best);219      out.push({ ...r, classification: `apprenti-${best}` });220    }221  }222  return out;223}224225/* --------------------------------------------------------------- connecteur */226227export const apchqConnector: CostConnector = {228  config: APCHQ_CONFIG,229230  async discover(opts) {231    const page = await scrape(APCHQ_LANDING, { links: true, timeoutMs: APCHQ_CONFIG.timeoutMs, retries: APCHQ_CONFIG.retries, rateLimitMs: APCHQ_CONFIG.rateLimitMs });232    return discoverFromLinks([...page.links, ...page.markdown.matchAll(/https:\/\/media\.apchq\.com\/[^\s)]+\.pdf/g).map((m) => m[0])], opts.maxPages ?? APCHQ_CONFIG.maxPages);233  },234235  async fetch(doc, opts) {236    const r = await scrape(doc.url, { timeoutMs: APCHQ_CONFIG.timeoutMs, retries: APCHQ_CONFIG.retries, rateLimitMs: APCHQ_CONFIG.rateLimitMs, concurrency: APCHQ_CONFIG.concurrency });237    const hash = contentHash(r.markdown);238    return { url: doc.url, fetchedAt: new Date().toISOString(), markdown: r.markdown, metadata: r.metadata, statusCode: r.statusCode, contentHash: hash, unchanged: !opts.force && lastHashFor(doc.url, opts.db) === hash, meta: doc.meta };239  },240241  async extract(raw, opts) {242    const effective = dateFromUrl(raw.url);243    const sector = sectorFromUrl(raw.url);244    if (!effective || !sector) return [];245    let rows = parseApchqMarkdown(raw.markdown);246    // lecture assistée (Claude Haiku) pour rétablir les libellés décalés par la conversion PDF → markdown ;247    // chaque ligne IA est vérifiée contre les couples (taux, total) du parseur déterministe.248    if (process.env.ANTHROPIC_API_KEY && !opts.onlyItems?.includes("no-ai")) {249      try {250        const pdf = await downloadPdf(raw.url, 60_000);251        const ai = await extractApchqWithClaude(pdf, rows, { url: raw.url, sector, effectiveDate: effective }, opts.db);252        if (ai.rows.length >= Math.min(20, rows.length * 0.6)) {253          opts.log?.({ connector: "apchq", url: raw.url, status: "ok", duration: ai.usage.latencyMs, observations: ai.rows.length, validationErrors: ai.dropped, timestamp: new Date().toISOString(), message: `lecture IA : ${ai.rows.length} lignes vérifiées, ${ai.dropped} rejetées (regex : ${rows.length})` });254          rows = ai.rows;255        } else {256          opts.log?.({ connector: "apchq", url: raw.url, status: "skipped", duration: ai.usage.latencyMs, observations: ai.rows.length, validationErrors: ai.dropped, timestamp: new Date().toISOString(), message: `lecture IA insuffisante (${ai.rows.length} lignes) — parseur déterministe conservé` });257        }258      } catch (e) {259        opts.log?.({ connector: "apchq", url: raw.url, status: "skipped", duration: 0, observations: 0, validationErrors: 0, timestamp: new Date().toISOString(), message: `lecture IA indisponible : ${(e as Error).message} — parseur déterministe conservé` });260      }261    }262    return rows.map((r) => ({263      externalId: `${sector}|${effective}|${fold(r.trade)}|${r.classification}`, sourceUrl: raw.url, retrievedAt: raw.fetchedAt, effectiveDate: effective,264      title: `${r.trade} — ${r.classification} (${sector})`, description: null, unit: "h", price: r.total, regularPrice: null, salePrice: null, currency: "CAD", location: "QC", category: "labour",265      payload: { ...r, sector },266    }));267  },268269  async normalize(obs) {270    const out: CanonicalObservation[] = [];271    for (const o of obs) {272      const p = o.payload as unknown as ApchqRow & { sector: string };273      const { code, matched } = tradeCodeFor(p.trade);274      const base = {275        kind: "labour" as const, sector: p.sector, classification: p.classification, region: "QC", effectiveFrom: o.effectiveDate ?? "", baseWage: p.baseWage, vacationCost: p.vacation, benefitsCost: p.benefits,276        employerContributions: p.employer, otherContributions: p.otherFixed, totalEmployerCost: p.total, sourceUrl: o.sourceUrl, confidence: (p.exactBreakdown ? 95 : 88) - (matched ? 0 : 10), raw: o,277      };278      out.push({ ...base, tradeCode: code, tradeNameFr: p.trade });279      for (const d of DERIVED) if (d.from === code) out.push({ ...base, tradeCode: d.to, tradeNameFr: d.label, confidence: base.confidence - 5 });280    }281    return out;282  },283284  validate(obs, opts): ValidationResult {285    return validateObservations(obs, { recentPrices: (c) => recentPricesFor(c, 120, opts.db), referencePrice: (c) => referencePriceFor(c, opts.db) });286  },287};288289/** Filtre les liens de la page APCHQ : grilles « coût main-d'œuvre », temps simple, léger/lourd/IC. */290export function discoverFromLinks(links: string[], maxPages: number): SourceDocument[] {291  const seen = new Set<string>();292  const docs: SourceDocument[] = [];293  for (const l of links) {294    const url = l.split("#")[0].trim();295    if (!/media\.apchq\.com\/.+cout-main-d-oeuvre.+\.pdf$/i.test(url)) continue;296    if (APCHQ_CONFIG.excludedPaths.some((re) => re.test(url))) continue;297    const d = dateFromUrl(url);298    const s = sectorFromUrl(url);299    if (!d || !s || seen.has(url)) continue;300    seen.add(url);301    docs.push({ url, title: `APCHQ ${s} ${d}`, meta: { effectiveDate: d, sector: s } });302  }303  return docs.sort((a, b) => String(a.meta?.effectiveDate).localeCompare(String(b.meta?.effectiveDate))).slice(0, maxPages);304}305306/** Pour les tests : construit un RawDocument depuis un markdown de fixture. */307export function rawDocFromMarkdown(url: string, markdown: string, meta?: RawDocument["meta"]): RawDocument {308  return { url, fetchedAt: new Date().toISOString(), markdown, metadata: {}, statusCode: 200, contentHash: contentHash(markdown), unchanged: false, meta };309}310311export type { RawObservation, RunOptions };312