// Auteur : Simon-Pierre Boucher — contact@spboucher.ai
/**
 * Connecteur APCHQ — grilles publiques « Coût horaire de la main-d'œuvre »
 * (PDF, media.apchq.com) : taux horaire conventionné (CCQ), indemnité de
 * vacances (13 %), avantages sociaux, cotisations (AE, RQAP, RRQ, FSS, CNESST),
 * prélèvements (CCQ, AECQ, fonds), TOTAL = coût employeur complet.
 *
 * Firecrawl rend le PDF en tableau markdown ; la mise en page fusionne parfois
 * deux lignes (compagnon + apprenti 1, ou apprenti 2 + apprenti 3) dans une
 * même rangée : le parseur lit les montants cellule par cellule et reconstitue
 * un enregistrement par valeur de « Taux horaire ». Les champs exacts sont le
 * taux horaire et le total ; la ventilation intermédiaire est lue quand la
 * cellule est nette, sinon reconstituée (vacances = 13 % réglementaire,
 * cotisations employeur = total − autres postes) et la confiance est réduite.
 */
import { TRADES } from "../taxonomy";
import { contentHash, parseMoney, scrape } from "./firecrawl";
import { lastHashFor } from "./store";
import { downloadPdf, extractApchqWithClaude } from "./apchq-ai";
import type { CanonicalObservation, ConnectorConfig, CostConnector, RawDocument, RawObservation, RunOptions, SourceDocument, ValidationResult } from "./types";
import { validateObservations } from "./validate";
import { recentPricesFor, referencePriceFor } from "./store";

export const APCHQ_PARSER_VERSION = "apchq-1.0";
export const APCHQ_LANDING = "https://www.apchq.com/nos-services/relations-du-travail/paie/couts-horaires-et-paie/";

export const APCHQ_CONFIG: ConnectorConfig = {
  key: "apchq", name: "APCHQ — Coûts horaires de la main-d'œuvre", domain: "apchq.com", maxPages: 40,
  allowedPaths: [/^https:\/\/media\.apchq\.com\/.+\.pdf$/i, /^https:\/\/www\.apchq\.com\/nos-services\/relations-du-travail\/paie\//],
  excludedPaths: [/temps-(demi|et-demi|double)/i, /exemple-de-paie/i, /avantages-(sociaux|imposable)/i, /residentiel-r-2/i, /chantiers?-isoles/i, /baie-james/i, /special-paie/i, /ic_special/i, /ic-speciale/i],
  refreshDays: 7, timeoutMs: 150_000, retries: 2, rateLimitMs: 1500, concurrency: 2,
};

const MONTHS: [RegExp, string][] = [
  [/^jan/, "01"], [/^f[eé]v/, "02"], [/^mar/, "03"], [/^a[vr]r?[iv]?/, "04"], [/^mai/, "05"], [/^juin/, "06"], [/^juil/, "07"], [/^ao[uû]/, "08"], [/^sep/, "09"], [/^oct/, "10"], [/^nov/, "11"], [/^d[eé]c/, "12"],
];

/** Date d'entrée en vigueur depuis le nom du PDF (« 26-arvil-2026 », « 1er-janvier-2024 »). */
export function dateFromUrl(url: string): string | null {
  const m = url.toLowerCase().match(/(\d{1,2})(?:er)?-([a-zéû]+)-(\d{4})/);
  if (!m) return null;
  const month = MONTHS.find(([re]) => re.test(m[2]))?.[1];
  if (!month) return null;
  return `${m[3]}-${month}-${m[1].padStart(2, "0")}`;
}

export function sectorFromUrl(url: string): "residentiel_leger" | "residentiel_lourd" | "ic" | null {
  const u = url.toLowerCase();
  if (/residentiel-leger/.test(u)) return "residentiel_leger";
  if (/residentiel-lourd/.test(u)) return "residentiel_lourd";
  if (/commercial/.test(u)) return "ic";
  return null;
}

const fold = (s: string) => s.replace(/œ/g, "oe").replace(/Œ/g, "oe").normalize("NFD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();

/** Libellés APCHQ (abrégés dans les PDF) → codes de métier internes. */
const ALIASES: Record<string, string> = {
  "charpentier menuisier": "charpentier",
  "briqueteur macon": "briqueteur",
  carreleur: "carreleur",
  "cimentier applicateur": "cimentier",
  "coffreur a beton": "coffreur",
  couvreur: "couvreur",
  electricien: "electricien",
  tuyauteur: "plombier",
  plombier: "plombier",
  ferblantier: "ferblantier",
  frigoriste: "frigoriste",
  peintre: "peintre",
  platrier: "platrier",
  "tir joints peint platr": "tireur_joints",
  "poseur de systemes interieurs": "poseur_systemes",
  "poseur de rev souples": "poseur_revetements",
  "poseur de revetements souples": "poseur_revetements",
  calorifugeur: "calorifugeur",
  "poseur d armature du beton": "ferrailleur",
  ferrailleur: "ferrailleur",
  "monteur mecanicien vitrier": "vitrier",
  "manoeuvre journalier": "manoeuvre",
  manoeuvre: "manoeuvre",
  "manoeuvre specialise": "manoeuvre_specialise",
  "op de p mec cl a": "operateur",
  "ope de p mec cl b": "operateur_b",
  "op d equip l classe a": "operateur_equipement_lourd",
  "op d equip l classe b": "operateur_equipement_lourd_b",
  "parqueteur sableur": "parqueteur",
  "grutier classe a": "grutier",
  "mecanicien de chantier": "mecanicien_chantier",
  "monteur assembleur": "monteur_assembleur",
  "poseur de fondation profonde": "poseur_fondation",
  "install de sys de securite": "installateur_securite",
  "mec protection incendie": "mecanicien_protection_incendie",
  soudeur: "soudeur",
  boutefeu: "boutefeu",
  foreur: "foreur",
};

/** Spécialités de la CCQ rattachées à un métier : le poseur de systèmes intérieurs est une spécialité du charpentier-menuisier. */
const DERIVED: { from: string; to: string; label: string }[] = [
  { from: "charpentier", to: "poseur_systemes", label: "Charpentier-menuisier — spécialité poseur de systèmes intérieurs" },
  { from: "couvreur", to: "poseur_bardeaux", label: "Couvreur — pose de bardeaux" },
];

export function tradeCodeFor(label: string): { code: string; matched: boolean } {
  const f = fold(label);
  if (ALIASES[f]) return { code: ALIASES[f], matched: true };
  for (const t of TRADES) if (t.apchq.some((a) => fold(a) === f)) return { code: t.code, matched: true };
  for (const [k, v] of Object.entries(ALIASES)) if (f.startsWith(k) || k.startsWith(f)) return { code: v, matched: true };
  return { code: f.replace(/ /g, "_").slice(0, 40) || "inconnu", matched: false };
}

/* ---------------------------------------------------------------- parseur */

export interface ApchqRow {
  trade: string; // libellé APCHQ
  classification: string; // compagnon | apprenti-1…
  baseWage: number;
  vacation: number;
  benefits: number;
  otherFixed: number;
  employer: number;
  total: number;
  exactBreakdown: boolean;
}

const FIXED_CANDIDATES = [0.26, 0.6, 0.65, 0.03, 0.02, 0.2, 0.043];
const MONEY_RE = /-?\d[\d\s]*(?:[.,]\d+)?\s*\$/g;
const APPR_RE = /apprenti[- ]p[eé]riode\s*(\d)/gi;

function cellValues(cell: string): number[] {
  return [...cell.matchAll(MONEY_RE)].map((m) => parseMoney(m[0])).filter((v): v is number => v != null);
}

function cellText(cell: string): string {
  return cell.replace(MONEY_RE, " ").replace(/&#x27;|&#39;|’/g, "'").replace(/\s+/g, " ").trim();
}

/** Parse le markdown d'une grille APCHQ en enregistrements (compagnon + apprentis). */
export function parseApchqMarkdown(md: string): ApchqRow[] {
  const rows: ApchqRow[] = [];
  let currentTrade: string | null = null;
  for (const line of md.split("\n")) {
    if (!line.startsWith("|")) continue;
    const cells = line.trim().replace(/^\|/, "").replace(/\|$/, "").split("|");
    const text = cells.map(cellText).filter(Boolean).join(" ").trim();
    if (!text || /^-+$/.test(text) || /après avoir|valeur unité|métiers et spécialités|occupations|^total$|horaire/i.test(text)) continue;
    let valueCells = cells.map(cellValues).filter((v) => v.length > 0);
    const apprs = [...text.matchAll(APPR_RE)].map((m) => Number(m[1]));
    const nameOnly = text.replace(APPR_RE, " ").replace(/\s+/g, " ").trim();
    const looksLikeName = nameOnly.length >= 3 && nameOnly.length <= 60 && !/%|\$/.test(nameOnly);
    if (!valueCells.length) {
      if (looksLikeName && !apprs.length) currentTrade = nameOnly;
      continue;
    }
    if (looksLikeName) currentTrade = nameOnly;
    if (!currentTrade) continue;
    // rangée aplatie : toutes les valeurs dans une seule cellule → un seul enregistrement
    if (valueCells.length === 1 && valueCells[0].length >= 12) valueCells = valueCells[0].map((v) => [v]);
    const first = valueCells[0];
    const last = valueCells[valueCells.length - 1];
    const k = Math.max(1, first.length);
    const classes: string[] = apprs.length ? apprs.map((n) => `apprenti-${n}`) : ["compagnon"];
    while (classes.length < k) {
      const prev = classes[classes.length - 1];
      classes.push(`apprenti-${prev === "compagnon" ? 1 : Number(prev.split("-")[1]) + 1}`);
    }
    const allValues = valueCells.flat();
    const otherFixed = FIXED_CANDIDATES.filter((c) => allValues.some((v) => Math.abs(v - c) < 0.0005)).reduce((s, v) => s + v, 0);
    for (let i = 0; i < k; i++) {
      const base = first[i];
      const total = last.length > i ? last[i] : last.length === 1 && i === 0 ? last[0] : null;
      if (base == null || total == null) continue;
      if (!(base > 10 && base < 200) || !(total > base * 1.25 && total < base * 2.1)) continue;
      const pick = (idx: number, lo: number, hi: number): number | null => {
        const c = valueCells[idx];
        if (!c) return null;
        const v = c.length > i ? c[i] : c.length === 1 ? c[0] : null;
        return v != null && v >= lo && v <= hi ? v : null;
      };
      const vac = pick(1, base * 0.1, base * 0.16);
      const ben = pick(2, 3, 14);
      const vacation = vac ?? Math.round(base * 0.13 * 100) / 100;
      const benefits = ben ?? 0;
      const employer = Math.max(0, Math.round((total - base - vacation - benefits - otherFixed) * 100) / 100);
      rows.push({ trade: currentTrade, classification: classes[i], baseWage: base, vacation, benefits, otherFixed: Math.round(otherFixed * 1000) / 1000, employer, total, exactBreakdown: vac != null && ben != null });
    }
  }
  return reclassify(rows);
}

/**
 * La mise en page du PDF décale parfois les libellés : on rétablit la
 * classification à partir des salaires. Le compagnon est le taux le plus élevé
 * du métier ; chaque apprenti reçoit la période dont le ratio conventionnel
 * (≈ 50 / 60 / 70 / 85 % du compagnon) est le plus proche, dans un régime à
 * 3, 4 ou 5 périodes selon le métier.
 */
function reclassify(rows: ApchqRow[]): ApchqRow[] {
  const groups = new Map<string, ApchqRow[]>();
  for (const r of rows) groups.set(fold(r.trade), [...(groups.get(fold(r.trade)) ?? []), r]);
  const out: ApchqRow[] = [];
  for (const g of groups.values()) {
    const uniq = [...new Map(g.map((r) => [r.baseWage.toFixed(2), r])).values()].sort((a, b) => b.baseWage - a.baseWage);
    const comp = uniq[0];
    out.push({ ...comp, classification: "compagnon" });
    const appr = uniq.slice(1).sort((a, b) => a.baseWage - b.baseWage);
    if (!appr.length) continue;
    const parsedMax = Math.max(...g.map((r) => Number(r.classification.split("-")[1]) || 0));
    const periods = Math.max(appr.length, Math.min(5, parsedMax || 0), 3);
    const scheme = periods >= 5 ? [0.5, 0.6, 0.7, 0.85, 0.85] : periods === 4 ? [0.5, 0.6, 0.7, 0.85] : [0.6, 0.7, 0.85];
    const used = new Set<number>();
    for (const r of appr) {
      const ratio = r.baseWage / comp.baseWage;
      let best = 1;
      let bestD = Infinity;
      scheme.forEach((x, idx) => { const d = Math.abs(x - ratio); if (d < bestD && !used.has(idx + 1)) { bestD = d; best = idx + 1; } });
      used.add(best);
      out.push({ ...r, classification: `apprenti-${best}` });
    }
  }
  return out;
}

/* --------------------------------------------------------------- connecteur */

export const apchqConnector: CostConnector = {
  config: APCHQ_CONFIG,

  async discover(opts) {
    const page = await scrape(APCHQ_LANDING, { links: true, timeoutMs: APCHQ_CONFIG.timeoutMs, retries: APCHQ_CONFIG.retries, rateLimitMs: APCHQ_CONFIG.rateLimitMs });
    return discoverFromLinks([...page.links, ...page.markdown.matchAll(/https:\/\/media\.apchq\.com\/[^\s)]+\.pdf/g).map((m) => m[0])], opts.maxPages ?? APCHQ_CONFIG.maxPages);
  },

  async fetch(doc, opts) {
    const r = await scrape(doc.url, { timeoutMs: APCHQ_CONFIG.timeoutMs, retries: APCHQ_CONFIG.retries, rateLimitMs: APCHQ_CONFIG.rateLimitMs, concurrency: APCHQ_CONFIG.concurrency });
    const hash = contentHash(r.markdown);
    return { url: doc.url, fetchedAt: new Date().toISOString(), markdown: r.markdown, metadata: r.metadata, statusCode: r.statusCode, contentHash: hash, unchanged: !opts.force && lastHashFor(doc.url, opts.db) === hash, meta: doc.meta };
  },

  async extract(raw, opts) {
    const effective = dateFromUrl(raw.url);
    const sector = sectorFromUrl(raw.url);
    if (!effective || !sector) return [];
    let rows = parseApchqMarkdown(raw.markdown);
    // lecture assistée (Claude Haiku) pour rétablir les libellés décalés par la conversion PDF → markdown ;
    // chaque ligne IA est vérifiée contre les couples (taux, total) du parseur déterministe.
    if (process.env.ANTHROPIC_API_KEY && !opts.onlyItems?.includes("no-ai")) {
      try {
        const pdf = await downloadPdf(raw.url, 60_000);
        const ai = await extractApchqWithClaude(pdf, rows, { url: raw.url, sector, effectiveDate: effective }, opts.db);
        if (ai.rows.length >= Math.min(20, rows.length * 0.6)) {
          opts.log?.({ connector: "apchq", url: raw.url, status: "ok", duration: ai.usage.latencyMs, observations: ai.rows.length, validationErrors: ai.dropped, timestamp: new Date().toISOString(), message: `lecture IA : ${ai.rows.length} lignes vérifiées, ${ai.dropped} rejetées (regex : ${rows.length})` });
          rows = ai.rows;
        } else {
          opts.log?.({ connector: "apchq", url: raw.url, status: "skipped", duration: ai.usage.latencyMs, observations: ai.rows.length, validationErrors: ai.dropped, timestamp: new Date().toISOString(), message: `lecture IA insuffisante (${ai.rows.length} lignes) — parseur déterministe conservé` });
        }
      } catch (e) {
        opts.log?.({ connector: "apchq", url: raw.url, status: "skipped", duration: 0, observations: 0, validationErrors: 0, timestamp: new Date().toISOString(), message: `lecture IA indisponible : ${(e as Error).message} — parseur déterministe conservé` });
      }
    }
    return rows.map((r) => ({
      externalId: `${sector}|${effective}|${fold(r.trade)}|${r.classification}`, sourceUrl: raw.url, retrievedAt: raw.fetchedAt, effectiveDate: effective,
      title: `${r.trade} — ${r.classification} (${sector})`, description: null, unit: "h", price: r.total, regularPrice: null, salePrice: null, currency: "CAD", location: "QC", category: "labour",
      payload: { ...r, sector },
    }));
  },

  async normalize(obs) {
    const out: CanonicalObservation[] = [];
    for (const o of obs) {
      const p = o.payload as unknown as ApchqRow & { sector: string };
      const { code, matched } = tradeCodeFor(p.trade);
      const base = {
        kind: "labour" as const, sector: p.sector, classification: p.classification, region: "QC", effectiveFrom: o.effectiveDate ?? "", baseWage: p.baseWage, vacationCost: p.vacation, benefitsCost: p.benefits,
        employerContributions: p.employer, otherContributions: p.otherFixed, totalEmployerCost: p.total, sourceUrl: o.sourceUrl, confidence: (p.exactBreakdown ? 95 : 88) - (matched ? 0 : 10), raw: o,
      };
      out.push({ ...base, tradeCode: code, tradeNameFr: p.trade });
      for (const d of DERIVED) if (d.from === code) out.push({ ...base, tradeCode: d.to, tradeNameFr: d.label, confidence: base.confidence - 5 });
    }
    return out;
  },

  validate(obs, opts): ValidationResult {
    return validateObservations(obs, { recentPrices: (c) => recentPricesFor(c, 120, opts.db), referencePrice: (c) => referencePriceFor(c, opts.db) });
  },
};

/** Filtre les liens de la page APCHQ : grilles « coût main-d'œuvre », temps simple, léger/lourd/IC. */
export function discoverFromLinks(links: string[], maxPages: number): SourceDocument[] {
  const seen = new Set<string>();
  const docs: SourceDocument[] = [];
  for (const l of links) {
    const url = l.split("#")[0].trim();
    if (!/media\.apchq\.com\/.+cout-main-d-oeuvre.+\.pdf$/i.test(url)) continue;
    if (APCHQ_CONFIG.excludedPaths.some((re) => re.test(url))) continue;
    const d = dateFromUrl(url);
    const s = sectorFromUrl(url);
    if (!d || !s || seen.has(url)) continue;
    seen.add(url);
    docs.push({ url, title: `APCHQ ${s} ${d}`, meta: { effectiveDate: d, sector: s } });
  }
  return docs.sort((a, b) => String(a.meta?.effectiveDate).localeCompare(String(b.meta?.effectiveDate))).slice(0, maxPages);
}

/** Pour les tests : construit un RawDocument depuis un markdown de fixture. */
export function rawDocFromMarkdown(url: string, markdown: string, meta?: RawDocument["meta"]): RawDocument {
  return { url, fetchedAt: new Date().toISOString(), markdown, metadata: {}, statusCode: 200, contentHash: contentHash(markdown), unchanged: false, meta };
}

export type { RawObservation, RunOptions };