Vrai-Prix — l'évaluation du vrai prix des propriétés résidentielles au Québec.
TypeScript 90.2%
JavaScript 3.5%
Python 3.4%
CSS 1.9%
HTML 0.6%
1// Auteur : Simon-Pierre Boucher — contact@spboucher.ai2/**3 * Connecteur détaillant générique (Canac, BMR, Patrick Morin, RONA, Home Depot) :4 * articles avec `retail_query` → URL produit (cost_item_sources ou découverte5 * par recherche + matching) → fiche (Firecrawl) → prix affiché (métadonnées6 * product:price:amount ou texte) → conversion vers l'unité canonique →7 * observation historisée (une par jour, par article et par source).8 * Prix régulier préféré au prix promotionnel ; jamais d'écrasement.9 */10import type Database from "better-sqlite3";11import { getCostDb, sourceIdByKey } from "../db";12import { convertPrice, isKnownUnit } from "../units";13import type { CanonicalUnit } from "../types";14import { contentHash, parseMoney, scrape, search, type ScrapeResult } from "./firecrawl";15import { aiMatch, deterministicMatch, upsertMapping, type Candidate } from "./matching";16import { lastHashFor, recentPricesFor, referencePriceFor } from "./store";17import type { CanonicalObservation, ConnectorConfig, ConnectorKey, CostConnector, RawDocument, RawObservation, RunOptions, SourceDocument, ValidationResult } from "./types";18import { validateObservations } from "./validate";1920export const RETAIL_PARSER_VERSION = "retail-1.0";2122export interface RetailAdapter {23 key: ConnectorKey;24 name: string;25 domain: string;26 searchPrefix: string; // ex. « canac.ca »27 isProductUrl(url: string): boolean;28 /** extrait prix affiché / régulier / promo / unité affichée depuis la fiche */29 parse(r: ScrapeResult): { title: string; price: number | null; regularPrice: number | null; salePrice: number | null; unitText: string | null; currency: string };30 active: boolean;31 config: Partial<ConnectorConfig>;32}3334interface ItemRow { id: number; canonical_code: string; name_fr: string; unit: CanonicalUnit; retail_query: string; retail_pack_qty: number | null; retail_pack_unit: string | null; reference_price: number | null }3536/* -------------------------------------------------------- aides d'analyse */3738/** Détecte un prix promotionnel dans le markdown (prix barré, « Spécial », « Économisez », « Prix régulier »). */39export function detectPromotion(mdFull: string, displayed: number | null): { regular: number | null; sale: number | null } {40 // seule la zone du produit principal compte (avant les produits associés / la description)41 const cut = mdFull.replace(/\u00a0/g, " ").search(/utile avec cet achat|produits (similaires|associ[ée]s|compl[ée]mentaires)|vous aimerez|articles connexes|\n#{1,3}\s*description/i);42 const md = cut > 0 ? mdFull.slice(0, cut) : mdFull.replace(/\u00a0/g, " ").slice(0, 3000);43 const regMatch = md.match(/(?:prix\s+r[ée]g(?:ulier)?\.?|r[ée]g\.?|was|prix\s+courant)\s*:?\s*(\d[\d\s]*(?:[.,]\d+)?)\s*\$/i) ?? md.match(/~~\s*(\d[\d\s]*(?:[.,]\d+)?)\s*\$\s*~~/);44 const regular = regMatch ? parseMoney(regMatch[1]) : null;45 // le mot « promo » ne compte que près du prix affiché (bandeaux de site exclus)46 const shown = displayed != null ? md.search(new RegExp(`${displayed.toFixed(2).replace(".", "[.,]")}\\s*\\$`)) : -1;47 const near = shown >= 0 ? md.slice(Math.max(0, shown - 250), shown + 250) : "";48 const promo = /(sp[ée]cial|[ée]conomisez|rabais|en solde|liquidation|prix\s+promo)/i.test(near);49 if (regular != null && displayed != null && regular > displayed) return { regular, sale: displayed };50 if (promo && displayed != null && regular == null) return { regular: null, sale: displayed };51 return { regular: displayed, sale: null };52}5354/** Contenu d'emballage lu sur la fiche, dans l'unité canonique (ex. « couvre 39,8 pi² », « 100 pi », « pqt/50 »). */55export function detectPackQty(text: string, unit: CanonicalUnit): number | null {56 const t = text.replace(/ /g, " ");57 const num = (s: string) => parseMoney(s);58 if (unit === "pi2") {59 const m = t.match(/(?:couvr\w*|couverture|superficie)\s*(?:de|:)?\s*(?:jusqu.à\s*)?(\d[\d\s]*(?:[.,]\d+)?)\s*(?:pi|ft)\s*(?:²|2|carr)/i) ?? t.match(/(\d[\d\s]*(?:[.,]\d+)?)\s*(?:pi|ft)\s*(?:²|2)\s*(?:par|\/)\s*(?:paquet|pqt|boîte|boite|rouleau|sac)/i);60 if (m) return num(m[1]);61 const sheet = t.match(/\b(\d+)\s*(?:pi|')\s*x\s*(\d+)\s*(?:pi|')\b/i);62 if (sheet) { const a = Number(sheet[1]) * Number(sheet[2]); if (a >= 8 && a <= 200) return a; }63 return null;64 }65 if (unit === "pi_lin") {66 const m = t.match(/(\d[\d\s]*(?:[.,]\d+)?)\s*(?:pi|ft)\b(?!\s*(?:²|2))/i);67 if (m) { const v = num(m[1]); if (v != null && v >= 4 && v <= 1000) return v; }68 const mm = t.match(/(\d+(?:[.,]\d+)?)\s*m\b/i);69 if (mm) { const v = num(mm[1]); if (v != null && v >= 5 && v <= 500) return Math.round(v * 3.28084 * 10) / 10; }70 return null;71 }72 if (unit === "lb") {73 const m = t.match(/(\d+(?:[.,]\d+)?)\s*lb\b/i);74 if (m) return num(m[1]);75 const kg = t.match(/(\d+(?:[.,]\d+)?)\s*kg\b/i);76 if (kg) { const v = num(kg[1]); return v != null ? Math.round(v * 2.20462 * 100) / 100 : null; }77 return null;78 }79 if (unit === "unit") {80 const m = t.match(/(?:pqt|paquet|bo[iî]te|ens\.?|ensemble|lot)\s*(?:de|\/|:)?\s*(\d{2,4})\b/i) ?? t.match(/\b(\d{2,4})\s*(?:unités|pièces|mcx|pcs|un\.)\b/i);81 if (m) { const v = Number(m[1]); return v >= 2 && v <= 5000 ? v : null; }82 }83 return null;84}8586export function unitTextToSourceUnit(unitText: string | null, unit: CanonicalUnit, packQty: number | null, defaultPackUnit: string | null): string {87 const t = (unitText ?? "").toLowerCase();88 if (/pi\s*²|pi2|sq\.?\s*ft/.test(t) && unit === "pi2") return "pi2";89 if (/pi\s*lin|lin\.?\s*ft|\/\s*pi\b/.test(t) && unit === "pi_lin") return "pi_lin";90 if (/m\s*²|m2/.test(t)) return "m2";91 if (/m\s*³|m3/.test(t)) return "m3";92 if (/\blb\b/.test(t)) return "lb";93 if (/\bkg\b/.test(t)) return "kg";94 if (/feuille|sheet/.test(t)) return "feuille";95 if (/paquet|pqt|pack|ballot|bundle/.test(t)) return "paquet";96 if (/rouleau|roll/.test(t)) return "rouleau";97 if (/bo[iî]te|box/.test(t)) return "boite";98 if (/sac|bag/.test(t)) return "sac";99 if (/seau|pail/.test(t)) return "seau";100 if (/chaque|each|\/\s*ch\b|unité/.test(t)) return packQty && unit !== "unit" ? (defaultPackUnit && isKnownUnit(defaultPackUnit) ? defaultPackUnit : "paquet") : "chaque";101 if (t.trim()) return t.trim(); // unité affichée non reconnue → rejet explicite en validation102 if (packQty && unit !== "unit") return defaultPackUnit && isKnownUnit(defaultPackUnit) ? defaultPackUnit : "paquet";103 return unit === "unit" ? "chaque" : unit;104}105106/* ------------------------------------------------------------ connecteur */107108/** Normalise les espaces insécables des fiches avant analyse. */109function clean(r: ScrapeResult): ScrapeResult {110 return { ...r, markdown: r.markdown.replace(/\u00a0/g, " ") };111}112113export function makeRetailConnector(a: RetailAdapter): CostConnector {114 const config: ConnectorConfig = {115 key: a.key, name: a.name, domain: a.domain, maxPages: 400, allowedPaths: [new RegExp(`^https?://(www\\.)?${a.domain.replace(/\./g, "\\.")}/`)], excludedPaths: [/\/(recherche|search|panier|cart|checkout|compte|account|sign-in|login)/i],116 refreshDays: 1, timeoutMs: 90_000, retries: 2, rateLimitMs: 1200, concurrency: 2, ...a.config,117 };118119 const itemsWithQuery = (d: Database.Database, only?: string[]): ItemRow[] => {120 const rows = d.prepare("SELECT id, canonical_code, name_fr, unit, retail_query, retail_pack_qty, retail_pack_unit, reference_price FROM cost_items WHERE active=1 AND retail_query IS NOT NULL ORDER BY canonical_code").all() as ItemRow[];121 return only?.length ? rows.filter((r) => only.includes(r.canonical_code)) : rows;122 };123124 return {125 config,126127 /** Une URL produit par article : source connue (cost_item_sources) sinon découverte par recherche + matching. */128 async discover(opts) {129 const d = opts.db ?? getCostDb();130 const sid = sourceIdByKey(a.key, d);131 const docs: SourceDocument[] = [];132 const known = d.prepare("SELECT s.product_url, s.pack_qty, s.pack_unit, i.canonical_code FROM cost_item_sources s JOIN cost_items i ON i.id=s.cost_item_id WHERE s.source_id=? AND s.active=1 AND s.approved=1").all(sid) as { product_url: string; pack_qty: number | null; pack_unit: string | null; canonical_code: string }[];133 const knownByItem = new Map(known.map((k) => [k.canonical_code, k]));134 const rejected = new Set((d.prepare("SELECT proposed_item_code FROM product_mappings WHERE source_id=? AND status IN ('pending','rejected') AND created_at >= datetime('now','-7 days')").all(sid) as { proposed_item_code: string | null }[]).map((r) => r.proposed_item_code));135 let discoveries = 0;136 const max = opts.maxPages ?? config.maxPages;137 for (const it of itemsWithQuery(d, opts.onlyItems)) {138 if (docs.length >= max) break;139 const k = knownByItem.get(it.canonical_code);140 if (k) { docs.push({ url: k.product_url, title: it.name_fr, meta: { itemCode: it.canonical_code, packQty: k.pack_qty ?? it.retail_pack_qty, packUnit: k.pack_unit ?? it.retail_pack_unit } }); continue; }141 if (rejected.has(it.canonical_code) && !opts.force) { opts.log?.({ connector: a.key, url: null, status: "skipped", duration: 0, observations: 0, validationErrors: 0, timestamp: new Date().toISOString(), message: `${it.canonical_code} : mapping en attente/rejeté récemment` }); continue; }142 const t0 = Date.now();143 try {144 const hits = await search(`${a.searchPrefix} ${it.retail_query}`, 6, { timeoutMs: 60_000, retries: 1, rateLimitMs: config.rateLimitMs });145 const cands: Candidate[] = hits.filter((h) => a.isProductUrl(h.url) && !config.excludedPaths.some((re) => re.test(h.url))).map((h) => ({ url: h.url.split("?")[0], title: h.title, description: h.description }));146 let dec = deterministicMatch(it.retail_query, cands);147 let packQty: number | null = null;148 if (dec.candidate && dec.confidence < 0.85 && cands.length && process.env.ANTHROPIC_API_KEY) {149 const ai = await aiMatch(it.name_fr, it.unit, it.retail_query, cands, d);150 dec = ai.candidate ? { ...ai, confidence: Math.max(ai.confidence, dec.confidence * 0.8) } : { candidate: null, confidence: 0, method: "ai", rationale: ai.rationale };151 packQty = ai.packQty;152 }153 const status = upsertMapping(a.key, it.canonical_code, dec, packQty ?? it.retail_pack_qty, it.retail_pack_unit, d);154 discoveries++;155 opts.log?.({ connector: a.key, url: dec.candidate?.url ?? null, status: status === "approved" ? "ok" : "skipped", duration: Date.now() - t0, observations: 0, validationErrors: 0, timestamp: new Date().toISOString(), message: `${it.canonical_code} → ${status} (${dec.method}, ${dec.confidence.toFixed(2)}) ${dec.rationale}` });156 if (status === "approved" && dec.candidate) docs.push({ url: dec.candidate.url, title: it.name_fr, meta: { itemCode: it.canonical_code, packQty: packQty ?? it.retail_pack_qty, packUnit: it.retail_pack_unit } });157 } catch (e) {158 opts.log?.({ connector: a.key, url: null, status: "error", duration: Date.now() - t0, observations: 0, validationErrors: 0, timestamp: new Date().toISOString(), message: `${it.canonical_code} : recherche échouée — ${(e as Error).message}` });159 }160 }161 void discoveries;162 return docs;163 },164165 async fetch(doc, opts) {166 const r = clean(await scrape(doc.url, { timeoutMs: config.timeoutMs, retries: config.retries, rateLimitMs: config.rateLimitMs, concurrency: config.concurrency }));167 const parsed = a.parse(r);168 // le hash inclut le prix affiché : un changement de prix = document changé169 const hash = contentHash(r.markdown.slice(0, 6000), { price: parsed.price, regular: parsed.regularPrice });170 return { url: doc.url, fetchedAt: new Date().toISOString(), markdown: r.markdown, metadata: { ...r.metadata, parsed }, statusCode: r.statusCode, contentHash: hash, unchanged: !opts.force && lastHashFor(doc.url, opts.db) === hash, meta: doc.meta };171 },172173 async extract(raw) {174 const p = (raw.metadata.parsed as ReturnType<RetailAdapter["parse"]>) ?? a.parse(clean({ markdown: raw.markdown, metadata: raw.metadata, statusCode: raw.statusCode, links: [] }));175 if (raw.statusCode != null && raw.statusCode >= 400) return [];176 if (p.price == null) return [];177 const promo = detectPromotion(raw.markdown, p.price);178 return [{179 externalId: String(raw.metadata["product:retailer_item_id"] ?? raw.metadata.sku ?? raw.url), sourceUrl: raw.url, retrievedAt: raw.fetchedAt, effectiveDate: raw.fetchedAt.slice(0, 10), title: p.title, description: null, unit: p.unitText,180 price: p.price, regularPrice: p.regularPrice ?? promo.regular, salePrice: p.salePrice ?? promo.sale, currency: p.currency, location: "QC", category: null,181 payload: { itemCode: raw.meta?.itemCode ?? null, packQty: raw.meta?.packQty ?? null, packUnit: raw.meta?.packUnit ?? null, statusCode: raw.statusCode, metaKeys: Object.keys(raw.metadata).slice(0, 30) },182 }];183 },184185 async normalize(obs, raw, opts) {186 const d = opts.db ?? getCostDb();187 const out: CanonicalObservation[] = [];188 for (const o of obs) {189 const code = o.payload.itemCode as string | null;190 if (!code) continue;191 const item = d.prepare("SELECT unit, retail_pack_qty, retail_pack_unit FROM cost_items WHERE canonical_code=?").get(code) as { unit: CanonicalUnit; retail_pack_qty: number | null; retail_pack_unit: string | null } | undefined;192 if (!item) continue;193 const regular = o.regularPrice != null && o.regularPrice > 0 ? o.regularPrice : null;194 const price = regular ?? o.price;195 if (price == null || price <= 0) continue;196 // contenu d'emballage lu sur le TITRE seulement (le corps de page mêle des produits associés)197 const detected = detectPackQty(o.title, item.unit);198 const packQty = (o.payload.packQty as number | null) ?? detected ?? item.retail_pack_qty;199 const sourceUnit = unitTextToSourceUnit(o.unit, item.unit, packQty, (o.payload.packUnit as string | null) ?? item.retail_pack_unit);200 try {201 const conv = convertPrice(price, sourceUnit, item.unit, packQty);202 out.push({ kind: "price", itemCode: code, sourceUrl: o.sourceUrl, observationDate: o.retrievedAt.slice(0, 10), sourceUnit: conv.conversion.sourceUnit, canonicalUnit: item.unit, conversionFactor: conv.conversion.factor, totalCost: Math.round(conv.price * 10000) / 10000, isRegularPrice: regular != null || o.salePrice == null, rawPrice: price, rawRegularPrice: o.regularPrice, rawSalePrice: o.salePrice, currency: o.currency, locationCode: "QC", title: o.title, raw: o });203 } catch (e) {204 // unité inconnue → observation canonique invalide (rejet explicite en validation)205 out.push({ kind: "price", itemCode: code, sourceUrl: o.sourceUrl, observationDate: o.retrievedAt.slice(0, 10), sourceUnit: isKnownUnit(sourceUnit) ? sourceUnit : `?${sourceUnit}`, canonicalUnit: item.unit, conversionFactor: 0, totalCost: 0, isRegularPrice: true, rawPrice: price, rawRegularPrice: o.regularPrice, rawSalePrice: o.salePrice, currency: o.currency, locationCode: "QC", title: `${o.title} — ${(e as Error).message}`, raw: o });206 }207 }208 return out;209 },210211 validate(obs, opts): ValidationResult {212 return validateObservations(obs, { recentPrices: (c) => recentPricesFor(c, 120, opts.db), referencePrice: (c) => referencePriceFor(c, opts.db) });213 },214 };215}216217/* ------------------------------------------------------------ adaptateurs */218219function metaPrice(md: Record<string, unknown>): number | null {220 for (const k of ["product:price:amount", "og:price:amount", "product:price", "price", "priceAmount"]) {221 const v = md[k];222 const p = parseMoney(Array.isArray(v) ? v[0] : v);223 if (p != null && p > 0) return p;224 }225 return null;226}227228function metaCurrency(md: Record<string, unknown>): string {229 const v = md["product:price:currency"] ?? md["og:price:currency"] ?? md.priceCurrency;230 const s = Array.isArray(v) ? v[0] : v;231 return typeof s === "string" && s ? s.toUpperCase() : "CAD";232}233234function metaTitle(md: Record<string, unknown>, fallback: string): string {235 const t = md.ogTitle ?? md["og:title"] ?? md.title;236 return typeof t === "string" && t ? t.replace(/\u00a0/g, " ").replace(/\s*[|\-–]\s*(Canac|BMR|Patrick Morin|RONA|Home Depot Canada)\s*$/i, "").trim() : fallback;237}238239export const canacAdapter: RetailAdapter = {240 key: "canac", name: "Canac", domain: "canac.ca", searchPrefix: "canac.ca", active: true, config: { rateLimitMs: 1200 },241 isProductUrl: (u) => /canac\.ca\/canac\/fr\/2\/p\//.test(u),242 parse(r0) {243 const r = clean(r0);244 const price = metaPrice(r.metadata) ?? parseMoney(r.markdown.match(/(\d[\d\s]*(?:[.,]\d{2}))\s*\$\s*\/\s*(?:Chaque|Feuille|Paquet|Boîte|Sac|Rouleau|Longueur|pi²|pi)/i)?.[1]);245 const unit = r.markdown.match(/\$\s*\/\s*([A-Za-zÀ-ÿ²]+(?:\s+de\s+\d+)?)/)?.[1] ?? null;246 const reg = r.markdown.match(/Prix\s+r[ée]gulier\s*:?\s*(\d[\d\s]*(?:[.,]\d{2}))\s*\$/i);247 return { title: metaTitle(r.metadata, r.markdown.match(/^#\s+(.+)$/m)?.[1] ?? ""), price, regularPrice: reg ? parseMoney(reg[1]) : null, salePrice: reg ? price : null, unitText: unit, currency: metaCurrency(r.metadata) };248 },249};250251export const bmrAdapter: RetailAdapter = {252 key: "bmr", name: "BMR", domain: "bmr.ca", searchPrefix: "bmr.ca", active: true, config: { rateLimitMs: 1500 },253 isProductUrl: (u) => /bmr\.ca\/(fr|en)\/[^/?#]+\.html$/.test(u.split("?")[0]) && !/\/search\//.test(u),254 parse(r0) {255 const r = clean(r0);256 const price = metaPrice(r.metadata) ?? parseMoney(r.markdown.match(/(\d[\d\s]*(?:[.,]\d{2}))\s*\$/)?.[1]);257 const reg = r.markdown.match(/(?:Prix\s+r[ée]g(?:ulier)?\.?|Rég\.)\s*:?\s*(\d[\d\s]*(?:[.,]\d{2}))\s*\$/i);258 const unit = r.markdown.match(/\$\s*\/\s*([A-Za-zÀ-ÿ²]+)/)?.[1] ?? null;259 return { title: metaTitle(r.metadata, ""), price, regularPrice: reg ? parseMoney(reg[1]) : null, salePrice: reg ? price : null, unitText: unit, currency: metaCurrency(r.metadata) };260 },261};262263export const patrickMorinAdapter: RetailAdapter = {264 key: "patrickmorin", name: "Patrick Morin", domain: "patrickmorin.com", searchPrefix: "patrickmorin.com", active: true, config: { rateLimitMs: 1500 },265 isProductUrl: (u) => /patrickmorin\.com\/(fr|en)\/[^/?#]+$/.test(u.split("?")[0]) && !/\/(fr|en)\/(recherche|search|catalogsearch)/.test(u),266 parse(r0) {267 const r = clean(r0);268 const price = metaPrice(r.metadata) ?? parseMoney(r.markdown.match(/(\d[\d\s]*(?:[.,]\d{2}))\s*\$/)?.[1]);269 const reg = r.markdown.match(/(?:Prix\s+r[ée]g(?:ulier)?\.?|Rég\.)\s*:?\s*(\d[\d\s]*(?:[.,]\d{2}))\s*\$/i);270 const unit = r.markdown.match(/\$\s*\/\s*([A-Za-zÀ-ÿ²]+)/)?.[1] ?? null;271 return { title: metaTitle(r.metadata, ""), price, regularPrice: reg ? parseMoney(reg[1]) : null, salePrice: reg ? price : null, unitText: unit, currency: metaCurrency(r.metadata) };272 },273};274275export const ronaAdapter: RetailAdapter = {276 key: "rona", name: "RONA", domain: "rona.ca", searchPrefix: "rona.ca", active: false, config: { rateLimitMs: 2500, concurrency: 1, retries: 1 },277 isProductUrl: (u) => /rona\.ca\/(fr|en)\/(produit|product)\//.test(u),278 parse(r0) {279 const r = clean(r0);280 const price = metaPrice(r.metadata) ?? parseMoney(r.markdown.match(/(\d[\d\s]*(?:[.,]\d{2}))\s*\$/)?.[1]);281 const reg = r.markdown.match(/(?:Prix\s+r[ée]g(?:ulier)?\.?|Was)\s*:?\s*\$?\s*(\d[\d\s]*(?:[.,]\d{2}))/i);282 return { title: metaTitle(r.metadata, ""), price, regularPrice: reg ? parseMoney(reg[1]) : null, salePrice: reg ? price : null, unitText: null, currency: metaCurrency(r.metadata) };283 },284};285286export const homeDepotAdapter: RetailAdapter = {287 key: "homedepot", name: "Home Depot Canada", domain: "homedepot.ca", searchPrefix: "homedepot.ca", active: false, config: { rateLimitMs: 2500, concurrency: 1, retries: 1 },288 isProductUrl: (u) => /homedepot\.ca\/(produit|product)\//.test(u),289 parse(r0) {290 const r = clean(r0);291 // prix dépendant du magasin sélectionné (défaut hors Québec) : conservé pour l'avenir, connecteur inactif292 const price = metaPrice(r.metadata) ?? parseMoney(r.markdown.match(/(\d[\d\s]*(?:[.,]\d{2}))\s*\$/)?.[1]);293 return { title: metaTitle(r.metadata, ""), price, regularPrice: null, salePrice: null, unitText: null, currency: metaCurrency(r.metadata) };294 },295};296297export type { RawDocument, RawObservation, RunOptions };298