// Auteur : Simon-Pierre Boucher — contact@spboucher.ai /** * Pipeline d'images d'annonce (§91-93, 160-163, 186-190) — SERVEUR : * téléchargement borné, type par octets magiques, dimensions par lecture des * en-têtes (sans dépendance), hash sha256, dédoublonnage, retrait des * miniatures, présélection ≤ MAX, manifeste transmis au modèle, cache disque * temporaire purgé selon LISTING_IMAGE_RETENTION_DAYS. Une photo qui échoue * n'arrête pas l'analyse. Les octets ne sont jamais redistribués. */ import { createHash } from "crypto"; import fs from "fs"; import path from "path"; export const MAX_IMAGE_BYTES = 4.5 * 1024 * 1024; export const MIN_IMAGE_PX = 400; export const FETCH_TIMEOUT_MS = 20_000; export type ImageMediaType = "image/jpeg" | "image/png" | "image/webp" | "image/gif"; export interface PreparedImage { id: string; // photo_01… position: number; // position d'origine dans l'annonce (1-based) sourceUrl: string; hash: string; width: number | null; height: number | null; bytes: number; mediaType: ImageMediaType; roomHint: string; qualityScore: number; data: Buffer; } export interface ImagePipelineResult { selected: PreparedImage[]; all: Omit[]; failed: { url: string; error: string }[]; duplicatesRemoved: number; thumbnailsRemoved: number; } export function cacheDir(uid: string): string { return path.join(process.cwd(), "tmp", "listing-images", uid.replace(/[^a-zA-Z0-9_:-]/g, "_")); } /* ------------------------------------------------------------ détection */ export function sniffMediaType(buf: Buffer): ImageMediaType | null { if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return "image/jpeg"; if (buf.length >= 8 && buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png"; if (buf.length >= 12 && buf.subarray(0, 4).toString("ascii") === "RIFF" && buf.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp"; if (buf.length >= 6 && buf.subarray(0, 3).toString("ascii") === "GIF") return "image/gif"; return null; } /** Dimensions lues dans les en-têtes JPEG (SOF), PNG (IHDR), WebP (VP8/VP8L/VP8X), GIF. */ export function readDimensions(buf: Buffer, type: ImageMediaType): { width: number; height: number } | null { try { if (type === "image/png" && buf.length >= 24) return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; if (type === "image/gif" && buf.length >= 10) return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) }; if (type === "image/webp" && buf.length >= 30) { const chunk = buf.subarray(12, 16).toString("ascii"); if (chunk === "VP8 ") return { width: buf.readUInt16LE(26) & 0x3fff, height: buf.readUInt16LE(28) & 0x3fff }; if (chunk === "VP8L") { const b0 = buf[21], b1 = buf[22], b2 = buf[23], b3 = buf[24]; return { width: 1 + (((b1 & 0x3f) << 8) | b0), height: 1 + (((b3 & 0xf) << 10) | (b2 << 2) | ((b1 & 0xc0) >> 6)) }; } if (chunk === "VP8X") return { width: 1 + buf.readUIntLE(24, 3), height: 1 + buf.readUIntLE(27, 3) }; } if (type === "image/jpeg") { let i = 2; while (i + 9 < buf.length) { if (buf[i] !== 0xff) { i++; continue; } const marker = buf[i + 1]; if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; } const len = buf.readUInt16BE(i + 2); if ((marker >= 0xc0 && marker <= 0xcf) && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) return { height: buf.readUInt16BE(i + 5), width: buf.readUInt16BE(i + 7) }; i += 2 + len; } } } catch { /* en-tête tronqué */ } return null; } export const sha256 = (b: Buffer | string): string => createHash("sha256").update(b).digest("hex"); /* --------------------------------------------------------------- fetch */ /** En-têtes ASCII seulement : un octet accentué dans le User-Agent fait répondre * HTTP 400 aux CDN stricts (Facebook Marketplace `*.fbcdn.net`, 2026-09-08). */ const UA_BOT = "Vrai-Prix/1.0 (building analysis; +https://www.vrai-prix.com)"; const UA_BROWSER = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"; const ACCEPT_IMG = "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"; /** Referer plausible pour les CDN qui l'exigent (photos servies depuis un domaine distinct du site). */ export function refererFor(url: string): string { try { const h = new URL(url).hostname; if (/\.fbcdn\.net$|facebook\.com$/.test(h)) return "https://www.facebook.com/"; if (/ebayimg\.com$|kijiji/.test(h)) return "https://www.kijiji.ca/"; if (/centris\.ca$/.test(h)) return "https://www.centris.ca/"; if (/duproprio\.com$/.test(h)) return "https://duproprio.com/"; const parts = h.split("."); return `https://www.${parts.slice(-2).join(".")}/`; } catch { return "https://www.vrai-prix.com/"; } } export async function fetchImage(url: string, timeoutMs = FETCH_TIMEOUT_MS): Promise<{ data: Buffer; mediaType: ImageMediaType }> { const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { // 1er essai : identité déclarée (bot) ; 2e essai (4xx) : en-têtes de navigateur + Referer. let res = await fetch(url, { signal: ctrl.signal, headers: { "User-Agent": UA_BOT, Accept: ACCEPT_IMG }, redirect: "follow" }); if (!res.ok && res.status >= 400 && res.status < 500) { res = await fetch(url, { signal: ctrl.signal, redirect: "follow", headers: { "User-Agent": UA_BROWSER, Accept: ACCEPT_IMG, "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8", Referer: refererFor(url), "Sec-Fetch-Dest": "image", "Sec-Fetch-Mode": "no-cors", "Sec-Fetch-Site": "cross-site" }, }); } if (!res.ok) throw new Error(`HTTP ${res.status} (${new URL(url).hostname})`); const len = Number(res.headers.get("content-length") ?? 0); if (len > MAX_IMAGE_BYTES) throw new Error(`trop volumineuse (${len} octets)`); const ab = await res.arrayBuffer(); if (ab.byteLength > MAX_IMAGE_BYTES) throw new Error(`trop volumineuse (${ab.byteLength} octets)`); const data = Buffer.from(ab); const mediaType = sniffMediaType(data); if (!mediaType) throw new Error("format d'image non reconnu"); return { data, mediaType }; } finally { clearTimeout(t); } } /* ----------------------------------------------------------- heuristiques */ /** Indice de pièce léger à partir de l'URL et de la position (les photos 1-3 sont presque toujours extérieures). */ export function roomHint(url: string, position: number, total: number): string { const u = url.toLowerCase(); const has = (...k: string[]) => k.some((x) => u.includes(x)); if (has("facade", "exterieur", "exterior", "front", "ext_")) return "exterior"; if (has("cuisine", "kitchen")) return "kitchen"; if (has("salle-de-bain", "bathroom", "sdb", "bain")) return "bathroom"; if (has("chambre", "bedroom")) return "bedroom"; if (has("salon", "living", "sejour")) return "living"; if (has("sous-sol", "basement")) return "basement"; if (has("garage")) return "garage"; if (has("cour", "backyard", "yard", "terrasse", "deck", "piscine", "pool")) return "exterior_rear"; if (has("plan", "floorplan")) return "floorplan"; if (position <= 2) return "exterior"; if (position === total && total > 8) return "exterior_rear"; return "unknown"; } /** Clé de regroupement : même photo servie en plusieurs tailles (-sm/-md/-lg, w=, /thumb/). */ export function sizeAgnosticKey(url: string): string { return url .replace(/[?#].*$/, "") .replace(/-(xs|sm|md|lg|xl|xxl|thumb|thumbnail|small|medium|large|original)(?=\.[a-z]{3,4}$)/i, "") .replace(/\/(thumb|thumbs|thumbnail|small|medium|large)\//i, "/") .replace(/_(\d{2,4})x(\d{2,4})(?=\.[a-z]{3,4}$)/i, "") .toLowerCase(); } /** Présélection : garde les extérieurs et une couverture régulière du reste. */ export function selectImages(imgs: T[], max: number): T[] { if (imgs.length <= max) return imgs; const ext = imgs.filter((i) => i.roomHint.startsWith("exterior") || i.roomHint === "floorplan").slice(0, Math.min(6, Math.floor(max / 4))); const rest = imgs.filter((i) => !ext.includes(i)); const slots = max - ext.length; const picked: T[] = []; for (let k = 0; k < slots; k++) picked.push(rest[Math.floor((k * rest.length) / slots)]); return [...ext, ...picked].sort((a, b) => a.position - b.position); } /* ------------------------------------------------------------- pipeline */ export async function prepareListingImages(uid: string, urls: string[], max: number, concurrency = 6): Promise { const dir = cacheDir(uid); fs.mkdirSync(dir, { recursive: true }); const failed: { url: string; error: string }[] = []; const fetched: (PreparedImage & { key: string })[] = []; let idx = 0; const worker = async () => { while (idx < urls.length) { const pos = idx++; const url = urls[pos]; try { const { data, mediaType } = await fetchImage(url); const hash = sha256(data); const dims = readDimensions(data, mediaType); const px = dims ? Math.min(dims.width, dims.height) : null; const ext = mediaType.split("/")[1]; const cached = path.join(dir, `${hash}.${ext}`); if (!fs.existsSync(cached)) fs.writeFileSync(cached, data); fetched.push({ id: "", position: pos + 1, sourceUrl: url, hash, width: dims?.width ?? null, height: dims?.height ?? null, bytes: data.length, mediaType, roomHint: roomHint(url, pos + 1, urls.length), qualityScore: px ? Math.min(1, px / 1200) : 0.5, data, key: sizeAgnosticKey(url) }); } catch (e) { failed.push({ url, error: (e as Error).message }); } } }; await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(1, urls.length)) }, worker)); fetched.sort((a, b) => a.position - b.position); // dédoublonnage : hash identique, puis même clé d'URL (on garde la plus grande) const byHash = new Map(); let duplicates = 0; for (const f of fetched) { if (byHash.has(f.hash)) { duplicates++; continue; } byHash.set(f.hash, f); } const byKey = new Map(); for (const f of byHash.values()) { const cur = byKey.get(f.key); if (!cur) byKey.set(f.key, f); else { duplicates++; if ((f.width ?? 0) * (f.height ?? 0) > (cur.width ?? 0) * (cur.height ?? 0)) byKey.set(f.key, f); } } let unique = [...byKey.values()].sort((a, b) => a.position - b.position); const before = unique.length; unique = unique.filter((i) => i.width == null || Math.min(i.width, i.height ?? i.width) >= MIN_IMAGE_PX); const thumbs = before - unique.length; const selected = selectImages(unique, max).map((img, i) => ({ ...img, id: `photo_${String(i + 1).padStart(2, "0")}` })); const all = unique.map((img) => { const sel = selected.find((s) => s.hash === img.hash); const { data: _d, key: _k, ...rest } = img; void _d; void _k; return { ...rest, id: sel?.id ?? "" }; }); return { selected: selected.map(({ key: _k, ...r }) => { void _k; return r; }), all, failed, duplicatesRemoved: duplicates, thumbnailsRemoved: thumbs }; } /** Purge du cache disque au-delà de la rétention (défaut 30 jours). */ export function purgeImageCache(retentionDays = Number(process.env.LISTING_IMAGE_RETENTION_DAYS ?? 30)): number { const root = path.join(process.cwd(), "tmp", "listing-images"); if (!fs.existsSync(root)) return 0; const cutoff = Date.now() - retentionDays * 86400000; let n = 0; for (const d of fs.readdirSync(root)) { const p = path.join(root, d); try { const st = fs.statSync(p); if (st.isDirectory() && st.mtimeMs < cutoff) { fs.rmSync(p, { recursive: true, force: true }); n++; } } catch { /* ignore */ } } return n; }