spb/vrai-prix

1// Auteur : Simon-Pierre Boucher — contact@spboucher.ai2/**3 * Pipeline d'images d'annonce (§91-93, 160-163, 186-190) — SERVEUR :4 * téléchargement borné, type par octets magiques, dimensions par lecture des5 * en-têtes (sans dépendance), hash sha256, dédoublonnage, retrait des6 * miniatures, présélection ≤ MAX, manifeste transmis au modèle, cache disque7 * temporaire purgé selon LISTING_IMAGE_RETENTION_DAYS. Une photo qui échoue8 * n'arrête pas l'analyse. Les octets ne sont jamais redistribués.9 */10import { createHash } from "crypto";11import fs from "fs";12import path from "path";1314export const MAX_IMAGE_BYTES = 4.5 * 1024 * 1024;15export const MIN_IMAGE_PX = 400;16export const FETCH_TIMEOUT_MS = 20_000;1718export type ImageMediaType = "image/jpeg" | "image/png" | "image/webp" | "image/gif";1920export interface PreparedImage {21  id: string; // photo_01…22  position: number; // position d'origine dans l'annonce (1-based)23  sourceUrl: string;24  hash: string;25  width: number | null;26  height: number | null;27  bytes: number;28  mediaType: ImageMediaType;29  roomHint: string;30  qualityScore: number;31  data: Buffer;32}3334export interface ImagePipelineResult {35  selected: PreparedImage[];36  all: Omit<PreparedImage, "data">[];37  failed: { url: string; error: string }[];38  duplicatesRemoved: number;39  thumbnailsRemoved: number;40}4142export function cacheDir(uid: string): string {43  return path.join(process.cwd(), "tmp", "listing-images", uid.replace(/[^a-zA-Z0-9_:-]/g, "_"));44}4546/* ------------------------------------------------------------ détection */4748export function sniffMediaType(buf: Buffer): ImageMediaType | null {49  if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return "image/jpeg";50  if (buf.length >= 8 && buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";51  if (buf.length >= 12 && buf.subarray(0, 4).toString("ascii") === "RIFF" && buf.subarray(8, 12).toString("ascii") === "WEBP") return "image/webp";52  if (buf.length >= 6 && buf.subarray(0, 3).toString("ascii") === "GIF") return "image/gif";53  return null;54}5556/** Dimensions lues dans les en-têtes JPEG (SOF), PNG (IHDR), WebP (VP8/VP8L/VP8X), GIF. */57export function readDimensions(buf: Buffer, type: ImageMediaType): { width: number; height: number } | null {58  try {59    if (type === "image/png" && buf.length >= 24) return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };60    if (type === "image/gif" && buf.length >= 10) return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) };61    if (type === "image/webp" && buf.length >= 30) {62      const chunk = buf.subarray(12, 16).toString("ascii");63      if (chunk === "VP8 ") return { width: buf.readUInt16LE(26) & 0x3fff, height: buf.readUInt16LE(28) & 0x3fff };64      if (chunk === "VP8L") { const b0 = buf[21], b1 = buf[22], b2 = buf[23], b3 = buf[24]; return { width: 1 + (((b1 & 0x3f) << 8) | b0), height: 1 + (((b3 & 0xf) << 10) | (b2 << 2) | ((b1 & 0xc0) >> 6)) }; }65      if (chunk === "VP8X") return { width: 1 + buf.readUIntLE(24, 3), height: 1 + buf.readUIntLE(27, 3) };66    }67    if (type === "image/jpeg") {68      let i = 2;69      while (i + 9 < buf.length) {70        if (buf[i] !== 0xff) { i++; continue; }71        const marker = buf[i + 1];72        if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { i += 2; continue; }73        const len = buf.readUInt16BE(i + 2);74        if ((marker >= 0xc0 && marker <= 0xcf) && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) return { height: buf.readUInt16BE(i + 5), width: buf.readUInt16BE(i + 7) };75        i += 2 + len;76      }77    }78  } catch { /* en-tête tronqué */ }79  return null;80}8182export const sha256 = (b: Buffer | string): string => createHash("sha256").update(b).digest("hex");8384/* --------------------------------------------------------------- fetch */8586/** En-têtes ASCII seulement : un octet accentué dans le User-Agent fait répondre87 *  HTTP 400 aux CDN stricts (Facebook Marketplace `*.fbcdn.net`, 2026-09-08). */88const UA_BOT = "Vrai-Prix/1.0 (building analysis; +https://www.vrai-prix.com)";89const UA_BROWSER = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36";90const ACCEPT_IMG = "image/avif,image/webp,image/apng,image/*,*/*;q=0.8";9192/** Referer plausible pour les CDN qui l'exigent (photos servies depuis un domaine distinct du site). */93export function refererFor(url: string): string {94  try {95    const h = new URL(url).hostname;96    if (/\.fbcdn\.net$|facebook\.com$/.test(h)) return "https://www.facebook.com/";97    if (/ebayimg\.com$|kijiji/.test(h)) return "https://www.kijiji.ca/";98    if (/centris\.ca$/.test(h)) return "https://www.centris.ca/";99    if (/duproprio\.com$/.test(h)) return "https://duproprio.com/";100    const parts = h.split(".");101    return `https://www.${parts.slice(-2).join(".")}/`;102  } catch {103    return "https://www.vrai-prix.com/";104  }105}106107export async function fetchImage(url: string, timeoutMs = FETCH_TIMEOUT_MS): Promise<{ data: Buffer; mediaType: ImageMediaType }> {108  const ctrl = new AbortController();109  const t = setTimeout(() => ctrl.abort(), timeoutMs);110  try {111    // 1er essai : identité déclarée (bot) ; 2e essai (4xx) : en-têtes de navigateur + Referer.112    let res = await fetch(url, { signal: ctrl.signal, headers: { "User-Agent": UA_BOT, Accept: ACCEPT_IMG }, redirect: "follow" });113    if (!res.ok && res.status >= 400 && res.status < 500) {114      res = await fetch(url, {115        signal: ctrl.signal,116        redirect: "follow",117        headers: { "User-Agent": UA_BROWSER, Accept: ACCEPT_IMG, "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8", Referer: refererFor(url), "Sec-Fetch-Dest": "image", "Sec-Fetch-Mode": "no-cors", "Sec-Fetch-Site": "cross-site" },118      });119    }120    if (!res.ok) throw new Error(`HTTP ${res.status} (${new URL(url).hostname})`);121    const len = Number(res.headers.get("content-length") ?? 0);122    if (len > MAX_IMAGE_BYTES) throw new Error(`trop volumineuse (${len} octets)`);123    const ab = await res.arrayBuffer();124    if (ab.byteLength > MAX_IMAGE_BYTES) throw new Error(`trop volumineuse (${ab.byteLength} octets)`);125    const data = Buffer.from(ab);126    const mediaType = sniffMediaType(data);127    if (!mediaType) throw new Error("format d'image non reconnu");128    return { data, mediaType };129  } finally {130    clearTimeout(t);131  }132}133134/* ----------------------------------------------------------- heuristiques */135136/** Indice de pièce léger à partir de l'URL et de la position (les photos 1-3 sont presque toujours extérieures). */137export function roomHint(url: string, position: number, total: number): string {138  const u = url.toLowerCase();139  const has = (...k: string[]) => k.some((x) => u.includes(x));140  if (has("facade", "exterieur", "exterior", "front", "ext_")) return "exterior";141  if (has("cuisine", "kitchen")) return "kitchen";142  if (has("salle-de-bain", "bathroom", "sdb", "bain")) return "bathroom";143  if (has("chambre", "bedroom")) return "bedroom";144  if (has("salon", "living", "sejour")) return "living";145  if (has("sous-sol", "basement")) return "basement";146  if (has("garage")) return "garage";147  if (has("cour", "backyard", "yard", "terrasse", "deck", "piscine", "pool")) return "exterior_rear";148  if (has("plan", "floorplan")) return "floorplan";149  if (position <= 2) return "exterior";150  if (position === total && total > 8) return "exterior_rear";151  return "unknown";152}153154/** Clé de regroupement : même photo servie en plusieurs tailles (-sm/-md/-lg, w=, /thumb/). */155export function sizeAgnosticKey(url: string): string {156  return url157    .replace(/[?#].*$/, "")158    .replace(/-(xs|sm|md|lg|xl|xxl|thumb|thumbnail|small|medium|large|original)(?=\.[a-z]{3,4}$)/i, "")159    .replace(/\/(thumb|thumbs|thumbnail|small|medium|large)\//i, "/")160    .replace(/_(\d{2,4})x(\d{2,4})(?=\.[a-z]{3,4}$)/i, "")161    .toLowerCase();162}163164/** Présélection : garde les extérieurs et une couverture régulière du reste. */165export function selectImages<T extends { roomHint: string; position: number; qualityScore: number }>(imgs: T[], max: number): T[] {166  if (imgs.length <= max) return imgs;167  const ext = imgs.filter((i) => i.roomHint.startsWith("exterior") || i.roomHint === "floorplan").slice(0, Math.min(6, Math.floor(max / 4)));168  const rest = imgs.filter((i) => !ext.includes(i));169  const slots = max - ext.length;170  const picked: T[] = [];171  for (let k = 0; k < slots; k++) picked.push(rest[Math.floor((k * rest.length) / slots)]);172  return [...ext, ...picked].sort((a, b) => a.position - b.position);173}174175/* ------------------------------------------------------------- pipeline */176177export async function prepareListingImages(uid: string, urls: string[], max: number, concurrency = 6): Promise<ImagePipelineResult> {178  const dir = cacheDir(uid);179  fs.mkdirSync(dir, { recursive: true });180  const failed: { url: string; error: string }[] = [];181  const fetched: (PreparedImage & { key: string })[] = [];182  let idx = 0;183  const worker = async () => {184    while (idx < urls.length) {185      const pos = idx++;186      const url = urls[pos];187      try {188        const { data, mediaType } = await fetchImage(url);189        const hash = sha256(data);190        const dims = readDimensions(data, mediaType);191        const px = dims ? Math.min(dims.width, dims.height) : null;192        const ext = mediaType.split("/")[1];193        const cached = path.join(dir, `${hash}.${ext}`);194        if (!fs.existsSync(cached)) fs.writeFileSync(cached, data);195        fetched.push({ id: "", position: pos + 1, sourceUrl: url, hash, width: dims?.width ?? null, height: dims?.height ?? null, bytes: data.length, mediaType, roomHint: roomHint(url, pos + 1, urls.length), qualityScore: px ? Math.min(1, px / 1200) : 0.5, data, key: sizeAgnosticKey(url) });196      } catch (e) {197        failed.push({ url, error: (e as Error).message });198      }199    }200  };201  await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(1, urls.length)) }, worker));202  fetched.sort((a, b) => a.position - b.position);203  // dédoublonnage : hash identique, puis même clé d'URL (on garde la plus grande)204  const byHash = new Map<string, PreparedImage & { key: string }>();205  let duplicates = 0;206  for (const f of fetched) {207    if (byHash.has(f.hash)) { duplicates++; continue; }208    byHash.set(f.hash, f);209  }210  const byKey = new Map<string, PreparedImage & { key: string }>();211  for (const f of byHash.values()) {212    const cur = byKey.get(f.key);213    if (!cur) byKey.set(f.key, f);214    else { duplicates++; if ((f.width ?? 0) * (f.height ?? 0) > (cur.width ?? 0) * (cur.height ?? 0)) byKey.set(f.key, f); }215  }216  let unique = [...byKey.values()].sort((a, b) => a.position - b.position);217  const before = unique.length;218  unique = unique.filter((i) => i.width == null || Math.min(i.width, i.height ?? i.width) >= MIN_IMAGE_PX);219  const thumbs = before - unique.length;220  const selected = selectImages(unique, max).map((img, i) => ({ ...img, id: `photo_${String(i + 1).padStart(2, "0")}` }));221  const all = unique.map((img) => { const sel = selected.find((s) => s.hash === img.hash); const { data: _d, key: _k, ...rest } = img; void _d; void _k; return { ...rest, id: sel?.id ?? "" }; });222  return { selected: selected.map(({ key: _k, ...r }) => { void _k; return r; }), all, failed, duplicatesRemoved: duplicates, thumbnailsRemoved: thumbs };223}224225/** Purge du cache disque au-delà de la rétention (défaut 30 jours). */226export function purgeImageCache(retentionDays = Number(process.env.LISTING_IMAGE_RETENTION_DAYS ?? 30)): number {227  const root = path.join(process.cwd(), "tmp", "listing-images");228  if (!fs.existsSync(root)) return 0;229  const cutoff = Date.now() - retentionDays * 86400000;230  let n = 0;231  for (const d of fs.readdirSync(root)) {232    const p = path.join(root, d);233    try {234      const st = fs.statSync(p);235      if (st.isDirectory() && st.mtimeMs < cutoff) { fs.rmSync(p, { recursive: true, force: true }); n++; }236    } catch { /* ignore */ }237  }238  return n;239}240