Python 52.4%
TeX 47.6%
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3"""4Analyse de la photo de façade (1/annonce) via Claude Haiku 4.5.5Attributs visuels OBJECTIFS. Parallèle, résumable (facade_specs.jsonl), retry.6"""7import json, os, time, threading, urllib.request, urllib.error8from concurrent.futures import ThreadPoolExecutor910SRC = "data/processed/maisons_avec_prix.json"11OUT = "data/processed/facade_specs.jsonl"12WORKERS = int(os.environ.get("WORKERS", "16"))13NSHARDS = int(os.environ.get("NSHARDS", "1"))14SHARD = int(os.environ.get("SHARD", "0"))15MODEL = "claude-haiku-4-5"1617for line in open(".env", encoding="utf-8"):18 if line.startswith("ANTHROPIC_API_KEY="):19 KEY = line.split("=", 1)[1].strip()2021SCHEMA = {22 "type": "object", "additionalProperties": False,23 "properties": {24 "is_exterior": {"type": "boolean", "description": "la photo montre-t-elle l'extérieur/façade"},25 "building_type_visual": {"type": "string", "enum": ["maison_detachee","jumele","rangee","condo_immeuble","chalet","terrain","autre"]},26 "condition_score": {"type": "integer", "description": "état visuel 1=délabré à 10=neuf/luxueux"},27 "curb_appeal": {"type": "integer", "description": "attrait extérieur 1-10"},28 "renovation_level": {"type": "string", "enum": ["a_renover","date","standard","renove","haut_de_gamme","inconnu"]},29 "exterior_materials": {"type": "array", "items": {"type": "string", "enum": ["brique","pierre","vinyle","bois","aluminium","stucco","crepi","beton","autre"]}},30 "landscaping": {"type": "string", "enum": ["aucun","minimal","entretenu","amenage_soigne"]},31 "has_garage_visible": {"type": "boolean"},32 "storeys_visual": {"type": "string", "enum": ["1","1.5","2","3+","inconnu"]},33 "setting": {"type": "string", "enum": ["urbain_dense","banlieue","rural","bord_eau","boise","inconnu"]},34 "photo_quality": {"type": "integer", "description": "qualité de la photo 1-10"},35 "curb_luxury": {"type": "boolean", "description": "signes de luxe (grande taille, matériaux nobles, prestige)"}36 },37 "required": ["is_exterior","building_type_visual","condition_score","curb_appeal",38 "renovation_level","exterior_materials","landscaping","has_garage_visible",39 "storeys_visual","setting","photo_quality","curb_luxury"]40}41SYS = ("Tu es un évaluateur immobilier objectif. Analyse la photo (censée être la "42 "façade d'une propriété) et renvoie des attributs FACTUELS et reproductibles "43 "au format JSON. Fonde les scores sur des critères concrets (état des "44 "surfaces, matériaux, aménagement), pas sur un goût personnel. Si la photo "45 "n'est pas un extérieur, mets is_exterior=false et remplis au mieux.")4647ROOM_KW = ("kitchen","cuisine","bathroom","salle de bain","bedroom","chambre",48 "living","salon","basement","sous-sol","dining","salle à manger",49 "garage","office","bureau","laundry","closet","stair")5051def pick_facade(d):52 imgs = d.get("images") or []53 if not imgs:54 return None55 if d.get("source") == "realtor":56 return imgs[0].get("url") # 1re photo = extérieur57 # duproprio : caption = adresse (sans mot-clé de pièce) = photo extérieure58 for im in imgs:59 cap = (im.get("caption") or "").lower()60 if cap and not any(k in cap for k in ROOM_KW):61 return im.get("url")62 return imgs[0].get("url")6364def call(url, tries=5):65 body = {"model": MODEL, "max_tokens": 400, "system": SYS,66 "messages": [{"role": "user", "content": [67 {"type": "image", "source": {"type": "url", "url": url}},68 {"type": "text", "text": "Analyse cette photo de propriété."}]}],69 "output_config": {"format": {"type": "json_schema", "schema": SCHEMA}}}70 data = json.dumps(body).encode()71 for t in range(tries):72 try:73 req = urllib.request.Request(74 "https://api.anthropic.com/v1/messages", data=data,75 headers={"x-api-key": KEY, "anthropic-version": "2023-06-01",76 "content-type": "application/json"})77 with urllib.request.urlopen(req, timeout=120) as r:78 out = json.load(r)79 txt = next(b["text"] for b in out["content"] if b["type"] == "text")80 return json.loads(txt)81 except urllib.error.HTTPError as e:82 # 400 inclus : fetch d'URL image transitoire côté Anthropic sous charge83 if e.code in (400, 429, 500, 502, 503, 504, 529) and t < tries - 1:84 time.sleep(2 ** t + 1); continue85 raise86 except (urllib.error.URLError, TimeoutError, ValueError):87 if t < tries - 1:88 time.sleep(2 ** t + 1); continue89 raise90 return None9192data = json.load(open(SRC, encoding="utf-8"))93done = set()94if os.path.exists(OUT):95 for line in open(OUT, encoding="utf-8"):96 try: done.add(json.loads(line)["uid"])97 except Exception: pass9899todo = []100for d in data:101 uid = f"{d.get('source')}:{d.get('listingId')}"102 if uid in done:103 continue104 url = pick_facade(d)105 if url:106 todo.append((uid, url))107108if NSHARDS > 1:109 todo = [t for i, t in enumerate(todo) if i % NSHARDS == SHARD]110111print(f"[shard {SHARD}/{NSHARDS}, workers={WORKERS}] Total={len(data)} "112 f"déjà fait={len(done)} à traiter (ce shard)={len(todo)}", flush=True)113114lock = threading.Lock()115c = {"n": 0, "err": 0}116fh = open(OUT, "a", encoding="utf-8")117118def work(item):119 uid, url = item120 try:121 rec = {"uid": uid, "facade": call(url)}122 except Exception as e:123 rec = {"uid": uid, "error": str(e)[:200]}124 with lock:125 fh.write(json.dumps(rec, ensure_ascii=False) + "\n"); fh.flush()126 c["n"] += 1127 if "error" in rec: c["err"] += 1128 if c["n"] % 250 == 0:129 print(f" {c['n']}/{len(todo)} (err={c['err']})", flush=True)130131t0 = time.time()132with ThreadPoolExecutor(max_workers=WORKERS) as ex:133 list(ex.map(work, todo))134fh.close()135print(f"Terminé : {c['n']} traités, {c['err']} erreurs, {time.time()-t0:.0f}s", flush=True)136