SPB Git

spb/vrai-prix

Public

Vrai-Prix — l'évaluation du vrai prix des propriétés résidentielles au Québec.

fix(analyse IA): sorties « placeholder » du modèle — tool_choice auto (puis any en relance) au lieu d'un outil forcé qui faisait répondre Sonnet 5 par {"property_analysis":{"placeholder":true}} ; ≥ 3 sections de premier niveau absentes = échec dur → relance avec consigne explicite (3 tentatives max), jamais une analyse « complétée » vide ; test missingTopLevelSections

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed (Sep 8, 2026) parent 4c0ba61 Browse files

3 changed files +49 −8

modified src/lib/cost/ai/ai.test.ts +17 −1
@@ -1,7 +1,7 @@
1 // Auteur : Simon-Pierre Boucher — contact@spboucher.ai 1 // Auteur : Simon-Pierre Boucher — contact@spboucher.ai
2 // Tests hors réseau de l'analyse multimodale : schéma/réparation, empreinte, fusion, mapping, vecteur, images. 2 // Tests hors réseau de l'analyse multimodale : schéma/réparation, empreinte, fusion, mapping, vecteur, images.
3 import { describe, expect, it } from "vitest"; 3 import { describe, expect, it } from "vitest";
4−import { buildModelSchema, isValid, schemaForApi, validateAndRepair, type ModelOutput } from "./schema"; 4+import { buildModelSchema, isValid, MAX_MISSING_SECTIONS, missingTopLevelSections, schemaForApi, validateAndRepair, type ModelOutput } from "./schema";
5 import { sampleOutput } from "./fixture"; 5 import { sampleOutput } from "./fixture";
6 import { inputHash } from "./context"; 6 import { inputHash } from "./context";
7 import { applyOverrides, listingFactsFromDetails, mergeFacts, type ListingFacts, type MamhFacts } from "./merge"; 7 import { applyOverrides, listingFactsFromDetails, mergeFacts, type ListingFacts, type MamhFacts } from "./merge";
@@ -47,6 +47,22 @@ describe("schema", () => {
47 // idempotence 47 // idempotence
48 expect(JSON.stringify(validateAndRepair(value, SCHEMA).value)).toBe(JSON.stringify(value)); 48 expect(JSON.stringify(validateAndRepair(value, SCHEMA).value)).toBe(JSON.stringify(value));
49 }); 49 });
50+
51+ it("un appel d'outil placeholder est détecté comme sortie vide (sections de premier niveau absentes)", () => {
52+ // observé avec Sonnet 5 en tool_choice forcé (2026-09-08) : le réparateur remplirait tout de nulls
53+ const placeholder = { property_analysis: { placeholder: true } };
54+ const { issues } = validateAndRepair(placeholder, SCHEMA);
55+ const missing = missingTopLevelSections(issues);
56+ expect(missing.length).toBeGreaterThan(MAX_MISSING_SECTIONS);
57+ expect(missing).toContain("property");
58+ expect(missing).toContain("exterior");
59+ // une vraie sortie complète : aucune section manquante
60+ expect(missingTopLevelSections(validateAndRepair(sampleOutput(), SCHEMA).issues)).toEqual([]);
61+ // une sortie à laquelle il ne manque qu'une section reste acceptable (réparée)
62+ const partial = { ...(sampleOutput() as unknown as Record<string, unknown>) };
63+ delete partial.renovations;
64+ expect(missingTopLevelSections(validateAndRepair(partial, SCHEMA).issues)).toEqual(["renovations"]);
65+ });
50 }); 66 });
51 67
52 describe("input hash", () => { 68 describe("input hash", () => {
modified src/lib/cost/ai/provider.ts +17 −7
@@ -10,7 +10,7 @@
10 import Anthropic from "@anthropic-ai/sdk"; 10 import Anthropic from "@anthropic-ai/sdk";
11 import fs from "fs"; 11 import fs from "fs";
12 import path from "path"; 12 import path from "path";
13−import { buildModelSchema, PROMPT_VERSION, schemaForApi, validateAndRepair, type ModelOutput, type ValidationIssue } from "./schema"; 13+import { buildModelSchema, MAX_MISSING_SECTIONS, missingTopLevelSections, PROMPT_VERSION, schemaForApi, validateAndRepair, type ModelOutput, type ValidationIssue } from "./schema";
14 import type { ImageMediaType } from "./images"; 14 import type { ImageMediaType } from "./images";
15 15
16 export const DEFAULT_MODEL = "claude-sonnet-5"; 16 export const DEFAULT_MODEL = "claude-sonnet-5";
@@ -78,10 +78,15 @@ export class ClaudeSonnet5Provider implements PropertyVisionModel {
78 let attempts = 0; 78 let attempts = 0;
79 let lastIssues: ValidationIssue[] = []; 79 let lastIssues: ValidationIssue[] = [];
80 let output: ModelOutput | null = null; 80 let output: ModelOutput | null = null;
81− while (attempts < 2 && !output) { 81+ // tool_choice : `auto` au 1er essai — un outil FORCÉ (`type:"tool"`) sans réflexion fait parfois
82+ // répondre Sonnet 5 par un placeholder ({"property_analysis":{"placeholder":true}}, 44 jetons) sur ce
83+ // schéma de ~150 champs (constaté 2026-09-08 sur une annonce Facebook Marketplace ; la même requête en
84+ // `auto` produit 21 Ko d'analyse). `any` en relance : le modèle doit appeler un outil, avec le retour d'erreur.
85+ const MAX_ATTEMPTS = 3;
86+ while (attempts < MAX_ATTEMPTS && !output) {
82 attempts++; 87 attempts++;
83 const stream = this.client.messages.stream({ 88 const stream = this.client.messages.stream({
84− model: this.model, max_tokens: 24000, system, messages, tools, tool_choice: { type: "tool", name: TOOL_NAME }, thinking: { type: "disabled" }, 89+ model: this.model, max_tokens: 24000, system, messages, tools, tool_choice: attempts === 1 ? { type: "auto" } : { type: "any" }, thinking: { type: "disabled" },
85 }); 90 });
86 const msg = await stream.finalMessage(); 91 const msg = await stream.finalMessage();
87 inTok += msg.usage.input_tokens; outTok += msg.usage.output_tokens; 92 inTok += msg.usage.input_tokens; outTok += msg.usage.output_tokens;
@@ -93,13 +98,18 @@ export class ClaudeSonnet5Provider implements PropertyVisionModel {
93 } else { 98 } else {
94 const { value, issues } = validateAndRepair(tu.input, schema); 99 const { value, issues } = validateAndRepair(tu.input, schema);
95 const hard = issues.filter((i) => !i.repaired); 100 const hard = issues.filter((i) => !i.repaired);
96− lastIssues = issues; 101+ const missing = missingTopLevelSections(issues);
97− if (!hard.length) { output = value as ModelOutput; break; } 102+ lastIssues = missing.length > MAX_MISSING_SECTIONS
103+ ? [{ path: "$", message: `sortie vide ou placeholder — ${missing.length} sections absentes (${missing.slice(0, 6).join(", ")}…)`, repaired: false }, ...issues]
104+ : issues;
105+ if (!hard.length && missing.length <= MAX_MISSING_SECTIONS) { output = value as ModelOutput; break; }
98 } 106 }
99− // relance unique avec les erreurs de validation 107+ if (attempts >= MAX_ATTEMPTS) break;
108+ // relance avec les erreurs de validation (la sortie fautive reste dans l'historique du tour)
100 messages.push({ role: "assistant", content: msg.content.map((b) => (b.type === "tool_use" ? { type: "tool_use" as const, id: b.id, name: b.name, input: b.input } : b.type === "text" ? { type: "text" as const, text: b.text } : { type: "text" as const, text: "" })).filter((b) => b.type !== "text" || b.text) }); 109 messages.push({ role: "assistant", content: msg.content.map((b) => (b.type === "tool_use" ? { type: "tool_use" as const, id: b.id, name: b.name, input: b.input } : b.type === "text" ? { type: "text" as const, text: b.text } : { type: "text" as const, text: "" })).filter((b) => b.type !== "text" || b.text) });
101 const toolId = msg.content.find((b): b is Anthropic.ToolUseBlock => b.type === "tool_use")?.id; 110 const toolId = msg.content.find((b): b is Anthropic.ToolUseBlock => b.type === "tool_use")?.id;
102− messages.push({ role: "user", content: toolId ? [{ type: "tool_result", tool_use_id: toolId, is_error: true, content: `Schema validation failed:\n${lastIssues.slice(0, 40).map((i) => `${i.path}: ${i.message}`).join("\n")}\nCall ${TOOL_NAME} again with a fully valid input.` }] : `Schema validation failed. Call ${TOOL_NAME} again with a fully valid input.` }); 111+ const feedback = `Schema validation failed:\n${lastIssues.slice(0, 40).map((i) => `${i.path}: ${i.message}`).join("\n")}\nThis was not an analysis. Look at every photo and the listing context, then call ${TOOL_NAME} again with the COMPLETE input: all top-level sections (property, geometry, construction, exterior, interior, kitchens, bathrooms, mechanical, electrical, plumbing, basement, garage, exterior_improvements, quality, condition, estimated_effective_age, renovations, estimated_quantities, assemblies, uncertainties, privacy) filled, null only where evidence is missing. Never return a placeholder.`;
112+ messages.push({ role: "user", content: toolId ? [{ type: "tool_result", tool_use_id: toolId, is_error: true, content: feedback }] : feedback });
103 } 113 }
104 if (!output) throw new Error(`Sortie du modèle invalide après ${attempts} tentative(s) : ${lastIssues.slice(0, 5).map((i) => `${i.path} ${i.message}`).join(" ; ")}`); 114 if (!output) throw new Error(`Sortie du modèle invalide après ${attempts} tentative(s) : ${lastIssues.slice(0, 5).map((i) => `${i.path} ${i.message}`).join(" ; ")}`);
105 return { 115 return {
modified src/lib/cost/ai/schema.ts +15 −0
@@ -175,6 +175,21 @@ export function validateAndRepair(data: unknown, schema: JS, path = "$"): { valu
175 return { value, issues }; 175 return { value, issues };
176 } 176 }
177 177
178+/**
179+ * Sections de premier niveau absentes de la sortie du modèle (`$.property`,
180+ * `$.exterior`…). Le réparateur les remplit de nulls, mais une sortie où
181+ * plusieurs sections manquent n'est PAS une analyse : c'est typiquement un appel
182+ * d'outil « placeholder » (`{"property_analysis":{"placeholder":true}}`, observé
183+ * avec Sonnet 5 en tool_choice forcé, 2026-09-08). À traiter comme un échec
184+ * dur → relance, jamais comme une analyse complète.
185+ */
186+export function missingTopLevelSections(issues: ValidationIssue[]): string[] {
187+ return issues.filter((i) => i.repaired && i.message === "champ manquant" && /^\$\.[a-z_]+$/.test(i.path)).map((i) => i.path.slice(2));
188+}
189+
190+/** Seuil au-delà duquel une sortie est jugée vide (placeholder) : ≥ 3 sections manquantes. */
191+export const MAX_MISSING_SECTIONS = 2;
192+
178 function typeList(schema: JS): string[] { 193 function typeList(schema: JS): string[] {
179 const t = schema.type; 194 const t = schema.type;
180 return Array.isArray(t) ? (t as string[]) : typeof t === "string" ? [t] : []; 195 return Array.isArray(t) ? (t as string[]) : typeof t === "string" ? [t] : [];
181 196
Menu

Theme