Commit
fix(worker): tolerate truncated/invalid batch results; raise rater max_tokens
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
modified
apps/worker/src/raters/batch.ts
+15 −2
@@ -26,7 +26,7 @@ export async function submitRatingBatch(tasks: RatingTask[], model: string): Pro | |||
| 26 | custom_id: ratingJobId(task.taskId, model, RATER_PROMPT_VERSION), | 26 | custom_id: ratingJobId(task.taskId, model, RATER_PROMPT_VERSION), |
| 27 | params: { | 27 | params: { |
| 28 | model, | 28 | model, |
| 29 | − max_tokens: 2048, | 29 | + max_tokens: 3000, |
| 30 | system: [ | 30 | system: [ |
| 31 | { | 31 | { |
| 32 | type: "text" as const, | 32 | type: "text" as const, |
@@ -80,13 +80,26 @@ export async function ingestBatchResults( | |||
| 80 | continue; | 80 | continue; |
| 81 | } | 81 | } |
| 82 | const message = entry.result.message; | 82 | const message = entry.result.message; |
| 83 | + // A max_tokens stop truncates the JSON mid-string — count as failed; the | ||
| 84 | + // task stays unrated and is resubmitted by the next rating run. | ||
| 85 | + if (message.stop_reason === "max_tokens") { | ||
| 86 | + failed += 1; | ||
| 87 | + continue; | ||
| 88 | + } | ||
| 83 | const textBlock = message.content.find((block) => block.type === "text"); | 89 | const textBlock = message.content.find((block) => block.type === "text"); |
| 84 | if (!textBlock || textBlock.type !== "text") { | 90 | if (!textBlock || textBlock.type !== "text") { |
| 85 | failed += 1; | 91 | failed += 1; |
| 86 | continue; | 92 | continue; |
| 87 | } | 93 | } |
| 88 | 94 | ||
| 89 | − const parsed = ratingResponseSchema.safeParse(JSON.parse(textBlock.text)); | 95 | + let json: unknown; |
| 96 | + try { | ||
| 97 | + json = JSON.parse(textBlock.text); | ||
| 98 | + } catch { | ||
| 99 | + failed += 1; | ||
| 100 | + continue; | ||
| 101 | + } | ||
| 102 | + const parsed = ratingResponseSchema.safeParse(json); | ||
| 90 | if (!parsed.success) { | 103 | if (!parsed.success) { |
| 91 | failed += 1; | 104 | failed += 1; |
| 92 | continue; | 105 | continue; |
| 93 | 106 | ||