Repository Public
1import { mkdir, readFile } from "node:fs/promises";2import path from "node:path";3import { z } from "zod";4import "./_env";5import { sql, j } from "../src/lib/db";6import { getPolicy } from "../src/lib/budget";7import { Bible, LANGS, Research, VOICES, type Script } from "../src/lib/contracts";8import { env } from "../src/lib/env";9import { assetPath, getAsset } from "../src/lib/storage";10import { TAGS } from "../src/lib/taxonomy";11import { providers } from "../src/providers";12import { IMAGE_EDIT_USD, estimateImage, estimateVideo } from "../src/providers/pricing";13import { frameAt } from "../src/pipeline/ffmpeg";14import * as prompts from "../src/pipeline/prompts";15import { runProviderJob } from "../src/pipeline/provider-job";16import {17 KEYFRAME, buildTimeline, generateKeyframe, generateShotVideo, generateVoice, getProject, getShots, latestFilmVersion, shotDuration,18 stageAssemble, stageQa, type ShotRow,19} from "../src/pipeline/stages";2021/**22 * `./clip` — the tools handed to the Claude Code director. Every command prints one JSON object.23 * Paid commands go through the same budget gate, idempotency and cost ledger as the rest of the studio,24 * and additionally respect the budget of the director run.25 */26const args = process.argv.slice(2);27const opt = (name: string) => { const i = args.indexOf(`--${name}`); return i < 0 ? undefined : args[i + 1]; };28const projectId = opt("project")!, runId = opt("run");29const VALUE_FLAGS = ["project", "run", "prompt-file", "narration", "alt"];30const rest: string[] = [];31for (let i = 0; i < args.length; i++) { if (args[i].startsWith("--")) { if (VALUE_FLAGS.includes(args[i].slice(2))) i++; } else rest.push(args[i]); }32const [cmd, ...params] = rest;33const workdir = process.env.CLIP_WORKDIR ?? process.cwd();34const viewDir = path.join(workdir, "view");3536const DirectorPlan = z.object({37 titleEn: z.string().min(2).max(80), titleFr: z.string().min(2).max(80),38 synopsisEn: z.string().min(10).max(500), synopsisFr: z.string().min(10).max(500),39 takeawayEn: z.string().max(240), takeawayFr: z.string().max(240),40 language: z.enum(LANGS), kind: z.enum(["fiction", "factual", "experimental"]).default("fiction"),41 voiceId: z.enum(VOICES), voiceDirection: z.string().max(240).default(""),42 tags: z.array(z.string()).max(12).default([]),43 bible: Bible,44 shots: z.array(z.object({45 title: z.string().max(80), purpose: z.enum(["hook", "setup", "tension", "turn", "payoff"]),46 lang: z.enum(["en", "fr"]), narration: z.string().min(3).max(260), narrationAlt: z.string().min(3).max(260),47 camera: z.string().max(240).default(""), action: z.string().max(400).default(""), transition: z.enum(["cut", "fade"]).default("cut"),48 imagePrompt: z.string().min(40).max(3000), videoPrompt: z.string().min(30).max(2000),49 })).min(2).max(6),50});5152class ToolError extends Error {}53const fail = (m: string): never => { throw new ToolError(m); };5455async function spend() {56 const [r] = await sql<{ usd: number }[]>`SELECT COALESCE(SUM(usd), 0)::float8 AS usd FROM cost_entries WHERE project_id = ${projectId}`;57 const [run] = runId ? await sql<{ budget_usd: number; stop_requested: boolean }[]>`SELECT budget_usd, stop_requested FROM director_runs WHERE id = ${runId}` : [];58 return { spent: r.usd, budget: run ? Number(run.budget_usd) : null, stop: run?.stop_requested ?? false };59}60/** Run-level budget on top of the studio policy. */61async function guard(estimate: number) {62 const s = await spend();63 if (s.stop) fail("The operator asked this run to stop. Do not start anything else; write your final report.");64 if (s.budget !== null && s.spent + estimate > s.budget + 1e-9) fail(`Run budget exceeded: $${s.spent.toFixed(2)} spent + $${estimate.toFixed(2)} > $${s.budget.toFixed(2)}. Finish with what exists (assemble, qa) or stop.`);65}66async function shotN(n: string | undefined): Promise<ShotRow> {67 const shots = await getShots(projectId);68 const s = shots[Number(n) - 1];69 if (!s) fail(`No shot ${n}: the plan has ${shots.length} shots (1-based).`);70 return s;71}72async function view(src: string, name: string, at?: number): Promise<string> {73 await mkdir(viewDir, { recursive: true });74 const out = path.join(viewDir, name);75 await frameAt(src, at ?? 0, out, 720);76 return path.relative(workdir, out);77}78async function clipFrames(assetId: string | null, prefix: string): Promise<string[]> {79 const a = await getAsset(assetId);80 if (!a) return [];81 const d = Number(a.duration ?? 5);82 return Promise.all([0.08, 0.5, 0.92].map((f, i) => view(assetPath(a), `${prefix}-${i + 1}.jpg`, d * f)));83}84async function newPromptVersion(shotId: string, kind: "image" | "video" | "voice", prompt: string) {85 await sql`INSERT INTO prompt_versions (shot_id, kind, version, prompt, author)86 SELECT ${shotId}, ${kind}, COALESCE(MAX(version), 0) + 1, ${prompt}, 'claude-code' FROM prompt_versions WHERE shot_id = ${shotId} AND kind = ${kind}`;87}88const promptFile = async () => { const f = opt("prompt-file"); return f ? (await readFile(path.resolve(workdir, f), "utf8")).trim() : undefined; };89const settle = async () => {90 const shots = await getShots(projectId);91 const status = shots.some((s) => !s.image_asset_id) ? "storyboarded" : shots.some((s) => !s.audio_asset_id) ? "keyframes" : shots.some((s) => !s.video_asset_id) ? "voiced" : "filmed";92 await sql`UPDATE projects SET status = ${status}, updated_at = now(), error = NULL WHERE id = ${projectId} AND status NOT IN ('published','archived')`;93};9495const commands: Record<string, () => Promise<unknown>> = {96 async status() {97 const [p, shots, s, policy, fv] = await Promise.all([getProject(projectId), getShots(projectId), spend(), getPolicy(), latestFilmVersion(projectId)]);98 return {99 project: { title: p.title, status: p.status, language: p.language },100 budget: { spentUsd: Number(s.spent.toFixed(3)), runBudgetUsd: s.budget, perShotCeilingUsd: Number(policy.per_shot_usd), perFilmCeilingUsd: Number(policy.per_film_usd), killSwitch: policy.kill_switch },101 prices: {102 keyframeFirst: estimateImage(env.imageModel, KEYFRAME.quality, KEYFRAME.resolution).usd, keyframeEdit: IMAGE_EDIT_USD,103 previewPerSecond: estimateVideo(env.videoDraftModel, "480p", 1).usd, finalPerSecond: estimateVideo(env.videoModel, env.videoResolution, 1).usd,104 },105 shots: shots.map((x) => ({106 n: x.idx + 1, title: x.title, narration: x.narration, keyframe: !!x.image_asset_id, voice: !!x.audio_asset_id, finalClip: !!x.video_asset_id,107 seconds: x.narration_duration === null ? null : shotDuration(x.narration_duration), regenerations: x.regen_count, error: x.error,108 })),109 cut: fv ? { version: fv.version, seconds: Number(fv.duration) } : null,110 };111 },112113 /** Claude Code is the writer and director: it submits the whole plan as one JSON file. */114 async plan() {115 const file = params[0] ?? fail("usage: ./clip plan <file.json>");116 let raw: unknown;117 try { raw = JSON.parse(await readFile(path.resolve(workdir, file), "utf8")); } catch (e) { fail(`Cannot read ${file} as JSON: ${(e as Error).message}`); }118 const parsed = DirectorPlan.safeParse(raw);119 if (!parsed.success) fail(`Plan rejected: ${parsed.error.issues.slice(0, 8).map((i) => `${i.path.join(".")}: ${i.message}`).join("; ")}`);120 const plan = parsed.data!;121 const p = await getProject(projectId);122 if (p.status === "published") fail("This project is published and cannot be re-planned.");123 const vocab = (kind: string) => plan.tags.filter((t) => TAGS.some((x) => x.slug === t && x.kind === kind));124 const script: Script = {125 titleEn: plan.titleEn, titleFr: plan.titleFr, synopsisEn: plan.synopsisEn, synopsisFr: plan.synopsisFr, takeawayEn: plan.takeawayEn, takeawayFr: plan.takeawayFr,126 voiceId: plan.voiceId, voiceDirection: plan.voiceDirection,127 beats: plan.shots.map((s) => ({ purpose: s.purpose, lang: s.lang, narration: s.narration, narrationAlt: s.narrationAlt, visual: s.action || s.title })),128 claims: [], tags: { genre: vocab("genre"), mood: vocab("mood"), topic: vocab("topic"), technique: vocab("technique"), style: vocab("style") },129 };130 await sql.begin(async (tx) => {131 const [{ v }] = await tx<{ v: number }[]>`SELECT COALESCE(MAX(version), 0) + 1 AS v FROM script_versions WHERE project_id = ${projectId}`;132 await tx`INSERT INTO script_versions (project_id, version, script, research, author) VALUES (${projectId}, ${v}, ${j(script)}, ${plan.kind === "factual" ? j({ claims: [], required: true, unresolved: 0, note: "Not yet checked: run ./clip factcheck" }) : null}, 'claude-code')`;133 const shotPlan = { bible: plan.bible, shots: plan.shots.map((s) => ({ title: s.title, camera: s.camera, action: s.action, continuity: "", transition: s.transition, imagePrompt: s.imagePrompt, videoPrompt: s.videoPrompt })) };134 await tx`INSERT INTO storyboards (project_id, version, plan) VALUES (${projectId}, ${v}, ${j(shotPlan)})`;135 await tx`INSERT INTO bibles (project_id, version, bible) VALUES (${projectId}, ${v}, ${j(plan.bible)})`;136 await tx`DELETE FROM shots WHERE project_id = ${projectId}`;137 for (const [idx, s] of plan.shots.entries()) {138 const [row] = await tx<{ id: string }[]>`139 INSERT INTO shots (project_id, idx, title, narration, narration_alt, camera, action, transition, image_prompt, video_prompt, timestamps)140 VALUES (${projectId}, ${idx}, ${s.title}, ${s.narration}, ${s.narrationAlt}, ${s.camera}, ${s.action}, ${idx === 0 ? "cut" : s.transition}, ${s.imagePrompt}, ${s.videoPrompt}, ${j({ lang: s.lang, words: [] })})141 RETURNING id`;142 for (const [kind, prompt] of [["image", s.imagePrompt], ["video", s.videoPrompt], ["voice", s.narration]] as const) {143 await tx`INSERT INTO prompt_versions (shot_id, kind, version, prompt, author) VALUES (${row.id}, ${kind}, 1, ${prompt}, 'claude-code')`;144 }145 }146 const brief = { ...(p.brief as object), language: plan.language, shots: plan.shots.length };147 await tx`UPDATE projects SET title = ${plan.titleEn}, language = ${plan.language}, brief = ${j(brief)}, status = 'storyboarded', error = NULL, updated_at = now() WHERE id = ${projectId}`;148 });149 return { ok: true, shots: plan.shots.length, next: "Generate keyframe 1 first (./clip keyframe 1), look at it, then the others." };150 },151152 async keyframe() {153 let shot = await shotN(params[0]);154 const prompt = await promptFile();155 const p = await getProject(projectId);156 if (prompt || shot.image_asset_id) {157 if (prompt) { await newPromptVersion(shot.id, "image", prompt); await sql`UPDATE shots SET image_prompt = ${prompt} WHERE id = ${shot.id}`; }158 await sql`UPDATE shots SET image_rev = image_rev + 1, image_asset_id = NULL, video_asset_id = NULL, regen_count = regen_count + ${shot.image_asset_id ? 1 : 0}, updated_at = now() WHERE id = ${shot.id}`;159 shot = await shotN(params[0]);160 }161 await guard(shot.idx === 0 ? 0.08 : IMAGE_EDIT_USD);162 await generateKeyframe(p, shot);163 await settle();164 const fresh = await shotN(params[0]);165 const asset = (await getAsset(fresh.image_asset_id))!;166 return {167 shot: shot.idx + 1, image: await view(assetPath(asset), `shot${shot.idx + 1}-keyframe.jpg`), mode: shot.idx === 0 ? "generated from text" : "edited from shot 1",168 note: shot.idx === 0 ? "Open the image with Read. Every other keyframe is derived from this one: if you regenerate it, regenerate the others too." : "Open the image with Read and check the character matches shot 1.",169 };170 },171172 async voice() {173 const p = await getProject(projectId);174 const [sv] = await sql<{ script: Script }[]>`SELECT script FROM script_versions WHERE project_id = ${projectId} ORDER BY version DESC LIMIT 1`;175 if (!sv) fail("Submit a plan first.");176 const n = params[0];177 const shots = n ? [await shotN(n)] : await getShots(projectId);178 const narration = opt("narration");179 if (n && narration) {180 await newPromptVersion(shots[0].id, "voice", narration);181 await sql`UPDATE shots SET narration = ${narration}, narration_alt = ${opt("alt") ?? shots[0].narration_alt}, voice_rev = voice_rev + 1, audio_asset_id = NULL, narration_duration = NULL, video_asset_id = NULL WHERE id = ${shots[0].id}`;182 shots[0] = await shotN(n);183 }184 await guard(0.01);185 for (const s of shots) if (!s.audio_asset_id) await generateVoice(p, s, sv.script.voiceId);186 await settle();187 return { voice: sv.script.voiceId, shots: (await getShots(projectId)).map((s) => ({ n: s.idx + 1, narrationSeconds: s.narration_duration, shotSeconds: shotDuration(s.narration_duration) })) };188 },189190 /** Cheap draft of the motion (draft model, 480p): look before paying for the final clip. */191 async preview() {192 const shot = await shotN(params[0]);193 const prompt = await promptFile();194 if (prompt) { await newPromptVersion(shot.id, "video", prompt); await sql`UPDATE shots SET video_prompt = ${prompt}, video_rev = video_rev + 1, video_asset_id = NULL WHERE id = ${shot.id}`; }195 const s = prompt ? await shotN(params[0]) : shot;196 if (!s.image_asset_id) fail("This shot has no keyframe yet.");197 const seconds = Math.min(6, Math.ceil(shotDuration(s.narration_duration)));198 await guard(estimateVideo(env.videoDraftModel, "480p", seconds).usd ?? 0.2);199 const r = await generateShotVideo(await getProject(projectId), s, { model: env.videoDraftModel, resolution: "480p", preview: true, maxSeconds: 6 });200 return { shot: s.idx + 1, model: env.videoDraftModel, seconds: r.duration, costUsd: r.costUsd, frames: await clipFrames(r.assetId, `shot${s.idx + 1}-preview`), note: "Three frames (start, middle, end). The final model moves better and is sharper; the preview tells you whether the action and camera idea work." };201 },202203 /** Final clip: master model at full resolution. Needs the voice first, because the clip is sized to its line. */204 async render() {205 const which = params[0] ?? fail("usage: ./clip render <n|all>");206 const prompt = await promptFile();207 const all = await getShots(projectId);208 if (all.some((s) => s.narration.trim() && !s.audio_asset_id)) fail("Run ./clip voice first: each clip lasts as long as its narration needs.");209 const targets = which === "all" ? all.filter((s) => !s.video_asset_id) : [await shotN(which)];210 const p = await getProject(projectId);211 const out = [];212 for (let s of targets) {213 if (!s.image_asset_id) fail(`Shot ${s.idx + 1} has no keyframe.`);214 if (which !== "all" && (prompt || s.video_asset_id)) {215 if (prompt) { await newPromptVersion(s.id, "video", prompt); await sql`UPDATE shots SET video_prompt = ${prompt} WHERE id = ${s.id}`; }216 await sql`UPDATE shots SET video_rev = video_rev + 1, video_asset_id = NULL, regen_count = regen_count + ${s.video_asset_id ? 1 : 0} WHERE id = ${s.id}`;217 s = await shotN(which);218 }219 const seconds = Math.ceil(shotDuration(s.narration_duration));220 await guard(estimateVideo(p.video_model ?? env.videoModel, env.videoResolution, seconds).usd ?? 2);221 const r = await generateShotVideo(p, s);222 out.push({ shot: s.idx + 1, seconds: r.duration, costUsd: r.costUsd, frames: await clipFrames(r.assetId, `shot${s.idx + 1}-final`) });223 }224 await settle();225 return { rendered: out, note: "Open the frames with Read. Re-render a shot only for a defect a viewer would notice." };226 },227228 /** Informational films: check the narration against sources with web search. */229 async factcheck() {230 const p = await getProject(projectId);231 const [sv] = await sql<{ id: string; script: Script }[]>`SELECT id, script FROM script_versions WHERE project_id = ${projectId} ORDER BY version DESC LIMIT 1`;232 if (!sv) fail("Submit a plan first.");233 const lines = sv.script.beats.map((b) => b.narration);234 const re = prompts.researchEditor(lines, `${sv.script.titleEn} — ${sv.script.takeawayEn}`);235 await guard(0.35);236 const r = await runProviderJob<{ data: z.infer<typeof Research>; citations?: string[] }>({237 projectId: p.id, stage: "research", kind: "search", provider: providers().name, model: env.llmModel, prompt: `${re.system}\n\n---\n\n${re.user}`, settings: { name: "research" },238 input: { system: re.system, user: re.user }, estimateUsd: 0.35,239 exec: async () => { const x = await providers().llm.research({ name: "research", schema: Research, ...re, effort: "medium" }); return { output: { data: x.data, citations: x.citations }, usage: x.usage }; },240 });241 const data = Research.parse(r.output.data);242 for (const c of data.claims) { c.sources = c.sources.filter((x) => /^https:\/\/[^\s]+$/.test(x.url)).slice(0, 3); if (c.factual && c.verdict === "supported" && !c.sources.length) c.verdict = "uncertain"; }243 const factual = data.claims.filter((c) => c.factual);244 const unresolved = factual.filter((c) => c.verdict !== "supported").length;245 await sql`UPDATE script_versions SET research = ${j({ claims: factual, citations: r.output.citations ?? [], required: true, unresolved, checkedAt: new Date().toISOString() })} WHERE id = ${sv.id}`;246 return { checked: factual.length, unresolved, claims: factual.map((c) => ({ claim: c.claim, verdict: c.verdict, correction: c.correction, sources: c.sources.map((s) => s.url) })), note: unresolved ? "Fix the lines (./clip voice <n> --narration \"…\" --alt \"…\") and run factcheck again: QA holds a film with unresolved claims." : "All factual lines are supported." };247 },248249 async assemble() {250 const shots = await getShots(projectId);251 if (shots.some((s) => !s.video_asset_id)) fail(`Shots without a final clip: ${shots.filter((s) => !s.video_asset_id).map((s) => s.idx + 1).join(", ")}. Run ./clip render all.`);252 await sql`UPDATE projects SET status = 'filmed' WHERE id = ${projectId} AND status NOT IN ('published','archived')`;253 await stageAssemble(projectId);254 const fv = (await latestFilmVersion(projectId))!;255 const master = (await getAsset(fv.master_asset_id))!;256 const tl = buildTimeline(shots);257 const frames = await Promise.all(tl.shots.map((t, i) => view(assetPath(master), `cut-v${fv.version}-shot${i + 1}.jpg`, t.start + t.duration * 0.5)));258 return { cut: fv.version, seconds: Number(fv.duration), frames, next: "./clip qa" };259 },260261 async qa() {262 await guard(0.06);263 await stageQa(projectId);264 const [q] = await sql<{ score: number | null; passed: boolean; technical: { reasons: string[]; checks: { label: string; ok: boolean; detail: string }[] }; critique: { summary: string; issues: unknown[] } | null }[]>`265 SELECT score, passed, technical, critique FROM quality_reviews WHERE project_id = ${projectId} ORDER BY created_at DESC LIMIT 1`;266 return {267 passed: q.passed, score: q.score, blockingReasons: q.technical.reasons, failedChecks: q.technical.checks.filter((c) => !c.ok).map((c) => `${c.label}: ${c.detail}`),268 critic: q.critique, note: q.passed ? "QA passed. The film now waits for a human approval in the studio: do not try to publish. Write your final report." : "QA failed. Fix the blocking issues if the budget allows (edit a prompt, re-render that shot, assemble, qa), otherwise report what is wrong.",269 };270 },271272 async note() {273 const text = params.join(" ").slice(0, 600);274 if (runId && text) await sql`UPDATE director_runs SET log = log || ${j([{ t: new Date().toISOString(), kind: "note", text }])} WHERE id = ${runId}`;275 return { ok: true };276 },277};278279async function main() {280 if (!projectId || !/^[0-9a-f-]{36}$/.test(projectId)) fail("missing --project");281 const fn = commands[cmd ?? ""];282 if (!fn) fail(`Unknown command "${cmd ?? ""}". Commands: ${Object.keys(commands).join(", ")}`);283 console.log(JSON.stringify(await fn(), null, 1));284}285main().then(() => sql.end(), async (e: Error) => {286 // Errors are answers too: the director reads them and adapts.287 console.log(JSON.stringify({ error: e.message.replace(/xai-[A-Za-z0-9]{20,}/g, "xai-[redacted]"), kind: e.name }));288 await sql.end();289 process.exit(e instanceof ToolError || e.name === "BudgetError" ? 2 : 1);290});291