Repository Public
1import { mkdir, readFile, writeFile } from "node:fs/promises";2import path from "node:path";3import { sql, j } from "../lib/db";4import { env } from "../lib/env";5import { getPolicy, observedVideoRate } from "../lib/budget";6import {7 CreativeBrief, Critique, PitchScores, PitchSet, Research, Script, ShotPlan, critiqueScore, pitchTotal,8 type FilmManifest, type Pitch, type TechnicalCheck,9} from "../lib/contracts";10import { cosine, embed } from "../lib/embedding";11import { assetPath, getAsset, saveAsset, storage, tmpDir, type AssetRow } from "../lib/storage";12import { SERIES } from "../lib/taxonomy";13import { log } from "../lib/log";14import { providers } from "../providers";15import { videoCaps } from "../providers/capabilities";16import { IMAGE_EDIT_USD, WEB_SEARCH_USD_PER_CALL, estimateImage, estimateSpeech, estimateVideo } from "../providers/pricing";17import { ProviderError, type CharTiming, type Usage } from "../providers/types";18import { cuesFromText, cuesFromWords, toVtt, wordsFromChars, type Cue } from "./captions";19import { assembleMaster, cleanup, conformShot, frameAt, probe, renderLow, stillFromImage } from "./ffmpeg";20import * as prompts from "./prompts";21import { runProviderJob } from "./provider-job";22import { technicalChecks, technicalPassed, technicalReport } from "./qa";2324export const STAGES = ["concept", "script", "storyboard", "keyframes", "voice", "video", "assemble", "qa"] as const;25export type Stage = (typeof STAGES)[number];26/** Stages that buy media. `assisted` mode pauses before the expensive one. */27export const PAID_MEDIA: Stage[] = ["video"];2829export interface ProjectRow {30 id: string; title: string; series_id: string | null; brief: unknown; language: "en" | "fr" | "bilingual"; mode: "manual" | "assisted" | "autonomous";31 status: string; running_stage: string | null; blocked_reason: string | null; error: string | null; video_model: string | null; origin: string;32 created_at: Date; updated_at: Date;33}34export interface ShotRow {35 id: string; project_id: string; idx: number; title: string; narration: string; narration_alt: string; camera: string; action: string; continuity: string;36 transition: string; planned_duration: number; image_prompt: string; video_prompt: string; status: string;37 image_asset_id: string | null; audio_asset_id: string | null; video_asset_id: string | null; narration_duration: number | null;38 timestamps: { lang: "en" | "fr"; words: Cue[] } | null; image_rev: number; video_rev: number; voice_rev: number; regen_count: number; regen_approved: boolean; error: string | null;39}4041/** Timing grammar: narration starts LEAD seconds into a shot and the shot holds TAIL seconds after it. */42export const LEAD = 0.5, TAIL = 0.9, MIN_SHOT = 5;43export const shotDuration = (narration: number | null) => Math.max(MIN_SHOT, Math.round(((narration ?? 0) + LEAD + TAIL) * 100) / 100);44export const TARGET = { width: 1080, height: 1920 };45/** Keyframes: 2k, medium quality (the provider's own advice when fine detail must be recognisable). */46export const KEYFRAME = { aspectRatio: "9:16", resolution: "2k", quality: "medium" } as const;4748export async function getProject(id: string): Promise<ProjectRow> {49 const [p] = await sql<ProjectRow[]>`SELECT * FROM projects WHERE id = ${id}`;50 if (!p) throw new Error("Project not found");51 return p;52}53export const getShots = (projectId: string) => sql<ShotRow[]>`SELECT * FROM shots WHERE project_id = ${projectId} ORDER BY idx`;54const setStatus = (id: string, status: string) => sql`UPDATE projects SET status = ${status}, updated_at = now(), error = NULL, blocked_reason = NULL WHERE id = ${id}`;55const briefOf = (p: ProjectRow) => CreativeBrief.parse(p.brief);56const seriesKind = (slug: string) => SERIES.find((s) => s.slug === slug)?.kind ?? "fiction";5758async function pool<T>(items: T[], n: number, fn: (item: T) => Promise<void>): Promise<void> {59 const errors: Error[] = [];60 let i = 0;61 await Promise.all(Array.from({ length: Math.max(1, Math.min(n, items.length)) }, async () => {62 while (i < items.length) { const item = items[i++]; try { await fn(item); } catch (e) { errors.push(e as Error); } }63 }));64 if (errors.length) {65 const budget = errors.find((e) => e.name === "BudgetError");66 if (budget) throw budget;67 if (errors.length === 1) throw errors[0]; // keep the original type (retryable or not)68 // Several shots failed: retry the stage if any failure is transient; finished shots are reused, not re-bought.69 const retryable = errors.some((e) => (e as ProviderError).retryable === true || e.name === "JobInProgressError");70 throw new ProviderError(`${errors.length} shots failed: ${errors.map((e) => e.message).join(" · ").slice(0, 600)}`, { retryable });71 }72}7374async function llmJob<T>(o: { project: ProjectRow; stage: string; name: string; prompt: { system: string; user: string }; input: unknown; exec: () => Promise<{ data: T; usage: Usage; model: string; citations?: string[] }>; kind?: "llm" | "search"; estimate?: number }) {75 return runProviderJob<{ data: T; citations?: string[] }>({76 projectId: o.project.id, stage: o.stage, kind: o.kind ?? "llm", provider: providers().name, model: env.llmModel,77 prompt: `${o.prompt.system}\n\n---\n\n${o.prompt.user}`, settings: { name: o.name },78 input: { name: o.name, system: o.prompt.system, user: o.prompt.user, extra: o.input, rev: await stageRev(o.project.id, o.stage) }, estimateUsd: o.estimate ?? 0.08,79 exec: async () => { const r = await o.exec(); return { output: { data: r.data, citations: r.citations }, usage: r.usage }; },80 });81}8283/** Revision counter per (project, stage): bumping it makes the LLM calls of that stage buy a fresh answer. */84export async function stageRev(projectId: string, stage: string): Promise<number> {85 const [r] = await sql<{ n: number }[]>`SELECT (value->>'n')::int AS n FROM settings WHERE key = ${`rev:${projectId}:${stage}`}`;86 return r?.n ?? 0;87}88export async function bumpStageRev(projectId: string, stage: string) {89 await sql`INSERT INTO settings (key, value) VALUES (${`rev:${projectId}:${stage}`}, ${j({ n: 1 })})90 ON CONFLICT (key) DO UPDATE SET value = jsonb_build_object('n', (settings.value->>'n')::int + 1), updated_at = now()`;91}9293// ───────────────────────── concept ─────────────────────────94export async function stageConcept(projectId: string) {95 const project = await getProject(projectId);96 const brief = briefOf(project);97 const prior = await sql<{ title: string; logline: string }[]>`98 SELECT c.pitch->>'title' AS title, c.pitch->>'logline' AS logline FROM concept_candidates c99 WHERE c.selected AND c.project_id <> ${projectId} ORDER BY c.created_at DESC LIMIT 40`;100 const [{ rounds }] = await sql<{ rounds: number }[]>`SELECT COALESCE((SELECT (value->>'n')::int FROM settings WHERE key = ${`concept_round:${projectId}`}), 0) AS rounds`;101 const cd = prompts.creativeDirector(brief, prior.map((p) => `${p.title}: ${p.logline}`), rounds + 1);102 const gen = await llmJob({ project, stage: "concept", name: "pitches", prompt: cd, input: { round: rounds },103 exec: () => providers().llm.generateStructured({ name: "pitches", schema: PitchSet, ...cd, effort: "medium" }) });104 const pitches: Pitch[] = PitchSet.parse(gen.output.data).pitches.slice(0, 5);105 if (pitches.length < 2) throw new ProviderError("The Creative Director returned fewer than two pitches", { retryable: true });106107 const judge = prompts.pitchJudge(brief, pitches);108 const scored = await llmJob({ project, stage: "concept", name: "pitch_scores", prompt: judge, input: {},109 exec: () => providers().llm.critique({ name: "pitch_scores", schema: PitchScores, ...judge }) });110 const scores = PitchScores.parse(scored.output.data).scores;111112 // Duplicate prevention: compare with published films and selected concepts (explainable lexical similarity).113 const corpus = [114 ...(await sql<{ label: string; text: string }[]>`SELECT title_en AS label, title_en || ' ' || synopsis_en || ' ' || takeaway_en AS text FROM films`),115 ...prior.map((p) => ({ label: p.title, text: `${p.title} ${p.logline}` })),116 ].map((c) => ({ label: c.label, v: embed(c.text) }));117118 await sql.begin(async (tx) => {119 await tx`DELETE FROM concept_candidates WHERE project_id = ${projectId}`;120 let best = { idx: 0, total: -1 };121 const rows = pitches.map((pitch, idx) => {122 const s = scores.find((x) => x.index === idx) ?? { index: idx, originality: 5, feasibility: 5, visualPotential: 5, narrativeClarity: 5, artifactRisk: 5, note: "not scored" };123 const v = embed(`${pitch.title} ${pitch.logline} ${pitch.thesis}`);124 const near = corpus.map((c) => ({ label: c.label, sim: cosine(v, c.v) })).sort((a, b) => b.sim - a.sim)[0];125 const duplicate = near && near.sim > 0.6 ? near : null;126 const total = Math.max(0, pitchTotal(s) - (duplicate ? 3 : 0));127 if (total > best.total) best = { idx, total };128 return { idx, pitch, scores: { ...s, duplicateOf: duplicate?.label ?? null, similarity: near ? Math.round(near.sim * 100) / 100 : 0 }, total };129 });130 for (const r of rows) {131 await tx`INSERT INTO concept_candidates (project_id, idx, pitch, scores, total, selected) VALUES (${projectId}, ${r.idx}, ${j(r.pitch)}, ${j(r.scores)}, ${r.total}, ${r.idx === best.idx})`;132 }133 await tx`INSERT INTO settings (key, value) VALUES (${`concept_round:${projectId}`}, ${j({ n: rounds + 1 })}) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`;134 });135 await setStatus(projectId, "pitched");136}137138// ───────────────────────── script (+ research) ─────────────────────────139export async function stageScript(projectId: string) {140 const project = await getProject(projectId);141 const brief = briefOf(project);142 const [cand] = await sql<{ pitch: Pitch }[]>`SELECT pitch FROM concept_candidates WHERE project_id = ${projectId} AND selected LIMIT 1`;143 if (!cand) throw new Error("Select a pitch before writing the script");144 const factual = seriesKind(brief.seriesSlug) === "factual";145146 const write = async (corrections?: string) => {147 const sw = prompts.screenwriter(brief, cand.pitch, corrections);148 const r = await llmJob({ project, stage: "script", name: "script", prompt: sw, input: {},149 exec: () => providers().llm.generateStructured({ name: "script", schema: Script, ...sw, effort: "medium" }) });150 const s = Script.parse(r.output.data);151 if (s.beats.length < 2) throw new ProviderError("The script has fewer than two beats", { retryable: true });152 s.beats = s.beats.slice(0, 6);153 if (brief.language !== "bilingual") s.beats = s.beats.map((b) => ({ ...b, lang: brief.language as "en" | "fr" }));154 return s;155 };156 const check = async (s: Script) => {157 // Informational films are checked line by line, not only on the claims the Screenwriter chose to declare.158 const lines = factual ? [...new Set([...s.beats.map((b) => b.narration), ...s.claims])] : s.claims;159 if (!lines.length) return null;160 const re = prompts.researchEditor(lines, `${cand.pitch.title} — ${cand.pitch.thesis}`);161 const r = await llmJob({ project, stage: "research", kind: "search", name: "research", prompt: re, input: {}, estimate: 0.3 + 10 * WEB_SEARCH_USD_PER_CALL,162 exec: () => providers().llm.research({ name: "research", schema: Research, ...re, effort: "medium" }) });163 const data = Research.parse(r.output.data);164 // Keep only sources that are well-formed https URLs; a factual claim with no usable source cannot be "supported".165 for (const c of data.claims) {166 c.sources = c.sources.filter((x) => /^https:\/\/[^\s]+$/.test(x.url)).slice(0, 3);167 if (c.factual && c.verdict === "supported" && !c.sources.length) c.verdict = "uncertain";168 }169 return { ...data, claims: data.claims.filter((c) => c.factual), descriptive: data.claims.filter((c) => !c.factual).length, citations: r.output.citations ?? [], checkedAt: new Date().toISOString() };170 };171172 let script = await write();173 let research = await check(script);174 const wrong = research?.claims.filter((c) => c.verdict === "inaccurate") ?? [];175 if (wrong.length) {176 script = await write(wrong.map((c) => `- "${c.claim}" → ${c.correction || "remove this claim"}`).join("\n"));177 research = await check(script);178 }179 const open = research?.claims.filter((c) => c.verdict !== "supported") ?? [];180 const researchRow = research ? { ...research, required: factual, unresolved: open.length } : (factual ? { claims: [], citations: [], required: true, unresolved: 0, note: "The script declared no factual claims." } : null);181182 const [{ v }] = await sql<{ v: number }[]>`SELECT COALESCE(MAX(version), 0) + 1 AS v FROM script_versions WHERE project_id = ${projectId}`;183 await sql`INSERT INTO script_versions (project_id, version, script, research) VALUES (${projectId}, ${v}, ${j(script)}, ${researchRow ? j(researchRow) : null})`;184 await sql`UPDATE projects SET title = ${script.titleEn}, updated_at = now() WHERE id = ${projectId}`;185 await setStatus(projectId, "scripted");186}187188export async function currentScript(projectId: string) {189 const [row] = await sql<{ id: string; version: number; script: Script; research: (Research & { required?: boolean; unresolved?: number; citations?: string[]; checkedAt?: string; note?: string }) | null; author: string }[]>`190 SELECT * FROM script_versions WHERE project_id = ${projectId} ORDER BY version DESC LIMIT 1`;191 return row ?? null;192}193194// ───────────────────────── storyboard + bibles + prompts ─────────────────────────195export async function stageStoryboard(projectId: string) {196 const project = await getProject(projectId);197 const brief = briefOf(project);198 const sv = await currentScript(projectId);199 if (!sv) throw new Error("Write the script before the storyboard");200 const [cand] = await sql<{ pitch: Pitch }[]>`SELECT pitch FROM concept_candidates WHERE project_id = ${projectId} AND selected LIMIT 1`;201 const sd = prompts.storyboardDirector(brief, cand.pitch, sv.script);202 const r = await llmJob({ project, stage: "storyboard", name: "shot_plan", prompt: sd, input: { scriptVersion: sv.version },203 exec: () => providers().llm.generateStructured({ name: "shot_plan", schema: ShotPlan, ...sd, effort: "medium" }) });204 const plan = ShotPlan.parse(r.output.data);205 if (plan.shots.length < sv.script.beats.length) throw new ProviderError(`The storyboard has ${plan.shots.length} shots for ${sv.script.beats.length} narration lines`, { retryable: true });206 plan.shots = plan.shots.slice(0, sv.script.beats.length);207208 await sql.begin(async (tx) => {209 const [{ v }] = await tx<{ v: number }[]>`SELECT COALESCE(MAX(version), 0) + 1 AS v FROM storyboards WHERE project_id = ${projectId}`;210 await tx`INSERT INTO storyboards (project_id, version, plan) VALUES (${projectId}, ${v}, ${j(plan)})`;211 await tx`INSERT INTO bibles (project_id, version, bible) VALUES (${projectId}, ${v}, ${j(plan.bible)})`;212 await tx`DELETE FROM shots WHERE project_id = ${projectId}`;213 for (const [idx, s] of plan.shots.entries()) {214 const beat = sv.script.beats[idx];215 const [shot] = await tx<{ id: string }[]>`216 INSERT INTO shots (project_id, idx, title, narration, narration_alt, camera, action, continuity, transition, image_prompt, video_prompt, timestamps)217 VALUES (${projectId}, ${idx}, ${s.title}, ${beat.narration}, ${beat.narrationAlt}, ${s.camera}, ${s.action}, ${s.continuity}, ${idx === 0 ? "cut" : s.transition}, ${s.imagePrompt}, ${s.videoPrompt}, ${j({ lang: beat.lang, words: [] })})218 RETURNING id`;219 for (const [kind, prompt] of [["image", s.imagePrompt], ["video", s.videoPrompt], ["voice", beat.narration]] as const) {220 await tx`INSERT INTO prompt_versions (shot_id, kind, version, prompt) VALUES (${shot.id}, ${kind}, 1, ${prompt})`;221 }222 }223 });224 await setStatus(projectId, "storyboarded");225}226227// ───────────────────────── keyframes ─────────────────────────228export async function generateKeyframe(project: ProjectRow, shot: ShotRow) {229 const model = env.imageModel;230 // Shot 1 is generated from text; every later shot is made by editing shot 1's keyframe, which is what keeps231 // the protagonist and the look identical from shot to shot.232 let anchor: AssetRow | null = null;233 if (shot.idx > 0) {234 const [first] = await sql<{ image_asset_id: string | null }[]>`SELECT image_asset_id FROM shots WHERE project_id = ${project.id} AND idx = 0`;235 anchor = await getAsset(first?.image_asset_id);236 if (!anchor) throw new Error("The first shot needs its keyframe before the others can be derived from it");237 }238 const r = await runProviderJob<{ width: number; height: number }>({239 projectId: project.id, shotId: shot.id, stage: "keyframes", kind: "image", provider: providers().name, model,240 prompt: shot.image_prompt, settings: { ...KEYFRAME, mode: anchor ? "edit-from-shot-1" : "generate" },241 input: { prompt: shot.image_prompt, ...KEYFRAME, anchor: anchor?.sha256 ?? null, rev: shot.image_rev, shot: shot.id },242 estimateUsd: anchor ? IMAGE_EDIT_USD : estimateImage(model, KEYFRAME.quality, KEYFRAME.resolution).usd,243 exec: async () => {244 const img = anchor245 ? await providers().image.edit({ prompt: shot.image_prompt, ...KEYFRAME, image: { bytes: await storage().read(anchor.sha256, anchor.ext), mime: anchor.mime } })246 : await providers().image.generate({ prompt: shot.image_prompt, ...KEYFRAME });247 const ext = img.mime.includes("png") ? "png" : img.mime.includes("webp") ? "webp" : "jpg";248 const saved = await saveAsset({ bytes: img.bytes, ext, kind: "keyframe", projectId: project.id, meta: { model: img.model, shot: shot.idx, derivedFrom: anchor?.sha256 ?? null } });249 const p = await probe(assetPath(saved));250 await sql`UPDATE assets SET width = ${p.width}, height = ${p.height} WHERE id = ${saved.id}`;251 return { output: { width: p.width, height: p.height }, assetId: saved.id, usage: img.usage };252 },253 });254 await sql`UPDATE shots SET image_asset_id = ${r.assetId}, status = 'keyframe', error = NULL, video_asset_id = NULL, updated_at = now() WHERE id = ${shot.id}`;255}256export async function stageKeyframes(projectId: string) {257 const project = await getProject(projectId);258 const shots = await getShots(projectId);259 if (!shots.length) throw new Error("Create the storyboard before generating keyframes");260 const { concurrency } = await getPolicy();261 const run = async (s: ShotRow) => {262 try { await generateKeyframe(project, s); } catch (e) { await sql`UPDATE shots SET error = ${(e as Error).message.slice(0, 500)}, updated_at = now() WHERE id = ${s.id}`; throw e; }263 };264 const missing = shots.filter((s) => !s.image_asset_id);265 const first = missing.find((s) => s.idx === 0);266 if (first) await run(first); // the anchor comes first267 await pool(missing.filter((s) => s.idx > 0), concurrency, run);268 await setStatus(projectId, "keyframes");269}270271// ───────────────────────── voice ─────────────────────────272export async function generateVoice(project: ProjectRow, shot: ShotRow, voiceId: string) {273 const lang = shot.timestamps?.lang ?? (project.language === "fr" ? "fr" : "en");274 const text = shot.narration.trim();275 if (!text) { await sql`UPDATE shots SET narration_duration = 0, audio_asset_id = NULL, updated_at = now() WHERE id = ${shot.id}`; return; }276 const r = await runProviderJob<{ duration: number; chars: CharTiming[] }>({277 projectId: project.id, shotId: shot.id, stage: "voice", kind: "tts", provider: providers().name, model: "xai-tts",278 prompt: text, settings: { voiceId, language: lang }, input: { text, voiceId, lang, rev: shot.voice_rev },279 estimateUsd: estimateSpeech(text.length).usd,280 exec: async () => {281 const a = await providers().speech.synthesize({ text, voiceId, language: lang });282 const saved = await saveAsset({ bytes: a.bytes, ext: "mp3", kind: "narration", projectId: project.id, duration: a.duration, meta: { voiceId, lang, shot: shot.idx } });283 return { output: { duration: a.duration, chars: a.chars }, assetId: saved.id, usage: a.usage };284 },285 });286 const asset = await getAsset(r.assetId);287 // Trust the decoded file for the duration (the reported value excludes encoder padding).288 const real = asset ? (await probe(assetPath(asset))).duration : r.output.duration;289 const words = wordsFromChars(r.output.chars);290 const spoken = Math.max(r.output.duration, words.at(-1)?.end ?? 0);291 await sql`UPDATE shots SET audio_asset_id = ${r.assetId}, narration_duration = ${Math.min(real, spoken + 0.25)}, timestamps = ${j({ lang, words })},292 status = CASE WHEN status IN ('planned','keyframe') THEN 'voiced' ELSE status END, updated_at = now() WHERE id = ${shot.id}`;293}294export async function stageVoice(projectId: string) {295 const project = await getProject(projectId);296 const sv = await currentScript(projectId);297 const shots = await getShots(projectId);298 if (!sv || !shots.length) throw new Error("Script and storyboard are required before voice");299 await pool(shots.filter((s) => !s.audio_asset_id && s.narration.trim()), 2, (s) => generateVoice(project, s, sv.script.voiceId));300 await setStatus(projectId, "voiced");301}302303// ───────────────────────── video ─────────────────────────304const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));305/**306 * Image-to-video for one shot. With `preview`, a cheap draft is rendered (other model / resolution) and stored as an307 * intermediate asset without touching the shot: the director looks at it before paying for the final clip.308 */309export async function generateShotVideo(project: ProjectRow, shot: ShotRow, o: { model?: string; resolution?: "480p" | "720p" | "1080p"; preview?: boolean; maxSeconds?: number } = {}): Promise<{ assetId: string | null; costUsd: number | null; duration: number }> {310 const model = o.model ?? project.video_model ?? env.videoModel;311 const caps = videoCaps(model);312 const image = await getAsset(shot.image_asset_id);313 if (!image) throw new Error(`Shot ${shot.idx + 1} has no keyframe`);314 const need = shotDuration(shot.narration_duration);315 const duration = Math.min(o.maxSeconds ?? caps.maxDuration, caps.maxDuration, Math.max(caps.minDuration, Math.ceil(need)));316 const resolution = o.resolution ?? env.videoResolution;317 const stage = o.preview ? "preview" : "video";318 const est = estimateVideo(model, resolution, duration, await observedVideoRate(model, resolution));319 const r = await runProviderJob<{ duration: number; moderationOk: boolean; width: number; height: number; hasAudio: boolean }>({320 projectId: project.id, shotId: shot.id, stage, kind: "video", provider: providers().name, model,321 prompt: shot.video_prompt, settings: { duration, resolution, mode: "image-to-video", estimateBasis: est.basis, preview: !!o.preview },322 input: { image: image.sha256, prompt: shot.video_prompt, duration, resolution, rev: shot.video_rev, preview: !!o.preview },323 estimateUsd: est.usd, approvedUnknown: shot.regen_approved,324 exec: async ({ job, setRequestId }) => {325 const video = providers().video;326 let requestId = job.request_id;327 if (!requestId) {328 const bytes = await storage().read(image.sha256, image.ext);329 requestId = (await video.imageToVideo({ model, prompt: shot.video_prompt, duration, resolution, image: { bytes, mime: image.mime } })).requestId;330 await setRequestId(requestId); // persisted before polling: a crash resumes here331 } else log("info", "video.resume", { requestId, shot: shot.idx });332 const deadline = Date.now() + 20 * 60_000;333 let outages = 0;334 for (;;) {335 // The job is already paid for: a network blip while polling must never lose it.336 let st: Awaited<ReturnType<typeof video.poll>>;337 try { st = await video.poll(requestId); outages = 0; }338 catch (e) {339 if (!(e as ProviderError).retryable || ++outages > 12 || Date.now() > deadline) throw e;340 log("warn", "video.poll_outage", { requestId, outages, error: (e as Error).message });341 await sleep(10_000);342 continue;343 }344 if (st.status === "done") {345 if (!st.url) throw Object.assign(new ProviderError("Provider reported done without a video URL", { retryable: false, billed: true }), { usage: st.usage });346 if (st.moderationOk === false) throw Object.assign(new ProviderError("The clip was filtered by provider moderation", { retryable: false, billed: true, code: "moderation" }), { usage: st.usage });347 const bytes = await video.download(st.url);348 const saved = await saveAsset({ bytes, ext: "mp4", kind: o.preview ? "intermediate" : "shot_video", projectId: project.id, meta: { model, shot: shot.idx, requestId, preview: !!o.preview } });349 const p = await probe(assetPath(saved));350 if (!p.hasVideo || p.duration < 1) throw Object.assign(new ProviderError("Downloaded clip is not a playable video", { retryable: false, billed: true }), { usage: st.usage });351 await sql`UPDATE assets SET width = ${p.width}, height = ${p.height}, duration = ${p.duration} WHERE id = ${saved.id}`;352 return { output: { duration: p.duration, moderationOk: true, width: p.width, height: p.height, hasAudio: p.hasAudio }, assetId: saved.id, usage: st.usage };353 }354 if (st.status === "failed") throw Object.assign(new ProviderError(`Video generation failed: ${st.error?.message ?? "unknown"} (${st.error?.code ?? "?"})`, { retryable: st.error?.code === "service_unavailable" || st.error?.code === "internal_error", code: st.error?.code }), { usage: st.usage });355 if (Date.now() > deadline) throw new ProviderError("Video generation timed out after 20 minutes; it will resume from the same request on retry", { retryable: true, code: "timeout" });356 await sleep(5000);357 }358 },359 });360 if (!o.preview) await sql`UPDATE shots SET video_asset_id = ${r.assetId}, status = 'filmed', error = NULL, updated_at = now() WHERE id = ${shot.id}`;361 return { assetId: r.assetId, costUsd: r.costUsd, duration: r.output.duration };362}363export async function stageVideo(projectId: string) {364 const project = await getProject(projectId);365 const shots = await getShots(projectId);366 if (!shots.length || shots.some((s) => !s.image_asset_id)) throw new Error("Every shot needs a keyframe before video generation");367 if (shots.some((s) => s.narration.trim() && !s.audio_asset_id)) throw new Error("Voice must be generated before video (shot durations follow the narration)");368 const { concurrency } = await getPolicy();369 await pool(shots.filter((s) => !s.video_asset_id), concurrency, async (s) => {370 try { await generateShotVideo(project, s); } catch (e) { await sql`UPDATE shots SET error = ${(e as Error).message.slice(0, 500)}, status = CASE WHEN ${(e as Error).name === "BudgetError"} THEN status ELSE 'failed' END, updated_at = now() WHERE id = ${s.id}`; throw e; }371 });372 await setStatus(projectId, "filmed");373}374375// ───────────────────────── assembly ─────────────────────────376export interface Timeline { shots: { idx: number; start: number; duration: number; narrationStart: number; narrationEnd: number; transition: string }[]; total: number }377export function buildTimeline(shots: Pick<ShotRow, "idx" | "narration_duration" | "transition">[]): Timeline {378 let t = 0;379 const out = shots.map((s) => {380 const duration = shotDuration(s.narration_duration);381 const row = { idx: s.idx, start: t, duration, narrationStart: t + LEAD, narrationEnd: t + LEAD + (s.narration_duration ?? 0), transition: s.transition };382 t = Math.round((t + duration) * 1000) / 1000;383 return row;384 });385 return { shots: out, total: t };386}387export function buildCaptions(shots: Pick<ShotRow, "narration" | "narration_alt" | "timestamps">[], tl: Timeline): Record<"en" | "fr", Cue[]> {388 const out: Record<"en" | "fr", Cue[]> = { en: [], fr: [] };389 shots.forEach((s, i) => {390 const t = tl.shots[i];391 const spoken = s.timestamps?.lang ?? "en";392 const other = spoken === "en" ? "fr" : "en";393 const words = s.timestamps?.words ?? [];394 out[spoken].push(...(words.length ? cuesFromWords(words, t.narrationStart) : cuesFromText(s.narration, t.narrationStart, t.narrationEnd)));395 out[other].push(...cuesFromText(s.narration_alt || s.narration, t.narrationStart, t.narrationEnd));396 });397 return out;398}399400export async function stageAssemble(projectId: string) {401 const project = await getProject(projectId);402 const sv = await currentScript(projectId);403 const shots = await getShots(projectId);404 if (!sv || !shots.length || shots.some((s) => !s.video_asset_id)) throw new Error("Every shot needs a generated clip before assembly");405 const tl = buildTimeline(shots);406 const [{ v: version }] = await sql<{ v: number }[]>`SELECT COALESCE(MAX(version), 0) + 1 AS v FROM film_versions WHERE project_id = ${projectId}`;407 const work = path.join(tmpDir(), `${projectId}-v${version}`);408 await mkdir(work, { recursive: true });409 try {410 const conformed: string[] = [];411 const narration: { file: string; at: number }[] = [];412 for (const [i, s] of shots.entries()) {413 const clip = (await getAsset(s.video_asset_id))!;414 const src = assetPath(clip);415 const p = await probe(src);416 const out = path.join(work, `shot-${i}.mkv`);417 const fadeIn = i === 0 ? 0.35 : s.transition === "fade" ? 0.3 : 0;418 const fadeOut = i === shots.length - 1 ? 0.5 : shots[i + 1].transition === "fade" ? 0.3 : 0;419 await conformShot({ input: src, output: out, duration: tl.shots[i].duration, width: TARGET.width, height: TARGET.height, fadeIn, fadeOut, hasAudio: p.hasAudio });420 conformed.push(out);421 const audio = await getAsset(s.audio_asset_id);422 if (audio) narration.push({ file: assetPath(audio), at: tl.shots[i].narrationStart });423 }424 const generatedAt = new Date().toISOString();425 const metadata = {426 title: sv.script.titleEn, artist: "CLIP — autonomous AI cinema", date: generatedAt.slice(0, 10),427 comment: "AI-generated film. Images and video: xAI Grok Imagine. Narration: synthetic voice (xAI TTS). Published by clip.spboucher.ai.",428 description: sv.script.synopsisEn, copyright: "© CLIP / Simon-Pierre Boucher. AI-generated media.",429 };430 const masterFile = path.join(work, "master.mp4"), lowFile = path.join(work, "low.mp4"), posterFile = path.join(work, "poster.jpg"), thumbFile = path.join(work, "thumb.jpg");431 await assembleMaster({ shots: conformed, narration, total: tl.total, output: masterFile, workDir: work, metadata });432 await renderLow(masterFile, lowFile, metadata);433 const cover = (await getAsset(shots[0].image_asset_id))!;434 await stillFromImage(assetPath(cover), posterFile, TARGET.width, TARGET.height, 3);435 await stillFromImage(assetPath(cover), thumbFile, 360, 640, 5);436437 const mp = await probe(masterFile);438 const master = await saveAsset({ file: masterFile, ext: "mp4", kind: "master", projectId, width: mp.width, height: mp.height, duration: mp.duration });439 const low = await saveAsset({ file: lowFile, ext: "mp4", kind: "delivery_low", projectId, duration: mp.duration });440 const poster = await saveAsset({ file: posterFile, ext: "jpg", kind: "poster", projectId, width: TARGET.width, height: TARGET.height });441 const thumb = await saveAsset({ file: thumbFile, ext: "jpg", kind: "thumb", projectId, width: 360, height: 640 });442443 const captions = buildCaptions(shots, tl);444 const vtt: Record<string, AssetRow> = {};445 for (const lang of ["en", "fr"] as const) {446 const f = path.join(work, `captions.${lang}.vtt`);447 await writeFile(f, toVtt(captions[lang]));448 vtt[lang] = await saveAsset({ file: f, ext: "vtt", kind: "captions", projectId, meta: { lang } });449 }450 const spokenLangs = [...new Set(shots.map((s) => s.timestamps?.lang ?? "en"))];451 const videoModel = project.video_model ?? env.videoModel;452 const manifest: FilmManifest = {453 schema: "clip.film/1",454 title: { en: sv.script.titleEn, fr: sv.script.titleFr },455 language: project.language, kind: seriesKind(briefOf(project).seriesSlug),456 duration: mp.duration, frame: { width: mp.width, height: mp.height, fps: Math.round(mp.fps) },457 shots: tl.shots.map((s) => ({ index: s.idx + 1, start: s.start, duration: s.duration, transition: s.transition })),458 audio: {459 narration: `Synthetic preset voice "${sv.script.voiceId}" (xAI text to speech); no voice cloning`,460 bed: "Audio generated with each clip by the video model, ducked under the narration",461 music: "none", loudnessTarget: "−14 LUFS integrated, −1.5 dBTP",462 },463 captions: (["en", "fr"] as const).map((l) => ({ language: l, kind: spokenLangs.includes(l) ? "spoken" : "translation" })),464 provenance: {465 aiGenerated: true,466 models: [467 { role: "concept, script, storyboard, critique", provider: "xAI", model: env.llmModel },468 { role: "keyframes", provider: "xAI", model: env.imageModel },469 { role: "image-to-video", provider: "xAI", model: videoModel },470 { role: "narration", provider: "xAI", model: "text-to-speech" },471 ],472 pipeline: ["concept", "research", "script", "storyboard", "keyframes", "voice", "video", "assembly", "automated QA", "human approval"],473 generatedAt, humanApproval: "pending",474 },475 };476 await sql.begin(async (tx) => {477 const [fv] = await tx<{ id: string }[]>`478 INSERT INTO film_versions (project_id, version, manifest, master_asset_id, low_asset_id, poster_asset_id, duration)479 VALUES (${projectId}, ${version}, ${j({ ...manifest, _assets: { thumb: thumb.id, captions: { en: vtt.en.id, fr: vtt.fr.id } }, _timeline: tl })}, ${master.id}, ${low.id}, ${poster.id}, ${mp.duration})480 RETURNING id`;481 for (const lang of ["en", "fr"] as const) {482 await tx`INSERT INTO subtitle_tracks (film_version_id, language, kind, asset_id) VALUES (${fv.id}, ${lang}, ${spokenLangs.includes(lang) ? "spoken" : "translation"}, ${vtt[lang].id})`;483 }484 await tx`INSERT INTO audio_tracks (film_version_id, kind, language, voice_id, asset_id, license) VALUES485 (${fv.id}, 'narration', ${spokenLangs.join("+")}, ${sv.script.voiceId}, NULL, 'xAI TTS preset voice; synthetic, no cloned voice'),486 (${fv.id}, 'bed', NULL, NULL, NULL, 'Generated by the video model with each clip; no third-party recording'),487 (${fv.id}, 'mix', NULL, NULL, ${master.id}, 'Original mix by CLIP')`;488 });489 await setStatus(projectId, "assembled");490 } finally { await cleanup(work); }491}492493export async function latestFilmVersion(projectId: string) {494 const [fv] = await sql<{ id: string; version: number; manifest: FilmManifest & { _assets: { thumb: string; captions: { en: string; fr: string } }; _timeline: Timeline }; master_asset_id: string; low_asset_id: string; poster_asset_id: string; duration: number; created_at: Date }[]>`495 SELECT * FROM film_versions WHERE project_id = ${projectId} ORDER BY version DESC LIMIT 1`;496 return fv ?? null;497}498499// ───────────────────────── QA ─────────────────────────500export async function stageQa(projectId: string) {501 const project = await getProject(projectId);502 const fv = await latestFilmVersion(projectId);503 const sv = await currentScript(projectId);504 const shots = await getShots(projectId);505 if (!fv || !sv) throw new Error("Assemble the film before QA");506 const master = (await getAsset(fv.master_asset_id))!;507 const tl = fv.manifest._timeline;508 const jobs = await sql<{ shot_id: string; output: { moderationOk?: boolean } | null }[]>`509 SELECT shot_id, output FROM provider_jobs WHERE project_id = ${projectId} AND kind = 'video' AND status = 'done'`;510 const captions = buildCaptions(shots, tl);511 const checks: TechnicalCheck[] = await technicalChecks({512 master: assetPath(master), expectedDuration: tl.total, width: TARGET.width, height: TARGET.height,513 shots: tl.shots.map((t, i) => ({ ...t, moderationOk: jobs.filter((x) => x.shot_id === shots[i].id).every((x) => x.output?.moderationOk !== false) })),514 captions,515 });516517 const work = path.join(tmpDir(), `${projectId}-qa`);518 await mkdir(work, { recursive: true });519 let critique: Critique | null = null, critiqueError: string | null = null;520 try {521 const images: { bytes: Buffer; mime: string }[] = [];522 for (const [i, t] of tl.shots.entries()) {523 for (const [k, frac] of [0.25, 0.8].entries()) {524 const f = path.join(work, `f-${i}-${k}.jpg`);525 await frameAt(assetPath(master), t.start + t.duration * frac, f);526 images.push({ bytes: await readFile(f), mime: "image/jpeg" });527 }528 }529 const [sb] = await sql<{ plan: ShotPlan }[]>`SELECT plan FROM storyboards WHERE project_id = ${projectId} ORDER BY version DESC LIMIT 1`;530 const research = sv.research ? (sv.research.claims?.length ? sv.research.claims.map((c) => `${c.verdict.toUpperCase()}: ${c.claim} (${c.sources.length} source${c.sources.length === 1 ? "" : "s"})`).join("\n") : "No factual claims declared.") : "Fiction: no research required.";531 const qc = prompts.qualityCritic({ titleEn: sv.script.titleEn, thesis: sv.script.takeawayEn, kind: fv.manifest.kind, plan: sb.plan, lines: shots.map((s) => s.narration), frames: images.length, technical: technicalReport(checks), research });532 const r = await runProviderJob<{ data: Critique }>({533 projectId, stage: "qa", kind: "llm", provider: providers().name, model: env.llmModel, prompt: `${qc.system}\n\n---\n\n${qc.user}`, settings: { name: "critique", frames: images.length },534 input: { master: master.sha256, system: qc.system, user: qc.user }, estimateUsd: 0.05,535 exec: async () => { const x = await providers().llm.critique({ name: "critique", schema: Critique, ...qc, images, effort: "medium" }); return { output: { data: x.data }, usage: x.usage }; },536 });537 critique = Critique.parse(r.output.data);538 } catch (e) {539 if ((e as Error).name === "BudgetError") throw e;540 critiqueError = (e as Error).message;541 } finally { await cleanup(work); }542543 const policy = await getPolicy();544 const score = critique ? critiqueScore(critique) : null;545 const blocking = critique?.issues.filter((x) => x.severity === "blocking") ?? [];546 const unresolved = sv.research?.required ? sv.research.unresolved ?? 0 : 0;547 const reasons = [548 ...checks.filter((c) => !c.ok && c.severity === "fail").map((c) => `${c.label}: ${c.detail}`),549 ...(critique ? [] : [`Automated critique unavailable: ${critiqueError}`]),550 ...(score !== null && score < policy.qa_threshold ? [`Critic score ${score} is below the threshold ${policy.qa_threshold}`] : []),551 ...(critique && critique.safety < 8 ? [`Safety score ${critique.safety} is below 8`] : []),552 ...blocking.map((b) => `Blocking issue (shot ${b.shot}): ${b.note}`),553 ...(unresolved ? [`${unresolved} factual claim${unresolved > 1 ? "s" : ""} not supported by sources`] : []),554 ];555 const passed = technicalPassed(checks) && reasons.length === 0;556 await sql`INSERT INTO quality_reviews (project_id, film_version_id, technical, critique, score, passed)557 VALUES (${projectId}, ${fv.id}, ${j({ checks, reasons })}, ${critique ? j(critique) : null}, ${score}, ${passed})`;558 await setStatus(projectId, passed ? "qa_passed" : "qa_failed");559 log("info", "qa.done", { project: project.id, passed, score, reasons: reasons.length });560}561562export const STAGE_FN: Record<Stage, (projectId: string) => Promise<void>> = {563 concept: stageConcept, script: stageScript, storyboard: stageStoryboard, keyframes: stageKeyframes, voice: stageVoice, video: stageVideo, assemble: stageAssemble, qa: stageQa,564};565