Skills › Visual Explainer › visual-explainer › render.mjs
render.mjs
175 lines · plugins/visual-explainer/video/render.mjs at 9d5f880
1#!/usr/bin/env node
2import { existsSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs";
3import { createRequire } from "node:module";
4import { homedir } from "node:os";
5import { basename, dirname, join, resolve } from "node:path";
6import { fileURLToPath, pathToFileURL } from "node:url";
7import { parseArgs } from "node:util";
8import { duration, encoder, lineId, lineLength, loadVoice, requireFfmpeg, writeTrack } from "./media.mjs";
10const DRIVER = fileURLToPath(new URL("./driver.js", import.meta.url));
11const FRAME = { x: 0, y: 0, width: 1280, height: 720, scale: 1.5 }; // 1920×1080 output
12const LEAD = 0.5, GAP = 0.3, TAIL = 0.9, BARE = 3; // pacing in seconds around and between lines
13const USAGE = `Usage: visual-explainer-video <deck.html> [out.mp4] [options]
15Renders a video deck (<section data-say> scenes) to a 1920×1080 MP4 on a virtual clock, so
16every frame lands on time. See references/video.md.
18Options:
19 --lines Print "<id><TAB><text>" for each narration line, then stop.
20 --voice <dir> Narrate with <dir>/<id>.mp3 (or .wav, .m4a, .ogg, .flac, .aiff). Without it
21 the video is silent with captions.
22 --captions Burn in captions with a voice too.
23 --stills Write one PNG per scene, fully played and captioned, to <name>.stills/, then stop.
24 --fps <n> Frames per second (default 30).`;
26async function main() {
27 const { values, positionals } = parseArgs({
28 allowPositionals: true,
29 options: { lines: { type: "boolean" }, voice: { type: "string" }, captions: { type: "boolean" }, stills: { type: "boolean" }, fps: { type: "string" }, help: { type: "boolean", short: "h" } },
30 });
31 if (values.help || !positionals.length) return console.log(USAGE);
32 if (positionals.length > 2) throw new Error(`Too many arguments: ${positionals.slice(2).join(" ")}\n\n${USAGE}`);
33 const home = (p) => resolve(p.replace(/^~(?=\/)/, homedir()));
34 const input = home(positionals[0]);
35 if (!existsSync(input)) throw new Error(`No such file: ${input}`);
36 const stem = input.replace(/\.[^./]+$/, "");
37 const out = home(positionals[1] ?? `${stem}.mp4`);
38 const fps = Number(values.fps ?? 30);
39 if (!Number.isInteger(fps) || fps < 1 || fps > 120) throw new Error("--fps must be a whole number from 1 to 120");
41 if (values.lines) {
42 const scenes = await withDeck(input, ({ scenes }) => scenes);
43 for (const text of new Set(scenes.flatMap((s) => s.lines))) console.log(`${lineId(text)}\t${text}`);
44 return;
45 }
46 if (values.stills) return await withDeck(input, (deck) => stills(deck, `${stem}.stills`));
47 if (!existsSync(dirname(out))) throw new Error(`Output folder does not exist: ${dirname(out)}`);
48 requireFfmpeg();
49 const voice = values.voice ? home(values.voice) : null;
50 await withDeck(input, (deck) => render(deck, { out, fps, voice, captions: values.captions || !voice }));
51 console.log(`wrote ${out} (${duration(out).toFixed(1)} s)`);
52}
54// Opens the deck at 1280×720 CSS pixels with driver.js injected, and passes it to fn.
55async function withDeck(input, fn) {
56 let chromium;
57 try {
58 ({ chromium } = await import("playwright-core"));
59 } catch (e) {
60 if (e.code !== "ERR_MODULE_NOT_FOUND") throw e;
61 // playwright-core is an optional peer, so only people who make videos download it.
62 throw new Error("playwright-core is not installed. Run: npx -y -p visual-explainer -p playwright-core visual-explainer-video <deck.html> …\nWith a global install, run npm install -g playwright-core once. In a repository checkout, run npm install first.");
63 }
64 let browser;
65 try {
66 browser = await chromium.launch();
67 } catch (e) {
68 if (!/Executable doesn't exist/.test(e.message)) throw e;
69 try {
70 browser = await chromium.launch({ channel: "chrome" });
71 } catch {
72 const { version } = createRequire(import.meta.url)("playwright-core/package.json");
73 throw new Error(`No browser found. Install Google Chrome, or run: npx playwright-core@${version} install chromium-headless-shell`);
74 }
75 }
76 try {
77 const ctx = await browser.newContext({ viewport: { width: 1280, height: 720 }, deviceScaleFactor: 1.5, reducedMotion: "no-preference" });
78 await ctx.addInitScript({ path: DRIVER });
79 const page = await ctx.newPage();
80 const errors = [];
81 page.on("pageerror", (e) => errors.push(e.message));
82 await page.goto(pathToFileURL(input).href, { waitUntil: "networkidle" });
83 await page.evaluate(() => document.fonts.ready);
84 const failed = () => { if (errors.length) throw new Error(`The deck threw: ${errors[0]}`); };
85 failed();
86 const scenes = await page.evaluate(() => __ve.scenes());
87 if (!scenes.length) throw new Error(`${basename(input)} has no <section data-say> scenes.`);
88 const cdp = await ctx.newCDPSession(page);
89 // The clip scale re-rasterizes at 1.5×; without it a CDP screenshot comes back at 1280×720.
90 const shot = async (format = "jpeg") =>
91 Buffer.from((await cdp.send("Page.captureScreenshot", { format, clip: FRAME, ...(format === "jpeg" && { quality: 92, optimizeForSpeed: true }) })).data, "base64");
92 return await fn({ page, scenes, shot, failed });
93 } finally {
94 await browser.close();
95 }
96}
98// Stills show each scene's last sentence as a caption, so captions can be checked against the layout.
99async function stills({ page, scenes, shot, failed }, dir) {
100 // A fresh, uniquely named folder per run: no stale files, no clash with another run.
101 const part = mkdtempSync(`${dir}.partial-`);
102 try {
103 let t = 0;
104 for (const [i, s] of scenes.entries()) {
105 for (let b = 0; b <= s.lines.length; b++) {
106 await page.evaluate(([i, b, t, text]) => { __ve.cue(t, i, b, text, true); __ve.tick(t + 1500); }, [i, b, t, s.lines[b - 1] ?? ""]);
107 t += 1500;
108 }
109 await page.evaluate((t) => __ve.tick(t), t += 3000);
110 failed();
111 writeFileSync(join(part, `scene-${String(i + 1).padStart(2, "0")}.png`), await shot("png"));
112 }
113 // Replace the previous stills only once every new one exists.
114 rmSync(dir, { recursive: true, force: true });
115 renameSync(part, dir);
116 } finally {
117 rmSync(part, { recursive: true, force: true });
118 }
119 console.log(`wrote ${scenes.length} stills to ${dir}`);
120}
122async function render({ page, scenes, shot, failed }, { out, fps, voice, captions }) {
123 const clips = voice ? loadVoice(voice, scenes.flatMap((s) => s.lines)) : null;
124 // Scene start, then each line in turn; a line's beat starts when its clip starts.
125 const cues = [], track = [];
126 let t = 0;
127 for (const [i, s] of scenes.entries()) {
128 cues.push({ t, scene: i, beat: 0, text: "" });
129 let at = t + LEAD;
130 for (const [j, text] of s.lines.entries()) {
131 cues.push({ t: at, scene: i, beat: j + 1, text });
132 if (clips) track.push({ at, pcm: clips.get(text) });
133 at += lineLength(text, clips?.get(text)) + GAP;
134 }
135 t = (s.lines.length ? at - GAP + TAIL : t + BARE) + s.hold;
136 }
138 // One fresh folder beside the output holds the voice track and the partial MP4. The MP4 is renamed
139 // into place on success, so a failed render keeps the last good one and two renders never share files.
140 const partDir = mkdtempSync(join(dirname(out), ".ve-partial-"));
141 const part = join(partDir, basename(out));
142 let enc = null;
143 try {
144 const frames = Math.ceil(t * fps);
145 const audio = clips ? join(partDir, "voice.wav") : null;
146 if (audio) writeTrack(audio, track, frames / fps); // as long as the video, so neither stream ends early
147 enc = encoder(part, { fps, audio });
148 let next = 0, shown = 0;
149 for (let f = 0; f < frames; f++) {
150 // Each cue runs at its own time between frames, so what it starts is timed from the cue.
151 const due = [];
152 while (next < cues.length && cues[next].t <= f / fps) due.push(cues[next++]);
153 await page.evaluate(([due, ms, captions]) => {
154 for (const c of due) __ve.cue(c.t * 1000, c.scene, c.beat, c.text, captions);
155 __ve.tick(ms);
156 }, [due, (f / fps) * 1000, captions]);
157 failed();
158 await enc.write(await shot());
159 const pct = Math.floor(((f + 1) / frames) * 10) * 10;
160 if (pct > shown) { shown = pct; console.log(`rendered ${pct}%`); }
161 }
162 await enc.done();
163 renameSync(part, out);
164 } catch (e) {
165 enc?.kill();
166 throw e;
167 } finally {
168 rmSync(partDir, { recursive: true, force: true });
169 }
170}
172main().catch((e) => {
173 console.error(e.message);
174 process.exit(1);
175});