Skills › HyperFrames › media-use › carve.mjs
carve.mjs
560 lines · skills/media-use/audio/scripts/carve.mjs at a595755
1#!/usr/bin/env node
2/**
3 * Apply a voiceover carve to a composition, from the command line.
4 *
5 * The carve is an analysis: it listens to a voice track, finds the bands it
6 * occupies, and writes a chain of dips into the music bed plus a level match. In
7 * Studio a panel runs it. This is the same analysis for an agent that has no
8 * panel to click — identical functions from `@hyperframes/core`, identical
9 * output, so a composition carved here and one carved in Studio are the same
10 * three attributes.
11 *
12 * node carve.mjs --comp index.html
13 * node carve.mjs --comp index.html --bed music-bed --voice narration \
14 * --voice interview-guest --strength 0.45
15 *
16 * With no --bed/--voice it works out the tracks itself: the bed, and every voice
17 * playing over it. `--voice` may be repeated to name them instead. Every named
18 * voice is analysed together, so a bed running under a narrator and an answer makes
19 * room for both.
20 *
21 * Needs `ffmpeg` on PATH (to decode the audio) and `@hyperframes/core` resolvable
22 * from the composition's project (`npm i -D @hyperframes/core`) — the CLI bundles
23 * core inline rather than shipping it as a package, so it cannot be borrowed from
24 * there.
25 */
27import { execFileSync } from "node:child_process";
28import { createRequire } from "node:module";
29import { readFileSync, writeFileSync } from "node:fs";
30import { dirname, resolve } from "node:path";
31import { pathToFileURL } from "node:url";
32import { isMainModule } from "./lib/main-module.mjs";
34/** Sample rate the analysis runs at. Matches Studio's own decode rate, so the
35 * bands and envelopes come out the same either way. */
36const SAMPLE_RATE = 48000;
38/** Default carve strength. 0.25 kept the bed present but still fighting the voice. */
39export const DEFAULT_STRENGTH = 0.8;
41const usage = `carve.mjs --comp <file.html> [--bed <elementId>] [--voice <elementId> ...]
42 [--strength 0..1] [--dry-run] [--core <dir>]
44 --bed id of the music track that gets carved (detected if omitted)
45 --voice id of a voice to make room for; repeatable (detected if omitted)
46 --strength how hard to carve, 0..1 (default ${DEFAULT_STRENGTH})
47 --dry-run report what it would write, touch nothing
48 --core directory to resolve @hyperframes/core from (default: the comp's)`;
50export function parseArgs(argv) {
51 const args = { strength: DEFAULT_STRENGTH, dryRun: false, voices: [] };
52 for (let i = 0; i < argv.length; i += 1) {
53 const flag = argv[i];
54 const next = () => {
55 const value = argv[i + 1];
56 if (value === undefined) fail(`${flag} needs a value`);
57 i += 1;
58 return value;
59 };
60 if (flag === "--comp") args.comp = next();
61 else if (flag === "--bed") args.bed = next();
62 else if (flag === "--voice") args.voices.push(next());
63 else if (flag === "--strength") args.strength = Number(next());
64 else if (flag === "--core") args.core = next();
65 else if (flag === "--dry-run") args.dryRun = true;
66 else if (flag === "-h" || flag === "--help") fail(usage, 0);
67 else fail(`unknown flag: ${flag}\n\n${usage}`);
68 }
69 if (!args.comp) fail(`--comp is required\n\n${usage}`);
70 if (!Number.isFinite(args.strength) || args.strength < 0 || args.strength > 1) {
71 fail("--strength must be a number from 0 to 1");
72 }
73 return args;
74}
76function fail(message, code = 1) {
77 process.stderr.write(`${message}\n`);
78 process.exit(code);
79}
81/**
82 * Load the carve analysis out of `@hyperframes/core`.
83 *
84 * Resolved from the project rather than from this script, which lives wherever
85 * the skill was installed — a sibling of the composition is what has the
86 * dependency.
87 */
88export async function loadCore(fromDir) {
89 const require = createRequire(pathToFileURL(resolve(fromDir, "package.json")));
91 /*
92 * Two constraints at once, and satisfying either alone is broken:
93 *
94 * 1. Anchored at the PROJECT, not at this script. This file lives wherever
95 * the skill was installed, which has no @hyperframes/core; the
96 * composition's project is what holds the dependency. So a bare
97 * `import("@hyperframes/core/audio-carve")` from here cannot work — bare
98 * specifiers resolve relative to the importing module.
99 * 2. Honouring the package's export CONDITIONS. `require.resolve` asks for
100 * "require"/"node". The workspace manifest declares `node`, so this
101 * resolved fine inside the monorepo — but the PUBLISHED manifest carries
102 * only `import` + `types`, so every consumer of the released package got
103 * ERR_PACKAGE_PATH_NOT_EXPORTED for a package that ships the file. That
104 * is the audience this skill is shipped to, so the script was broken
105 * everywhere except where it was developed.
106 *
107 * Keep the project anchor; fall back to the package's declared `import`
108 * target when no require-resolvable condition exists.
109 */
110 const load = async (subpath) => {
111 const spec = `@hyperframes/core/${subpath}`;
112 try {
113 return await import(pathToFileURL(require.resolve(spec)).href);
114 } catch (error) {
115 if (error?.code !== "ERR_PACKAGE_PATH_NOT_EXPORTED") throw error;
116 // `./package.json` is exported by every manifest, so this always resolves
117 // and gives us the package root without guessing at node_modules layout.
118 const pkgPath = require.resolve("@hyperframes/core/package.json");
119 const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
120 const entry = pkg.exports?.[`./${subpath}`];
121 const target = typeof entry === "string" ? entry : (entry?.import ?? entry?.default ?? null);
122 if (!target) {
123 fail(
124 `@hyperframes/core does not export ./${subpath}\n` +
125 ` found at: ${pkgPath} (version ${pkg.version})\n` +
126 ` update it: npm i -D @hyperframes/core`,
127 );
128 }
129 return import(pathToFileURL(resolve(dirname(pkgPath), target)).href);
130 }
131 };
133 try {
134 return {
135 carve: await load("audio-carve"),
136 fx: await load("audio-fx"),
137 };
138 } catch (error) {
139 fail(
140 `cannot load @hyperframes/core from ${fromDir}\n` +
141 ` is it installed there? npm i -D @hyperframes/core\n` +
142 ` or point at one: --core <dir containing node_modules/@hyperframes/core>\n` +
143 ` (${error.code ?? "error"}: ${error.message.split("\n")[0]})`,
144 );
145 }
146}
148/**
149 * The `sources` a carve should record for these voices, on this bed.
150 *
151 * SKILL.md states the invariant: "A carve against more than one clip id is
152 * wrong. Group the clips and carve against the group." Naming the group lets
153 * `resolveCarveSourceIds` resolve membership at analysis time, so a voice added
154 * later is covered without editing `sources` — whereas a list of clip ids rots
155 * silently the moment a fourth narration clip appears. The lint rule
156 * `audio_carve_ungrouped_sources` enforces exactly this.
157 *
158 * This script was writing clip ids unconditionally, so it violated its own
159 * skill's invariant and tripped its own lint rule on every run. When every
160 * voice shares one group, record the group. Mixed or ungrouped voices keep
161 * their ids, and the lint rule then correctly tells the author to group them.
162 *
163 * The bed has to be part of the decision, because the group form resolves
164 * LATER and wider than it looks. If the bed is itself a member of the voices'
165 * group, `resolveCarveSourceIds` expands that id to every current member on the
166 * next analysis — including the bed — and the bed ends up carved against
167 * itself, which SKILL.md calls a bug rather than a mix choice. This run cannot
168 * see it: `main()` sums the voice list it detected and never round-trips
169 * through group resolution, so the first pass is correct and only the next
170 * re-analysis in Studio is wrong. So decline the group form there and fall back
171 * to clip ids, which is exactly the case `audio_carve_ungrouped_sources` exists
172 * to put in front of the author.
173 *
174 * Only an `<audio>` bed can trip it: group membership is audio-only
175 * (`audioGroupOf`), so `data-audio-group` on a `<video>` bed is ignored by core
176 * and expanding a group can never pull it in.
177 */
178export function carveSources(voices, bed, members) {
179 const group = sharedVoiceGroup(voices);
180 return group && !groupSourceRefusal(voices, bed, members) ? [group] : voices.map((v) => v.id);
181}
183/** The one group every voice belongs to, or null if they do not share exactly one. */
184function sharedVoiceGroup(voices) {
185 const groups = voices.map((v) => attrOf(v.tag, "data-audio-group"));
186 const first = groups[0];
187 return Boolean(first) && groups.every((g) => g === first) ? first : null;
188}
190/**
191 * Why naming the voices' shared group would persist something this run did not
192 * analyse — or null when the group is safe to name.
193 *
194 * `members` is every `<audio>` in the composition as `{id, group, nameKind}`,
195 * with `nameKind` from core's `classifyAudioName`, so this and Studio's picker
196 * classify the same way.
197 *
198 * Required, deliberately not defaulting to `[]`. With an empty list the `mixed`
199 * refusal below cannot fire, so a call that forgot the argument would return the
200 * group form and restore the exact behaviour this function exists to prevent —
201 * silently, because the first CLI pass is correct either way and only a later
202 * Studio re-analysis is wrong. A missing argument throws on `members.filter`
203 * instead.
204 *
205 * Two refusals, and both exist because the group form resolves LATER and WIDER
206 * than the analysis: `resolveCarveSourceIds` expands a group id to every current
207 * member on every analysis, and `resolveCarveVoices` keeps any audio member with
208 * a src. `main()` meanwhile sums the voice list `detectTracks` returned, so the
209 * first pass looks correct however wrong the persisted attribute is.
210 *
211 * `bed` — the bed is a member, so it would be handed to itself as a voice
212 * and carved against its own content.
213 * `mixed` — a member classified music or sfx is not a voice this run measured,
214 * so it would enter the sidechain on the next analysis and duck the
215 * bed under a whoosh.
216 *
217 * Deliberately NOT a refusal: a member classified `voice` or `unknown` that this
218 * run left out. That is the group form working as designed — `detectTracks` only
219 * takes voices that overlap the bed, and picking up a clip that starts playing
220 * later without an edit to `sources` is the whole reason SKILL.md says to name
221 * the group. Refusing there would collapse the group form into clip ids for
222 * every ordinary narration sequence.
223 */
224export function groupSourceRefusal(voices, bed, members) {
225 const group = sharedVoiceGroup(voices);
226 if (!group) return null;
227 if (bed?.kind === "audio" && attrOf(bed.tag, "data-audio-group") === group) {
228 return { group, reason: "bed", ids: [bed.id] };
229 }
230 const analysed = new Set(voices.map((v) => v.id));
231 const strays = members
232 .filter(
233 (m) =>
234 m.group === group &&
235 !analysed.has(m.id) &&
236 (m.nameKind === "music" || m.nameKind === "sfx"),
237 )
238 .map((m) => m.id);
239 return strays.length > 0 ? { group, reason: "mixed", ids: strays } : null;
240}
242/** Mono float PCM for one media file, via ffmpeg. */
243function decode(path) {
244 let raw;
245 try {
246 raw = execFileSync(
247 "ffmpeg",
248 [
249 "-v",
250 "error",
251 "-i",
252 path,
253 "-vn",
254 "-ac",
255 "1",
256 "-ar",
257 String(SAMPLE_RATE),
258 "-f",
259 "f32le",
260 "-",
261 ],
262 { maxBuffer: 1 << 30 },
263 );
264 } catch (error) {
265 fail(`could not decode ${path}\n ${error.message.split("\n")[0]}`);
266 }
267 if (raw.length === 0) fail(`no audio in ${path}`);
268 return new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
269}
271const attrOf = (tag, name) => tag.match(new RegExp(`\\s${name}="([^"]*)"`, "i"))?.[1] ?? null;
273const unescapeAttr = (value) =>
274 value
275 .replace(/"/g, '"')
276 .replace(/'/g, "'")
277 .replace(/&/g, "&");
279const escapeAttr = (value) => value.replace(/&/g, "&").replace(/"/g, """);
281/** Every media element with a src, as {id, tag, kind}. */
282function mediaElements(html) {
283 const found = [];
284 for (const match of html.matchAll(/<(audio|video)\b[^>]*>/gi)) {
285 const tag = match[0];
286 // `\sid=` and not `id=`: `data-hf-id` would match first.
287 const id = tag.match(/\sid="([^"]+)"/)?.[1];
288 if (id && attrOf(tag, "src")) found.push({ id, tag, kind: match[1].toLowerCase() });
289 }
290 return found;
291}
293/**
294 * Work out which track is the bed and which tracks are its voices.
295 *
296 * Names first, because they are what the author already told us and the answer is
297 * explainable: a track whose id or filename looks like music is the bed, ones that
298 * look like speech are voices, SFX-shaped names are neither. `classifyAudioName`
299 * comes from core so Studio's own picker and this cannot disagree.
300 *
301 * EVERY voice over the bed, not one of them. A bed usually runs under a whole
302 * sequence, and they are analysed together — so there is nothing to disambiguate,
303 * which is why this no longer refuses when several tracks look like speech.
304 *
305 * Only tracks that actually play while the bed does: one somewhere else on the
306 * timeline cannot mask it. It still refuses when it cannot find a bed at all, or
307 * finds no voice to make room for.
308 */
309function detectTracks(html, given, classify, overlaps) {
310 const all = mediaElements(html);
311 const kindOf = (el) => classify(el.id, unescapeAttr(attrOf(el.tag, "src") ?? ""));
312 const spanOf = (el) => {
313 const raw = attrOf(el.tag, "data-duration");
314 const n = raw === null ? Number.NaN : Number(raw);
315 return {
316 start: startOf(el.tag),
317 duration: Number.isFinite(n) ? n : null,
318 };
319 };
320 const pick = (id, what) => {
321 const found = all.find((el) => el.id === id);
322 if (!found) fail(`no <audio>/<video> with id="${id}" in the composition`);
323 return { ...found, why: `--${what}` };
324 };
326 let bed = given.bed ? pick(given.bed, "bed") : null;
327 const named = given.voices.map((id) => pick(id, "voice"));
329 if (!bed) {
330 const others = all.filter((el) => !named.some((v) => v.id === el.id));
331 const music = others.filter((el) => kindOf(el) === "music");
332 if (music.length === 1) bed = { ...music[0], why: "name looks like music" };
333 else if (music.length > 1) {
334 fail(
335 `several tracks look like music (${music.map((el) => el.id).join(", ")}) — name one with --bed`,
336 );
337 } else if (others.length === 1) {
338 bed = { ...others[0], why: "only track left" };
339 } else {
340 fail(
341 `cannot tell which track is the music bed\n` +
342 ` media in the composition: ${all.map((el) => el.id).join(", ") || "none"}\n` +
343 ` name it with --bed`,
344 );
345 }
346 }
348 const bedSpan = spanOf(bed);
349 const overlapping = (el) => overlaps(bedSpan, spanOf(el));
350 const plausible = all
351 .filter((el) => el.id !== bed.id && kindOf(el) !== "music" && kindOf(el) !== "sfx")
352 .filter(overlapping);
353 // A voiceover is normally its own <audio>. Video counts only when no audio track
354 // is left to be the voice — a talking-head recut — because otherwise every B-roll
355 // clip in the composition reads as somebody talking.
356 const spoken = plausible.filter((el) => el.kind === "audio");
357 const pool = spoken.length > 0 ? spoken : plausible;
358 const voices = named.length
359 ? named
360 : pool.map((el) => ({
361 ...el,
362 why: kindOf(el) === "voice" ? "name looks like a voice" : "plays over the bed",
363 }));
365 const usable = voices.filter((el) => attrOf(el.tag, "src"));
366 if (usable.length === 0) {
367 fail(
368 `no voice to make room for on ${bed.id}\n` +
369 ` media in the composition: ${all.map((el) => el.id).join(", ") || "none"}\n` +
370 ` name one with --voice`,
371 );
372 }
373 return { bed, voices: usable, all };
374}
376const startOf = (tag) => {
377 const raw = Number(attrOf(tag, "data-start"));
378 return Number.isFinite(raw) ? raw : 0;
379};
381async function main() {
382 const args = parseArgs(process.argv.slice(2));
383 const compPath = resolve(args.comp);
384 const compDir = dirname(compPath);
385 const { carve: carveApi, fx: fxApi } = await loadCore(args.core ? resolve(args.core) : compDir);
387 const html = readFileSync(compPath, "utf-8");
388 const {
389 bed: bedEl,
390 voices,
391 all: media,
392 } = detectTracks(html, args, carveApi.classifyAudioName, carveApi.clipsOverlap);
393 // Group membership + name classification for every audio track, so the source
394 // decision can see what the group will resolve to later and not just what this
395 // run analysed.
396 const members = media
397 .filter((el) => el.kind === "audio")
398 .map((el) => ({
399 id: el.id,
400 group: attrOf(el.tag, "data-audio-group"),
401 nameKind: carveApi.classifyAudioName(el.id, unescapeAttr(attrOf(el.tag, "src") ?? "")),
402 }));
403 const bedTag = bedEl.tag;
404 const bedSrc = attrOf(bedTag, "src");
405 process.stdout.write(
406 `bed ${bedEl.id} (${bedEl.why})\n` +
407 voices.map((v) => `voice ${v.id} (${v.why})`).join("\n") +
408 "\n",
409 );
411 const profile = carveApi.carveProfile(args.strength);
412 // Every voice summed onto the BED's clock before anything is measured. One
413 // question — where and when is speech masking this bed — with one answer, even
414 // when the answer comes from several people at different times.
415 const voice = carveApi.mixCarveSources(
416 voices.map((v) => ({
417 samples: decode(resolve(compDir, unescapeAttr(attrOf(v.tag, "src")))),
418 offsetSeconds: startOf(v.tag) - startOf(bedTag),
419 })),
420 SAMPLE_RATE,
421 );
422 if (voice.length === 0) fail("the voices do not overlap the bed, so there is nothing to carve");
423 const bands = carveApi.analyseCarveBands(voice, SAMPLE_RATE, profile);
425 // The level half of the carve needs both sides: "how far over the speech is this
426 // bed" cannot be answered by listening to one of them. No offset — the mix is
427 // already on the bed's clock.
428 const bed = profile.duckDb > 0 ? decode(resolve(compDir, unescapeAttr(bedSrc))) : null;
429 const duck = bed ? carveApi.analyseCarveDuck(voice, bed, SAMPLE_RATE, profile, 0) : [];
431 // Anything the author built by hand survives a carve; only the previous
432 // carve's own nodes are replaced. That is what `fromCarve` is for.
433 const existingChain = attrOf(bedTag, "data-fx-chain");
434 const existingNodes = existingChain
435 ? fxApi.parseAudioFxChain(unescapeAttr(existingChain)).nodes
436 : [];
437 const kept = existingNodes.filter((n) => !n.fromCarve);
438 // Lanes belonging to the carve being replaced, addressed by the ids the OLD
439 // nodes had. Taken before anything is minted: those ids are freed by the
440 // replacement and a new node can be handed one of them, so reading them off the
441 // new chain would keep exactly the stale lanes it is supposed to drop.
442 const stalePrefixes = existingNodes.filter((n) => n.fromCarve && n.id).map((n) => `fx.${n.id}.`);
444 let claimed = { version: 1, nodes: kept };
445 const mint = (node) => {
446 const withId = { ...node, id: fxApi.mintAudioFxNodeId(claimed), fromCarve: true };
447 claimed = { version: 1, nodes: [...claimed.nodes, withId] };
448 return withId;
449 };
450 const bandNodes = bands.map((band) => mint(carveApi.carveBandsToChain([band]).nodes[0]));
452 const duckNode =
453 duck.length > 0
454 ? mint({
455 type: "gain",
456 enabled: true,
457 params: {
458 ...fxApi.defaultAudioFxParams("gain"),
459 gain: 0,
460 },
461 })
462 : null;
463 const chain = {
464 version: 1,
465 nodes: [...bandNodes, ...(duckNode ? [duckNode] : []), ...kept],
466 };
468 /**
469 * One carve envelope as a lane on the BED's clock.
470 *
471 * Nothing to shift: the voices were summed onto that clock before the analysis
472 * ran. A lane does hold its first value backwards to the start of its clip, so an
473 * envelope that begins later needs an explicit "no cut" at zero or the bed starts
474 * out ducked.
475 */
476 const laneFor = (id, points) => {
477 const timed = points
478 .map((p) => ({ t: Number(p.t.toFixed(3)), v: p.v }))
479 .filter((p) => p.t >= 0);
480 if ((timed[0]?.t ?? 0) > 0) timed.unshift({ t: 0, v: 0 });
481 return timed.length > 1 ? [{ target: `fx.${id}.gain`, points: timed }] : [];
482 };
484 // Every carve follows the speech: a fixed depth thins the bed through every pause.
485 const carvedLanes = [
486 ...carveApi
487 .analyseCarveDynamics(voice, SAMPLE_RATE, bands)
488 .flatMap((dyn, i) => (bandNodes[i]?.id ? laneFor(bandNodes[i].id, dyn.points) : [])),
489 ...(duckNode?.id && duck.length > 0 ? laneFor(duckNode.id, duck) : []),
490 ];
492 // Hand-drawn lanes are kept the same way hand-built nodes are: by dropping only
493 // the ones that addressed the previous carve's nodes.
494 const existingAutomation = attrOf(bedTag, "data-automation");
495 const carriedLanes = existingAutomation
496 ? (JSON.parse(unescapeAttr(existingAutomation)).lanes ?? []).filter(
497 (lane) => !stalePrefixes.some((prefix) => String(lane.target).startsWith(prefix)),
498 )
499 : [];
500 const lanes = [...carriedLanes, ...carvedLanes];
502 const settings = {
503 enabled: true,
504 sources: carveSources(voices, bedEl, members),
505 strength: args.strength,
506 };
507 // Say why the group form was declined, or the lint rule tells the author to
508 // group clips they have already grouped.
509 const refusal = groupSourceRefusal(voices, bedEl, members);
510 if (refusal) {
511 process.stderr.write(
512 refusal.reason === "bed"
513 ? `note bed ${bedEl.id} is in group "${refusal.group}" with the voices, so\n` +
514 ` sources are clip ids: naming that group would carve the bed\n` +
515 ` against itself on the next analysis. Move the bed to its own group.\n`
516 : `note group "${refusal.group}" also holds ${refusal.ids.join(", ")}, which this run\n` +
517 ` did not analyse (music/sfx by name), so sources are clip ids: naming\n` +
518 ` the group would pull them into the sidechain on the next analysis.\n` +
519 ` Move them out of the voice group.\n`,
520 );
521 }
522 const written =
523 ` data-fx-carve="${escapeAttr(JSON.stringify(settings))}"` +
524 ` data-fx-chain="${escapeAttr(fxApi.serializeAudioFxChain(chain))}"` +
525 (lanes.length > 0
526 ? ` data-automation="${escapeAttr(JSON.stringify({ version: 1, lanes }))}"`
527 : "");
529 process.stdout.write(
530 `carve strength ${args.strength}, ${voices.length} voice${voices.length === 1 ? "" : "s"}\n` +
531 `bands ${bands.map((b) => `${b.freq}Hz ${b.gainDb}dB q${b.q}`).join(", ")}\n` +
532 `level ${
533 duckNode
534 ? `${duck.length}-point envelope, floor ${Math.min(...duck.map((p) => p.v))} dB`
535 : "no level match at this strength"
536 }\n` +
537 `lanes ${carvedLanes.length} carve${carriedLanes.length ? ` + ${carriedLanes.length} kept` : ""}\n`,
538 );
539 if (args.dryRun) {
540 process.stdout.write("dry run: nothing written\n");
541 return;
542 }
544 let stripped = bedTag;
545 for (const attr of ["data-fx-carve", "data-fx-chain", "data-automation"]) {
546 stripped = stripped.replace(new RegExp(`\\s${attr}="[^"]*"`, "i"), "");
547 }
548 // Inserted before the tag's own closing ">", which is the only place they can
549 // go: `stripped` is the opening tag alone, so appending would land outside it.
550 const nextTag = stripped.replace(/\/?>$/, (close) => `${written}${close}`);
551 if (nextTag === stripped) fail("attribute write produced no change — refusing to save");
552 writeFileSync(compPath, html.replace(bedTag, nextTag));
553 process.stdout.write(`wrote ${args.comp} (id="${bedEl.id}")\n`);
554}
556// Only run as a CLI. Guarded so the pure helpers above can be unit-tested by
557// importing this module (`skills/**/*.test.mjs`, run by `bun run test:skills`).
558if (isMainModule(import.meta.url)) {
559 await main();
560}