Skills › HyperFrames › media-use › audio.mjs
audio.mjs
310 lines · skills/media-use/audio/scripts/audio.mjs at a595755
1#!/usr/bin/env node
2// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
3// BGM + SFX. Callers do NOT vendor a copy: they write a neutral
4// `audio_request.json` (mapping their storyboard/scenes into it) and call:
5//
6// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
7//
8// TTS and BGM degrade on ONE switch — whether HeyGen is configured (credential
9// present, NOT the CLI). SFX needs no credential. This mirrors ../SKILL.md:
10//
11// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
12// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
13// SFX : Openverse (CC0 / CC BY) → (offline, rate limited, no match) bundled library
14//
15// ── audio_request.json (input) ────────────────────────────────────────────────
16// {
17// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider)
18// "lang": "en", "speed": 1.0,
19// "lines": [ // one TTS unit each; id joins back to the caller's model
20// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
21// ],
22// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
23// "query": "calm cinematic underscore", // mood for retrieval
24// "prompt": null, // full prompt for generation (else inferred)
25// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
26// }
27//
28// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
29// { tts_provider, voice_id,
30// bgm: { path, volume, mode, query?, duration_s? } | null,
31// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
32// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
33// sfx: [ { id, name, file, source, offset_s, duration_s, volume, license?, credit? } ],
34// total_duration_s }
35//
36// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
37// caller can do TTS+BGM early, then SFX later once cues exist). When BGM uses
38// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
39// before assembling.
41import { existsSync, mkdirSync, readFileSync } from "node:fs";
42import { dirname, join, resolve } from "node:path";
43import { fileURLToPath } from "node:url";
44import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
45import {
46 ffprobeDuration,
47 pickProvider,
48 resolveVoiceId,
49 synthesizeOne,
50 transcribeWav,
51 withWordIds,
52} from "./lib/tts.mjs";
53import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
54import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
55import { mapWithConcurrency } from "./lib/concurrency.mjs";
56import { openAudioMeta } from "./lib/audio-meta.mjs";
57import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
59const HERE = dirname(fileURLToPath(import.meta.url));
60const argv = process.argv.slice(2);
61const flag = (name, def) => {
62 const i = argv.indexOf(`--${name}`);
63 return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
64};
65const has = (name) => argv.includes(`--${name}`);
66const die = (m) => {
67 console.error(`✗ audio engine: ${m}`);
68 process.exit(1);
69};
70const r3 = (x) => Number(x.toFixed(3));
72// Two independent reports of an unbounded Promise.all over TTS lines
73// overwhelming a machine: one OOM'd 12/13 concurrent Kokoro TTS +
74// whisper-transcribe lines on a resource-constrained laptop, the other saw
75// 7/8 lines fail on first run (concurrent cold-start model loads) and pass on
76// retry once the model was cached. Kokoro/Whisper each load their own local
77// model per subprocess, so firing every line at once multiplies that cost by
78// the line count. mapWithConcurrency caps how many run at once — still
79// parallel, just bounded.
80const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENCY) || 4);
82const hyperframesDir = resolve(flag("hyperframes", "."));
83const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
84const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
85const sfxLibDir = resolve(flag("sfx-lib", BUNDLED_SFX_DIR));
86const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
87const onlyArg = flag("only", "tts,bgm,sfx");
88const only = new Set(
89 onlyArg
90 .split(",")
91 .map((s) => s.trim())
92 .filter(Boolean),
93);
94const providerOverride = flag("provider", null);
95const bgmModeOverride = flag("bgm-mode", null);
96const noBgm = has("no-bgm");
97const voiceOverride = flag("voice", null);
98const speedOverride = flag("speed", null);
99const langOverride = flag("lang", null);
100const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
102if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
103let request;
104try {
105 request = JSON.parse(readFileSync(requestPath, "utf8"));
106} catch (e) {
107 die(`audio_request.json parse: ${e.message}`);
108}
109const lines = Array.isArray(request.lines) ? request.lines : [];
110const lang = langOverride || request.lang || "en";
111const speedInput = speedOverride ?? request.speed ?? 1;
112const speed = Number(speedInput);
113if (!(speed > 0 && speed <= 3))
114 die(`speed must be above 0 and at most 3, got ${JSON.stringify(speedInput)}`);
116// ── env + HeyGen availability (the single switch) ─────────────────────────────
117loadEnvFromDir(hyperframesDir);
118const heygenOK = heygenCredential() !== null;
120// ── merge base: preserve sections not selected by --only ──────────────────────
121const audioMeta = openAudioMeta(outPath);
122const prev = audioMeta.value;
123const anomalies = [];
125// ── TTS ───────────────────────────────────────────────────────────────────────
126let voices = prev.voices ?? [];
127let ttsProvider = prev.tts_provider ?? null;
128let voiceId = prev.voice_id ?? null;
129if (only.has("tts") && lines.length) {
130 try {
131 ttsProvider = pickProvider(
132 providerOverride || (request.provider === "auto" ? null : request.provider),
133 );
134 } catch (e) {
135 die(e.message);
136 }
137 voiceId = await resolveVoiceId({
138 provider: ttsProvider,
139 userVoice: voiceOverride || request.voice,
140 lang,
141 });
142 console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
143 const paths = voicePaths(hyperframesDir, lines, anomalies);
144 const synthLine = async (line) => {
145 const id = String(line.id);
146 const text = String(line.text ?? "").trim();
147 if (!text) {
148 anomalies.push(`line ${id}: empty text — skipped`);
149 return null;
150 }
151 const rel = paths.get(id);
152 const abs = join(hyperframesDir, rel);
153 const { ok, words, error } = await synthesizeOne({
154 provider: ttsProvider,
155 text,
156 voiceId,
157 lang,
158 speed,
159 model: flag("tts-model", request.tts_model),
160 style: flag("style", line.style ?? request.style),
161 wavAbs: abs,
162 hyperframesDir,
163 });
164 if (!ok) {
165 anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
166 return null;
167 }
168 let wordArr = words; // heygen: native; else transcribe
169 if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
170 const dur = ffprobeDuration(abs);
171 if (!isFinite(dur) || dur <= 0) {
172 anomalies.push(`line ${id}: bad voice duration — omitted`);
173 return null;
174 }
175 return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
176 };
177 const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
178 voices = results.filter(Boolean);
179 for (const v of voices)
180 console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
181}
182const hasVoice = voices.length > 0;
183const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
185// ── BGM ─────────────────────────────────────────────────────────────────────
186let bgm = prev.bgm ?? null;
187const bgmFields = {
188 bgm_pending: prev.bgm_pending ?? false,
189 bgm_provider: prev.bgm_provider ?? null,
190 bgm_pid: prev.bgm_pid ?? null,
191 bgm_log: prev.bgm_log ?? null,
192 bgm_mode: prev.bgm_mode ?? null,
193 bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
194 bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
195 bgm_loop_count: prev.bgm_loop_count ?? null,
196};
197if (only.has("bgm")) {
198 bgm = null;
199 Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
200 // Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
201 // "retrieve" means retrieve-or-nothing — it never silently starts a detached
202 // generate (a caller with no wait-bgm step must not get a pending job it
203 // can't await). Only the UNSET/auto default picks generate
204 // when HeyGen is absent.
205 const explicitMode = bgmModeOverride || request.bgm?.mode || null;
206 let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
207 if (mode === "retrieve" && !heygenOK) {
208 anomalies.push(
209 "bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
210 );
211 mode = "none";
212 }
214 if (mode === "none") {
215 console.error(`· bgm: disabled`);
216 } else if (mode === "retrieve") {
217 try {
218 bgm = await retrieveBgm({
219 query: request.bgm?.query,
220 headers: heygenAuthHeaders(),
221 hyperframesDir,
222 hasVoice,
223 anomalies,
224 });
225 if (bgm) {
226 bgmFields.bgm_provider = "heygen";
227 bgmFields.bgm_mode = "retrieve";
228 console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
229 } else {
230 anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
231 }
232 } catch (e) {
233 anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
234 }
235 } else {
236 // generate
237 const prompt = inferBgmPrompt({
238 userPrompt: request.bgm?.prompt,
239 blob: request.bgm?.blob || request.bgm?.query,
240 archetype: request.bgm?.archetype,
241 arc: request.bgm?.arc,
242 });
243 const gen = generateBgmDetached({
244 prompt,
245 durationS: totalDuration || 30,
246 hyperframesDir,
247 lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
248 seedSeconds,
249 hasVoice,
250 anomalies,
251 });
252 if (gen.disabled) {
253 anomalies.push(`bgm: ${gen.reason}`);
254 } else {
255 bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
256 bgmFields.bgm_pending = true;
257 bgmFields.bgm_provider = gen.provider;
258 bgmFields.bgm_pid = gen.pid;
259 bgmFields.bgm_log = gen.log;
260 bgmFields.bgm_mode = gen.mode;
261 bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
262 bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
263 bgmFields.bgm_loop_count = gen.loop_count ?? null;
264 console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
265 }
266 }
267}
269// ── SFX ─────────────────────────────────────────────────────────────────────
270let sfx = prev.sfx ?? [];
271if (only.has("sfx")) {
272 const cues = lines.flatMap((l) =>
273 (Array.isArray(l.sfx) ? l.sfx : [])
274 .map((name) => ({ id: String(l.id), name: String(name).trim() }))
275 .filter((c) => c.name),
276 );
277 const res = await resolveSfx({ cues, hyperframesDir, sfxLibDir });
278 sfx = res.sfx;
279 anomalies.push(...res.anomalies);
280 const openverse = sfx.filter((s) => s.source === "openverse").length;
281 console.error(
282 `· sfx: ${sfx.length} cue(s) resolved (${openverse} Openverse, ${sfx.length - openverse} bundled library)`,
283 );
284}
286// ── write audio_meta.json ─────────────────────────────────────────────────────
287const meta = {
288 tts_provider: ttsProvider,
289 voice_id: voiceId,
290 bgm,
291 ...bgmFields,
292 voices,
293 sfx,
294 total_duration_s: totalDuration,
295};
296mkdirSync(dirname(outPath), { recursive: true });
297audioMeta.write(meta);
298const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx });
299anomalies.push(...recordInManifest(hyperframesDir, written));
301console.log(`✓ audio engine → ${outPath}`);
302console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
303console.log(
304 ` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
305);
306console.log(` total voice duration: ${totalDuration}s`);
307if (anomalies.length) {
308 console.log(`\nanomalies (non-fatal):`);
309 for (const a of anomalies) console.log(` - ${a}`);
310}