Skills › HyperFrames › media-use › bgm.mjs

bgm.mjs

257 lines · skills/media-use/audio/scripts/lib/bgm.mjs at a595755

Open on GitHub

1// bgm.mjs — background music for the media audio engine. Two routes, gated the

2// same way as TTS/SFX:

3//

4// retrieve (default when HeyGen is configured) — search HeyGen's music library

5// by mood, download the top track. Synchronous. assets/bgm/track.mp3.

6// generate (the alternative; the automatic choice when HeyGen is absent) —

7// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,

8// else local MusicGen (facebook/musicgen-small via transformers). Spawned

9// DETACHED so the engine can return while audio renders; the caller marks

10// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.

11//

12// Missing/failed BGM never blocks a render.

14import { spawn, spawnSync } from "node:child_process";

15import { existsSync, mkdirSync, openSync, closeSync, rmSync } from "node:fs";

16import { join } from "node:path";

17import { downloadTo, searchSounds } from "./heygen.mjs";

18import { agentWritePath } from "./media-record.mjs";

19import { pythonInvocation } from "./python.mjs";

21const r3 = (x) => Number(x.toFixed(3));

22const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";

24import { bgmDefaultVolume } from "./bgm-volume.mjs";

25export { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm-volume.mjs";

27const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];

28const BGM_PY_PROBE =

29 "import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";

30const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];

31const LYRIA_PY_PROBE = "import google.genai";

33function pyOk(probe) {

34 const { cmd, args } = pythonInvocation(["-c", probe]);

35 return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;

36}

37// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often

38// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently

39// no-ops (ENOENT) and the documented "auto-installed on demand" path never

40// actually installs. `-m pip` also guarantees the packages land in the SAME

41// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a

42// different Python installation than `python3` if more than one is on PATH.

43function pipInstall(deps) {

44 const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);

45 return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;

46}

48// ── retrieval (HeyGen music library) ──────────────────────────────────────────

49export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice, anomalies }) {

50 const q = query || "calm cinematic underscore";

51 const results = await searchSounds(q, "music", headers, { limit: 5 });

52 if (!results.length) return null;

53 const top = results[0];

54 const rel = agentWritePath(hyperframesDir, "assets/bgm/track.mp3", { anomalies });

55 await downloadTo(top.audio_url, join(hyperframesDir, rel));

56 return {

57 path: rel,

58 volume: bgmDefaultVolume(hasVoice),

59 query: q,

60 mode: "retrieve",

61 duration_s: typeof top.duration === "number" ? r3(top.duration) : null,

62 };

63}

65// ── mood inference (for the generate path's prompt) ──────────────────────────

66// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a

67// workflow adapter can build a rich prompt from its own narrative metadata; the

68// engine also calls it when generate has only a plain mood query.

69export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {

70 if (userPrompt) return userPrompt;

71 const b = String(blob).toLowerCase();

72 let base;

73 let bpm;

74 if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {

75 base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";

76 bpm = 100;

77 } else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {

78 base = "calm cinematic, soft strings, subtle piano, restrained percussion";

79 bpm = 92;

80 } else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {

81 base = "playful electronic, warm pads, light percussion";

82 bpm = 115;

83 } else {

84 base = "uplifting corporate tech, bright modern piano with synth pads";

85 bpm = 108;

86 }

87 const at = String(archetype).toLowerCase();

88 const ar = String(arc).toLowerCase();

89 if (/\bpas\b|pain.agitate|pain.+solve/.test(at))

90 return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;

91 if (/\bbab\b|before.after|future.pac|vision/.test(at))

92 return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;

93 if (/cascade|feature.benefit/.test(at))

94 return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;

95 if (/demo.loop|question.+answer/.test(at))

96 return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;

98 return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;

99 if (/excit|awe|power|triumph/.test(ar))

100 return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;

101 if (/trust|ease|clarity|reassur/.test(ar))

102 return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;

103 return `${base}, BPM ${bpm}, MAJOR`;

104}

105

106// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────

107// Returns a bgmMeta the caller folds into audio_meta:

108// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,

109// loop_count, pending:true } on success, or { disabled:true, reason }.

110export function generateBgmDetached({

111 prompt,

112 durationS,

113 hyperframesDir,

114 lyriaRecipe,

115 seedSeconds = 28,

116 hasVoice,

117 anomalies,

118}) {

119 const rel = agentWritePath(hyperframesDir, "assets/bgm/track.wav", { anomalies });

120 const abs = join(hyperframesDir, rel);

121 mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });

122 const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);

123 const targetS = Math.max(1, durationS);

124 const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };

125

126 const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);

127

128 // Make a backend runnable: prefer Lyria when configured (install google-genai

129 // on demand), else ensure local MusicGen deps. Installs are synchronous here —

130 // generation itself is detached, so the engine still returns promptly.

131 if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);

132 const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);

133 if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);

134

135 const fd = openSync(log, "w");

136 if (useLyria) {

137 const { cmd, args } = pythonInvocation([

138 lyriaRecipe,

139 "--output",

140 abs,

141 "--duration",

142 String(targetS),

143 "--prompt",

144 prompt,

145 ]);

146 rmSync(abs, { force: true }); // wait-bgm takes any file here as the finished track

147 const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });

148 proc.unref();

149 closeSync(fd);

150 return {

151 ...baseMeta,

152 mode: "detached-single",

153 provider: "lyria",

154 pid: proc.pid,

155 log,

156 target_duration_s: r3(targetS),

157 };

158 }

159

160 if (pyOk(BGM_PY_PROBE)) {

161 const seedS = Math.min(Math.max(seedSeconds, 10), 30);

162 const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;

163 const script = musicgenScript({ prompt, abs, targetS, seedS });

164 const { cmd, args } = pythonInvocation(["-c", script]);

165 rmSync(abs, { force: true });

166 const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });

167 proc.unref();

168 closeSync(fd);

169 return {

170 ...baseMeta,

171 mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",

172 provider: "musicgen",

173 pid: proc.pid,

174 log,

175 target_duration_s: r3(targetS),

176 seed_duration_s: seedS,

177 loop_count: loops,

178 };

179 }

180

181 closeSync(fd);

182 return {

183 disabled: true,

184 reason: lyriaConfigured

185 ? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`

186 : `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,

187 };

188}

189

190// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's

191// positional limit), then trim it down or crossfade-loop it up to the target.

192function musicgenScript({ prompt, abs, targetS, seedS }) {

193 return `

194import math, os, sys, traceback

195from pathlib import Path

196import numpy as np

197import soundfile as sf

198from transformers import MusicgenForConditionalGeneration, AutoProcessor

199

200prompt = ${JSON.stringify(prompt)}

201out_path = ${JSON.stringify(abs)}

202target_s = float(${targetS.toFixed(3)})

203seed_s = float(${seedS.toFixed(3)})

204token_rate = 50

205crossfade_s = 0.3

206

207def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):

208 n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)

209 n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)

210 if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")

211 if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")

212 return arr

213

214def loop_crossfade(seed, target_len, xf):

215 if seed.shape[0] >= target_len: return seed[:target_len]

216 xf = min(xf, seed.shape[0] // 2)

217 if xf < 1:

218 reps = int(math.ceil(target_len / seed.shape[0]))

219 return np.tile(seed, reps)[:target_len]

220 t = np.linspace(0.0, 1.0, xf, dtype="float32")

221 fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))

222 out = seed.copy()

223 while out.shape[0] < target_len:

224 tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in

225 out = np.concatenate([out[:-xf], tail + head, seed[xf:]])

226 return out[:target_len]

227

228try:

229 Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)

230 processor = AutoProcessor.from_pretrained("facebook/musicgen-small")

231 model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")

232 model.eval()

233 sr = int(model.config.audio_encoder.sampling_rate)

234 gen_s = min(seed_s, target_s)

235 tokens = max(1, int(math.ceil(gen_s * token_rate)))

236 print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)

237 inputs = processor(text=[prompt], padding=True, return_tensors="pt")

238 audio = model.generate(**inputs, max_new_tokens=tokens)

239 seed = audio[0, 0].detach().cpu().numpy().astype("float32")

240 peak = float(np.max(np.abs(seed)))

241 if peak > 1e-6: seed = seed * (0.89 / peak)

242 want = max(1, int(round(target_s * sr)))

243 if seed.shape[0] >= want:

244 final = seed[:want].copy()

245 else:

246 final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))

247 if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))

248 else: final = final[:want]

249 final = apply_fade(final, sr)

250 peak = float(np.max(np.abs(final)))

251 if peak > 1.0: final = final / peak

252 sf.write(out_path, final, sr)

253 print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)

254except Exception:

255 traceback.print_exc(); sys.exit(1)

256`;

257}