Skills › HyperFrames › media-use › sfx.mjs
sfx.mjs
196 lines · skills/media-use/audio/scripts/lib/sfx.mjs at a595755
1// sfx.mjs — sound effects for the media audio engine. Each cue is searched on
2// Openverse (CC0 / CC BY only, see openverse.mjs). A cue Openverse cannot answer,
3// and every cue after Openverse is unreachable or rate limited, resolves against
4// the bundled 21-file library (assets/sfx/manifest.json) instead.
5//
6// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
7// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
9import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
10import { dirname, join } from "node:path";
11import { fileURLToPath } from "node:url";
12import { downloadTo } from "./heygen.mjs";
13import { OpenverseUnavailableError, searchOpenverseSfx, writeSfxCredit } from "./openverse.mjs";
14import { latestRecordFor } from "../../../scripts/lib/manifest.mjs";
15import { agentWritePath } from "./media-record.mjs";
17export const BUNDLED_SFX_DIR = join(
18 dirname(fileURLToPath(import.meta.url)),
19 "..",
20 "..",
21 "assets",
22 "sfx",
23);
24const SFX_VOLUME = 0.35;
25const slug = (s) =>
26 s
27 .toLowerCase()
28 .replace(/[^a-z0-9]+/g, "-")
29 .replace(/^-+|-+$/g, "")
30 .slice(0, 40) || "x";
31const r3 = (x) => Number(x.toFixed(3));
33// Lookups by manifest key, file basename, and slug of either, so a cue can name
34// "whoosh", "whoosh.mp3", or "ui click" (→ slug match). { error } when unusable.
35function loadBundledLibrary(sfxLibDir) {
36 const manifestPath = join(sfxLibDir, "manifest.json");
37 if (!existsSync(manifestPath)) return { error: `no bundled SFX library at ${sfxLibDir}` };
38 let manifest;
39 try {
40 manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
41 } catch (e) {
42 return { error: `bundled SFX manifest parse failed (${e.message})` };
43 }
44 const byKey = new Map();
45 for (const [key, entry] of Object.entries(manifest)) {
46 if (!entry?.file || !isFinite(entry.duration)) continue;
47 const rec = { key, file: entry.file, duration: entry.duration };
48 byKey.set(key, rec);
49 byKey.set(entry.file, rec);
50 byKey.set(slug(key), rec);
51 byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
52 }
53 return { byKey };
54}
56// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
57// { sfx: [{ id, name, file, source, offset_s, duration_s, volume, license?, credit? }], anomalies }.
58export async function resolveSfx({ cues, hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
59 const sfx = [];
60 const anomalies = [];
61 const destDir = join(hyperframesDir, "assets", "sfx");
62 // Each effect's file this run: one effect named twice gets one file, and two effects never share one.
63 const fileFor = new Map();
64 const taken = () => new Set(fileFor.values());
65 // One Openverse sound per effect name this run, so a repeated cue costs no second request.
66 const openverseFor = new Map();
67 let openverseDown = false;
68 let library;
70 // Dedupe identical (id,name) cues — the same effect named twice in one line
71 // downloads/copies once.
72 const seen = new Set();
73 const uniq = cues.filter((c) => {
74 const k = `${c.id}:${c.name}`;
75 if (seen.has(k)) return false;
76 seen.add(k);
77 return true;
78 });
80 async function fromOpenverse({ id, name }) {
81 const key = `openverse:${slug(name)}`;
82 let entry = openverseFor.get(key);
83 if (!entry) {
84 const sound = await searchOpenverseSfx(name);
85 if (!sound) return null;
86 const file = agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}${sound.ext}`, {
87 anomalies,
88 taken: taken(),
89 });
90 try {
91 await downloadTo(sound.url, join(hyperframesDir, file));
92 } catch (e) {
93 anomalies.push(`sfx "${name}" (id ${id}): Openverse download failed — ${e.message}`);
94 return null;
95 }
96 fileFor.set(key, file);
97 writeSfxCredit(join(destDir, "CREDITS.md"), file, sound);
98 entry = {
99 file,
100 source: "openverse",
101 duration_s: r3(sound.durationS),
102 license: sound.license,
103 credit: sound.credit,
104 };
105 openverseFor.set(key, entry);
106 }
107 return { id, name, ...entry, offset_s: 0, volume: SFX_VOLUME };
108 }
110 function fromBundled({ id, name }) {
111 library ??= loadBundledLibrary(sfxLibDir);
112 if (library.error) {
113 anomalies.push(`sfx "${name}" (id ${id}): ${library.error} — skipped`);
114 return null;
115 }
116 const { byKey } = library;
117 const hit = byKey.get(name) ?? byKey.get(slug(name));
118 if (!hit) {
119 const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
120 anomalies.push(
121 `sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
122 );
123 return null;
124 }
125 mkdirSync(destDir, { recursive: true });
126 const src = join(sfxLibDir, hit.file);
127 const bytes = existsSync(src) ? readFileSync(src) : null;
128 const isLibraryCopy = (rel) =>
129 bytes &&
130 existsSync(join(hyperframesDir, rel)) &&
131 readFileSync(join(hyperframesDir, rel)).equals(bytes);
132 const destRel =
133 fileFor.get(hit.file) ??
134 agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, {
135 anomalies,
136 taken: taken(),
137 // An unrecorded copy of the library file is one an engine run made before the manifest had it.
138 reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel),
139 });
140 const dest = join(hyperframesDir, destRel);
141 // Some installs of the skill ship manifest.json without the mp3s. An entry
142 // pointing at a file we never copied would silently drop downstream, so
143 // skip the cue with a loud anomaly instead.
144 if (bytes) {
145 if (!isLibraryCopy(destRel)) copyFileSync(src, dest);
146 } else if (!existsSync(dest)) {
147 anomalies.push(
148 `sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
149 `library (${sfxLibDir}) — skipped. Ask the person to reinstall the media-use ` +
150 `skill to restore assets/sfx/*.mp3.`,
151 );
152 return null;
153 } // else the engine's earlier copy at dest stands in for the file this install lacks
154 fileFor.set(hit.file, destRel);
155 return {
156 id,
157 name,
158 file: destRel,
159 source: "local",
160 offset_s: 0,
161 duration_s: r3(hit.duration),
162 volume: SFX_VOLUME,
163 };
164 }
166 for (const cue of uniq) {
167 let entry = null;
168 if (!openverseDown) {
169 try {
170 entry = await fromOpenverse(cue);
171 } catch (e) {
172 if (!(e instanceof OpenverseUnavailableError)) throw e;
173 openverseDown = true;
174 anomalies.push(`${e.message}: the remaining SFX cues use the bundled library`);
175 }
176 }
177 entry ??= fromBundled(cue);
178 if (entry) sfx.push(entry);
179 }
180 return { sfx, anomalies };
181}
183/**
184 * One sound effect for `query`, saved under the project's assets/sfx/ (the entry point for hosts).
185 * { sound: { file, source, duration_s, license?, credit? } | null, anomalies }.
186 */
187export async function findSoundEffect(query, { hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
188 const { sfx, anomalies } = await resolveSfx({
189 cues: [{ id: "1", name: query }],
190 hyperframesDir,
191 sfxLibDir,
192 });
193 if (!sfx.length) return { sound: null, anomalies };
194 const { file, source, duration_s, license, credit } = sfx[0];
195 return { sound: { file, source, duration_s, ...(license && { license, credit }) }, anomalies };
196}