Skill Tracer
Skills › HyperFrames › media-use › tts.mjs

tts.mjs

396 lines · skills/media-use/audio/scripts/lib/tts.mjs at a595755

Open on GitHub
1import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
2import { refuseUnderLimit } from "../../../scripts/lib/credit-limit.mjs";
3// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
4// auto-detected from env, is the one documented in ../SKILL.md:
5//
6// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.hyperframes.
7// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
8// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
9// in the same call, so no separate transcribe pass.
10// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
11// word timings → caller chains transcribeWav().
12// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
13// CLI. No word timings → caller chains transcribeWav().
14//
15// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
16// by the CLI — see the note above.
18import { spawn, spawnSync } from "node:child_process";
19import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
20import { tmpdir } from "node:os";
21import { dirname, join } from "node:path";
22import { heygenAuthHeaders, heygenCredential, heygenJSON, heygenMessage } from "./heygen.mjs";
23import { pythonInvocation } from "./python.mjs";
24import { synthesizeGemini } from "./gemini-tts.mjs";
25import { geminiConfigured } from "./gemini-auth.mjs";
27// ── provider detection ────────────────────────────────────────────────────────
28export function heygenAvailable() {
29 return heygenCredential() !== null;
31export function elevenlabsAvailable() {
32 if (!process.env.ELEVENLABS_API_KEY) return false;
33 const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
34 const r = spawnSync(cmd, args, {
35 stdio: "ignore",
36 });
37 return r.status === 0;
40// First available provider wins; an explicit choice is honored (and validated).
41export function pickProvider(userProvider) {
42 if (userProvider) {
43 if (!["heygen", "elevenlabs", "kokoro", "gemini"].includes(userProvider))
44 throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro | gemini)`);
45 if (userProvider === "gemini" && !geminiConfigured())
46 throw new Error(
47 "provider=gemini needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
48 );
49 // heygenAuthHeaders owns the reason and its fix: no credential, or a credentials path that cannot be read.
50 if (userProvider === "heygen" && !heygenAvailable()) heygenAuthHeaders();
51 if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
52 throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
53 return userProvider;
54 }
55 return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
58// ── voice resolution ──────────────────────────────────────────────────────────
59// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
60// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
61// their own defaults.
62export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
63 if (userVoice) return userVoice;
64 if (provider === "gemini") return "Kore";
65 if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
66 if (provider === "kokoro") {
67 if (lang === "en") return "am_michael";
68 throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
69 }
70 // heygen — pin a fixed English default so the choice is deterministic. The old
71 // "first English voice the API returns" drifts whenever HeyGen re-sorts the
72 // public catalog. Marcia (mature, low female). Override with --voice / request.voice.
73 if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
74 // Non-English: no fixed default — fall back to the first matching catalog voice.
75 const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
76 headers: heygenAuthHeaders(),
77 });
78 const voices = payload.data ?? payload.voices ?? [];
79 const pick = voices.find((v) => v.language === "English") ?? voices[0];
80 if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
81 return pick.voice_id;
84// ── helpers ─────────────────────────────────────────────────────────────────
85export function withWordIds(words) {
86 return (words ?? []).map((w, i) => ({
87 id: `w${i}`,
88 text: w.text,
89 start: w.start,
90 end: w.end,
91 }));
94// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
95// though it exits non-zero with no output requested. Parsing pulled out as
96// a pure function so the ENOENT fallback below can be tested without
97// depending on whether ffprobe/ffmpeg are actually installed on the
98// machine running the tests.
99export function parseFfmpegDurationBanner(stderrText) {
100 const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
101 if (!match) return NaN;
102 const [, hours, minutes, seconds] = match;
103 return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
106// Some "essentials"-style ffmpeg distributions (common on Windows) ship
107// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
108// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
109// already-successfully-synthesized TTS line, rather than "the tool for
110// measuring it is missing".
111function ffmpegDurationFallback(absPath) {
112 const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
113 return parseFfmpegDurationBanner(r.stderr);
116export function ffprobeDuration(absPath) {
117 const r = spawnSync(
118 "ffprobe",
119 ["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", "--", absPath],
120 { encoding: "utf8" },
121 );
122 if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
123 if (r.status !== 0) return NaN;
124 return parseFloat(String(r.stdout).trim());
127export function resolveNpxCliFromNpmExecPath(
128 npmExecPath = process.env.npm_execpath,
129 pathExists = existsSync,
130) {
131 if (!npmExecPath) return null;
132 const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
133 const npxCliPath =
134 fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
135 return pathExists(npxCliPath) ? npxCliPath : null;
138export function resolveNpxCliPath(
139 npmExecPath = process.env.npm_execpath,
140 nodeExecPath = process.env.npm_node_execpath || process.execPath,
141 pathExists = existsSync,
142) {
143 const fromNpm = resolveNpxCliFromNpmExecPath(npmExecPath, pathExists);
144 if (fromNpm) return fromNpm;
145 const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js");
146 return pathExists(besideNode) ? besideNode : null;
149export function resolveSpawnCommand(
150 cmd,
151 args,
152 opts = {},
153 platform = process.platform,
154 env = process.env,
155 pathExists = existsSync,
156) {
157 if (cmd !== "npx" || platform !== "win32") {
158 return { cmd, args, opts: { stdio: "ignore", ...opts } };
159 }
161 // On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
162 // Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
163 // node, preserving request-provided values as argv data instead of shell text.
164 const nodeExecPath = env.npm_node_execpath || process.execPath;
165 const npxCliPath = resolveNpxCliPath(env.npm_execpath, nodeExecPath, pathExists);
166 if (!npxCliPath) return null;
167 return {
168 cmd: nodeExecPath,
169 args: [npxCliPath, ...args.map((arg) => String(arg))],
170 opts: { stdio: "ignore", windowsHide: true, ...opts },
171 };
174// `platform`/`spawnFn` params (default process.platform / the real spawn)
175// exist so tests can exercise the win32 branch without mocking node:child_process
176// (its ESM exports are non-configurable, so mock.method can't patch it).
177// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
178let _warnedNpxResolution = false;
179/** Test-only: reset the one-shot npx-resolution warning latch. */
180export function _resetNpxResolutionWarnForTests() {
181 _warnedNpxResolution = false;
184export function spawnP(
185 cmd,
186 args,
187 opts = {},
188 platform = process.platform,
189 spawnFn = spawn,
190 env = process.env,
191 pathExists = existsSync,
192) {
193 const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
194 if (!resolved) {
195 // resolveSpawnCommand only returns null for the npx-on-win32 case where
196 // neither npm's configured CLI nor the beside-node fallback exists. Without
197 // this, every call silently returns status:-1 and stdio:"ignore" hides why.
198 if (!_warnedNpxResolution) {
199 _warnedNpxResolution = true;
200 const reason = env.npm_execpath
201 ? `npm_execpath (${env.npm_execpath}) and the beside-node npm fallback could not be found`
202 : "npm_execpath is unset and the beside-node npm fallback could not be found";
203 console.error(
204 `[media-use] Cannot run "${cmd}" on Windows: ${reason}. ` +
205 `Every "${cmd}" call is being skipped. Install npm with Node, or run via ` +
206 `\`npx\`/\`npm run\` with a valid npm_execpath.`,
207 );
208 }
209 return Promise.resolve({ status: -1 });
210 }
211 return new Promise((resolve) => {
212 const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
213 p.on("exit", (code) => resolve({ status: code ?? -1 }));
214 p.on("error", () => resolve({ status: -1 }));
215 });
218// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
219function transcodeToWav(bytes, destWav) {
220 const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
221 const tmp = join(td, "a.mp3");
222 writeFileSync(tmp, bytes);
223 mkdirSync(dirname(destWav), { recursive: true });
224 const ff = spawnSync(
225 "ffmpeg",
226 ["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
227 { stdio: "ignore" },
228 );
229 rmSync(td, { recursive: true, force: true });
230 return ff.status === 0 && existsSync(destWav);
233const ELEVENLABS_PY = `
234import os, sys
235from elevenlabs.client import ElevenLabs
236from elevenlabs import save
237client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
238text = open(sys.argv[1], encoding="utf-8").read()
239audio = client.text_to_speech.convert(
240 text=text, voice_id=sys.argv[2],
241 model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
243save(audio, sys.argv[3])
244`;
246// ── synthesize one line ───────────────────────────────────────────────────────
247// Writes wav at wavAbs. Returns { ok, words, error } — words is the raw
248// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
249// (caller must transcribeWav). Never throws; failures return { ok:false, error }
250// where `error` states WHY (so the caller can surface it, not a bare "TTS failed").
251export async function synthesizeOne({
252 provider,
253 text,
254 voiceId,
255 lang = "en",
256 speed = 1.0,
257 model,
258 style,
259 wavAbs,
260 hyperframesDir,
261}) {
262 if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
263 if (provider === "gemini")
264 return synthesizeGemini({ text, voiceId, model, style, speed, wavAbs });
265 if (provider === "elevenlabs") {
266 // The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
267 // and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
268 // so on a fresh project (no assets/voice/ yet) the save fails and the line is
269 // silently dropped as "TTS failed - omitted". Create it first, like the other
270 // providers do. Guarded so a mkdir failure (EACCES/EROFS) returns
271 // { ok:false } like the rest of this branch rather than throwing (the
272 // function's contract is "never throws; failures return { ok:false }").
273 try {
274 mkdirSync(dirname(wavAbs), { recursive: true });
275 } catch {
276 return { ok: false, words: null };
277 }
278 return withTmpText(text, async (textPath) => {
279 const { cmd, args } = pythonInvocation(["-c", ELEVENLABS_PY, textPath, voiceId, wavAbs]);
280 const r = await spawnP(cmd, args, {});
281 return synthResult(r, wavAbs, "elevenlabs (python)");
282 });
283 }
284 // kokoro — via the published CLI; --output is relative to the project dir.
285 const wavRel = relTo(hyperframesDir, wavAbs);
286 return withTmpText(text, async (textPath) => {
287 const args = ["hyperframes", "tts", textPath, "--voice", voiceId, "--output", wavRel];
288 if (lang !== "en") args.push("--lang", lang);
289 if (speed !== 1) args.push("--speed", String(speed));
290 const r = await spawnP("npx", args, { cwd: hyperframesDir });
291 return synthResult(r, wavAbs, "kokoro (npx hyperframes tts)");
292 });
295// Shape a spawn result into { ok, words, error }, naming why on failure so the
296// caller surfaces it instead of a bare "TTS failed".
297export function synthResult(r, wavAbs, label) {
298 if (r.status === 0 && existsSync(wavAbs)) return { ok: true, words: null };
299 const why =
300 r.status !== 0 ? `${label} exited with status ${r.status}` : `${label} produced no wav file`;
301 return { ok: false, words: null, error: why };
304// `deps` is injectable for tests; production uses the real network/ffmpeg impls.
305// Every failure path returns an `error` string so the caller can surface WHY a
306// line was dropped instead of the bare "TTS failed" that hid the real cause
307// (e.g. an HTTP 402 plan_upgrade_required thrown by heygenJSON was swallowed).
308export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, deps = {}) {
309 const requestJSON = deps.heygenJSON ?? heygenJSON;
310 const authHeaders = deps.heygenAuthHeaders ?? heygenAuthHeaders;
311 const fetchImpl = deps.fetch ?? fetch;
312 const transcode = deps.transcodeToWav ?? transcodeToWav;
313 try {
314 refuseUnderLimit(deps.projectDir ?? process.cwd(), "a HeyGen voice", "voice");
315 const body = { text, voice_id: voiceId, speed };
316 if (lang !== "en") body.language = lang;
317 const payload = await requestJSON(`/voices/speech`, {
318 method: "POST",
319 headers: authHeaders(),
320 body,
321 });
322 const inner = payload.data ?? payload;
323 if (!inner.audio_url) {
324 return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" };
325 }
326 const res = await fetchMedia(inner.audio_url, { fetchImpl });
327 if (!res.ok) {
328 return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` };
329 }
330 const bytes = Buffer.from(await res.arrayBuffer());
331 // .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
332 // engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
333 if (wavAbs.endsWith(".wav")) {
334 if (!transcode(bytes, wavAbs)) {
335 return {
336 ok: false,
337 words: null,
338 error: "wav transcode failed (ffmpeg)",
339 };
340 }
341 } else {
342 mkdirSync(dirname(wavAbs), { recursive: true });
343 writeFileSync(wavAbs, bytes);
344 }
345 const words = Array.isArray(inner.word_timestamps)
346 ? inner.word_timestamps
347 .filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
348 .filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
349 .map((w) => ({ text: w.word, start: w.start, end: w.end }))
350 : [];
351 return { ok: true, words };
352 } catch (e) {
353 return { ok: false, words: null, error: heygenMessage(e) };
354 }
357// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
358// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
359// --dir so parallel scenes don't collide on transcript.json.
360export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
361 const model = lang === "en" ? "small.en" : "small";
362 const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
363 const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
364 if (lang !== "en") args.push("--language", lang);
365 const r = await spawnP("npx", args, { cwd: hyperframesDir });
366 let words = null;
367 if (r.status === 0) {
368 const src = join(td, "transcript.json");
369 if (existsSync(src)) {
370 try {
371 const arr = JSON.parse(readFileSync(src, "utf8"));
372 if (Array.isArray(arr) && arr.length) words = arr;
373 } catch {}
374 }
375 }
376 rmSync(td, { recursive: true, force: true });
377 return words;
380// ── tiny local utils ──────────────────────────────────────────────────────────
381async function withTmpText(text, run) {
382 const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
383 try {
384 const p = join(td, "line.txt");
385 writeFileSync(p, text);
386 return await run(p);
387 } finally {
388 // Cleanup must preserve the synthesis result or the original write/spawn error.
389 try {
390 rmSync(td, { recursive: true, force: true });
391 } catch {}
392 }
394function relTo(base, abs) {
395 return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;