Skills › HyperFrames › media-use › extract-audio-data.py
extract-audio-data.py
199 lines · skills/media-use/audio/scripts/extract-audio-data.py at a595755
1#!/usr/bin/env python3
2"""
3Extract per-frame audio visualization data from an audio or video file.
5Outputs JSON with RMS amplitude and frequency band data at the target FPS,
6ready to embed in a HyperFrames composition.
8Usage:
9 python extract-audio-data.py input.mp3 -o audio-data.json
10 python extract-audio-data.py input.mp4 --fps 30 --bands 16 -o audio-data.json
12Requirements:
13 - Python 3.9+
14 - ffmpeg (for decoding audio)
15 - numpy (pip install numpy)
16"""
18import argparse
19import json
20import subprocess
21import sys
23import numpy as np
25# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
26# every platform; say so rather than depending on the console's code page. Carry
27# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
28# stderr "backslashreplace" so the diagnostic path can never itself raise.
29for _stream in (sys.stdout, sys.stderr):
30 if hasattr(_stream, "reconfigure"):
31 _stream.reconfigure(encoding="utf-8", errors=_stream.errors)
33# ---------------------------------------------------------------------------
34# FFT parameters
35#
36# A 4096-sample window gives ~10.8 Hz per bin at 44100Hz — enough to resolve
37# low-frequency bands cleanly. The per-frame audio slice (44100/30 = 1470
38# samples at 30fps) is too small and causes low bands to map to the same bins.
39#
40# Frequency range 30Hz–16kHz covers the useful range for music. Below 30Hz is
41# sub-bass most speakers can't reproduce; above 16kHz is noise/harmonics that
42# don't contribute to perceived rhythm or melody.
43# ---------------------------------------------------------------------------
45SAMPLE_RATE = 44100
46FFT_SIZE = 4096
47MIN_FREQ = 30.0
48MAX_FREQ = 16000.0
51def decode_audio(path: str) -> np.ndarray:
52 """Decode audio to mono float32 samples via ffmpeg."""
53 cmd = [
54 "ffmpeg", "-i", path,
55 "-vn", "-ac", "1", "-ar", str(SAMPLE_RATE),
56 "-f", "s16le", "-acodec", "pcm_s16le",
57 "-loglevel", "error",
58 "pipe:1",
59 ]
60 result = subprocess.run(cmd, capture_output=True)
61 if result.returncode != 0:
62 # Decoding ffmpeg's diagnostics strictly makes the reporter the thing that
63 # crashes: a Windows ffmpeg emits cp1252 bytes, and UnicodeDecodeError here
64 # would bury the actual failure it was trying to report.
65 print(f"ffmpeg error: {result.stderr.decode('utf-8', errors='replace')}", file=sys.stderr)
66 sys.exit(1)
67 return np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) / 32768.0
70def compute_band_edges(n_bands: int) -> np.ndarray:
71 """Logarithmically-spaced frequency band edges from MIN_FREQ to MAX_FREQ."""
72 return np.array([
73 MIN_FREQ * (MAX_FREQ / MIN_FREQ) ** (i / n_bands)
74 for i in range(n_bands + 1)
75 ])
78def compute_fft_bands(
79 windowed: np.ndarray, freq_per_bin: float, n_bins: int,
80 band_edges: np.ndarray, n_bands: int,
81) -> np.ndarray:
82 """Compute peak magnitude in logarithmically-spaced frequency bands."""
83 magnitudes = np.abs(np.fft.rfft(windowed))
85 bands = np.zeros(n_bands)
86 for b in range(n_bands):
87 low_bin = max(0, int(band_edges[b] / freq_per_bin))
88 high_bin = min(n_bins, int(band_edges[b + 1] / freq_per_bin))
89 if high_bin <= low_bin:
90 high_bin = low_bin + 1
91 # Clamp to valid range to avoid empty slices
92 low_bin = min(low_bin, n_bins - 1)
93 high_bin = min(high_bin, n_bins)
94 bands[b] = np.max(magnitudes[low_bin:high_bin])
96 return bands
99def extract(path: str, fps: int, n_bands: int) -> dict:
100 """Extract per-frame audio data."""
101 print(f"Decoding audio from {path}...", file=sys.stderr)
102 samples = decode_audio(path)
103 duration = len(samples) / SAMPLE_RATE
104 frame_step = SAMPLE_RATE // fps
105 total_frames = int(duration * fps)
107 print(f"Duration: {duration:.1f}s, {total_frames} frames at {fps}fps", file=sys.stderr)
108 print(f"FFT window: {FFT_SIZE} samples ({SAMPLE_RATE / FFT_SIZE:.1f} Hz/bin)", file=sys.stderr)
109 print(f"Frequency range: {MIN_FREQ:.0f}-{MAX_FREQ:.0f} Hz, {n_bands} bands", file=sys.stderr)
111 # Precompute constants
112 hann = np.hanning(FFT_SIZE)
113 band_edges = compute_band_edges(n_bands)
114 freq_per_bin = SAMPLE_RATE / FFT_SIZE
115 n_bins = FFT_SIZE // 2 + 1
116 half_fft = FFT_SIZE // 2
118 # Pass 1: extract raw values
119 rms_values = np.zeros(total_frames)
120 band_values = np.zeros((total_frames, n_bands))
122 for f in range(total_frames):
123 # RMS from the frame's audio slice
124 rms_start = f * frame_step
125 rms_end = rms_start + frame_step
126 frame_slice = samples[rms_start:min(rms_end, len(samples))]
127 if len(frame_slice) > 0:
128 rms_values[f] = np.sqrt(np.mean(frame_slice ** 2))
130 # FFT from a centered 4096-sample window
131 center = rms_start + frame_step // 2
132 win_start = center - half_fft
133 win_end = center + half_fft
135 if win_start >= 0 and win_end <= len(samples):
136 window = samples[win_start:win_end] * hann
137 else:
138 # Zero-pad at edges
139 padded = np.zeros(FFT_SIZE)
140 src_start = max(0, win_start)
141 src_end = min(len(samples), win_end)
142 dst_start = src_start - win_start
143 dst_end = dst_start + (src_end - src_start)
144 padded[dst_start:dst_end] = samples[src_start:src_end]
145 window = padded * hann
147 band_values[f] = compute_fft_bands(window, freq_per_bin, n_bins, band_edges, n_bands)
149 # Pass 2: normalize
150 peak_rms = rms_values.max() if total_frames > 0 else 1.0
151 if peak_rms > 0:
152 rms_values /= peak_rms
154 # Per-band normalization so treble is visible alongside louder bass
155 band_peaks = band_values.max(axis=0)
156 band_peaks[band_peaks == 0] = 1.0
157 band_values /= band_peaks
159 # Build output
160 frames = []
161 for f in range(total_frames):
162 frames.append({
163 "time": round(f / fps, 4),
164 "rms": round(float(rms_values[f]), 4),
165 "bands": [round(float(b), 4) for b in band_values[f]],
166 })
168 return {
169 "duration": round(duration, 4),
170 "fps": fps,
171 "bands": n_bands,
172 "totalFrames": total_frames,
173 "frames": frames,
174 }
177def main():
178 parser = argparse.ArgumentParser(description="Extract per-frame audio visualization data")
179 parser.add_argument("input", help="Audio or video file")
180 parser.add_argument("-o", "--output", default="audio-data.json", help="Output JSON path")
181 parser.add_argument("--fps", type=int, default=30, help="Frames per second (default: 30)")
182 parser.add_argument("--bands", type=int, default=16, help="Number of frequency bands (default: 16)")
183 args = parser.parse_args()
185 if args.fps < 1:
186 parser.error("--fps must be at least 1")
187 if args.bands < 1:
188 parser.error("--bands must be at least 1")
190 data = extract(args.input, args.fps, args.bands)
192 with open(args.output, "w", encoding="utf-8") as f:
193 json.dump(data, f)
195 print(f"Wrote {args.output} ({data['totalFrames']} frames, {data['bands']} bands)", file=sys.stderr)
198if __name__ == "__main__":
199 main()