Skills › HyperFrames › media-use › manifest.mjs
manifest.mjs
282 lines · skills/media-use/scripts/lib/manifest.mjs at a595755
1import {
2 readFileSync,
3 appendFileSync,
4 mkdirSync,
5 existsSync,
6 readdirSync,
7 openSync,
8 closeSync,
9 writeFileSync,
10 rmSync,
11 statSync,
12} from "node:fs";
13import { randomBytes } from "node:crypto";
14import { basename, join } from "node:path";
16const MANIFEST_FILE = "manifest.jsonl";
17const INDEX_FILE = "index.md";
19const TYPE_DIRS = {
20 bgm: "audio/bgm",
21 sfx: "audio/sfx",
22 voice: "audio/voice",
23 image: "images",
24 icon: "images",
25 logo: "images",
26 brand: "images",
27 video: "video",
28 grade: "luts",
29 lut: "luts",
30 recipe: "recipes",
31};
33export function mediaDir(projectDir) {
34 return join(projectDir, ".media");
35}
37export function manifestPath(projectDir) {
38 return join(mediaDir(projectDir), MANIFEST_FILE);
39}
41export function indexPath(projectDir) {
42 return join(mediaDir(projectDir), INDEX_FILE);
43}
45export function typeSubdir(type) {
46 const sub = TYPE_DIRS[type];
47 if (!sub) throw new Error(`unknown media type: ${type}`);
48 return sub;
49}
51export function typeDirPath(projectDir, type) {
52 return join(mediaDir(projectDir), typeSubdir(type));
53}
55export function readManifest(projectDir) {
56 const p = manifestPath(projectDir);
57 if (!existsSync(p)) return [];
58 const raw = readFileSync(p, "utf8");
59 const records = [];
60 for (const line of raw.split(/\r?\n/)) {
61 const trimmed = line.trim();
62 if (!trimmed) continue;
63 try {
64 records.push(JSON.parse(trimmed));
65 } catch {
66 // ponytail: skip malformed lines, don't crash
67 }
68 }
69 return records;
70}
72export function appendRecord(projectDir, record) {
73 const dir = mediaDir(projectDir);
74 mkdirSync(dir, { recursive: true });
75 const typeDir = typeDirPath(projectDir, record.type);
76 mkdirSync(typeDir, { recursive: true });
78 const p = manifestPath(projectDir);
79 const line = JSON.stringify(record) + "\n";
80 appendFileSync(p, line);
81}
83/** Sources that mean the agent made or fetched the file; any other file is the person's own. */
84export const AGENT_SOURCES = ["generated", "search", "bundled"];
86/** The record a path has now: the manifest only appends, so the last one for a path wins. */
87export function latestRecordFor(projectDir, path) {
88 return readManifest(projectDir).findLast((record) => record.path === path);
89}
91/** Records with every older record for the same path dropped, since the last one for a path is its record. */
92export function currentRecords(projectDir) {
93 const records = readManifest(projectDir);
94 const last = new Map(records.map((record, index) => [record.path, index]));
95 return records.filter((record, index) => !record.path || last.get(record.path) === index);
96}
98/** Records a file already in the project where it is, unless its current record already says the same thing. */
99export function recordInPlace(
100 projectDir,
101 { type, path, source, description, duration, provenance },
102) {
103 const fields = {
104 type,
105 path,
106 source,
107 description: description || basename(path),
108 ...(duration != null && { duration: Math.round(duration * 10) / 10 }),
109 };
110 return withMediaLock(projectDir, () => {
111 const latest = latestRecordFor(projectDir, path);
112 const same = ["source", "description", "duration"].every(
113 (key) => latest?.[key] === fields[key],
114 );
115 if (latest && same) return latest;
116 const record = { id: nextFreeId(projectDir, type), ...fields, provenance };
117 appendRecord(projectDir, record);
118 return record;
119 });
120}
122// Match prompts forgivingly. Agents rarely re-emit a byte-identical intent, so
123// keying cache lookups on exact equality meant "Calm piano" and "calm piano"
124// re-searched and re-downloaded. Normalize (trim, lowercase, collapse internal
125// whitespace) on both sides; the raw prompt is still stored for audit.
126export function normalizePrompt(prompt) {
127 return String(prompt ?? "")
128 .trim()
129 .toLowerCase()
130 .replace(/\s+/g, " ");
131}
133export function findByPrompt(projectDir, prompt, type) {
134 const key = normalizePrompt(prompt);
135 if (!key) return null;
136 const records = currentRecords(projectDir);
137 return (
138 records.find(
139 (r) => normalizePrompt(r.provenance?.prompt) === key && (type == null || r.type === type),
140 ) || null
141 );
142}
144export function findByEntity(projectDir, entity) {
145 const lower = entity.toLowerCase();
146 const records = currentRecords(projectDir);
147 return records.find((r) => r.entity && r.entity.toLowerCase() === lower) || null;
148}
150export function nextId(projectDir, type) {
151 const records = readManifest(projectDir);
152 const prefix = type;
153 let max = 0;
154 for (const r of records) {
155 if (r.type !== type) continue;
156 const m = r.id?.match(new RegExp(`^${prefix}_(\\d+)$`));
157 if (m) max = Math.max(max, parseInt(m[1], 10));
158 }
159 return `${prefix}_${String(max + 1).padStart(3, "0")}`;
160}
162// Sync sleep (no busy-spin) for the allocation lock retry.
163function sleepMs(ms) {
164 Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
165}
167// Coarse per-project lock so concurrent resolves don't race on id allocation.
168// ponytail: one lock file with a 15s stale-steal (a crashed holder can't wedge
169// the project); fine for agent-scale concurrency — revisit if throughput needs
170// finer locking. Date.now() is available here (a normal Node CLI, not a
171// workflow DSL), so mtime-based staleness is safe.
172const LOCK_STALE_MS = 15000;
173const LOCK_TIMEOUT_MS = 20000;
175function withLock(dir, fn) {
176 const lock = join(dir, ".lock");
177 const start = Date.now();
178 for (;;) {
179 try {
180 closeSync(openSync(lock, "wx")); // O_EXCL: atomic acquire
181 break;
182 } catch (err) {
183 if (err.code !== "EEXIST") throw err;
184 try {
185 if (Date.now() - statSync(lock).mtimeMs > LOCK_STALE_MS) {
186 rmSync(lock, { force: true }); // steal a stale lock from a dead holder
187 continue;
188 }
189 } catch {
190 continue; // lock vanished between check and stat — retry the acquire
191 }
192 if (Date.now() - start > LOCK_TIMEOUT_MS) {
193 throw new Error("media-use: timed out acquiring .media/.lock");
194 }
195 sleepMs(25);
196 }
197 }
198 try {
199 return fn();
200 } finally {
201 rmSync(lock, { force: true });
202 }
203}
205/** Runs `fn` holding the project's `.media/.lock`, so concurrent resolves take turns on what it guards. */
206export function withMediaLock(projectDir, fn) {
207 mkdirSync(mediaDir(projectDir), { recursive: true });
208 return withLock(mediaDir(projectDir), fn);
209}
211// Atomically allocate the next free id for `type` AND reserve it on disk, so a
212// slow download/copy between allocation and appendRecord can't let a concurrent
213// caller grab the same id (the MU-23 clobber). Under the lock we take the max id
214// across BOTH the manifest and any already-reserved files in the type dir, then
215// O_EXCL-create an empty marker under core's atomic temp name, which Studio's
216// project history skips. Returns { id, localPath, markerPath }.
217export function allocateId(projectDir, type, ext) {
218 mkdirSync(typeDirPath(projectDir, type), { recursive: true });
219 return withMediaLock(projectDir, () => {
220 const id = nextFreeId(projectDir, type);
221 const localPath = `.media/${typeSubdir(type)}/${id}${ext}`;
222 // Same shape as core atomicTempPath: an empty file at the final name would be committed.
223 const markerPath = `${join(projectDir, localPath)}.hf${randomBytes(3).toString("hex")}.tmp`;
224 writeFileSync(markerPath, "", { flag: "wx" }); // durable reservation
225 return { id, localPath, markerPath };
226 });
227}
229// Call under the lock: counts recorded ids and ids reserved by a file not yet recorded.
230function nextFreeId(projectDir, type) {
231 const re = new RegExp(`^${type}_(\\d+)`);
232 let max = 0;
233 for (const r of readManifest(projectDir)) {
234 if (r.type !== type) continue;
235 const m = r.id?.match(re);
236 if (m) max = Math.max(max, parseInt(m[1], 10));
237 }
238 const typeDir = typeDirPath(projectDir, type);
239 for (const f of existsSync(typeDir) ? readdirSync(typeDir) : []) {
240 const m = f.match(re);
241 if (m) max = Math.max(max, parseInt(m[1], 10));
242 }
243 return `${type}_${String(max + 1).padStart(3, "0")}`;
244}
246function reservedFile(projectDir, type, ext) {
247 const allocation = allocateId(projectDir, type, ext);
248 return { ...allocation, fullPath: join(projectDir, allocation.localPath) };
249}
251function releaseReservation(reservation, committed) {
252 if (!committed) rmSync(reservation.fullPath, { force: true });
253 rmSync(reservation.markerPath, { force: true });
254}
256// A reservation is committed only when populate returns a non-null value.
257// Throwing/rejecting or returning null means no usable asset was produced, so
258// the reservation and any partial asset are released. Keeping this transaction
259// beside allocateId prevents provider/cache/LUT paths from forgetting the rollback.
260export function withReservedFileSync(projectDir, type, ext, populate) {
261 const reservation = reservedFile(projectDir, type, ext);
262 try {
263 const result = populate(reservation);
264 releaseReservation(reservation, result != null);
265 return result;
266 } catch (error) {
267 releaseReservation(reservation, false);
268 throw error;
269 }
270}
272export async function withReservedFile(projectDir, type, ext, populate) {
273 const reservation = reservedFile(projectDir, type, ext);
274 try {
275 const result = await populate(reservation);
276 releaseReservation(reservation, result != null);
277 return result;
278 } catch (error) {
279 releaseReservation(reservation, false);
280 throw error;
281 }
282}