Skills › HyperFrames › media-use › eval.mjs
eval.mjs
369 lines · skills/media-use/scripts/eval.mjs at a595755
1#!/usr/bin/env node
3/**
4 * media-use eval — compare baseline (no media-use) vs. with media-use
5 * on real registry blocks. Produces an HTML report.
6 */
8import {
9 mkdtempSync,
10 cpSync,
11 rmSync,
12 readFileSync,
13 readdirSync,
14 existsSync,
15 writeFileSync,
16} from "node:fs";
17import { join, basename, resolve, dirname } from "node:path";
18import { execFileSync } from "node:child_process";
19import { tmpdir } from "node:os";
20import { fileURLToPath } from "node:url";
22const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
23const REPO_ROOT = resolve(SCRIPT_DIR, "..", "..", "..");
24const RESOLVE_SCRIPT = join(SCRIPT_DIR, "resolve.mjs");
26const TEST_BLOCKS = [
27 "registry/blocks/nyc-paris-flight",
28 "registry/blocks/macos-tahoe-liquid-glass",
29 "registry/blocks/blue-sweater-intro-video",
30 "registry/blocks/vpn-youtube-spot",
31 "registry/blocks/apple-money-count",
32 "registry/blocks/liquid-glass-notification",
33 "registry/blocks/instagram-follow",
34];
36// Run resolve.mjs with args as a literal argv array (no shell), so values
37// interpolated from manifest metadata (--intent prompt, --type) can't inject
38// shell. Mirrors the execFileSync fix in probe.mjs / heygen-search.mjs.
39function run(args, opts = {}) {
40 try {
41 return {
42 ok: true,
43 output: execFileSync(process.execPath, [RESOLVE_SCRIPT, ...args], {
44 encoding: "utf8",
45 timeout: 15000,
46 stdio: "pipe",
47 ...opts,
48 }).trim(),
49 };
50 } catch (err) {
51 return { ok: false, output: (err.stdout || "") + (err.stderr || ""), code: err.status };
52 }
53}
55function countAssetFiles(dir) {
56 const assetsDir = join(dir, "assets");
57 if (!existsSync(assetsDir)) return { count: 0, files: [] };
58 const files = [];
59 function walk(d, base = "") {
60 for (const e of readdirSync(d, { withFileTypes: true })) {
61 const rel = base ? `${base}/${e.name}` : e.name;
62 if (e.isDirectory()) walk(join(d, e.name), rel);
63 else files.push(rel);
64 }
65 }
66 walk(assetsDir);
67 return { count: files.length, files };
68}
70function evalBlock(blockPath) {
71 const fullPath = join(REPO_ROOT, blockPath);
72 if (!existsSync(fullPath)) return null;
74 const name = basename(blockPath);
75 const tmp = mkdtempSync(join(tmpdir(), `mu-eval-${name}-`));
77 try {
78 cpSync(fullPath, tmp, { recursive: true });
80 // baseline: what the agent sees WITHOUT media-use
81 const baseline = countAssetFiles(tmp);
82 const htmlFiles = readdirSync(tmp).filter((f) => f.endsWith(".html"));
84 // parse compositions for asset references
85 const assetRefs = [];
86 for (const hf of htmlFiles) {
87 const html = readFileSync(join(tmp, hf), "utf8");
88 const srcMatches = html.matchAll(/src=["']([^"']+?)["']/g);
89 for (const m of srcMatches) {
90 const ref = m[1];
91 if (ref.startsWith("data:") || ref.startsWith("http")) continue;
92 assetRefs.push({ composition: hf, ref });
93 }
94 const urlMatches = html.matchAll(/url\(["']?([^"')]+?)["']?\)/g);
95 for (const m of urlMatches) {
96 const ref = m[1];
97 if (ref.startsWith("data:") || ref.startsWith("http") || ref.startsWith("#")) continue;
98 assetRefs.push({ composition: hf, ref });
99 }
100 }
102 // with media-use: run --adopt
103 const adoptResult = run(["--adopt", "--project", tmp, "--json"]);
104 let adopted = { ok: false, adopted: 0, assets: [] };
105 if (adoptResult.ok) {
106 try {
107 adopted = JSON.parse(adoptResult.output);
108 } catch {
109 /* */
110 }
111 }
113 // read the generated index
114 const indexPath = join(tmp, ".media", "index.md");
115 const indexContent = existsSync(indexPath)
116 ? readFileSync(indexPath, "utf8")
117 : "(no index generated)";
119 // read manifest for detail
120 const manifestPath = join(tmp, ".media", "manifest.jsonl");
121 const manifest = existsSync(manifestPath)
122 ? readFileSync(manifestPath, "utf8")
123 .trim()
124 .split("\n")
125 .map((l) => {
126 try {
127 return JSON.parse(l);
128 } catch {
129 return null;
130 }
131 })
132 .filter(Boolean)
133 : [];
135 // test resolve cache hit: try resolving something that was adopted
136 let resolveTest = null;
137 if (manifest.length > 0) {
138 const first = manifest[0];
139 const prompt = first.provenance?.prompt || first.description;
140 const r = run(["--type", first.type, "--intent", prompt, "--project", tmp, "--json"]);
141 if (r.ok) {
142 try {
143 resolveTest = JSON.parse(r.output);
144 } catch {
145 /* */
146 }
147 }
148 }
150 // test resolve miss: try resolving something that doesn't exist
151 const missResult = run([
152 "--type",
153 "bgm",
154 "--intent",
155 "nonexistent query xyz",
156 "--project",
157 tmp,
158 "--json",
159 ]);
160 let resolveMiss = null;
161 if (!missResult.ok) {
162 try {
163 resolveMiss = JSON.parse(missResult.output);
164 } catch {
165 /* */
166 }
167 }
169 // coverage: which composition refs are covered by the manifest
170 const manifestPaths = new Set(manifest.map((m) => m.path));
171 const coverage = assetRefs.map((r) => ({
172 ...r,
173 covered: manifestPaths.has(r.ref),
174 }));
176 return {
177 name,
178 baseline: { fileCount: baseline.count, files: baseline.files, htmlCount: htmlFiles.length },
179 compositions: htmlFiles,
180 assetRefs: coverage,
181 adopted: { count: adopted.adopted, assets: adopted.assets || [] },
182 index: indexContent,
183 manifest,
184 resolveTest,
185 resolveMiss,
186 };
187 } finally {
188 rmSync(tmp, { recursive: true, force: true });
189 }
190}
192function generateReport(results) {
193 const all = results.filter(Boolean);
194 const passed = all.filter((r) => r.adopted.count > 0);
196 const rows = results
197 .filter(Boolean)
198 .map((r) => {
199 const hasMetadata = r.manifest.some((m) => m.duration || m.width);
200 const cacheHit = r.resolveTest?._source === "cached";
201 const missHandled = r.resolveMiss?.ok === false;
203 return `<tr>
204 <td><strong>${r.name}</strong></td>
205 <td>${r.baseline.fileCount} files, ${r.baseline.htmlCount} comp${r.baseline.htmlCount === 1 ? "" : "s"}</td>
206 <td>${r.adopted.count} adopted</td>
207 <td>${hasMetadata ? "<span class='pass'>with metadata</span>" : "<span class='warn'>no metadata</span>"}</td>
208 <td>${cacheHit ? "<span class='pass'>cache hit</span>" : "<span class='warn'>no hit</span>"}</td>
209 <td>${missHandled ? "<span class='pass'>handled</span>" : "<span class='fail'>unexpected</span>"}</td>
210 </tr>`;
211 })
212 .join("\n");
214 const details = results
215 .filter(Boolean)
216 .filter((r) => r.adopted.count > 0)
217 .map((r) => {
218 const assetRows = r.manifest
219 .map((m) => {
220 const dur = m.duration != null ? `${m.duration}s` : "—";
221 const dims = m.width && m.height ? `${m.width}×${m.height}` : "—";
222 return `<tr><td>${m.id}</td><td>${m.type}</td><td>${dur}</td><td>${dims}</td><td class="path">${m.path}</td><td>${m.description || ""}</td></tr>`;
223 })
224 .join("\n");
226 const coveredCount = r.assetRefs.filter((c) => c.covered).length;
227 const totalRefs = r.assetRefs.length;
228 const coveragePct = totalRefs > 0 ? Math.round((coveredCount / totalRefs) * 100) : 100;
230 const refRows = r.assetRefs
231 .map(
232 (c) =>
233 `<tr><td class="path">${c.composition}</td><td class="path">${c.ref}</td><td>${c.covered ? "<span class='pass'>covered</span>" : "<span class='warn'>not in manifest</span>"}</td></tr>`,
234 )
235 .join("\n");
237 return `<div class="block-detail">
238 <h3>${r.name}</h3>
239 <p style="font-size:13px;color:var(--muted)">${r.compositions.length} composition${r.compositions.length === 1 ? "" : "s"}: ${r.compositions.join(", ")}</p>
241 <div class="comparison">
242 <div class="col">
243 <h4>Baseline (no media-use)</h4>
244 <p>Agent sees: ${r.baseline.fileCount} raw files in assets/<br>No metadata, no type info, no relationship to compositions.</p>
245 <pre class="file-list">${r.baseline.files.join("\n") || "(no assets)"}</pre>
246 </div>
247 <div class="col">
248 <h4>With media-use (after --adopt)</h4>
249 <p>Agent reads index.md — structured, typed, with metadata:</p>
250 <pre class="index">${escapeHtml(r.index)}</pre>
251 </div>
252 </div>
254 ${
255 totalRefs > 0
256 ? `<h4>Composition → asset coverage <span class="${coveragePct === 100 ? "pass" : "warn"}">${coveragePct}%</span> (${coveredCount}/${totalRefs} refs)</h4>
257 <table class="manifest">
258 <thead><tr><th>composition</th><th>asset reference</th><th>in manifest?</th></tr></thead>
259 <tbody>${refRows}</tbody>
260 </table>`
261 : ""
262 }
264 <h4>Manifest records</h4>
265 <table class="manifest">
266 <thead><tr><th>id</th><th>type</th><th>dur</th><th>dims</th><th>path</th><th>description</th></tr></thead>
267 <tbody>${assetRows}</tbody>
268 </table>
269 </div>`;
270 })
271 .join("\n");
273 return `<title>media-use eval report</title>
274<style>
275:root { --bg: #fafaf7; --text: #1b1b18; --muted: #7a756a; --accent: #0d7377; --good: #1a7a3a; --warn: #b45309; --fail: #dc2626; --border: #e8e5df; --surface: #fff; --mono: ui-monospace, 'SF Mono', Menlo, Consolas, monospace; --sans: system-ui, -apple-system, sans-serif; --serif: Georgia, serif }
276* { box-sizing: border-box; margin: 0 } body { background: var(--bg); color: var(--text); font-family: var(--serif); line-height: 1.6; font-size: 15px; padding: 40px 24px }
277.wrap { max-width: 1100px; margin: 0 auto }
278h1 { font-family: var(--sans); font-size: 28px; font-weight: 700; margin-bottom: 8px; letter-spacing: -.02em }
279h2 { font-family: var(--sans); font-size: 20px; font-weight: 650; margin: 32px 0 12px; letter-spacing: -.01em }
280h3 { font-family: var(--sans); font-size: 17px; font-weight: 650; margin: 24px 0 8px }
281h4 { font-family: var(--sans); font-size: 14px; font-weight: 600; margin: 16px 0 6px; color: var(--muted) }
282p { margin-bottom: 10px }
283.meta { font-family: var(--mono); font-size: 12px; color: var(--muted); margin-bottom: 24px }
284.summary { display: flex; gap: 16px; margin: 16px 0; flex-wrap: wrap }
285.stat { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 14px 18px; flex: 1; min-width: 140px }
286.stat .num { font-family: var(--sans); font-size: 28px; font-weight: 700; color: var(--accent) }
287.stat .label { font-family: var(--mono); font-size: 11px; color: var(--muted); text-transform: uppercase; letter-spacing: .1em }
288table { width: 100%; border-collapse: collapse; font-size: 13px; font-family: var(--sans); margin: 8px 0 }
289th { text-align: left; font-family: var(--mono); font-size: 10px; letter-spacing: .08em; text-transform: uppercase; color: var(--muted); border-bottom: 2px solid var(--border); padding: 6px 8px; font-weight: 700 }
290td { border-bottom: 1px solid var(--border); padding: 7px 8px; vertical-align: top }
291td.path { font-family: var(--mono); font-size: 12px; color: var(--muted); max-width: 300px; overflow: hidden; text-overflow: ellipsis }
292.pass { color: var(--good); font-weight: 600 } .warn { color: var(--warn); font-weight: 600 } .fail { color: var(--fail); font-weight: 600 }
293.comparison { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin: 12px 0 }
294@media(max-width:700px) { .comparison { grid-template-columns: 1fr } }
295.col { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 14px 16px }
296.col h4 { margin-top: 0 }
297pre { font-family: var(--mono); font-size: 12px; background: #1b1b18; color: #d4d0c8; border-radius: 6px; padding: 12px 14px; overflow-x: auto; margin: 6px 0; line-height: 1.5 }
298pre.file-list { background: var(--bg); color: var(--muted); border: 1px solid var(--border) }
299pre.index { white-space: pre; }
300.block-detail { border-top: 1px solid var(--border); padding-top: 20px; margin-top: 20px }
301.verdict { margin-top: 24px; padding: 16px 20px; border-radius: 8px; font-family: var(--sans); font-size: 15px }
302.verdict.ship { background: #edfbf0; border: 1px solid #1a7a3a; color: #1a7a3a }
303.verdict.wait { background: #fff3ec; border: 1px solid #d94f04; color: #d94f04 }
304</style>
305<div class="wrap">
306<h1>media-use eval report</h1>
307<p class="meta">${new Date().toISOString().slice(0, 10)} · ${all.length} blocks evaluated · baseline vs. media-use --adopt</p>
309<div class="summary">
310 <div class="stat"><div class="num">${all.length}</div><div class="label">blocks tested</div></div>
311 <div class="stat"><div class="num">${passed.length}</div><div class="label">with assets</div></div>
312 <div class="stat"><div class="num">${all.reduce((s, r) => s + r.adopted.count, 0)}</div><div class="label">assets adopted</div></div>
313 <div class="stat"><div class="num">${all.filter((r) => r.manifest.some((m) => m.duration || m.width)).length}</div><div class="label">with ffprobe metadata</div></div>
314 <div class="stat"><div class="num">${(() => {
315 const refs = all.flatMap((r) => r.assetRefs);
316 const covered = refs.filter((c) => c.covered).length;
317 return refs.length > 0 ? Math.round((covered / refs.length) * 100) + "%" : "—";
318 })()}</div><div class="label">composition coverage</div></div>
319</div>
321<h2>Results matrix</h2>
322<table>
323 <thead><tr><th>Block</th><th>Baseline</th><th>Adopted</th><th>Metadata</th><th>Cache hit</th><th>Miss handling</th></tr></thead>
324 <tbody>${rows}</tbody>
325</table>
327<h2>Before / after comparisons</h2>
328${details}
330<div class="verdict ${passed.length >= 3 ? "ship" : "wait"}">
331 ${
332 passed.length >= 3
333 ? `<strong>Ship it.</strong> ${passed.length}/${all.length} blocks adopted successfully with metadata. Resolve cache hits work. Miss handling is clean.`
334 : `<strong>Needs work.</strong> Only ${passed.length} blocks adopted. Check the failures above.`
335 }
336</div>
337</div>`;
338}
340function escapeHtml(str) {
341 return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
342}
344console.log("media-use eval · running against registry blocks...\n");
346const results = [];
347for (const block of TEST_BLOCKS) {
348 const fullPath = join(REPO_ROOT, block);
349 if (!existsSync(fullPath)) {
350 console.log(` skip ${basename(block)} (not found)`);
351 results.push(null);
352 continue;
353 }
354 process.stdout.write(` ${basename(block)}...`);
355 const result = evalBlock(block);
356 if (result) {
357 console.log(
358 ` ${result.adopted.count} adopted, ${result.manifest.filter((m) => m.duration || m.width).length} with metadata`,
359 );
360 } else {
361 console.log(" failed");
362 }
363 results.push(result);
364}
366const report = generateReport(results);
367const outPath = join(SCRIPT_DIR, "..", "eval-report.html");
368writeFileSync(outPath, report);
369console.log(`\nReport: ${outPath}`);