Skills › HyperFrames › hyperframes › locate.mjs

locate.mjs

228 lines · skills/hyperframes/scripts/locate.mjs at a595755

Open on GitHub

1#!/usr/bin/env node

2/* locate.mjs — find "X" in an image WITHOUT any detector API or key.

3 *

4 * You (the author model) are weak at regressing pixel coordinates but strong at

5 * "which numbered strip is the target in?" — so localization = RSVP grid-index

6 * (ACL 2025 long.715): pick strips on a numbered grid, reconstruct the box

7 * geometrically. Zero npm deps — node + ffmpeg (both already required by

8 * hyperframes). Accuracy notes: references/film.md.

9 *

10 * THE LOOP (you read images between steps):

11 * 1) node locate.mjs overlay <img> --out DIR

12 * → DIR/gv.png (vertical strips 1..9) + DIR/gh.png (horizontal 1..9).

13 * READ both, pick the strip numbers the target spans.

14 * 2) node locate.mjs region <img> --vids 4,5 --hids 6,7 --out DIR

15 * → prints coarse region + writes DIR/gc.png (region cropped+upscaled,

16 * finer 6×6 grid). READ it, pick the finer strips.

17 * 3) node locate.mjs final <img> --region x0,y0,x1,y1 --vids 3,4 --hids 3,4

18 * → {box, center} normalized to the FULL image.

19 * 4) node locate.mjs mark <img> --box x0,y0,x1,y1 --out DIR/check.png

20 * → VERIFY: read check.png — is the red box on the target? If off,

21 * redo step 2/3 with corrected strips. Never skip this.

22 */

23import { execFileSync } from "node:child_process";

24import { existsSync, mkdirSync, copyFileSync } from "node:fs";

26const N1 = 9,

27 PAD1 = 0.4; // stage 1: strips per axis, padding in STRIP units

28const N2 = 6,

29 PAD2 = 0.25; // stage 2 (crop)

30const GREEN = "0x3CDC5A",

31 BLUE = "0x3C82FF",

32 RED = "0xFF3232";

34const ff = (args) =>

35 execFileSync("ffmpeg", ["-v", "error", "-y", ...args], {

36 stdio: ["ignore", "ignore", "inherit"],

37 });

38function probe(img) {

39 const out = execFileSync("ffprobe", [

40 "-v",

41 "error",

42 "-select_streams",

43 "v:0",

44 "-show_entries",

45 "stream=width,height",

46 "-of",

47 "csv=p=0",

48 "--",

49 img,

50 ])

51 .toString()

52 .trim();

53 const [w, h] = out.split(",").map(Number);

54 return { w, h };

55}

56function font() {

57 const dst = "/tmp/locate-font.ttf";

58 if (!existsSync(dst)) {

59 for (const c of [

60 "/System/Library/Fonts/Supplemental/Arial Bold.ttf",

61 "/System/Library/Fonts/Supplemental/Arial.ttf",

62 "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",

63 ]) {

64 if (existsSync(c)) {

65 copyFileSync(c, dst);

66 break;

67 }

68 }

69 }

70 return dst;

71}

72function gridFilters(w, h, n, axis, fz) {

73 const F = font(),

74 fs = [];

75 if (axis === "v" || axis === "both") {

76 for (let i = 1; i < n; i++)

77 fs.push(`drawbox=x=${Math.round((i * w) / n)}:y=0:w=2:h=${h}:color=${GREEN}@0.8:t=fill`);

78 for (let i = 0; i < n; i++)

79 fs.push(

80 `drawtext=fontfile=${F}:text='${i + 1}':x=${Math.round(((i + 0.5) * w) / n - fz * 0.3)}:y=6:fontsize=${fz}:fontcolor=black:box=1:boxcolor=white@0.95:boxborderw=5`,

81 );

82 }

83 if (axis === "h" || axis === "both") {

84 for (let j = 1; j < n; j++)

85 fs.push(`drawbox=x=0:y=${Math.round((j * h) / n)}:w=${w}:h=2:color=${BLUE}@0.8:t=fill`);

86 for (let j = 0; j < n; j++)

87 fs.push(

88 `drawtext=fontfile=${F}:text='${j + 1}':x=6:y=${Math.round(((j + 0.5) * h) / n - fz * 0.5)}:fontsize=${fz}:fontcolor=black:box=1:boxcolor=white@0.95:boxborderw=5`,

89 );

90 }

91 return fs;

92}

93const ids = (s) => String(s).split(",").filter(Boolean).map(Number);

94const regionOf = (v, h, n, pad) => [

95 Math.max(Math.min(...v) - 1 - pad, 0) / n,

96 Math.max(Math.min(...h) - 1 - pad, 0) / n,

97 Math.min(Math.max(...v) + pad, n) / n,

98 Math.min(Math.max(...h) + pad, n) / n,

99];

100const r4 = (x) => Math.round(x * 1e4) / 1e4;

101const out = (o) => console.log(JSON.stringify(o));

102

103// --- tiny arg parsing: locate.mjs <cmd> <img?> --k v ---

104const [cmd, ...rest] = process.argv.slice(2);

105const pos = rest.filter((a) => !a.startsWith("--"));

106const opt = {};

107for (let i = 0; i < rest.length; i++)

108 if (rest[i].startsWith("--")) opt[rest[i].slice(2)] = rest[i + 1];

109const img = pos[0];

110

111if (cmd === "overlay") {

112 const dir = opt.out || ".";

113 mkdirSync(dir, { recursive: true });

114 const n = Number(opt.n || N1);

115 const { w, h } = probe(img);

116 const fz = Math.max(16, Math.round(Math.min(w, h) / 22));

117 ff(["-i", img, "-vf", gridFilters(w, h, n, "v", fz).join(","), `${dir}/gv.png`]);

118 ff(["-i", img, "-vf", gridFilters(w, h, n, "h", fz).join(","), `${dir}/gh.png`]);

119 out({

120 w,

121 h,

122 n,

123 vertical: `${dir}/gv.png`,

124 horizontal: `${dir}/gh.png`,

125 next: "READ gv.png + gh.png, then: locate.mjs region <img> --vids .. --hids .. --out " + dir,

126 });

127} else if (cmd === "region") {

128 const dir = opt.out || ".";

129 mkdirSync(dir, { recursive: true });

130 const n = Number(opt.n || N1),

131 nf = Number(opt.nf || N2);

132 const reg = regionOf(ids(opt.vids), ids(opt.hids), n, PAD1);

133 const { w, h } = probe(img);

134 const cx = Math.floor(reg[0] * w),

135 cy = Math.floor(reg[1] * h);

136 const cw = Math.max(2, Math.floor((reg[2] - reg[0]) * w)),

137 ch = Math.max(2, Math.floor((reg[3] - reg[1]) * h));

138 const uw = Math.max(cw, 640),

139 uh = Math.round((ch * uw) / cw);

140 const fz = Math.max(15, Math.round(Math.min(uw, uh) / 16));

141 ff([

142 "-i",

143 img,

144 "-vf",

145 [

146 `crop=${cw}:${ch}:${cx}:${cy}`,

147 `scale=${uw}:${uh}`,

148 ...gridFilters(uw, uh, nf, "both", fz),

149 ].join(","),

150 `${dir}/gc.png`,

151 ]);

152 out({

153 region: reg.map(r4),

154 crop_grid: `${dir}/gc.png`,

155 nf,

156 next: `READ gc.png, then: locate.mjs final <img> --region ${reg.map(r4).join(",")} --vids .. --hids ..`,

157 });

158} else if (cmd === "final") {

159 const nf = Number(opt.nf || N2);

160 const reg = opt.region.split(",").map(Number);

161 const rl = regionOf(ids(opt.vids), ids(opt.hids), nf, PAD2);

162 const [x0, y0, x1, y1] = reg;

163 const box = [

164 x0 + rl[0] * (x1 - x0),

165 y0 + rl[1] * (y1 - y0),

166 x0 + rl[2] * (x1 - x0),

167 y0 + rl[3] * (y1 - y0),

168 ].map(r4);

169 out({

170 box,

171 center: [r4((box[0] + box[2]) / 2), r4((box[1] + box[3]) / 2)],

172 next:

173 "VERIFY: locate.mjs mark <img> --box " +

174 box.join(",") +

175 " --out <dir>/check.png — then READ it",

176 });

177} else if (cmd === "mark") {

178 const box = opt.box.split(",").map(Number);

179 const { w, h } = probe(img);

180 const bx = Math.round(box[0] * w),

181 by = Math.round(box[1] * h);

182 const bw = Math.max(2, Math.round((box[2] - box[0]) * w)),

183 bh = Math.max(2, Math.round((box[3] - box[1]) * h));

184 const dst = opt.out || "./check.png";

185 ff(["-i", img, "-vf", `drawbox=x=${bx}:y=${by}:w=${bw}:h=${bh}:color=${RED}@1:t=6`, dst]);

186 out({

187 marked: dst,

188 next: "READ it — red box on the target? If off, redo region/final with corrected strips.",

189 });

190} else if (cmd === "auto") {

191 // OPTIONAL fast path — only if a strong detector key happens to exist. Never assume it.

192 const key = process.env.GEMINI_API_KEY;

193 if (!key) {

194 console.error(

195 "auto needs GEMINI_API_KEY; use the grid loop instead (overlay→region→final→mark)",

196 );

197 process.exit(1);

198 }

199 const target = pos[1];

200 const b64 = execFileSync("base64", ["-i", img]).toString().replace(/\n/g, "");

201 const mime = img.match(/\.png$/i) ? "image/png" : "image/jpeg";

202 const body = {

203 contents: [

204 {

205 parts: [

206 { inline_data: { mime_type: mime, data: b64 } },

207 {

208 text: `Detect "${target}". Return ONLY {"box_2d":[ymin,xmin,ymax,xmax]} integers 0-1000.`,

209 },

210 ],

211 },

212 ],

213 generationConfig: { temperature: 0, response_mime_type: "application/json" },

214 };

215 const res = await fetch(

216 `https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent?key=${key}`,

217 { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) },

218 );

219 const t = (await res.json()).candidates[0].content.parts[0].text;

220 let bb = JSON.parse(t.match(/\{[\s\S]*\}/)[0]).box_2d;

221 if (Array.isArray(bb[0])) bb = bb[0];

222 const s = Math.max(...bb) > 1.5 ? 1000 : 1;

223 const box = [bb[1] / s, bb[0] / s, bb[3] / s, bb[2] / s].map(r4);

224 out({ box, center: [r4((box[0] + box[2]) / 2), r4((box[1] + box[3]) / 2)] });

225} else {

226 console.error("usage: locate.mjs overlay|region|final|mark|auto … (see header comment)");

227 process.exit(1);

228}