Skill Tracer
Skills › Compound Engineering › ce-optimize › decide.mjs

decide.mjs

676 lines · skills/ce-optimize/scripts/decide.mjs at 3449230

Open on GitHub
1#!/usr/bin/env node
2// Decide keep / revert / inconclusive / censored / degenerate for one
3// candidate against the current best. Owns multi-objective eligibility,
4// noise-aware comparison, and the measurement-ladder next step.
5//
6// Usage:
7// node decide.mjs # JSON on stdin
8// node decide.mjs <input.json>
9//
10// A spec with no `objectives` and no `ladder` reproduces the legacy
11// single-primary + absolute noise_threshold rule, except that a delta
12// inside the threshold is `inconclusive` rather than a silent revert.
13//
14// Input: { spec, baseline, candidate, holdout? }. `holdout` is a second
15// { baseline, candidate } snapshot pair from the held-out set. When the spec
16// configures a holdout and the pair is absent, a would-be keep returns
17// `next_measurement: "holdout"` instead of keeping. When either snapshot
18// carries a `cases` object, the output lists `regressions` (report only).
20export function median(values) {
21 if (!values.length) return null
22 const sorted = [...values].sort((a, b) => a - b)
23 const mid = Math.floor(sorted.length / 2)
24 return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]
27function parseCheck(check) {
28 const match = String(check).trim().match(/^(>=|<=|==|!=|>|<)\s*(.+)$/)
29 if (!match) {
30 throw new Error(`invalid gate check: ${check}`)
31 }
32 return { op: match[1], threshold: Number(match[2]) }
35export function gatePasses(value, check) {
36 const { op, threshold } = parseCheck(check)
37 const n = finiteNumber(value)
38 if (n == null || !Number.isFinite(threshold)) return false
39 switch (op) {
40 case ">=":
41 return n >= threshold
42 case "<=":
43 return n <= threshold
44 case ">":
45 return n > threshold
46 case "<":
47 return n < threshold
48 case "==":
49 return n === threshold
50 case "!=":
51 return n !== threshold
52 default:
53 return false
54 }
57function signedDelta(baseline, candidate, direction) {
58 return direction === "minimize" ? baseline - candidate : candidate - baseline
61function finiteNumber(value) {
62 if (value == null || value === "") return null
63 const n = typeof value === "number" ? value : Number(value)
64 return Number.isFinite(n) ? n : null
67function firstNonNegativeNumber(values, fallback) {
68 for (const value of values) {
69 const n = finiteNumber(value)
70 if (n != null && n >= 0) return n
71 }
72 return fallback
75function positiveInteger(value, fallback) {
76 const n = typeof value === "number" ? value : Number(value)
77 if (!Number.isInteger(n) || n < 1) return fallback
78 return n
81function verdictFromSigned(delta, threshold) {
82 if (delta > threshold) return "improved"
83 if (delta < -threshold) return "regressed"
84 return "inconclusive"
87function closedResult(fields) {
88 return {
89 eligible: false,
90 next_measurement: "none",
91 target_reached: false,
92 improved_objectives: [],
93 violated_objectives: [],
94 comparisons: {},
95 primary_delta: null,
96 rank_score: 0,
97 regressions: [],
98 holdout: null,
99 ...fields,
100 }
103function casePassed(value) {
104 return value === true || value === 1 || value === "1" || value === "true" || value === "pass"
107// Cases the reference passed and the candidate fails. Report only: this never
108// changes the decision, and a missing `cases` object on either side yields [].
109export function regressions(baseline, candidate) {
110 const before = baseline?.cases
111 const after = candidate?.cases
112 if (!before || typeof before !== "object" || !after || typeof after !== "object") return []
113 return Object.keys(before)
114 .filter((id) => casePassed(before[id]) && id in after && !casePassed(after[id]))
115 .sort()
118function holdoutConfigured(spec) {
119 if (spec.measurement?.holdout?.command) return true
120 const seed = spec.judge?.confirmation_seed
121 if (seed == null) return false
122 return spec.primary?.type === "judge" && seed !== (spec.judge?.sample_seed ?? 42)
125function configuredAggregation(value) {
126 return value === "mean" || value === "min" || value === "max" ? value : "median"
129function aggregateSamples(samples, method) {
130 if (!samples.length) return null
131 if (method === "mean") return samples.reduce((sum, value) => sum + value, 0) / samples.length
132 if (method === "min") return Math.min(...samples)
133 if (method === "max") return Math.max(...samples)
134 return median(samples)
137function valueBundle(raw, aggregation = "median") {
138 if (raw && typeof raw === "object" && !Array.isArray(raw)) {
139 const samples = Array.isArray(raw.samples) ? raw.samples.map(finiteNumber) : []
140 if (samples.some((n) => n == null)) return { aggregate: null, samples: [] }
141 const aggregate = samples.length
142 ? aggregateSamples(samples, aggregation)
143 : finiteNumber(raw.aggregate)
144 return { aggregate: aggregate ?? null, samples }
145 }
146 if (raw != null && typeof raw !== "object") {
147 const n = finiteNumber(raw)
148 return n == null ? { aggregate: null, samples: [] } : { aggregate: n, samples: [] }
149 }
150 return null
153function metricBundle(source, name, aggregation, type) {
154 if (!source) return null
155 const containers =
156 type === "judge"
157 ? [source.judge?.[name], source.metrics?.[name], source.diagnostics?.[name], source.gates?.[name]]
158 : [source.metrics?.[name], source.judge?.[name], source.diagnostics?.[name], source.gates?.[name]]
159 for (const raw of containers) {
160 if (raw === undefined) continue
161 return valueBundle(raw, aggregation)
162 }
163 return null
166function normalizeSpec(spec) {
167 const metric = spec.metric ?? {}
168 const measurement = spec.measurement ?? {}
169 const stability = spec.stability ?? measurement.stability ?? {}
170 const primary = spec.primary ?? metric.primary ?? {}
171 const judge = spec.judge ?? metric.judge
172 const comparison = spec.comparison ?? stability.comparison
173 return {
174 ...spec,
175 primary,
176 objectives: spec.objectives ?? metric.objectives,
177 degenerate_gates: spec.degenerate_gates ?? metric.degenerate_gates,
178 judge,
179 comparison,
180 ladder: spec.ladder ?? stability.ladder ?? {},
181 stability_mode: spec.stability_mode ?? stability.mode,
182 aggregation: configuredAggregation(spec.aggregation ?? stability.aggregation),
183 repeat_count: spec.repeat_count ?? stability.repeat_count,
184 noise_threshold: spec.noise_threshold ?? stability.noise_threshold,
185 minimum_improvement:
186 spec.minimum_improvement ?? comparison?.minimum_improvement ?? judge?.minimum_improvement,
187 measurement,
188 stability,
189 }
192function comparisonDefaults(spec) {
193 const stability = spec.stability ?? spec.measurement?.stability ?? {}
194 const comparison = spec.comparison ?? stability.comparison ?? {}
195 const usesJudge = spec.primary?.type === "judge" || spec.judge != null
196 return {
197 method: comparison.method ?? "absolute",
198 noise_threshold: firstNonNegativeNumber(
199 [comparison.noise_threshold, spec.noise_threshold, stability.noise_threshold],
200 0.02,
201 ),
202 relative_threshold: firstNonNegativeNumber([comparison.relative_threshold], 0.05),
203 minimum_improvement: firstNonNegativeNumber(
204 [comparison.minimum_improvement, spec.minimum_improvement, spec.judge?.minimum_improvement],
205 usesJudge ? 0.3 : null,
206 ),
207 }
210function requiredObjectives(spec) {
211 const primary = spec.primary ?? {}
212 const listed = Array.isArray(spec.objectives) ? spec.objectives : []
213 const extras = listed
214 .map((objective) => ({
215 name: objective.name,
216 direction: objective.direction ?? primary.direction ?? "maximize",
217 role: objective.role ?? "required",
218 type: objective.type ?? "hard",
219 target: objective.target ?? null,
220 max_regression: objective.max_regression ?? null,
221 }))
222 .filter((objective) => objective.role !== "secondary")
223 if (!primary.name) return extras
224 const listedPrimary = extras.find((objective) => objective.name === primary.name)
225 return [
226 {
227 name: primary.name,
228 direction: primary.direction ?? listedPrimary?.direction ?? "maximize",
229 role: "required",
230 type: primary.type ?? listedPrimary?.type ?? "hard",
231 target: primary.target ?? listedPrimary?.target ?? null,
232 max_regression: listedPrimary?.max_regression ?? primary.max_regression ?? null,
233 },
234 ...extras.filter((objective) => objective.name !== primary.name),
235 ]
238export function compareObjective({
239 baselineValue,
240 candidateValue,
241 baselineSamples,
242 candidateSamples,
243 direction,
244 type,
245 comparison,
246 maxRegression,
247}) {
248 const delta = signedDelta(baselineValue, candidateValue, direction)
249 const denom = Math.abs(baselineValue)
250 const relative = denom > 0 ? delta / denom : 0
251 const absThreshold =
252 type === "judge" && comparison.minimum_improvement != null
253 ? comparison.minimum_improvement
254 : comparison.noise_threshold
255 const relativeThreshold = comparison.relative_threshold
257 let verdict
258 if ((comparison.method === "relative" || comparison.method === "paired") && denom <= 0) {
259 verdict = "inconclusive"
260 } else if (comparison.method === "relative") {
261 verdict = verdictFromSigned(relative, relativeThreshold)
262 } else if (comparison.method === "paired") {
263 const baseSamples = baselineSamples?.filter((n) => n != null) ?? []
264 const candSamples = candidateSamples?.filter((n) => n != null) ?? []
265 const threshold = relativeThreshold * denom
266 if (!baseSamples.length || !candSamples.length) {
267 verdict =
268 type === "judge"
269 ? verdictFromSigned(delta, absThreshold)
270 : verdictFromSigned(delta, threshold) === "regressed"
271 ? "regressed"
272 : "inconclusive"
273 } else if (candSamples.length > baseSamples.length) {
274 verdict = "inconclusive"
275 } else {
276 const diffs = candSamples.map((value, index) =>
277 signedDelta(baseSamples[index], value, direction),
278 )
279 const lo = Math.min(...diffs)
280 const hi = Math.max(...diffs)
281 if (verdictFromSigned(lo, threshold) === "improved") verdict = "improved"
282 else if (verdictFromSigned(hi, threshold) === "regressed") verdict = "regressed"
283 else verdict = "inconclusive"
284 }
285 } else {
286 verdict = verdictFromSigned(delta, absThreshold)
287 }
289 if (
290 type === "judge" &&
291 comparison.minimum_improvement != null &&
292 verdict === "improved" &&
293 delta <= comparison.minimum_improvement
294 ) {
295 verdict = "inconclusive"
296 }
298 let violated = verdict === "regressed"
299 if (maxRegression && verdict !== "improved") {
300 const rawBound = Number(maxRegression.value)
301 if (Number.isFinite(rawBound) && rawBound >= 0) {
302 const bound = maxRegression.type === "relative" ? rawBound * denom : rawBound
303 violated = -delta > bound
304 if (violated) verdict = "regressed"
305 }
306 }
308 return { verdict, delta, relative, violated }
311function evaluateGates(spec, candidate) {
312 const gates = Array.isArray(spec.degenerate_gates) ? spec.degenerate_gates : []
313 const values = candidate?.gates ?? {}
314 const failures = []
315 for (const gate of gates) {
316 if (!gatePasses(values[gate.name], gate.check)) {
317 failures.push(gate.name)
318 }
319 }
320 return failures
323function futilityBound(futility, baselineValue, direction) {
324 const factor = finiteNumber(futility.worse_factor ?? 1.2)
325 const baseline = finiteNumber(baselineValue)
326 if (factor == null || factor <= 1 || baseline == null || baseline <= 0) return null
327 return direction === "minimize" ? baseline * factor : baseline / factor
330function isFutile({
331 ladder,
332 direction,
333 baselineValue,
334 candidateValue,
335 elapsedSeconds,
336 sampleCount,
337 enabled,
338}) {
339 const futility = ladder.futility
340 if (!enabled || futility == null || typeof futility !== "object") return false
342 const afterElapsed = finiteNumber(futility.after_elapsed_seconds)
343 if (
344 afterElapsed != null &&
345 afterElapsed > 0 &&
346 elapsedSeconds != null &&
347 Number(elapsedSeconds) >= afterElapsed &&
348 signedDelta(baselineValue, candidateValue, direction) <= 0
349 ) {
350 return true
351 }
353 const bound = futilityBound(futility, baselineValue, direction)
354 if (bound == null || candidateValue == null) return false
355 const worse = signedDelta(bound, candidateValue, direction) <= 0
356 return worse && (sampleCount ?? 1) <= (ladder.exploratory_pairs ?? 1)
359function compareRequired({
360 required,
361 comparison,
362 aggregation,
363 baseline,
364 candidate,
365 ladderEnabled,
366 confirmationRepeats,
367}) {
368 const comparisons = {}
369 const improved = []
370 const violated = []
371 const missing = []
372 const incompleteBaselines = []
373 const candidateBundles = {}
374 const baselineBundles = {}
376 for (const objective of required) {
377 const base = metricBundle(baseline, objective.name, aggregation, objective.type)
378 const cand = metricBundle(candidate, objective.name, aggregation, objective.type)
379 baselineBundles[objective.name] = base
380 candidateBundles[objective.name] = cand
381 if (!base || base.aggregate == null || !cand || cand.aggregate == null) {
382 missing.push(objective.name)
383 continue
384 }
385 if (comparison.method === "paired" && base.samples.length && cand.samples.length) {
386 const requiredSamples = Math.max(cand.samples.length, ladderEnabled ? confirmationRepeats : 1)
387 if (base.samples.length < requiredSamples) {
388 incompleteBaselines.push(
389 `${objective.name} (${base.samples.length} observed, ${requiredSamples} required)`,
390 )
391 continue
392 }
393 }
394 const result = compareObjective({
395 baselineValue: base.aggregate,
396 candidateValue: cand.aggregate,
397 baselineSamples: base.samples,
398 candidateSamples: cand.samples,
399 direction: objective.direction,
400 type: objective.type,
401 comparison,
402 maxRegression: objective.max_regression,
403 })
404 comparisons[objective.name] = result
405 if (result.verdict === "improved") improved.push({ name: objective.name, ...result })
406 if (result.violated) violated.push(objective.name)
407 }
409 return { comparisons, improved, violated, missing, incompleteBaselines, candidateBundles, baselineBundles }
412// The held-out pair is scored with the same gates, objectives, and thresholds
413// as the selection pair. It confirms a keep; it never selects. A holdout that
414// does not improve an objective the selection improved withholds the keep.
415function confirmHoldout({ spec, holdout, required, comparison, aggregation, selectionImproved }) {
416 const gateFailures = evaluateGates(spec, holdout.candidate)
417 if (gateFailures.length) {
418 return {
419 comparisons: {},
420 improved_objectives: [],
421 violated_objectives: gateFailures,
422 agrees: false,
423 decision: "degenerate",
424 reason: `holdout degenerate gate failed: ${gateFailures.join(", ")}`,
425 }
426 }
427 const compared = compareRequired({
428 required,
429 comparison,
430 aggregation,
431 baseline: holdout.baseline ?? {},
432 candidate: holdout.candidate ?? {},
433 ladderEnabled: false,
434 confirmationRepeats: 1,
435 })
436 const summary = {
437 comparisons: compared.comparisons,
438 improved_objectives: compared.improved.map((item) => item.name),
439 violated_objectives: compared.violated,
440 }
441 if (compared.missing.length) {
442 return {
443 ...summary,
444 agrees: false,
445 decision: "error",
446 reason: `holdout missing required metric: ${compared.missing.join(", ")}`,
447 }
448 }
449 if (compared.incompleteBaselines.length) {
450 return {
451 ...summary,
452 agrees: false,
453 decision: "error",
454 reason: `holdout has insufficient paired baseline samples: ${compared.incompleteBaselines.join(", ")}`,
455 }
456 }
457 if (compared.violated.length) {
458 return {
459 ...summary,
460 agrees: false,
461 decision: "revert",
462 reason: `holdout regressed ${compared.violated.join(", ")}`,
463 }
464 }
465 if (!compared.improved.some((item) => selectionImproved.includes(item.name))) {
466 return {
467 ...summary,
468 agrees: false,
469 decision: "inconclusive",
470 reason: `holdout did not confirm the selection gain in ${selectionImproved.join(", ")}`,
471 }
472 }
473 return { ...summary, agrees: true, decision: "keep", reason: null }
476function rankScore(primaryComparison, improved) {
477 if (primaryComparison?.verdict === "improved") return primaryComparison.relative ?? 0
478 if (!improved.length) return primaryComparison?.relative ?? 0
479 return Math.max(...improved.map((item) => item.relative ?? 0))
482export function decide(input) {
483 const spec = normalizeSpec(input.spec ?? {})
484 const baseline = input.baseline ?? {}
485 const candidate = input.candidate ?? {}
486 const primary = spec.primary ?? {}
487 if (!primary.name) {
488 return closedResult({ decision: "error", reason: "missing primary metric" })
489 }
490 const comparison = comparisonDefaults(spec)
491 const required = requiredObjectives(spec)
492 const ladder = spec.ladder ?? {}
493 const ladderEnabled = Boolean(ladder.enabled || spec.stability_mode === "ladder")
494 const exploratoryPairs = positiveInteger(ladder.exploratory_pairs, 1)
495 let confirmationRepeats = positiveInteger(ladder.confirmation_repeats ?? spec.repeat_count, 5)
496 if (confirmationRepeats < exploratoryPairs) confirmationRepeats = exploratoryPairs
498 if (candidate.smoke_passed === false) {
499 return closedResult({ decision: "degenerate", reason: "smoke test failed" })
500 }
502 const gateFailures = evaluateGates(spec, candidate)
503 if (gateFailures.length) {
504 return closedResult({
505 decision: "degenerate",
506 violated_objectives: gateFailures,
507 reason: `degenerate gate failed: ${gateFailures.join(", ")}`,
508 })
509 }
511 const aggregation = spec.aggregation ?? "median"
512 const compared = compareRequired({
513 required,
514 comparison,
515 aggregation,
516 baseline,
517 candidate,
518 ladderEnabled,
519 confirmationRepeats,
520 })
521 const { comparisons, improved, violated, missing, incompleteBaselines, candidateBundles, baselineBundles } =
522 compared
524 if (missing.length) {
525 return closedResult({
526 decision: "error",
527 comparisons,
528 reason: `missing required metric: ${missing.join(", ")}`,
529 })
530 }
532 if (incompleteBaselines.length) {
533 // Ladder next steps collect candidate samples; they cannot repair the baseline.
534 return closedResult({
535 decision: "error",
536 comparisons,
537 reason: `insufficient paired baseline samples: ${incompleteBaselines.join(", ")}`,
538 })
539 }
541 const primaryBundle =
542 candidateBundles[primary.name] ?? metricBundle(candidate, primary.name, aggregation, primary.type)
543 const baselinePrimary =
544 baselineBundles[primary.name] ?? metricBundle(baseline, primary.name, aggregation, primary.type)
545 const primaryComparison = comparisons[primary.name] ?? null
546 const eligible = improved.length > 0 && violated.length === 0
547 const improvedNames = improved.map((item) => item.name)
548 const stillContending = required.some(
549 (objective) => comparisons[objective.name]?.verdict === "inconclusive",
550 )
551 const sampleCount = Math.min(
552 ...required.map((objective) => {
553 const fromSamples = candidateBundles[objective.name]?.samples?.length
554 if (Number.isInteger(fromSamples) && fromSamples > 0) return fromSamples
555 return positiveInteger(candidate.sample_count, 1)
556 }),
557 )
559 if (
560 !eligible &&
561 !stillContending &&
562 isFutile({
563 ladder,
564 direction: primary.direction,
565 baselineValue: baselinePrimary?.aggregate,
566 candidateValue: primaryBundle?.aggregate,
567 elapsedSeconds: candidate.elapsed_seconds,
568 sampleCount,
569 enabled: ladderEnabled,
570 })
571 ) {
572 return closedResult({
573 decision: "censored",
574 improved_objectives: improvedNames,
575 violated_objectives: violated,
576 comparisons,
577 primary_delta: primaryComparison?.delta ?? null,
578 reason: "noncompetitive under the predeclared futility bound",
579 })
580 }
582 const withTargets = required.filter((objective) => objective.target != null)
583 const targetReached =
584 withTargets.length > 0 &&
585 withTargets.every((objective) => {
586 const value = candidateBundles[objective.name]?.aggregate
587 if (value == null) return false
588 return signedDelta(objective.target, value, objective.direction) >= 0
589 })
591 let decision
592 if (eligible) decision = "keep"
593 else if (violated.length) decision = "revert"
594 else if (stillContending) decision = "inconclusive"
595 else decision = "revert"
597 let nextMeasurement = "none"
598 if (ladderEnabled && ladder.smoke_command && candidate.smoke_passed == null) {
599 nextMeasurement = "smoke"
600 } else if (ladderEnabled && (decision === "keep" || decision === "inconclusive")) {
601 const confirming = decision === "keep"
602 const sampleBudget = confirming
603 ? confirmationRepeats
604 : Math.min(exploratoryPairs + 1, confirmationRepeats)
605 if (sampleCount < sampleBudget) {
606 if (confirming) decision = "promising"
607 if (sampleCount < exploratoryPairs) nextMeasurement = "exploratory"
608 else nextMeasurement = confirming ? "confirm" : "add_sample"
609 }
610 }
612 let reason = "no required objective improved"
613 if (eligible) {
614 reason = `improved ${improvedNames.join(", ")} without violating other required objectives`
615 } else if (decision === "inconclusive") {
616 reason = "delta inside the comparison threshold"
617 } else if (violated.length) {
618 reason = `violated ${violated.join(", ")}`
619 }
621 // A holdout is scored only once the selection comparison would keep.
622 let holdout = null
623 let keepEligible = eligible
624 if (decision === "keep" && nextMeasurement === "none" && holdoutConfigured(spec)) {
625 if (input.holdout == null) {
626 decision = "promising"
627 nextMeasurement = "holdout"
628 reason = "selection comparison would keep; held-out confirmation still missing"
629 } else {
630 holdout = confirmHoldout({
631 spec,
632 holdout: input.holdout,
633 required,
634 comparison,
635 aggregation,
636 selectionImproved: improvedNames,
637 })
638 if (!holdout.agrees) {
639 decision = holdout.decision
640 keepEligible = false
641 reason = holdout.reason
642 }
643 }
644 }
646 return {
647 decision,
648 eligible: keepEligible,
649 next_measurement: nextMeasurement,
650 target_reached: targetReached,
651 improved_objectives: improvedNames,
652 violated_objectives: violated,
653 comparisons,
654 primary_delta: primaryComparison?.delta ?? null,
655 rank_score: rankScore(primaryComparison, improved),
656 regressions: regressions(baseline, candidate),
657 holdout,
658 reason,
659 }
662function isCli(argv = process.argv) {
663 const entry = argv[1] ?? ""
664 return entry.endsWith("decide.mjs")
667async function runCli(argv = process.argv, io = process) {
668 const { readFileSync } = await import("node:fs")
669 const source = argv[2] && argv[2] !== "-" ? argv[2] : 0
670 const input = JSON.parse(readFileSync(source, "utf8"))
671 io.stdout.write(`${JSON.stringify(decide(input), null, 2)}\n`)
674if (isCli()) {
675 await runCli()