Skills › Compound Engineering › ce-optimize › example-expensive-benchmark-spec.yaml
example-expensive-benchmark-spec.yaml
88 lines · skills/ce-optimize/references/example-expensive-benchmark-spec.yaml at 3449230
1# Expensive-benchmark template (test-suite wall time, CI critical path, runner-minutes).
2# Use this shape when each evaluation costs minutes and "better" is more than one hard target.
3# Existing single-primary specs stay valid; this file is an opt-in example, not a new default.
5name: reduce-test-suite-wall-time
6description: >
7 Reduce local full-suite wall time without raising the CI critical path
8 or aggregate runner-minutes. A change that helps only CI is eligible
9 if it does not regress the other required targets.
11metric:
12 primary:
13 type: hard
14 name: local_wall_seconds
15 direction: minimize
16 target: 300
17 objectives:
18 - name: local_wall_seconds
19 direction: minimize
20 role: required
21 target: 300
22 - name: ci_critical_path_seconds
23 direction: minimize
24 role: required
25 target: 90
26 - name: runner_minutes
27 direction: minimize
28 role: required
29 target: 40
30 degenerate_gates:
31 - name: suite_passed
32 check: "== 1"
33 description: The full suite must stay green
34 diagnostics:
35 - name: python_group_seconds
36 - name: go_group_seconds
38measurement:
39 command: "python tools/eval/measure_suite.py"
40 timeout_seconds: 1200
41 working_directory: "."
42 stability:
43 mode: ladder
44 repeat_count: 5
45 aggregation: median
46 noise_threshold: 10
47 comparison:
48 method: relative
49 relative_threshold: 0.05
50 ladder:
51 smoke_command: "python tools/eval/measure_suite.py --smoke"
52 exploratory_pairs: 1
53 confirmation_repeats: 5
54 futility:
55 worse_factor: 1.2
57scope:
58 mutable:
59 - "Makefile"
60 - "scripts/test/"
61 - ".github/workflows/"
62 immutable:
63 - "tools/eval/measure_suite.py"
64 - "tests/fixtures/"
66execution:
67 mode: serial
68 backend: worktree
69 max_concurrent: 1
71parallel:
72 port_strategy: none
73 shared_files: []
75dependencies:
76 approved: []
78constraints:
79 - "Do not skip required tests to win a timing metric"
80 - "Do not change the measurement harness"
82stopping:
83 max_iterations: 12
84 max_hours: 8
85 plateau_iterations: 6
86 target_reached: true
88max_runner_up_merges_per_batch: 0