Skills › Compound Engineering › ce-optimize › example-text-target-spec.yaml
example-text-target-spec.yaml
80 lines · skills/ce-optimize/references/example-text-target-spec.yaml at 3449230
1# Instruction-text target: a SKILL.md scored on a fixed case set.
2# Start here when the mutable files are instructions a model runs under.
3# See references/text-targets.md for the eval-set coverage rule, model roles,
4# and the hypothesis moves this spec's categories name.
6name: tighten-triage-skill
7description: Raise the pass rate of the issue-triage skill on recorded failures without growing the prompt
9metric:
10 primary:
11 type: hard
12 name: pass_rate # fraction of selection cases the task model passes
13 direction: maximize
14 target: 0.95 # a 1.0 here means the case set stopped separating candidates
15 objectives:
16 - name: prompt_tokens
17 direction: minimize
18 role: secondary # logged on every experiment; make it required to forbid growth
19 degenerate_gates:
20 - name: cases_run
21 check: ">= 24"
22 description: Every selection case ran; a partial run is not a measurement
23 - name: skill_parses
24 check: "== 1"
25 description: Frontmatter and body still parse as a skill
26 diagnostics:
27 - name: mean_output_tokens
28 - name: cases_ambiguous
30measurement:
31 # Runs every case in eval/triage/selection.jsonl under the pinned task model
32 # and prints {"pass_rate", "prompt_tokens", "cases_run", "skill_parses",
33 # "mean_output_tokens", "cases_ambiguous", "cases": {"<id>": true|false}}.
34 command: "python eval/triage/run_cases.py --cases eval/triage/selection.jsonl --model task-model-pinned-in-script"
35 timeout_seconds: 900
36 working_directory: "."
37 per_case: true # decide.mjs reports cases the baseline passed and a candidate fails
38 holdout:
39 # Same script, a case file the selection command never reads. Scored only
40 # before a keep and at final confirmation.
41 command: "python eval/triage/run_cases.py --cases eval/triage/holdout.jsonl --model task-model-pinned-in-script"
42 stability:
43 mode: repeat
44 repeat_count: 3 # the task model is not deterministic; aggregate
45 aggregation: median
46 comparison:
47 method: absolute
48 noise_threshold: 0.04 # one case in 25
50scope:
51 mutable:
52 - "skills/triage/SKILL.md"
53 - "skills/triage/references/"
54 immutable:
55 - "eval/triage/" # cases, holdout, and the runner are the artifact the run keeps
56 - "tests/"
58execution:
59 mode: serial
60 backend: worktree
61 max_concurrent: 1
63dependencies:
64 approved: []
66constraints:
67 - "Keep the skill's name and description frontmatter fields unchanged"
68 - "Every rule kept in the body must have at least one case in eval/triage/ that fails without it"
70# Hypothesis categories for this target are the moves in text-targets.md:
71# add-examples, bootstrap-examples, rewrite-from-feedback, shorten,
72# restructure, decompose, vote, external-optimizer.
74stopping:
75 max_iterations: 8
76 max_hours: 2
77 plateau_iterations: 4
78 target_reached: true
80max_runner_up_merges_per_batch: 0