Skills › Compound Engineering › ce-optimize › example-judge-spec.yaml
example-judge-spec.yaml
83 lines · skills/ce-optimize/references/example-judge-spec.yaml at 3449230
1# Minimal first-run template for qualitative metrics.
2# Start here when true quality requires semantic judgment, not a proxy metric.
4name: improve-search-relevance
5description: Improve semantic relevance of search results without obvious failures
7metric:
8 primary:
9 type: judge
10 name: mean_score
11 direction: maximize
12 degenerate_gates:
13 - name: result_count
14 check: ">= 5"
15 description: Return enough results to judge quality
16 - name: empty_query_failures
17 check: "== 0"
18 description: Empty or trivial queries must not fail
19 diagnostics:
20 - name: latency_ms
21 - name: recall_at_10
22 judge:
23 rubric: |
24 Rate each result set from 1-5 for relevance:
25 - 5: Results are directly relevant and well ordered
26 - 4: Mostly relevant with minor ordering issues
27 - 3: Mixed relevance or one obvious miss
28 - 2: Weak relevance, several misses, or poor ordering
29 - 1: Mostly irrelevant
30 Also report: ambiguous (boolean)
31 scoring:
32 primary: mean_score
33 secondary:
34 - ambiguous_rate
35 model: cheap # capability tier; the harness picks the concrete model
36 sample_size: 10
37 batch_size: 5
38 sample_seed: 42
39 confirmation_seed: 7 # held-out sample, scored only before a keep and at final confirmation
40 calibration:
41 labels: "tools/eval/judge-labels.yaml" # optional: your own scores for some outputs; omit the block and Phase 1 offers a guided check
42 min_agreement: 0.8
43 minimum_improvement: 0.2
44 max_total_cost_usd: 5
46measurement:
47 command: "python eval_search.py"
48 timeout_seconds: 300
49 working_directory: "tools/eval"
51scope:
52 mutable:
53 - "src/search/"
54 - "config/search.yaml"
55 immutable:
56 - "tools/eval/eval_search.py"
57 - "tools/eval/judge-labels.yaml"
58 - "tests/fixtures/"
59 - "docs/"
61execution:
62 mode: serial
63 backend: worktree
64 max_concurrent: 1
66parallel:
67 port_strategy: none
68 shared_files: []
70dependencies:
71 approved: []
73constraints:
74 - "Preserve the existing search response shape"
75 - "Do not add new dependencies on the first run"
77stopping:
78 max_iterations: 4
79 max_hours: 1
80 plateau_iterations: 3
81 target_reached: true
83max_runner_up_merges_per_batch: 0