judge-calibration-configs.ts
117 lines · scripts/judge-calibration-configs.ts at 3f79d81
1/**
2 * W2 calibration request configurations, keyed by distinct request (prompt
3 * builder, model, max_tokens, stream, effort, thresholds), not by test case.
4 * Each config builds prompts through the same builder the eval sends; "old" is
5 * today's request and "new" adds only the JSON schema (comparison 1).
6 */
7import {
8 ARM_JUDGE_MODEL, ARM_JUDGE_SCHEMA, buildArmJudgePrompt, parseArmJudgeResponse,
9 buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt,
10 buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt,
11 JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA,
12 VOICE_DIRECTIVE_JUDGE_SCHEMA, VOICE_DIRECTIVE_DIMENSIONS, JUDGE_SCORE_DIMENSIONS, DEFAULT_JUDGE_MAX_TOKENS,
13 judgePanelMedian, judgePanelMajority,
14} from '../test/helpers/llm-judge';
15import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
16import { COOKIE_WORKFLOW_JUDGE } from '../test/helpers/cookie-workflow-judge-input';
17import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
18import type { CalibrationConfig, Verdict } from './judge-calibration';
20type Sample = Record<string, unknown>;
22/** What production rejects: judgePanelMedian needs finite numbers, judgePanelMajority needs booleans. */
23function requireFields(value: unknown, numbers: readonly string[], booleans: readonly string[] = []): void {
24 const sample = (value ?? {}) as Sample;
25 for (const key of numbers) {
26 if (typeof sample[key] !== 'number' || !Number.isFinite(sample[key])) throw new Error(`non-numeric ${key}: ${JSON.stringify(sample[key])}`);
27 }
28 for (const key of booleans) {
29 if (typeof sample[key] !== 'boolean') throw new Error(`non-boolean ${key}: ${JSON.stringify(sample[key])}`);
30 }
31}
33const meets = (median: Record<string, number>, floors: Record<string, number>): Verdict =>
34 Object.entries(floors).every(([key, floor]) => median[key]! >= floor) ? 'pass' : 'fail';
36const QA_FLOORS = { clarity: 3, completeness: 3, actionability: 4 };
37const scoreJudge = (floors: Record<string, number>) => ({
38 validate: (value: unknown) => requireFields(value, JUDGE_SCORE_DIMENSIONS),
39 verdict: (samples: unknown[]) => meets(judgePanelMedian<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
40});
42const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
43 id, priority, cases,
44 model: extra.model ?? CLAUDE_JUDGE_EVAL_MODEL,
45 build: inputs => buildWorkflowJudgePrompt({ judgeContext: inputs.judgeContext!, judgeGoal: inputs.judgeGoal!, agentCapability: extra.agentCapability },
46 { files: [], text: inputs.bundle! }),
47 oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
48 newOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
49 ...scoreJudge(extra.floors ?? QA_FLOORS),
50 estimatedOutputTokens: 1200,
51});
53export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
54 {
55 id: 'arm', priority: 1, cases: ['arm-benchmark (skill-e2e-arm-benchmark.test.ts)'], model: ARM_JUDGE_MODEL,
56 build: inputs => buildArmJudgePrompt(inputs.task!.trimEnd(), inputs.diff!),
57 oldOptions: {},
58 newOptions: { jsonSchema: ARM_JUDGE_SCHEMA },
59 validate: value => { parseArmJudgeResponse(value); },
60 verdict: samples => {
61 const scores = samples.map(sample => parseArmJudgeResponse(sample).over_engineering).sort((a, b) => a - b);
62 return scores[Math.floor(scores.length / 2)]! <= 1 ? 'pass' : 'fail';
63 },
64 estimatedOutputTokens: 300,
65 },
66 {
67 id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_JUDGE_EVAL_MODEL,
68 build: inputs => buildQaWorkflowJudgePrompt(inputs.section!),
69 oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
70 ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
71 },
72 {
73 id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_JUDGE_EVAL_MODEL,
74 build: inputs => buildQaHealthRubricJudgePrompt(inputs.section!),
75 oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
76 ...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
77 },
78 {
79 id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_JUDGE_EVAL_MODEL,
80 build: inputs => buildQaAntiRefusalJudgePrompt(inputs.diffAwareSection!, inputs.rulesSection!),
81 oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
82 validate: value => requireFields(value, ['confidence'], ['would_browse']),
83 verdict: samples => judgePanelMajority<string>(samples as Sample[], 'would_browse')
84 && judgePanelMedian<string>(samples as Sample[], ['confidence']).confidence >= 4 ? 'pass' : 'fail',
85 estimatedOutputTokens: 1200,
86 },
87 {
88 id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_JUDGE_EVAL_MODEL,
89 build: inputs => buildCrossSkillConsistencyJudgePrompt(inputs.collected!),
90 oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
91 validate: value => requireFields(value, ['score'], ['consistent']),
92 verdict: samples => judgePanelMajority<string>(samples as Sample[], 'consistent')
93 && judgePanelMedian<string>(samples as Sample[], ['score']).score >= 4 ? 'pass' : 'fail',
94 estimatedOutputTokens: 1200,
95 },
96 {
97 id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_JUDGE_EVAL_MODEL,
98 build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
99 oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
100 validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),
101 verdict: samples => meets(judgePanelMedian<string>(samples as Sample[], VOICE_DIRECTIVE_DIMENSIONS),
102 Object.fromEntries(VOICE_DIRECTIVE_DIMENSIONS.map(key => [key, 4]))),
103 estimatedOutputTokens: 1200,
104 },
105 workflow('workflow-default', 7, ['document-release/SKILL.md workflow', 'plan-ceo-review/SKILL.md modes', 'plan-eng-review/SKILL.md sections',
106 'plan-design-review/SKILL.md passes', 'design-review/SKILL.md fix loop', 'design-consultation/SKILL.md research',
107 'land-and-deploy/SKILL.md workflow', 'canary/SKILL.md monitoring loop', 'benchmark/SKILL.md perf collection',
108 'setup-deploy/SKILL.md platform setup', 'sync-gbrain/SKILL.md read-only readiness', 'retro/SKILL.md instructions',
109 'qa-only/SKILL.md workflow', 'gstack-upgrade/SKILL.md upgrade flow']),
110 {
111 id: 'cookie-workflow', priority: 8, cases: ['setup-browser-cookies/SKILL.md workflow'], model: COOKIE_WORKFLOW_JUDGE.model,
112 build: inputs => buildWorkflowJudgePrompt(COOKIE_WORKFLOW_JUDGE, { files: [], text: inputs.bundle! }),
113 oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
114 newOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
115 ...scoreJudge(COOKIE_WORKFLOW_JUDGE.thresholds), estimatedOutputTokens: 1200,
116 },
117];