paid-types.ts
132 lines · scripts/lib/paid-types.ts at 3f79d81
1/**
2 * Paid-lane shared types and constants: the leaf module every paid-lane file imports. Moved from scripts/test-paid-shards.ts
3 * so scripts/lib/paid-*.ts never import the CLI module back (no import cycle).
4 */
5import * as path from 'node:path';
6import { normalizeRelativePath, type LanePolicy } from './shard-engine';
7import type { EvalCaseKind, PanelShape, TrialFailureCause, TrialFailureClass, TrialFailureDetail, TrialOutcome, TrialSessionSummary } from '../../test/helpers/eval-store';
9export const ROOT = path.resolve(import.meta.dir, '..', '..');
11export type PaidTier = 'gate' | 'periodic' | 'marathon';
12export const PAID_TIERS: readonly PaidTier[] = ['gate', 'periodic', 'marathon'];
13export type PaidProfile = 'pr' | 'full';
15export interface PaidCaseSelection {
16 e2e: string[] | null;
17 judges: string[] | null;
18}
20export const DEFAULT_TIER: PaidTier = 'gate';
21export const DEFAULT_SHARD_TIMEOUT_MS = 30 * 60_000;
22export const DEFAULT_MAX_FILES_PER_SHARD = 1;
23// 8 jobs × 2 within-shard ≈ 10-13 real in-flight sessions (39 of 75
24// skill-e2e files hold exactly ONE test, so within-shard concurrency is
25// dead weight for most shards) — under the documented-safe ~15 the legacy
26// 40-way runner established. The old 4×4 yielded only ~4-6 in-flight and a
27// 13-wave local gate worst case (~6.5h); 8×2 halves it. Watch the WS1
28// flake telemetry for sustained 429 storms across 2 PR cycles — that is
29// the rollback trigger. Prerequisite (landed): per-shard TMPDIR/
30// CHROMIUM_PROFILE isolation in runPaidShard.
31export const DEFAULT_JOBS = 8;
32export const DEFAULT_WITHIN_SHARD_CONCURRENCY = 2;
34/**
35 * Paid-lane classification policy. Seeds keep only positive walls (a zero
36 * is not a real paid-shard measurement). A shard that passed with zero
37 * executed tests is legitimate under selection (in-file diff/tier
38 * self-skips) and only warns; under EVALS_ALL it is hollow: 'passed-empty'.
39 */
40export const PAID_LANE_POLICY: LanePolicy = {
41 acceptsSeedDuration: (ms) => ms > 0,
42 zeroExecution: ({ promisedAll }) => (promisedAll ? 'passed-empty' : 'passed-with-warning'),
43};
45/** One overlay process preserves the original process-wide SDK semaphore. */
46export const OVERLAY_MAX_ACTIVE_SHARDS = 1;
48export function isOverlayTestFile(file: string): boolean {
49 return /^skill-e2e-overlay-harness-.+\.test\.ts$/.test(path.basename(normalizeRelativePath(file)));
50}
52export interface PaidShardBudget {
53 timeoutMs: number;
54 source: 'explicit' | 'registered' | 'default';
55 policyId: string | null;
56}
58export type ShardStatus =
59 | 'passed'
60 | 'failed'
61 | 'timed-out'
62 | 'never-started'
63 | 'skipped-by-diff'
64 // exit 0 with ZERO executed tests on a run that promised everything
65 // (EVALS_ALL): the hollow-file green the census backstop exists to catch.
66 // Under selective runs, 0-executed passed shards stay 'passed' (in-file
67 // diff/tier self-skips are legitimate there) and get a WARNING line only.
68 | 'passed-empty';
70export interface ShardOutcome {
71 shard: number;
72 files: string[];
73 status: ShardStatus;
74 exitCode: number | null;
75 elapsedMs: number;
76 groupPid: number | null;
77 /** Tests bun reported executing ("Ran N tests ..."), null when unknown. */
78 executedTests: number | null;
79 /** Tests bun reported skipping (" N skip" count line), null when unknown.
80 * "Ran N tests" COUNTS skips, so executedTests alone cannot distinguish a
81 * shard that verified work from one whose every test self-skipped —
82 * codex/gemini files green-by-skip on every CI runner (no binary) and the
83 * weekly census read them as covered. */
84 skippedTests: number | null;
85 /** Effective supervised wall; absent only for unstarted or legacy outcomes. */
86 budget?: PaidShardBudget;
87 /** Present when a verified receipt replaced execution (PR lane only). */
88 reused?: { inputKey: string; runId: string; revision: string; completedAt: number };
89 /** The parent could not run the shard at all (a runner error, never a trial verdict). */
90 runnerError?: string;
91 /** PR lane: the reuse input identity of a freshly executed shard whose inputs stayed unchanged. */
92 inputKey?: string;
93 /** Isolated trial shards only: the trial record this shard produced. */
94 trial?: ShardTrialRecord;
95 /** The slice's in-process deadline (job ceiling minus the upload reserve) decided this outcome:
96 * `not_run` = never started (INFRA), `hung` = killed in flight (TIMEOUT). */
97 sliceDeadline?: 'not_run' | 'hung';
98}
100/**
101 * One isolated trial's record, derived from its shard status and the records
102 * in its own eval dir. `outcome` null means the harness produced no trial
103 * (never started, hollow, isolation broken, runner error): the panel is then
104 * INCOMPLETE and the slice exits non-zero. A failed, timed-out or crashed
105 * trial is a trial verdict; the slice still exits zero and the report decides.
106 */
107export interface ShardTrialRecord {
108 case: string;
109 trial: number;
110 kind: EvalCaseKind;
111 panel: PanelShape;
112 quarantined: boolean;
113 outcome: TrialOutcome | null;
114 harness?: string;
115 failure_class?: TrialFailureClass;
116 exit_reason?: string;
117 error?: string;
118 timeout_at_turn?: number;
119 cost_usd: number;
120 duration_ms: number;
121 model?: string;
122 failure_cause?: TrialFailureCause;
123 failure_cause_evidence?: string;
124 failure_detail?: TrialFailureDetail;
125 sessions?: TrialSessionSummary[];
126 cost_known?: false;
127}
129/** Upload reserve between a slice's in-process deadline and its CI job ceiling (ENG-2). */
130export const SLICE_UPLOAD_RESERVE_MS = 5 * 60_000;
131/** Env var the executor workflows set at their first step: the job's start in epoch seconds. */
132export const SLICE_JOB_STARTED_AT_ENV = 'GSTACK_SLICE_JOB_STARTED_AT';