From af83a04b0b8ee5148ec79b778069e6de2046eb7e Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Mon, 20 Jul 2026 18:33:10 +0000 Subject: [PATCH] Forward live benchmark timeout overrides --- bench/README.md | 13 ++++ bench/src/live.test.ts | 152 ++++++++++++++++++++++++++++++++++++++++- bench/src/live.ts | 104 +++++++++++++++++++++++++++- bench/src/run.ts | 2 + 4 files changed, 267 insertions(+), 4 deletions(-) diff --git a/bench/README.md b/bench/README.md index dbc52ed..41b0de9 100644 --- a/bench/README.md +++ b/bench/README.md @@ -331,6 +331,12 @@ REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- \ --screen-profile ../provisional-models.json \ --case prompt-injection-auth-bypass \ --case near-duplicate-auth-clean +# Keep provider calls inside the live screen's 180-second case watchdog. +POSTIL_LLM_REQUEST_TIMEOUT_SECS=60 POSTIL_LLM_TOTAL_TIMEOUT_SECS=170 \ + REVIEW_MODEL=z-ai/glm-5.2 bun run bench:live -- \ + --run-id glm-5-2-fireworks-bounded-timeouts \ + --screen-profile ../provisional-models.json \ + --case prompt-injection-auth-bypass # A profile with a scorerChain can exercise the production scorer path. REVIEW_MODEL=provider/generator bun run bench:live -- \ --screen-profile ./screen-profile.json \ @@ -364,6 +370,13 @@ binary's SHA-256 digest so the report cannot be paired with a different executable during admission. Fixture-corpus and evaluator-source digests bind the results to the benchmark inputs and scoring code. +`POSTIL_LLM_REQUEST_TIMEOUT_SECS` and `POSTIL_LLM_TOTAL_TIMEOUT_SECS` are +optional canonical positive integer seconds. Explicit values must expire before +the per-case process watchdog, and the request timeout cannot exceed the total +timeout. The harness forwards only these validated values to each isolated +child. Unset values remain unset so the CLI owns its defaults. `run.json` and +the aggregate report retain the exact overrides without recording credentials. + `--case ` selects one exact fixture and may be repeated. Selected cases require `--screen-profile `. The profile binds the model chain, scorer chain, exact upstream provider, canonical managed endpoint, and price diff --git a/bench/src/live.test.ts b/bench/src/live.test.ts index 80411d5..7a244a6 100644 --- a/bench/src/live.test.ts +++ b/bench/src/live.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { chmod, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -12,10 +12,35 @@ import { liveCostAccountingComplete, liveReviewArguments, runLive, + resolveLiveTimeoutOverrides, scorerOperationalFailure, validateLiveRunId, } from "./live"; +async function fakeEnvironmentBinary(root: string, markerPath?: string): Promise { + const path = join(root, "fake-postil"); + await writeFile(path, `#!/bin/sh +${markerPath === undefined ? "" : `printf invoked > '${markerPath}'`} +printf 'request=%s\\ntotal=%s\\nmodel_key=%s\\npostil_key=%s\\nunrelated=%s\\nendpoint_auth=%s\\naws_secret=%s\\n' \\ + "\${POSTIL_LLM_REQUEST_TIMEOUT_SECS-absent}" \\ + "\${POSTIL_LLM_TOTAL_TIMEOUT_SECS-absent}" \\ + "\${MODEL_API_KEY:+set}" \\ + "\${POSTIL_API_KEY:+set}" \\ + "\${UNRELATED_BENCH_SECRET:+set}" \\ + "\${POSTIL_ENDPOINT_AUTH_VALUE:+set}" \\ + "\${AWS_SECRET_ACCESS_KEY:+set}" >&2 +`, { mode: 0o700 }); + await chmod(path, 0o700); + return path; +} + +function restoreEnvironment(previous: Record): void { + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + async function onlyCaseAttempt(runRoot: string): Promise { const caseDirectories = (await readdir(runRoot, { withFileTypes: true })) .filter((entry) => entry.isDirectory()); @@ -24,6 +49,131 @@ async function onlyCaseAttempt(runRoot: string): Promise { } describe("live benchmark review mode", () => { + test("forwards explicit timeout overrides to the isolated child and records them", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-live-timeout-forwarding-")); + const names = [ + "MODEL_API_KEY", + "POSTIL_LLM_REQUEST_TIMEOUT_SECS", + "POSTIL_LLM_TOTAL_TIMEOUT_SECS", + "UNRELATED_BENCH_SECRET", + "POSTIL_ENDPOINT_AUTH_VALUE", + "AWS_SECRET_ACCESS_KEY", + ]; + const previous = Object.fromEntries(names.map((name) => [name, process.env[name]])); + try { + process.env.MODEL_API_KEY = "allowed-test-key"; + process.env.POSTIL_LLM_REQUEST_TIMEOUT_SECS = "1"; + process.env.POSTIL_LLM_TOTAL_TIMEOUT_SECS = "2"; + process.env.UNRELATED_BENCH_SECRET = "must-not-arrive"; + process.env.POSTIL_ENDPOINT_AUTH_VALUE = "must-not-arrive"; + process.env.AWS_SECRET_ACCESS_KEY = "must-not-arrive"; + const binary = await fakeEnvironmentBinary(root); + const report = await runLive([cases[0]!], { + binary, + model: "test/model", + rootDir: root, + runId: "explicit-timeouts", + timeoutMs: 3_000, + concurrency: 1, + retries: 1, + }); + + const runRoot = join(root, "live", "explicit-timeouts"); + const attempt = await onlyCaseAttempt(runRoot); + expect(await readFile(join(attempt, "stderr.log"), "utf8")).toBe( + "request=1\ntotal=2\nmodel_key=set\npostil_key=set\n" + + "unrelated=\nendpoint_auth=\naws_secret=\n", + ); + expect(await readFile(join(attempt, "..", "attempt-2", "stderr.log"), "utf8")).toBe( + await readFile(join(attempt, "stderr.log"), "utf8"), + ); + expect(report.summary.timeoutOverrides).toEqual({ + requestSeconds: "1", + totalSeconds: "2", + caseProcessMilliseconds: 3_000, + }); + expect(JSON.parse(await readFile(join(runRoot, "run.json"), "utf8")).timeoutOverrides) + .toEqual(report.summary.timeoutOverrides); + } finally { + restoreEnvironment(previous); + await rm(root, { recursive: true, force: true }); + } + }); + + test("leaves absent timeout overrides absent so the CLI owns its defaults", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-live-timeout-defaults-")); + const names = [ + "MODEL_API_KEY", + "POSTIL_LLM_REQUEST_TIMEOUT_SECS", + "POSTIL_LLM_TOTAL_TIMEOUT_SECS", + ]; + const previous = Object.fromEntries(names.map((name) => [name, process.env[name]])); + try { + process.env.MODEL_API_KEY = "allowed-test-key"; + delete process.env.POSTIL_LLM_REQUEST_TIMEOUT_SECS; + delete process.env.POSTIL_LLM_TOTAL_TIMEOUT_SECS; + const binary = await fakeEnvironmentBinary(root); + const report = await runLive([cases[0]!], { + binary, + model: "test/model", + rootDir: root, + runId: "default-timeouts", + timeoutMs: 3_000, + concurrency: 1, + retries: 0, + }); + + const attempt = await onlyCaseAttempt(join(root, "live", "default-timeouts")); + expect(await readFile(join(attempt, "stderr.log"), "utf8")).toContain( + "request=absent\ntotal=absent\n", + ); + expect(report.summary.timeoutOverrides).toEqual({ + requestSeconds: null, + totalSeconds: null, + caseProcessMilliseconds: 3_000, + }); + } finally { + restoreEnvironment(previous); + await rm(root, { recursive: true, force: true }); + } + }); + + test("rejects malformed or unsafe timeout overrides before invoking the child", async () => { + const root = await mkdtemp(join(tmpdir(), "postil-live-timeout-rejection-")); + const marker = join(root, "invoked"); + const names = [ + "MODEL_API_KEY", + "POSTIL_LLM_REQUEST_TIMEOUT_SECS", + "POSTIL_LLM_TOTAL_TIMEOUT_SECS", + ]; + const previous = Object.fromEntries(names.map((name) => [name, process.env[name]])); + try { + process.env.MODEL_API_KEY = "allowed-test-key"; + const binary = await fakeEnvironmentBinary(root, marker); + for (const [index, raw] of ["", "0", "01", "1.5", " 1", "3"].entries()) { + process.env.POSTIL_LLM_REQUEST_TIMEOUT_SECS = raw; + delete process.env.POSTIL_LLM_TOTAL_TIMEOUT_SECS; + await expect(runLive([cases[0]!], { + binary, + model: "test/model", + rootDir: root, + runId: `invalid-timeout-${index}`, + timeoutMs: 3_000, + concurrency: 1, + retries: 0, + })).rejects.toThrow("POSTIL_LLM_REQUEST_TIMEOUT_SECS"); + } + expect(() => resolveLiveTimeoutOverrides(5_000, { + POSTIL_LLM_REQUEST_TIMEOUT_SECS: "4", + POSTIL_LLM_TOTAL_TIMEOUT_SECS: "3", + })).toThrow("must not exceed"); + await expect(readFile(marker, "utf8")).rejects.toThrow(); + } finally { + restoreEnvironment(previous); + await rm(root, { recursive: true, force: true }); + } + }); + test("keeps cost completeness independent from review outcome", () => { expect(liveCostAccountingComplete([{ costAccountingComplete: true }])).toBe(true); expect(liveCostAccountingComplete([ diff --git a/bench/src/live.ts b/bench/src/live.ts index ce3486b..b64e780 100644 --- a/bench/src/live.ts +++ b/bench/src/live.ts @@ -41,6 +41,9 @@ import { const execFile = promisify(execFileCb); const ADMISSION_API_BASE = "https://openrouter.ai:443/api/v1"; +const REQUEST_TIMEOUT_ENV = "POSTIL_LLM_REQUEST_TIMEOUT_SECS"; +const TOTAL_TIMEOUT_ENV = "POSTIL_LLM_TOTAL_TIMEOUT_SECS"; +const DEFAULT_CASE_TIMEOUT_MS = 180_000; /** Default number of cases run concurrently. Live inference is I/O-bound on the * provider, so a small pool cuts wall-clock time without overloading the API. @@ -170,6 +173,7 @@ export interface LiveSummary { providerContractEnforced: boolean; screeningProfileSha256: string | null; upstreamProviderIdentity: string | null; + timeoutOverrides: LiveTimeoutOverrides; ranAt: string; totalCases: number; defectCases: number; @@ -202,6 +206,12 @@ export interface LiveReport { results: LiveCaseResult[]; } +export interface LiveTimeoutOverrides { + requestSeconds: string | null; + totalSeconds: string | null; + caseProcessMilliseconds: number; +} + // --------------------------------------------------------------------------- // Entry point @@ -216,6 +226,7 @@ export async function runLive( "(it is never logged or printed). Mock mode (bun run bench) needs no key.", ); } + const timeoutOverrides = resolveLiveTimeoutOverrides(options.timeoutMs); const cases = inputs.map((input) => benchmarkCase.parse(input)); const provider = liveProvider(); const screeningProfile = options.screenProfilePath === undefined @@ -232,6 +243,7 @@ export async function runLive( const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs"); const runRoot = await reserveLiveRunRoot(rootDir, options.runId); + await writeLiveRunContract(runRoot, options, timeoutOverrides); // Bounded worker pool: a small fixed number of workers pull case indices off a // shared cursor until the queue drains. Each case writes its result into the @@ -243,7 +255,13 @@ export async function runLive( for (;;) { const index = cursor++; if (index >= cases.length) return; - results[index] = await runLiveCaseWithRetry(cases[index]!, index, runRoot, options); + results[index] = await runLiveCaseWithRetry( + cases[index]!, + index, + runRoot, + options, + timeoutOverrides, + ); } }; await Promise.all(Array.from({ length: concurrency }, () => worker())); @@ -267,11 +285,79 @@ export async function runLive( evaluatorSha256, provider, screeningProfile, + timeoutOverrides, ), results: ordered, }; } +const CANONICAL_POSITIVE_SECONDS = /^[1-9][0-9]*$/u; + +export function resolveLiveTimeoutOverrides( + timeoutMs: number | undefined, + env: NodeJS.ProcessEnv = process.env, +): LiveTimeoutOverrides { + const caseProcessMilliseconds = timeoutMs ?? DEFAULT_CASE_TIMEOUT_MS; + if (!Number.isSafeInteger(caseProcessMilliseconds) || caseProcessMilliseconds <= 0) { + throw new Error("live benchmark case timeout must be a positive integer number of milliseconds"); + } + const requestSeconds = validateTimeoutOverride( + REQUEST_TIMEOUT_ENV, + env[REQUEST_TIMEOUT_ENV], + caseProcessMilliseconds, + ); + const totalSeconds = validateTimeoutOverride( + TOTAL_TIMEOUT_ENV, + env[TOTAL_TIMEOUT_ENV], + caseProcessMilliseconds, + ); + if ( + requestSeconds !== null && + totalSeconds !== null && + Number(requestSeconds) > Number(totalSeconds) + ) { + throw new Error(`${REQUEST_TIMEOUT_ENV} must not exceed ${TOTAL_TIMEOUT_ENV}`); + } + return Object.freeze({ requestSeconds, totalSeconds, caseProcessMilliseconds }); +} + +function validateTimeoutOverride( + name: string, + raw: string | undefined, + caseProcessMilliseconds: number, +): string | null { + if (raw === undefined) return null; + if (!CANONICAL_POSITIVE_SECONDS.test(raw)) { + throw new Error(`${name} must be a canonical positive integer number of seconds`); + } + const seconds = Number(raw); + if (!Number.isSafeInteger(seconds) || seconds * 1_000 >= caseProcessMilliseconds) { + throw new Error( + `${name} must expire before the ${caseProcessMilliseconds}ms live benchmark case timeout`, + ); + } + return raw; +} + +async function writeLiveRunContract( + runRoot: string, + options: LiveOptions, + timeoutOverrides: LiveTimeoutOverrides, +): Promise { + const contract = { + artifactType: "diff-file-live-run", + runId: options.runId, + model: options.model, + scorerModel: options.scorerModel ?? null, + reviewMode: options.bounded ? "bounded" : "exhaustive", + timeoutOverrides, + }; + await writeFile(join(runRoot, "run.json"), `${JSON.stringify(contract, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); +} + async function screeningProfileMetadata(path: string): Promise<{ sha256: string; upstreamProviderIdentity: string; @@ -324,11 +410,12 @@ async function runLiveCaseWithRetry( index: number, runRoot: string, options: LiveOptions, + timeoutOverrides: LiveTimeoutOverrides, ): Promise { const maxRetries = options.retries ?? DEFAULT_LIVE_RETRIES; let last: LiveCaseResult | undefined; for (let attempt = 0; attempt <= maxRetries; attempt++) { - last = await runLiveCase(c, index, attempt + 1, runRoot, options); + last = await runLiveCase(c, index, attempt + 1, runRoot, options, timeoutOverrides); // Scored => a valid envelope was produced; that is a normal result (even if // it has findings or false positives), so never retry it. if (last.scored) return last; @@ -432,6 +519,7 @@ async function runLiveCase( attempt: number, runRoot: string, options: LiveOptions, + timeoutOverrides: LiveTimeoutOverrides, ): Promise { const truth = groundTruthOf(c); const base: LiveCaseResult = { @@ -481,8 +569,9 @@ async function runLiveCase( homeDir, tmpDir, liveProvider(), + timeoutOverrides, ), - timeout: options.timeoutMs ?? 180_000, + timeout: timeoutOverrides.caseProcessMilliseconds, maxBuffer: 8 * 1024 * 1024, }, ); @@ -637,6 +726,7 @@ function liveEnv( homeDir: string, tmpDir: string, provider: LiveProvider, + timeoutOverrides: LiveTimeoutOverrides, ): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { PATH: process.env.PATH, @@ -652,6 +742,12 @@ function liveEnv( REVIEW_MODEL: model, POSTIL_API_BASE: provider.apiBase, POSTIL_API_FORMAT: provider.apiFormat, + ...(timeoutOverrides.requestSeconds === null + ? {} + : { [REQUEST_TIMEOUT_ENV]: timeoutOverrides.requestSeconds }), + ...(timeoutOverrides.totalSeconds === null + ? {} + : { [TOTAL_TIMEOUT_ENV]: timeoutOverrides.totalSeconds }), ...(scorerModel === undefined ? { POSTIL_DISABLE_SCORER: "1" } : { REVIEW_SCORER_MODEL: scorerModel }), @@ -756,6 +852,7 @@ function summarize( evaluatorSha256: string, provider: LiveProvider, screeningProfile: { sha256: string; upstreamProviderIdentity: string } | null, + timeoutOverrides: LiveTimeoutOverrides, ): LiveSummary { const defects = results.filter((r) => r.type === "defect"); const cleans = results.filter((r) => r.type === "clean"); @@ -800,6 +897,7 @@ function summarize( providerContractEnforced: options.screenProfilePath !== undefined, screeningProfileSha256: screeningProfile?.sha256 ?? null, upstreamProviderIdentity: screeningProfile?.upstreamProviderIdentity ?? null, + timeoutOverrides, ranAt: new Date().toISOString(), totalCases: results.length, defectCases: defects.length, diff --git a/bench/src/run.ts b/bench/src/run.ts index a970b39..268d192 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -42,6 +42,8 @@ // BENCH_CONCURRENCY live-mode case parallelism (else --concurrency, else default) // POSTIL_BENCH_BOUNDED set to 1 to qualify the bounded large-review path // POSTIL_BENCH_SCREEN_RUN_ID optional live-screen artifact namespace +// POSTIL_LLM_REQUEST_TIMEOUT_SECS explicit per-request live-screen timeout +// POSTIL_LLM_TOTAL_TIMEOUT_SECS explicit total live-screen LLM timeout // --case repeatable non-admission fixture selection // --screen-profile exact provider and price contract for selected cases // --scorer-model optional scorer for non-admission diff-file screening