From a6ca8c1b2569b52f879a5db426b6c01cf6849474 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Thu, 16 Jul 2026 12:44:37 +0000 Subject: [PATCH] Align qualification budget with runtime exposure --- .github/workflows/bench-live.yml | 2 +- bench/README.md | 2 +- bench/src/livemodels-score.ts | 2 +- bench/src/livemodels.test.ts | 13 +++++++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index 472db9f..1a2d3af 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -14,7 +14,7 @@ on: cost_cap_usd: description: Abort when projected qualification spend exceeds this amount required: true - default: "55" + default: "70" concurrency: group: bench-live-admission diff --git a/bench/README.md b/bench/README.md index 10608df..9141547 100644 --- a/bench/README.md +++ b/bench/README.md @@ -214,7 +214,7 @@ plan for every fixture before inference. It includes bounded planner, selected source and synthesis requests, scoring, consensus, fallback, repair, and transport retries. It rejects missing prices, more than six models, a review above the $1 hosted operation cap, a total above the configured qualification -cap, or a cap outside `(0, $55]`. A single model used for more than one role is +cap, or a cap outside `(0, $70]`. A single model used for more than one role is priced for each planned invocation. Atomic attribution accepts at most three findings anchored in one authored region. More is a fidelity failure. Each decision is limited to a 4 KiB input, diff --git a/bench/src/livemodels-score.ts b/bench/src/livemodels-score.ts index d816c1c..b7e4f88 100644 --- a/bench/src/livemodels-score.ts +++ b/bench/src/livemodels-score.ts @@ -917,7 +917,7 @@ export function pricingFromZdrCatalog( return out; } -export const MAX_GENERATOR_COST_CAP_USD = 55; +export const MAX_GENERATOR_COST_CAP_USD = 70; export const MAX_GENERATOR_CANDIDATES = 6; /** Normalize candidate ids once before pricing, job creation, or aggregation. */ diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 2d201d2..5efad15 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -44,6 +44,7 @@ import { } from "./livemodels"; import { compareCanonicalDecimals, + MAX_GENERATOR_COST_CAP_USD, parseCanonicalDecimal, type QualificationPair, } from "./livemodels-score"; @@ -305,9 +306,9 @@ describe("pair qualification configuration", () => { binary: "/missing/postil", pairs: [pair], pricing: new Map(), - costCapUsd: 56, + costCapUsd: 71, upstreamProvider: "PinnedProvider", - })).rejects.toThrow("cost cap must be greater than zero and at most $55"); + })).rejects.toThrow("cost cap must be greater than zero and at most $70"); const pairs = Array.from({ length: 7 }, (_, index) => ({ generatorModel: `generator/${index}`, @@ -357,7 +358,7 @@ describe("pair qualification configuration", () => { pricing, apiBase: normalizeApiBase("https://openrouter.ai/api/v1"), apiFormat: "openai-compatible", - costCapUsdDecimal: "55", + costCapUsdDecimal: "70", upstreamProvider: "PinnedProvider", }); expect(compareCanonicalDecimals( @@ -366,7 +367,7 @@ describe("pair qualification configuration", () => { )).toBeGreaterThan(0); expect(compareCanonicalDecimals( parseCanonicalDecimal(projected), - parseCanonicalDecimal("55"), + parseCanonicalDecimal("70"), )).toBeLessThanOrEqual(0); } finally { if (inheritedModelKey === undefined) delete process.env.MODEL_API_KEY; @@ -823,6 +824,10 @@ describe("managed admission workflow", () => { expect(workflow).toContain("POSTIL_API_FORMAT: openai-compatible"); expect(workflow).toContain("POSTIL_BENCH_REPEATS: \"3\""); expect(workflow).toContain("POSTIL_BENCH_PAIRS: ${{ inputs.pairs }}"); + expect(workflow).toMatch(new RegExp( + `^ {6}cost_cap_usd:\\n(?: {8}.*\\n){2} {8}default: "${MAX_GENERATOR_COST_CAP_USD}"$`, + "mu", + )); expect(workflow).toContain("upstream_provider:"); expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }}"); expect(workflow).toContain('echo "POSTIL_MANIFEST_OUT=${RUNNER_TEMP}/postil-qualified-models-${suffix}.json"');