Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/bench-live.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ on:
cost_cap_usd:
description: Abort when projected qualification spend exceeds this amount
required: true
default: "55"
default: "70"

concurrency:
group: bench-live-admission
Expand Down
2 changes: 1 addition & 1 deletion bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ plan for every fixture before inference. It includes bounded planner, selected
source and synthesis requests, scoring, consensus, fallback, repair, and
transport retries. It rejects missing prices, more than six models, a review
above the $1 hosted operation cap, a total above the configured qualification
cap, or a cap outside `(0, $55]`. A single model used for more than one role is
cap, or a cap outside `(0, $70]`. A single model used for more than one role is
priced for each planned invocation.
Atomic attribution accepts at most three findings anchored in one authored
region. More is a fidelity failure. Each decision is limited to a 4 KiB input,
Expand Down
2 changes: 1 addition & 1 deletion bench/src/livemodels-score.ts
Original file line number Diff line number Diff line change
Expand Up @@ -917,7 +917,7 @@ export function pricingFromZdrCatalog(
return out;
}

export const MAX_GENERATOR_COST_CAP_USD = 55;
export const MAX_GENERATOR_COST_CAP_USD = 70;
export const MAX_GENERATOR_CANDIDATES = 6;

/** Normalize candidate ids once before pricing, job creation, or aggregation. */
Expand Down
13 changes: 9 additions & 4 deletions bench/src/livemodels.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import {
} from "./livemodels";
import {
compareCanonicalDecimals,
MAX_GENERATOR_COST_CAP_USD,
parseCanonicalDecimal,
type QualificationPair,
} from "./livemodels-score";
Expand Down Expand Up @@ -305,9 +306,9 @@ describe("pair qualification configuration", () => {
binary: "/missing/postil",
pairs: [pair],
pricing: new Map(),
costCapUsd: 56,
costCapUsd: 71,
upstreamProvider: "PinnedProvider",
})).rejects.toThrow("cost cap must be greater than zero and at most $55");
})).rejects.toThrow("cost cap must be greater than zero and at most $70");

const pairs = Array.from({ length: 7 }, (_, index) => ({
generatorModel: `generator/${index}`,
Expand Down Expand Up @@ -357,7 +358,7 @@ describe("pair qualification configuration", () => {
pricing,
apiBase: normalizeApiBase("https://openrouter.ai/api/v1"),
apiFormat: "openai-compatible",
costCapUsdDecimal: "55",
costCapUsdDecimal: "70",
upstreamProvider: "PinnedProvider",
});
expect(compareCanonicalDecimals(
Expand All @@ -366,7 +367,7 @@ describe("pair qualification configuration", () => {
)).toBeGreaterThan(0);
expect(compareCanonicalDecimals(
parseCanonicalDecimal(projected),
parseCanonicalDecimal("55"),
parseCanonicalDecimal("70"),
)).toBeLessThanOrEqual(0);
} finally {
if (inheritedModelKey === undefined) delete process.env.MODEL_API_KEY;
Expand Down Expand Up @@ -823,6 +824,10 @@ describe("managed admission workflow", () => {
expect(workflow).toContain("POSTIL_API_FORMAT: openai-compatible");
expect(workflow).toContain("POSTIL_BENCH_REPEATS: \"3\"");
expect(workflow).toContain("POSTIL_BENCH_PAIRS: ${{ inputs.pairs }}");
expect(workflow).toMatch(new RegExp(
`^ {6}cost_cap_usd:\\n(?: {8}.*\\n){2} {8}default: "${MAX_GENERATOR_COST_CAP_USD}"$`,
"mu",
));
expect(workflow).toContain("upstream_provider:");
expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }}");
expect(workflow).toContain('echo "POSTIL_MANIFEST_OUT=${RUNNER_TEMP}/postil-qualified-models-${suffix}.json"');
Expand Down
Loading