From b8d51f64e486e395bc63b83f92a1ab2a8c8b5261 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Mon, 20 Jul 2026 04:44:55 +0000 Subject: [PATCH 1/3] Prepare source-exact DeepSeek qualification candidate --- .github/workflows/bench-live.yml | 2 + bench/src/livemodels.test.ts | 142 +++++++++++++++++++++++++++++++ bench/src/livemodels.ts | 2 +- config.toml | 8 +- qualified-models.json | 2 +- src/config.rs | 10 ++- tests/e2e.rs | 38 +++++---- 7 files changed, 179 insertions(+), 25 deletions(-) diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index 601a2a2..7aa6f0f 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -8,9 +8,11 @@ on: pairs: description: Exact generators::consensus::scorer+fallback profile required: true + default: "deepseek/deepseek-v4-flash::1::z-ai/glm-5.2" upstream_provider: description: Exact ZDR provider identity serving every model in the profile required: true + default: "Fireworks" cost_cap_usd: description: Abort when projected qualification spend exceeds this amount required: true diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 9ab1508..2502058 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -10,6 +10,7 @@ import { benchmarkCase, envelopeV1, type BenchmarkCase } from "./harness"; import type { AttributionCallEvidence } from "./attribution"; import { admissionManifestCandidate, + assertBinaryMatchesQualificationWorktree, assertPromptInjectionCleanAdmissionRegression, assertManagedAdmissionCapacityPreflight, assertGitTreeSourceAuthority, @@ -53,6 +54,7 @@ import { verifyPrivateEvidenceBundle, withImmutableQualificationBinary, type LiveModelsReport, + type BinaryQualificationMetadata, } from "./livemodels"; import { compareCanonicalDecimals, @@ -64,6 +66,28 @@ import { } from "./livemodels-score"; const pair: QualificationPair = { generatorModel: "test/generator", scorerModel: "test/scorer" }; +const intendedManagedPair: QualificationPair = { + generatorModel: "deepseek/deepseek-v4-flash", + scorerModel: "z-ai/glm-5.2", + consensus: 1, +}; + +const intendedManagedPricing = new Map([ + [intendedManagedPair.generatorModel, { + providerIdentity: "Fireworks", + promptUsdPerToken: 0.00000014, + completionUsdPerToken: 0.00000028, + inputMicrosPerMillionTokens: 140_000, + outputMicrosPerMillionTokens: 280_000, + }], + [intendedManagedPair.scorerModel, { + providerIdentity: "Fireworks", + promptUsdPerToken: 0.0000014, + completionUsdPerToken: 0.0000044, + inputMicrosPerMillionTokens: 1_400_000, + outputMicrosPerMillionTokens: 4_400_000, + }], +]); function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); @@ -473,6 +497,47 @@ describe("pair qualification configuration", () => { ).POSTIL_QUALIFICATION_CANDIDATE_PROFILE).toBe("/tmp/candidate.json"); }); + test("binds the intended managed candidate to Fireworks and exact price ceilings", () => { + const apiBase = normalizeApiBase("https://openrouter.ai/api/v1"); + expect(qualificationCandidateDocument( + intendedManagedPair, + intendedManagedPricing, + apiBase, + "openai-compatible", + "Fireworks", + )).toEqual({ + benchmarkProviderIdentity: MANAGED_OPENROUTER_PROVIDER_IDENTITY, + apiBase, + apiFormat: "openai-compatible", + upstreamProviderIdentity: "Fireworks", + generatorChain: ["deepseek/deepseek-v4-flash"], + consensus: 1, + scorerChain: ["z-ai/glm-5.2"], + modelPriceBounds: [ + { + model: "deepseek/deepseek-v4-flash", + inputMicrosPerMillionTokens: 140_000, + outputMicrosPerMillionTokens: 280_000, + }, + { + model: "z-ai/glm-5.2", + inputMicrosPerMillionTokens: 1_400_000, + outputMicrosPerMillionTokens: 4_400_000, + }, + ], + }); + expect(() => assertPricingProviderIdentity( + intendedManagedPricing, + [intendedManagedPair.generatorModel, intendedManagedPair.scorerModel], + "Fireworks", + )).not.toThrow(); + expect(() => assertPricingProviderIdentity( + intendedManagedPricing, + [intendedManagedPair.generatorModel, intendedManagedPair.scorerModel], + "OtherProvider", + )).toThrow("not bound to upstream provider OtherProvider"); + }); + test("activates the exact candidate profile for evaluator-bank calls", async () => { const root = await mkdtemp(resolve(tmpdir(), "postil-evaluator-profile-")); const pricing = new Map([ @@ -640,6 +705,34 @@ describe("pair qualification configuration", () => { } }, 60_000); + test("projects the intended managed pair without provider access", async () => { + const root = await mkdtemp(resolve(tmpdir(), "postil-intended-runtime-preflight-")); + const inheritedModelKey = process.env.MODEL_API_KEY; + try { + process.env.MODEL_API_KEY = "postil-plan-only-fixture"; + const binary = process.env.POSTIL_BIN === undefined + ? resolve(import.meta.dir, "..", "..", "target", "release", "postil") + : resolve(process.env.POSTIL_BIN); + const projected = await assertRuntimeShapedQualificationPreflight({ + binary, + rootDir: root, + cases: fixtureInputs.map((input) => benchmarkCase.parse(input)), + pairs: normalizeQualificationPairs([intendedManagedPair]), + repeats: 3, + pricing: intendedManagedPricing, + apiBase: normalizeApiBase("https://openrouter.ai/api/v1"), + apiFormat: "openai-compatible", + costCapUsdDecimal: "70", + upstreamProvider: "Fireworks", + }); + expect(projected).toBe("61.457763"); + } finally { + if (inheritedModelKey === undefined) delete process.env.MODEL_API_KEY; + else process.env.MODEL_API_KEY = inheritedModelKey; + await rm(root, { recursive: true, force: true }); + } + }, 60_000); + test("settles every started preflight worker before cleaning a failed workspace", async () => { const root = await mkdtemp(resolve(tmpdir(), "postil-runtime-preflight-failure-")); const binary = resolve(root, "fixture-postil"); @@ -951,6 +1044,53 @@ describe("qualification Git source authority", () => { }); describe("immutable qualification binary", () => { + test("accepts only the intended embedded generator and scorer identity", async () => { + const configHash = createHash("sha256") + .update(await readFile(resolve(import.meta.dir, "..", "..", "config.toml"))) + .digest("hex"); + const metadata: BinaryQualificationMetadata = { + qualificationIssuedAtUnixSeconds: null, + qualificationExpiresAtUnixSeconds: null, + qualificationMaxAgeDays: null, + modelDefaultsSha256: configHash, + reviewContractSha256: "b".repeat(64), + fixtureSetSha256: "c".repeat(64), + evaluatorContractSha256: "d".repeat(64), + evaluatorRuntimeIdentity: "bun@1.3.14", + defaultApiBase: normalizeApiBase("https://openrouter.ai/api/v1"), + defaultApiFormat: "openai-compatible", + generatorChain: [intendedManagedPair.generatorModel], + consensus: 1, + scorerChain: [intendedManagedPair.scorerModel], + hostedOperationCostCapMicros: 1_000_000, + attributionMaxInputBytes: 4 * 1024, + attributionMaxProviderRequestBytes: 5_000, + admittedProfile: null, + }; + const authority = { + metadata, + fixtureHash: metadata.fixtureSetSha256, + reviewContractHash: metadata.reviewContractSha256, + evaluatorContractHash: metadata.evaluatorContractSha256, + evaluatorRuntimeIdentity: metadata.evaluatorRuntimeIdentity, + configHash, + apiBase: metadata.defaultApiBase, + apiFormat: metadata.defaultApiFormat, + } as const; + expect(() => assertBinaryMatchesQualificationWorktree({ + ...authority, + pairs: normalizeQualificationPairs([intendedManagedPair]), + })).not.toThrow(); + expect(() => assertBinaryMatchesQualificationWorktree({ + ...authority, + pairs: normalizeQualificationPairs([{ + generatorModel: "z-ai/glm-5.2", + scorerModel: "z-ai/glm-5.2", + consensus: 1, + }]), + })).toThrow("qualification pair does not match the supplied binary's embedded default profile"); + }); + test("rejects source authority drift before broader qualification spend", () => { const expected = { sourceSha: "a".repeat(40), @@ -1224,11 +1364,13 @@ describe("managed admission workflow", () => { expect(workflow).toContain("POSTIL_API_FORMAT: openai-compatible"); expect(workflow).toContain("POSTIL_BENCH_REPEATS: \"3\""); expect(workflow).toContain("POSTIL_BENCH_PAIRS: ${{ inputs.pairs }}"); + expect(workflow).toMatch(/pairs:\n(?: {8}.*\n){2} {8}default: "deepseek\/deepseek-v4-flash::1::z-ai\/glm-5\.2"/u); expect(workflow).toMatch(new RegExp( `^ {6}cost_cap_usd:\\n(?: {8}.*\\n){2} {8}default: "${MAX_GENERATOR_COST_CAP_USD}"$`, "mu", )); expect(workflow).toContain("upstream_provider:"); + expect(workflow).toMatch(/upstream_provider:\n(?: {8}.*\n){2} {8}default: "Fireworks"/u); expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }}"); expect(workflow).toContain("OPENROUTER_MANAGEMENT_API_KEY: ${{ secrets.OPENROUTER_MANAGEMENT_API_KEY }}"); expect(workflow).toContain("OPENROUTER_QUALIFICATION_KEY_SHA256: ${{ secrets.OPENROUTER_QUALIFICATION_KEY_SHA256 }}"); diff --git a/bench/src/livemodels.ts b/bench/src/livemodels.ts index 01fda23..3dc333c 100644 --- a/bench/src/livemodels.ts +++ b/bench/src/livemodels.ts @@ -2596,7 +2596,7 @@ export async function resolveBinaryQualificationMetadata(binary: string): Promis return metadata; } -function assertBinaryMatchesQualificationWorktree(args: { +export function assertBinaryMatchesQualificationWorktree(args: { metadata: BinaryQualificationMetadata; fixtureHash: string; reviewContractHash: string; diff --git a/config.toml b/config.toml index 1a72a86..b9882c3 100644 --- a/config.toml +++ b/config.toml @@ -1,12 +1,12 @@ version = 3 -default_model = "z-ai/glm-5.2" +default_model = "deepseek/deepseek-v4-flash" cascade = [] consensus = 1 api_base = "https://openrouter.ai/api/v1" api_format = "openai-compatible" [scorer] -enabled = false -default_model = "" +enabled = true +default_model = "z-ai/glm-5.2" fallback = "" -qualification_candidates = [] +qualification_candidates = ["z-ai/glm-5.2"] diff --git a/qualified-models.json b/qualified-models.json index 08dfd48..a40d3dc 100644 --- a/qualified-models.json +++ b/qualified-models.json @@ -4,6 +4,6 @@ "qualificationIssuedAtUnixSeconds": null, "qualificationExpiresAtUnixSeconds": null, "qualificationMaxAgeDays": null, - "modelDefaultsSha256": "98de5a44a3f45557e746ed36afd5d7aae55942aa4f57d74a77335894fd21e9f3", + "modelDefaultsSha256": "1479ff69457cb2d12b2db5225559130f38e2ddc25ccf7dd4ec8f100a45694919", "profiles": [] } diff --git a/src/config.rs b/src/config.rs index d1d27bb..2222755 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2986,11 +2986,15 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid #[test] fn hosted_candidate_matches_the_qualification_profile() { let defaults = model_defaults(); - assert_eq!(defaults.default_model, "z-ai/glm-5.2"); + assert_eq!(defaults.default_model, "deepseek/deepseek-v4-flash"); assert!(defaults.cascade.is_empty()); - assert!(defaults.scorer_model.is_empty()); + assert!(defaults.scorer_enabled); + assert_eq!(defaults.scorer_model, "z-ai/glm-5.2"); assert!(defaults.scorer_fallback.is_empty()); - assert!(defaults.scorer_qualification_candidates.is_empty()); + assert_eq!( + defaults.scorer_qualification_candidates, + vec!["z-ai/glm-5.2"] + ); } #[test] diff --git a/tests/e2e.rs b/tests/e2e.rs index 4423c97..69f406c 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1385,8 +1385,8 @@ fn qualification_candidate_preflights_the_bounded_hosted_path_without_provider_c std::fs::write(&diff_path, diff).unwrap(); let metadata = postil_cli::config::qualification_metadata(); - let generator_chain = vec!["openai/gpt-5-mini".to_string()]; - let scorer_chain = vec!["openai/gpt-5-mini".to_string()]; + let generator_chain = metadata.generator_chain.clone(); + let scorer_chain = metadata.scorer_chain.clone(); let mut models = generator_chain.clone(); models.extend(scorer_chain.clone()); models.sort(); @@ -1452,6 +1452,10 @@ fn qualification_candidate_admits_worst_case_json_escaped_hosted_batches() { std::fs::write(&diff_path, diff).unwrap(); let metadata = postil_cli::config::qualification_metadata(); + let mut models = metadata.generator_chain.clone(); + models.extend(metadata.scorer_chain.clone()); + models.sort(); + models.dedup(); let profile_path = dir.path().join("candidate.json"); std::fs::write( &profile_path, @@ -1460,14 +1464,14 @@ fn qualification_candidate_admits_worst_case_json_escaped_hosted_batches() { "upstreamProviderIdentity": "test-provider", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, - "generatorChain": ["openai/gpt-5-mini"], - "consensus": 1, - "scorerChain": ["openai/gpt-5-mini"], - "modelPriceBounds": [{ - "model": "openai/gpt-5-mini", + "generatorChain": metadata.generator_chain, + "consensus": metadata.consensus, + "scorerChain": metadata.scorer_chain, + "modelPriceBounds": models.into_iter().map(|model| json!({ + "model": model, "inputMicrosPerMillionTokens": 1, "outputMicrosPerMillionTokens": 1 - }] + })).collect::>() })) .unwrap(), ) @@ -1591,6 +1595,8 @@ fn qualification_candidate_admits_fixture_51_shape_at_fireworks_price_bounds() { std::fs::write(&diff_path, diff).unwrap(); let metadata = postil_cli::config::qualification_metadata(); + assert_eq!(metadata.generator_chain, vec!["deepseek/deepseek-v4-flash"]); + assert_eq!(metadata.scorer_chain, vec!["z-ai/glm-5.2"]); let profile_path = dir.path().join("candidate.json"); std::fs::write( &profile_path, @@ -1599,19 +1605,19 @@ fn qualification_candidate_admits_fixture_51_shape_at_fireworks_price_bounds() { "upstreamProviderIdentity": "Fireworks", "apiBase": metadata.default_api_base, "apiFormat": metadata.default_api_format, - "generatorChain": ["deepseek/deepseek-v4-pro"], - "consensus": 1, - "scorerChain": ["z-ai/glm-5.2"], + "generatorChain": metadata.generator_chain, + "consensus": metadata.consensus, + "scorerChain": metadata.scorer_chain, "modelPriceBounds": [ { - "model": "deepseek/deepseek-v4-pro", - "inputMicrosPerMillionTokens": 1_740_000, - "outputMicrosPerMillionTokens": 3_480_000 + "model": "deepseek/deepseek-v4-flash", + "inputMicrosPerMillionTokens": 140_000, + "outputMicrosPerMillionTokens": 280_000 }, { "model": "z-ai/glm-5.2", - "inputMicrosPerMillionTokens": 2_100_000, - "outputMicrosPerMillionTokens": 6_600_000 + "inputMicrosPerMillionTokens": 1_400_000, + "outputMicrosPerMillionTokens": 4_400_000 } ] })) From 58e1a48db37c2735f211b3620ba372ab617abe1d Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Mon, 20 Jul 2026 05:13:14 +0000 Subject: [PATCH 2/3] Pin qualification authority and keep candidate inactive --- .github/workflows/bench-live.yml | 6 +--- bench/src/livemodels.test.ts | 28 +++++++++++++----- bench/src/livemodels.ts | 17 ++++++++--- bench/src/run.ts | 8 +++--- bench/src/scorer-eval.test.ts | 8 +++--- bench/src/verify-admission.test.ts | 46 +++++++++++++++++++++++++++--- bench/src/verify-admission.ts | 10 +++---- src/config.rs | 28 ++++++++++++++---- tests/e2e.rs | 7 +++-- 9 files changed, 117 insertions(+), 41 deletions(-) diff --git a/.github/workflows/bench-live.yml b/.github/workflows/bench-live.yml index 7aa6f0f..a9e4d81 100644 --- a/.github/workflows/bench-live.yml +++ b/.github/workflows/bench-live.yml @@ -9,10 +9,6 @@ on: description: Exact generators::consensus::scorer+fallback profile required: true default: "deepseek/deepseek-v4-flash::1::z-ai/glm-5.2" - upstream_provider: - description: Exact ZDR provider identity serving every model in the profile - required: true - default: "Fireworks" cost_cap_usd: description: Abort when projected qualification spend exceeds this amount required: true @@ -33,7 +29,7 @@ jobs: env: POSTIL_BENCH_MODE: live POSTIL_BENCH_PAIRS: ${{ inputs.pairs }} - POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }} + POSTIL_BENCH_UPSTREAM_PROVIDER: Fireworks POSTIL_BENCH_REPEATS: "3" POSTIL_BENCH_COST_CAP_USD: ${{ inputs.cost_cap_usd }} POSTIL_API_BASE: https://openrouter.ai/api/v1 diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 2502058..87e6718 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -29,7 +29,9 @@ import { liveEnv, liveModelsQualificationExitCode, liveModelsCostAccountingComplete, + managedQualificationUpstreamProvider, MANAGED_OPENROUTER_PROVIDER_IDENTITY, + MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY, modelPriceBoundsFor, modelExecutionIntegrityFailures, normalizeApiBase, @@ -164,6 +166,18 @@ async function close(server: Server): Promise { } describe("pair qualification configuration", () => { + test("accepts only the source-pinned managed upstream provider", () => { + expect(managedQualificationUpstreamProvider()).toBe( + MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY, + ); + expect(managedQualificationUpstreamProvider("Fireworks")).toBe("Fireworks"); + for (const requested of ["", " Fireworks ", "OtherProvider"]) { + expect(() => managedQualificationUpstreamProvider(requested)).toThrow( + "live qualification requires the source-pinned Fireworks upstream provider", + ); + } + }); + test("always dispatches three clean canaries before broader work", () => { const allCases = fixtureInputs.map((input) => benchmarkCase.parse(input)); const canary = allCases.find((candidate) => @@ -504,12 +518,12 @@ describe("pair qualification configuration", () => { intendedManagedPricing, apiBase, "openai-compatible", - "Fireworks", + MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY, )).toEqual({ benchmarkProviderIdentity: MANAGED_OPENROUTER_PROVIDER_IDENTITY, apiBase, apiFormat: "openai-compatible", - upstreamProviderIdentity: "Fireworks", + upstreamProviderIdentity: MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY, generatorChain: ["deepseek/deepseek-v4-flash"], consensus: 1, scorerChain: ["z-ai/glm-5.2"], @@ -529,7 +543,7 @@ describe("pair qualification configuration", () => { expect(() => assertPricingProviderIdentity( intendedManagedPricing, [intendedManagedPair.generatorModel, intendedManagedPair.scorerModel], - "Fireworks", + MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY, )).not.toThrow(); expect(() => assertPricingProviderIdentity( intendedManagedPricing, @@ -723,7 +737,7 @@ describe("pair qualification configuration", () => { apiBase: normalizeApiBase("https://openrouter.ai/api/v1"), apiFormat: "openai-compatible", costCapUsdDecimal: "70", - upstreamProvider: "Fireworks", + upstreamProvider: MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY, }); expect(projected).toBe("61.457763"); } finally { @@ -1369,9 +1383,9 @@ describe("managed admission workflow", () => { `^ {6}cost_cap_usd:\\n(?: {8}.*\\n){2} {8}default: "${MAX_GENERATOR_COST_CAP_USD}"$`, "mu", )); - expect(workflow).toContain("upstream_provider:"); - expect(workflow).toMatch(/upstream_provider:\n(?: {8}.*\n){2} {8}default: "Fireworks"/u); - expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }}"); + expect(workflow).not.toContain("upstream_provider:"); + expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: Fireworks"); + expect(workflow).not.toContain("inputs.upstream_provider"); expect(workflow).toContain("OPENROUTER_MANAGEMENT_API_KEY: ${{ secrets.OPENROUTER_MANAGEMENT_API_KEY }}"); expect(workflow).toContain("OPENROUTER_QUALIFICATION_KEY_SHA256: ${{ secrets.OPENROUTER_QUALIFICATION_KEY_SHA256 }}"); expect(workflow).toContain("COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}"); diff --git a/bench/src/livemodels.ts b/bench/src/livemodels.ts index 3dc333c..e86c578 100644 --- a/bench/src/livemodels.ts +++ b/bench/src/livemodels.ts @@ -96,6 +96,7 @@ export const QUALIFICATION_MAX_AGE_SECONDS = QUALIFICATION_MAX_AGE_DAYS * 24 * 6 const MAX_QUALIFICATION_SOURCE_BYTES = 16 * 1024 * 1024; const MANAGED_OPENROUTER_API_BASE = "https://openrouter.ai:443/api/v1"; export const MANAGED_OPENROUTER_PROVIDER_IDENTITY = "openrouter:managed-routing"; +export const MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY = "Fireworks"; export const LIVE_MODELS_REPORT_SCHEMA_VERSION = 3; export const LIVE_MODELS_PRIVATE_EVIDENCE_SCHEMA_VERSION = 1; export const PROMPT_INJECTION_CLEAN_ADMISSION_CASE_ID = "prompt-injection-comment-clean"; @@ -125,6 +126,17 @@ export class ManagedAdmissionCapacityError extends Error { } } +export function managedQualificationUpstreamProvider( + requested?: string, +): typeof MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY { + if (requested !== undefined && requested !== MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY) { + throw new Error( + `live qualification requires the source-pinned ${MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY} upstream provider`, + ); + } + return MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY; +} + export function managedAdmissionCapacityFailure( error: unknown, ): ManagedAdmissionCapacityError | null { @@ -720,6 +732,7 @@ export async function runLiveModels( validateGeneratorQualificationBounds(models, costCapUsd); const cases = inputs.map((input) => benchmarkCase.parse(input)); assertExactQualificationFixtures(cases); + const upstreamProvider = managedQualificationUpstreamProvider(options.upstreamProvider); if (!resolveApiKeyName()) { throw new Error( `live mode needs a real model key: set ${API_KEY_ENV_NAMES_TEXT} in the ` + @@ -731,10 +744,6 @@ export async function runLiveModels( if (benchmarkProviderIdentityFor(apiBase, apiFormat) !== MANAGED_OPENROUTER_PROVIDER_IDENTITY) { throw new Error("live qualification requires the canonical managed OpenRouter endpoint"); } - const upstreamProvider = options.upstreamProvider.trim(); - if (upstreamProvider.length === 0) { - throw new Error("live qualification requires an exact pinned upstream provider identity"); - } const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs", "live-models"); const suppliedPricing = options.pricing; return withImmutableQualificationBinary(options.binary, rootDir, async (immutableBinary) => { diff --git a/bench/src/run.ts b/bench/src/run.ts index 6f7402f..a47f0b7 100644 --- a/bench/src/run.ts +++ b/bench/src/run.ts @@ -56,6 +56,7 @@ import { formatLiveModelsReport, liveModelsQualificationExitCode, MANAGED_OPENROUTER_PROVIDER_IDENTITY, + managedQualificationUpstreamProvider, managedAdmissionCapacityFailure, managedAdmissionCapacityFailureCategories, parseLiveModelsReport, @@ -245,10 +246,9 @@ async function main() { const repeatsRaw = process.env.POSTIL_BENCH_REPEATS ?? flagValue(args, "--repeats"); const apiFormat = qualificationApiFormat(process.env.POSTIL_API_FORMAT); const pricingFile = process.env.POSTIL_BENCH_PRICING_FILE ?? flagValue(args, "--pricing-file"); - const upstreamProvider = process.env.POSTIL_BENCH_UPSTREAM_PROVIDER ?? flagValue(args, "--upstream-provider"); - if (!upstreamProvider?.trim()) { - throw new Error("live-models admission needs POSTIL_BENCH_UPSTREAM_PROVIDER or --upstream-provider"); - } + const requestedUpstreamProvider = + process.env.POSTIL_BENCH_UPSTREAM_PROVIDER ?? flagValue(args, "--upstream-provider"); + const upstreamProvider = managedQualificationUpstreamProvider(requestedUpstreamProvider); const qualificationSourceSha = await resolveQualificationSourceSha( resolve(import.meta.dir, "..", ".."), ); diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index a385b33..ac96d47 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -199,11 +199,11 @@ describe("parseModels", () => { expect(parseModels(" a/model, ,b/model,a/model ", [])).toEqual(["a/model", "b/model"]); }); - test("does not invent an embedded scorer candidate when scoring is disabled", async () => { + test("uses the explicitly enabled embedded scorer candidate", async () => { const defaults = await loadEmbeddedScorerDefaults(); - expect(defaults.enabled).toBe(false); - expect(defaults.qualification_candidates).toEqual([]); - expect(parseModels(undefined, defaults.qualification_candidates)).toEqual([]); + expect(defaults.enabled).toBe(true); + expect(defaults.qualification_candidates).toEqual(["z-ai/glm-5.2"]); + expect(parseModels(undefined, defaults.qualification_candidates)).toEqual(["z-ai/glm-5.2"]); }); }); diff --git a/bench/src/verify-admission.test.ts b/bench/src/verify-admission.test.ts index d0234ff..f0fc1a1 100644 --- a/bench/src/verify-admission.test.ts +++ b/bench/src/verify-admission.test.ts @@ -44,13 +44,51 @@ async function temporaryDirectory(): Promise { } describe("admission attestation verification", () => { - test("verifies the repository provisional hosted release profile", async () => { + test("keeps the repository candidate configuration non-provisional", async () => { const repositoryRoot = join(import.meta.dir, "..", ".."); - expect(await verifyProvisionalRelease( + await expect(verifyProvisionalRelease( join(repositoryRoot, "qualified-models.json"), join(repositoryRoot, "config.toml"), join(repositoryRoot, "provisional-models.json"), - )).toBe("provisional"); + )).rejects.toThrow("no provisional hosted profile is active for embedded model defaults"); + }); + + test("verifies a provisional release only when its profile matches the embedded defaults", async () => { + const repositoryRoot = join(import.meta.dir, "..", ".."); + const directory = await temporaryDirectory(); + const manifest = join(directory, "qualified-models.json"); + const config = join(directory, "config.toml"); + const profile = join(directory, "provisional-models.json"); + await writeFile(manifest, await readFile(join(repositoryRoot, "qualified-models.json"))); + await writeFile(config, await readFile(join(repositoryRoot, "config.toml"))); + const exact = JSON.parse( + await readFile(join(repositoryRoot, "provisional-models.json"), "utf8"), + ) as { + generatorChain: string[]; + scorerChain: string[]; + modelPriceBounds: Array<{ + model: string; + inputMicrosPerMillionTokens: number; + outputMicrosPerMillionTokens: number; + }>; + }; + exact.generatorChain = ["deepseek/deepseek-v4-flash"]; + exact.scorerChain = ["z-ai/glm-5.2"]; + exact.modelPriceBounds = [ + { + model: "deepseek/deepseek-v4-flash", + inputMicrosPerMillionTokens: 140_000, + outputMicrosPerMillionTokens: 280_000, + }, + { + model: "z-ai/glm-5.2", + inputMicrosPerMillionTokens: 1_400_000, + outputMicrosPerMillionTokens: 4_400_000, + }, + ]; + await writeFile(profile, JSON.stringify(exact)); + + expect(await verifyProvisionalRelease(manifest, config, profile)).toBe("provisional"); }); test("keeps attestation verification active after a profile is formally admitted", async () => { @@ -97,7 +135,7 @@ describe("admission attestation verification", () => { await writeFile(profile, JSON.stringify(altered)); await expect(verifyProvisionalRelease(manifest, config, profile)).rejects.toThrow( - "does not exactly match embedded model defaults", + "no provisional hosted profile is active for embedded model defaults", ); }); diff --git a/bench/src/verify-admission.ts b/bench/src/verify-admission.ts index e6d05f9..d8b9fbb 100644 --- a/bench/src/verify-admission.ts +++ b/bench/src/verify-admission.ts @@ -254,6 +254,10 @@ export async function verifyProvisionalRelease( if (scorer.fallback.length > 0 && scorer.fallback === scorer.default_model) { throw new Error("scorer fallback must differ from scorer.defaultModel"); } + const defaultsSha = createHash("sha256").update(configSource).digest("hex"); + if (manifest.modelDefaultsSha256 !== defaultsSha) { + throw new Error("empty qualification manifest does not match embedded model defaults"); + } if ( config.default_model !== profile.generatorChain[0] || JSON.stringify(config.cascade) !== JSON.stringify(profile.generatorChain.slice(1)) || @@ -265,11 +269,7 @@ export async function verifyProvisionalRelease( scorer.fallback !== (profile.scorerChain[1] ?? "") || JSON.stringify(scorer.qualification_candidates) !== JSON.stringify(profile.scorerChain) ) { - throw new Error("provisional hosted profile does not exactly match embedded model defaults"); - } - const defaultsSha = createHash("sha256").update(configSource).digest("hex"); - if (manifest.modelDefaultsSha256 !== defaultsSha) { - throw new Error("empty qualification manifest does not match embedded model defaults"); + throw new Error("no provisional hosted profile is active for embedded model defaults"); } return "provisional"; } diff --git a/src/config.rs b/src/config.rs index 2222755..7a39e9e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1482,11 +1482,27 @@ fn provisional_hosted_roster_enabled() -> bool { provisional_hosted_roster_allowed( std::env::var(PROVISIONAL_HOSTED_ROSTER_ENV).as_deref() == Ok("1"), qualification_manifest().profiles.is_empty(), + provisional_hosted_profile_matches_model_defaults(), ) } -fn provisional_hosted_roster_allowed(requested: bool, qualification_manifest_empty: bool) -> bool { - requested && qualification_manifest_empty +fn provisional_hosted_roster_allowed( + requested: bool, + qualification_manifest_empty: bool, + provisional_profile_matches_model_defaults: bool, +) -> bool { + requested && qualification_manifest_empty && provisional_profile_matches_model_defaults +} + +fn provisional_hosted_profile_matches_model_defaults() -> bool { + let defaults = model_defaults(); + let profile = provisional_hosted_profile(); + let (generator_chain, scorer_chain, api_base) = qualification_defaults(defaults); + generator_chain == profile.generator_chain + && scorer_chain == profile.scorer_chain + && defaults.consensus == profile.consensus + && defaults.api_format == profile.api_format + && api_base == profile.api_base } pub(crate) fn bounded_review_selection_mode() -> bool { @@ -2995,6 +3011,7 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid defaults.scorer_qualification_candidates, vec!["z-ai/glm-5.2"] ); + assert!(!provisional_hosted_profile_matches_model_defaults()); } #[test] @@ -3054,9 +3071,10 @@ scorer = { enabled = true, default_model = "provider/scorer", fallback = "provid #[test] fn provisional_hosted_roster_cannot_override_formal_admission() { - assert!(provisional_hosted_roster_allowed(true, true)); - assert!(!provisional_hosted_roster_allowed(true, false)); - assert!(!provisional_hosted_roster_allowed(false, true)); + assert!(provisional_hosted_roster_allowed(true, true, true)); + assert!(!provisional_hosted_roster_allowed(true, true, false)); + assert!(!provisional_hosted_roster_allowed(true, false, true)); + assert!(!provisional_hosted_roster_allowed(false, true, true)); } #[test] diff --git a/tests/e2e.rs b/tests/e2e.rs index 69f406c..a7c5b35 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -3213,7 +3213,7 @@ fn hosted_config_ignores_repository_model_provider_and_scorer() { } #[test] -fn provisional_hosted_config_uses_only_the_baked_roster() { +fn provisional_hosted_config_stays_inactive_when_embedded_defaults_do_not_match() { let dir = tempfile::tempdir().unwrap(); std::fs::write( dir.path().join(".postil.yaml"), @@ -3233,12 +3233,13 @@ fn provisional_hosted_config_uses_only_the_baked_roster() { .success(); let stdout = String::from_utf8(out.get_output().stdout.clone()).unwrap(); - assert!(stdout.contains("model.name: z-ai/glm-5.2")); + assert!(stdout.contains("model.name: ")); assert!(stdout.contains("model.cascade: []")); assert!(stdout.contains("model.scorer: ")); - assert!(stdout.contains("model.apiBase: https://openrouter.ai:443/api/v1")); + assert!(stdout.contains("model.apiBase: https://openrouter.ai/api/v1")); assert!(stdout.contains("model.apiFormat: openai-compatible")); assert!(stdout.contains("model.consensus: 1")); + assert!(!stdout.contains("z-ai/glm-5.2")); assert!(!stdout.contains("attacker")); assert!(!stdout.contains("stale/")); } From 0cacee57a07f96046efda7a67aa52716ddf2166c Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Mon, 20 Jul 2026 05:27:59 +0000 Subject: [PATCH 3/3] Release inactive model candidates safely --- .github/workflows/release.yml | 4 ++-- bench/src/livemodels.test.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 95dd3c9..21e712a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,9 +20,9 @@ jobs: - name: Install admission verifier dependencies working-directory: bench run: bun install --frozen-lockfile - - name: Verify embedded model admission + - name: Verify non-activating or formally admitted model manifest working-directory: bench - run: bun run verify-admission --allow-provisional + run: bun run verify-admission - name: Match release tag to package version env: RELEASE_TAG: ${{ github.ref_name }} diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 87e6718..04d26fb 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1434,6 +1434,8 @@ describe("managed admission workflow", () => { resolve(import.meta.dir, "..", "..", ".github", "workflows", "release.yml"), ).text(); expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n build:\n/u); + expect(release).toContain("run: bun run verify-admission"); + expect(release).not.toContain("bun run verify-admission --allow-provisional"); expect(release).toMatch(/build:\n\s+needs: validate-tag/u); let checkedReferences = 0; const workflowGlob = new Bun.Glob("*.yml");