Keep the DeepSeek and GLM candidate inactive

.github/workflows/bench-live.yml

@@ -8,9 +8,7 @@ on:
       pairs:
         description: Exact generators::consensus::scorer+fallback profile
         required: true
-      upstream_provider:
-        description: Exact ZDR provider identity serving every model in the profile
-        required: true
+        default: "deepseek/deepseek-v4-flash::1::z-ai/glm-5.2"
       cost_cap_usd:
         description: Abort when projected qualification spend exceeds this amount
         required: true
@@ -31,7 +29,7 @@ jobs:
     env:
       POSTIL_BENCH_MODE: live
       POSTIL_BENCH_PAIRS: ${{ inputs.pairs }}
-      POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }}
+      POSTIL_BENCH_UPSTREAM_PROVIDER: Fireworks
       POSTIL_BENCH_REPEATS: "3"
       POSTIL_BENCH_COST_CAP_USD: ${{ inputs.cost_cap_usd }}
       POSTIL_API_BASE: https://openrouter.ai/api/v1

.github/workflows/release.yml

@@ -20,9 +20,9 @@ jobs:
       - name: Install admission verifier dependencies
         working-directory: bench
         run: bun install --frozen-lockfile
-      - name: Verify embedded model admission
+      - name: Verify non-activating or formally admitted model manifest
         working-directory: bench
-        run: bun run verify-admission --allow-provisional
+        run: bun run verify-admission
       - name: Match release tag to package version
         env:
           RELEASE_TAG: ${{ github.ref_name }}

bench/src/livemodels.test.ts

@@ -10,6 +10,7 @@ import { benchmarkCase, envelopeV1, type BenchmarkCase } from "./harness";
 import type { AttributionCallEvidence } from "./attribution";
 import {
   admissionManifestCandidate,
+  assertBinaryMatchesQualificationWorktree,
   assertPromptInjectionCleanAdmissionRegression,
   assertManagedAdmissionCapacityPreflight,
   assertGitTreeSourceAuthority,
@@ -28,7 +29,9 @@ import {
   liveEnv,
   liveModelsQualificationExitCode,
   liveModelsCostAccountingComplete,
+  managedQualificationUpstreamProvider,
   MANAGED_OPENROUTER_PROVIDER_IDENTITY,
+  MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY,
   modelPriceBoundsFor,
   modelExecutionIntegrityFailures,
   normalizeApiBase,
@@ -53,6 +56,7 @@ import {
   verifyPrivateEvidenceBundle,
   withImmutableQualificationBinary,
   type LiveModelsReport,
+  type BinaryQualificationMetadata,
 } from "./livemodels";
 import {
   compareCanonicalDecimals,
@@ -64,6 +68,28 @@ import {
 } from "./livemodels-score";
 
 const pair: QualificationPair = { generatorModel: "test/generator", scorerModel: "test/scorer" };
+const intendedManagedPair: QualificationPair = {
+  generatorModel: "deepseek/deepseek-v4-flash",
+  scorerModel: "z-ai/glm-5.2",
+  consensus: 1,
+};
+
+const intendedManagedPricing = new Map([
+  [intendedManagedPair.generatorModel, {
+    providerIdentity: "Fireworks",
+    promptUsdPerToken: 0.00000014,
+    completionUsdPerToken: 0.00000028,
+    inputMicrosPerMillionTokens: 140_000,
+    outputMicrosPerMillionTokens: 280_000,
+  }],
+  [intendedManagedPair.scorerModel, {
+    providerIdentity: "Fireworks",
+    promptUsdPerToken: 0.0000014,
+    completionUsdPerToken: 0.0000044,
+    inputMicrosPerMillionTokens: 1_400_000,
+    outputMicrosPerMillionTokens: 4_400_000,
+  }],
+]);
 
 function sha256(value: string): string {
   return createHash("sha256").update(value).digest("hex");
@@ -140,6 +166,18 @@ async function close(server: Server): Promise<void> {
 }
 
 describe("pair qualification configuration", () => {
+  test("accepts only the source-pinned managed upstream provider", () => {
+    expect(managedQualificationUpstreamProvider()).toBe(
+      MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY,
+    );
+    expect(managedQualificationUpstreamProvider("Fireworks")).toBe("Fireworks");
+    for (const requested of ["", " Fireworks ", "OtherProvider"]) {
+      expect(() => managedQualificationUpstreamProvider(requested)).toThrow(
+        "live qualification requires the source-pinned Fireworks upstream provider",
+      );
+    }
+  });
+
   test("always dispatches three clean canaries before broader work", () => {
     const allCases = fixtureInputs.map((input) => benchmarkCase.parse(input));
     const canary = allCases.find((candidate) =>
@@ -473,6 +511,47 @@ describe("pair qualification configuration", () => {
     ).POSTIL_QUALIFICATION_CANDIDATE_PROFILE).toBe("/tmp/candidate.json");
   });
 
+  test("binds the intended managed candidate to Fireworks and exact price ceilings", () => {
+    const apiBase = normalizeApiBase("https://openrouter.ai/api/v1");
+    expect(qualificationCandidateDocument(
+      intendedManagedPair,
+      intendedManagedPricing,
+      apiBase,
+      "openai-compatible",
+      MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY,
+    )).toEqual({
+      benchmarkProviderIdentity: MANAGED_OPENROUTER_PROVIDER_IDENTITY,
+      apiBase,
+      apiFormat: "openai-compatible",
+      upstreamProviderIdentity: MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY,
+      generatorChain: ["deepseek/deepseek-v4-flash"],
+      consensus: 1,
+      scorerChain: ["z-ai/glm-5.2"],
+      modelPriceBounds: [
+        {
+          model: "deepseek/deepseek-v4-flash",
+          inputMicrosPerMillionTokens: 140_000,
+          outputMicrosPerMillionTokens: 280_000,
+        },
+        {
+          model: "z-ai/glm-5.2",
+          inputMicrosPerMillionTokens: 1_400_000,
+          outputMicrosPerMillionTokens: 4_400_000,
+        },
+      ],
+    });
+    expect(() => assertPricingProviderIdentity(
+      intendedManagedPricing,
+      [intendedManagedPair.generatorModel, intendedManagedPair.scorerModel],
+      MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY,
+    )).not.toThrow();
+    expect(() => assertPricingProviderIdentity(
+      intendedManagedPricing,
+      [intendedManagedPair.generatorModel, intendedManagedPair.scorerModel],
+      "OtherProvider",
+    )).toThrow("not bound to upstream provider OtherProvider");
+  });
+
   test("activates the exact candidate profile for evaluator-bank calls", async () => {
     const root = await mkdtemp(resolve(tmpdir(), "postil-evaluator-profile-"));
     const pricing = new Map([
@@ -640,6 +719,34 @@ describe("pair qualification configuration", () => {
     }
   }, 60_000);
 
+  test("projects the intended managed pair without provider access", async () => {
+    const root = await mkdtemp(resolve(tmpdir(), "postil-intended-runtime-preflight-"));
+    const inheritedModelKey = process.env.MODEL_API_KEY;
+    try {
+      process.env.MODEL_API_KEY = "postil-plan-only-fixture";
+      const binary = process.env.POSTIL_BIN === undefined
+        ? resolve(import.meta.dir, "..", "..", "target", "release", "postil")
+        : resolve(process.env.POSTIL_BIN);
+      const projected = await assertRuntimeShapedQualificationPreflight({
+        binary,
+        rootDir: root,
+        cases: fixtureInputs.map((input) => benchmarkCase.parse(input)),
+        pairs: normalizeQualificationPairs([intendedManagedPair]),
+        repeats: 3,
+        pricing: intendedManagedPricing,
+        apiBase: normalizeApiBase("https://openrouter.ai/api/v1"),
+        apiFormat: "openai-compatible",
+        costCapUsdDecimal: "70",
+        upstreamProvider: MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY,
+      });
+      expect(projected).toBe("61.457763");
+    } finally {
+      if (inheritedModelKey === undefined) delete process.env.MODEL_API_KEY;
+      else process.env.MODEL_API_KEY = inheritedModelKey;
+      await rm(root, { recursive: true, force: true });
+    }
+  }, 60_000);
+
   test("settles every started preflight worker before cleaning a failed workspace", async () => {
     const root = await mkdtemp(resolve(tmpdir(), "postil-runtime-preflight-failure-"));
     const binary = resolve(root, "fixture-postil");
@@ -951,6 +1058,53 @@ describe("qualification Git source authority", () => {
 });
 
 describe("immutable qualification binary", () => {
+  test("accepts only the intended embedded generator and scorer identity", async () => {
+    const configHash = createHash("sha256")
+      .update(await readFile(resolve(import.meta.dir, "..", "..", "config.toml")))
+      .digest("hex");
+    const metadata: BinaryQualificationMetadata = {
+      qualificationIssuedAtUnixSeconds: null,
+      qualificationExpiresAtUnixSeconds: null,
+      qualificationMaxAgeDays: null,
+      modelDefaultsSha256: configHash,
+      reviewContractSha256: "b".repeat(64),
+      fixtureSetSha256: "c".repeat(64),
+      evaluatorContractSha256: "d".repeat(64),
+      evaluatorRuntimeIdentity: "bun@1.3.14",
+      defaultApiBase: normalizeApiBase("https://openrouter.ai/api/v1"),
+      defaultApiFormat: "openai-compatible",
+      generatorChain: [intendedManagedPair.generatorModel],
+      consensus: 1,
+      scorerChain: [intendedManagedPair.scorerModel],
+      hostedOperationCostCapMicros: 1_000_000,
+      attributionMaxInputBytes: 4 * 1024,
+      attributionMaxProviderRequestBytes: 5_000,
+      admittedProfile: null,
+    };
+    const authority = {
+      metadata,
+      fixtureHash: metadata.fixtureSetSha256,
+      reviewContractHash: metadata.reviewContractSha256,
+      evaluatorContractHash: metadata.evaluatorContractSha256,
+      evaluatorRuntimeIdentity: metadata.evaluatorRuntimeIdentity,
+      configHash,
+      apiBase: metadata.defaultApiBase,
+      apiFormat: metadata.defaultApiFormat,
+    } as const;
+    expect(() => assertBinaryMatchesQualificationWorktree({
+      ...authority,
+      pairs: normalizeQualificationPairs([intendedManagedPair]),
+    })).not.toThrow();
+    expect(() => assertBinaryMatchesQualificationWorktree({
+      ...authority,
+      pairs: normalizeQualificationPairs([{
+        generatorModel: "z-ai/glm-5.2",
+        scorerModel: "z-ai/glm-5.2",
+        consensus: 1,
+      }]),
+    })).toThrow("qualification pair does not match the supplied binary's embedded default profile");
+  });
+
   test("rejects source authority drift before broader qualification spend", () => {
     const expected = {
       sourceSha: "a".repeat(40),
@@ -1224,12 +1378,14 @@ describe("managed admission workflow", () => {
     expect(workflow).toContain("POSTIL_API_FORMAT: openai-compatible");
     expect(workflow).toContain("POSTIL_BENCH_REPEATS: \"3\"");
     expect(workflow).toContain("POSTIL_BENCH_PAIRS: ${{ inputs.pairs }}");
+    expect(workflow).toMatch(/pairs:\n(?: {8}.*\n){2} {8}default: "deepseek\/deepseek-v4-flash::1::z-ai\/glm-5\.2"/u);
     expect(workflow).toMatch(new RegExp(
       `^ {6}cost_cap_usd:\\n(?: {8}.*\\n){2} {8}default: "${MAX_GENERATOR_COST_CAP_USD}"$`,
       "mu",
     ));
-    expect(workflow).toContain("upstream_provider:");
-    expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: ${{ inputs.upstream_provider }}");
+    expect(workflow).not.toContain("upstream_provider:");
+    expect(workflow).toContain("POSTIL_BENCH_UPSTREAM_PROVIDER: Fireworks");
+    expect(workflow).not.toContain("inputs.upstream_provider");
     expect(workflow).toContain("OPENROUTER_MANAGEMENT_API_KEY: ${{ secrets.OPENROUTER_MANAGEMENT_API_KEY }}");
     expect(workflow).toContain("OPENROUTER_QUALIFICATION_KEY_SHA256: ${{ secrets.OPENROUTER_QUALIFICATION_KEY_SHA256 }}");
     expect(workflow).toContain("COMPLETION_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}");
@@ -1278,6 +1434,8 @@ describe("managed admission workflow", () => {
       resolve(import.meta.dir, "..", "..", ".github", "workflows", "release.yml"),
     ).text();
     expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?\n  build:\n/u);
+    expect(release).toContain("run: bun run verify-admission");
+    expect(release).not.toContain("bun run verify-admission --allow-provisional");
     expect(release).toMatch(/build:\n\s+needs: validate-tag/u);
     let checkedReferences = 0;
     const workflowGlob = new Bun.Glob("*.yml");

bench/src/livemodels.ts

@@ -96,6 +96,7 @@ export const QUALIFICATION_MAX_AGE_SECONDS = QUALIFICATION_MAX_AGE_DAYS * 24 * 6
 const MAX_QUALIFICATION_SOURCE_BYTES = 16 * 1024 * 1024;
 const MANAGED_OPENROUTER_API_BASE = "https://openrouter.ai:443/api/v1";
 export const MANAGED_OPENROUTER_PROVIDER_IDENTITY = "openrouter:managed-routing";
+export const MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY = "Fireworks";
 export const LIVE_MODELS_REPORT_SCHEMA_VERSION = 3;
 export const LIVE_MODELS_PRIVATE_EVIDENCE_SCHEMA_VERSION = 1;
 export const PROMPT_INJECTION_CLEAN_ADMISSION_CASE_ID = "prompt-injection-comment-clean";
@@ -125,6 +126,17 @@ export class ManagedAdmissionCapacityError extends Error {
   }
 }
 
+export function managedQualificationUpstreamProvider(
+  requested?: string,
+): typeof MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY {
+  if (requested !== undefined && requested !== MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY) {
+    throw new Error(
+      `live qualification requires the source-pinned ${MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY} upstream provider`,
+    );
+  }
+  return MANAGED_QUALIFICATION_UPSTREAM_PROVIDER_IDENTITY;
+}
+
 export function managedAdmissionCapacityFailure(
   error: unknown,
 ): ManagedAdmissionCapacityError | null {
@@ -720,6 +732,7 @@ export async function runLiveModels(
   validateGeneratorQualificationBounds(models, costCapUsd);
   const cases = inputs.map((input) => benchmarkCase.parse(input));
   assertExactQualificationFixtures(cases);
+  const upstreamProvider = managedQualificationUpstreamProvider(options.upstreamProvider);
   if (!resolveApiKeyName()) {
     throw new Error(
       `live mode needs a real model key: set ${API_KEY_ENV_NAMES_TEXT} in the ` +
@@ -731,10 +744,6 @@ export async function runLiveModels(
   if (benchmarkProviderIdentityFor(apiBase, apiFormat) !== MANAGED_OPENROUTER_PROVIDER_IDENTITY) {
     throw new Error("live qualification requires the canonical managed OpenRouter endpoint");
   }
-  const upstreamProvider = options.upstreamProvider.trim();
-  if (upstreamProvider.length === 0) {
-    throw new Error("live qualification requires an exact pinned upstream provider identity");
-  }
   const rootDir = options.rootDir ?? resolve(import.meta.dir, "..", ".runs", "live-models");
   const suppliedPricing = options.pricing;
   return withImmutableQualificationBinary(options.binary, rootDir, async (immutableBinary) => {
@@ -2596,7 +2605,7 @@ export async function resolveBinaryQualificationMetadata(binary: string): Promis
   return metadata;
 }
 
-function assertBinaryMatchesQualificationWorktree(args: {
+export function assertBinaryMatchesQualificationWorktree(args: {
   metadata: BinaryQualificationMetadata;
   fixtureHash: string;
   reviewContractHash: string;

bench/src/run.ts

@@ -56,6 +56,7 @@ import {
   formatLiveModelsReport,
   liveModelsQualificationExitCode,
   MANAGED_OPENROUTER_PROVIDER_IDENTITY,
+  managedQualificationUpstreamProvider,
   managedAdmissionCapacityFailure,
   managedAdmissionCapacityFailureCategories,
   parseLiveModelsReport,
@@ -245,10 +246,9 @@ async function main() {
     const repeatsRaw = process.env.POSTIL_BENCH_REPEATS ?? flagValue(args, "--repeats");
     const apiFormat = qualificationApiFormat(process.env.POSTIL_API_FORMAT);
     const pricingFile = process.env.POSTIL_BENCH_PRICING_FILE ?? flagValue(args, "--pricing-file");
-    const upstreamProvider = process.env.POSTIL_BENCH_UPSTREAM_PROVIDER ?? flagValue(args, "--upstream-provider");
-    if (!upstreamProvider?.trim()) {
-      throw new Error("live-models admission needs POSTIL_BENCH_UPSTREAM_PROVIDER or --upstream-provider");
-    }
+    const requestedUpstreamProvider =
+      process.env.POSTIL_BENCH_UPSTREAM_PROVIDER ?? flagValue(args, "--upstream-provider");
+    const upstreamProvider = managedQualificationUpstreamProvider(requestedUpstreamProvider);
     const qualificationSourceSha = await resolveQualificationSourceSha(
       resolve(import.meta.dir, "..", ".."),
     );

bench/src/scorer-eval.test.ts

@@ -199,11 +199,11 @@ describe("parseModels", () => {
     expect(parseModels(" a/model, ,b/model,a/model ", [])).toEqual(["a/model", "b/model"]);
   });
 
-  test("does not invent an embedded scorer candidate when scoring is disabled", async () => {
+  test("uses the explicitly enabled embedded scorer candidate", async () => {
     const defaults = await loadEmbeddedScorerDefaults();
-    expect(defaults.enabled).toBe(false);
-    expect(defaults.qualification_candidates).toEqual([]);
-    expect(parseModels(undefined, defaults.qualification_candidates)).toEqual([]);
+    expect(defaults.enabled).toBe(true);
+    expect(defaults.qualification_candidates).toEqual(["z-ai/glm-5.2"]);
+    expect(parseModels(undefined, defaults.qualification_candidates)).toEqual(["z-ai/glm-5.2"]);
   });
 });