fix: wait for console asset version overrides to converge

docs/deploy-cloudflare.md

@@ -1325,6 +1325,13 @@ readback, never by blindly repeating the write.
    `Cloudflare-Workers-Version-Overrides: clawrouter-edge="<green UUID>"`.
    `/v1/health.versionId` and Worker response headers must identify green; a
    silently ignored override is not a successful smoke.
+   Each console asset must also prove its requested version and exact bytes,
+   through both green and blue (the initial legacy bootstrap keeps its documented
+   exception). A successful response from another runtime is
+   retried for at most 30 seconds per asset because overrides can still propagate
+   after health passes. Missing/invalid version proof, HTTP failures and corrupt
+   bytes fail immediately. Failure receipts include the public asset path, HTTP
+   status and expected/observed version; no response content is logged.
 2. Green 10% for at least two minutes, then green 50% for at least three minutes.
    Full override smoke runs immediately and waits at most 30 seconds between
    checks through each hold. It includes a paid golden-provider request and

scripts/deploy-console-assets.mjs

@@ -2,10 +2,12 @@ import { createHash } from 'node:crypto';
 import { execFileSync } from 'node:child_process';
 import { readdirSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
 
 const pattern = /^[A-Za-z0-9_.-]+-[A-Za-z0-9_-]{8,}\.(js|css|woff2?|png|jpe?g|svg|webp|gif|ico)$/;
 const types = { js: 'text/javascript', css: 'text/css', woff: 'font/woff', woff2: 'font/woff2', png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', svg: 'image/svg+xml', webp: 'image/webp', gif: 'image/gif', ico: 'image/x-icon' };
 const digest = (bytes) => createHash('sha256').update(bytes).digest('hex');
+const versionPattern = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i;
 
 export function assertAssetBootstrap(blueSha, cwd = process.cwd()) {
   const git = (...args) => execFileSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000 }).trim();
@@ -58,13 +60,49 @@ export async function publishConsoleAssets({ sha, env = process.env, directory =
   return receipts;
 }
 
-export async function verifyConsoleAssets({ assets, baseUrl, version, legacy = false, fetchImpl = fetch }) {
+export async function verifyConsoleAssets({ assets, baseUrl, version, legacy = false, fetchImpl = fetch,
+  readinessTimeoutMs = 30_000, now = () => performance.now(), wait = sleep, log = console.log }) {
+  if (!Number.isInteger(readinessTimeoutMs) || readinessTimeoutMs < 1 || readinessTimeoutMs > 30_000) throw new Error('Invalid console asset readiness timeout');
   for (const asset of assets) {
     if (legacy && asset.probe) continue; // legacy blue has no shared fallback yet
-    const response = await fetchImpl(`${baseUrl}${asset.path}`, { headers: { 'Cloudflare-Workers-Version-Overrides': `clawrouter-edge="${version}"` }, redirect: 'error', signal: AbortSignal.timeout(30_000) });
-    const actualVersion = response.headers.get('x-clawrouter-version-id');
-    if (!response.ok || (actualVersion !== version && !(legacy && actualVersion === null))) { await response.body?.cancel(); throw new Error('Cross-version console asset response failed'); }
-    const body = Buffer.from(await response.arrayBuffer());
-    if (body.length !== asset.bytes || digest(body) !== asset.sha256) throw new Error('Cross-version console asset bytes differ');
+    if (typeof asset.path !== 'string' || !asset.path.startsWith('/assets/') || !pattern.test(asset.path.slice('/assets/'.length))) throw new Error('Invalid console asset receipt path');
+    const deadline = now() + readinessTimeoutMs;
+    let attempt = 0;
+    while (true) {
+      const details = { path: asset.path, expectedVersion: version, actualVersion: null, status: null, attempt: ++attempt };
+      const failure = (reason, extra = {}, kind = 'response failed') => new Error(`Cross-version console asset ${kind}: ${JSON.stringify({ ...details, reason, ...extra })}`);
+      const remaining = Math.ceil(deadline - now());
+      if (remaining <= 0) throw failure('request_deadline', { failure: 'readiness_timeout' });
+      let response;
+      try {
+        response = await fetchImpl(`${baseUrl}${asset.path}`, { method: 'GET', headers: { 'Cloudflare-Workers-Version-Overrides': `clawrouter-edge="${version}"` }, redirect: 'error', signal: AbortSignal.timeout(remaining) });
+      } catch { throw failure('request_failed'); } // Never expose transport messages, headers or response bodies.
+      const actualVersion = response.headers.get('x-clawrouter-version-id');
+      details.status = response.status;
+      details.actualVersion = actualVersion === null ? null : versionPattern.test(actualVersion) ? actualVersion : 'invalid';
+      if (!response.ok || (actualVersion !== version && !(legacy && actualVersion === null))) {
+        await response.body?.cancel().catch(() => {});
+        const reason = response.ok ? 'version_mismatch' : 'http_status';
+        // An override can converge independently for each asset after health passes.
+        // Retry only successful responses proving another runtime, never missing
+        // proof, HTTP errors, corrupt bytes, or a mutation.
+        if (response.ok && versionPattern.test(actualVersion ?? '')) {
+          const delayMs = Math.min(1_000, Math.max(0, deadline - now()));
+          if (delayMs > 0) {
+            log(failure(reason, { retryInMs: delayMs }).message);
+            await wait(delayMs);
+            if (now() < deadline) continue;
+          }
+          throw failure(reason, { failure: 'readiness_timeout' });
+        }
+        throw failure(reason);
+      }
+      let body;
+      try { body = Buffer.from(await response.arrayBuffer()); }
+      catch { throw failure('body_read_failed'); }
+      const sha256 = digest(body);
+      if (body.length !== asset.bytes || sha256 !== asset.sha256) throw failure('bytes_mismatch', { expectedBytes: asset.bytes, actualBytes: body.length, expectedSha256: asset.sha256, actualSha256: sha256 }, 'bytes differ');
+      break;
+    }
   }
 }

test/deploy-console-assets.test.mjs

@@ -3,6 +3,7 @@ import { execFileSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
+import { createServer } from 'node:http';
 import test from 'node:test';
 import { assertAssetBootstrap, publishConsoleAssets, verifyConsoleAssets } from '../scripts/deploy-console-assets.mjs';
 
@@ -52,6 +53,135 @@ test('asset smoke validates actual version and bytes, including the R2-only prob
   await assert.rejects(verifyConsoleAssets({ ...options, fetchImpl: async () => new Response(null, { status: 404 }) }), /response failed/);
 });
 
+const blue = '11111111-1111-1111-1111-111111111111';
+const green = '22222222-2222-2222-2222-222222222222';
+function assetFixture() {
+  const bytes = Buffer.from('shared console asset');
+  const asset = { path: '/assets/index-Fixture123.js', bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
+  let time = 0;
+  const waits = [], logs = [];
+  return { bytes, asset, waits, logs, options: {
+    assets: [asset], baseUrl: 'https://router.example', version: green,
+    readinessTimeoutMs: 2_000, now: () => time,
+    wait: async ms => { waits.push(ms); time += ms; }, log: message => logs.push(message),
+  } };
+}
+
+test('an asset override can converge after health; only exact-version bytes pass', async () => {
+  const f = assetFixture();
+  const responses = [
+    new Response(f.bytes, { headers: { 'x-clawrouter-version-id': blue } }),
+    new Response(f.bytes, { headers: { 'x-clawrouter-version-id': green } }),
+  ];
+  let calls = 0;
+  await verifyConsoleAssets({ ...f.options, fetchImpl: async (url, init) => {
+    assert.equal(url, 'https://router.example' + f.asset.path);
+    assert.equal(init.method ?? 'GET', 'GET');
+    assert.equal(init.redirect, 'error');
+    assert.equal(init.headers['Cloudflare-Workers-Version-Overrides'], `clawrouter-edge="${green}"`);
+    return responses[calls++];
+  } });
+  assert.equal(calls, 2);
+  assert.equal(responses[0].bodyUsed, true, 'discard the unselected response');
+  assert.deepEqual(f.waits, [1_000]);
+  assert.match(f.logs[0], /version_mismatch/);
+});
+
+test('persistent fallback fails within the readiness budget with actionable metadata', async () => {
+  const f = assetFixture();
+  let calls = 0;
+  await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => {
+    calls++;
+    return new Response(f.bytes, { headers: { 'x-clawrouter-version-id': blue } });
+  } }), error => {
+    assert.match(error.message, /response failed/);
+    assert.match(error.message, /version_mismatch/);
+    assert.match(error.message, /readiness_timeout/);
+    assert.match(error.message, /index-Fixture123.js/);
+    assert.match(error.message, new RegExp(green));
+    assert.match(error.message, new RegExp(blue));
+    assert.match(error.message, /"status":200/);
+    return true;
+  });
+  assert.equal(calls, 2);
+  assert.deepEqual(f.waits, [1_000, 1_000]);
+});
+
+test('selected-version HTTP and byte failures stay fatal and never log response bodies', async () => {
+  for (const status of [404, 503]) {
+    const f = assetFixture();
+    let calls = 0;
+    await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => {
+      calls++;
+      return new Response('fixture private error body', { status, headers: { 'x-clawrouter-version-id': green } });
+    } }), error => {
+      assert.match(error.message, new RegExp(`"status":${status}`));
+      assert.match(error.message, /http_status/);
+      assert.match(error.message, /index-Fixture123.js/);
+      assert.doesNotMatch(error.message, /fixture private error body/);
+      return true;
+    });
+    assert.equal(calls, 1);
+    assert.deepEqual(f.waits, []);
+  }
+  const f = assetFixture();
+  await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => new Response('fixture private bytes', { headers: { 'x-clawrouter-version-id': green } }) }), error => {
+    assert.match(error.message, /bytes differ/);
+    assert.match(error.message, /index-Fixture123.js/);
+    assert.match(error.message, /expectedSha256/);
+    assert.doesNotMatch(error.message, /fixture private bytes/);
+    return true;
+  });
+  assert.deepEqual(f.waits, []);
+});
+
+test('missing version proof and network failures retain safe diagnostics without retry', async () => {
+  const f = assetFixture();
+  await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => new Response(f.bytes) }), error => {
+    assert.match(error.message, /"actualVersion":null/);
+    assert.match(error.message, /version_mismatch/);
+    return true;
+  });
+  await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => { throw new Error('fixture secret in transport error'); } }), error => {
+    assert.match(error.message, /request_failed/);
+    assert.match(error.message, /index-Fixture123.js/);
+    assert.doesNotMatch(error.message, /fixture secret/);
+    return true;
+  });
+  assert.deepEqual(f.waits, []);
+});
+
+test('a selected-version asset with a stalled body is bounded by the request deadline', { timeout: 3_000 }, async t => {
+  const server = createServer((_request, response) => {
+    response.writeHead(200, { 'x-clawrouter-version-id': green });
+    response.write('unfinished fixture content');
+  });
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
+  const f = assetFixture();
+  await assert.rejects(verifyConsoleAssets({ assets: [f.asset], baseUrl: `http://127.0.0.1:${server.address().port}`, version: green, readinessTimeoutMs: 500 }), error => {
+    assert.match(error.message, /body_read_failed/);
+    assert.match(error.message, /"status":200/);
+    assert.doesNotMatch(error.message, /unfinished fixture content/);
+    return true;
+  });
+});
+
+test('diagnostics reject private paths and redact malformed version headers', async () => {
+  const f = assetFixture();
+  await assert.rejects(verifyConsoleAssets({ ...f.options, assets: [{ ...f.asset, path: '/assets/fixture.js?private=fixture-only-secret' }], fetchImpl: async () => assert.fail('invalid path must not dispatch') }), error => {
+    assert.match(error.message, /Invalid console asset receipt path/);
+    assert.doesNotMatch(error.message, /fixture-only-secret/);
+    return true;
+  });
+  await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => new Response(f.bytes, { headers: { 'x-clawrouter-version-id': 'fixture-only-secret' } }) }), error => {
+    assert.match(error.message, /"actualVersion":"invalid"/);
+    assert.doesNotMatch(error.message, /fixture-only-secret/);
+    return true;
+  });
+  assert.deepEqual(f.waits, []);
+});
+
 test('legacy bootstrap refuses changed asset inputs until both versions support shared assets', (t) => {
   mkdirSync('.artifacts', { recursive: true });
   const directory = mkdtempSync(join(process.cwd(), '.artifacts/asset-source-'));

test/deploy-train.test.mjs

@@ -11,6 +11,8 @@ import { inspectSource } from '../scripts/deploy-train-source.mjs';
 import { command, runTrain, uploadVersion } from '../scripts/deploy-train.mjs';
 import { assertSecretLineage, uploadMessage } from '../scripts/deploy-train-secrets.mjs';
 import { recoverRemote, failureIssue } from '../scripts/deploy-train-recover.mjs';
+import { createHash } from 'node:crypto';
+import { verifyConsoleAssets } from '../scripts/deploy-console-assets.mjs';
 
 const blue = '11111111-1111-1111-1111-111111111111';
 const green = '22222222-2222-2222-2222-222222222222';
@@ -83,6 +85,38 @@ test('zero-only success removes candidate and leaves exactly blue100', async ()
   assert.deepEqual(f.deployment.versions, [{ version_id: blue, percentage: 100 }]);
 });
 
+test('zero-smoke waits for asset version convergence before declaring the rehearsal passed', async () => {
+  const f = fixture();
+  const bytes = Buffer.from('fixture console asset');
+  const asset = { path: '/assets/index-Fixture123.js', bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
+  let calls = 0;
+  await runTrain({ ...f.options, mode: 'zero-smoke', smoke: async () => {
+    await verifyConsoleAssets({ assets: [asset], baseUrl: 'https://router.example', version: green,
+      readinessTimeoutMs: 2_000, now: f.options.now, wait: f.options.wait, log() {},
+      fetchImpl: async () => {
+        assert.equal(f.state.percentage, 0, 'never promote during asset readiness');
+        return new Response(bytes, { headers: { 'x-clawrouter-version-id': calls++ === 0 ? blue : green } });
+      } });
+  } });
+  assert.equal(calls, 2);
+  assert.equal(f.state.status, 'rehearsed');
+  assert.deepEqual(posts(f).map(p => p.versions), [splitVersions(blue, green, 0), [{ version_id: blue, percentage: 100 }]]);
+});
+
+test('a selected-version asset failure preserves diagnostic metadata and restores blue at zero traffic', async () => {
+  const f = fixture();
+  const asset = { path: '/assets/index-Fixture123.js', bytes: 0, sha256: createHash('sha256').update('').digest('hex') };
+  await assert.rejects(runTrain({ ...f.options, smoke: () => verifyConsoleAssets({
+    assets: [asset], baseUrl: 'https://router.example', version: green, log() {},
+    fetchImpl: async () => new Response('fixture private response', { status: 503, headers: { 'x-clawrouter-version-id': green } }),
+  }) }), /http_status/);
+  assert.equal(f.state.status, 'rolled-back');
+  assert.match(f.state.failure, /index-Fixture123.js/);
+  assert.match(f.state.failure, /"status":503/);
+  assert.doesNotMatch(f.state.failure, /fixture private response/);
+  assert.deepEqual(posts(f).map(p => p.versions), [splitVersions(blue, green, 0), [{ version_id: blue, percentage: 100 }]]);
+});
+
 for (const [name, percentage] of [['initial override', 0], ['10 percent', 10], ['50 percent', 50], ['final full cutover', 100]]) {
   test(`smoke breach at ${name} restores blue and throws`, async () => {
     const f = fixture({ failSmokePercentage: percentage });