From d0b534c01290a851b9fd623b3c3eec17bcab1b61 Mon Sep 17 00:00:00 2001 From: joshavant <830519+joshavant@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:46:24 -0700 Subject: [PATCH] fix: tolerate asset version propagation with safe diagnostics --- docs/deploy-cloudflare.md | 7 ++ scripts/deploy-console-assets.mjs | 50 +++++++++-- test/deploy-console-assets.test.mjs | 130 ++++++++++++++++++++++++++++ test/deploy-train.test.mjs | 34 ++++++++ 4 files changed, 215 insertions(+), 6 deletions(-) diff --git a/docs/deploy-cloudflare.md b/docs/deploy-cloudflare.md index 51b21f69..d303aa82 100644 --- a/docs/deploy-cloudflare.md +++ b/docs/deploy-cloudflare.md @@ -1325,6 +1325,13 @@ readback, never by blindly repeating the write. `Cloudflare-Workers-Version-Overrides: clawrouter-edge=""`. `/v1/health.versionId` and Worker response headers must identify green; a silently ignored override is not a successful smoke. + Each console asset must also prove its requested version and exact bytes, + through both green and blue (the initial legacy bootstrap keeps its documented + exception). A successful response from another runtime is + retried for at most 30 seconds per asset because overrides can still propagate + after health passes. Missing/invalid version proof, HTTP failures and corrupt + bytes fail immediately. Failure receipts include the public asset path, HTTP + status and expected/observed version; no response content is logged. 2. Green 10% for at least two minutes, then green 50% for at least three minutes. Full override smoke runs immediately and waits at most 30 seconds between checks through each hold. It includes a paid golden-provider request and diff --git a/scripts/deploy-console-assets.mjs b/scripts/deploy-console-assets.mjs index 46613500..3fd1f085 100644 --- a/scripts/deploy-console-assets.mjs +++ b/scripts/deploy-console-assets.mjs @@ -2,10 +2,12 @@ import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { readdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; const pattern = /^[A-Za-z0-9_.-]+-[A-Za-z0-9_-]{8,}\.(js|css|woff2?|png|jpe?g|svg|webp|gif|ico)$/; const types = { js: 'text/javascript', css: 'text/css', woff: 'font/woff', woff2: 'font/woff2', png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', svg: 'image/svg+xml', webp: 'image/webp', gif: 'image/gif', ico: 'image/x-icon' }; const digest = (bytes) => createHash('sha256').update(bytes).digest('hex'); +const versionPattern = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i; export function assertAssetBootstrap(blueSha, cwd = process.cwd()) { const git = (...args) => execFileSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000 }).trim(); @@ -58,13 +60,49 @@ export async function publishConsoleAssets({ sha, env = process.env, directory = return receipts; } -export async function verifyConsoleAssets({ assets, baseUrl, version, legacy = false, fetchImpl = fetch }) { +export async function verifyConsoleAssets({ assets, baseUrl, version, legacy = false, fetchImpl = fetch, + readinessTimeoutMs = 30_000, now = () => performance.now(), wait = sleep, log = console.log }) { + if (!Number.isInteger(readinessTimeoutMs) || readinessTimeoutMs < 1 || readinessTimeoutMs > 30_000) throw new Error('Invalid console asset readiness timeout'); for (const asset of assets) { if (legacy && asset.probe) continue; // legacy blue has no shared fallback yet - const response = await fetchImpl(`${baseUrl}${asset.path}`, { headers: { 'Cloudflare-Workers-Version-Overrides': `clawrouter-edge="${version}"` }, redirect: 'error', signal: AbortSignal.timeout(30_000) }); - const actualVersion = response.headers.get('x-clawrouter-version-id'); - if (!response.ok || (actualVersion !== version && !(legacy && actualVersion === null))) { await response.body?.cancel(); throw new Error('Cross-version console asset response failed'); } - const body = Buffer.from(await response.arrayBuffer()); - if (body.length !== asset.bytes || digest(body) !== asset.sha256) throw new Error('Cross-version console asset bytes differ'); + if (typeof asset.path !== 'string' || !asset.path.startsWith('/assets/') || !pattern.test(asset.path.slice('/assets/'.length))) throw new Error('Invalid console asset receipt path'); + const deadline = now() + readinessTimeoutMs; + let attempt = 0; + while (true) { + const details = { path: asset.path, expectedVersion: version, actualVersion: null, status: null, attempt: ++attempt }; + const failure = (reason, extra = {}, kind = 'response failed') => new Error(`Cross-version console asset ${kind}: ${JSON.stringify({ ...details, reason, ...extra })}`); + const remaining = Math.ceil(deadline - now()); + if (remaining <= 0) throw failure('request_deadline', { failure: 'readiness_timeout' }); + let response; + try { + response = await fetchImpl(`${baseUrl}${asset.path}`, { method: 'GET', headers: { 'Cloudflare-Workers-Version-Overrides': `clawrouter-edge="${version}"` }, redirect: 'error', signal: AbortSignal.timeout(remaining) }); + } catch { throw failure('request_failed'); } // Never expose transport messages, headers or response bodies. + const actualVersion = response.headers.get('x-clawrouter-version-id'); + details.status = response.status; + details.actualVersion = actualVersion === null ? null : versionPattern.test(actualVersion) ? actualVersion : 'invalid'; + if (!response.ok || (actualVersion !== version && !(legacy && actualVersion === null))) { + await response.body?.cancel().catch(() => {}); + const reason = response.ok ? 'version_mismatch' : 'http_status'; + // An override can converge independently for each asset after health passes. + // Retry only successful responses proving another runtime, never missing + // proof, HTTP errors, corrupt bytes, or a mutation. + if (response.ok && versionPattern.test(actualVersion ?? '')) { + const delayMs = Math.min(1_000, Math.max(0, deadline - now())); + if (delayMs > 0) { + log(failure(reason, { retryInMs: delayMs }).message); + await wait(delayMs); + if (now() < deadline) continue; + } + throw failure(reason, { failure: 'readiness_timeout' }); + } + throw failure(reason); + } + let body; + try { body = Buffer.from(await response.arrayBuffer()); } + catch { throw failure('body_read_failed'); } + const sha256 = digest(body); + if (body.length !== asset.bytes || sha256 !== asset.sha256) throw failure('bytes_mismatch', { expectedBytes: asset.bytes, actualBytes: body.length, expectedSha256: asset.sha256, actualSha256: sha256 }, 'bytes differ'); + break; + } } } diff --git a/test/deploy-console-assets.test.mjs b/test/deploy-console-assets.test.mjs index 7af4e839..4e25408a 100644 --- a/test/deploy-console-assets.test.mjs +++ b/test/deploy-console-assets.test.mjs @@ -3,6 +3,7 @@ import { execFileSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; +import { createServer } from 'node:http'; import test from 'node:test'; import { assertAssetBootstrap, publishConsoleAssets, verifyConsoleAssets } from '../scripts/deploy-console-assets.mjs'; @@ -52,6 +53,135 @@ test('asset smoke validates actual version and bytes, including the R2-only prob await assert.rejects(verifyConsoleAssets({ ...options, fetchImpl: async () => new Response(null, { status: 404 }) }), /response failed/); }); +const blue = '11111111-1111-1111-1111-111111111111'; +const green = '22222222-2222-2222-2222-222222222222'; +function assetFixture() { + const bytes = Buffer.from('shared console asset'); + const asset = { path: '/assets/index-Fixture123.js', bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }; + let time = 0; + const waits = [], logs = []; + return { bytes, asset, waits, logs, options: { + assets: [asset], baseUrl: 'https://router.example', version: green, + readinessTimeoutMs: 2_000, now: () => time, + wait: async ms => { waits.push(ms); time += ms; }, log: message => logs.push(message), + } }; +} + +test('an asset override can converge after health; only exact-version bytes pass', async () => { + const f = assetFixture(); + const responses = [ + new Response(f.bytes, { headers: { 'x-clawrouter-version-id': blue } }), + new Response(f.bytes, { headers: { 'x-clawrouter-version-id': green } }), + ]; + let calls = 0; + await verifyConsoleAssets({ ...f.options, fetchImpl: async (url, init) => { + assert.equal(url, 'https://router.example' + f.asset.path); + assert.equal(init.method ?? 'GET', 'GET'); + assert.equal(init.redirect, 'error'); + assert.equal(init.headers['Cloudflare-Workers-Version-Overrides'], `clawrouter-edge="${green}"`); + return responses[calls++]; + } }); + assert.equal(calls, 2); + assert.equal(responses[0].bodyUsed, true, 'discard the unselected response'); + assert.deepEqual(f.waits, [1_000]); + assert.match(f.logs[0], /version_mismatch/); +}); + +test('persistent fallback fails within the readiness budget with actionable metadata', async () => { + const f = assetFixture(); + let calls = 0; + await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => { + calls++; + return new Response(f.bytes, { headers: { 'x-clawrouter-version-id': blue } }); + } }), error => { + assert.match(error.message, /response failed/); + assert.match(error.message, /version_mismatch/); + assert.match(error.message, /readiness_timeout/); + assert.match(error.message, /index-Fixture123.js/); + assert.match(error.message, new RegExp(green)); + assert.match(error.message, new RegExp(blue)); + assert.match(error.message, /"status":200/); + return true; + }); + assert.equal(calls, 2); + assert.deepEqual(f.waits, [1_000, 1_000]); +}); + +test('selected-version HTTP and byte failures stay fatal and never log response bodies', async () => { + for (const status of [404, 503]) { + const f = assetFixture(); + let calls = 0; + await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => { + calls++; + return new Response('fixture private error body', { status, headers: { 'x-clawrouter-version-id': green } }); + } }), error => { + assert.match(error.message, new RegExp(`"status":${status}`)); + assert.match(error.message, /http_status/); + assert.match(error.message, /index-Fixture123.js/); + assert.doesNotMatch(error.message, /fixture private error body/); + return true; + }); + assert.equal(calls, 1); + assert.deepEqual(f.waits, []); + } + const f = assetFixture(); + await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => new Response('fixture private bytes', { headers: { 'x-clawrouter-version-id': green } }) }), error => { + assert.match(error.message, /bytes differ/); + assert.match(error.message, /index-Fixture123.js/); + assert.match(error.message, /expectedSha256/); + assert.doesNotMatch(error.message, /fixture private bytes/); + return true; + }); + assert.deepEqual(f.waits, []); +}); + +test('missing version proof and network failures retain safe diagnostics without retry', async () => { + const f = assetFixture(); + await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => new Response(f.bytes) }), error => { + assert.match(error.message, /"actualVersion":null/); + assert.match(error.message, /version_mismatch/); + return true; + }); + await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => { throw new Error('fixture secret in transport error'); } }), error => { + assert.match(error.message, /request_failed/); + assert.match(error.message, /index-Fixture123.js/); + assert.doesNotMatch(error.message, /fixture secret/); + return true; + }); + assert.deepEqual(f.waits, []); +}); + +test('a selected-version asset with a stalled body is bounded by the request deadline', { timeout: 3_000 }, async t => { + const server = createServer((_request, response) => { + response.writeHead(200, { 'x-clawrouter-version-id': green }); + response.write('unfinished fixture content'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); })); + const f = assetFixture(); + await assert.rejects(verifyConsoleAssets({ assets: [f.asset], baseUrl: `http://127.0.0.1:${server.address().port}`, version: green, readinessTimeoutMs: 500 }), error => { + assert.match(error.message, /body_read_failed/); + assert.match(error.message, /"status":200/); + assert.doesNotMatch(error.message, /unfinished fixture content/); + return true; + }); +}); + +test('diagnostics reject private paths and redact malformed version headers', async () => { + const f = assetFixture(); + await assert.rejects(verifyConsoleAssets({ ...f.options, assets: [{ ...f.asset, path: '/assets/fixture.js?private=fixture-only-secret' }], fetchImpl: async () => assert.fail('invalid path must not dispatch') }), error => { + assert.match(error.message, /Invalid console asset receipt path/); + assert.doesNotMatch(error.message, /fixture-only-secret/); + return true; + }); + await assert.rejects(verifyConsoleAssets({ ...f.options, fetchImpl: async () => new Response(f.bytes, { headers: { 'x-clawrouter-version-id': 'fixture-only-secret' } }) }), error => { + assert.match(error.message, /"actualVersion":"invalid"/); + assert.doesNotMatch(error.message, /fixture-only-secret/); + return true; + }); + assert.deepEqual(f.waits, []); +}); + test('legacy bootstrap refuses changed asset inputs until both versions support shared assets', (t) => { mkdirSync('.artifacts', { recursive: true }); const directory = mkdtempSync(join(process.cwd(), '.artifacts/asset-source-')); diff --git a/test/deploy-train.test.mjs b/test/deploy-train.test.mjs index 2f8477ff..8406464a 100644 --- a/test/deploy-train.test.mjs +++ b/test/deploy-train.test.mjs @@ -11,6 +11,8 @@ import { inspectSource } from '../scripts/deploy-train-source.mjs'; import { command, runTrain, uploadVersion } from '../scripts/deploy-train.mjs'; import { assertSecretLineage, uploadMessage } from '../scripts/deploy-train-secrets.mjs'; import { recoverRemote, failureIssue } from '../scripts/deploy-train-recover.mjs'; +import { createHash } from 'node:crypto'; +import { verifyConsoleAssets } from '../scripts/deploy-console-assets.mjs'; const blue = '11111111-1111-1111-1111-111111111111'; const green = '22222222-2222-2222-2222-222222222222'; @@ -83,6 +85,38 @@ test('zero-only success removes candidate and leaves exactly blue100', async () assert.deepEqual(f.deployment.versions, [{ version_id: blue, percentage: 100 }]); }); +test('zero-smoke waits for asset version convergence before declaring the rehearsal passed', async () => { + const f = fixture(); + const bytes = Buffer.from('fixture console asset'); + const asset = { path: '/assets/index-Fixture123.js', bytes: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') }; + let calls = 0; + await runTrain({ ...f.options, mode: 'zero-smoke', smoke: async () => { + await verifyConsoleAssets({ assets: [asset], baseUrl: 'https://router.example', version: green, + readinessTimeoutMs: 2_000, now: f.options.now, wait: f.options.wait, log() {}, + fetchImpl: async () => { + assert.equal(f.state.percentage, 0, 'never promote during asset readiness'); + return new Response(bytes, { headers: { 'x-clawrouter-version-id': calls++ === 0 ? blue : green } }); + } }); + } }); + assert.equal(calls, 2); + assert.equal(f.state.status, 'rehearsed'); + assert.deepEqual(posts(f).map(p => p.versions), [splitVersions(blue, green, 0), [{ version_id: blue, percentage: 100 }]]); +}); + +test('a selected-version asset failure preserves diagnostic metadata and restores blue at zero traffic', async () => { + const f = fixture(); + const asset = { path: '/assets/index-Fixture123.js', bytes: 0, sha256: createHash('sha256').update('').digest('hex') }; + await assert.rejects(runTrain({ ...f.options, smoke: () => verifyConsoleAssets({ + assets: [asset], baseUrl: 'https://router.example', version: green, log() {}, + fetchImpl: async () => new Response('fixture private response', { status: 503, headers: { 'x-clawrouter-version-id': green } }), + }) }), /http_status/); + assert.equal(f.state.status, 'rolled-back'); + assert.match(f.state.failure, /index-Fixture123.js/); + assert.match(f.state.failure, /"status":503/); + assert.doesNotMatch(f.state.failure, /fixture private response/); + assert.deepEqual(posts(f).map(p => p.versions), [splitVersions(blue, green, 0), [{ version_id: blue, percentage: 100 }]]); +}); + for (const [name, percentage] of [['initial override', 0], ['10 percent', 10], ['50 percent', 50], ['final full cutover', 100]]) { test(`smoke breach at ${name} restores blue and throws`, async () => { const f = fixture({ failSmokePercentage: percentage });