fix: renew workspace callback tokens for sessions awake past 24h

.claude/skills/api-reference/SKILL.md

@@ -220,6 +220,7 @@ The MCP `create_trigger` tool intentionally creates cron triggers only. Generic
 - `POST /api/workspaces/:id/provisioning-failed` — Workspace provisioning failure callback (sets workspace to `error`)
 - `POST /api/workspaces/:id/heartbeat` — Workspace activity heartbeat callback
 - `GET /api/workspaces/:id/runtime` — Workspace runtime metadata callback (repository/branch for recovery)
+- `POST /api/workspaces/:id/callback-token/renew` — Renew a workspace callback token before it expires. Two proofs: the current, unexpired workspace token in `Authorization`, and `{ nodeId, nodeToken }` (the hosting node's node-scoped token) in the body. The workspace must be `creating`/`running`/`recovery`, bound to that VM node (never an Instant `cf-container` node), owned by the node's user, and on a non-terminal node; otherwise 401/403/410 (`NODE_CALLBACK_UNAUTHORIZED`/`NODE_CALLBACK_FORBIDDEN` when only the node proof failed). Authenticated attempts count against `RATE_LIMIT_CALLBACK_TOKEN_RENEWAL` per workspace per window (429 `RATE_LIMIT_EXCEEDED` with `Retry-After`). A token younger than `CALLBACK_TOKEN_REFRESH_THRESHOLD_RATIO` of its lifetime gets `{ renewed: false }`; otherwise `{ renewed: true, token, expiresAt }`, where the token keeps the chain's first `iat` as `gen_iat`. Responses carry `Cache-Control: no-store`
 - `POST /api/workspaces/:id/boot-log` — Workspace boot progress log callback
 - `POST /api/workspaces/:id/agent-settings` — Workspace agent settings callback (model, permissionMode)
 - `POST /api/projects/:id/workspaces/:workspaceId/eviction` — VM-agent callback JWT endpoint that validates node/workspace/runtime-generation identity and successful container stop, atomically marks the workspace `evicted` and closes usage/agent sessions, then serializes replay-safe ProjectData finalization through NodeLifecycle. A stale generation returns 410; failed finalization is retryable. Explicit restart requires renewed capacity admission and rotates the runtime generation

.claude/skills/env-reference/SKILL.md

@@ -534,6 +534,13 @@ by the read-only cron-liveness check.
 - `MCP_INCIDENT_LIST_MAX` — Maximum result count accepted by the private `list_incident_queue` MCP tool (default: 50)
 - `HETZNER_MAX_LIST_PAGES` — Maximum pages per Hetzner list request (default: 100)
 
+### Callback Tokens
+
+- `CALLBACK_TOKEN_EXPIRY_MS` — Lifetime of node- and workspace-scoped VM callback JWTs (default: `86400000` / 24h). Changing it does not extend tokens already issued.
+- `CALLBACK_TOKEN_REFRESH_THRESHOLD_RATIO` — Fraction of a callback token's lifetime after which it may be renewed (default: `0.5`, clamped to `0.1`–`0.9`). Gates both the node token refresh in `POST /api/nodes/:id/heartbeat` and workspace token renewal in `POST /api/workspaces/:id/callback-token/renew`; a token younger than this is not re-minted.
+- `RATE_LIMIT_CALLBACK_TOKEN_RENEWAL` — Authenticated workspace callback-token renewal attempts allowed per workspace per window (default: `12`). Counted atomically in D1 (`workspace_callback_token_renewal_rate_limits`) only after both proofs and the node binding pass; a healthy agent asks about once per half token lifetime.
+- `RATE_LIMIT_CALLBACK_TOKEN_RENEWAL_WINDOW_SECONDS` — Window for `RATE_LIMIT_CALLBACK_TOKEN_RENEWAL` (default: `3600`).
+
 ### Timeouts
 
 - `ORCHESTRATOR_STOP_CAS_MAX_ATTEMPTS` — Maximum task-status compare-and-set attempts after a parent hard-stops a child runtime (default: 2)
@@ -727,6 +734,16 @@ Generated deployments validate and pass these values through cloud-init to newly
 - `SESSION_SNAPSHOT_PROGRESS_REPORT_INTERVAL` — Minimum interval between progress callbacks while a checkpoint continues making progress (default: `15s`)
 - `SESSION_SNAPSHOT_PROGRESS_REPORT_TIMEOUT` — Timeout for each best-effort progress callback to the control plane (default: `5s`)
 
+### Workspace Callback Token Renewal
+
+The agent renews each workspace callback token after a successful node heartbeat once the token is past the refresh ratio (`internal/server/workspace_callback_token_renewal.go`). These use their defaults unless set in the agent service environment.
+
+- `WORKSPACE_CALLBACK_TOKEN_REFRESH_RATIO` — Fraction of a workspace token's lifetime after which the agent renews it (default: `0.5`, clamped to `0.1`–`0.9`; the control plane's `CALLBACK_TOKEN_REFRESH_THRESHOLD_RATIO` still decides)
+- `WORKSPACE_CALLBACK_TOKEN_RENEWAL_TIMEOUT` — Timeout for one renewal request (default: `15s`)
+- `WORKSPACE_CALLBACK_TOKEN_RENEWAL_RETRY_INITIAL` — First backoff after a transient renewal failure (default: `1m`)
+- `WORKSPACE_CALLBACK_TOKEN_RENEWAL_RETRY_MAX` — Backoff ceiling, also the wait after a "not yet due" answer (default: `30m`)
+- `MSG_AUTH_RENEWAL_WAIT` — How long chat-message delivery may stay paused on a rejected (401) workspace token before the pause is reported as an error (default: `15m`). Held messages are kept either way.
+
 ### File Operations
 
 - `FILE_LIST_TIMEOUT` — Timeout for file listing commands (default: 10s)

apps/api/.env.example

@@ -492,6 +492,12 @@ DEBUG_AGENT_MODEL_OUTPUT_TOKENS=4096
 # MAX_CLIENT_ERROR_BATCH_SIZE=25
 # MAX_CLIENT_ERROR_BODY_BYTES=65536
 
+# VM agent callback tokens (node- and workspace-scoped callback JWTs)
+# CALLBACK_TOKEN_EXPIRY_MS=86400000                    # Token lifetime (default: 24h)
+# CALLBACK_TOKEN_REFRESH_THRESHOLD_RATIO=0.5           # Lifetime fraction before renewal (default: 0.5, clamped 0.1-0.9)
+# RATE_LIMIT_CALLBACK_TOKEN_RENEWAL=12                 # Workspace token renewal attempts per workspace per window (default: 12)
+# RATE_LIMIT_CALLBACK_TOKEN_RENEWAL_WINDOW_SECONDS=3600  # Renewal rate-limit window in seconds (default: 3600)
+
 # VM agent error reporting
 # MAX_VM_AGENT_ERROR_BODY_BYTES=32768
 # MAX_VM_AGENT_ERROR_BATCH_SIZE=10

apps/api/src/db/migrations/0180_workspace_callback_token_renewal_rate_limits.sql

@@ -0,0 +1,15 @@
+-- Atomic per-workspace counters for workspace callback-token renewal
+-- (POST /api/workspaces/:id/callback-token/renew).
+--
+-- A credential rotation endpoint needs a per-principal limit whose state cannot
+-- lose increments under concurrent requests, which KV read-modify-write cannot
+-- guarantee (.claude/rules/28). One row per workspace; a new window replaces the
+-- counter in place, and deleting the workspace deletes its row.
+--
+-- Additive only: a new table. No DROP, no table rebuild.
+
+CREATE TABLE workspace_callback_token_renewal_rate_limits (
+  workspace_id TEXT PRIMARY KEY NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
+  window_start INTEGER NOT NULL CHECK (window_start >= 0),
+  count INTEGER NOT NULL CHECK (count >= 0)
+);

apps/api/src/db/schema.ts

@@ -160,6 +160,20 @@ export const aiSpendRateLimits = sqliteTable(
   })
 );
 
+// =============================================================================
+// Workspace Callback Token Renewal Rate Limits
+// =============================================================================
+export const workspaceCallbackTokenRenewalRateLimits = sqliteTable(
+  'workspace_callback_token_renewal_rate_limits',
+  {
+    workspaceId: text('workspace_id')
+      .primaryKey()
+      .references(() => workspaces.id, { onDelete: 'cascade' }),
+    windowStart: integer('window_start').notNull(),
+    count: integer('count').notNull(),
+  }
+);
+
 // =============================================================================
 // Sessions (BetterAuth)
 // =============================================================================

apps/api/src/env.ts

@@ -334,6 +334,8 @@ export interface Env extends WebhookTriggerEnv, TaskRecoveryEnv {
   RATE_LIMIT_SESSION_SUMMARIZE_WINDOW_SECONDS?: string;
   RATE_LIMIT_IDENTITY_TOKEN?: string;
   RATE_LIMIT_IDENTITY_TOKEN_WINDOW_SECONDS?: string;
+  RATE_LIMIT_CALLBACK_TOKEN_RENEWAL?: string; // Authenticated workspace callback-token renewal attempts per workspace per window (default: 12)
+  RATE_LIMIT_CALLBACK_TOKEN_RENEWAL_WINDOW_SECONDS?: string; // Window for RATE_LIMIT_CALLBACK_TOKEN_RENEWAL (default: 3600)
   /**
    * Max Codex refresh requests per user per window. Defaults to 30. Enforced
    * atomically by CodexRefreshLock DO using ctx.storage (not KV). See

apps/api/src/middleware/rate-limit.ts

@@ -54,6 +54,10 @@ export const DEFAULT_RATE_LIMITS = {
   // Voice transcription runs Workers AI Whisper on every request. Per MINUTE, not per hour
   // (`DEFAULT_TRANSCRIBE_WINDOW_SECONDS`): dictation is bursty, and the budget is for abuse.
   TRANSCRIBE: 30,
+  // Workspace callback-token renewal, per workspace. A healthy VM agent asks about once
+  // per half token lifetime, so this only bounds a holder of both proofs replaying them
+  // (`services/workspace-callback-token-renewal-rate-limit.ts`).
+  CALLBACK_TOKEN_RENEWAL: 12,
 } as const;
 
 /** Default time window (1 hour in seconds) */

apps/api/src/routes/_stale-callback-guard.ts

@@ -1,6 +1,5 @@
-import { decodeJwt } from 'jose';
-
 import type { Env } from '../env';
+import { callbackTokenGenerationIssuedAtSeconds } from '../services/callback-token-claims';
 
 /**
  * Staleness guard for VM-agent → control-plane DESTRUCTIVE callbacks (S2).
@@ -37,26 +36,24 @@ export const DEFAULT_INSTANT_STALE_CALLBACK_MARGIN_MS = 60_000;
 export function getInstantStaleCallbackMarginMs(env: Env): number {
   const raw = env.INSTANT_STALE_CALLBACK_MARGIN_MS;
   const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
-  return Number.isFinite(parsed) && parsed >= 0
-    ? parsed
-    : DEFAULT_INSTANT_STALE_CALLBACK_MARGIN_MS;
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : DEFAULT_INSTANT_STALE_CALLBACK_MARGIN_MS;
 }
 
 /**
- * Read the `iat` (issued-at) claim from an ALREADY-VERIFIED callback token and
+ * Read the issue time of an ALREADY-VERIFIED callback token's generation and
  * return it in milliseconds. The token MUST have been verified by
  * `verifyCallbackToken` first — `decodeJwt` does not verify the signature; it is
- * used here only to read a claim `verifyCallbackToken` does not surface.
+ * used here only to read claims `verifyCallbackToken` does not surface.
+ *
+ * A renewed workspace token keeps its chain's first `iat` in the `gen_iat` claim
+ * (`callbackTokenGenerationIssuedAtSeconds`), so renewal never makes a superseded
+ * container generation look newer than the recovery that replaced it.
  *
- * Returns null when the token cannot be decoded or has no numeric `iat`.
+ * Returns null when the token cannot be decoded or has no usable issue time.
  */
 export function callbackTokenIssuedAtMs(token: string): number | null {
-  try {
-    const claims = decodeJwt(token);
-    return typeof claims.iat === 'number' ? claims.iat * 1000 : null;
-  } catch {
-    return null;
-  }
+  const generationIssuedAtSeconds = callbackTokenGenerationIssuedAtSeconds(token);
+  return generationIssuedAtSeconds === null ? null : generationIssuedAtSeconds * 1000;
 }
 
 export interface SupersededInstantCallbackInput {

apps/api/src/routes/workspaces/_helpers.ts

@@ -11,6 +11,11 @@ import { errors } from '../../middleware/error';
 import { signCallbackToken, verifyCallbackToken } from '../../services/jwt';
 import { createWorkspaceOnNode } from '../../services/node-agent';
 import { nodeStatusTerminatesCallbacks } from '../../services/node-callback-auth';
+import {
+  sameWorkspaceCallbackIdentity,
+  WORKSPACE_CALLBACK_ACTIVE_STATUSES,
+  type WorkspaceCallbackIdentitySnapshot,
+} from '../../services/workspace-callback-identity';
 import {
   signalWorkspaceDeletionUnconfirmedCallback,
   type WorkspaceDeletionCallbackKind,
@@ -20,27 +25,18 @@ import {
   type WorkspaceGitSourceProject,
 } from '../../services/workspace-git-source';
 
+export {
+  sameWorkspaceCallbackIdentity,
+  WORKSPACE_CALLBACK_ACTIVE_STATUSES,
+  type WorkspaceCallbackIdentitySnapshot,
+} from '../../services/workspace-callback-identity';
+
 export const ACTIVE_WORKSPACE_STATUSES = new Set(['running', 'recovery'] as const);
-export const WORKSPACE_CALLBACK_ACTIVE_STATUSES: ReadonlySet<string> = new Set([
-  'creating',
-  'running',
-  'recovery',
-]);
 export const WORKSPACE_CALLBACK_PROVISIONING_FAILURE_STATUSES: ReadonlySet<string> = new Set([
   'creating',
   'error',
 ]);
 
-export interface WorkspaceCallbackIdentitySnapshot {
-  workspaceId: string;
-  userId: string;
-  projectId: string | null;
-  chatSessionId: string | null;
-  status: string;
-  nodeId: string | null;
-  nodeStatus: string | null;
-}
-
 export function isActiveWorkspaceStatus(status: string): boolean {
   return ACTIVE_WORKSPACE_STATUSES.has(status as 'running' | 'recovery');
 }
@@ -117,21 +113,6 @@ export async function loadWorkspaceCallbackIdentity(
   return rows[0] ?? null;
 }
 
-export function sameWorkspaceCallbackIdentity(
-  current: WorkspaceCallbackIdentitySnapshot,
-  expected: WorkspaceCallbackIdentitySnapshot
-): boolean {
-  return (
-    current.workspaceId === expected.workspaceId &&
-    current.userId === expected.userId &&
-    current.projectId === expected.projectId &&
-    current.chatSessionId === expected.chatSessionId &&
-    current.status === expected.status &&
-    current.nodeId === expected.nodeId &&
-    current.nodeStatus === expected.nodeStatus
-  );
-}
-
 interface WorkspaceCallbackTransitionValues {
   status: string;
   updatedAt: string;
@@ -448,8 +429,13 @@ export async function scheduleWorkspaceCreateOnNode(
       },
       { beforeExternalMutation: assertCurrent }
     );
-    if (options.durableRetry && (!acknowledgement || typeof acknowledgement !== 'object'
-      || !('workspaceId' in acknowledgement) || acknowledgement.workspaceId !== workspaceId)) {
+    if (
+      options.durableRetry &&
+      (!acknowledgement ||
+        typeof acknowledgement !== 'object' ||
+        !('workspaceId' in acknowledgement) ||
+        acknowledgement.workspaceId !== workspaceId)
+    ) {
       throw new Error('Node agent did not acknowledge the expected workspace identity');
     }
     await assertCurrent();

apps/api/src/routes/workspaces/callback-token-renewal.ts

@@ -0,0 +1,74 @@
+/**
+ * POST /api/workspaces/:id/callback-token/renew — VM agent callback.
+ *
+ * The VM agent calls this (packages/vm-agent/internal/server/workspace_callback_token_renewal.go)
+ * before a workspace callback token expires, so a workspace that stays awake longer than
+ * CALLBACK_TOKEN_EXPIRY_MS keeps working. Auth is two callback JWTs, never a session cookie
+ * (.claude/rules/34): the workspace's current token in `Authorization`, and the hosting
+ * node's id and node token in the JSON body. See
+ * `services/workspace-callback-token-renewal.ts` for the renewal rules and
+ * `services/workspace-callback-token-binding.ts` for the node binding they enforce.
+ *
+ * `workspacesRoutes` applies no session middleware, so this callback route is safe to
+ * mount there next to the other workspace callbacks (`/:id/messages`, `/:id/git-token`).
+ */
+import { Hono } from 'hono';
+import * as v from 'valibot';
+
+import type { Env } from '../../env';
+import { extractBearerToken } from '../../lib/auth-helpers';
+import { errors } from '../../middleware/error';
+import { RateLimitError } from '../../middleware/rate-limit';
+import {
+  renewWorkspaceCallbackToken,
+  type WorkspaceCallbackTokenRenewalResult,
+} from '../../services/workspace-callback-token-renewal';
+
+const MAX_NODE_ID_LENGTH = 128;
+const MAX_NODE_TOKEN_LENGTH = 16 * 1024;
+
+const RenewalRequestSchema = v.object({
+  nodeId: v.pipe(v.string(), v.trim(), v.minLength(1), v.maxLength(MAX_NODE_ID_LENGTH)),
+  nodeToken: v.pipe(v.string(), v.minLength(1), v.maxLength(MAX_NODE_TOKEN_LENGTH)),
+});
+
+const callbackTokenRenewalRoutes = new Hono<{ Bindings: Env }>();
+
+callbackTokenRenewalRoutes.post('/:id/callback-token/renew', async (c) => {
+  // A success response carries a credential; no intermediary may store any response.
+  // Set first so error responses from the global handler carry it too.
+  c.header('Cache-Control', 'no-store');
+  const workspaceId = c.req.param('id');
+  const workspaceToken = extractBearerToken(c.req.header('Authorization'));
+
+  // The body carries a credential, so validation failures must never echo it back
+  // (jsonValidator interpolates offending values into its 400 message).
+  let raw: unknown;
+  try {
+    raw = await c.req.json();
+  } catch {
+    throw errors.badRequest('Invalid callback token renewal request');
+  }
+  const parsed = v.safeParse(RenewalRequestSchema, raw);
+  if (!parsed.success) {
+    throw errors.badRequest('Invalid callback token renewal request');
+  }
+
+  let result: WorkspaceCallbackTokenRenewalResult;
+  try {
+    result = await renewWorkspaceCallbackToken(c.env, {
+      workspaceId,
+      workspaceToken,
+      nodeId: parsed.output.nodeId,
+      nodeToken: parsed.output.nodeToken,
+    });
+  } catch (err) {
+    if (err instanceof RateLimitError) {
+      c.header('Retry-After', Math.max(1, err.retryAfter).toString());
+    }
+    throw err;
+  }
+  return c.json(result);
+});
+
+export { callbackTokenRenewalRoutes };

apps/api/src/routes/workspaces/index.ts

@@ -3,6 +3,7 @@ import { Hono } from 'hono';
 import type { Env } from '../../env';
 import { agentSessionSuspendResumeRoutes } from './agent-session-suspend-resume';
 import { agentSessionRoutes } from './agent-sessions';
+import { callbackTokenRenewalRoutes } from './callback-token-renewal';
 import { crudRoutes } from './crud';
 import { lifecycleRoutes } from './lifecycle';
 import { localForwardRoutes } from './local-forward';
@@ -17,5 +18,6 @@ workspacesRoutes.route('/', agentSessionRoutes);
 workspacesRoutes.route('/', agentSessionSuspendResumeRoutes);
 workspacesRoutes.route('/', runtimeRoutes);
 workspacesRoutes.route('/', sessionSnapshotRoutes);
+workspacesRoutes.route('/', callbackTokenRenewalRoutes);
 
 export { workspacesRoutes };

apps/api/src/services/callback-token-claims.ts

@@ -0,0 +1,48 @@
+/**
+ * Claim readers for callback JWTs that `verifyCallbackToken` does not surface.
+ *
+ * Kept apart from `jwt.ts` (signing and verification) so the guards that read these
+ * claims do not depend on the signer module. Every function here uses `decodeJwt`,
+ * which does NOT verify the signature: callers must verify the token first.
+ */
+import { decodeJwt } from 'jose';
+
+/**
+ * Claim carried by a RENEWED workspace callback token: the `iat` (seconds) of the first
+ * token in its renewal chain. Renewal must not move a token's generation forward, because
+ * the Instant stale-callback guard compares the generation's issue time with the most
+ * recent recovery (see `routes/_stale-callback-guard.ts`). First issuance omits the claim;
+ * the token's own `iat` is then the generation.
+ */
+export const CALLBACK_TOKEN_GENERATION_ISSUED_AT_CLAIM = 'gen_iat';
+
+function positiveIntegerClaim(value: unknown): number | null {
+  return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null;
+}
+
+/**
+ * Generation issue time (seconds) of an already-verified callback token: the `gen_iat`
+ * claim a renewal preserved, else the token's own `iat`. Null when neither is a positive
+ * integer.
+ */
+export function callbackTokenGenerationIssuedAtSeconds(token: string): number | null {
+  try {
+    const claims = decodeJwt(token);
+    return (
+      positiveIntegerClaim(claims[CALLBACK_TOKEN_GENERATION_ISSUED_AT_CLAIM]) ??
+      positiveIntegerClaim(claims.iat)
+    );
+  } catch {
+    return null;
+  }
+}
+
+/** `exp` (ms) of an already-verified callback token; null if absent. */
+export function callbackTokenExpiresAtMs(token: string): number | null {
+  try {
+    const exp = positiveIntegerClaim(decodeJwt(token).exp);
+    return exp === null ? null : exp * 1000;
+  } catch {
+    return null;
+  }
+}