From 69d00ccf938896582f204a499b02b8649e24b055 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 07:59:12 +0200 Subject: [PATCH 01/19] feat(observability): persistence, indexing and security log-port parity; store fingerprints in the support bundle (#3239) Wave 4a of epic #3233. StoreFingerprint (keiko-contracts) with a fail-closed guard; read-only store opens and collectStoreFingerprints in keiko-server, composed by keiko support export into the manifest (storeFingerprints / storesUnavailable); store.opened, store.encryption-migrated (outside the migration transaction), memory-vault key source and quarantine events; chunker configuration, ANN runtime and index-invalidation lines, repository.fingerprint-diff.completed in Local Knowledge; an independent SecurityLogSink in keiko-security with keychain fallback, shard-unreadable and key-tier events wired at every composition site with removal-fails tests; ServerLogCategory gains security. Refs #3239 Refs #3233 Co-Authored-By: Claude Fable 5 --- ...-log-v2-machine-reconstruction-contract.md | 26 ++ docs/observability/op-catalog.generated.json | 152 +++++++--- packages/keiko-cli/src/support-export.test.ts | 50 ++++ packages/keiko-cli/src/support-export.ts | 33 +++ .../src/support.fingerprints.e2e.test.ts | 278 ++++++++++++++++++ packages/keiko-cli/src/support.test.ts | 121 +++++++- packages/keiko-cli/src/support.ts | 23 +- packages/keiko-contracts/src/index.ts | 8 + .../src/store-fingerprint.test.ts | 114 +++++++ .../keiko-contracts/src/store-fingerprint.ts | 149 ++++++++++ packages/keiko-local-knowledge/src/index.ts | 2 + .../src/indexing/orchestrator.test.ts | 32 ++ .../src/indexing/orchestrator.ts | 16 + .../src/repository-pod.test.ts | 72 +++++ .../src/repository-pod.ts | 32 +- .../retrieval/local-vector-index-port.test.ts | 47 +++ .../src/retrieval/local-vector-index-port.ts | 38 ++- .../src/retrieval/usearch-ann-index.test.ts | 48 +++ .../src/retrieval/usearch-ann-index.ts | 37 ++- .../src/retrieval/vector-index.ts | 8 + .../src/store-content-encryption.ts | 73 ++++- .../keiko-local-knowledge/src/store.test.ts | 163 +++++++++- packages/keiko-local-knowledge/src/store.ts | 86 +++++- packages/keiko-memory-vault/src/db.test.ts | 203 ++++++++++++- packages/keiko-memory-vault/src/db.ts | 187 +++++++++++- packages/keiko-memory-vault/src/index.ts | 12 + .../src/migrate-encrypt.test.ts | 129 ++++++++ .../keiko-memory-vault/src/migrate-encrypt.ts | 77 ++++- packages/keiko-memory-vault/src/schema.ts | 23 +- .../src/vault-keychain-log-wiring.test.ts | 97 ++++++ .../keiko-memory-vault/src/vault-log.test.ts | 262 +++++++++++++++++ packages/keiko-memory-vault/src/vault-log.ts | 162 ++++++++++ packages/keiko-memory-vault/src/vault.test.ts | 72 +++++ packages/keiko-memory-vault/src/vault.ts | 94 +++++- packages/keiko-security/src/index.ts | 6 + packages/keiko-security/src/log-port.test.ts | 262 +++++++++++++++++ packages/keiko-security/src/log-port.ts | 160 ++++++++++ .../keiko-security/src/macos-keychain.test.ts | 130 +++++++- packages/keiko-security/src/macos-keychain.ts | 58 +++- .../keiko-security/src/secret-vault.test.ts | 269 ++++++++++++++++- packages/keiko-security/src/secret-vault.ts | 92 +++++- .../keiko-security/src/sqlite-corruption.ts | 15 +- .../src/atlassian/credentialVault.ts | 5 + packages/keiko-server/src/atlassian/wiring.ts | 5 + .../src/conversation-attachment-store.test.ts | 41 ++- .../src/conversation-attachment-store.ts | 11 + .../keiko-server/src/credentialPersistence.ts | 10 + packages/keiko-server/src/credentialVault.ts | 15 + ...achment-history-securitylog-wiring.test.ts | 171 +++++++++++ .../deps-vault-key-securitylog-wiring.test.ts | 268 +++++++++++++++++ packages/keiko-server/src/deps.test.ts | 43 ++- packages/keiko-server/src/deps.ts | 42 ++- .../keiko-server/src/editor/hotExitStore.ts | 5 + .../localHistory/localHistoryStore.test.ts | 38 +++ .../editor/localHistory/localHistoryStore.ts | 12 +- ...setup-vault-key-securitylog-wiring.test.ts | 154 ++++++++++ packages/keiko-server/src/gateway-setup.ts | 2 + packages/keiko-server/src/index.ts | 14 + .../src/localKnowledgeKeyProvider.ts | 5 + ...memory-handlers-securitylog-wiring.test.ts | 99 +++++++ packages/keiko-server/src/memory-handlers.ts | 15 + .../src/observability/server-log.ts | 7 +- .../src/observability/server-logger.ts | 1 + ...tOrchestration-keychain-log-wiring.test.ts | 98 ++++++ .../src/qualityIntelligence/figma/index.ts | 1 + .../figmaSnapshotOrchestration.ts | 19 +- .../src/store-fingerprints.test.ts | 201 +++++++++++++ .../keiko-server/src/store-fingerprints.ts | 177 +++++++++++ packages/keiko-server/src/store/db.test.ts | 144 ++++++++- packages/keiko-server/src/store/db.ts | 187 +++++++++++- packages/keiko-server/src/store/index.ts | 3 + .../src/workspace-index-provider.ts | 5 + 72 files changed, 5602 insertions(+), 114 deletions(-) create mode 100644 packages/keiko-cli/src/support.fingerprints.e2e.test.ts create mode 100644 packages/keiko-contracts/src/store-fingerprint.test.ts create mode 100644 packages/keiko-contracts/src/store-fingerprint.ts create mode 100644 packages/keiko-memory-vault/src/migrate-encrypt.test.ts create mode 100644 packages/keiko-memory-vault/src/vault-keychain-log-wiring.test.ts create mode 100644 packages/keiko-memory-vault/src/vault-log.test.ts create mode 100644 packages/keiko-memory-vault/src/vault-log.ts create mode 100644 packages/keiko-security/src/log-port.test.ts create mode 100644 packages/keiko-security/src/log-port.ts create mode 100644 packages/keiko-server/src/deps-attachment-history-securitylog-wiring.test.ts create mode 100644 packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts create mode 100644 packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts create mode 100644 packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts create mode 100644 packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotOrchestration-keychain-log-wiring.test.ts create mode 100644 packages/keiko-server/src/store-fingerprints.test.ts create mode 100644 packages/keiko-server/src/store-fingerprints.ts diff --git a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md index 9cad737238..df4a4746a5 100644 --- a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md +++ b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md @@ -454,6 +454,32 @@ copy without a single source of truth. that two packages could otherwise structurally agree on without importing each other. `keiko-contracts` (the leaf) gains only pure wire/data shapes used by more than one package (the client-diagnostics ingest request, a store-fingerprint data shape) — never logic. +- **Wave 4a** (epic #3233 §8) added two more ports of that same shape: `SecurityLogSink` + (`keiko-security/src/log-port.ts`, categories `security`/`diagnostic`) and `MemoryVaultLogSink` + (`keiko-memory-vault/src/vault-log.ts`, categories `memory`/`diagnostic`). `keiko-server`'s + `processServerLogSink()` supplies both at every call site: `keiko-memory-vault`'s `cipher.ts` + (`keyFromKeychain`, threaded through `createMemoryVault`'s `securityLogSink` option from + `memory-handlers.ts`'s `createBffMemoryVault`), `qualityIntelligence/figmaSnapshotOrchestration.ts`, + `conversation-attachment-store.ts`, and `editor/localHistory/localHistoryStore.ts` (the latter two + via `deps.ts`). Each site degrades to a silent no-op sink when unwired, so a missing composition + edge fails closed rather than throwing. +- **Gap g18** (epic #3233 §8, later in Wave 4a): `resolveLocalVaultKey` + (`keiko-security/src/secret-vault.ts`), the shared env -> macOS Keychain -> keyfile key-tier + resolver every local vault composes, had no `sink` parameter at all, so none of its production + callers could ever report which tier answered or that the keychain tier fell back — independent + of the `SecurityLogSink` port existing. It now emits `security.vault.key-resolved` + (`extra.source: "env" | "keychain" | "keyfile"`) on every resolution, and its own keychain reader + (`createKeychainVaultKeyAccess`, a separate implementation from `macos-keychain.ts`'s + `readMacosKeychainSecret` — it spawns `security` through an injectable `KeychainCommandRunner` + rather than that function) reports `security.keychain.fallback` via the same + `emitKeychainFallback` helper `macos-keychain.ts` exports, so the two keychain surfaces cannot + report the fallback shape differently. `processServerLogSink()` reaches it through every caller: + `credentialVault.ts` and `gateway-setup.ts`'s `persistGatewayConfig`/`durableStoredGatewayConfig` + (the provider-credential vault), `atlassian/credentialVault.ts` via `atlassian/wiring.ts`, + `editor/hotExitStore.ts`, `localKnowledgeKeyProvider.ts`, `workspace-index-provider.ts` (all five + via `deps.ts`), and the `conversation-attachment-store.ts`/`editor/localHistory/localHistoryStore.ts` + sink options wired in the earlier bullet, which reached the sharded vault's shard reads but not + this key-resolution layer until now. - **ADR-0048** (evidence artifact confidentiality) classified evidence artifacts into confidentiality tiers and mandated write-time permission enforcement. The support bundle is a new artifact class in that same spirit: every log line it carries was already redacted before this contract existed diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index d22b3c042f..483f001454 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -113,175 +113,175 @@ { "op": "embedding.preflight.cache-hit", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2675", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2690", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.completed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2656", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2671", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2647", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2662", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2786", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2801", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.identity-adopted", "category": "embedding", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2723", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2738", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.identity-refreshed", "category": "embedding", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2735", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2750", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.identity-rejected", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2692", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2707", "package": "keiko-local-knowledge" }, { "op": "embedding.preflight.started", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2640", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2655", "package": "keiko-local-knowledge" }, { "op": "indexing.chunking.failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1425", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1440", "package": "keiko-local-knowledge" }, { "op": "indexing.chunking.failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1763", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1778", "package": "keiko-local-knowledge" }, { "op": "indexing.chunking.failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:967", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:982", "package": "keiko-local-knowledge" }, { "op": "indexing.discovery.limit-reached", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2868", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2883", "package": "keiko-local-knowledge" }, { "op": "indexing.discovery.scope-error", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2216", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2231", "package": "keiko-local-knowledge" }, { "op": "indexing.document.chunked", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1302", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1317", "package": "keiko-local-knowledge" }, { "op": "indexing.document.embedded", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1498", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1513", "package": "keiko-local-knowledge" }, { "op": "indexing.document.embedding-started", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1054", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1069", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extracted", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1298", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1313", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extracted", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1985", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2000", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extraction-failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1094", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1109", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extraction-started", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2202", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2217", "package": "keiko-local-knowledge" }, { "op": "indexing.document.failed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1467", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1482", "package": "keiko-local-knowledge" }, { "op": "indexing.document.skipped", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1067", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1082", "package": "keiko-local-knowledge" }, { "op": "indexing.document.skipped", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1116", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1131", "package": "keiko-local-knowledge" }, { "op": "indexing.document.skipped", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1988", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2003", "package": "keiko-local-knowledge" }, { "op": "indexing.job.finished", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:3193", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:3209", "package": "keiko-local-knowledge" }, { "op": "indexing.job.received", "category": "indexing", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:3006", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:3022", "package": "keiko-local-knowledge" }, { "op": "indexing.job.started", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2946", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2961", "package": "keiko-local-knowledge" }, { "op": "indexing.source.completed", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2325", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2340", "package": "keiko-local-knowledge" }, { "op": "indexing.source.started", "category": "unknown", - "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2307", + "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2322", "package": "keiko-local-knowledge" }, { @@ -293,15 +293,63 @@ { "op": "knowledge.store.encryption-rejected", "category": "diagnostic", - "site": "packages/keiko-local-knowledge/src/store.ts:510", + "site": "packages/keiko-local-knowledge/src/store.ts:514", "package": "keiko-local-knowledge" }, { "op": "knowledge.store.quarantined", "category": "diagnostic", - "site": "packages/keiko-local-knowledge/src/store.ts:473", + "site": "packages/keiko-local-knowledge/src/store.ts:477", "package": "keiko-local-knowledge" }, + { + "op": "repository.fingerprint-diff.completed", + "category": "indexing", + "site": "packages/keiko-local-knowledge/src/repository-pod.ts:341", + "package": "keiko-local-knowledge" + }, + { + "op": "search.index-invalidated-for-capsule", + "category": "search", + "site": "packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.ts:167", + "package": "keiko-local-knowledge" + }, + { + "op": "search.native-runtime-resolved", + "category": "search", + "site": "packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.ts:411", + "package": "keiko-local-knowledge" + }, + { + "op": "store.encryption-migrated", + "category": "diagnostic", + "site": "packages/keiko-local-knowledge/src/store-content-encryption.ts:305", + "package": "keiko-local-knowledge" + }, + { + "op": "memory-vault.log.sink-failed", + "category": "diagnostic", + "site": "packages/keiko-memory-vault/src/vault-log.ts:140", + "package": "keiko-memory-vault" + }, + { + "op": "memory-vault.store.opened", + "category": "memory", + "site": "packages/keiko-memory-vault/src/vault.ts:420", + "package": "keiko-memory-vault" + }, + { + "op": "memory-vault.store.quarantined", + "category": "diagnostic", + "site": "packages/keiko-memory-vault/src/db.ts:121", + "package": "keiko-memory-vault" + }, + { + "op": "store.encryption-migrated", + "category": "diagnostic", + "site": "packages/keiko-memory-vault/src/migrate-encrypt.ts:117", + "package": "keiko-memory-vault" + }, { "op": "", "category": "embedding", @@ -590,6 +638,30 @@ "site": "packages/keiko-model-gateway/src/http.ts:1217", "package": "keiko-model-gateway" }, + { + "op": "security.keychain.fallback", + "category": "security", + "site": "packages/keiko-security/src/macos-keychain.ts:151", + "package": "keiko-security" + }, + { + "op": "security.log.sink-failed", + "category": "diagnostic", + "site": "packages/keiko-security/src/log-port.ts:138", + "package": "keiko-security" + }, + { + "op": "security.vault.key-resolved", + "category": "security", + "site": "packages/keiko-security/src/secret-vault.ts:272", + "package": "keiko-security" + }, + { + "op": "security.vault.shard-unreadable", + "category": "security", + "site": "packages/keiko-security/src/secret-vault.ts:537", + "package": "keiko-security" + }, { "op": "", "category": "diagnostic", @@ -599,25 +671,25 @@ { "op": "", "category": "diagnostic", - "site": "packages/keiko-server/src/observability/server-log.ts:714", + "site": "packages/keiko-server/src/observability/server-log.ts:719", "package": "keiko-server" }, { "op": "", "category": "unknown", - "site": "packages/keiko-server/src/observability/server-logger.ts:143", + "site": "packages/keiko-server/src/observability/server-logger.ts:144", "package": "keiko-server" }, { "op": "", "category": "unknown", - "site": "packages/keiko-server/src/observability/server-logger.ts:169", + "site": "packages/keiko-server/src/observability/server-logger.ts:170", "package": "keiko-server" }, { "op": "", "category": "unknown", - "site": "packages/keiko-server/src/observability/server-logger.ts:170", + "site": "packages/keiko-server/src/observability/server-logger.ts:171", "package": "keiko-server" }, { @@ -785,13 +857,19 @@ { "op": "server-log.write-failed", "category": "diagnostic", - "site": "packages/keiko-server/src/observability/server-log.ts:312", + "site": "packages/keiko-server/src/observability/server-log.ts:317", "package": "keiko-server" }, { "op": "server-log.write-failed", "category": "diagnostic", - "site": "packages/keiko-server/src/observability/server-log.ts:338", + "site": "packages/keiko-server/src/observability/server-log.ts:343", + "package": "keiko-server" + }, + { + "op": "store.opened", + "category": "setup", + "site": "packages/keiko-server/src/store/db.ts:919", "package": "keiko-server" } ], diff --git a/packages/keiko-cli/src/support-export.test.ts b/packages/keiko-cli/src/support-export.test.ts index c97eb664d6..5eb554a448 100644 --- a/packages/keiko-cli/src/support-export.test.ts +++ b/packages/keiko-cli/src/support-export.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { SERVER_LOG_SCHEMA_VERSION } from "@oscharko-dev/keiko-server"; +import type { StoreFingerprint } from "@oscharko-dev/keiko-contracts"; import type { AuditResult } from "./audit.js"; import { @@ -47,6 +48,8 @@ function baseManifestInput( skippedLogFiles: [], auditSummary: HEALTHY_AUDIT, evidenceIndexCount: 0, + storeFingerprints: [], + storesUnavailable: [], ...overrides, }; } @@ -320,6 +323,8 @@ describe("buildSupportBundleManifest", () => { sectionsExcluded: [], auditSummary: REDACTED_HEALTHY_AUDIT, evidenceIndexCount: 3, + storeFingerprints: [], + storesUnavailable: [], }); }); @@ -336,6 +341,51 @@ describe("buildSupportBundleManifest", () => { expect(JSON.stringify(manifest)).not.toContain(HEALTHY_AUDIT.stateDir); }); + // Wave 4a (epic #3233 §6.2/§8): a valid fingerprint and an unavailable-store entry both pass + // through to the manifest unchanged. + it("carries valid storeFingerprints and storesUnavailable entries through unchanged", () => { + const validFingerprint: StoreFingerprint = { + store: "ui", + schemaVersion: 19, + migrationsApplied: ["v1"], + tableRowCounts: { projects: 2 }, + quickCheckOk: true, + encryptionMode: "plaintext", + }; + + const manifest = buildSupportBundleManifest( + baseManifestInput({ + storeFingerprints: [validFingerprint], + storesUnavailable: [{ store: "memory-vault", reasonKind: "missing" }], + }), + ); + + expect(manifest.storeFingerprints).toEqual([validFingerprint]); + expect(manifest.storesUnavailable).toEqual([{ store: "memory-vault", reasonKind: "missing" }]); + }); + + // Defense-in-depth (this file's own header discipline, and the redaction doctrine every other + // guard in this repo follows): a fingerprint that fails the shared `isStoreFingerprint` guard — + // here, a negative row count nothing in this codebase could produce — must never reach the + // exported bundle, even though `buildSupportBundleManifest`'s own caller is the only producer + // today. Proves the filter is live, not merely a type-level assumption. + it("drops a fingerprint that fails isStoreFingerprint rather than embedding it", () => { + const malformed = { + store: "local-knowledge", + schemaVersion: 1, + migrationsApplied: [], + tableRowCounts: { capsules: -1 }, + quickCheckOk: true, + encryptionMode: "plaintext", + } as unknown as StoreFingerprint; + + const manifest = buildSupportBundleManifest( + baseManifestInput({ storeFingerprints: [malformed] }), + ); + + expect(manifest.storeFingerprints).toEqual([]); + }); + // `buildSupportBundleManifest` forwards `schemaVersion` verbatim rather than deriving its own // copy (see `ManifestInput.schemaVersion`'s doc comment) — the real value comes from // `packages/keiko-server/src/observability/server-log.ts`'s own `SERVER_LOG_SCHEMA_VERSION`, diff --git a/packages/keiko-cli/src/support-export.ts b/packages/keiko-cli/src/support-export.ts index 9e1bee47cd..9768146510 100644 --- a/packages/keiko-cli/src/support-export.ts +++ b/packages/keiko-cli/src/support-export.ts @@ -16,6 +16,7 @@ import { readFileSync, readdirSync, statSync } from "node:fs"; import { join } from "node:path"; +import { isStoreFingerprint, type StoreFingerprint } from "@oscharko-dev/keiko-contracts"; import type { AuditResult } from "./audit.js"; export const CURRENT_LOG_FILE_NAME = "server.log"; @@ -226,6 +227,24 @@ function redactedAuditSummary(audit: AuditResult): RedactedAuditSummary { return { ok: audit.ok, classes: audit.classes }; } +// ─── Store fingerprints (Wave 4a, epic #3233 §6.2/§8) ────────────────────────────────────────── +// +// `support.ts` opens each of the three stores (ui, local-knowledge, memory-vault) through that +// store package's genuinely read-only open (`openNodeUiDatabaseReadOnly` / +// `openKnowledgeStoreReadOnly` / `openMemoryDatabaseReadOnly`, `node:sqlite`'s `readOnly: true`) — +// never the mutating production open path, which migrates, recovers, re-encrypts, or quarantines as +// an ordinary part of opening — against the resolved state-dir paths, and calls that store +// package's own `computeStoreFingerprint(db)`. A store that does not exist yet (never used from +// this state dir) or that cannot be opened (corrupt, or a vault key the operator has not supplied) +// contributes no fingerprint — its name and a closed-vocabulary reason go to `storesUnavailable` +// instead, never a path or the underlying error's message. +export type StoreUnavailableReasonKind = "missing" | "open-failed"; + +export interface StoreUnavailableEntry { + readonly store: StoreFingerprint["store"]; + readonly reasonKind: StoreUnavailableReasonKind; +} + export interface SupportBundleManifest { readonly $section: "manifest"; readonly schemaVersion: 2; @@ -253,6 +272,12 @@ export interface SupportBundleManifest { readonly sectionsExcluded: readonly string[]; readonly auditSummary: RedactedAuditSummary; readonly evidenceIndexCount: number; + // Wave 4a additions (epic #3233 §6.2/§8), additive over the Wave 1 shape above. A bundle + // produced by a pre-Wave-4a build simply lacks `storeFingerprints`; `storesUnavailable` is + // always present once this exporter runs, even when every store fingerprinted cleanly (empty + // array), so a reader never has to distinguish "not attempted" from "nothing to report". + readonly storeFingerprints?: readonly StoreFingerprint[]; + readonly storesUnavailable: readonly StoreUnavailableEntry[]; } export interface ManifestInput { @@ -280,6 +305,8 @@ export interface ManifestInput { readonly skippedLogFiles: readonly SkippedLogFile[]; readonly auditSummary: AuditResult; readonly evidenceIndexCount: number; + readonly storeFingerprints: readonly StoreFingerprint[]; + readonly storesUnavailable: readonly StoreUnavailableEntry[]; } export function buildSupportBundleManifest(input: ManifestInput): SupportBundleManifest { @@ -301,6 +328,12 @@ export function buildSupportBundleManifest(input: ManifestInput): SupportBundleM sectionsExcluded: [], auditSummary: redactedAuditSummary(input.auditSummary), evidenceIndexCount: input.evidenceIndexCount, + // Defense-in-depth (never trust the producer unconditionally, matching this repo's redaction + // doctrine): re-validated against the same closed structural guard the manifest's own bundle + // reader would use, so a malformed fingerprint (a future producer bug, a version-skewed + // dependency) is silently dropped rather than embedded in a customer-facing artifact. + storeFingerprints: input.storeFingerprints.filter(isStoreFingerprint), + storesUnavailable: input.storesUnavailable, }; } diff --git a/packages/keiko-cli/src/support.fingerprints.e2e.test.ts b/packages/keiko-cli/src/support.fingerprints.e2e.test.ts new file mode 100644 index 0000000000..9dcc8d8b3d --- /dev/null +++ b/packages/keiko-cli/src/support.fingerprints.e2e.test.ts @@ -0,0 +1,278 @@ +// Wave 4a acceptance test (epic #3233 §6.2/§8, w4a-bundle-manifest-fingerprints): seeds a real, +// on-disk instance of all three stores (ui, local-knowledge, memory-vault) under one temp state +// dir using each store package's own create/open helpers — never a hand-rolled fixture that +// re-derives a schema this test does not own — then runs `keiko support export` in-process and +// asserts the manifest's `storeFingerprints` array reports all three with row counts matching +// what was seeded, and that none of the seeded row content reaches the serialised bundle. +// +// This is deliberately the one test in this work item that exercises the real store-opening path +// end to end (every other test — support.test.ts, support-export.test.ts — proves the "missing" +// path and the manifest-assembly logic against synthetic inputs); this file proves the "present +// and healthy" path actually works against the real packages, not a mock of them. + +import { randomBytes } from "node:crypto"; +import { + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + isStoreFingerprint, + type EmbeddingModelIdentity, + type KnowledgeCapsuleId, + type StoreFingerprint, +} from "@oscharko-dev/keiko-contracts"; +import type { MemoryId, UserId } from "@oscharko-dev/keiko-contracts/memory"; +import { createMemoryVault, MEMORY_DB_FILENAME } from "@oscharko-dev/keiko-memory-vault"; +import { + createCapsule, + openKnowledgeStore, + resolveKnowledgeStorePath, + type CreateCapsuleInput, +} from "@oscharko-dev/keiko-local-knowledge"; +import { createNodeUiStore, UI_DB_FILENAME } from "@oscharko-dev/keiko-server"; + +import type { AuditResult } from "./audit.js"; +import type { CliIo } from "./runner.js"; +import { runSupportCli, type SupportCliDeps } from "./support.js"; + +const HEALTHY_AUDIT: AuditResult = { + ok: true, + stateDir: "/irrelevant/.keiko", + classes: [{ id: "creds", title: "Credential references", status: "pass", findings: [] }], +}; + +const AUDIT_ENV = { KEIKO_LOCAL_STATE_AUDITOR: "/opt/keiko/scripts/lib/local-state-audit.mjs" }; + +function healthyAuditDeps(): SupportCliDeps["auditDeps"] { + return { loadAuditor: () => Promise.resolve({ auditLocalState: () => HEALTHY_AUDIT }) }; +} + +function makeIo(): { io: CliIo; out: () => string; err: () => string } { + const outChunks: string[] = []; + const errChunks: string[] = []; + return { + io: { + out: (text: string): void => { + outChunks.push(text); + }, + err: (text: string): void => { + errChunks.push(text); + }, + }, + out: (): string => outChunks.join(""), + err: (): string => errChunks.join(""), + }; +} + +// Distinctive, never-otherwise-present markers for each store's seeded row content — the test's +// proof that the bundle carries counts, never bodies. +const UI_PROJECT_MARKER = "e2e-ui-project-marker-3233"; +const CAPSULE_MARKER = "e2e-local-knowledge-capsule-marker-3233"; +const MEMORY_BODY_MARKER = "e2e-memory-vault-body-marker-3233"; + +const EMBEDDING_IDENTITY: EmbeddingModelIdentity = { + provider: "openai", + modelId: "text-embedding-3-small", + vectorDimensions: 1536, + vectorMetric: "cosine", + normalization: "l2", + instructionVersion: "keiko-embedding-input-v1", + embeddingSpaceFingerprint: "keiko-embedding-space-fingerprint-v1:3233-e2e", +}; + +function seedUiStore(stateDir: string): void { + const dbPath = join(stateDir, "ui", UI_DB_FILENAME); + const store = createNodeUiStore(dbPath); + const projectDir1 = mkdtempSync(join(stateDir, `${UI_PROJECT_MARKER}-1-`)); + const projectDir2 = mkdtempSync(join(stateDir, `${UI_PROJECT_MARKER}-2-`)); + store.createProject(projectDir1, UI_PROJECT_MARKER); + store.createProject(projectDir2, UI_PROJECT_MARKER); + store.close(); +} + +function capsuleInput(id: string): CreateCapsuleInput { + return { + id: id as KnowledgeCapsuleId, + displayName: CAPSULE_MARKER, + tags: [], + retrievalEffort: "default", + outputMode: "answers", + answerGroundingPolicy: "require-citations", + embeddingModelIdentity: EMBEDDING_IDENTITY, + lifecycleState: "draft", + storageReference: `${CAPSULE_MARKER}/${id}`, + }; +} + +function seedLocalKnowledgeStore(stateDir: string): void { + const dbPath = resolveKnowledgeStorePath({ runtimeStateDir: stateDir }); + const store = openKnowledgeStore({ dbPath }); + createCapsule(store, capsuleInput("cap-1")); + createCapsule(store, capsuleInput("cap-2")); + store.close(); +} + +function seedMemoryVault(stateDir: string, memoryKeyBase64: string): void { + const memoryDir = join(stateDir, "memory"); + const vault = createMemoryVault({ + memoryDir, + env: { KEIKO_MEMORY_DIR: memoryDir, KEIKO_MEMORY_KEY: memoryKeyBase64 }, + }); + const t = 1_700_000_000_000; + const memory = (id: string): Parameters[0] => ({ + id: id as MemoryId, + schemaVersion: "1", + scope: { kind: "user", userId: "u-1" as UserId }, + type: "preference", + body: MEMORY_BODY_MARKER, + provenance: { + sourceKind: "explicit-user-instruction", + capturedAt: t, + confidence: 0.9, + sensitivity: "confidential", + }, + validity: { validFrom: t }, + status: "accepted", + pinned: false, + tags: [], + createdAt: t, + updatedAt: t, + }); + vault.insertMemory(memory("m1")); + vault.insertMemory(memory("m2")); + vault.close(); +} + +describe("keiko support export — store fingerprints acceptance (Wave 4a)", () => { + let stateDir: string; + let outDir: string; + let memoryKeyBase64: string; + + beforeEach(() => { + // Realpath the tmpdir root: on macOS both /tmp and /var are symlinks, and the memory vault's + // own path guard (mirrored from keiko-server's UI-db guard) refuses a path with a symlinked + // ancestor — the same reason this package's own vault.test.ts realpaths its tmp root. + const root = realpathSync(tmpdir()); + stateDir = mkdtempSync(join(root, "keiko-support-fp-e2e-state-")); + outDir = mkdtempSync(join(root, "keiko-support-fp-e2e-out-")); + memoryKeyBase64 = randomBytes(32).toString("base64"); + seedUiStore(stateDir); + seedLocalKnowledgeStore(stateDir); + seedMemoryVault(stateDir, memoryKeyBase64); + }); + + afterEach(() => { + rmSync(stateDir, { recursive: true, force: true }); + rmSync(outDir, { recursive: true, force: true }); + }); + + it("reports three valid fingerprints with the seeded row counts, and leaks no row content", async () => { + const c = makeIo(); + const outPath = join(outDir, "bundle.jsonl"); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", outPath], + c.io, + { ...AUDIT_ENV, KEIKO_MEMORY_KEY: memoryKeyBase64 }, + { auditDeps: healthyAuditDeps() }, + ); + + expect(code).toBe(0); + const written = readFileSync(outPath, "utf8"); + const manifestLine = written.split("\n")[0] ?? "{}"; + const manifest = JSON.parse(manifestLine) as { + readonly storeFingerprints: readonly unknown[]; + readonly storesUnavailable: readonly unknown[]; + }; + + expect(manifest.storesUnavailable).toEqual([]); + expect(manifest.storeFingerprints).toHaveLength(3); + expect(manifest.storeFingerprints.every((entry) => isStoreFingerprint(entry))).toBe(true); + + const byStore = new Map( + (manifest.storeFingerprints as readonly StoreFingerprint[]).map((entry) => [ + entry.store, + entry, + ]), + ); + expect(byStore.get("ui")?.tableRowCounts.projects).toBe(2); + expect(byStore.get("local-knowledge")?.tableRowCounts.capsules).toBe(2); + expect(byStore.get("memory-vault")?.tableRowCounts.memories).toBe(2); + for (const store of ["ui", "local-knowledge", "memory-vault"]) { + expect(byStore.get(store)?.quickCheckOk).toBe(true); + } + + // The fingerprint is counts and closed-vocabulary labels only — never the row content that + // produced them. + expect(written).not.toContain(UI_PROJECT_MARKER); + expect(written).not.toContain(CAPSULE_MARKER); + expect(written).not.toContain(MEMORY_BODY_MARKER); + expect(written).not.toContain(memoryKeyBase64); + }); + + // RED (before fix): computing a store's fingerprint went through that store package's mutating + // production open path, which quarantines confirmed SQLite corruption as an ordinary part of + // opening — renaming the corrupt file aside and silently creating an empty replacement. A + // diagnostic export must never destroy the very corruption evidence an operator ran it to + // capture. Genuinely SQLite-corrupt bytes (not just an EISDIR, which the "open-failed" test in + // support.test.ts already covers and which never reaches the quarantine path at all) for all + // three stores. + it("never quarantines a genuinely SQLite-corrupt store file while computing its fingerprint", async () => { + const uiDbPath = join(stateDir, "ui", UI_DB_FILENAME); + const localKnowledgeDbPath = resolveKnowledgeStorePath({ runtimeStateDir: stateDir }); + const memoryDbPath = join(stateDir, "memory", MEMORY_DB_FILENAME); + const corruptBytes = "garbage that is not a sqlite header"; + for (const dbPath of [uiDbPath, localKnowledgeDbPath, memoryDbPath]) { + mkdirSync(dirname(dbPath), { recursive: true }); + writeFileSync(dbPath, corruptBytes); + } + + const c = makeIo(); + const outPath = join(outDir, "corrupt-bundle.jsonl"); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", outPath], + c.io, + { ...AUDIT_ENV, KEIKO_MEMORY_KEY: memoryKeyBase64 }, + { auditDeps: healthyAuditDeps() }, + ); + + expect(code).toBe(0); + // The corrupt files are untouched: same bytes, and no quarantine sidecar (`.corrupt.`) + // appeared next to any of them. + for (const dbPath of [uiDbPath, localKnowledgeDbPath, memoryDbPath]) { + expect(readFileSync(dbPath, "utf8")).toBe(corruptBytes); + const siblingNames = readdirSync(dirname(dbPath)); + expect(siblingNames.some((name) => name.includes(".corrupt."))).toBe(false); + } + + // Every store must land on the "unhealthy" side of the manifest for genuinely corrupt bytes: + // either an entry in `storesUnavailable` (the read-only open itself failed) or a present + // fingerprint whose `quickCheckOk` is `false` (the open succeeded but every read degraded) — + // never a fingerprint that reads as healthy, and never a thrown error out of the CLI either + // way (`code` is 0 above). + const manifest = JSON.parse(readFileSync(outPath, "utf8").split("\n")[0] ?? "{}") as { + readonly storeFingerprints: readonly StoreFingerprint[]; + readonly storesUnavailable: readonly { + readonly store: string; + readonly reasonKind: string; + }[]; + }; + const unavailableStores = new Set(manifest.storesUnavailable.map((entry) => entry.store)); + for (const store of ["ui", "local-knowledge", "memory-vault"] as const) { + const fingerprint = manifest.storeFingerprints.find((entry) => entry.store === store); + if (fingerprint === undefined) { + expect(unavailableStores.has(store)).toBe(true); + } else { + expect(fingerprint.quickCheckOk).toBe(false); + } + } + }); +}); diff --git a/packages/keiko-cli/src/support.test.ts b/packages/keiko-cli/src/support.test.ts index 70a605b2b6..0bfaa96c72 100644 --- a/packages/keiko-cli/src/support.test.ts +++ b/packages/keiko-cli/src/support.test.ts @@ -1,4 +1,5 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -8,7 +9,11 @@ import { type EvidenceManifest, type EvidenceStore, } from "@oscharko-dev/keiko-evidence"; -import { SERVER_LOG_SCHEMA_VERSION } from "@oscharko-dev/keiko-server"; +import { + createNodeUiStore, + SERVER_LOG_SCHEMA_VERSION, + UI_DB_FILENAME, +} from "@oscharko-dev/keiko-server"; import type { AuditResult } from "./audit.js"; import type { CliIo } from "./runner.js"; import { parseSupportArgs, runSupportCli, type SupportCliDeps } from "./support.js"; @@ -349,6 +354,120 @@ describe("runSupportCli export", () => { expect(c.err()).not.toContain(badOutPath); expect(existsSync(badOutPath)).toBe(false); }); + + // Wave 4a (epic #3233 §6.2/§8): none of the three stores has ever been created under this + // state dir, so the export must still succeed — each store is named in storesUnavailable with + // reasonKind "missing", and none of their (nonexistent) db files or directories are created as + // a side effect of merely running an export. + it("reports all three stores as missing, never creating them, when none exist under --state-dir", async () => { + const c = makeIo(); + const outPath = join(outDir, "no-stores.jsonl"); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", outPath], + c.io, + AUDIT_ENV, + { + auditDeps: healthyAuditDeps(), + evidenceStore: createInMemoryEvidenceStore(), + }, + ); + + expect(code).toBe(0); + const manifest: Record = JSON.parse( + readFileSync(outPath, "utf8").split("\n")[0] ?? "{}", + ) as Record; + expect(manifest.storeFingerprints).toEqual([]); + expect(manifest.storesUnavailable).toEqual( + expect.arrayContaining([ + { store: "ui", reasonKind: "missing" }, + { store: "local-knowledge", reasonKind: "missing" }, + { store: "memory-vault", reasonKind: "missing" }, + ]), + ); + expect(existsSync(join(stateDir, "ui"))).toBe(false); + expect(existsSync(join(stateDir, "memory"))).toBe(false); + expect(existsSync(join(stateDir, "local-knowledge"))).toBe(false); + }); + + // Wave 4a: a store whose db path exists but cannot be opened as a database (here, a directory + // sitting where the db file is expected — not classified as SQLite corruption, so the store's + // own quarantine-and-recover path never kicks in) reports reasonKind "open-failed", and the + // export still succeeds for the other two stores. + it("reports open-failed (never throwing) when a store's db path exists but cannot be opened", async () => { + mkdirSync(join(stateDir, "ui", UI_DB_FILENAME), { recursive: true }); + + const c = makeIo(); + const outPath = join(outDir, "open-failed.jsonl"); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", outPath], + c.io, + AUDIT_ENV, + { auditDeps: healthyAuditDeps(), evidenceStore: createInMemoryEvidenceStore() }, + ); + + expect(code).toBe(0); + const manifest: Record = JSON.parse( + readFileSync(outPath, "utf8").split("\n")[0] ?? "{}", + ) as Record; + expect(manifest.storesUnavailable).toEqual( + expect.arrayContaining([{ store: "ui", reasonKind: "open-failed" }]), + ); + expect(manifest.storeFingerprints).toEqual([]); + }); + + // RED (before fix): computing the ui store's fingerprint called `openNodeUiDatabase`, the + // mutating production open path — which unconditionally runs `sqlRecoverInterruptedClientTurns`, + // flipping any `client_turn_state = 'pending'` row to `'failed'`. That is exactly the evidence an + // operator running `keiko support export` to diagnose a stuck chat turn needs preserved. This + // goes through the real, on-disk ui store (no mock of the open path) so a regression in which + // fingerprint collection touches the production open path again cannot hide behind a fixture. + it("does not flip a pending client turn to failed as a side effect of computing the ui store fingerprint", async () => { + const uiDataDir = join(stateDir, "ui"); + mkdirSync(uiDataDir, { recursive: true }); + const dbPath = join(uiDataDir, UI_DB_FILENAME); + const projectDir = mkdtempSync(join(stateDir, "pending-turn-project-")); + const store = createNodeUiStore(dbPath); + store.createProject(projectDir); + const chat = store.createChat(projectDir, "Chat", "example-chat-model"); + const admission = store.admitChatTurn("turn-stuck-in-flight", { + chatId: chat.id, + role: "user", + content: "message stuck in flight", + timestamp: 1, + runId: undefined, + workflowId: undefined, + workflowStatus: undefined, + shortResult: undefined, + taskType: undefined, + }); + expect(admission.kind).toBe("admitted"); + if (admission.kind !== "admitted") throw new Error("expected canonical admission"); + store.close(); + + const c = makeIo(); + const outPath = join(outDir, "pending-turn.jsonl"); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", outPath], + c.io, + AUDIT_ENV, + { auditDeps: healthyAuditDeps(), evidenceStore: createInMemoryEvidenceStore() }, + ); + expect(code).toBe(0); + + const inspector = new DatabaseSync(dbPath, { readOnly: true }); + const stored = inspector + .prepare("SELECT client_turn_state FROM chat_messages WHERE id = ?") + .get(admission.userMessage.id) as { client_turn_state: string }; + inspector.close(); + expect(stored.client_turn_state).toBe("pending"); + + const manifest: Record = JSON.parse( + readFileSync(outPath, "utf8").split("\n")[0] ?? "{}", + ) as Record; + expect(manifest.storesUnavailable).toEqual( + expect.not.arrayContaining([{ store: "ui", reasonKind: "open-failed" }]), + ); + }); }); describe("runSupportCli analyze", () => { diff --git a/packages/keiko-cli/src/support.ts b/packages/keiko-cli/src/support.ts index f5b760d802..6993ad8ff2 100644 --- a/packages/keiko-cli/src/support.ts +++ b/packages/keiko-cli/src/support.ts @@ -17,7 +17,10 @@ import type { EvidenceStore } from "@oscharko-dev/keiko-evidence"; import type { EnvSource } from "@oscharko-dev/keiko-model-gateway"; import { type AuditCliDeps, AuditLoadError, auditLocalStateResult } from "./audit.js"; // GEN-PERF-CLI-001 — the evidence graph (and, below, the server module graph) load at dispatch, -// and only for `export`; `analyze` never needs either. +// and only for `export`; `analyze` never needs either. Store-fingerprint collection (ui, +// local-knowledge, memory-vault) is owned by keiko-server (ADR-0019 direction rule 7: keiko-cli +// is a leaf consumer and must not import keiko-local-knowledge directly) and reached through the +// same lazily-loaded server module, via `server.collectStoreFingerprints`. import { loadEvidence, loadServer } from "./lazy-modules.js"; import type { CliIo } from "./runner.js"; import { resolveStateDir } from "./state-paths.js"; @@ -46,11 +49,14 @@ const USAGE = `Usage: keiko support analyze FILE [--correlation-id ID] [--json] export writes a redacted .jsonl support bundle: a manifest line (local-state audit summary, -evidence-index count, exactly which log files were copied) followed by every line of -/logs/server*.log, copied byte-for-byte. Default --out is -./keiko-support-.jsonl (colons replaced with '-'); default --max-bytes is 50MB — -the oldest log files are dropped first when the cap would be exceeded, and always named in the -manifest's truncatedLogFiles. +evidence-index count, exactly which log files were copied, and a redacted schema/integrity +fingerprint for each of the ui, local-knowledge, and memory-vault stores found under --state-dir) +followed by every line of /logs/server*.log, copied byte-for-byte. A store that has +never been used from this state dir, or that cannot be opened (corrupt, or a vault key the +operator has not supplied), is named in the manifest's storesUnavailable instead of failing the +export. Default --out is ./keiko-support-.jsonl (colons replaced with '-'); default +--max-bytes is 50MB — the oldest log files are dropped first when the cap would be exceeded, and +always named in the manifest's truncatedLogFiles. analyze reads FILE (a support bundle or a raw server.log — auto-detected), groups its lines by correlationId, and prints one reconstructed timeline per id. Each process lifetime is ordered by @@ -304,6 +310,9 @@ async function runSupportExport( return reportAuditFailure(error, io); } + // Deferred until after the audit's own fail-closed check: opening three real stores is real + // I/O, wasted if the export is about to be refused anyway. + const storeFingerprintCollection = await server.collectStoreFingerprints({ stateDir, env }); const generatedAtDate = now(); const manifest = buildSupportBundleManifest({ schemaVersion: server.SERVER_LOG_SCHEMA_VERSION, @@ -319,6 +328,8 @@ async function runSupportExport( skippedLogFiles: logContent.skippedLogFiles, auditSummary, evidenceIndexCount, + storeFingerprints: storeFingerprintCollection.fingerprints, + storesUnavailable: storeFingerprintCollection.unavailable, }); const lines = serializeBundleLines(manifest, logContent.contentLines); diff --git a/packages/keiko-contracts/src/index.ts b/packages/keiko-contracts/src/index.ts index 8d39e0bf33..760660cc36 100644 --- a/packages/keiko-contracts/src/index.ts +++ b/packages/keiko-contracts/src/index.ts @@ -4768,3 +4768,11 @@ export { PROMPT_CANDIDATE_RANKING_EXPECTED_ORDER, PROMPT_CANDIDATE_RANKING_FIXTURE, } from "./prompt-enhancer-ranking-fixture.js"; + +// ─── Store fingerprint (Epic #3233 §6.2, Wave 4a) ──────────────────────────────── +// A redacted, point-in-time snapshot of one persisted store's schema/integrity state, embedded +// in the support bundle manifest's `storeFingerprints` array. `isStoreFingerprint` is the +// fail-closed guard the manifest assembler uses to refuse a malformed value instead of embedding +// it. +export type { StoreFingerprint } from "./store-fingerprint.js"; +export { isStoreFingerprint } from "./store-fingerprint.js"; diff --git a/packages/keiko-contracts/src/store-fingerprint.test.ts b/packages/keiko-contracts/src/store-fingerprint.test.ts new file mode 100644 index 0000000000..dd9f55c472 --- /dev/null +++ b/packages/keiko-contracts/src/store-fingerprint.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; + +import { isStoreFingerprint, type StoreFingerprint } from "./store-fingerprint.js"; + +function validFingerprint(): StoreFingerprint { + return { + store: "local-knowledge", + schemaVersion: 7, + migrationsApplied: ["0001-initial", "0002-add-index"], + tableRowCounts: { documents: 12, chunks: 480 }, + quickCheckOk: true, + encryptionMode: "encrypted", + keySource: "keychain", + }; +} + +describe("isStoreFingerprint", () => { + it("accepts a fully-populated, valid fingerprint", () => { + expect(isStoreFingerprint(validFingerprint())).toBe(true); + }); + + it("accepts a valid fingerprint with keySource omitted (plaintext store)", () => { + // exactOptionalPropertyTypes forbids `keySource: undefined`; destructure-to-exclude instead. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { keySource: _keySource, ...rest } = validFingerprint(); + expect(isStoreFingerprint({ ...rest, encryptionMode: "plaintext" })).toBe(true); + }); + + it("accepts every closed-vocabulary store, encryptionMode, and keySource value", () => { + for (const store of ["ui", "local-knowledge", "memory-vault"] as const) { + expect(isStoreFingerprint({ ...validFingerprint(), store })).toBe(true); + } + for (const encryptionMode of ["plaintext", "encrypted", "migrating"] as const) { + expect(isStoreFingerprint({ ...validFingerprint(), encryptionMode })).toBe(true); + } + for (const keySource of ["env", "keychain", "keyfile"] as const) { + expect(isStoreFingerprint({ ...validFingerprint(), keySource })).toBe(true); + } + }); + + it("rejects a non-object, null, and an array", () => { + expect(isStoreFingerprint(undefined)).toBe(false); + expect(isStoreFingerprint(null)).toBe(false); + expect(isStoreFingerprint("not an object")).toBe(false); + expect(isStoreFingerprint([validFingerprint()])).toBe(false); + }); + + it("rejects an unknown store, encryptionMode, or keySource value", () => { + expect(isStoreFingerprint({ ...validFingerprint(), store: "vector-db" })).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), encryptionMode: "unknown" })).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), keySource: "hsm" })).toBe(false); + }); + + it("rejects a non-integer, negative, or non-finite schemaVersion", () => { + expect(isStoreFingerprint({ ...validFingerprint(), schemaVersion: 1.5 })).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), schemaVersion: -1 })).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), schemaVersion: Number.NaN })).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), schemaVersion: Infinity })).toBe(false); + }); + + it("rejects a migrationsApplied entry that is not a bounded identifier", () => { + expect( + isStoreFingerprint({ + ...validFingerprint(), + migrationsApplied: ["a whole sentence with spaces"], + }), + ).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), migrationsApplied: ["0001", 42] })).toBe( + false, + ); + expect( + isStoreFingerprint({ + ...validFingerprint(), + migrationsApplied: Array.from({ length: 513 }, (_unused, index) => `m${String(index)}`), + }), + ).toBe(false); + }); + + it("rejects a tableRowCounts entry with a bad table name or a bad count", () => { + expect( + isStoreFingerprint({ + ...validFingerprint(), + tableRowCounts: { "not a valid table name": 1 }, + }), + ).toBe(false); + expect(isStoreFingerprint({ ...validFingerprint(), tableRowCounts: { documents: -1 } })).toBe( + false, + ); + expect(isStoreFingerprint({ ...validFingerprint(), tableRowCounts: { documents: 1.5 } })).toBe( + false, + ); + expect( + isStoreFingerprint({ ...validFingerprint(), tableRowCounts: { documents: Number.NaN } }), + ).toBe(false); + }); + + it("rejects tableRowCounts given as an array instead of a record", () => { + expect(isStoreFingerprint({ ...validFingerprint(), tableRowCounts: [1, 2, 3] })).toBe(false); + }); + + it("rejects a non-boolean quickCheckOk", () => { + expect(isStoreFingerprint({ ...validFingerprint(), quickCheckOk: 1 })).toBe(false); + }); + + it("rejects an unexpected field riding along on an otherwise-valid fingerprint", () => { + expect(isStoreFingerprint({ ...validFingerprint(), extra: "smuggled" })).toBe(false); + }); + + it("rejects an object missing a required field", () => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { store: _store, ...withoutStore } = validFingerprint(); + expect(isStoreFingerprint(withoutStore)).toBe(false); + }); +}); diff --git a/packages/keiko-contracts/src/store-fingerprint.ts b/packages/keiko-contracts/src/store-fingerprint.ts new file mode 100644 index 0000000000..2dc301e58e --- /dev/null +++ b/packages/keiko-contracts/src/store-fingerprint.ts @@ -0,0 +1,149 @@ +// StoreFingerprint — a redacted, point-in-time snapshot of one persisted store's schema and +// integrity state (Wave 4a, epic #3233 §6.2). `keiko bundle export`'s manifest assembly (in +// `keiko-server`, which already depends on all three store packages) calls each store's own +// `computeStoreFingerprint(db)` and embeds the resulting array in the support bundle manifest's +// `storeFingerprints` field. +// +// Every field is a count, a closed-vocabulary label, or a bounded identifier drawn from this +// repository's own fixed table/migration lists — never a row, a path, a key, a secret, or free +// text (ADR-0128 D6 redaction vocabulary). `isStoreFingerprint` lets the manifest assembler +// refuse a malformed value before it is embedded, mirroring the fail-closed guard style already +// established in `atlassian-connectors-validation.ts`. +// +// Leaf-package rule (ADR-0019 direction 1): no `@oscharko-dev/keiko-*` imports, pure functions +// only, zero logic beyond the shape and its guard. + +/** + * A point-in-time, redacted snapshot of one persisted store's schema/integrity state, computed + * by each store package's own `computeStoreFingerprint(db)` and assembled into the support + * bundle manifest's `storeFingerprints` array (Wave 4a). + */ +export interface StoreFingerprint { + readonly store: "ui" | "local-knowledge" | "memory-vault"; + /** `PRAGMA user_version`. */ + readonly schemaVersion: number; + /** Migration-group names, already tracked by the store's own migration runner. */ + readonly migrationsApplied: readonly string[]; + /** + * `COUNT(*)` over a FIXED, closed table-name list the owning package already declares — never + * a dynamic table walk. + */ + readonly tableRowCounts: Readonly>; + /** `PRAGMA quick_check` summary — pass/fail only, never the raw check output. */ + readonly quickCheckOk: boolean; + readonly encryptionMode: "plaintext" | "encrypted" | "migrating"; + /** The already-computed-then-discarded key-resolution tier, when the store is encrypted. */ + readonly keySource?: "env" | "keychain" | "keyfile" | undefined; +} + +// ─── Closed-vocabulary sets (S7776: a Set, never `.includes()` on a constant array) ──────────── +const STORE_FINGERPRINT_STORE_SET = new Set(["ui", "local-knowledge", "memory-vault"]); +const STORE_FINGERPRINT_ENCRYPTION_MODE_SET = new Set([ + "plaintext", + "encrypted", + "migrating", +]); +const STORE_FINGERPRINT_KEY_SOURCE_SET = new Set(["env", "keychain", "keyfile"]); +const STORE_FINGERPRINT_KEY_SET = new Set([ + "store", + "schemaVersion", + "migrationsApplied", + "tableRowCounts", + "quickCheckOk", + "encryptionMode", + "keySource", +]); + +// A migration-group name or SQL table name is identifier-shaped and named by our own migration +// runners and schema declarations, never a caller — but the manifest assembler still refuses +// anything sentence-shaped before it is embedded, mirroring `ERROR_KIND_PATTERN`'s guard against +// an echoed payload riding an envelope field (ADR-0173 D11). Migration-group names commonly lead +// with a numeric sequence prefix (e.g. "0001-initial"), so — unlike `ERROR_KIND_PATTERN` — the +// leading character allows a digit too. +const BOUNDED_IDENTIFIER_PATTERN = /^\w[\w.-]{0,127}$/u; + +// Defensive ceilings: both lists are FIXED and package-owned (never attacker- or user-grown), but +// a validator that embeds the result in a customer-facing bundle still bounds array/object size +// rather than trusting the producer unconditionally. +const STORE_FINGERPRINT_MAX_MIGRATIONS = 512; +const STORE_FINGERPRINT_MAX_TABLES = 128; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isFiniteNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function isBoundedIdentifier(value: unknown): value is string { + return typeof value === "string" && BOUNDED_IDENTIFIER_PATTERN.test(value); +} + +function isStoreFingerprintStore(value: unknown): value is StoreFingerprint["store"] { + return typeof value === "string" && STORE_FINGERPRINT_STORE_SET.has(value); +} + +function isStoreFingerprintEncryptionMode( + value: unknown, +): value is StoreFingerprint["encryptionMode"] { + return typeof value === "string" && STORE_FINGERPRINT_ENCRYPTION_MODE_SET.has(value); +} + +function isOptionalStoreFingerprintKeySource( + value: unknown, +): value is StoreFingerprint["keySource"] { + if (value === undefined) return true; + return typeof value === "string" && STORE_FINGERPRINT_KEY_SOURCE_SET.has(value); +} + +function isMigrationsApplied(value: unknown): value is readonly string[] { + return ( + Array.isArray(value) && + value.length <= STORE_FINGERPRINT_MAX_MIGRATIONS && + value.every((entry) => isBoundedIdentifier(entry)) + ); +} + +function isTableRowCounts(value: unknown): value is Readonly> { + if (!isRecord(value)) return false; + const entries = Object.entries(value); + return ( + entries.length <= STORE_FINGERPRINT_MAX_TABLES && + entries.every( + ([table, count]) => isBoundedIdentifier(table) && isFiniteNonNegativeInteger(count), + ) + ); +} + +function hasKnownStoreFingerprintKeys(value: Record): boolean { + return Object.keys(value).every((key) => STORE_FINGERPRINT_KEY_SET.has(key)); +} + +function isStoreFingerprintCore(value: Record): boolean { + return ( + isStoreFingerprintStore(value.store) && + isFiniteNonNegativeInteger(value.schemaVersion) && + isMigrationsApplied(value.migrationsApplied) && + isTableRowCounts(value.tableRowCounts) + ); +} + +function isStoreFingerprintEncryptionShape(value: Record): boolean { + return ( + typeof value.quickCheckOk === "boolean" && + isStoreFingerprintEncryptionMode(value.encryptionMode) && + isOptionalStoreFingerprintKeySource(value.keySource) + ); +} + +/** + * Fail-closed structural guard for {@link StoreFingerprint}: every union is checked against its + * closed vocabulary, every count is a finite non-negative integer, every table/migration name is + * a bounded identifier, and no unexpected field rides through. The manifest assembler uses this + * to refuse a malformed fingerprint rather than embed it. + */ +export function isStoreFingerprint(value: unknown): value is StoreFingerprint { + if (!isRecord(value) || !hasKnownStoreFingerprintKeys(value)) return false; + return isStoreFingerprintCore(value) && isStoreFingerprintEncryptionShape(value); +} diff --git a/packages/keiko-local-knowledge/src/index.ts b/packages/keiko-local-knowledge/src/index.ts index 094a5a9446..79c47b1c2e 100644 --- a/packages/keiko-local-knowledge/src/index.ts +++ b/packages/keiko-local-knowledge/src/index.ts @@ -27,7 +27,9 @@ export { } from "./knowledge-log.js"; export { resolveKnowledgeStorePath, type ResolveKnowledgeStorePathOptions } from "./store-paths.js"; export { + computeStoreFingerprint, openKnowledgeStore, + openKnowledgeStoreReadOnly, type KnowledgeStoreKeyProvider, type KnowledgeStoreKeyProviderContext, type KnowledgeStoreProtectionOptions, diff --git a/packages/keiko-local-knowledge/src/indexing/orchestrator.test.ts b/packages/keiko-local-knowledge/src/indexing/orchestrator.test.ts index e9ab1addfc..06dd5d2966 100644 --- a/packages/keiko-local-knowledge/src/indexing/orchestrator.test.ts +++ b/packages/keiko-local-knowledge/src/indexing/orchestrator.test.ts @@ -2869,6 +2869,38 @@ describe("runIndexingJob — activity log", () => { } }); + it("states the resolved chunker profile on indexing.job.started, reflecting an operator override", async () => { + // `IndexingOptions.chunkingOptions` is operator-overridable but, before this test, never + // reached the run's spine line — an operator-supplied budget was invisible to anyone + // reading the activity log. Asserting on a NON-default override (rather than the defaults + // every other test in this suite exercises implicitly) proves the values are read from the + // resolved run configuration, not hard-coded defaults that would pass even if the wiring + // were dropped. + const fixture = buildFixture({ + "alpha.txt": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(12), + }); + const log = recordingSink(); + try { + await drain( + runIndexingJob( + buildOptions(fixture, { + logSink: log.sink, + idSource: () => "job-chunker-profile", + chunkingOptions: { maxTokens: 96, minTokens: 8, overlapTokens: 12 }, + }), + ), + ); + expect(extraOf(requireLine(log, "indexing.job.started"))).toMatchObject({ + minChunkTokens: 8, + maxChunkTokens: 96, + overlapTokens: 12, + tokenizerKind: "estimator", + }); + } finally { + fixture.cleanup(); + } + }); + it("correlates every line to the job and identifies capsule and document by digest only", async () => { const fixture = buildFixture({ "alpha.txt": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ".repeat(12), diff --git a/packages/keiko-local-knowledge/src/indexing/orchestrator.ts b/packages/keiko-local-knowledge/src/indexing/orchestrator.ts index 98ef591286..3c2df6f8a3 100644 --- a/packages/keiko-local-knowledge/src/indexing/orchestrator.ts +++ b/packages/keiko-local-knowledge/src/indexing/orchestrator.ts @@ -350,6 +350,21 @@ function endpointHostExtra(state: RunState): Readonly> { return host === undefined ? {} : { endpointHost: host }; } +// The run's chunker profile, resolved through the exact same two calls the pipeline itself +// takes to decide chunk boundaries (`chunkingOptionsForState` + `resolveChunkingOptions`) — no +// new computation, just surfacing values the run already derives. `IndexingOptions.chunkingOptions` +// is operator-overridable, so without this an operator-supplied budget was invisible on the one +// line that states the run's shape. +function chunkerConfigExtra(state: RunState): Readonly> { + const resolved = resolveChunkingOptions(chunkingOptionsForState(state)); + return { + minChunkTokens: resolved.minTokens, + maxChunkTokens: resolved.maxTokens, + overlapTokens: resolved.overlapTokens, + tokenizerKind: resolved.tokenizer.kind, + }; +} + function documentLogContext(state: RunState, documentId: DocumentId): IndexingLogContext { return { ...state.logContext, documentIdDigest: logDigest(String(documentId)) }; } @@ -2951,6 +2966,7 @@ function emitJobStarted(state: RunState, sources: readonly KnowledgeSource[]): I force: state.options.force === true, resume: state.options.resume === true, contextualRetrieval: state.options.contextualRetrieval?.enabled === true, + ...chunkerConfigExtra(state), ...endpointHostExtra(state), }, }); diff --git a/packages/keiko-local-knowledge/src/repository-pod.test.ts b/packages/keiko-local-knowledge/src/repository-pod.test.ts index 4dd2fe31e8..4b3b67e355 100644 --- a/packages/keiko-local-knowledge/src/repository-pod.test.ts +++ b/packages/keiko-local-knowledge/src/repository-pod.test.ts @@ -34,6 +34,7 @@ import { resolveRepositoryChunkLineRange, } from "./indexing/repository-chunk-lines.js"; import { readRepositoryFileFingerprints } from "./indexing/repository-fingerprints.js"; +import type { KnowledgeLogEvent, KnowledgeLogSink } from "./knowledge-log.js"; import { isCodeSymbolDefinitionLine } from "./parsers/code-parser.js"; import { createDefaultParserRegistry } from "./parsers/index.js"; import { @@ -512,6 +513,77 @@ function stubChatGateway(): { }; } +describe("repository pod fingerprint-diff activity log", () => { + function recordingSink(): { + readonly sink: KnowledgeLogSink; + readonly events: KnowledgeLogEvent[]; + } { + const events: KnowledgeLogEvent[] = []; + return { + sink: { + write: (event): void => { + events.push(event); + }, + }, + events, + }; + } + + it("logs repository.fingerprint-diff.completed with the run's delta counts, correlated to the run id", async () => { + createShell(); + const adapter = countingAdapter(); + const { sink, events } = recordingSink(); + + await refreshRepositoryPod(indexingDeps(adapter, { logSink: sink }), { + runId: "fp-diff-initial", + }); + const initialLine = events.find( + (event) => event.op === "repository.fingerprint-diff.completed", + ); + expect(initialLine).toBeDefined(); + expect(initialLine?.category).toBe("indexing"); + expect(initialLine?.correlationId).toBe("fp-diff-initial"); + expect(initialLine?.extra).toEqual({ + added: 4, + changed: 0, + removed: 0, + moved: 0, + unchanged: 0, + }); + + events.length = 0; + writeFileSync( + join(repositoryRoot, "src", "service.py"), + ["class Service:", " pass", "", "def load():", ' return "changed"', ""].join("\n"), + "utf8", + ); + await refreshRepositoryPod(indexingDeps(adapter, { logSink: sink }), { + runId: "fp-diff-refresh", + }); + const refreshLine = events.find( + (event) => event.op === "repository.fingerprint-diff.completed", + ); + expect(refreshLine).toBeDefined(); + expect(refreshLine?.correlationId).toBe("fp-diff-refresh"); + expect(refreshLine?.extra).toEqual({ + added: 0, + changed: 1, + removed: 0, + moved: 0, + unchanged: 3, + }); + }); + + it("writes nothing when no logSink is supplied", async () => { + createShell(); + const adapter = countingAdapter(); + // No logSink override — must not throw, and the deps default carries none. + await expect( + refreshRepositoryPod(indexingDeps(adapter), { runId: "fp-diff-no-sink" }), + ).resolves.toBeDefined(); + }); +}); + describe("repository pod capsule setting parity (M2.12)", () => { it("honours contextualRetrieval when threaded through refreshRepositoryPod", async (): Promise => { createShell(); diff --git a/packages/keiko-local-knowledge/src/repository-pod.ts b/packages/keiko-local-knowledge/src/repository-pod.ts index 7ec134ceff..87401306ab 100644 --- a/packages/keiko-local-knowledge/src/repository-pod.ts +++ b/packages/keiko-local-knowledge/src/repository-pod.ts @@ -31,7 +31,7 @@ import { type DiscoveryOptions, } from "./discovery/types.js"; import { KnowledgeStoreError } from "./errors.js"; -import { diffFingerprintSets } from "./fingerprint-diff.js"; +import { diffFingerprintSets, type FingerprintSetDelta } from "./fingerprint-diff.js"; import { readRepositoryFileFingerprints, replaceRepositoryFileFingerprints, @@ -41,12 +41,12 @@ import { } from "./indexing/repository-fingerprints.js"; import type { ContextualRetrievalOptions } from "./indexing/contextual-retrieval.js"; import { runIndexingJob, type IndexingEvent, type IndexingResult } from "./indexing/index.js"; +import { emitKnowledgeLogEvent, type KnowledgeLogSink } from "./knowledge-log.js"; import { buildKnowledgePodSummary } from "./knowledge-pods.js"; import type { ParserRegistry } from "./parsers/index.js"; import type { AuditEventSink } from "./privacy/index.js"; import { addSourceToCapsule, listCapsuleSources } from "./source-lifecycle.js"; import type { KnowledgeStore } from "./store.js"; -import type { KnowledgeLogSink } from "./knowledge-log.js"; export interface RepositoryPodDeps { readonly store: KnowledgeStore; @@ -327,7 +327,32 @@ function pruneRemovedDocuments( } } +// Job-scoped: correlated to the refresh run, not the document, since a fingerprint diff is a +// whole-source-scan fact rather than a per-document one. Category "indexing" matches the op +// prefix and every other repository-pod-adjacent line this package already writes from +// `orchestrator.ts`'s `logIndexing`. +function logFingerprintDiffCompleted( + deps: RepositoryPodDeps, + runId: string, + delta: FingerprintSetDelta, +): void { + emitKnowledgeLogEvent(deps.logSink, { + category: "indexing", + op: "repository.fingerprint-diff.completed", + correlationId: runId, + extra: { + added: delta.added, + changed: delta.changed, + removed: delta.removed, + moved: delta.moved, + unchanged: delta.unchanged, + }, + }); +} + function runCounts( + deps: RepositoryPodDeps, + runId: string, prior: ReadonlyMap, next: readonly RepositoryFileFingerprint[], result: IndexingResult, @@ -337,6 +362,7 @@ function runCounts( const delta = diffFingerprintSets(fingerprintMap([...prior.values()]), fingerprintMap(next), { detectMoves: false, }); + logFingerprintDiffCompleted(deps, runId, delta); return { addedFiles: delta.added, changedFiles: delta.changed, @@ -444,6 +470,8 @@ export async function refreshRepositoryPod( } const persisted = applied ? next : [...prior.values()]; const counts = runCounts( + deps, + runId, prior, scan.fingerprints, drained.result, diff --git a/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.test.ts b/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.test.ts index 07149d444b..313b9a827e 100644 --- a/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.test.ts +++ b/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.test.ts @@ -22,6 +22,7 @@ import { describe, expect, it } from "vitest"; import { DEFAULT_EMBEDDING, freshStore, sampleCapsuleInput } from "../_support.js"; import { createCapsule } from "../capsule-lifecycle.js"; +import type { KnowledgeLogEvent, KnowledgeLogSink } from "../knowledge-log.js"; import type { KnowledgeStore } from "../store.js"; // Deep-import the port implementation so vitest's v8 coverage attributes execution to the @@ -203,6 +204,52 @@ describe("createLocalKnowledgeStoreVectorIndexPort", () => { } }); + it("logs search.index-invalidated-for-capsule on identity mismatch, capsule id digested only", async () => { + const fixture = freshStore(); + try { + createTestCapsule(fixture.store, "cap-port-a"); + const events: KnowledgeLogEvent[] = []; + const logSink: KnowledgeLogSink = { + write: (event): void => { + events.push(event); + }, + }; + const port = createLocalKnowledgeStoreVectorIndexPort({ + namespace: "knowledge", + store: fixture.store, + logSink, + }); + + const roguesIdentity: EmbeddingModelIdentity = { + ...DEFAULT_EMBEDDING, + embeddingSpaceFingerprint: "keiko-embedding-space-fingerprint-v2:rogue", + }; + const result = await port.search(baseQuery({ identity: roguesIdentity })); + expect(result.ok).toBe(false); + + const lines = events.filter((event) => event.op === "search.index-invalidated-for-capsule"); + expect(lines).toHaveLength(1); + expect(lines[0]).toMatchObject({ + level: "warn", + category: "search", + extra: { namespace: "knowledge" }, + }); + // The raw capsule id never reaches the log — only a digest of it. + const capsuleIdDigest = lines[0]?.extra?.capsuleIdDigest; + expect(capsuleIdDigest).toMatch(/^[0-9a-f]{16}$/u); + expect(JSON.stringify(lines[0])).not.toContain("cap-port-a"); + + // A successful search never emits this line. + events.length = 0; + await port.search(baseQuery()); + expect( + events.filter((event) => event.op === "search.index-invalidated-for-capsule"), + ).toHaveLength(0); + } finally { + fixture.cleanup(); + } + }); + it("fails closed with invalid-partition-key when the encoded partition cannot be decoded", async () => { // `parsePartitionKey` uses `decodeURIComponent` and returns `undefined` on any decode // failure — a malformed key MUST fail closed instead of silently narrowing to a partial diff --git a/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.ts b/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.ts index 07e20428fd..8152203009 100644 --- a/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.ts +++ b/packages/keiko-local-knowledge/src/retrieval/local-vector-index-port.ts @@ -12,6 +12,8 @@ // require touching `tryVectorIndexForCapsule` or `searchVectorIndex`. Every refusal is // content-free `ok: false` — never an exception, never a body, never a path. +import { createHash } from "node:crypto"; + import { embeddingIdentityKey, isValidVectorIndexQuery, @@ -25,6 +27,7 @@ import { } from "@oscharko-dev/keiko-contracts"; import { getCapsule } from "../capsule-lifecycle.js"; +import { emitKnowledgeLogEvent, type KnowledgeLogSink } from "../knowledge-log.js"; import type { KnowledgeStore } from "../store.js"; import type { RetrievalVectorIndexDiagnostics } from "./types.js"; @@ -138,6 +141,34 @@ function portIdentityMismatch(): VectorIndexResult { }; } +const CAPSULE_ID_DIGEST_LENGTH = 16; + +// A caller-supplied capsule id is never logged raw — only its digest (matches the convention +// `orchestrator.ts`'s `logDigest` establishes for the same reason: the id is caller-chosen and +// must not become a durable, searchable identifier in the log). +function capsuleIdDigest(capsuleId: KnowledgeCapsuleId): string { + const digest = createHash("sha256").update(String(capsuleId)).digest("hex"); + return digest.slice(0, CAPSULE_ID_DIGEST_LENGTH); +} + +// Fires exactly where the identity-mismatch VALUE is already computed (`portIdentityMismatch`'s +// caller, just below) — never a second, independent identity comparison. This is the same fact +// that later reconstructs into the adapter shim's `sawIdentityIncompatible: true` and, further +// up, a capsule's `vectorCompatible: false` / `staleReasons` health projection: the capsule's +// persisted index no longer matches the identity a caller is querying with and needs a reindex. +function logIndexInvalidatedForCapsule( + logSink: KnowledgeLogSink | undefined, + namespace: LocalKnowledgeStoreNamespace, + capsuleId: KnowledgeCapsuleId, +): void { + emitKnowledgeLogEvent(logSink, { + level: "warn", + category: "search", + op: "search.index-invalidated-for-capsule", + extra: { namespace, capsuleIdDigest: capsuleIdDigest(capsuleId) }, + }); +} + function toPortDiagnostics(source: RetrievalVectorIndexDiagnostics): VectorIndexDiagnostics { return { ...diagnostic(source.provider, source.status, source.reason), @@ -175,6 +206,10 @@ export interface CreateLocalKnowledgeStoreVectorIndexPortOptions { // set here is cleared before dispatch: the LK adapter shim wraps THIS port, so keeping an // adapter would re-enter the port through itself. readonly vectorIndexOptions?: VectorIndexOptions | undefined; + // Content-free activity log (ADR-0019 seam, `knowledge-log.ts`). Absent → nothing is written. + // Covers the port's OWN identity-mismatch refusal, which happens before `vectorIndexOptions` + // is ever handed to `searchVectorIndex` (Wave 4a, epic #3233 §8). + readonly logSink?: KnowledgeLogSink | undefined; } // Build the `VectorIndexPort` implementation over an owned Local Knowledge store. @@ -197,7 +232,7 @@ export interface CreateLocalKnowledgeStoreVectorIndexPortOptions { export function createLocalKnowledgeStoreVectorIndexPort( options: CreateLocalKnowledgeStoreVectorIndexPortOptions, ): VectorIndexPort { - const { namespace, store } = options; + const { namespace, store, logSink } = options; return { async search(query: VectorIndexQuery): Promise { if (!isValidVectorIndexQuery(query)) return portInvalidQuery(); @@ -210,6 +245,7 @@ export function createLocalKnowledgeStoreVectorIndexPort( embeddingIdentityKey(query.identity) !== embeddingIdentityKey(capsule.embeddingModelIdentity) ) { + logIndexInvalidatedForCapsule(logSink, namespace, capsule.id); return portIdentityMismatch(); } const request: VectorIndexSearchRequest = { diff --git a/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts b/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts index ce137c13d6..32b8a551b4 100644 --- a/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts +++ b/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts @@ -5,6 +5,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { EmbeddingModelIdentity } from "@oscharko-dev/keiko-contracts"; +import type { KnowledgeLogEvent, KnowledgeLogSink } from "../knowledge-log.js"; + import { __resetTargetRuntimeCacheForTests, clearUsearchAnnCacheForTests, @@ -402,6 +404,52 @@ describe("USearch ANN index", () => { expect(readsAfterWarm).toBe(readsAfterCold); }); + it("logs search.native-runtime-resolved on the cold path only, content-free", async () => { + const corpus = clusteredCorpus(64, IDENTITY); + const queryVector = corpus.entries[0]?.vector; + if (queryVector === undefined) throw new Error("test corpus must contain a query vector"); + const binary = runtimePath(); + const events: KnowledgeLogEvent[] = []; + const logSink: KnowledgeLogSink = { + write: (event): void => { + events.push(event); + }, + }; + const request = { + partition: partition(corpus.entries, "native-runtime-resolved-logging"), + queryVector, + candidateLimit: 5, + exactScanThreshold: 0, + binaryPath: binary, + logSink, + }; + __resetTargetRuntimeCacheForTests(); + + const cold = await searchUsearchAnnIndex(request); + expect(cold.ok).toBe(true); + const coldLines = events.filter((event) => event.op === "search.native-runtime-resolved"); + // resolvedIndex() and buildSearchIndex() both call targetRuntime() for the same request; + // the second call is already a warm hit against the cache the first call just populated, + // so exactly one line — not two — is written for one logical search call. + expect(coldLines).toHaveLength(1); + expect(coldLines[0]).toMatchObject({ + level: "info", + category: "search", + extra: { + targetKey: usearchRuntimeTargetKey(process.platform, process.arch), + resolved: true, + version: USEARCH_RUNTIME_MANIFEST.version, + }, + }); + // Content-free: never the resolved filesystem path or the raw SHA-256 digest. + expect(JSON.stringify(coldLines[0])).not.toContain(binary); + + events.length = 0; + const warm = await searchUsearchAnnIndex(request); + expect(warm.ok).toBe(true); + expect(events.filter((event) => event.op === "search.native-runtime-resolved")).toHaveLength(0); + }); + it("serializes concurrent callers through queryQueue without cross-contaminating results (KEIKO-0360)", async () => { // KEIKO-0360: coverage pin for the ADR-0164 D2 single-Worker queryQueue serialization // at annSearch()/annSearchExclusive(). Twelve concurrent Promise.all searches against diff --git a/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.ts b/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.ts index 9326ab2596..5189235b3d 100644 --- a/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.ts +++ b/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.ts @@ -9,6 +9,8 @@ import { type EmbeddingVectorMetric, } from "@oscharko-dev/keiko-contracts"; +import { emitKnowledgeLogEvent, type KnowledgeLogSink } from "../knowledge-log.js"; + import { USEARCH_RUNTIME_MANIFEST, type UsearchRuntimeApproval, @@ -45,6 +47,10 @@ export interface UsearchAnnSearchRequest { readonly binaryPath?: string; readonly exactScanThreshold?: number; readonly maxIndexBytes?: number; + // Content-free activity log (ADR-0019 seam, `knowledge-log.ts`). Absent → nothing is written. + // Threaded down to `targetRuntime()` so a fresh native-addon resolution is visible in + // `server.log` beside the search that triggered it (Wave 4a, epic #3233 §8). + readonly logSink?: KnowledgeLogSink; } export interface UsearchAnnCandidate { @@ -390,7 +396,31 @@ function cacheEntryStillMatches( ); } -function targetRuntime(binaryPath: string | undefined): TargetRuntimeResult { +// Content-free: `targetKey` is a closed platform:arch label this package owns +// (`usearchRuntimeTargetKey`), never the resolved filesystem path or the raw SHA-256 digest. +// Emitted only on the COLD path below — a warm cache hit never re-logs, matching the reasoning +// that already justifies not re-hashing on every request (KEIKO-0409). +function logNativeRuntimeResolved( + logSink: KnowledgeLogSink | undefined, + targetKey: string, + result: TargetRuntimeResult, +): void { + emitKnowledgeLogEvent(logSink, { + level: typeof result === "string" ? "warn" : "info", + category: "search", + op: "search.native-runtime-resolved", + extra: { + targetKey, + resolved: typeof result !== "string", + ...(typeof result === "string" ? { reason: result } : { version: result.expectedVersion }), + }, + }); +} + +function targetRuntime( + binaryPath: string | undefined, + logSink?: KnowledgeLogSink, +): TargetRuntimeResult { const targetKey = usearchRuntimeTargetKey(process.platform, process.arch); if (targetKey === undefined) return "unavailable"; const path = resolvedRuntimePath(binaryPath, targetKey); @@ -412,6 +442,7 @@ function targetRuntime(binaryPath: string | undefined): TargetRuntimeResult { return cached.result; } const result = verifyRuntimeAt(path); + logNativeRuntimeResolved(logSink, targetKey, result); // PR-review follow-up: only memoize SUCCESSFUL verifications. A transient failure // (EMFILE, EIO, temporarily-tightened permissions) that later heals must NOT be cached // against the same tuple — otherwise a recovered runtime is permanently invisible to every @@ -678,7 +709,7 @@ async function buildSearchIndex( byteSize: estimate, }); } - const runtime = targetRuntime(request.binaryPath); + const runtime = targetRuntime(request.binaryPath, request.logSink); if (runtime === "unavailable") return { ok: false, reason: "runtime-unavailable" }; if (runtime === "invalid") return { ok: false, reason: "runtime-integrity-failed" }; const worker = await startWorker(request.partition, vectors, runtime, HNSW_MAX_RESULTS); @@ -727,7 +758,7 @@ async function resolvedIndex( // cost that motivated the finding is neutralised by memoization inside targetRuntime() // (keyed on the resolved (path, mtimeMs, size)), so a warm hit no longer re-reads the // multi-MB addon on the Node.js event loop. - const runtime = targetRuntime(request.binaryPath); + const runtime = targetRuntime(request.binaryPath, request.logSink); if (runtime === "unavailable") return { ok: false, reason: "runtime-unavailable" }; if (runtime === "invalid") return { ok: false, reason: "runtime-integrity-failed" }; } diff --git a/packages/keiko-local-knowledge/src/retrieval/vector-index.ts b/packages/keiko-local-knowledge/src/retrieval/vector-index.ts index cb19844745..073eb445e1 100644 --- a/packages/keiko-local-knowledge/src/retrieval/vector-index.ts +++ b/packages/keiko-local-knowledge/src/retrieval/vector-index.ts @@ -13,6 +13,7 @@ import { writeVectorIndexState, type VectorIndexStateRecord, } from "../indexing/vector-index-state.js"; +import type { KnowledgeLogSink } from "../knowledge-log.js"; import type { KnowledgeStore } from "../store.js"; import type { RetrievalVectorIndexDiagnostics } from "./types.js"; @@ -70,6 +71,10 @@ export interface VectorIndexOptions { readonly now?: () => number; readonly newCorrelationId?: () => string; readonly onUnexpectedFailure?: (diagnostic: VectorIndexUnexpectedFailureDiagnostic) => void; + // Content-free activity log (ADR-0019 seam, `knowledge-log.ts`). Absent → nothing is written. + // Threaded down to the USearch ANN layer so a fresh native-runtime resolution is visible in + // `server.log` beside the search that triggered it (Wave 4a, epic #3233 §8). + readonly logSink?: KnowledgeLogSink; } export interface VectorIndexEnvironment { @@ -85,6 +90,7 @@ interface ResolvedVectorIndexOptions { readonly now: () => number; readonly newCorrelationId: () => string; readonly onUnexpectedFailure?: (diagnostic: VectorIndexUnexpectedFailureDiagnostic) => void; + readonly logSink?: KnowledgeLogSink; } interface VectorIndexStampRow { @@ -368,6 +374,7 @@ export function resolveVectorIndexOptions( ...(supplied.onUnexpectedFailure !== undefined ? { onUnexpectedFailure: supplied.onUnexpectedFailure } : {}), + ...(supplied.logSink !== undefined ? { logSink: supplied.logSink } : {}), }; } @@ -567,6 +574,7 @@ async function runBuiltInSearch( candidateLimit: request.candidateLimit, ...(options.usearchBinaryPath !== undefined ? { binaryPath: options.usearchBinaryPath } : {}), maxIndexBytes: options.maxIndexedVectorBytes, + ...(options.logSink !== undefined ? { logSink: options.logSink } : {}), }); } diff --git a/packages/keiko-local-knowledge/src/store-content-encryption.ts b/packages/keiko-local-knowledge/src/store-content-encryption.ts index ea35a81828..da7936f31f 100644 --- a/packages/keiko-local-knowledge/src/store-content-encryption.ts +++ b/packages/keiko-local-knowledge/src/store-content-encryption.ts @@ -22,9 +22,21 @@ import type { DatabaseSync } from "node:sqlite"; import { KnowledgeStoreError } from "./errors.js"; +import { + emitKnowledgeLogEvent, + startKnowledgeLogTimer, + type KnowledgeLogSink, +} from "./knowledge-log.js"; import { sectionPathHashFromJson } from "./section-path-hash.js"; import type { StoreContentCipher } from "./store-content-cipher.js"; +// Sentinel `fromScope` reported on a store that had never been encrypted before this migration — +// there is no prior `content_encryption_scope` value to report, and this reads clearly next to +// the real scope-version strings (`ENCRYPTION_SCOPE_VALUE` and its predecessors). +const UNENCRYPTED_SCOPE_LABEL = "plaintext"; +// Reported when an already-encrypted store predates the scope-marker key entirely (pre-v2). +const UNSCOPED_ENCRYPTED_SCOPE_LABEL = "unscoped"; + const ENCRYPTION_MARKER_KEY = "content_encryption"; const ENCRYPTION_MARKER_VALUE = "aes-256-gcm/v1"; const ENCRYPTION_PROBE_KEY = "content_encryption_probe"; @@ -277,7 +289,31 @@ function flushPlaintextResidue(db: DatabaseSync): void { db.exec("VACUUM"); } -function migrateToEncrypted(db: DatabaseSync, cipher: StoreContentCipher): void { +// Fires only after the migration function it is called from has ALREADY returned without +// throwing — never inside the transactional try/catch above it, and never for a branch of +// `applyStoreContentEncryption` that migrates nothing (a store already at the current scope). +// `durationMs` rides the envelope's own field, not `extra`, matching every other timed line this +// package writes (`startKnowledgeLogTimer`, ADR-0019 seam). +function logEncryptionMigrated( + logSink: KnowledgeLogSink | undefined, + fromScope: string, + toScope: string, + durationMs: number, +): void { + emitKnowledgeLogEvent(logSink, { + category: "diagnostic", + op: "store.encryption-migrated", + durationMs, + extra: { fromScope, toScope }, + }); +} + +function migrateToEncrypted( + db: DatabaseSync, + cipher: StoreContentCipher, + logSink?: KnowledgeLogSink, +): void { + const elapsed = startKnowledgeLogTimer(); // Phase 1 (transactional): seal every content row and write the sealed key-verification probe. The // completion MARKER is deliberately NOT written here — see phase 2. db.exec("BEGIN"); @@ -303,9 +339,16 @@ function migrateToEncrypted(db: DatabaseSync, cipher: StoreContentCipher): void flushPlaintextResidue(db); writeSchemaMeta(db, ENCRYPTION_MARKER_KEY, ENCRYPTION_MARKER_VALUE); writeSchemaMeta(db, ENCRYPTION_SCOPE_KEY, ENCRYPTION_SCOPE_VALUE); + logEncryptionMigrated(logSink, UNENCRYPTED_SCOPE_LABEL, ENCRYPTION_SCOPE_VALUE, elapsed()); } -function upgradeEncryptedScope(db: DatabaseSync, cipher: StoreContentCipher): void { +function upgradeEncryptedScope( + db: DatabaseSync, + cipher: StoreContentCipher, + fromScope: string, + logSink?: KnowledgeLogSink, +): void { + const elapsed = startKnowledgeLogTimer(); db.exec("BEGIN"); try { ensureSectionPathHashes(db, cipher); @@ -323,6 +366,7 @@ function upgradeEncryptedScope(db: DatabaseSync, cipher: StoreContentCipher): vo } flushPlaintextResidue(db); writeSchemaMeta(db, ENCRYPTION_SCOPE_KEY, ENCRYPTION_SCOPE_VALUE); + logEncryptionMigrated(logSink, fromScope, ENCRYPTION_SCOPE_VALUE, elapsed()); } function verifyProbe(db: DatabaseSync, cipher: StoreContentCipher): void { @@ -364,7 +408,11 @@ function assertSupportedEncryptionScope(scope: string | undefined): void { // Reconciles the store's on-disk encryption state with the resolved cipher. Called once from // openKnowledgeStore after migrations and before the handle is returned. -export function applyStoreContentEncryption(db: DatabaseSync, cipher: StoreContentCipher): void { +export function applyStoreContentEncryption( + db: DatabaseSync, + cipher: StoreContentCipher, + logSink?: KnowledgeLogSink, +): void { const marker = readSchemaMeta(db, ENCRYPTION_MARKER_KEY); const probe = readSchemaMeta(db, ENCRYPTION_PROBE_KEY); const scope = readSchemaMeta(db, ENCRYPTION_SCOPE_KEY); @@ -382,7 +430,7 @@ export function applyStoreContentEncryption(db: DatabaseSync, cipher: StoreConte verifyProbe(db, cipher); assertSupportedEncryptionScope(scope); if (scope !== ENCRYPTION_SCOPE_VALUE) { - upgradeEncryptedScope(db, cipher); + upgradeEncryptedScope(db, cipher, scope ?? UNSCOPED_ENCRYPTED_SCOPE_LABEL, logSink); } return; } @@ -394,14 +442,27 @@ export function applyStoreContentEncryption(db: DatabaseSync, cipher: StoreConte ); } verifyProbe(db, cipher); - migrateToEncrypted(db, cipher); + migrateToEncrypted(db, cipher, logSink); return; } if (!cipher.isEncrypted) { ensureSectionPathHashes(db, cipher); return; } - migrateToEncrypted(db, cipher); + migrateToEncrypted(db, cipher, logSink); +} + +export type StoreContentEncryptionMode = "plaintext" | "encrypted" | "migrating"; + +// Read-only snapshot of the store's on-disk encryption state, independent of any resolved cipher +// and never throwing — used by `computeStoreFingerprint` (store.ts) to report `encryptionMode` +// in the support-bundle manifest (Wave 4a, epic #3233 §6.2) without re-deriving the marker/probe +// schema_meta keys a second time. Mirrors the case matrix documented at the top of this file: a +// malformed marker VALUE is still reported as "encrypted" here — validating it is +// `applyStoreContentEncryption`'s job, not a read-only reporter's. +export function readStoreEncryptionMode(db: DatabaseSync): StoreContentEncryptionMode { + if (readSchemaMeta(db, ENCRYPTION_MARKER_KEY) !== undefined) return "encrypted"; + return readSchemaMeta(db, ENCRYPTION_PROBE_KEY) !== undefined ? "migrating" : "plaintext"; } export const STORE_CONTENT_ENCRYPTION_TEST_CONSTANTS = { diff --git a/packages/keiko-local-knowledge/src/store.test.ts b/packages/keiko-local-knowledge/src/store.test.ts index 043ebc057c..4c3e255139 100644 --- a/packages/keiko-local-knowledge/src/store.test.ts +++ b/packages/keiko-local-knowledge/src/store.test.ts @@ -23,7 +23,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { KnowledgeStoreError } from "./errors.js"; import type { KnowledgeLogEvent, KnowledgeLogSink } from "./knowledge-log.js"; -import { LK_STORE_BUSY_TIMEOUT_MS, openKnowledgeStore } from "./store.js"; +import { + computeStoreFingerprint, + LK_STORE_BUSY_TIMEOUT_MS, + openKnowledgeStore, + type KnowledgeStoreKeyProvider, +} from "./store.js"; +import { STORE_CONTENT_ENCRYPTION_TEST_CONSTANTS } from "./store-content-encryption.js"; interface CountRow { readonly n: number; @@ -781,4 +787,159 @@ describe("openKnowledgeStore — activity log", () => { expect(rejection?.level).toBe("error"); expect(rejection?.extra).toEqual({ protectionMode: "encrypted-key-provider" }); }); + + function testKeyProvider(fill: number): KnowledgeStoreKeyProvider { + return { + providerId: `test-${String(fill)}`, + resolveKey: () => new Uint8Array(32).fill(fill), + }; + } + + it("records store.encryption-migrated on a fresh forward migration to encrypted storage", () => { + const dbPath = join(tmp, "capsules.db"); + const { sink, events } = recordingSink(); + + const store = openKnowledgeStore({ + dbPath, + logSink: sink, + protection: { mode: "encrypted-key-provider", keyProvider: testKeyProvider(7) }, + }); + store.close(); + + const migrated = events.find((event) => event.op === "store.encryption-migrated"); + expect(migrated).toBeDefined(); + expect(migrated?.category).toBe("diagnostic"); + expect(migrated?.durationMs).toBeGreaterThanOrEqual(0); + expect(migrated?.extra).toEqual({ + fromScope: "plaintext", + toScope: STORE_CONTENT_ENCRYPTION_TEST_CONSTANTS.scopeValue, + }); + }); + + it("records store.encryption-migrated with the prior scope on a scope upgrade", () => { + const dbPath = join(tmp, "capsules.db"); + const store = openKnowledgeStore({ + dbPath, + protection: { mode: "encrypted-key-provider", keyProvider: testKeyProvider(9) }, + }); + store.close(); + + const raw = new DatabaseSync(dbPath); + try { + raw + .prepare("UPDATE schema_meta SET value = ? WHERE key = ?") + .run("reconstructive-columns/v2", STORE_CONTENT_ENCRYPTION_TEST_CONSTANTS.scopeKey); + } finally { + raw.close(); + } + + const { sink, events } = recordingSink(); + const upgraded = openKnowledgeStore({ + dbPath, + logSink: sink, + protection: { mode: "encrypted-key-provider", keyProvider: testKeyProvider(9) }, + }); + upgraded.close(); + + const migrated = events.find((event) => event.op === "store.encryption-migrated"); + expect(migrated).toBeDefined(); + expect(migrated?.extra).toEqual({ + fromScope: "reconstructive-columns/v2", + toScope: STORE_CONTENT_ENCRYPTION_TEST_CONSTANTS.scopeValue, + }); + }); + + it("never writes store.encryption-migrated when nothing needed migrating", () => { + const dbPath = join(tmp, "capsules.db"); + const provider = testKeyProvider(11); + openKnowledgeStore({ + dbPath, + protection: { mode: "encrypted-key-provider", keyProvider: provider }, + }).close(); + + const { sink, events } = recordingSink(); + openKnowledgeStore({ + dbPath, + logSink: sink, + protection: { mode: "encrypted-key-provider", keyProvider: provider }, + }).close(); + + expect(events.find((event) => event.op === "store.encryption-migrated")).toBeUndefined(); + }); +}); + +describe("computeStoreFingerprint", () => { + it("reports schema version, applied migrations, table row counts, and quick_check on a fresh plaintext store", () => { + const store = openKnowledgeStore({ dbPath: join(tmp, "capsules.db") }); + try { + const fingerprint = computeStoreFingerprint(store._internal.db); + expect(fingerprint.store).toBe("local-knowledge"); + expect(fingerprint.schemaVersion).toBe(LOCAL_KNOWLEDGE_DB_SCHEMA_VERSION); + expect(fingerprint.migrationsApplied).toEqual( + KNOWLEDGE_CAPSULE_MIGRATIONS.map((migration) => `v${String(migration.version)}`), + ); + expect(Object.keys(fingerprint.tableRowCounts).sort()).toEqual( + [...KNOWLEDGE_CAPSULE_TABLES].sort(), + ); + expect(Object.values(fingerprint.tableRowCounts).every((count) => count === 0)).toBe(true); + expect(fingerprint.quickCheckOk).toBe(true); + expect(fingerprint.encryptionMode).toBe("plaintext"); + expect(fingerprint.keySource).toBeUndefined(); + } finally { + store.close(); + } + }); + + it("counts existing rows and reports encryptionMode: encrypted for an encrypted store", () => { + const dbPath = join(tmp, "capsules.db"); + const store = openKnowledgeStore({ + dbPath, + protection: { + mode: "encrypted-key-provider", + keyProvider: { providerId: "fp-test", resolveKey: () => new Uint8Array(32).fill(3) }, + }, + }); + try { + store._internal.db + .prepare( + `INSERT INTO capsules (id, display_name, tags_json, retrieval_effort, output_mode, + answer_grounding_policy, lifecycle_state, storage_reference, + embedding_model_provider, embedding_model_id, vector_dimensions, vector_metric, + created_at, updated_at) + VALUES ('cap-fp', 'Fingerprint capsule', '[]', 'default', 'answers', + 'require-citations-or-state-no-evidence', 'draft', 'capsules/cap-fp', + 'test', 'model', 8, 'cosine', 1, 1)`, + ) + .run(); + const fingerprint = computeStoreFingerprint(store._internal.db); + expect(fingerprint.tableRowCounts.capsules).toBe(1); + expect(fingerprint.encryptionMode).toBe("encrypted"); + expect(fingerprint.quickCheckOk).toBe(true); + } finally { + store.close(); + } + }); + + it("degrades a failing quick_check read to quickCheckOk: false rather than throwing", () => { + const dbPath = join(tmp, "capsules.db"); + const store = openKnowledgeStore({ dbPath }); + // A quick_check failure must degrade, never propagate — a bundle export must not crash + // because the very store it is reporting on is unhealthy. Overriding `prepare` for just the + // one statement (rather than corrupting the on-disk file, which `openKnowledgeStore` already + // quarantines at open time) isolates the ONE read this function's `quickCheckOkFor` helper + // must swallow, without disturbing every other prepared statement `computeStoreFingerprint` + // also issues. + const originalPrepare = store._internal.db.prepare.bind(store._internal.db); + store._internal.db.prepare = (sql: string): ReturnType => { + if (sql === "PRAGMA quick_check") throw new Error("simulated quick_check read failure"); + return originalPrepare(sql); + }; + try { + const fingerprint = computeStoreFingerprint(store._internal.db); + expect(fingerprint.quickCheckOk).toBe(false); + } finally { + store._internal.db.prepare = originalPrepare; + store.close(); + } + }); }); diff --git a/packages/keiko-local-knowledge/src/store.ts b/packages/keiko-local-knowledge/src/store.ts index 36370c6517..e32799cefa 100644 --- a/packages/keiko-local-knowledge/src/store.ts +++ b/packages/keiko-local-knowledge/src/store.ts @@ -22,6 +22,7 @@ import { KNOWLEDGE_CAPSULE_V1_TABLES, LOCAL_KNOWLEDGE_DB_SCHEMA_VERSION, type KnowledgeCapsuleMigration, + type StoreFingerprint, } from "@oscharko-dev/keiko-contracts"; // Shared fs-hardening owner [GEN-MAINT-COUPLING-005]: the single 0o700/0o600 hardening pair. import { @@ -49,7 +50,10 @@ import { PLAINTEXT_CONTENT_CIPHER, type StoreContentCipher, } from "./store-content-cipher.js"; -import { applyStoreContentEncryption } from "./store-content-encryption.js"; +import { + applyStoreContentEncryption, + readStoreEncryptionMode, +} from "./store-content-encryption.js"; import type { VectorIndexOptions } from "./retrieval/vector-index.js"; export interface OpenKnowledgeStoreOptions { @@ -498,7 +502,7 @@ export function openKnowledgeStore(opts: OpenKnowledgeStoreOptions): KnowledgeSt let contentCipher: StoreContentCipher; try { contentCipher = resolveContentCipher(opts, currentUserVersion(db)); - applyStoreContentEncryption(db, contentCipher); + applyStoreContentEncryption(db, contentCipher, opts.logSink); } catch (cause) { db.close(); // Fail-closed: a wrong key or a missing provider for an already-encrypted store. The throw @@ -523,3 +527,81 @@ export function openKnowledgeStore(opts: OpenKnowledgeStoreOptions): KnowledgeSt _internal: { db: handle, now, contentCipher }, }; } + +// ─── Store fingerprint (Wave 4a, epic #3233 §6.2) ────────────────────────────── +// +// A redacted, point-in-time snapshot of this store's schema/integrity state, embedded in the +// support bundle manifest's `storeFingerprints` array. Read-only and never throwing: a bundle +// export must not fail because the store it is reporting on is itself unhealthy — an unreadable +// signal degrades to its own closed-vocabulary "unavailable" reading rather than aborting the +// whole fingerprint. + +// Genuinely read-only open for a diagnostic snapshot. Unlike `openKnowledgeStore` above, this +// never runs `runMigrations`, `applyStoreContentEncryption`, or the corruption-quarantine reopen +// loop — every one of those is a write, and a fingerprint export must not migrate, re-encrypt, or +// quarantine the very store an operator is trying to inspect. It also returns the raw handle +// rather than a `KnowledgeStore`, so a caller never needs to reach into `KnowledgeStore._internal` +// (deliberately package-private, see the doc comment on `KnowledgeStore` above) just to fingerprint +// a store. `computeStoreFingerprint` below needs only `PRAGMA user_version`/`quick_check` and fixed +// `SELECT COUNT(*)` reads, none of which need write access. +export function openKnowledgeStoreReadOnly(dbPath: string): DatabaseSync { + return new DatabaseSync(dbPath, { readOnly: true }); +} + +// Reuses the SAME source list and comparison `runMigrations` already applies (`version` vs the +// current `PRAGMA user_version`), just inverted: applied, not pending. Migration-group names are +// `v` — the manifest only carries `version`/`reason`, and `reason` is a free-text +// sentence unsafe to embed verbatim in a redacted manifest field. +function migrationsAppliedThrough(schemaVersion: number): readonly string[] { + return KNOWLEDGE_CAPSULE_MIGRATIONS.filter((migration) => migration.version <= schemaVersion).map( + (migration) => `v${String(migration.version)}`, + ); +} + +interface RowCount { + readonly n: number; +} + +// `KNOWLEDGE_CAPSULE_TABLES` is the same FIXED, package-owned table list `expectedTablesPresent` +// already validates post-migration — never a dynamic walk of `sqlite_master`, and never a +// caller-influenced name reaching SQL text. +function tableRowCountsFor(db: DatabaseSync): Readonly> { + const counts: Record = {}; + for (const table of KNOWLEDGE_CAPSULE_TABLES) { + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as RowCount | undefined; + counts[table] = row?.n ?? 0; + } + return counts; +} + +// A boolean projection of `assertQuickCheckOk`'s pass/fail — never the raw check-output rows, +// and never throwing: a fingerprint reports a bad quick_check as `false` rather than aborting the +// whole manifest assembly over one store's integrity. +function quickCheckOkFor(db: DatabaseSync): boolean { + try { + assertQuickCheckOk(db); + return true; + } catch { + return false; + } +} + +/** + * Computes a {@link StoreFingerprint} for this package's capsule store. Read-only, pure + * introspection over an already-open handle — never mutates, never throws. + * + * `keySource` is intentionally omitted: `KnowledgeStoreKeyProvider` carries only `providerId`, + * with no key-resolution-tier concept to report (unlike the vault-backed stores this field also + * covers), and this function takes only `db` so it has no provider to ask in any case. + */ +export function computeStoreFingerprint(db: DatabaseSync): StoreFingerprint { + const schemaVersion = currentUserVersion(db); + return { + store: "local-knowledge", + schemaVersion, + migrationsApplied: migrationsAppliedThrough(schemaVersion), + tableRowCounts: tableRowCountsFor(db), + quickCheckOk: quickCheckOkFor(db), + encryptionMode: readStoreEncryptionMode(db), + }; +} diff --git a/packages/keiko-memory-vault/src/db.test.ts b/packages/keiko-memory-vault/src/db.test.ts index 435c03e177..7b5759973c 100644 --- a/packages/keiko-memory-vault/src/db.test.ts +++ b/packages/keiko-memory-vault/src/db.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { DatabaseSync } from "node:sqlite"; import { existsSync, @@ -13,9 +13,16 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { Worker } from "node:worker_threads"; -import { chmodIfPresent, openMemoryDatabase, quarantineCorruptDb } from "./db.js"; +import { + chmodIfPresent, + computeStoreFingerprint, + openMemoryDatabase, + quarantineCorruptDb, +} from "./db.js"; import { MEMORY_VAULT_SCHEMA_VERSION } from "./schema.js"; -import { TEST_CIPHER } from "./_support.js"; +import { insertMemoryRow } from "./memories.js"; +import { makeRecord, memId, TEST_CIPHER } from "./_support.js"; +import type { MemoryVaultLogEvent, MemoryVaultLogSink } from "./vault-log.js"; const cleanups: string[] = []; @@ -23,6 +30,7 @@ afterEach(() => { for (const path of cleanups.splice(0)) { rmSync(path, { recursive: true, force: true }); } + vi.restoreAllMocks(); }); function freshDir(): string { @@ -165,6 +173,195 @@ describe("openMemoryDatabase corruption path", () => { } expect(readdirSync(dir).some((e) => e.includes(".corrupt."))).toBe(false); }); + + // RED (before fix): openMemoryDatabase had no third parameter at all, so a quarantine — a + // data-losing recovery decision — was completely unobservable from the activity log. + it("emits exactly one memory-vault.store.quarantined event with reopened:true", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + writeFileSync(dbPath, "garbage that is not a sqlite header"); + const events: MemoryVaultLogEvent[] = []; + const sink: MemoryVaultLogSink = { + write: (event): void => { + events.push(event); + }, + }; + + const db = openMemoryDatabase(dbPath, TEST_CIPHER, sink); + db.close(); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + level: "error", + category: "diagnostic", + op: "memory-vault.store.quarantined", + extra: { reopened: true }, + }); + expect(typeof events[0]?.errorKind).toBe("string"); + }); + + it("never lets a throwing sink surface as an open failure", () => { + vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + writeFileSync(dbPath, "garbage that is not a sqlite header"); + const dead: MemoryVaultLogSink = { + write: (): never => { + throw new Error("sink is down"); + }, + }; + + let db: DatabaseSync | undefined; + expect(() => { + db = openMemoryDatabase(dbPath, TEST_CIPHER, dead); + }).not.toThrow(); + db?.close(); + }); +}); + +// RED (before fix): `runMigrations` never forwarded a sink into `encryptExistingContent`, so this +// event could never fire through the real `openMemoryDatabase` path — only through a direct, +// bypassing call to `encryptExistingContent` itself (see migrate-encrypt.test.ts). This test goes +// through the real production entry point end to end, per AGENTS.md's fixture rule: a fixture that +// never reaches the production entry point cannot detect a wiring gap between two functions that +// both individually work. +describe("openMemoryDatabase — store.encryption-migrated wiring", () => { + // Mirrors `encryption-at-rest.test.ts`'s `downgradeToLegacyPlaintext`, at the level this suite + // already operates on (a raw `DatabaseSync`, not the public vault API): bring the DB to schema + // head first, insert a row, downgrade its content back to plaintext, then roll the v3+ DDL back + // to its v1 shape (a genuine v1 DB predates it) so the reopen's pending-DDL replay does not + // collide with tables/columns that already exist. + function seedLegacyPlaintextDb(dbPath: string): void { + openMemoryDatabase(dbPath, TEST_CIPHER).close(); + const db = new DatabaseSync(dbPath); + insertMemoryRow(db, makeRecord({ id: memId("m1") }), TEST_CIPHER); + db.prepare("UPDATE memories SET body = ? WHERE id = ?").run("plaintext body", "m1"); + db.exec("DROP TABLE memory_access"); + db.exec("DROP TABLE memory_tombstones"); + db.exec(` + CREATE TABLE memory_tombstones ( + id TEXT NOT NULL PRIMARY KEY, + memory_id TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_coordinate TEXT NOT NULL, + type TEXT NOT NULL, + forgotten_at INTEGER NOT NULL, + forgetter_surface TEXT NOT NULL, + reason TEXT + ) STRICT; + CREATE INDEX idx_tombstones_scope ON memory_tombstones(scope_kind, scope_coordinate); + CREATE INDEX idx_tombstones_memory_id ON memory_tombstones(memory_id); + `); + db.exec("PRAGMA user_version = 1"); + db.close(); + } + + it("emits store.encryption-migrated when opening a legacy plaintext DB", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + seedLegacyPlaintextDb(dbPath); + const events: MemoryVaultLogEvent[] = []; + const sink: MemoryVaultLogSink = { + write: (event): void => { + events.push(event); + }, + }; + + const db = openMemoryDatabase(dbPath, TEST_CIPHER, sink); + db.close(); + + const migrated = events.filter((event) => event.op === "store.encryption-migrated"); + expect(migrated).toHaveLength(1); + expect(migrated[0]).toMatchObject({ category: "diagnostic" }); + const extra = migrated[0]?.extra as { rowsMigrated?: unknown } | undefined; + expect(typeof extra?.rowsMigrated).toBe("number"); + expect(extra?.rowsMigrated as number).toBeGreaterThanOrEqual(1); + }); + + it("does not emit store.encryption-migrated for a fresh DB with nothing to migrate", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + const events: MemoryVaultLogEvent[] = []; + const sink: MemoryVaultLogSink = { + write: (event): void => { + events.push(event); + }, + }; + + openMemoryDatabase(dbPath, TEST_CIPHER, sink).close(); + + expect(events.some((event) => event.op === "store.encryption-migrated")).toBe(false); + }); +}); + +describe("computeStoreFingerprint", () => { + it("reports schemaVersion, table row counts, quickCheckOk, encryptionMode and keySource for a healthy vault", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + const db = openMemoryDatabase(dbPath, TEST_CIPHER); + db.prepare( + "INSERT INTO memory_vault_secrets (name, value) VALUES ('probe', 'kv1.probe-value')", + ).run(); + + const fingerprint = computeStoreFingerprint(db, "keychain"); + + expect(fingerprint.store).toBe("memory-vault"); + expect(fingerprint.schemaVersion).toBe(MEMORY_VAULT_SCHEMA_VERSION); + expect(fingerprint.migrationsApplied).toContain("v1"); + expect(fingerprint.migrationsApplied).toContain(`v${String(MEMORY_VAULT_SCHEMA_VERSION)}`); + expect(fingerprint.tableRowCounts.memory_vault_secrets).toBe(1); + expect(fingerprint.tableRowCounts.memories).toBe(0); + expect(fingerprint.quickCheckOk).toBe(true); + expect(fingerprint.encryptionMode).toBe("encrypted"); + expect(fingerprint.keySource).toBe("keychain"); + db.close(); + }); + + it("omits keySource when the caller supplies none (an injected cipher/vaultKey test seam)", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + const db = openMemoryDatabase(dbPath, TEST_CIPHER); + + const fingerprint = computeStoreFingerprint(db, undefined); + + expect(fingerprint.keySource).toBeUndefined(); + db.close(); + }); + + it("never throws on a corrupt/garbage file and reports quickCheckOk:false", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + writeFileSync(dbPath, "garbage that is not a sqlite header, definitely not a real db file"); + const raw = new DatabaseSync(dbPath); + + let fingerprint: ReturnType | undefined; + expect(() => { + fingerprint = computeStoreFingerprint(raw, undefined); + }).not.toThrow(); + + expect(fingerprint?.store).toBe("memory-vault"); + expect(fingerprint?.quickCheckOk).toBe(false); + // Every fixed table read fails against a garbage file (not a database at all), so each + // reports the safe default of 0 rather than throwing or being omitted. + expect(Object.values(fingerprint?.tableRowCounts ?? {})).toEqual([0, 0, 0, 0, 0, 0]); + raw.close(); + }); + + it("is read-only: computing a fingerprint does not change table row counts", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + const db = openMemoryDatabase(dbPath, TEST_CIPHER); + db.prepare("INSERT INTO memory_vault_secrets (name, value) VALUES ('a', 'kv1.a')").run(); + + computeStoreFingerprint(db, undefined); + computeStoreFingerprint(db, undefined); + + const row = db.prepare("SELECT COUNT(*) AS n FROM memory_vault_secrets").get() as { + readonly n: number; + }; + expect(row.n).toBe(1); + db.close(); + }); }); describe("chmodIfPresent", () => { diff --git a/packages/keiko-memory-vault/src/db.ts b/packages/keiko-memory-vault/src/db.ts index 90f76a3e48..436801fde6 100644 --- a/packages/keiko-memory-vault/src/db.ts +++ b/packages/keiko-memory-vault/src/db.ts @@ -21,8 +21,14 @@ import { errorRecord, isSqliteCorruptionError, } from "@oscharko-dev/keiko-security/sqlite-corruption"; +import type { StoreFingerprint } from "@oscharko-dev/keiko-contracts"; import { runMigrations } from "./schema.js"; -import type { MemoryContentCipher } from "./cipher.js"; +import type { MemoryContentCipher, VaultKeySource } from "./cipher.js"; +import { + emitMemoryVaultLogEvent, + memoryVaultErrorKind, + type MemoryVaultLogSink, +} from "./vault-log.js"; export { chmodIfPresent, ensureDirHardened }; @@ -99,13 +105,36 @@ export function quarantineCorruptDb( ); } -export function openMemoryDatabase(dbPath: string, cipher: MemoryContentCipher): DatabaseSync { +// Quarantine is a DATA-LOSING decision (the old file is rotated aside and a fresh, empty DB takes +// its place), so it is recorded at `error` even when the reopen succeeds — mirroring +// `packages/keiko-local-knowledge/src/store.ts`'s `logStoreQuarantine`. `sink` is optional +// (ADR-0019 — see `vault-log.ts`); `emitMemoryVaultLogEvent` never throws, so a dead sink can +// never turn a successful recovery into a new failure. +function logStoreQuarantined( + sink: MemoryVaultLogSink | undefined, + cause: unknown, + reopened: boolean, +): void { + emitMemoryVaultLogEvent(sink, { + level: "error", + category: "diagnostic", + op: "memory-vault.store.quarantined", + errorKind: memoryVaultErrorKind(cause), + extra: { reopened }, + }); +} + +export function openMemoryDatabase( + dbPath: string, + cipher: MemoryContentCipher, + sink?: MemoryVaultLogSink, +): DatabaseSync { ensureDirHardened(dirname(dbPath)); let db = preparedDatabase(dbPath); try { configureWalDatabase(db); assertQuickCheckOk(db); - runMigrations(db, cipher); + runMigrations(db, cipher, sink); } catch (error) { // SQLite's close() on a WAL-enabled handle may checkpoint and unlink -wal/-shm, // so we must SNAPSHOT sidecar existence BEFORE close, then close, then rename @@ -118,13 +147,157 @@ export function openMemoryDatabase(dbPath: string, cipher: MemoryContentCipher): throw error; } quarantineCorruptDb(dbPath, { hadWal, hadShm }, error); - db = preparedDatabase(dbPath); - configureWalDatabase(db); - assertQuickCheckOk(db); - runMigrations(db, cipher); + let reopened = false; + try { + db = preparedDatabase(dbPath); + configureWalDatabase(db); + assertQuickCheckOk(db); + runMigrations(db, cipher, sink); + reopened = true; + } finally { + logStoreQuarantined(sink, error, reopened); + } } chmodIfPresent(dbPath, FILE_MODE); chmodIfPresent(`${dbPath}-wal`, FILE_MODE); chmodIfPresent(`${dbPath}-shm`, FILE_MODE); return db; } + +// Genuinely read-only open for a diagnostic snapshot (Wave 4a, epic #3233 §6.2/§8). Unlike +// `openMemoryDatabase` above, this never runs `runMigrations` (which can trigger the v1->v2 +// encryption sweep, rewriting every content row) or the corruption-quarantine reopen loop — every +// one of those is a write, and a fingerprint export must not migrate, re-encrypt, or quarantine the +// very vault an operator is trying to inspect. `computeStoreFingerprint` below needs only +// `PRAGMA user_version`/`quick_check` and fixed `SELECT COUNT(*)` reads, none of which need write +// access. +export function openMemoryDatabaseReadOnly(dbPath: string): DatabaseSync { + return new DatabaseSync(dbPath, { readOnly: true }); +} + +// ─── computeStoreFingerprint (Wave 4a, epic #3233 §6.2) ──────────────────────────────────────── +// +// A read-only, NEVER-THROWING snapshot of this vault's schema/integrity state for `keiko bundle +// export`'s support-bundle manifest (`StoreFingerprint`, `@oscharko-dev/keiko-contracts`). Every +// field is a count, a closed-vocabulary label, or a bounded identifier drawn from this package's +// own fixed table/migration list — never a row, a path, a key, a secret, or free text (ADR-0128 +// D6). `keySource` is the already-computed key-resolution tier from `resolveVaultKey` (cipher.ts), +// passed in by the caller rather than recomputed here — recomputing it would mean touching the +// keychain/keyfile tiers a second time purely for a diagnostic read. + +// FIXED, closed table list — mirrors schema.ts's CREATE TABLE statements (v1: memories, +// memory_edges, memory_embeddings, memory_tombstones; v3: memory_access; v9: memory_vault_secrets) +// — never a dynamic sqlite_master walk, per `StoreFingerprint`'s own contract. A future migration +// that adds a table updates this list alongside schema.ts: the same accepted, documented- +// duplication tradeoff `ERROR_KIND_PATTERN` used across three packages before ADR-0173 +// consolidated it, needed here because schema.ts does not export its private table/migration +// lists and this function must stay read-only over db + PRAGMA state without reaching into them. +const STORE_FINGERPRINT_TABLES: readonly string[] = [ + "memories", + "memory_edges", + "memory_embeddings", + "memory_tombstones", + "memory_access", + "memory_vault_secrets", +]; + +// FIXED, closed migration-version list — mirrors schema.ts's `MIGRATIONS` array's version numbers +// (frozen, append-only history; v2 is the encryption-only bump and has no discrete DDL entry, so +// it is intentionally absent here too, exactly as it is absent from schema.ts's own array). +const STORE_FINGERPRINT_MIGRATION_VERSIONS: readonly number[] = [1, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + +// The schema version at which this store's content became encrypted-at-rest. Mirrors schema.ts's +// private `ENCRYPTION_VERSION`, frozen at 2 since that migration already shipped and schema.ts's +// migration history is append-only. `openMemoryDatabase` always drives a DB to this version or +// higher before returning, so `"migrating"` is structurally unobservable for this store today — +// the value exists in `StoreFingerprint`'s closed vocabulary for the OTHER store packages that +// share the type, not because this one can produce it. +const ENCRYPTION_SCHEMA_VERSION = 2; + +function safeUserVersion(db: DatabaseSync): number { + try { + const row = db.prepare("PRAGMA user_version").get() as { user_version?: number } | undefined; + return typeof row?.user_version === "number" ? row.user_version : 0; + } catch { + return 0; + } +} + +function safeQuickCheckOk(db: DatabaseSync): boolean { + try { + const rows = db.prepare("PRAGMA quick_check").all() as readonly Record[]; + const values = rows + .map((row) => Object.values(row)[0]) + .filter((value): value is string => typeof value === "string"); + return values.length === 1 && values[0] === "ok"; + } catch { + return false; + } +} + +function safeTableRowCount(db: DatabaseSync, table: string): number { + try { + // `table` comes only from the hard-coded STORE_FINGERPRINT_TABLES list above, never from + // caller data, so the interpolation is not an injection surface (the same rule + // migrate-encrypt.ts and schema.ts rely on for their own fixed identifier lists). + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as + { readonly n?: number } | undefined; + return typeof row?.n === "number" ? row.n : 0; + } catch { + // Table absent (fresh or pre-migration DB) or unreadable (corruption) — 0 is the safe, + // honest count for "nothing confirmed present", never a thrown diagnostic-read failure. + return 0; + } +} + +function safeTableRowCounts(db: DatabaseSync): Readonly> { + const counts: Record = {}; + for (const table of STORE_FINGERPRINT_TABLES) { + counts[table] = safeTableRowCount(db, table); + } + return counts; +} + +function migrationsAppliedUpTo(schemaVersion: number): readonly string[] { + return STORE_FINGERPRINT_MIGRATION_VERSIONS.filter((version) => version <= schemaVersion).map( + (version) => `v${String(version)}`, + ); +} + +function fallbackStoreFingerprint(keySource: VaultKeySource | undefined): StoreFingerprint { + return { + store: "memory-vault", + schemaVersion: 0, + migrationsApplied: [], + tableRowCounts: {}, + quickCheckOk: false, + encryptionMode: "plaintext", + keySource, + }; +} + +/** + * A point-in-time, redacted snapshot of this vault's schema/integrity state (Wave 4a, epic #3233 + * §6.2). Read-only and NEVER THROWS — a corrupt or unreadable file yields a safe, all-failed + * fingerprint (`quickCheckOk: false`) instead of propagating, so a support-bundle export can + * always attach one entry per store even when a store is unhealthy. + */ +export function computeStoreFingerprint( + db: DatabaseSync, + keySource: VaultKeySource | undefined, +): StoreFingerprint { + try { + const schemaVersion = safeUserVersion(db); + return { + store: "memory-vault", + schemaVersion, + migrationsApplied: migrationsAppliedUpTo(schemaVersion), + tableRowCounts: safeTableRowCounts(db), + quickCheckOk: safeQuickCheckOk(db), + encryptionMode: schemaVersion >= ENCRYPTION_SCHEMA_VERSION ? "encrypted" : "plaintext", + keySource, + }; + } catch { + return fallbackStoreFingerprint(keySource); + } +} diff --git a/packages/keiko-memory-vault/src/index.ts b/packages/keiko-memory-vault/src/index.ts index ebbde8662c..71d3f69470 100644 --- a/packages/keiko-memory-vault/src/index.ts +++ b/packages/keiko-memory-vault/src/index.ts @@ -22,6 +22,18 @@ export { } from "./paths.js"; export { MEMORY_VAULT_SCHEMA_VERSION } from "./schema.js"; export { memoryBodySuppressionHash } from "./body-fingerprint.js"; +// Read-only diagnostic seam for `keiko bundle export` (Wave 4a, epic #3233 §6.2): the exporter +// resolves the vault key and opens the store itself via `openMemoryDatabaseReadOnly` (rather than +// through `createMemoryVault`, whose returned `MemoryVaultStore` intentionally exposes no `db` +// handle or `keySource`, or through the mutating `openMemoryDatabase`) so it can call +// `computeStoreFingerprint` directly without migrating, re-encrypting, or quarantining the vault. +export { + createMemoryContentCipher, + resolveVaultKey, + type MemoryContentCipher, + type VaultKeySource, +} from "./cipher.js"; +export { computeStoreFingerprint, openMemoryDatabase, openMemoryDatabaseReadOnly } from "./db.js"; export type { DeleteMemoryOptions, ListMemoriesOptions, diff --git a/packages/keiko-memory-vault/src/migrate-encrypt.test.ts b/packages/keiko-memory-vault/src/migrate-encrypt.test.ts new file mode 100644 index 0000000000..6b5244e098 --- /dev/null +++ b/packages/keiko-memory-vault/src/migrate-encrypt.test.ts @@ -0,0 +1,129 @@ +// Regression tests for the `store.encryption-migrated` activity-log event (w4a-memory-vault-fingerprint, +// epic #3233 §8/g19). Before this change `encryptExistingContent` re-sealed plaintext content +// completely silently — an operator had no way to see, from `server.log`, that a vault had just +// undergone the one-way v1 -> v2 encryption sweep. +// +// The event fires only on a REAL transition (at least one row was actually sealed this call), never +// on a fresh empty DB or a re-run over already-sealed content — both sweep zero rows. + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { insertMemoryRow } from "./memories.js"; +import { encryptExistingContent } from "./migrate-encrypt.js"; +import { makeRecord, memId, openTestDb, TEST_CIPHER } from "./_support.js"; +import type { MemoryVaultLogEvent, MemoryVaultLogSink } from "./vault-log.js"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function recordingSink(): { sink: MemoryVaultLogSink; events: MemoryVaultLogEvent[] } { + const events: MemoryVaultLogEvent[] = []; + return { + sink: { + write: (event): void => { + events.push(event); + }, + }, + events, + }; +} + +// Bypasses the row layer's own seal step so the column holds genuine plaintext, mirroring how +// `encryption-at-rest.test.ts`'s `downgradeToLegacyPlaintext` simulates a pre-migration DB. +function overwriteBodyWithPlaintext( + db: ReturnType, + id: string, + body: string, +): void { + db.prepare("UPDATE memories SET body = ? WHERE id = ?").run(body, id); +} + +describe("encryptExistingContent — store.encryption-migrated event", () => { + it("does not emit on a fresh DB with no rows to migrate", () => { + const db = openTestDb(); + const { sink, events } = recordingSink(); + + encryptExistingContent(db, TEST_CIPHER, sink); + + expect(events).toHaveLength(0); + db.close(); + }); + + // RED (before fix): encryptExistingContent had no sink parameter at all, so this migration was + // unobservable from the activity log. + it("emits exactly one event, with fromScope/toScope/durationMs, when content is re-sealed", () => { + const db = openTestDb(); + insertMemoryRow(db, makeRecord({ id: memId("m1") }), TEST_CIPHER); + overwriteBodyWithPlaintext(db, "m1", "plaintext body"); + const { sink, events } = recordingSink(); + + encryptExistingContent(db, TEST_CIPHER, sink); + + expect(events).toHaveLength(1); + const event = events[0]; + expect(event).toMatchObject({ + category: "diagnostic", + op: "store.encryption-migrated", + }); + expect(event?.extra).toMatchObject({ fromScope: "plaintext", toScope: "encrypted" }); + const rowsMigrated = (event?.extra as { rowsMigrated?: unknown } | undefined)?.rowsMigrated; + expect(typeof rowsMigrated).toBe("number"); + expect(rowsMigrated as number).toBeGreaterThanOrEqual(1); + expect(typeof event?.durationMs).toBe("number"); + + const row = db.prepare("SELECT body FROM memories WHERE id = ?").get("m1") as { + readonly body: string; + }; + expect(TEST_CIPHER.isSealed(row.body)).toBe(true); + expect(TEST_CIPHER.openString(row.body)).toBe("plaintext body"); + db.close(); + }); + + it("is idempotent: a sweep over already-sealed content emits nothing", () => { + const db = openTestDb(); + insertMemoryRow(db, makeRecord({ id: memId("m1") }), TEST_CIPHER); + overwriteBodyWithPlaintext(db, "m1", "plaintext body"); + encryptExistingContent(db, TEST_CIPHER); // first sweep, no sink — must not throw either. + + const { sink, events } = recordingSink(); + encryptExistingContent(db, TEST_CIPHER, sink); + + expect(events).toHaveLength(0); + db.close(); + }); + + it("never throws when no sink is supplied", () => { + const db = openTestDb(); + insertMemoryRow(db, makeRecord({ id: memId("m1") }), TEST_CIPHER); + overwriteBodyWithPlaintext(db, "m1", "plaintext body"); + + expect(() => { + encryptExistingContent(db, TEST_CIPHER); + }).not.toThrow(); + db.close(); + }); + + // The migration itself must never fail because its OWN logging failed — the same rule + // `vault-log.test.ts` proves for the seam in isolation, pinned again here at the real call site. + it("never lets a throwing sink surface as a migration failure", () => { + vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const db = openTestDb(); + insertMemoryRow(db, makeRecord({ id: memId("m1") }), TEST_CIPHER); + overwriteBodyWithPlaintext(db, "m1", "plaintext body"); + const dead: MemoryVaultLogSink = { + write: (): never => { + throw new Error("sink is down"); + }, + }; + + expect(() => { + encryptExistingContent(db, TEST_CIPHER, dead); + }).not.toThrow(); + const row = db.prepare("SELECT body FROM memories WHERE id = ?").get("m1") as { + readonly body: string; + }; + expect(TEST_CIPHER.isSealed(row.body)).toBe(true); + db.close(); + }); +}); diff --git a/packages/keiko-memory-vault/src/migrate-encrypt.ts b/packages/keiko-memory-vault/src/migrate-encrypt.ts index c27ca115cf..3676d21517 100644 --- a/packages/keiko-memory-vault/src/migrate-encrypt.ts +++ b/packages/keiko-memory-vault/src/migrate-encrypt.ts @@ -9,6 +9,11 @@ import type { DatabaseSync } from "node:sqlite"; import type { MemoryContentCipher } from "./cipher.js"; +import { + emitMemoryVaultLogEvent, + startMemoryVaultLogTimer, + type MemoryVaultLogSink, +} from "./vault-log.js"; interface IdStringRow { readonly rowid: number; @@ -47,22 +52,30 @@ function isAlreadySealed(cipher: MemoryContentCipher, value: string): boolean { } } +// Returns the count of rows actually sealed in THIS call (rows that were plaintext and are now +// sealed) — never rows merely visited. This is the signal `encryptExistingContent` uses to decide +// whether a `store.encryption-migrated` event describes a real transition: a fresh, empty table +// (every column skipped as "already sealed" trivially, zero rows) and a re-run against an +// already-encrypted DB (every row already sealed) both correctly report zero. function sweepStringColumn( db: DatabaseSync, table: string, column: string, cipher: MemoryContentCipher, -): void { +): number { // Identifiers come from the hard-coded STRING_TARGETS list, never from caller data, so the // interpolation is not an injection surface (the same rule schema.ts relies on for PRAGMA). const rows = db .prepare(`SELECT rowid AS rowid, ${column} AS value FROM ${table}`) .all() as unknown as readonly IdStringRow[]; const update = db.prepare(`UPDATE ${table} SET ${column} = ? WHERE rowid = ?`); + let sealed = 0; for (const row of rows) { if (row.value === null || isAlreadySealed(cipher, row.value)) continue; update.run(cipher.sealString(row.value), row.rowid); + sealed += 1; } + return sealed; } // Unlike the string columns, the embedding BLOB has NO unambiguous "already sealed" marker: a @@ -70,8 +83,9 @@ function sweepStringColumn( // would wrongly skip it and leave plaintext that then fails to decrypt. Correctness instead rests on // the user_version gate in runMigrations: this sweep runs exactly once, in the same transaction that // sets user_version = 2, at which point EVERY embedding row is still v1 plaintext. So we seal all of -// them unconditionally. An interrupted run rolls back (user_version stays < 2) and re-seals cleanly. -function sweepEmbeddingVectors(db: DatabaseSync, cipher: MemoryContentCipher): void { +// them unconditionally — meaning the row count IS the sealed count — and an interrupted run rolls +// back (user_version stays < 2) and re-seals cleanly. +function sweepEmbeddingVectors(db: DatabaseSync, cipher: MemoryContentCipher): number { const rows = db .prepare("SELECT memory_id, vector FROM memory_embeddings") .all() as unknown as readonly EmbeddingBlobRow[]; @@ -79,11 +93,62 @@ function sweepEmbeddingVectors(db: DatabaseSync, cipher: MemoryContentCipher): v for (const row of rows) { update.run(cipher.sealBytes(Buffer.from(row.vector)), row.memory_id); } + return rows.length; } -export function encryptExistingContent(db: DatabaseSync, cipher: MemoryContentCipher): void { +/** + * Emits `store.encryption-migrated` for a genuine transition (`rowsMigrated > 0`); a no-op + * otherwise. Exported so `schema.ts`'s `runMigrations` can call it AFTER its own transaction + * commits (see that file): the sweep runs inside `BEGIN`/`COMMIT` and can still roll back, so + * emitting from inside the sweep itself — as this package used to — could log a migration that + * never actually committed. + */ +export function emitEncryptionMigrated( + sink: MemoryVaultLogSink | undefined, + rowsMigrated: number, + durationMs: number, +): void { + if (rowsMigrated <= 0) return; + emitMemoryVaultLogEvent(sink, { + // "diagnostic", matching the same op emitted by Local Knowledge's + // `store-content-encryption.ts` for the identical event — one shared op name should carry one + // shared category so an analyzer grouping by (category, op) sees one cluster, not two. + category: "diagnostic", + op: "store.encryption-migrated", + durationMs, + extra: { fromScope: "plaintext", toScope: "encrypted", rowsMigrated }, + }); +} + +/** + * Eager v1→v2 encryption sweep (see file header), WITHOUT emitting: the row rewrite is the only + * effect, and the count of rows actually re-sealed (never rows merely visited) is returned so a + * caller can decide, on its own success, whether and when to report it. `schema.ts`'s + * `runMigrations` calls this (never `encryptExistingContent` below) from inside its migration + * transaction, then calls `emitEncryptionMigrated` itself only after `COMMIT` succeeds. + */ +export function sweepExistingContent(db: DatabaseSync, cipher: MemoryContentCipher): number { + let rowsMigrated = 0; for (const target of STRING_TARGETS) { - sweepStringColumn(db, target.table, target.column, cipher); + rowsMigrated += sweepStringColumn(db, target.table, target.column, cipher); } - sweepEmbeddingVectors(db, cipher); + rowsMigrated += sweepEmbeddingVectors(db, cipher); + return rowsMigrated; +} + +/** + * Convenience wrapper over {@link sweepExistingContent} that also emits + * `store.encryption-migrated` itself, timed around the sweep. `sink` is optional (ADR-0019 — this + * package declares its own `MemoryVaultLogSink` port rather than importing the server's logger). + * Kept for callers that sweep OUTSIDE a transaction they do not otherwise control (this package's + * own direct tests); `schema.ts`'s `runMigrations` does NOT use this — see {@link sweepExistingContent}. + */ +export function encryptExistingContent( + db: DatabaseSync, + cipher: MemoryContentCipher, + sink?: MemoryVaultLogSink, +): void { + const elapsedMs = startMemoryVaultLogTimer(); + const rowsMigrated = sweepExistingContent(db, cipher); + emitEncryptionMigrated(sink, rowsMigrated, elapsedMs()); } diff --git a/packages/keiko-memory-vault/src/schema.ts b/packages/keiko-memory-vault/src/schema.ts index a9732c1f24..c49e0b1d0d 100644 --- a/packages/keiko-memory-vault/src/schema.ts +++ b/packages/keiko-memory-vault/src/schema.ts @@ -19,7 +19,8 @@ import type { DatabaseSync } from "node:sqlite"; import type { MemoryContentCipher } from "./cipher.js"; -import { encryptExistingContent } from "./migrate-encrypt.js"; +import { emitEncryptionMigrated, sweepExistingContent } from "./migrate-encrypt.js"; +import { startMemoryVaultLogTimer, type MemoryVaultLogSink } from "./vault-log.js"; // v2 = encryption-at-rest (ADR-0035). v1 stored content columns in plaintext; v2 seals them via an // eager code sweep (no column changes). The bump is one-way: a v2 DB is unreadable by v1 code. @@ -233,7 +234,11 @@ function setUserVersion(db: DatabaseSync, v: number): void { db.exec(`PRAGMA user_version = ${String(v)}`); } -export function runMigrations(db: DatabaseSync, cipher: MemoryContentCipher): void { +export function runMigrations( + db: DatabaseSync, + cipher: MemoryContentCipher, + sink?: MemoryVaultLogSink, +): void { const start = currentUserVersion(db); if (start > MEMORY_VAULT_SCHEMA_VERSION) { throw new Error( @@ -248,6 +253,8 @@ export function runMigrations(db: DatabaseSync, cipher: MemoryContentCipher): vo // An EXISTING (already-created) DB crossing into the encryption version had plaintext on disk; // its superseded pages must be purged from the WAL so the plaintext does not linger after upgrade. const upgradedExistingDb = start > 0 && needsEncryption; + const elapsedMs = startMemoryVaultLogTimer(); + let rowsMigrated = 0; db.exec("BEGIN"); try { for (const m of pendingDdl) { @@ -258,8 +265,11 @@ export function runMigrations(db: DatabaseSync, cipher: MemoryContentCipher): vo // Idempotent: skips values already sealed, so a fresh DB (no rows) and a re-run are no-ops. // The encryption sweep is keyed to ENCRYPTION_VERSION (2) but is NOT a user_version write: // post-v2 migrations (v3+) own the version. Setting the version is deferred to the line below - // so encryption never regresses a DB that already applied a later DDL migration. - encryptExistingContent(db, cipher); + // so encryption never regresses a DB that already applied a later DDL migration. The sweep + // itself never emits (see migrate-encrypt.ts's `sweepExistingContent`): a log line for a + // migration that then rolls back would be a false report, so the emission below waits for + // this transaction's own COMMIT to actually succeed. + rowsMigrated = sweepExistingContent(db, cipher); } // Pin the final version to the current schema head once every pending DDL and the encryption // sweep have run. A fresh DB applies v1 + later DDL and the encryption sweep, then lands on @@ -270,6 +280,11 @@ export function runMigrations(db: DatabaseSync, cipher: MemoryContentCipher): vo db.exec("ROLLBACK"); throw error; } + // Outside the transaction, same reasoning as the WAL checkpoint below: a real transition is + // reported only once the migration has actually committed, never for a sweep that then rolled + // back. `emitEncryptionMigrated` itself is a no-op when `rowsMigrated` is 0 (fresh DB, or a + // re-run over already-sealed content). + emitEncryptionMigrated(sink, rowsMigrated, elapsedMs()); if (upgradedExistingDb) { // Outside the transaction (checkpoint cannot run inside one): truncate the WAL so pages that // held the now-re-encrypted plaintext are reclaimed immediately, not at the next close. diff --git a/packages/keiko-memory-vault/src/vault-keychain-log-wiring.test.ts b/packages/keiko-memory-vault/src/vault-keychain-log-wiring.test.ts new file mode 100644 index 0000000000..2ec47eede1 --- /dev/null +++ b/packages/keiko-memory-vault/src/vault-keychain-log-wiring.test.ts @@ -0,0 +1,97 @@ +// Wiring test for `createMemoryVault`'s `securityLogSink` option (Wave 4a, epic #3233 §8). +// +// WHY THIS IS ITS OWN FILE +// +// `CreateMemoryVaultOptions` deliberately exposes no `keychainAccess` test seam — production has +// none either, `resolveVaultKey`'s injectable third argument is built internally by +// `resolveCipherWithSource`, not forwarded from the public factory. That means the real OS +// boundary (`cipher.ts`'s `keyFromKeychain`, backed by `@oscharko-dev/keiko-security`'s bounded +// `/usr/bin/security` spawn) cannot be forced to fail deterministically from `createMemoryVault`'s +// public surface alone. The only hermetic way to observe, from that public surface, whether +// `securityLogSink` actually reaches the keychain tier is to replace `keyFromKeychain` itself with +// a fake that reports "unavailable" while forwarding whatever `sink` option it was called with — +// exactly the contract the real reader documents (`macos-keychain.ts`'s `emitKeychainFallback`). +// `vi.mock` is file-scoped, so this lives apart from `vault.test.ts` to avoid forcing every other +// test in that file through the fake keychain reader. +// +// THE FAILURE THIS PINS +// +// Before the wiring, `resolveCipherWithSource` called `resolveVaultKey(env, memoryDir)` with no +// third argument, so `resolveVaultKey`'s own default (`keyFromKeychain` with no options) ran — +// `options.sink` was always `undefined`, so a keychain fallback never reached ANY sink no matter +// what a caller passed as `securityLogSink`. Reverting the `sink: securityLogSink` forwarding in +// `resolveCipherWithSource` (`vault.ts`) reproduces that: the fake keychain reader below would +// then be called with `options.sink === undefined`, write nothing, and the first test's assertion +// on `events` would fail — exactly the FAILS-BEFORE/PASSES-AFTER property a wiring test needs. + +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { SecurityLogEvent, SecurityLogSink } from "@oscharko-dev/keiko-security"; + +vi.mock("./cipher.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // Stands in for a keychain that never answers (the 0.3.0 boot-hang class of failure, see + // `macos-keychain.ts`'s file header): reports `unavailable` (`undefined`, falls through to the + // keyfile tier) while emitting `security.keychain.fallback` on whatever sink it was given — + // the same observable contract the real bounded reader has. + keyFromKeychain: (options: { readonly sink?: SecurityLogSink } = {}): Buffer | undefined => { + options.sink?.write({ + level: "warn", + category: "security", + op: "security.keychain.fallback", + extra: { reasonKind: "ETIMEDOUT", boundedExitKind: "timeout" }, + }); + return undefined; + }, + }; +}); + +// Imported AFTER the mock declaration so `createMemoryVault`'s internal `keyFromKeychain` import +// binds to the fake above. +const { createMemoryVault } = await import("./vault.js"); + +let dir: string; + +beforeEach(() => { + dir = mkdtempSync(join(realpathSync(tmpdir()), "keiko-memvault-keychain-log-")); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +function recordingSink(): { sink: SecurityLogSink; events: SecurityLogEvent[] } { + const events: SecurityLogEvent[] = []; + return { sink: { write: (event): void => void events.push(event) }, events }; +} + +describe("createMemoryVault — securityLogSink wiring to the shared keychain tier", () => { + it("forwards securityLogSink into the keychain tier so a fallback reaches the caller's sink", () => { + const { sink, events } = recordingSink(); + + const vault = createMemoryVault({ + memoryDir: dir, + env: {}, // No KEIKO_MEMORY_KEY: forces the (mocked) keychain tier, which falls through to keyfile. + securityLogSink: sink, + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + category: "security", + op: "security.keychain.fallback", + }); + vault.close(); + }); + + it("never throws, and the vault still opens, when securityLogSink is omitted", () => { + expect(() => { + const vault = createMemoryVault({ memoryDir: dir, env: {} }); + vault.close(); + }).not.toThrow(); + }); +}); diff --git a/packages/keiko-memory-vault/src/vault-log.test.ts b/packages/keiko-memory-vault/src/vault-log.test.ts new file mode 100644 index 0000000000..ec8aa15964 --- /dev/null +++ b/packages/keiko-memory-vault/src/vault-log.test.ts @@ -0,0 +1,262 @@ +// Tests for the package's activity-log seam. Four properties are load-bearing: +// +// * the default sink is inert — an unwired caller must never pay for, or fail because of, +// instrumentation; +// * `memoryVaultErrorKind` classifies WITHOUT reading `message`, which is where a thrown error +// routinely carries a path or a fragment of the value that failed, and without letting a +// hostile property ACCESSOR throw out of the classification; +// * a failing sink neither surfaces to the operation being logged nor disappears silently; +// * the timer is monotonic and never reports a negative duration. + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + emitMemoryVaultLogEvent, + memoryVaultErrorKind, + nullMemoryVaultLogSink, + startMemoryVaultLogTimer, + type MemoryVaultLogEvent, + type MemoryVaultLogSink, +} from "./vault-log.js"; + +// The specs below replace platform functions — `performance.now`, `process.emitWarning`. A spy +// restored on the last line of its own test is only restored when that test PASSES: an assertion +// that throws first leaves the platform patched for every later test in this worker, turning one +// red into a cascade that hides its own cause. Neither vitest config in this repository sets +// `restoreMocks`, so the hook is what guarantees it. +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("nullMemoryVaultLogSink", () => { + it("accepts an event without throwing and returns a shared instance", () => { + const sink = nullMemoryVaultLogSink(); + expect(() => { + sink.write({ category: "memory", op: "test.op" }); + }).not.toThrow(); + expect(nullMemoryVaultLogSink()).toBe(sink); + }); +}); + +describe("memoryVaultErrorKind", () => { + it("prefers a coded `code` over the constructor name", () => { + const error = Object.assign(new TypeError("boom"), { code: "SQLITE_CORRUPT" }); + expect(memoryVaultErrorKind(error)).toBe("SQLITE_CORRUPT"); + }); + + it("falls back to the constructor name when there is no code", () => { + expect(memoryVaultErrorKind(new RangeError("boom"))).toBe("RangeError"); + }); + + it("reports `unknown` for a primitive throw and for an empty-string code", () => { + expect(memoryVaultErrorKind("a raw string throw")).toBe("unknown"); + expect(memoryVaultErrorKind(undefined)).toBe("unknown"); + expect(memoryVaultErrorKind(null)).toBe("unknown"); + expect(memoryVaultErrorKind({ code: "", name: "" })).toBe("unknown"); + }); + + // `errorKind`/`extra.reasonKind` are ENVELOPE fields written before `extra` redaction runs, so a + // provider- or OS-controlled `code`/`name` that carries a sentence must never pass through. + it("refuses a `code` that is a sentence rather than a taxonomy code", () => { + const echoed = Object.assign(new Error("boom"), { + code: "permission denied for /Users/someone/memory/vault.db", + name: "AccessError", + }); + const kind = memoryVaultErrorKind(echoed); + expect(kind).toBe("AccessError"); + expect(kind).not.toContain("/Users"); + expect(kind).not.toContain(" "); + }); + + it("degrades to the next candidate when a property accessor throws", () => { + const hostile = { name: "TransportError" }; + Object.defineProperty(hostile, "code", { + get(): string { + throw new Error("accessor refused"); + }, + enumerable: true, + }); + expect(() => memoryVaultErrorKind(hostile)).not.toThrow(); + expect(memoryVaultErrorKind(hostile)).toBe("TransportError"); + }); + + it("never reads `message` — the field that carries content", () => { + const readFields: string[] = []; + const probe = new Proxy( + { code: undefined, name: "ProbeError", message: "/Users/someone/secret-memory" }, + { + get(target, property, receiver): unknown { + readFields.push(String(property)); + return Reflect.get(target, property, receiver); + }, + }, + ); + expect(memoryVaultErrorKind(probe)).toBe("ProbeError"); + expect(readFields).not.toContain("message"); + }); +}); + +describe("emitMemoryVaultLogEvent", () => { + const failure = (): Error => Object.assign(new Error("no space left"), { code: "ENOSPC" }); + + function recordingSinkThatFailsOn(failingOps: readonly string[]): { + sink: MemoryVaultLogSink; + events: MemoryVaultLogEvent[]; + } { + const events: MemoryVaultLogEvent[] = []; + return { + sink: { + write: (event): void => { + if (failingOps.includes(event.op)) throw failure(); + events.push(event); + }, + }, + events, + }; + } + + it("does nothing at all when no sink is wired", () => { + expect(() => { + emitMemoryVaultLogEvent(undefined, { category: "memory", op: "memory-vault.store.opened" }); + }).not.toThrow(); + }); + + // The rule callers depend on: `openMemoryDatabase` logs from inside a corruption-recovery path, + // and `encryptExistingContent` logs after a migration that already committed. A write that threw + // there would replace a diagnosable outcome with a logging failure. + it("never lets a sink failure surface to the operation being logged", () => { + vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const dead: MemoryVaultLogSink = { + write: (): never => { + throw failure(); + }, + }; + expect(() => { + emitMemoryVaultLogEvent(dead, { category: "memory", op: "memory-vault.store.opened" }); + }).not.toThrow(); + }); + + it("reports a failing sink once, through the sink itself, as an envelope-only notice", () => { + const { sink, events } = recordingSinkThatFailsOn(["memory-vault.store.opened"]); + + emitMemoryVaultLogEvent(sink, { + category: "memory", + op: "memory-vault.store.opened", + errorKind: "Error", + extra: { keySource: "keychain" }, + }); + emitMemoryVaultLogEvent(sink, { category: "memory", op: "memory-vault.store.opened" }); + + expect(events).toHaveLength(1); + expect(events[0]).toEqual({ + level: "error", + category: "diagnostic", + op: "memory-vault.log.sink-failed", + errorKind: "ENOSPC", + extra: { droppedOp: "memory-vault.store.opened" }, + }); + }); + + it("keeps writing subsequent lines a recovered sink can take", () => { + const { sink, events } = recordingSinkThatFailsOn(["memory-vault.store.opened"]); + + emitMemoryVaultLogEvent(sink, { category: "memory", op: "memory-vault.store.opened" }); + emitMemoryVaultLogEvent(sink, { category: "memory", op: "store.encryption-migrated" }); + + expect(events.map((event) => event.op)).toEqual([ + "memory-vault.log.sink-failed", + "store.encryption-migrated", + ]); + }); + + // A sink that refuses the notice too is a dead transport, not a rejected shape. The report then + // leaves by the one channel that is not the broken one. + it("falls back to one process warning per sink when the transport itself is down", () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const dead: MemoryVaultLogSink = { + write: (): never => { + throw failure(); + }, + }; + + emitMemoryVaultLogEvent(dead, { category: "memory", op: "memory-vault.store.opened" }); + emitMemoryVaultLogEvent(dead, { category: "memory", op: "store.encryption-migrated" }); + + expect(warn).toHaveBeenCalledTimes(1); + const calls: readonly (readonly unknown[])[] = warn.mock.calls; + expect(calls[0]?.[1]).toMatchObject({ + code: "KEIKO_LOG_SINK_FAILED", + detail: "op=memory-vault.store.opened errorKind=ENOSPC", + }); + + // Per sink, not per process: a replaced sink that also fails is a new fact about the log. + const replacement: MemoryVaultLogSink = { + write: (): never => { + throw failure(); + }, + }; + emitMemoryVaultLogEvent(replacement, { + category: "memory", + op: "store.encryption-migrated", + }); + expect(warn).toHaveBeenCalledTimes(2); + }); + + it("carries no body into the fallback report", () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const secret = ["memory", "body", "value"].join("-"); + const dead: MemoryVaultLogSink = { + write: (): never => { + throw new Error(`writing failed for ${secret}`); + }, + }; + + emitMemoryVaultLogEvent(dead, { + category: "memory", + op: "memory-vault.store.opened", + extra: { note: secret }, + }); + + const reported = JSON.stringify(warn.mock.calls); + expect(reported).not.toContain(secret); + expect(reported).toContain("errorKind=Error"); + }); +}); + +describe("startMemoryVaultLogTimer", () => { + it("reports a non-negative elapsed duration rounded to three decimals", () => { + const nowSpy = vi.spyOn(performance, "now"); + nowSpy.mockReturnValueOnce(1000).mockReturnValueOnce(1042.98765); + const elapsed = startMemoryVaultLogTimer(); + expect(elapsed()).toBe(42.988); + }); + + it("is driven by performance.now, so a backwards wall clock cannot go negative", () => { + vi.spyOn(Date, "now").mockReturnValue(0); + const elapsed = startMemoryVaultLogTimer(); + expect(elapsed()).toBeGreaterThanOrEqual(0); + }); +}); + +describe("MemoryVaultLogEvent", () => { + it("carries only the envelope a redacting sink expects", () => { + const event: MemoryVaultLogEvent = { + level: "warn", + category: "memory", + op: "memory-vault.store.opened", + errorKind: "SQLITE_CORRUPT", + status: 500, + durationMs: 12.5, + extra: { keySource: "keychain" }, + }; + expect(Object.keys(event).sort()).toEqual([ + "category", + "durationMs", + "errorKind", + "extra", + "level", + "op", + "status", + ]); + }); +}); diff --git a/packages/keiko-memory-vault/src/vault-log.ts b/packages/keiko-memory-vault/src/vault-log.ts new file mode 100644 index 0000000000..60229eb17f --- /dev/null +++ b/packages/keiko-memory-vault/src/vault-log.ts @@ -0,0 +1,162 @@ +// Content-free activity-log seam for the memory-vault package (epic #3233, w4a-memory-vault-fingerprint). +// +// WHY THIS FILE EXISTS INSTEAD OF AN IMPORT +// +// `packages/keiko-local-knowledge/src/knowledge-log.ts` and `packages/keiko-security/src/log-port.ts` +// already declare structurally identical seams. This file is NOT an import of either: ADR-0019 +// forbids a domain package from depending on a SIBLING domain package, and `keiko-memory-vault` +// sits below `keiko-server` in the dependency direction, same as those two — pulling in a sibling +// package merely because the event shape matches would invert that arrow and hand this package's +// callers a dependency on an unrelated subsystem's failure modes. `keiko-memory-vault` therefore +// declares its own structural port, narrowed to the categories this package can legitimately emit. +// The server is the composition root that supplies the real implementation (`processServerLogSink`, +// `packages/keiko-server/src/process-log-sink.ts`); every call site degrades to a no-op when +// nothing is wired. +// +// The event shape is deliberately a STRUCTURAL SUBSET of the server's `ServerLogEvent` (same +// fields, same optionality), and `"memory"` is already a member of `ServerLogCategory` (added, +// per that file's own header, "for the vault/retrieval surface") — so a `ServerLogSink` is +// assignable to `MemoryVaultLogSink` with no adapter and no shared import, exactly the property +// `knowledge-log.ts` and `log-port.ts` document for their own ports. +// +// REDACTION IS STRUCTURAL, NOT A CALLER PROMISE +// +// A field here carries counts, durations, and shape-gated error kinds (ADR-0128 D6: identifiers, +// counts, and hashes only). A memory body, a tag, a scope coordinate, a vault key, and a +// filesystem path never reach a field on this event. + +import { classifyErrorKind } from "@oscharko-dev/keiko-contracts"; + +export type MemoryVaultLogLevel = "debug" | "info" | "warn" | "error"; + +export type MemoryVaultLogCategory = "memory" | "diagnostic"; + +export interface MemoryVaultLogEvent { + // Omitted means `info`, matching the server sink's own default. + readonly level?: MemoryVaultLogLevel | undefined; + readonly category: MemoryVaultLogCategory; + readonly op: string; + readonly correlationId?: string | undefined; + readonly durationMs?: number | undefined; + readonly status?: number | undefined; + readonly errorKind?: string | undefined; + readonly extra?: Readonly> | undefined; +} + +export interface MemoryVaultLogSink { + readonly write: (event: MemoryVaultLogEvent) => void; +} + +const NULL_SINK: MemoryVaultLogSink = { + write(_event: MemoryVaultLogEvent): void { + // Explicit no-op: the default whenever no caller wired a sink. + }, +}; + +export function nullMemoryVaultLogSink(): MemoryVaultLogSink { + return NULL_SINK; +} + +// Monotonic elapsed milliseconds, rounded to 3 decimals so a line stays stable in width. +// `performance.now()` rather than `Date.now()` so a clock step cannot produce a negative +// duration across a vault-open or an encryption sweep. +export function startMemoryVaultLogTimer(): () => number { + const startedAt = performance.now(); + return (): number => Math.round((performance.now() - startedAt) * 1000) / 1000; +} + +// The shape an error KIND may have is `classifyErrorKind` (ADR-0173 D11), imported from +// `keiko-contracts` rather than declared here, so this reducer and the ones in `knowledge-log.ts`, +// `log-port.ts`, `keiko-server/src/observability/server-log.ts` and +// `keiko-model-gateway/src/observability.ts` cannot drift into accepting different things: an +// identifier, a taxonomy code, a constructor name — never a sentence. +// +// READING the property is itself a call into foreign code — `code`/`name` are ordinary properties +// on an object this layer did not build, so an accessor or Proxy trap can throw. Every call site +// classifies a cause while already handling a failure (a corrupt DB open, a migration sweep that +// just threw), so a throw here would replace a diagnosable failure with a classification failure. +// An unreadable property degrades to the next candidate, exactly like a property whose value fails +// the shape gate. +function errorKindProperty(error: object, key: string): string | undefined { + let value: unknown; + try { + value = (error as Record)[key]; + } catch { + return undefined; + } + return classifyErrorKind(value); +} + +// Classification only: the coded `code`, else the constructor `name`, else "unknown". The message +// is NEVER read — a thrown error's message routinely carries a path or a fragment of the value +// that failed — and neither `code` nor `name` is trusted just because it is called one: both must +// pass `classifyErrorKind`'s shape gate first. +export function memoryVaultErrorKind(error: unknown): string { + if (typeof error !== "object" || error === null) return "unknown"; + return errorKindProperty(error, "code") ?? errorKindProperty(error, "name") ?? "unknown"; +} + +// ─── Writing a line must never become the failure it describes ──────────────── +// +// Same two rules as `knowledge-log.ts`/`log-port.ts`, restated here because this package's callers +// sit on the same kind of failure path: a corrupt-DB quarantine and an encryption sweep both run +// while the caller is already mid-recovery or mid-migration. Neither may have a logging failure +// replace the outcome it is reporting, and neither may go permanently silent without any signal +// reaching the operator. +// +// 1. A sink failure must never surface to the operation being logged. +// 2. A permanently broken sink must never be invisible. +// +// Both are satisfied by degrading in place — first a minimal envelope-only notice back through the +// same sink, then, only if that also throws, a single body-free `process.emitWarning` — reported +// once per sink instance via a `WeakSet` so a batch of failures cannot flood stderr and a replaced +// sink is reported again. +const REPORTED_FAILED_SINKS = new WeakSet(); + +export function emitMemoryVaultLogEvent( + sink: MemoryVaultLogSink | undefined, + event: MemoryVaultLogEvent, +): void { + if (sink === undefined) return; + try { + sink.write(event); + } catch (cause) { + reportFailedMemoryVaultLogSink(sink, event.op, cause); + } +} + +function reportFailedMemoryVaultLogSink( + sink: MemoryVaultLogSink, + droppedOp: string, + cause: unknown, +): void { + if (REPORTED_FAILED_SINKS.has(sink)) return; + REPORTED_FAILED_SINKS.add(sink); + const errorKind = memoryVaultErrorKind(cause); + try { + sink.write({ + level: "error", + category: "diagnostic", + op: "memory-vault.log.sink-failed", + errorKind, + extra: { droppedOp }, + }); + return; + } catch { + // The transport refuses this shape too, so it is the transport that is down — fall through to + // the only channel left. + } + warnFailedMemoryVaultLogSink(droppedOp, errorKind); +} + +function warnFailedMemoryVaultLogSink(droppedOp: string, errorKind: string): void { + try { + process.emitWarning("Keiko memory-vault log sink is failing; log lines are being dropped.", { + type: "KeikoActivityLog", + code: "KEIKO_LOG_SINK_FAILED", + detail: `op=${droppedOp} errorKind=${errorKind}`, + }); + } catch { + // The process warning channel is the last one there is; a report beyond it does not exist. + } +} diff --git a/packages/keiko-memory-vault/src/vault.test.ts b/packages/keiko-memory-vault/src/vault.test.ts index a9b71b3911..a8fdf09782 100644 --- a/packages/keiko-memory-vault/src/vault.test.ts +++ b/packages/keiko-memory-vault/src/vault.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { DatabaseSync } from "node:sqlite"; +import { randomBytes } from "node:crypto"; import { existsSync, mkdtempSync, @@ -32,6 +33,7 @@ import { type MemoryEvent, type MemoryVaultStore, } from "./index.js"; +import type { MemoryVaultLogEvent, MemoryVaultLogSink } from "./vault-log.js"; // Deterministic injected key so the vault tests never touch the OS keychain or write a keyfile, // and so encrypted-at-rest reads are reproducible across the suite (ADR-0035). @@ -1236,3 +1238,73 @@ describe("replaceAllEmbeddings concurrent-write detection", () => { v.close(); }); }); + +// w4a-memory-vault-fingerprint (epic #3233 §8, g18): `resolveVaultKey` returns `{ key, source }` +// but createMemoryVault used to destructure only `{ key }`, discarding `source` entirely — the +// key-resolution tier an operator needs to tell "opened via KEIKO_MEMORY_KEY" from "fell through +// to the weaker keyfile tier" was computed and then thrown away. +describe("activity-log seam: memory-vault.store.opened retains the key-resolution tier", () => { + function recordingSink(): { sink: MemoryVaultLogSink; events: MemoryVaultLogEvent[] } { + const events: MemoryVaultLogEvent[] = []; + return { + sink: { + write: (event): void => { + events.push(event); + }, + }, + events, + }; + } + + // RED (before fix): createMemoryVault had no `logSink` option and `resolveCipher` returned only + // the cipher, so this event did not exist at all. + it('emits exactly one event carrying keySource:"env" when KEIKO_MEMORY_KEY resolves the key', () => { + const dir = freshDir(); + const { sink, events } = recordingSink(); + const key = randomBytes(32); + + const v = createMemoryVault({ + memoryDir: dir, + env: { KEIKO_MEMORY_DIR: dir, KEIKO_MEMORY_KEY: key.toString("base64") }, + logSink: sink, + }); + + const opened = events.filter((event) => event.op === "memory-vault.store.opened"); + expect(opened).toHaveLength(1); + expect(opened[0]).toMatchObject({ category: "memory", op: "memory-vault.store.opened" }); + expect(opened[0]?.extra).toEqual({ keySource: "env" }); + expect(typeof opened[0]?.durationMs).toBe("number"); + v.close(); + }); + + // A test-injected vaultKey/cipher never touches resolveVaultKey at all, so there is no tier to + // report — the event still fires (the vault still opened), but without a keySource field. + it("omits keySource from the event when a vaultKey/cipher test seam bypassed key resolution", () => { + const dir = freshDir(); + const { sink, events } = recordingSink(); + + const v = createMemoryVault({ + memoryDir: dir, + env: { KEIKO_MEMORY_DIR: dir }, + vaultKey: Buffer.alloc(32, 7), + logSink: sink, + }); + + const opened = events.filter((event) => event.op === "memory-vault.store.opened"); + expect(opened).toHaveLength(1); + expect(opened[0]?.extra).toBeUndefined(); + v.close(); + }); + + it("never throws when no logSink is supplied (fully backward-compatible)", () => { + const dir = freshDir(); + expect(() => { + const v = createMemoryVault({ + memoryDir: dir, + env: { KEIKO_MEMORY_DIR: dir }, + vaultKey: Buffer.alloc(32, 7), + }); + v.close(); + }).not.toThrow(); + }); +}); diff --git a/packages/keiko-memory-vault/src/vault.ts b/packages/keiko-memory-vault/src/vault.ts index ee00dc8101..b22d7a7726 100644 --- a/packages/keiko-memory-vault/src/vault.ts +++ b/packages/keiko-memory-vault/src/vault.ts @@ -20,9 +20,21 @@ import type { MemoryScope, MemoryStatus, } from "@oscharko-dev/keiko-contracts/memory"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import { chmodIfPresent, openMemoryDatabase } from "./db.js"; import { resolveMemoryDir, resolveMemoryDbPath } from "./paths.js"; -import { createMemoryContentCipher, resolveVaultKey, type MemoryContentCipher } from "./cipher.js"; +import { + createMemoryContentCipher, + keyFromKeychain, + resolveVaultKey, + type MemoryContentCipher, + type VaultKeySource, +} from "./cipher.js"; +import { + emitMemoryVaultLogEvent, + startMemoryVaultLogTimer, + type MemoryVaultLogSink, +} from "./vault-log.js"; import { scopeCoordinateOf, scopeKindOf } from "./scope-key.js"; import { deleteMemoryRow, @@ -361,17 +373,54 @@ function resolveBodySuppressionKey(db: DatabaseSync, cipher: MemoryContentCipher return key; } +interface ResolvedCipherWithSource { + readonly cipher: MemoryContentCipher; + // `undefined` for a test-injected cipher/vaultKey, where no key-resolution tier ran at all — + // never a made-up label for a tier that was never actually consulted. + readonly keySource: VaultKeySource | undefined; +} + // Resolve the content cipher. Precedence: an explicitly injected cipher (tests), then an injected // raw key (tests/CI), then the real tiered resolver (KEIKO_MEMORY_KEY > keychain > keyfile). The // public factory never requires any of these — production callers get the tiered resolver. -function resolveCipher( +// +// The resolved tier is RETAINED here rather than discarded (previously `resolveVaultKey(...).key` +// threw the `source` half away — the store-open event this file now emits is exactly the field an +// operator needs to tell "the vault opened using an env override" from "the vault fell all the way +// through to the weaker keyfile tier" without re-deriving it from a second key-resolution call). +// +// `securityLogSink` (Wave 4a, epic #3233 §8) is threaded into the keychain tier's own bounded +// reader (`keyFromKeychain`, `cipher.ts`) rather than into `resolveVaultKey` itself: neither an +// injected cipher nor an injected raw key ever touches the keychain, so building the sink-wired +// closure only on the path that actually resolves through it keeps a test-injected key from paying +// for, or depending on, a sink it will never call. +function resolveCipherWithSource( opts: MemoryVaultFactoryOptions | undefined, env: Readonly>, -): MemoryContentCipher { - if (opts?.cipher !== undefined) return opts.cipher; - if (opts?.vaultKey !== undefined) return createMemoryContentCipher(opts.vaultKey); + securityLogSink: SecurityLogSink | undefined, +): ResolvedCipherWithSource { + if (opts?.cipher !== undefined) return { cipher: opts.cipher, keySource: undefined }; + if (opts?.vaultKey !== undefined) { + return { cipher: createMemoryContentCipher(opts.vaultKey), keySource: undefined }; + } const memoryDir = resolveMemoryDir(opts?.memoryDir, env); - return createMemoryContentCipher(resolveVaultKey(env, memoryDir).key); + const resolved = resolveVaultKey(env, memoryDir, () => + keyFromKeychain({ sink: securityLogSink }), + ); + return { cipher: createMemoryContentCipher(resolved.key), keySource: resolved.source }; +} + +function emitVaultOpened( + sink: MemoryVaultLogSink | undefined, + keySource: VaultKeySource | undefined, + durationMs: number, +): void { + emitMemoryVaultLogEvent(sink, { + category: "memory", + op: "memory-vault.store.opened", + durationMs, + ...(keySource === undefined ? {} : { extra: { keySource } }), + }); } // Validate-then-redact for inserts. The validator runs on the CALLER-SUPPLIED record so a @@ -1127,11 +1176,38 @@ function buildStore(db: DatabaseSync, opts: ResolvedOptions): MemoryVaultStore { }; } -export function createMemoryVault(options?: MemoryVaultFactoryOptions): MemoryVaultStore { +// `logSink`/`securityLogSink` are intentionally not on `MemoryVaultFactoryOptions` itself: an +// intersection at the factory's own call boundary is enough for full external usability, since a +// re-export preserves the exact function type TypeScript infers here, and object-literal +// assignability at a call site is checked structurally rather than by the type's name being +// importable — the same property that lets a bare `ServerLogSink` object literal satisfy either +// seam with no adapter (see `log-port.ts`/`vault-log.ts`). +export type CreateMemoryVaultOptions = MemoryVaultFactoryOptions & { + /** + * Optional activity-log seam (ADR-0019; see `vault-log.ts`). When wired, vault-open records the + * retained key-resolution tier on `memory-vault.store.opened`, and a corruption recovery emits + * `memory-vault.store.quarantined`. Omitted or `undefined` keeps the vault exactly as silent as + * before this change. + */ + readonly logSink?: MemoryVaultLogSink; + /** + * Optional activity-log seam for the shared macOS Keychain tier (ADR-0019, Wave 4a epic #3233 + * §8; see `@oscharko-dev/keiko-security/log-port.ts` and `cipher.ts`'s `keyFromKeychain`). When + * wired, a keychain spawn that falls through to the keyfile tier emits one + * `security.keychain.fallback` event on this sink instead of failing silently. A `ServerLogSink` + * (`processServerLogSink()`) is structurally assignable here with no adapter, mirroring `logSink` + * above. Omitted or `undefined` keeps the keychain tier exactly as silent as before this change. + */ + readonly securityLogSink?: SecurityLogSink; +}; + +export function createMemoryVault(options?: CreateMemoryVaultOptions): MemoryVaultStore { const env = options?.env ?? defaultEnv(); const dbPath = resolveMemoryDbPath(options?.memoryDir, env); - const cipher = resolveCipher(options, env); - const db = openMemoryDatabase(dbPath, cipher); + const { cipher, keySource } = resolveCipherWithSource(options, env, options?.securityLogSink); + const elapsedMs = startMemoryVaultLogTimer(); + const db = openMemoryDatabase(dbPath, cipher, options?.logSink); + emitVaultOpened(options?.logSink, keySource, elapsedMs()); const bodySuppressionKey = resolveBodySuppressionKey(db, cipher); hardenVaultSidecars(dbPath); return buildStore(db, resolveOptions(options, cipher, dbPath, bodySuppressionKey)); diff --git a/packages/keiko-security/src/index.ts b/packages/keiko-security/src/index.ts index 79b5fd77be..29c468dc82 100644 --- a/packages/keiko-security/src/index.ts +++ b/packages/keiko-security/src/index.ts @@ -48,6 +48,12 @@ export { errorRecord, } from "./sqlite-corruption.js"; +// Content-free activity-log seam for this package (ADR-0019, w4a-security-log-port) — independent +// of `keiko-local-knowledge`'s `KnowledgeLogSink`. Wired into `readMacosKeychainSecret` and +// `createShardedLocalSecretVault`'s shard reads; the composition root supplies the real sink. +export type { SecurityLogEvent, SecurityLogSink } from "./log-port.js"; +export { nullSecurityLogSink } from "./log-port.js"; + // Prompt Enhancer authoritative injection / unsafe-content detector (#1313, ADR-0044 §1/§5). export type { PromptInjectionSignalCode, diff --git a/packages/keiko-security/src/log-port.test.ts b/packages/keiko-security/src/log-port.test.ts new file mode 100644 index 0000000000..26dfcb97ff --- /dev/null +++ b/packages/keiko-security/src/log-port.test.ts @@ -0,0 +1,262 @@ +// Tests for the package's activity-log seam. Four properties are load-bearing: +// +// * the default sink is inert — an unwired caller must never pay for, or fail because of, +// instrumentation; +// * `securityErrorKind` classifies WITHOUT reading `message`, which is where a thrown error +// routinely carries a path or a fragment of the value that failed, and without letting a +// hostile property ACCESSOR throw out of the classification; +// * a failing sink neither surfaces to the operation being logged nor disappears silently; +// * the timer is monotonic and never reports a negative duration. + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + emitSecurityLogEvent, + nullSecurityLogSink, + securityErrorKind, + startSecurityLogTimer, + type SecurityLogEvent, + type SecurityLogSink, +} from "./log-port.js"; + +// The specs below replace platform functions — `performance.now`, `process.emitWarning`. A spy +// restored on the last line of its own test is only restored when that test PASSES: an assertion +// that throws first leaves the platform patched for every later test in this worker, turning one +// red into a cascade that hides its own cause. Neither vitest config in this repository sets +// `restoreMocks`, so the hook is what guarantees it. +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("nullSecurityLogSink", () => { + it("accepts an event without throwing and returns a shared instance", () => { + const sink = nullSecurityLogSink(); + expect(() => { + sink.write({ category: "security", op: "test.op" }); + }).not.toThrow(); + expect(nullSecurityLogSink()).toBe(sink); + }); +}); + +describe("securityErrorKind", () => { + it("prefers a coded `code` over the constructor name", () => { + const error = Object.assign(new TypeError("boom"), { code: "ETIMEDOUT" }); + expect(securityErrorKind(error)).toBe("ETIMEDOUT"); + }); + + it("falls back to the constructor name when there is no code", () => { + expect(securityErrorKind(new RangeError("boom"))).toBe("RangeError"); + }); + + it("reports `unknown` for a primitive throw and for an empty-string code", () => { + expect(securityErrorKind("a raw string throw")).toBe("unknown"); + expect(securityErrorKind(undefined)).toBe("unknown"); + expect(securityErrorKind(null)).toBe("unknown"); + expect(securityErrorKind({ code: "", name: "" })).toBe("unknown"); + }); + + // `errorKind`/`extra.reasonKind` are ENVELOPE fields written before `extra` redaction runs, so a + // provider- or OS-controlled `code`/`name` that carries a sentence must never pass through. + it("refuses a `code` that is a sentence rather than a taxonomy code", () => { + const echoed = Object.assign(new Error("boom"), { + code: "permission denied for /Users/someone/vault/shard.bin", + name: "AccessError", + }); + const kind = securityErrorKind(echoed); + expect(kind).toBe("AccessError"); + expect(kind).not.toContain("/Users"); + expect(kind).not.toContain(" "); + }); + + it("degrades to the next candidate when a property accessor throws", () => { + const hostile = { name: "TransportError" }; + Object.defineProperty(hostile, "code", { + get(): string { + throw new Error("accessor refused"); + }, + enumerable: true, + }); + expect(() => securityErrorKind(hostile)).not.toThrow(); + expect(securityErrorKind(hostile)).toBe("TransportError"); + }); + + it("never reads `message` — the field that carries content", () => { + const readFields: string[] = []; + const probe = new Proxy( + { code: undefined, name: "ProbeError", message: "/Users/someone/secret-shard" }, + { + get(target, property, receiver): unknown { + readFields.push(String(property)); + return Reflect.get(target, property, receiver); + }, + }, + ); + expect(securityErrorKind(probe)).toBe("ProbeError"); + expect(readFields).not.toContain("message"); + }); +}); + +describe("emitSecurityLogEvent", () => { + const failure = (): Error => Object.assign(new Error("no space left"), { code: "ENOSPC" }); + + function recordingSinkThatFailsOn(failingOps: readonly string[]): { + sink: SecurityLogSink; + events: SecurityLogEvent[]; + } { + const events: SecurityLogEvent[] = []; + return { + sink: { + write: (event): void => { + if (failingOps.includes(event.op)) throw failure(); + events.push(event); + }, + }, + events, + }; + } + + it("does nothing at all when no sink is wired", () => { + expect(() => { + emitSecurityLogEvent(undefined, { category: "security", op: "security.keychain.fallback" }); + }).not.toThrow(); + }); + + // The rule callers depend on: `readMacosKeychainSecret` logs from inside a catch on the boot + // path, and `readShardEnvelope` logs from inside a catch a caller is about to treat as "absent". + // A write that threw there would replace a diagnosable outcome with a logging failure. + it("never lets a sink failure surface to the operation being logged", () => { + vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const dead: SecurityLogSink = { + write: (): never => { + throw failure(); + }, + }; + expect(() => { + emitSecurityLogEvent(dead, { category: "security", op: "security.keychain.fallback" }); + }).not.toThrow(); + }); + + it("reports a failing sink once, through the sink itself, as an envelope-only notice", () => { + const { sink, events } = recordingSinkThatFailsOn(["security.vault.shard-unreadable"]); + + emitSecurityLogEvent(sink, { + category: "security", + op: "security.vault.shard-unreadable", + errorKind: "Error", + extra: { count: 1 }, + }); + emitSecurityLogEvent(sink, { category: "security", op: "security.vault.shard-unreadable" }); + + expect(events).toHaveLength(1); + expect(events[0]).toEqual({ + level: "error", + category: "diagnostic", + op: "security.log.sink-failed", + errorKind: "ENOSPC", + extra: { droppedOp: "security.vault.shard-unreadable" }, + }); + }); + + it("keeps writing subsequent lines a recovered sink can take", () => { + const { sink, events } = recordingSinkThatFailsOn(["security.keychain.fallback"]); + + emitSecurityLogEvent(sink, { category: "security", op: "security.keychain.fallback" }); + emitSecurityLogEvent(sink, { category: "security", op: "security.vault.shard-unreadable" }); + + expect(events.map((event) => event.op)).toEqual([ + "security.log.sink-failed", + "security.vault.shard-unreadable", + ]); + }); + + // A sink that refuses the notice too is a dead transport, not a rejected shape. The report then + // leaves by the one channel that is not the broken one. + it("falls back to one process warning per sink when the transport itself is down", () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const dead: SecurityLogSink = { + write: (): never => { + throw failure(); + }, + }; + + emitSecurityLogEvent(dead, { category: "security", op: "security.keychain.fallback" }); + emitSecurityLogEvent(dead, { category: "security", op: "security.vault.shard-unreadable" }); + + expect(warn).toHaveBeenCalledTimes(1); + const calls: readonly (readonly unknown[])[] = warn.mock.calls; + expect(calls[0]?.[1]).toMatchObject({ + code: "KEIKO_LOG_SINK_FAILED", + detail: "op=security.keychain.fallback errorKind=ENOSPC", + }); + + // Per sink, not per process: a replaced sink that also fails is a new fact about the log. + const replacement: SecurityLogSink = { + write: (): never => { + throw failure(); + }, + }; + emitSecurityLogEvent(replacement, { + category: "security", + op: "security.vault.shard-unreadable", + }); + expect(warn).toHaveBeenCalledTimes(2); + }); + + it("carries no body into the fallback report", () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const secret = ["shard", "reference", "value"].join("-"); + const dead: SecurityLogSink = { + write: (): never => { + throw new Error(`writing failed for ${secret}`); + }, + }; + + emitSecurityLogEvent(dead, { + category: "security", + op: "security.vault.shard-unreadable", + extra: { note: secret }, + }); + + const reported = JSON.stringify(warn.mock.calls); + expect(reported).not.toContain(secret); + expect(reported).toContain("errorKind=Error"); + }); +}); + +describe("startSecurityLogTimer", () => { + it("reports a non-negative elapsed duration rounded to three decimals", () => { + const nowSpy = vi.spyOn(performance, "now"); + nowSpy.mockReturnValueOnce(1000).mockReturnValueOnce(1042.98765); + const elapsed = startSecurityLogTimer(); + expect(elapsed()).toBe(42.988); + }); + + it("is driven by performance.now, so a backwards wall clock cannot go negative", () => { + vi.spyOn(Date, "now").mockReturnValue(0); + const elapsed = startSecurityLogTimer(); + expect(elapsed()).toBeGreaterThanOrEqual(0); + }); +}); + +describe("SecurityLogEvent", () => { + it("carries only the envelope a redacting sink expects", () => { + const event: SecurityLogEvent = { + level: "warn", + category: "security", + op: "security.keychain.fallback", + errorKind: "ETIMEDOUT", + status: 504, + durationMs: 12.5, + extra: { reasonKind: "ETIMEDOUT", boundedExitKind: "timeout" }, + }; + expect(Object.keys(event).sort()).toEqual([ + "category", + "durationMs", + "errorKind", + "extra", + "level", + "op", + "status", + ]); + }); +}); diff --git a/packages/keiko-security/src/log-port.ts b/packages/keiko-security/src/log-port.ts new file mode 100644 index 0000000000..fb471d2aca --- /dev/null +++ b/packages/keiko-security/src/log-port.ts @@ -0,0 +1,160 @@ +// Content-free activity-log seam for the security package (epic #2902, w4a-security-log-port). +// +// WHY THIS FILE EXISTS INSTEAD OF AN IMPORT +// +// `packages/keiko-local-knowledge/src/knowledge-log.ts` already declares a structurally identical +// seam. This file is NOT an import of it: ADR-0019 forbids a domain package from depending on a +// SIBLING domain package, and `keiko-security` sits at the leaf, depended upon by every store +// surface — pulling in `keiko-local-knowledge` merely because the event shape matches would invert +// that arrow and hand this package's callers a dependency on an unrelated subsystem's failure +// modes. `keiko-security` therefore declares its own structural port, narrowed to the categories +// this package can legitimately emit, and accepts an optional sink through the same options-object +// pattern every call site already threads test seams through. The server wires its own sink in at +// the composition root; every call site degrades to a no-op when nothing is wired. +// +// The event shape is deliberately a STRUCTURAL SUBSET of the server's `ServerLogEvent` (same +// fields, same optionality) so a `ServerLogSink` is assignable to `SecurityLogSink` with no +// adapter and no shared import — exactly the property `knowledge-log.ts` documents for its own +// port. +// +// REDACTION IS STRUCTURAL, NOT A CALLER PROMISE +// +// A field here carries counts, durations, and shape-gated error kinds (ADR-0128 D6: identifiers, +// counts, and hashes only). A keychain service/account name, a vault reference, a shard file path, +// and secret material never reach a field on this event. + +import { classifyErrorKind } from "@oscharko-dev/keiko-contracts"; + +export type SecurityLogLevel = "debug" | "info" | "warn" | "error"; + +export type SecurityLogCategory = "security" | "diagnostic"; + +export interface SecurityLogEvent { + // Omitted means `info`, matching the server sink's own default. + readonly level?: SecurityLogLevel | undefined; + readonly category: SecurityLogCategory; + readonly op: string; + readonly correlationId?: string | undefined; + readonly durationMs?: number | undefined; + readonly status?: number | undefined; + readonly errorKind?: string | undefined; + readonly extra?: Readonly> | undefined; +} + +export interface SecurityLogSink { + readonly write: (event: SecurityLogEvent) => void; +} + +const NULL_SINK: SecurityLogSink = { + write(_event: SecurityLogEvent): void { + // Explicit no-op: the default whenever no caller wired a sink. + }, +}; + +export function nullSecurityLogSink(): SecurityLogSink { + return NULL_SINK; +} + +// Monotonic elapsed milliseconds, rounded to 3 decimals so a line stays stable in width. +// `performance.now()` rather than `Date.now()` so a clock step cannot produce a negative duration +// while a bounded keychain spawn is in flight. +export function startSecurityLogTimer(): () => number { + const startedAt = performance.now(); + return (): number => Math.round((performance.now() - startedAt) * 1000) / 1000; +} + +// The shape an error KIND may have is `classifyErrorKind` (ADR-0173 D11), imported from +// `keiko-contracts` rather than declared here, so this reducer and the ones in +// `knowledge-log.ts`, `keiko-server/src/observability/server-log.ts` and +// `keiko-model-gateway/src/observability.ts` cannot drift into accepting different things: an +// identifier, a taxonomy code, a constructor name — never a sentence. +// +// READING the property is itself a call into foreign code — `code`/`name` are ordinary properties +// on an object this layer did not build, so an accessor or Proxy trap can throw. Every call site +// classifies a cause while already handling a failure (a keychain spawn that just failed, a shard +// read that just threw), so a throw here would replace a diagnosable failure with a classification +// failure. An unreadable property degrades to the next candidate, exactly like a property whose +// value fails the shape gate. +function errorKindProperty(error: object, key: string): string | undefined { + let value: unknown; + try { + value = (error as Record)[key]; + } catch { + return undefined; + } + return classifyErrorKind(value); +} + +// Classification only: the coded `code`, else the constructor `name`, else "unknown". The message +// is NEVER read — a thrown error's message routinely carries a path or a fragment of the value +// that failed — and neither `code` nor `name` is trusted just because it is called one: both must +// pass `classifyErrorKind`'s shape gate first. +export function securityErrorKind(error: unknown): string { + if (typeof error !== "object" || error === null) return "unknown"; + return errorKindProperty(error, "code") ?? errorKindProperty(error, "name") ?? "unknown"; +} + +// ─── Writing a line must never become the failure it describes ──────────────── +// +// Same two rules as `knowledge-log.ts`, restated here because this package's callers sit on the +// same kind of failure path: `readMacosKeychainSecret` is on the boot-path key-resolution tier, and +// `readShardEnvelope` runs inside a per-entry read a caller is treating as "absent" on failure. +// Neither may have a logging failure replace the outcome it is reporting, and neither may go +// permanently silent without any signal reaching the operator. +// +// 1. A sink failure must never surface to the operation being logged. +// 2. A permanently broken sink must never be invisible. +// +// Both are satisfied by degrading in place — first a minimal envelope-only notice back through the +// same sink, then, only if that also throws, a single body-free `process.emitWarning` — reported +// once per sink instance via a `WeakSet` so a large batch of shard reads cannot flood stderr and a +// replaced sink is reported again. +const REPORTED_FAILED_SINKS = new WeakSet(); + +export function emitSecurityLogEvent( + sink: SecurityLogSink | undefined, + event: SecurityLogEvent, +): void { + if (sink === undefined) return; + try { + sink.write(event); + } catch (cause) { + reportFailedSecurityLogSink(sink, event.op, cause); + } +} + +function reportFailedSecurityLogSink( + sink: SecurityLogSink, + droppedOp: string, + cause: unknown, +): void { + if (REPORTED_FAILED_SINKS.has(sink)) return; + REPORTED_FAILED_SINKS.add(sink); + const errorKind = securityErrorKind(cause); + try { + sink.write({ + level: "error", + category: "diagnostic", + op: "security.log.sink-failed", + errorKind, + extra: { droppedOp }, + }); + return; + } catch { + // The transport refuses this shape too, so it is the transport that is down — fall through to + // the only channel left. + } + warnFailedSecurityLogSink(droppedOp, errorKind); +} + +function warnFailedSecurityLogSink(droppedOp: string, errorKind: string): void { + try { + process.emitWarning("Keiko security log sink is failing; log lines are being dropped.", { + type: "KeikoActivityLog", + code: "KEIKO_LOG_SINK_FAILED", + detail: `op=${droppedOp} errorKind=${errorKind}`, + }); + } catch { + // The process warning channel is the last one there is; a report beyond it does not exist. + } +} diff --git a/packages/keiko-security/src/macos-keychain.test.ts b/packages/keiko-security/src/macos-keychain.test.ts index 032372abe0..7246d8b843 100644 --- a/packages/keiko-security/src/macos-keychain.test.ts +++ b/packages/keiko-security/src/macos-keychain.test.ts @@ -1,7 +1,8 @@ import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import type { SecurityLogEvent, SecurityLogSink } from "./log-port.js"; import { KEYCHAIN_SPAWN_TIMEOUT_MS, readMacosKeychainSecret, @@ -256,3 +257,130 @@ describe("writeMacosKeychainSecret", () => { expect(stored).toBe(false); }, 20_000); }); + +// The wiring the 0.3.0 boot-hang incident (file header) was missing: a fallback now emits ONE +// event, carrying only closed-vocabulary/duration fields — never the service/account name, and +// never the executable path or the OS error's message. +describe("readMacosKeychainSecret sink wiring", () => { + function recordingSink(): { sink: SecurityLogSink; events: SecurityLogEvent[] } { + const events: SecurityLogEvent[] = []; + return { + sink: { + write: (event): void => { + events.push(event); + }, + }, + events, + }; + } + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("emits one security.keychain.fallback event, with only the documented fields, when the keychain refuses immediately", () => { + const { sink, events } = recordingSink(); + + const read = readMacosKeychainSecret("svc", "acct", { + executable: DENIES, + platform: "darwin", + timeoutMs: 30_000, + sink, + }); + + expect(read).toEqual({ kind: "unavailable" }); + expect(events).toHaveLength(1); + const [event] = events; + expect(event).toMatchObject({ + level: "warn", + category: "security", + op: "security.keychain.fallback", + extra: { reasonKind: "Error", boundedExitKind: "exit-status" }, + }); + expect(typeof event?.durationMs).toBe("number"); + expect(event?.durationMs).toBeGreaterThanOrEqual(0); + expect(Object.keys(event ?? {}).sort()).toEqual([ + "category", + "durationMs", + "extra", + "level", + "op", + ]); + // Never the service/account this call was made with, never a path. + expect(JSON.stringify(event)).not.toContain("svc"); + expect(JSON.stringify(event)).not.toContain("acct"); + }); + + it("classifies a timed-out spawn distinctly from an immediate refusal", () => { + const { sink, events } = recordingSink(); + + readMacosKeychainSecret("svc", "acct", { + executable: HANGS, + platform: "darwin", + timeoutMs: 250, + sink, + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + op: "security.keychain.fallback", + extra: { reasonKind: "ETIMEDOUT", boundedExitKind: "timeout" }, + }); + }, 15_000); + + it("does not emit for the ordinary first-run case (item not found)", () => { + const { sink, events } = recordingSink(); + + const read = readMacosKeychainSecret("svc", "acct", { + executable: REFUSES, + platform: "darwin", + timeoutMs: 30_000, + sink, + }); + + expect(read).toEqual({ kind: "absent" }); + expect(events).toHaveLength(0); + }); + + it("does not emit when the call succeeds", () => { + const { sink, events } = recordingSink(); + + readMacosKeychainSecret("svc", "acct", { + executable: ANSWERS, + platform: "darwin", + timeoutMs: 30_000, + sink, + }); + + expect(events).toHaveLength(0); + }); + + it("degrades a throwing sink without ever failing the read (degrade-once idiom)", () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const dead: SecurityLogSink = { + write: (): never => { + throw new Error("sink transport is down"); + }, + }; + + const read = readMacosKeychainSecret("svc", "acct", { + executable: DENIES, + platform: "darwin", + timeoutMs: 30_000, + sink: dead, + }); + + expect(read).toEqual({ kind: "unavailable" }); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it("stays exactly as silent as before when no sink is wired", () => { + expect(() => { + readMacosKeychainSecret("svc", "acct", { + executable: DENIES, + platform: "darwin", + timeoutMs: 30_000, + }); + }).not.toThrow(); + }); +}); diff --git a/packages/keiko-security/src/macos-keychain.ts b/packages/keiko-security/src/macos-keychain.ts index 1914f3691b..56e15423c5 100644 --- a/packages/keiko-security/src/macos-keychain.ts +++ b/packages/keiko-security/src/macos-keychain.ts @@ -18,6 +18,13 @@ import { execFileSync } from "node:child_process"; +import { + emitSecurityLogEvent, + securityErrorKind, + startSecurityLogTimer, + type SecurityLogSink, +} from "./log-port.js"; + const MACOS_SECURITY_EXECUTABLE = "/usr/bin/security"; /** @@ -34,6 +41,14 @@ export interface MacosKeychainOptions { readonly timeoutMs?: number | undefined; /** Test seam, so the tier's behaviour is assertable on a non-darwin host. */ readonly platform?: NodeJS.Platform | undefined; + /** + * Optional activity-log seam (ADR-0019; see `log-port.ts`). When wired, a spawn failure that + * collapses to `{ kind: "unavailable" }` emits one `security.keychain.fallback` event instead of + * discarding the OS error silently — the gap the 0.3.0 boot-hang incident (see file header) + * exposed: the tier now returns in time, but still logged nothing about WHY it fell back. + * Omitted or `undefined` keeps this tier exactly as silent as before. + */ + readonly sink?: SecurityLogSink | undefined; } /** @@ -104,6 +119,44 @@ function readOutcome(error: unknown): MacosKeychainRead { return keychainItemNotFound(error) ? { kind: "absent" } : { kind: "unavailable" }; } +// Structural classification of HOW the bounded spawn ended, read from the same properties Node's +// `execFileSync` sets on a `spawnSync`-family failure — never from `message`, which can carry the +// resolved executable path. `"timeout"` is the one value an operator should treat as a recurrence +// of the 0.3.0 boot-hang incident (file header): the keychain did not answer inside the bound and +// had to be killed, exactly the shape a blocking OS modal produces. Closed vocabulary, verified +// against the shipped `execFileSync` error shape for a killed timeout, a plain non-zero exit, and a +// missing executable (ENOENT). +function classifyBoundedExit(error: unknown): string { + if (typeof error !== "object" || error === null) return "unknown"; + const e = error as { code?: unknown; signal?: unknown; status?: unknown }; + if (e.code === "ETIMEDOUT") return "timeout"; + if (typeof e.signal === "string") return "signal"; + if (typeof e.status === "number") return "exit-status"; + if (typeof e.code === "string") return "spawn-error"; + return "unknown"; +} + +// Exported so the sibling keychain surface in `secret-vault.ts` — which spawns `security` through +// its own injectable `KeychainCommandRunner` rather than calling `readMacosKeychainSecret` — can +// report the identical `security.keychain.fallback` shape instead of growing a second copy of the +// classification logic [GEN-MAINT-COUPLING-006]. +export function emitKeychainFallback( + sink: SecurityLogSink | undefined, + error: unknown, + elapsedMs: () => number, +): void { + emitSecurityLogEvent(sink, { + level: "warn", + category: "security", + op: "security.keychain.fallback", + durationMs: elapsedMs(), + extra: { + reasonKind: securityErrorKind(error), + boundedExitKind: classifyBoundedExit(error), + }, + }); +} + /** Reads the generic password for `service`/`account`. Never throws. */ export function readMacosKeychainSecret( service: string, @@ -112,6 +165,7 @@ export function readMacosKeychainSecret( ): MacosKeychainRead { const spawn = resolveSpawn(options); if (!spawn.darwin) return { kind: "unavailable" }; + const elapsedMs = startSecurityLogTimer(); try { const secret = execFileSync( spawn.executable, @@ -125,7 +179,9 @@ export function readMacosKeychainSecret( ).trim(); return { kind: "found", secret }; } catch (error) { - return readOutcome(error); + const outcome = readOutcome(error); + if (outcome.kind === "unavailable") emitKeychainFallback(options.sink, error, elapsedMs); + return outcome; } } diff --git a/packages/keiko-security/src/secret-vault.test.ts b/packages/keiko-security/src/secret-vault.test.ts index ebd75ca64c..40bddf44a1 100644 --- a/packages/keiko-security/src/secret-vault.test.ts +++ b/packages/keiko-security/src/secret-vault.test.ts @@ -13,8 +13,9 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { SecretboxError } from "./errors/secretbox.js"; +import type { SecurityLogEvent, SecurityLogSink } from "./log-port.js"; import { NO_LOCAL_VAULT_KEYCHAIN, SecretVaultStoreError, @@ -187,6 +188,102 @@ describe("resolveLocalVaultKey — KEYFILE tier", () => { }); }); +// gap g18: the key-source fact (which tier answered) was previously invisible even on the +// ordinary, no-fallback path. `security.vault.key-resolved` closes that — it fires every time a +// tier resolves, independent of whether anything went wrong. +describe("resolveLocalVaultKey — security.vault.key-resolved sink wiring", () => { + function recordingSink(): { sink: SecurityLogSink; events: SecurityLogEvent[] } { + const events: SecurityLogEvent[] = []; + return { sink: { write: (event): void => void events.push(event) }, events }; + } + + it("emits source=env, with only the documented fields, on the env tier", () => { + const { sink, events } = recordingSink(); + resolveLocalVaultKey({ + env: { KEIKO_TEST_VAULT_KEY: Buffer.alloc(32, 5).toString("base64") }, + vaultDir: dir, + envVarName: "KEIKO_TEST_VAULT_KEY", + keychainService: "keiko-test-vault", + keyfileName: "test-vault.key", + keychainAccess: NO_LOCAL_VAULT_KEYCHAIN, + sink, + }); + + expect(events).toHaveLength(1); + const [event] = events; + expect(event).toMatchObject({ + level: "info", + category: "security", + op: "security.vault.key-resolved", + extra: { source: "env" }, + }); + expect(Object.keys(event ?? {}).sort()).toEqual(["category", "extra", "level", "op"]); + }); + + it("emits source=keychain when the keychain tier answers", () => { + const { sink, events } = recordingSink(); + resolveLocalVaultKey({ + env: {}, + vaultDir: dir, + envVarName: "KEIKO_TEST_VAULT_KEY", + keychainService: "keiko-test-vault", + keyfileName: "test-vault.key", + keychainAccess: () => Buffer.alloc(32, 9), + sink, + }); + + expect(events).toEqual([ + expect.objectContaining({ op: "security.vault.key-resolved", extra: { source: "keychain" } }), + ]); + }); + + it("emits source=keyfile when env and keychain are both absent", () => { + const { sink, events } = recordingSink(); + resolveLocalVaultKey({ + env: {}, + vaultDir: dir, + envVarName: "KEIKO_TEST_VAULT_KEY", + keychainService: "keiko-test-vault", + keyfileName: "test-vault.key", + keychainAccess: NO_LOCAL_VAULT_KEYCHAIN, + sink, + }); + + expect(events).toEqual([ + expect.objectContaining({ op: "security.vault.key-resolved", extra: { source: "keyfile" } }), + ]); + }); + + it("emits nothing when the tier computation throws (a malformed env key)", () => { + const { sink, events } = recordingSink(); + expect(() => + resolveLocalVaultKey({ + env: { KEIKO_TEST_VAULT_KEY: Buffer.alloc(16, 3).toString("base64") }, + vaultDir: dir, + envVarName: "KEIKO_TEST_VAULT_KEY", + keychainService: "keiko-test-vault", + keyfileName: "test-vault.key", + keychainAccess: NO_LOCAL_VAULT_KEYCHAIN, + sink, + }), + ).toThrow("32 bytes"); + expect(events).toHaveLength(0); + }); + + it("stays exactly as silent as before when no sink is wired", () => { + expect(() => + resolveLocalVaultKey({ + env: {}, + vaultDir: dir, + envVarName: "KEIKO_TEST_VAULT_KEY", + keychainService: "keiko-test-vault", + keyfileName: "test-vault.key", + keychainAccess: NO_LOCAL_VAULT_KEYCHAIN, + }), + ).not.toThrow(); + }); +}); + // --------------------------------------------------------------------------- // createLocalSecretVault — CRUD behaviour // --------------------------------------------------------------------------- @@ -627,6 +724,73 @@ describe("createKeychainVaultKeyAccess", () => { }; expect(createKeychainVaultKeyAccess("svc", runner)()).toBeUndefined(); }); + + // The keychain key-tier path spawns `security` through its own injectable `KeychainCommandRunner` + // rather than calling `readMacosKeychainSecret`, so it needs its own proof that a read failure + // (other than "no such item") reports `security.keychain.fallback` — the same shape + // `readMacosKeychainSecret` reports for its own read tier (macos-keychain.test.ts). + describe("security.keychain.fallback sink wiring", () => { + function recordingSink(): { sink: SecurityLogSink; events: SecurityLogEvent[] } { + const events: SecurityLogEvent[] = []; + return { sink: { write: (event): void => void events.push(event) }, events }; + } + + it("emits one security.keychain.fallback event when the read refuses for a reason other than 'not found'", () => { + setPlatform("darwin"); + const { sink, events } = recordingSink(); + const runner = (): string => { + throw Object.assign(new Error("keychain refused"), { status: 45 }); + }; + + expect(createKeychainVaultKeyAccess("svc", runner, sink)()).toBeUndefined(); + + expect(events).toHaveLength(1); + const [event] = events; + expect(event).toMatchObject({ + level: "warn", + category: "security", + op: "security.keychain.fallback", + extra: { reasonKind: "Error", boundedExitKind: "exit-status" }, + }); + expect(typeof event?.durationMs).toBe("number"); + // Never the service name this call was made with. + expect(JSON.stringify(event)).not.toContain("svc"); + }); + + it("does not emit for the ordinary first-run case (item not found)", () => { + setPlatform("darwin"); + const { sink, events } = recordingSink(); + const runner = (args: readonly string[]): string => { + if (args[0] === "find-generic-password") throw itemNotFound(); + return ""; + }; + + const key = createKeychainVaultKeyAccess("svc", runner, sink)(); + + expect(key?.length).toBe(32); + expect(events).toHaveLength(0); + }); + + it("does not emit when the read succeeds", () => { + setPlatform("darwin"); + const { sink, events } = recordingSink(); + const stored = Buffer.alloc(32, 5).toString("base64"); + const runner = (): string => `${stored}\n`; + + createKeychainVaultKeyAccess("svc", runner, sink)(); + + expect(events).toHaveLength(0); + }); + + it("stays exactly as silent as before when no sink is wired", () => { + setPlatform("darwin"); + const runner = (): string => { + throw Object.assign(new Error("keychain refused"), { status: 45 }); + }; + + expect(() => createKeychainVaultKeyAccess("svc", runner)()).not.toThrow(); + }); + }); }); // --------------------------------------------------------------------------- @@ -760,6 +924,109 @@ describe("createShardedLocalSecretVault — CRUD parity with the single-file lay expect(vault.has("cred:a")).toBe(false); }); + function recordingSink(): { sink: SecurityLogSink; events: SecurityLogEvent[] } { + const events: SecurityLogEvent[] = []; + return { + sink: { + write: (event): void => { + events.push(event); + }, + }, + events, + }; + } + + it("emits one security.vault.shard-unreadable event, with only the documented fields, for an EISDIR shard", () => { + const storeDir = join(dir, "sharded"); + const { sink, events } = recordingSink(); + const vault = createShardedLocalSecretVault({ key: KEY, storeDir, sink }); + vault.set("cred:a", "secret-A"); + const [name] = readdirSync(storeDir); + // Same fixture as the unwired test above: a directory where the entry file belongs makes + // readFileSync raise EISDIR without depending on process privilege (unlike EACCES, which a + // root-run test worker would never actually be refused). + rmSync(join(storeDir, name ?? "missing")); + mkdirSync(join(storeDir, name ?? "missing"), { recursive: true }); + + expect(vault.get("cred:a")).toBeUndefined(); + + expect(events).toHaveLength(1); + const [event] = events; + expect(event).toMatchObject({ + level: "warn", + category: "security", + op: "security.vault.shard-unreadable", + errorKind: "Error", + extra: { count: 1 }, + }); + expect(Object.keys(event ?? {}).sort()).toEqual([ + "category", + "errorKind", + "extra", + "level", + "op", + ]); + // Never the reference, never the shard's filename/path. + expect(JSON.stringify(event)).not.toContain("cred:a"); + expect(JSON.stringify(event)).not.toContain(storeDir); + }); + + // EACCES specifically, as distinct proof from the EISDIR case above — skipped only when the + // worker itself runs as root (uid 0), where a permission bit never actually refuses a read. + it.skipIf(process.getuid?.() === 0)( + "emits security.vault.shard-unreadable for an EACCES shard, and never breaks the read when the sink throws", + () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + const storeDir = join(dir, "sharded"); + const dead: SecurityLogSink = { + write: (): never => { + throw new Error("sink transport is down"); + }, + }; + const vault = createShardedLocalSecretVault({ key: KEY, storeDir, sink: dead }); + vault.set("cred:a", "secret-A"); + const [name] = readdirSync(storeDir); + const shardPath = join(storeDir, name ?? "missing"); + chmodSync(shardPath, 0o000); + + let read: string | undefined; + try { + expect(() => { + read = vault.get("cred:a"); + }).not.toThrow(); + } finally { + chmodSync(shardPath, 0o600); + vi.restoreAllMocks(); + } + expect(read).toBeUndefined(); + // Degrade-once: the dead sink's own write threw, so the report fell through to the one + // channel left — exactly once, never silently and never per-line. + expect(warn).toHaveBeenCalledTimes(1); + }, + ); + + it("does not emit for an entry that simply was never set", () => { + const storeDir = join(dir, "sharded"); + const { sink, events } = recordingSink(); + const vault = createShardedLocalSecretVault({ key: KEY, storeDir, sink }); + + expect(vault.get("cred:never-set")).toBeUndefined(); + expect(events).toHaveLength(0); + }); + + it("stays exactly as silent as before when no sink is wired", () => { + const storeDir = join(dir, "sharded"); + const vault = createShardedLocalSecretVault({ key: KEY, storeDir }); + vault.set("cred:a", "secret-A"); + const [name] = readdirSync(storeDir); + rmSync(join(storeDir, name ?? "missing")); + mkdirSync(join(storeDir, name ?? "missing"), { recursive: true }); + + expect(() => { + vault.get("cred:a"); + }).not.toThrow(); + }); + it("never lists a filename it would refuse to read or delete under that reference", () => { const storeDir = join(dir, "sharded"); const vault = shardedVaultAt(storeDir); diff --git a/packages/keiko-security/src/secret-vault.ts b/packages/keiko-security/src/secret-vault.ts index a99e5c01f3..eaa2204074 100644 --- a/packages/keiko-security/src/secret-vault.ts +++ b/packages/keiko-security/src/secret-vault.ts @@ -43,7 +43,17 @@ import { isSealed, openString, sealString } from "./secretbox.js"; // pair; import it from the sibling module (relative — we ARE keiko-security). import { chmodIfPresent, ensureDirHardened, FILE_MODE } from "./fs-hardening.js"; // Shared keychain-spawn bound [GEN-MAINT-COUPLING-006], so the three surfaces cannot drift apart. -import { KEYCHAIN_SPAWN_TIMEOUT_MS, keychainItemNotFound } from "./macos-keychain.js"; +// `emitKeychainFallback` is the shared `security.keychain.fallback` reporter so this file's own +// keychain reader (below) reports the identical shape as `readMacosKeychainSecret`'s. +import { + emitKeychainFallback, + KEYCHAIN_SPAWN_TIMEOUT_MS, + keychainItemNotFound, +} from "./macos-keychain.js"; +// Independent activity-log seam (ADR-0019, w4a-security-log-port) and the hardened error-class +// classifier this package already applies to its persisted quarantine diagnostic. +import { emitSecurityLogEvent, startSecurityLogTimer, type SecurityLogSink } from "./log-port.js"; +import { hardenedErrorClass } from "./sqlite-corruption.js"; const KEY_BYTES = 32; const STORE_VERSION = 1; @@ -94,6 +104,18 @@ export interface ResolveLocalVaultKeyOptions { readonly keyfileName: string; // Test/non-darwin seam. Defaults to the real `security` CLI reader scoped to keychainService. readonly keychainAccess?: LocalVaultKeychainAccess | undefined; + /** + * Optional activity-log seam (ADR-0019; see `log-port.ts`), mirroring + * {@link ShardedLocalSecretVaultDeps.sink}. When wired: + * - one `security.vault.key-resolved` event fires every time a key tier resolves, carrying + * which tier won (`extra.source`) — visible even on the ordinary path where no fallback + * ever happens (gap g18); + * - a keychain read that fails for a reason other than "no such item" additionally emits + * `security.keychain.fallback`, the same shape `readMacosKeychainSecret` reports for its + * own read tier. + * Omitted or `undefined` keeps this resolver exactly as silent as before. + */ + readonly sink?: SecurityLogSink | undefined; } export interface LocalSecretVault { @@ -164,10 +186,12 @@ const defaultKeychainCommandRunner = createKeychainCommandRunner(); export function createKeychainVaultKeyAccess( keychainService: string, runCommand: KeychainCommandRunner = defaultKeychainCommandRunner, + sink?: SecurityLogSink, ): LocalVaultKeychainAccess { return (): Buffer | undefined => { if (process.platform !== "darwin") return undefined; const account = userInfo().username; + const elapsedMs = startSecurityLogTimer(); let found: string; try { found = runCommand([ @@ -182,9 +206,11 @@ export function createKeychainVaultKeyAccess( // Only "the item is not there" invites a write. A read that timed out, or a keychain that // refused outright, means a store attempt would meet the same wall and spend a SECOND bounded // wait — on a path a caller may be blocking on. Same rule, same predicate, as the shared owner. - return keychainItemNotFound(error) - ? generateKeychainKey(keychainService, account, runCommand) - : undefined; + if (keychainItemNotFound(error)) { + return generateKeychainKey(keychainService, account, runCommand); + } + emitKeychainFallback(sink, error, elapsedMs); + return undefined; } try { return decodeKeyOrThrow(found); @@ -237,10 +263,24 @@ function keyFromKeyfile(vaultDir: string, keyfileName: string): Buffer { } export function resolveLocalVaultKey(options: ResolveLocalVaultKeyOptions): ResolvedLocalVaultKey { + const resolved = computeLocalVaultKey(options); + // Fires only once the tier has genuinely resolved — a thrown error (a malformed env key, a + // symlinked keyfile path) skips this and reports nothing, exactly as before this event existed. + emitSecurityLogEvent(options.sink, { + level: "info", + category: "security", + op: "security.vault.key-resolved", + extra: { source: resolved.source }, + }); + return resolved; +} + +function computeLocalVaultKey(options: ResolveLocalVaultKeyOptions): ResolvedLocalVaultKey { const fromEnv = keyFromEnv(options.env, options.envVarName); if (fromEnv !== undefined) return { key: fromEnv, source: "env" }; const keychainAccess = - options.keychainAccess ?? createKeychainVaultKeyAccess(options.keychainService); + options.keychainAccess ?? + createKeychainVaultKeyAccess(options.keychainService, undefined, options.sink); const fromKeychain = keychainAccess(); if (fromKeychain !== undefined) return { key: fromKeychain, source: "keychain" }; return { key: keyFromKeyfile(options.vaultDir, options.keyfileName), source: "keyfile" }; @@ -403,6 +443,14 @@ export interface ShardedLocalSecretVaultDeps { readonly key: Buffer; // Directory that holds this vault's sealed entry files. It is created hardened on first write. readonly storeDir: string; + /** + * Optional activity-log seam (ADR-0019; see `log-port.ts`). When wired, a shard file that fails + * to read for a reason other than the symlink guarantee (EACCES, EISDIR, EIO — previously + * collapsed to `undefined`, indistinguishable from "never set") emits one + * `security.vault.shard-unreadable` event instead of failing silently. Omitted or `undefined` + * keeps this layout exactly as silent as before. + */ + readonly sink?: SecurityLogSink | undefined; } // References are opaque, NON-SECRET identifiers, so the filename may carry one — the single-file @@ -469,7 +517,32 @@ function writeShard(dir: string, filePath: string, envelope: string): void { } } -function readShardEnvelope(filePath: string): string | undefined { +// ENOENT is the ORDINARY case — no such shard file, no such store directory yet, exactly what +// every never-set reference looks like — and must never itself produce a log line, or every plain +// miss on a fresh vault would emit one. Only a read that fails for a reason OTHER than "not there" +// (EISDIR, EACCES, EIO) is the genuine unreadable-entry case this event exists to surface. +function isEnoent(cause: unknown): boolean { + return ( + typeof cause === "object" && + cause !== null && + "code" in cause && + (cause as NodeJS.ErrnoException).code === "ENOENT" + ); +} + +function emitShardUnreadable(sink: SecurityLogSink | undefined, cause: unknown): void { + emitSecurityLogEvent(sink, { + level: "warn", + category: "security", + op: "security.vault.shard-unreadable", + errorKind: hardenedErrorClass(cause), + // A single unreadable file per call; never the filename (it decodes to the reference) or the + // read error's message (it can carry the resolved path). + extra: { count: 1 }, + }); +} + +function readShardEnvelope(filePath: string, sink?: SecurityLogSink): string | undefined { // The symlink refusal is a guarantee and still throws. A read that fails for any other reason // (EISDIR, EACCES, EIO) is one unreadable entry, which says nothing about the others — reporting // it as absent is what this layout documents, and what `get`/`has` promise their callers. @@ -477,7 +550,8 @@ function readShardEnvelope(filePath: string): string | undefined { let envelope: string; try { envelope = readFileSync(filePath, "utf8"); - } catch { + } catch (cause) { + if (!isEnoent(cause)) emitShardUnreadable(sink, cause); return undefined; } return isSealed(envelope) ? envelope : undefined; @@ -522,7 +596,7 @@ function listShardReferences(dir: string): readonly string[] { * bodies against its own index. */ export function createShardedLocalSecretVault(deps: ShardedLocalSecretVaultDeps): LocalSecretVault { - const { key } = deps; + const { key, sink } = deps; const storeDir = resolve(deps.storeDir); // A reference with no representable filename can hold no entry, so reads and deletes report // "absent" rather than throwing — matching the single-file layout, where an unknown reference is @@ -557,7 +631,7 @@ export function createShardedLocalSecretVault(deps: ShardedLocalSecretVaultDeps) const envelopeFor = (reference: string): string | undefined => { const filePath = readPath(reference); - return filePath === undefined ? undefined : readShardEnvelope(filePath); + return filePath === undefined ? undefined : readShardEnvelope(filePath, sink); }; return { diff --git a/packages/keiko-security/src/sqlite-corruption.ts b/packages/keiko-security/src/sqlite-corruption.ts index 2758b2262b..af037b8271 100644 --- a/packages/keiko-security/src/sqlite-corruption.ts +++ b/packages/keiko-security/src/sqlite-corruption.ts @@ -88,7 +88,11 @@ const ERROR_CLASS_SHAPE = /^[A-Za-z][A-Za-z0-9_$]*$/; // Truncation happens AFTER redaction: cutting first could split a secret so that no pattern matches // the remaining prefix, and the point of the cap is size, not concealment. -function boundedRedactedText(value: string): string { +// +// Exported so `log-port.ts`'s callers (`macos-keychain.ts`, `secret-vault.ts`) can bound an +// activity-log field with the exact same redaction and cap this module already applies to the +// persisted quarantine diagnostic, rather than growing a second copy [w4a-security-log-port]. +export function boundedRedactedText(value: string): string { const redacted = redact(value); return redacted.length <= MAX_ERROR_RECORD_TEXT_CHARS ? redacted @@ -96,7 +100,8 @@ function boundedRedactedText(value: string): string { } // Reflective reads over a thrown value are hostile-input reads: a getter or proxy trap may throw. -function safeName(value: object): string | undefined { +// Exported for the same reuse reason as `boundedRedactedText` above [w4a-security-log-port]. +export function safeName(value: object): string | undefined { try { const name: unknown = Reflect.get(value, "name"); return typeof name === "string" ? name : undefined; @@ -107,7 +112,11 @@ function safeName(value: object): string | undefined { // The class label comes from code (a class declaration), never from request data — but only once it is // shape- and length-checked, because `name` is writable. Anything else degrades to "Error". -function hardenedErrorClass(cause: unknown): string { +// +// Exported so `secret-vault.ts`'s `readShardEnvelope` can classify an fs failure (EACCES, EISDIR, +// EIO) for `security.vault.shard-unreadable` with the same hardened classifier this module already +// uses for the persisted quarantine diagnostic [w4a-security-log-port]. +export function hardenedErrorClass(cause: unknown): string { if (!(cause instanceof Error)) return typeof cause; const name = safeName(cause); if (name === undefined || name.length > MAX_ERROR_CLASS_CHARS || !ERROR_CLASS_SHAPE.test(name)) { diff --git a/packages/keiko-server/src/atlassian/credentialVault.ts b/packages/keiko-server/src/atlassian/credentialVault.ts index abb63c6ba8..dc3c4de03d 100644 --- a/packages/keiko-server/src/atlassian/credentialVault.ts +++ b/packages/keiko-server/src/atlassian/credentialVault.ts @@ -19,6 +19,8 @@ import { type LocalSecretVault, type LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; +// Type-only: the sink itself is supplied by the composition root (wiring.ts), never resolved here. +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import type { EnvSource } from "@oscharko-dev/keiko-model-gateway"; const CREDENTIALS_SUBDIR = "credentials"; @@ -41,6 +43,8 @@ export interface OpenAtlassianCredentialVaultOptions { // Test/non-darwin seam (mirrors the provider vault): inject NO_LOCAL_VAULT_KEYCHAIN to force // the keyfile tier deterministically without touching the real login keychain. readonly keychainAccess?: LocalVaultKeychainAccess | undefined; + // Optional activity-log seam (ADR-0019); wired with `processServerLogSink()` in wiring.ts. + readonly securityLogSink?: SecurityLogSink | undefined; } export function openAtlassianCredentialVault( @@ -54,6 +58,7 @@ export function openAtlassianCredentialVault( keychainService: ATLASSIAN_KEYCHAIN_SERVICE, keyfileName: ATLASSIAN_KEYFILE, ...(options.keychainAccess !== undefined ? { keychainAccess: options.keychainAccess } : {}), + sink: options.securityLogSink, }); return createLocalSecretVault({ key, diff --git a/packages/keiko-server/src/atlassian/wiring.ts b/packages/keiko-server/src/atlassian/wiring.ts index d47409556a..453f7c26e0 100644 --- a/packages/keiko-server/src/atlassian/wiring.ts +++ b/packages/keiko-server/src/atlassian/wiring.ts @@ -15,6 +15,7 @@ import type { LocalSecretVault, LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import type { EnvSource } from "@oscharko-dev/keiko-model-gateway"; import type { OutboundHttpEgressConfig } from "@oscharko-dev/keiko-model-gateway/internal/http"; import { openAtlassianCredentialVault } from "./credentialVault.js"; @@ -33,6 +34,9 @@ export interface BuildAtlassianConnectorCredentialDepsOptions { // tier; production leaves it undefined so the darwin keychain tier stays available. readonly keychainAccess?: LocalVaultKeychainAccess | undefined; readonly fetchImpl?: typeof fetch | undefined; + // Optional activity-log seam (ADR-0019); the deps.ts composition root supplies + // `processServerLogSink()`. + readonly securityLogSink?: SecurityLogSink | undefined; } // Lazy vault port: key resolution (which may create a keychain entry or keyfile) happens on the @@ -46,6 +50,7 @@ function lazyVaultPort( configPath: options.configPath, env: options.env, ...(options.keychainAccess === undefined ? {} : { keychainAccess: options.keychainAccess }), + securityLogSink: options.securityLogSink, })); return { get: (reference: string): string | undefined => open().get(reference), diff --git a/packages/keiko-server/src/conversation-attachment-store.test.ts b/packages/keiko-server/src/conversation-attachment-store.test.ts index db34f41e8e..79e7c04b79 100644 --- a/packages/keiko-server/src/conversation-attachment-store.test.ts +++ b/packages/keiko-server/src/conversation-attachment-store.test.ts @@ -1,10 +1,11 @@ -import { mkdtempSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync } from "node:fs"; import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { MAX_ATTACHMENT_MIME_BYTES } from "@oscharko-dev/keiko-contracts"; import type { LocalSecretVault } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogEvent, SecurityLogSink } from "@oscharko-dev/keiko-security"; import { ConversationAttachmentStoreError, createConversationAttachmentStore, @@ -557,3 +558,41 @@ describe("conversation attachment store", () => { expect(vault.has(CORRUPT_REF)).toBe(true); }); }); + +// Wiring test for `securityLogSink` (Wave 4a, epic #3233 §8): every test above that exercises the +// REAL sharded vault supplies `KEIKO_CONVERSATION_ATTACHMENT_KEY` (env tier) so it never touches +// the keychain, and none supplies `securityLogSink` — this is the one test that forces the +// sharded vault's OWN failure mode, `security.vault.shard-unreadable`, and proves it reaches the +// caller's sink. `options.vault` is deliberately NOT injected: that seam bypasses +// `createShardedLocalSecretVault` entirely, so it cannot exercise this wiring. +// +// THE FAILURE THIS PINS: dropping `sink: options.securityLogSink` from the `createShardedLocalSecretVault` +// call in `createConversationAttachmentStore` (`conversation-attachment-store.ts`) makes `events` +// stay empty below. +describe("createConversationAttachmentStore — securityLogSink wiring to the sharded vault", () => { + it("records shard-unreadable when a shard file cannot be read for a reason other than absent", () => { + const root = realpathSync(mkdtempSync(join(tmpdir(), "keiko-chat-attachments-secloG-"))); + const events: SecurityLogEvent[] = []; + const sink: SecurityLogSink = { write: (event): void => void events.push(event) }; + const store = createConversationAttachmentStore({ + runtimeStateDir: root, + env: { KEIKO_CONVERSATION_ATTACHMENT_KEY: Buffer.alloc(32, 9).toString("base64") }, + securityLogSink: sink, + }); + + const uploaded = store.put({ ...binding(), bytes: BYTES }); + + // Replace the just-written shard FILE with a DIRECTORY of the same name: the next read fails + // with EISDIR, a reason other than "absent" (ENOENT), which is exactly what the sharded vault's + // own `readShardEnvelope` treats as "one unreadable entry" and reports on the sink. + const shardName = `entry-${Buffer.from(uploaded.ref, "utf8").toString("hex")}.sealed`; + const shardPath = join(root, "conversation-attachments", shardName); + rmSync(shardPath, { force: true }); + mkdirSync(shardPath); + + expect(() => store.resolve(uploaded.ref, binding())).toThrow(ConversationAttachmentStoreError); + expect(events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.shard-unreadable" }), + ); + }); +}); diff --git a/packages/keiko-server/src/conversation-attachment-store.ts b/packages/keiko-server/src/conversation-attachment-store.ts index 04ce9a3e14..78d447b56b 100644 --- a/packages/keiko-server/src/conversation-attachment-store.ts +++ b/packages/keiko-server/src/conversation-attachment-store.ts @@ -10,6 +10,7 @@ import { resolveLocalVaultKey, type LocalSecretVault, } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; const STORE_DIR = "conversation-attachments"; const KEY_ENV = "KEIKO_CONVERSATION_ATTACHMENT_KEY"; @@ -68,6 +69,14 @@ export interface CreateConversationAttachmentStoreOptions { readonly ttlMs?: number | undefined; /** Maximum aggregate decoded attachment-content bytes retained as live records. */ readonly totalContentBytes?: number | undefined; + /** + * Optional activity-log seam (ADR-0019, Wave 4a epic #3233 §8; see + * `@oscharko-dev/keiko-security/log-port.ts`). When wired, a shard file this vault cannot read + * for a reason other than "absent" emits one `security.vault.shard-unreadable` event instead of + * failing silently. Ignored when `vault` is injected directly (tests). Omitted or `undefined` + * keeps the vault exactly as silent as before this change. + */ + readonly securityLogSink?: SecurityLogSink | undefined; } export class ConversationAttachmentStoreError extends Error { @@ -368,8 +377,10 @@ export function createConversationAttachmentStore( envVarName: KEY_ENV, keychainService: KEY_SERVICE, keyfileName: KEY_FILE, + sink: options.securityLogSink, }).key, storeDir: join(options.runtimeStateDir, STORE_DIR), + sink: options.securityLogSink, }); return cachedVault; }; diff --git a/packages/keiko-server/src/credentialPersistence.ts b/packages/keiko-server/src/credentialPersistence.ts index f8f6aeccbf..bcd04d5a77 100644 --- a/packages/keiko-server/src/credentialPersistence.ts +++ b/packages/keiko-server/src/credentialPersistence.ts @@ -15,6 +15,7 @@ import { existsSync, readFileSync } from "node:fs"; import type { EnvSource } from "@oscharko-dev/keiko-model-gateway"; import type { LocalVaultKeychainAccess } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import { savePrivateJson } from "./private-json.js"; import { hasPlaintextGatewayCredentials, @@ -32,6 +33,9 @@ export interface SealGatewayConfigContext { readonly evidenceDir: string; readonly keychainAccess?: LocalVaultKeychainAccess | undefined; readonly figmaKeychainAccess?: FigmaKeychainAccess | undefined; + // Optional activity-log seam (ADR-0019); the deps.ts/gateway-setup.ts composition roots supply + // `processServerLogSink()`. + readonly securityLogSink?: SecurityLogSink | undefined; } function isRecord(value: unknown): value is Record { @@ -77,6 +81,7 @@ export function persistSealedGatewayConfig( env: ctx.env, configPath: ctx.storagePath, ...(ctx.keychainAccess !== undefined ? { keychainAccess: ctx.keychainAccess } : {}), + securityLogSink: ctx.securityLogSink, }); const withSealedProviders = { ...raw, providers: sealedProviders.providers }; const withSealedCredentials = @@ -90,6 +95,7 @@ export function persistSealedGatewayConfig( env: ctx.env, configPath: ctx.storagePath, ...(ctx.keychainAccess !== undefined ? { keychainAccess: ctx.keychainAccess } : {}), + securityLogSink: ctx.securityLogSink, }, sealedProviders.activeSecretRefs, ); @@ -101,6 +107,9 @@ export interface MigrateCredentialsOptions { readonly evidenceDir: string; readonly keychainAccess?: LocalVaultKeychainAccess | undefined; readonly figmaKeychainAccess?: FigmaKeychainAccess | undefined; + // Optional activity-log seam (ADR-0019); the deps.ts composition root supplies + // `processServerLogSink()`. + readonly securityLogSink?: SecurityLogSink | undefined; } export interface MigrateCredentialsOutcome { @@ -131,6 +140,7 @@ export function migrateLocalConfigCredentials( ...(options.figmaKeychainAccess !== undefined ? { figmaKeychainAccess: options.figmaKeychainAccess } : {}), + securityLogSink: options.securityLogSink, }); return { migrated: true }; } catch { diff --git a/packages/keiko-server/src/credentialVault.ts b/packages/keiko-server/src/credentialVault.ts index 8803a162e1..654643d474 100644 --- a/packages/keiko-server/src/credentialVault.ts +++ b/packages/keiko-server/src/credentialVault.ts @@ -21,6 +21,12 @@ import { type LocalSecretVault, type LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; +// Type-only: this module is also published under the lightweight `credential-vault` subpath (see +// package.json) that `keiko repair`/`keiko run` import without the rest of the BFF runtime, so it +// must never pick up a VALUE import of `process-log-sink.ts`/`observability`. A type import erases +// at compile time and costs nothing there; the real `processServerLogSink()` call happens only at +// the full-server composition sites (deps.ts, gateway-setup.ts, credentialPersistence.ts). +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import type { EnvSource } from "@oscharko-dev/keiko-model-gateway"; // Structurally identical to the gateway's ProviderSecretResolver (kept local so keiko-server does not @@ -58,6 +64,13 @@ export interface OpenCredentialVaultOptions { readonly configPath: string; readonly env: EnvSource; readonly keychainAccess?: LocalVaultKeychainAccess | undefined; + /** + * Optional activity-log seam (ADR-0019; see `@oscharko-dev/keiko-security/log-port.ts`). Wired + * with `processServerLogSink()` at the full-server composition sites only — the `keiko repair`/ + * `keiko run` CLI paths that import this module through the lightweight `credential-vault` + * subpath omit it and stay exactly as silent as before. + */ + readonly securityLogSink?: SecurityLogSink | undefined; } export function openProviderCredentialVault(options: OpenCredentialVaultOptions): LocalSecretVault { @@ -69,6 +82,7 @@ export function openProviderCredentialVault(options: OpenCredentialVaultOptions) keychainService: CREDENTIALS_KEYCHAIN_SERVICE, keyfileName: CREDENTIALS_KEYFILE, ...(options.keychainAccess !== undefined ? { keychainAccess: options.keychainAccess } : {}), + sink: options.securityLogSink, }); return createLocalSecretVault({ key, storePath: credentialStorePath(options.configPath) }); } @@ -148,6 +162,7 @@ export interface SealProviderApiKeysOptions { readonly env: EnvSource; readonly configPath: string; readonly keychainAccess?: LocalVaultKeychainAccess | undefined; + readonly securityLogSink?: SecurityLogSink | undefined; } export interface SealedProviderApiKeys { diff --git a/packages/keiko-server/src/deps-attachment-history-securitylog-wiring.test.ts b/packages/keiko-server/src/deps-attachment-history-securitylog-wiring.test.ts new file mode 100644 index 0000000000..683bb0c2ec --- /dev/null +++ b/packages/keiko-server/src/deps-attachment-history-securitylog-wiring.test.ts @@ -0,0 +1,171 @@ +// Wiring test for `buildUiHandlerDeps`'s composition of the conversation-attachment store and the +// editor local-history store (Wave 4a, epic #3233 §8). +// +// WHAT THIS PINS +// +// Both stores' own `securityLogSink` → `createShardedLocalSecretVault` wiring is already pinned, +// with its own FAILS-BEFORE/PASSES-AFTER proof, in `conversation-attachment-store.test.ts` and +// `editor/localHistory/localHistoryStore.test.ts`. This file pins the ONE remaining link: that the +// real composition root (`deps.ts`'s `buildBaseUiHandlerDeps`) actually supplies +// `securityLogSink: processServerLogSink()` when building each store's production default — the +// same #3230-class regression (a port declared but never wired at composition) the other sites in +// this wave guard against. +// +// THE FAILURE THIS PINS: dropping either `securityLogSink: processServerLogSink()` line from +// `buildBaseUiHandlerDeps` (`deps.ts`) leaves the corresponding store silent on a forced +// shard-unreadable failure, and the matching assertion below fails. + +import { mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { buildUiHandlerDeps } from "./deps.js"; +import { inspectWorkspaceRootIdentity } from "./workspace-root-identity.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; + +const tmpDirs: string[] = []; + +function tmp(prefix: string): string { + const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix))); + tmpDirs.push(dir); + return dir; +} + +let sink: BufferedServerLogSink; + +beforeEach(() => { + sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); +}); + +afterEach(() => { + resetServerLogger(); + for (const dir of tmpDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +// Finds the ONE file under `root` whose name matches the sharded-vault filename shape +// (`entry-.sealed`) — avoids hard-coding either store's internal directory layout. +function findShardFile(root: string): string | undefined { + for (const entry of readdirSync(root, { recursive: true, withFileTypes: true })) { + if (entry.isFile() && /^entry-[0-9a-f]+\.sealed$/u.test(entry.name)) { + return join(entry.parentPath, entry.name); + } + } + return undefined; +} + +// Replaces a shard FILE with a DIRECTORY of the same name: the next read fails with EISDIR, a +// reason other than "absent" (ENOENT) — exactly what the sharded vault's own `readShardEnvelope` +// treats as "one unreadable entry" and reports on the sink, rather than treating it as never-set. +function corruptOneShard(root: string): void { + const shardPath = findShardFile(root); + if (shardPath === undefined) throw new Error("fixture wrote no shard file"); + rmSync(shardPath, { force: true }); + mkdirSync(shardPath); +} + +describe("buildUiHandlerDeps — conversationAttachmentStore wires securityLogSink", () => { + it("records shard-unreadable on server.log for a forced non-ENOENT shard failure", () => { + const uiDir = tmp("deps-attach-secloG-"); + const evidenceDir = tmp("deps-attach-secloG-ev-"); + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + }); + if (deps.conversationAttachmentStore === undefined) { + throw new Error("production wiring did not build a conversationAttachmentStore"); + } + try { + const bytes = Buffer.from("PNGX"); + const binding = { + sessionId: "session-1", + sessionRotationCount: 0, + projectPath: "/workspace/project", + chatId: "chat-1", + mimeType: "image/png", + sizeBytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }; + const put = deps.conversationAttachmentStore.put({ ...binding, bytes }); + // The first write resolves the vault's key exactly once (memoized thereafter), so the + // key-tier resolution's own `sink` wiring (gap g18, epic #3233 §8) is provable from the + // same call that already proves the sharded vault's own sink above. + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.key-resolved" }), + ); + corruptOneShard(uiDir); + + expect(() => deps.conversationAttachmentStore?.resolve(put.ref, binding)).toThrow(); + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.shard-unreadable" }), + ); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); + +describe("buildUiHandlerDeps — editorLocalHistoryStore wires securityLogSink", () => { + it("records shard-unreadable on server.log for a forced non-ENOENT shard failure", () => { + const uiDir = tmp("deps-history-secloG-"); + const evidenceDir = tmp("deps-history-secloG-ev-"); + const workspaceRoot = tmp("deps-history-secloG-ws-"); + mkdirSync(join(workspaceRoot, "src")); + writeFileSync(join(workspaceRoot, "src", "app.ts"), "initial\n", "utf8"); + const identity = inspectWorkspaceRootIdentity(workspaceRoot); + if (identity.objectIdentityDigest === undefined) { + throw new Error("fixture filesystem has no durable object identity"); + } + + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + }); + if (deps.editorLocalHistoryStore === undefined) { + throw new Error("production wiring did not build an editorLocalHistoryStore"); + } + try { + const scope = { + workspaceId: "workspace-deps-secloG-test", + rootRef: identity.rootRef, + rootIdentityDigest: identity.identityDigest, + objectIdentityDigest: identity.objectIdentityDigest, + }; + const absolutePath = join(workspaceRoot, "src", "app.ts"); + const captured = deps.editorLocalHistoryStore.capture({ + ...scope, + realRoot: workspaceRoot, + relativePath: "src/app.ts", + absolutePath, + content: "wired\n", + origin: "user-save", + nowMs: 1_000, + }).entry; + // The first capture resolves the vault's key exactly once (memoized thereafter), so the + // key-tier resolution's own `sink` wiring (gap g18, epic #3233 §8) is provable from the same + // call that already proves the sharded vault's own sink above. + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.key-resolved" }), + ); + corruptOneShard(uiDir); + + expect(() => deps.editorLocalHistoryStore?.read(scope, captured.entryRef, 2_000)).toThrow(); + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.shard-unreadable" }), + ); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); diff --git a/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts b/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts new file mode 100644 index 0000000000..4d439bd8e6 --- /dev/null +++ b/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts @@ -0,0 +1,268 @@ +// Wiring test for `buildUiHandlerDeps`'s composition of the five keychain key-tier callers that, +// until this change, could resolve a local vault key (env -> keychain -> keyfile, +// `@oscharko-dev/keiko-security/secret-vault`'s `resolveLocalVaultKey`) without ever being able to +// report which tier answered or that the keychain tier fell back (Wave 4a, epic #3233 §8, gap g18): +// `editorHotExitStore`, `localKnowledgeKeyProvider`, `atlassianConnectorCredentials`, +// `workspaceIndexForRoot`, and the provider-credential vault reached through +// `migrateLocalConfigCredentials`/`createProviderSecretResolver` inside `loadRuntimeGatewayConfig`. +// +// WHAT THIS PINS +// +// Each caller's own `securityLogSink` -> `resolveLocalVaultKey({ sink })` wiring is unit-pinned in +// its own file (`credentialVault.test.ts`-adjacent files do not yet exist for every caller, so the +// closed-vocabulary and fallback shapes are pinned once, centrally, in +// `@oscharko-dev/keiko-security/secret-vault.test.ts`). This file pins the ONE remaining link per +// caller: that the real composition root (`deps.ts`) actually supplies +// `securityLogSink: processServerLogSink()` when building each caller's production default — the +// same #3230-class regression (a port declared but never wired at composition) the sibling +// `deps-attachment-history-securitylog-wiring.test.ts` guards against for the two callers wired in +// an earlier part of this wave. +// +// `@oscharko-dev/keiko-security/secret-vault` is module-mocked to CAPTURE every `resolveLocalVaultKey` +// call's options (real behaviour is preserved via `importOriginal` + delegation) so each assertion +// below can inspect exactly what `deps.ts` supplied, without needing to force a real keychain +// failure or touch the developer's login keychain — hermetic per AGENTS.md. +// +// THE FAILURE THIS PINS: dropping any one `securityLogSink: processServerLogSink()` line from +// `deps.ts` leaves the matching call's `sink` field `undefined`, and that caller's assertion below +// fails. + +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + EDITOR_HOT_EXIT_SCHEMA_VERSION, + type EditorHotExitSnapshotV1, +} from "@oscharko-dev/keiko-contracts"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; +import { createInMemoryUiStore } from "./store/index.js"; + +type ResolveLocalVaultKeyOptions = Parameters< + typeof import("@oscharko-dev/keiko-security/secret-vault").resolveLocalVaultKey +>[0]; + +let calls: ResolveLocalVaultKeyOptions[]; + +vi.mock("@oscharko-dev/keiko-security/secret-vault", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveLocalVaultKey: ( + options: ResolveLocalVaultKeyOptions, + ): ReturnType => { + calls.push(options); + return actual.resolveLocalVaultKey(options); + }, + }; +}); + +// Imported AFTER the mock declaration so `deps.ts`'s internal `resolveLocalVaultKey` import (via +// every caller it composes) binds to the capturing wrapper above. +const { buildUiHandlerDeps } = await import("./deps.js"); + +const tmpDirs: string[] = []; + +function tmp(prefix: string): string { + const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix))); + tmpDirs.push(dir); + return dir; +} + +let sink: BufferedServerLogSink; + +beforeEach(() => { + calls = []; + sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); +}); + +afterEach(() => { + resetServerLogger(); + for (const dir of tmpDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +// Every call captured for `envVarName` must carry a `sink` — checked with `every`, not merely the +// first match, because the provider-credential vault is reached through TWO independent call sites +// (`migrateLocalConfigCredentials` and `createProviderSecretResolver`) that share one env var name; +// `.find()` would let either site's wiring regress unnoticed behind the other's. Each sink must not +// merely be present but IS the process-wide activity log: a write through it must reach +// `server.log`, exactly like a real fallback would. +function expectRealServerLogSink(envVarName: string, op: string): void { + const matching = calls.filter((c) => c.envVarName === envVarName); + expect(matching.length).toBeGreaterThan(0); + expect(matching.every((c) => c.sink !== undefined)).toBe(true); + matching[0]?.sink?.write({ level: "info", category: "security", op, extra: {} }); + expect(sink.events).toContainEqual(expect.objectContaining({ category: "security", op })); +} + +function hotExitSnapshot(): EditorHotExitSnapshotV1 { + return { + schemaVersion: EDITOR_HOT_EXIT_SCHEMA_VERSION, + workspaceRoot: "/repo", + relativePath: "src/app.ts", + content: "const x = 1;\n", + baseVersion: { sizeBytes: 16, modifiedAt: 1, contentHash: "a".repeat(64) }, + contentHash: "b".repeat(64), + savedContentHash: "a".repeat(64), + updatedAt: 1_000, + paneId: "pane-1", + windowId: "editor-1", + }; +} + +// The composition root types these surfaces as optional (a deployment may omit them); a wiring +// test is about the deployment that HAS them, so their absence is a failure, not a skip. +function present(value: T | undefined, name: string): T { + if (value === undefined) throw new Error(`${name} was not composed`); + return value; +} + +describe("buildUiHandlerDeps — editorHotExitStore wires resolveLocalVaultKey's sink", () => { + it("supplies a real server-log sink to the hot-exit vault's key resolution", () => { + const uiDir = tmp("deps-hotexit-vaultkey-"); + const evidenceDir = tmp("deps-hotexit-vaultkey-ev-"); + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + }); + try { + const snapshot = hotExitSnapshot(); + const hotExitStore = present(deps.editorHotExitStore, "editorHotExitStore"); + const ref = hotExitStore.snapshotRefFor(snapshot.workspaceRoot, snapshot.relativePath); + hotExitStore.write(snapshot, ref); + + expectRealServerLogSink("KEIKO_EDITOR_HOT_EXIT_KEY", "security.vault.key-resolved"); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); + +describe("buildUiHandlerDeps — localKnowledgeKeyProvider wires resolveLocalVaultKey's sink", () => { + it("supplies a real server-log sink to the knowledge store's key resolution", () => { + const uiDir = tmp("deps-knowledge-vaultkey-"); + const evidenceDir = tmp("deps-knowledge-vaultkey-ev-"); + const knowledgeDir = tmp("deps-knowledge-vaultkey-store-"); + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + }); + try { + present(deps.localKnowledgeKeyProvider, "localKnowledgeKeyProvider").resolveKey({ + dbPath: join(knowledgeDir, "capsules.db"), + schemaVersion: 1, + }); + + expectRealServerLogSink("KEIKO_LOCAL_KNOWLEDGE_KEY", "security.vault.key-resolved"); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); + +describe("buildUiHandlerDeps — workspaceIndexForRoot wires resolveLocalVaultKey's sink", () => { + it("supplies a real server-log sink to the workspace index's key resolution", () => { + const uiDir = tmp("deps-wsindex-vaultkey-"); + const evidenceDir = tmp("deps-wsindex-vaultkey-ev-"); + const workspaceRoot = tmp("deps-wsindex-vaultkey-root-"); + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + }); + try { + if (deps.workspaceIndexForRoot === undefined) { + throw new Error("production wiring did not build a workspaceIndexForRoot"); + } + deps.workspaceIndexForRoot(workspaceRoot); + + expectRealServerLogSink("KEIKO_WORKSPACE_INDEX_KEY", "security.vault.key-resolved"); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); + +describe("buildUiHandlerDeps — atlassianConnectorCredentials wires resolveLocalVaultKey's sink", () => { + it("supplies a real server-log sink to the Atlassian custody vault's key resolution", () => { + const uiDir = tmp("deps-atlassian-vaultkey-"); + const evidenceDir = tmp("deps-atlassian-vaultkey-ev-"); + const configDir = tmp("deps-atlassian-vaultkey-cfg-"); + const deps = buildUiHandlerDeps({ + configPath: join(configDir, "keiko.config.json"), + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + }); + try { + if (deps.atlassianConnectorCredentials === undefined) { + throw new Error("production wiring did not build atlassianConnectorCredentials"); + } + deps.atlassianConnectorCredentials.custody.create({ + provider: "jira", + displayName: "Wiring Test Jira", + baseUrl: "https://wiring-test.example.com", + authScheme: "basic-api-token", + accountEmail: "wiring-test@example.com", + apiToken: ["synthetic", "wiring", "token", "0123456789"].join("-"), + }); + + expectRealServerLogSink( + "KEIKO_ATLASSIAN_CONNECTOR_CREDENTIALS_KEY", + "security.vault.key-resolved", + ); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); + +describe("buildUiHandlerDeps — provider-credential vault wires resolveLocalVaultKey's sink", () => { + it("supplies a real server-log sink when migrating a plaintext config into the vault", () => { + const uiDir = tmp("deps-credvault-vaultkey-"); + const evidenceDir = tmp("deps-credvault-vaultkey-ev-"); + const configPath = join(evidenceDir, "keiko.config.json"); + writeFileSync( + configPath, + JSON.stringify({ + providers: [ + { + modelId: "wiring-test-model", + baseUrl: "https://wiring-test.example.invalid/openai/v1", + apiKey: "plaintext-wiring-test-secret", + timeoutMs: 30_000, + maxRetries: 2, + retryBaseDelayMs: 500, + }, + ], + circuitBreaker: { failureThreshold: 5, cooldownMs: 30_000, halfOpenProbes: 2 }, + }), + "utf8", + ); + + const deps = buildUiHandlerDeps({ + configPath, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + store: createInMemoryUiStore(), + }); + try { + expectRealServerLogSink("KEIKO_PROVIDER_CREDENTIALS_KEY", "security.vault.key-resolved"); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); diff --git a/packages/keiko-server/src/deps.test.ts b/packages/keiko-server/src/deps.test.ts index 9b92568db6..c64da07dd8 100644 --- a/packages/keiko-server/src/deps.test.ts +++ b/packages/keiko-server/src/deps.test.ts @@ -49,7 +49,11 @@ import { reconcileTaskWorkspacesAtStartup, type UiHandlerDeps, } from "./deps.js"; -import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; +import { + DEFAULT_SERVER_DIAGNOSTIC_SUMMARY, + type ServerDiagnosticRecord, + type ServerDiagnosticSink, +} from "./diagnostics-log.js"; import type { WorkspaceReconciliationService } from "./task-workspace/types.js"; import { TASK_WORKSPACE_SCHEMA_VERSION, @@ -1139,6 +1143,43 @@ describe("buildUiHandlerDeps — UiStore wiring (ADR-0013)", () => { deps.store.close(); deps.memoryVault?.close(); }); + + // Wave 4a (epic #3233 §8): a UiStoreSchemaVersionError previously crashed startup as a bare, + // undiagnosed exception. Fail-closed is still correct here — this binary genuinely cannot open a + // newer schema — but the crash must be diagnosable, mirroring every other composition-root + // boundary in this module (see "diagnoses why the managed workspace boundary..." above). + it("diagnoses why the UI store could not be opened, and still fails closed", () => { + const stateDir = tmp("ui-store-open-diagnostic-"); + const uiDbPath = join(stateDir, "keiko-ui.db"); + const seed = new DatabaseSync(uiDbPath); + seed.exec("PRAGMA journal_mode = WAL"); + seed.exec("PRAGMA user_version = 9999"); + seed.close(); + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (entry) => { + records.push(entry); + }, + }; + + expect(() => + buildUiHandlerDeps({ + configPath: undefined, + evidenceDir: tmp("ui-store-open-diagnostic-evidence-"), + env: {}, + uiDbPath, + diagnostics, + }), + ).toThrow(/newer than this binary supports/); + + expect(records).toHaveLength(1); + expect(records[0]?.source).toBe("deps.composePersistence"); + expect(records[0]?.operation).toBe("server.composition"); + expect(records[0]?.message).toBe(DEFAULT_SERVER_DIAGNOSTIC_SUMMARY); + expect(records[0]?.correlationId).toMatch(/^[A-Za-z0-9._-]{8,128}$/); + // Content-free: the state directory path never enters the record. + expect(JSON.stringify(records)).not.toContain(stateDir); + }); }); describe("buildUiHandlerDeps — coding-sidecar model-source wiring", () => { diff --git a/packages/keiko-server/src/deps.ts b/packages/keiko-server/src/deps.ts index fa807d78bd..0d4679b672 100644 --- a/packages/keiko-server/src/deps.ts +++ b/packages/keiko-server/src/deps.ts @@ -70,6 +70,7 @@ import { type WorkspaceInstance, } from "@oscharko-dev/keiko-contracts"; import type { IncomingMessage } from "node:http"; +import type { DatabaseSync } from "node:sqlite"; import { detectWorkspaceAt, isWithinWorkspace } from "@oscharko-dev/keiko-workspace"; import { nodeWorkspaceFs } from "@oscharko-dev/keiko-workspace/internal/fs"; import { basename, delimiter, dirname, isAbsolute, join, resolve } from "node:path"; @@ -84,6 +85,7 @@ import { import { createRunRegistry } from "./runs.js"; import type { ChatTurnSerializer } from "./chat-turn-serializer.js"; import { + DEFAULT_SERVER_DIAGNOSTIC_SUMMARY, defaultServerDiagnosticSink, evidenceRetentionDiagnosticObserver, emitServerDiagnostic, @@ -91,6 +93,7 @@ import { type ServerDiagnosticSink, type ServerDiagnosticSummary, } from "./diagnostics-log.js"; +import { processServerLogSink } from "./process-log-sink.js"; import type { CodexSubscriptionProfileCoordinator } from "./coding-codex-subscription.js"; import { assertUiDbOutsideProject, @@ -1781,12 +1784,36 @@ interface ComposedPersistence { readonly codingRuntimeSnapshotStore: CodingRuntimeSnapshotStore | undefined; } +// A `UiStoreSchemaVersionError` or unrecoverable corruption here crashes startup — correctly: this +// store cannot silently continue without its schema — but that crash must not be a bare, +// undiagnosed exception. `emitCompositionDiagnostic` records it before the throw propagates, so an +// operator sees WHY the process refused to start instead of only an unhandled trace, exactly like +// every other composition-root boundary in this module. `processServerLogSink()` is wired in as +// the store's own `store.opened` activity-log sink so a successful open is recorded too. +function openUiDatabaseForComposition( + resolvedUiDbPath: string, + diagnostics: ServerDiagnosticSink | undefined, +): DatabaseSync { + try { + return openNodeUiDatabase(resolvedUiDbPath, processServerLogSink()); + } catch (error) { + emitCompositionDiagnostic( + diagnostics, + "deps.composePersistence", + DEFAULT_SERVER_DIAGNOSTIC_SUMMARY, + error, + ); + throw error; + } +} + function composePersistence( injected: UiStore | undefined, injectedCodingRuntimeSnapshots: CodingRuntimeSnapshotStore | undefined, resolvedUiDbPath: string, redactString: (value: string) => string, env: EnvSource, + diagnostics: ServerDiagnosticSink | undefined, ): ComposedPersistence { if (injected !== undefined) { return { @@ -1798,7 +1825,7 @@ function composePersistence( codingRuntimeSnapshotStore: injectedCodingRuntimeSnapshots, }; } - const db = openNodeUiDatabase(resolvedUiDbPath); + const db = openUiDatabaseForComposition(resolvedUiDbPath, diagnostics); const store = buildUiStoreOverDatabase(db, { redactString }); const relationship: RelationshipHandlerDeps = { scopeResolver: (): { readonly workspaceId: string } => ({ @@ -2766,12 +2793,14 @@ function buildPeripherals(args: BuildPeripheralsArgs): PeripheralManagers { createEditorHotExitStore({ stateDir: args.runtimeStateDir, env: args.options.env, + securityLogSink: processServerLogSink(), }), editorLocalHistoryStore: args.options.editorLocalHistoryStore ?? createEditorLocalHistoryStore({ stateDir: args.runtimeStateDir, env: args.options.env, + securityLogSink: processServerLogSink(), }), managedLspControl, debugActivationControl, @@ -2827,10 +2856,12 @@ function loadRuntimeGatewayConfig( configPath: effectiveConfigPath, env: options.env, evidenceDir: resolvedEvidenceDir, + securityLogSink: processServerLogSink(), }); const secretResolver = createProviderSecretResolver({ configPath: effectiveConfigPath, env: options.env, + securityLogSink: processServerLogSink(), }); const resolved = resolveConfig( options.configPath, @@ -3022,6 +3053,7 @@ function buildPersistenceBundle( resolvedUiDbPath, redactString, options.env, + options.diagnostics, ); const { store, dispose, relationship, codingRuntimeSnapshotStore } = persistence; try { @@ -3169,6 +3201,7 @@ function atlassianConnectorCredentialFields( configPath: args.runtimeConfig.storagePath, env: args.options.env, egress: () => args.runtimeConfig.current()?.egress ?? args.egress, + securityLogSink: processServerLogSink(), }), }; } @@ -3593,6 +3626,7 @@ function buildBaseUiHandlerDeps(args: UiHandlerDepsAssemblyArgs): BaseUiHandlerD createConversationAttachmentStore({ runtimeStateDir: dirname(args.resolvedUiDbPath), env: args.options.env, + securityLogSink: processServerLogSink(), }), }; } @@ -3679,6 +3713,7 @@ function buildIntegrationUiHandlerDeps(args: UiHandlerDepsAssemblyArgs): Integra runtimeStateDir: dirname(args.resolvedUiDbPath), env: args.options.env, diagnostics: args.options.diagnostics, + securityLogSink: processServerLogSink(), }), consolidationJobs: createConsolidationJobRegistry({ evidenceStore: args.evidenceStore }), }; @@ -4088,7 +4123,10 @@ export function buildUiHandlerDeps(options: BuildHandlerDepsOptions): UiHandlerD createNodeEvidenceStore(join(resolvedEvidenceDir, "coding-workbench")); const redactString = runtimeRedactString(options.env, runtimeConfig, egress); const liveRedactor = (value: unknown): unknown => deepRedactStrings(value, redactString); - const localKnowledgeKeyProvider = createLocalKnowledgeKeyProvider({ env: options.env }); + const localKnowledgeKeyProvider = createLocalKnowledgeKeyProvider({ + env: options.env, + securityLogSink: processServerLogSink(), + }); const bundle = buildPersistenceBundle(options, resolvedUiDbPath, redactString, evidenceStore); const contextProfileForModel = buildContextProfileResolver(() => runtimeConfig.current()); reconcileNodeStoreAtStartup(options, bundle); diff --git a/packages/keiko-server/src/editor/hotExitStore.ts b/packages/keiko-server/src/editor/hotExitStore.ts index 2a5524877d..f606cc7153 100644 --- a/packages/keiko-server/src/editor/hotExitStore.ts +++ b/packages/keiko-server/src/editor/hotExitStore.ts @@ -7,6 +7,7 @@ import { type LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; import { SecretboxError } from "@oscharko-dev/keiko-security/errors"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import { EDITOR_HOT_EXIT_TTL_MS, type EditorDocumentVersion, @@ -69,6 +70,9 @@ export interface CreateEditorHotExitStoreOptions { // sample lands close enough to match -- two independent real-clock reads that are never // actually synchronized (AGENTS.md hermetic-tests rule). readonly receiptClock?: (() => number) | undefined; + // Optional activity-log seam (ADR-0019); the deps.ts composition root supplies + // `processServerLogSink()`. + readonly securityLogSink?: SecurityLogSink | undefined; } interface StoredItem { @@ -126,6 +130,7 @@ function hotExitVault(options: CreateEditorHotExitStoreOptions): LocalSecretVaul keychainService: HOT_EXIT_KEYCHAIN_SERVICE, keyfileName: HOT_EXIT_KEYFILE, ...(options.keychainAccess !== undefined ? { keychainAccess: options.keychainAccess } : {}), + sink: options.securityLogSink, }); return createLocalSecretVault({ key, storePath: join(vaultDir, HOT_EXIT_STORE_FILE) }); } diff --git a/packages/keiko-server/src/editor/localHistory/localHistoryStore.test.ts b/packages/keiko-server/src/editor/localHistory/localHistoryStore.test.ts index b1135bff3b..61d01c8535 100644 --- a/packages/keiko-server/src/editor/localHistory/localHistoryStore.test.ts +++ b/packages/keiko-server/src/editor/localHistory/localHistoryStore.test.ts @@ -17,10 +17,12 @@ import { createShardedLocalSecretVault, type LocalSecretVault, } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogEvent, SecurityLogSink } from "@oscharko-dev/keiko-security"; import type { EditorLocalHistoryOrigin } from "@oscharko-dev/keiko-contracts"; import { inspectWorkspaceRootIdentity } from "../../workspace-root-identity.js"; import { createEditorLocalHistoryStore, + EditorLocalHistoryError, EDITOR_LOCAL_HISTORY_INDEX_MAX_BYTES, type EditorLocalHistoryCaptureInput, type EditorLocalHistoryRootScope, @@ -664,3 +666,39 @@ describe("editor local-history store", () => { ); }); }); + +// Wiring test for `securityLogSink` (Wave 4a, epic #3233 §8): every test above supplies +// `KEIKO_EDITOR_LOCAL_HISTORY_KEY` (env tier, via `storeOptions`) so none of them touch the +// keychain, and none supplies `securityLogSink`. This is the one test that forces the sharded +// vault's own failure mode, `security.vault.shard-unreadable`, and proves it reaches the caller's +// sink through the real `createVault` composition (no `vaultFactory` override — that seam bypasses +// `createShardedLocalSecretVault` entirely). +// +// THE FAILURE THIS PINS: dropping `sink: options.securityLogSink` from the +// `createShardedLocalSecretVault` call in `createVault` (`localHistoryStore.ts`) makes `events` +// stay empty below. +describe("createEditorLocalHistoryStore — securityLogSink wiring to the sharded vault", () => { + it("records shard-unreadable when a checkpoint body cannot be read for a reason other than absent", () => { + const fx = fixture(); + const events: SecurityLogEvent[] = []; + const sink: SecurityLogSink = { write: (event): void => void events.push(event) }; + const store = createEditorLocalHistoryStore({ ...storeOptions(fx), securityLogSink: sink }); + + const captured = store.capture(captureInput(fx, "wired\n", "user-save", 1_000)).entry; + const checkpointsDir = join(workspaceStateDir(fx.stateDir), "checkpoints"); + const [shardName] = bodyFiles(fx.stateDir); + if (shardName === undefined) throw new Error("fixture wrote no checkpoint body"); + const shardPath = join(checkpointsDir, shardName); + + // Replace the just-written shard FILE with a DIRECTORY of the same name: the next read fails + // with EISDIR, a reason other than "absent" (ENOENT), which is exactly what the sharded + // vault's own `readShardEnvelope` treats as "one unreadable entry" and reports on the sink. + rmSync(shardPath, { force: true }); + mkdirSync(shardPath); + + expect(() => store.read(fx.scope, captured.entryRef, 2_000)).toThrow(EditorLocalHistoryError); + expect(events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.shard-unreadable" }), + ); + }); +}); diff --git a/packages/keiko-server/src/editor/localHistory/localHistoryStore.ts b/packages/keiko-server/src/editor/localHistory/localHistoryStore.ts index 5375eb796c..19941bb4ce 100644 --- a/packages/keiko-server/src/editor/localHistory/localHistoryStore.ts +++ b/packages/keiko-server/src/editor/localHistory/localHistoryStore.ts @@ -15,7 +15,7 @@ import { type LocalSecretVault, type LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; -import { containsRedactableSecret } from "@oscharko-dev/keiko-security"; +import { containsRedactableSecret, type SecurityLogSink } from "@oscharko-dev/keiko-security"; import { containsPath } from "@oscharko-dev/keiko-git"; import { EDITOR_LOCAL_HISTORY_ENCRYPTION, @@ -212,6 +212,14 @@ export interface CreateEditorLocalHistoryStoreOptions { readonly vaultFactory?: ((workspaceDir: string) => LocalSecretVault) | undefined; readonly saveIndex?: ((path: string, value: Record) => void) | undefined; readonly limits?: Partial | undefined; + /** + * Optional activity-log seam (ADR-0019, Wave 4a epic #3233 §8; see + * `@oscharko-dev/keiko-security/log-port.ts`). When wired, a shard file this vault cannot read + * for a reason other than "absent" emits one `security.vault.shard-unreadable` event instead of + * failing silently. Ignored when `vaultFactory` is injected directly (tests). Omitted or + * `undefined` keeps the vault exactly as silent as before this change. + */ + readonly securityLogSink?: SecurityLogSink | undefined; } function digest(domain: string, parts: readonly string[]): string { @@ -510,10 +518,12 @@ function createVault(options: CreateEditorLocalHistoryStoreOptions, dir: string) keychainService: HISTORY_KEYCHAIN_SERVICE, keyfileName: HISTORY_KEYFILE, keychainAccess: options.keychainAccess, + sink: options.securityLogSink, }); return createShardedLocalSecretVault({ key: resolved.key, storeDir: join(dir, HISTORY_BODIES_DIR), + sink: options.securityLogSink, }); } diff --git a/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts b/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts new file mode 100644 index 0000000000..8c73c83d88 --- /dev/null +++ b/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts @@ -0,0 +1,154 @@ +// Wiring test for `gateway-setup.ts`'s two `securityLogSink: processServerLogSink()` call sites +// that feed the provider-credential vault's key resolution (Wave 4a, epic #3233 §8, gap g18): +// `persistGatewayConfig`'s `persistSealedGatewayConfig` call, and `durableStoredGatewayConfig`'s +// `createProviderSecretResolver` call. Both ultimately reach +// `@oscharko-dev/keiko-security/secret-vault`'s `resolveLocalVaultKey`, which — before this +// change — had no `sink` parameter at all, so neither site could ever report which key tier +// answered (`security.vault.key-resolved`) or that the keychain tier fell back +// (`security.keychain.fallback`). +// +// `@oscharko-dev/keiko-security/secret-vault` is module-mocked to CAPTURE every +// `resolveLocalVaultKey` call's options (real behaviour is preserved via `importOriginal` + +// delegation), so both sites are provable without forcing a real keychain failure — hermetic per +// AGENTS.md. Every captured call sharing `envVarName: "KEIKO_PROVIDER_CREDENTIALS_KEY"` is checked +// with `.every(...)`, not merely the first, because a second `handleGatewaySetup` call exercises +// BOTH sites and either one regressing behind the other must still fail the assertion. +// +// THE FAILURE THIS PINS: dropping `securityLogSink: processServerLogSink()` from either +// `persistGatewayConfig` or `durableStoredGatewayConfig` in `gateway-setup.ts` leaves the matching +// call's `sink` field `undefined`, and the assertion below fails. + +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import type { IncomingMessage } from "node:http"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; +import type { RouteContext } from "./routes.js"; + +type ResolveLocalVaultKeyOptions = Parameters< + typeof import("@oscharko-dev/keiko-security/secret-vault").resolveLocalVaultKey +>[0]; + +let calls: ResolveLocalVaultKeyOptions[]; + +vi.mock("@oscharko-dev/keiko-security/secret-vault", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveLocalVaultKey: ( + options: ResolveLocalVaultKeyOptions, + ): ReturnType => { + calls.push(options); + return actual.resolveLocalVaultKey(options); + }, + }; +}); + +// Imported AFTER the mock declaration so `gateway-setup.ts`'s internal `resolveLocalVaultKey` +// import (via `credentialVault.ts`/`credentialPersistence.ts`) binds to the capturing wrapper. +const { buildUiHandlerDeps } = await import("./deps.js"); +const { handleGatewaySetup } = await import("./gateway-setup.js"); + +const tmpDirs: string[] = []; + +function tmp(prefix: string): string { + const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix))); + tmpDirs.push(dir); + return dir; +} + +function ctx(body: unknown, correlationId: string): RouteContext { + return { + req: Readable.from([Buffer.from(JSON.stringify(body), "utf8")]) as IncomingMessage, + res: {} as RouteContext["res"], + params: {}, + url: new URL("http://127.0.0.1/api/gateway/setup"), + correlationId, + }; +} + +let sink: BufferedServerLogSink; + +beforeEach(() => { + calls = []; + sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); +}); + +afterEach(() => { + resetServerLogger(); + for (const dir of tmpDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("gateway-setup.ts — provider-credential vault wires resolveLocalVaultKey's sink", () => { + it("supplies a real server-log sink from both persistGatewayConfig and durableStoredGatewayConfig", async () => { + const uiDir = tmp("gwsetup-vaultkey-ui-"); + const evidenceDir = tmp("gwsetup-vaultkey-ev-"); + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { KEIKO_UI_DATA_DIR: uiDir }, + gatewayModelDiscovery: () => Promise.resolve(["wiring-test-model"]), + gatewayEmbeddingProbe: (_config, ids) => Promise.resolve(ids), + gatewaySetupTester: (_config, modelIds) => + Promise.resolve([modelIds[0] ?? "wiring-test-model"]), + }); + try { + // First call: a fresh setup with a plaintext apiKey. `current` is undefined, so this goes + // through `verifyAndSaveGatewaySetup` -> `persistGatewayConfig` -> `persistSealedGatewayConfig` + // (gateway-setup.ts site #1). `durableStoredGatewayConfig` short-circuits (no stored file yet + // and no `current`), so site #2 is not exercised here. + const first = await handleGatewaySetup( + ctx( + { baseUrl: "https://wiring-test.example.invalid", apiKey: "plaintext-wiring-secret" }, + "corr-gw-vaultkey-1", + ), + deps, + ); + expect(first.status, JSON.stringify(first.body)).toBe(200); + + // Second call on the SAME deps: `current` is now the just-saved config, and the persisted + // file holds only a reference (site #1 stripped the plaintext), so + // `durableStoredGatewayConfig` must resolve the vault to classify it (gateway-setup.ts + // site #2). `verifyGateway: false` also exercises the "existing config update" branch. + const second = await handleGatewaySetup( + ctx( + { + baseUrl: "https://wiring-test.example.invalid", + apiKey: "plaintext-wiring-secret", + verifyGateway: false, + }, + "corr-gw-vaultkey-2", + ), + deps, + ); + expect(second.status, JSON.stringify(second.body)).toBe(200); + + const matching = calls.filter((c) => c.envVarName === "KEIKO_PROVIDER_CREDENTIALS_KEY"); + expect(matching.length).toBeGreaterThan(1); + expect(matching.every((c) => c.sink !== undefined)).toBe(true); + + // Prove the captured sink is not merely present but IS the process-wide activity log. + matching[0]?.sink?.write({ + level: "info", + category: "security", + op: "security.vault.key-resolved", + extra: { source: "env" }, + }); + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.vault.key-resolved" }), + ); + } finally { + deps.store.close(); + deps.memoryVault?.close(); + } + }); +}); diff --git a/packages/keiko-server/src/gateway-setup.ts b/packages/keiko-server/src/gateway-setup.ts index feac26bb05..5fa3c0dd1b 100644 --- a/packages/keiko-server/src/gateway-setup.ts +++ b/packages/keiko-server/src/gateway-setup.ts @@ -1982,6 +1982,7 @@ function persistGatewayConfig( env: deps.env, storagePath, evidenceDir: resolveEvidenceDir(deps.evidenceDir, deps.env), + securityLogSink: processServerLogSink(), }, ); } @@ -5273,6 +5274,7 @@ function durableStoredGatewayConfig( secretResolver: createProviderSecretResolver({ configPath: storagePath, env: deps.env, + securityLogSink: processServerLogSink(), }), }); // Inside the success path on purpose: on a fall-back the returned config is the RUNTIME one, diff --git a/packages/keiko-server/src/index.ts b/packages/keiko-server/src/index.ts index 5291e4c89c..a7a30fe62b 100644 --- a/packages/keiko-server/src/index.ts +++ b/packages/keiko-server/src/index.ts @@ -118,14 +118,18 @@ export { } from "./evidence.js"; // ADR-0013 — UI-local SQLite persistence: ports, factories, and route handlers. export { + computeStoreFingerprint, createInMemoryUiStore, createNodeUiStore, isProjectAvailable, + openNodeUiDatabase, + openNodeUiDatabaseReadOnly, resolveUiDbPath, runMigrations, SCHEMA_VERSION, UI_DB_DIRNAME, UI_DB_FILENAME, + UI_STORE_FINGERPRINT_TABLES, UiStoreError, validateProjectPath, type Chat, @@ -143,6 +147,16 @@ export { type WorkspaceTrustRecordRow, type WorkspaceTrustRecordRowInput, } from "./store/index.js"; +// Wave 4a, epic #3233 §6.2/§8 — `keiko support export`'s per-store schema/integrity snapshot. +// Lives here (not in keiko-cli) because this is the one package already depending on all three +// store packages; see store-fingerprints.ts's header for the full ADR-0019 rationale. +export { + collectStoreFingerprints, + type CollectStoreFingerprintsInput, + type CollectStoreFingerprintsResult, + type StoreFingerprintUnavailableEntry, + type StoreFingerprintUnavailableReasonKind, +} from "./store-fingerprints.js"; export { handleListProjects, handleCreateProject, diff --git a/packages/keiko-server/src/localKnowledgeKeyProvider.ts b/packages/keiko-server/src/localKnowledgeKeyProvider.ts index 7b6a05dd4b..ff8b0623ea 100644 --- a/packages/keiko-server/src/localKnowledgeKeyProvider.ts +++ b/packages/keiko-server/src/localKnowledgeKeyProvider.ts @@ -19,6 +19,7 @@ import { resolveLocalVaultKey, type LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import type { KnowledgeStoreKeyProvider, KnowledgeStoreKeyProviderContext, @@ -37,6 +38,9 @@ export interface CreateLocalKnowledgeKeyProviderOptions { // Test/non-darwin seam, mirroring the credential vault: inject a stub to force the keyfile tier // deterministically without touching the real login keychain. readonly keychainAccess?: LocalVaultKeychainAccess | undefined; + // Optional activity-log seam (ADR-0019); the deps.ts composition root supplies + // `processServerLogSink()`. + readonly securityLogSink?: SecurityLogSink | undefined; } // Builds the key provider keiko-server injects into openKnowledgeStore so production capsule stores @@ -55,6 +59,7 @@ export function createLocalKnowledgeKeyProvider( keychainService: LOCAL_KNOWLEDGE_KEYCHAIN_SERVICE, keyfileName: LOCAL_KNOWLEDGE_KEYFILE, ...(options.keychainAccess !== undefined ? { keychainAccess: options.keychainAccess } : {}), + sink: options.securityLogSink, }); return key; }, diff --git a/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts b/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts new file mode 100644 index 0000000000..9fa4309e89 --- /dev/null +++ b/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts @@ -0,0 +1,99 @@ +// Wiring test for `createBffMemoryVault`'s `securityLogSink` option (Wave 4a, epic #3233 §8). +// +// WHAT THIS PINS +// +// `createBffMemoryVault` (`memory-handlers.ts`) is the ONE production caller of +// `createMemoryVault` (`@oscharko-dev/keiko-memory-vault`) in this codebase's BFF composition. It +// already threads `logSink: processServerLogSink()`; this pins that it ALSO threads +// `securityLogSink: processServerLogSink()`, the option `createMemoryVault` (Wave 4a, epic #3233 +// §8) forwards into the shared bounded keychain reader so a keychain fallback lands on +// `server.log` instead of being silently dropped. +// +// `@oscharko-dev/keiko-memory-vault` is module-mocked (kept elsewhere real: every other test in +// this file's package uses the genuine vault) so the OPTIONS OBJECT `createBffMemoryVault` builds +// is directly observable without needing to force a real OS keychain failure — the vault's own +// `securityLogSink` → `keyFromKeychain` wiring is separately pinned in +// `packages/keiko-memory-vault/src/vault-keychain-log-wiring.test.ts`; this file's job is only the +// SERVER composition boundary between the two. +// +// THE FAILURE THIS PINS: dropping `securityLogSink: processServerLogSink()` from +// `createBffMemoryVault`'s call to `createMemoryVault` makes the captured options carry no +// `securityLogSink`, and the first assertion below fails. + +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; + +type CreateMemoryVaultOptions = Parameters< + typeof import("@oscharko-dev/keiko-memory-vault").createMemoryVault +>[0]; + +let captured: CreateMemoryVaultOptions; + +vi.mock("@oscharko-dev/keiko-memory-vault", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createMemoryVault: ( + options: CreateMemoryVaultOptions, + ): ReturnType => { + captured = options; + return actual.createMemoryVault(options); + }, + }; +}); + +// Imported AFTER the mock declaration so `createBffMemoryVault`'s internal `createMemoryVault` +// import binds to the capturing wrapper above. +const { createBffMemoryVault } = await import("./memory-handlers.js"); + +let dir: string; +let sink: BufferedServerLogSink; + +beforeEach(() => { + dir = mkdtempSync(join(realpathSync(tmpdir()), "keiko-memory-handlers-secloG-")); + sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + captured = undefined; +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + resetServerLogger(); + vi.restoreAllMocks(); +}); + +describe("createBffMemoryVault — securityLogSink composition wiring", () => { + it("passes securityLogSink to createMemoryVault, and it reaches server.log when written to", () => { + const vault = createBffMemoryVault((value: string) => value, undefined, undefined, { + KEIKO_MEMORY_DIR: dir, + }); + try { + expect(captured?.securityLogSink).toBeDefined(); + + // Prove the captured sink is not merely present but IS the process-wide activity log: a + // write through it must reach `server.log`, exactly like the wired keychain tier would + // produce on a real fallback. + captured?.securityLogSink?.write({ + level: "warn", + category: "security", + op: "security.keychain.fallback", + extra: { reasonKind: "ETIMEDOUT", boundedExitKind: "timeout" }, + }); + + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.keychain.fallback" }), + ); + } finally { + vault.close(); + } + }); +}); diff --git a/packages/keiko-server/src/memory-handlers.ts b/packages/keiko-server/src/memory-handlers.ts index 337da9cde7..e38425bd08 100644 --- a/packages/keiko-server/src/memory-handlers.ts +++ b/packages/keiko-server/src/memory-handlers.ts @@ -79,6 +79,7 @@ import { type MemoryCaptureDecision, } from "./memory-capture-projection.js"; import { refreshMemoryEmbeddingAfterBodyEdit } from "./memory-embedding.js"; +import { processServerLogSink } from "./process-log-sink.js"; // ─── Constants ──────────────────────────────────────────────────────────────── @@ -2111,10 +2112,24 @@ export function createBffMemoryVault( // Optional onMemoryEvent (#214) wires every successful vault mutation into the audit // ledger. When undefined, the vault still fires its internal NOOP sink, so the absence // of an audit hook is fully backward-compatible with the pre-#214 BFF wiring. + // + // `logSink` (w4a-memory-vault-fingerprint, epic #3233 §8/g18): wires the process-wide activity + // log into the vault's own structural `MemoryVaultLogSink` port (ADR-0019 — see + // `keiko-memory-vault/src/vault-log.ts`), so vault-open (with the retained key-resolution + // tier), a corruption quarantine, and an encryption migration all land in `server.log` instead + // of being unobservable, mirroring `local-knowledge-store-open.ts`'s identical wiring. + // + // `securityLogSink` (Wave 4a, epic #3233 §8): the SAME sink threaded into the vault's own + // structural `SecurityLogSink` port (`@oscharko-dev/keiko-security/log-port.ts`) so the shared + // bounded macOS Keychain tier (`cipher.ts`'s `keyFromKeychain`) reports a fall-through to the + // keyfile tier as `security.keychain.fallback` instead of failing silently. `keiko-memory-vault` + // depends on `keiko-security` (ADR-0019), so importing the port's type here is legal. return createMemoryVault({ redactString, ...(onMemoryEvent === undefined ? {} : { onMemoryEvent }), ...(onDeleteEventsBeforeCommit === undefined ? {} : { onDeleteEventsBeforeCommit }), ...(env === undefined ? {} : { env }), + logSink: processServerLogSink(), + securityLogSink: processServerLogSink(), }); } diff --git a/packages/keiko-server/src/observability/server-log.ts b/packages/keiko-server/src/observability/server-log.ts index e4e62500ef..b3ae7f27a0 100644 --- a/packages/keiko-server/src/observability/server-log.ts +++ b/packages/keiko-server/src/observability/server-log.ts @@ -89,7 +89,11 @@ export { redactRoutePath } from "./route-template.js"; // Coarse routing label. Kept a closed union so a typo cannot invent a category an operator's // grep will never find; `memory` was added for the vault/retrieval surface, `process` for the // process-lifecycle lines (`process.started`/`process.heartbeat`/`process.exiting`) envelope v2 -// adds. +// adds, and `security` (Wave 4a, epic #3233 §8) for `keiko-security`'s own structural +// `SecurityLogSink` port (`packages/keiko-security/src/log-port.ts`) — without this member here, +// `SecurityLogEvent`'s `category` union is not a subset of this one, so `ServerLogSink` (via +// `processServerLogSink()`) is not structurally assignable to `SecurityLogSink` at all, unlike the +// `MemoryVaultLogSink`/`KnowledgeLogSink` ports, whose categories are already subsets of this union. export type ServerLogCategory = | "http" | "gateway" @@ -98,6 +102,7 @@ export type ServerLogCategory = | "setup" | "search" | "memory" + | "security" | "diagnostic" | "process"; diff --git a/packages/keiko-server/src/observability/server-logger.ts b/packages/keiko-server/src/observability/server-logger.ts index 1da0c3d6b5..571eea8e74 100644 --- a/packages/keiko-server/src/observability/server-logger.ts +++ b/packages/keiko-server/src/observability/server-logger.ts @@ -87,6 +87,7 @@ const KNOWN_CATEGORIES = new Set([ "setup", "search", "memory", + "security", "diagnostic", "process", ]); diff --git a/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotOrchestration-keychain-log-wiring.test.ts b/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotOrchestration-keychain-log-wiring.test.ts new file mode 100644 index 0000000000..678a93f59a --- /dev/null +++ b/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotOrchestration-keychain-log-wiring.test.ts @@ -0,0 +1,98 @@ +// Wiring test for the Figma PAT vault's keychain-tier activity-log seam (Wave 4a, epic #3233 §8). +// +// WHAT THIS PINS +// +// `readFigmaVaultToken`/`figmaTokenStoreFor` (the real, exported composition functions +// `figmaSnapshotOrchestration.ts` hands to route/orchestration callers) omit `deps.keychainAccess` +// in production — it exists only as a test/CI seam, supplied by every OTHER test in +// `figmaSnapshotOrchestration.test.ts`. Production instead falls through this file's own +// `productionKeychainAccess` helper, which wires `processServerLogSink()` into the shared bounded +// keychain reader (`figmaKeychainReader`, `figma/figmaTokenStore.ts`'s `keyFromKeychain`) so a +// keychain that never answers is recorded as `security.keychain.fallback` on `server.log` instead +// of failing silently. +// +// `figmaKeychainReader` is module-mocked here (kept, real, elsewhere: every existing test in this +// package's suite exercises it through an injected `deps.keychainAccess`, which this test +// deliberately omits) so the forced "keychain unavailable" outcome is hermetic and +// platform-independent — the real reader would otherwise spawn the actual OS `security` binary. +// +// THE FAILURE THIS PINS: reverting `processServerLogSink()` to `undefined` (or dropping +// `productionKeychainAccess` entirely, back to passing `deps.keychainAccess` bare) makes the mock +// below observe `options.sink === undefined`, write nothing, and the assertion on `events` fails — +// the FAILS-BEFORE/PASSES-AFTER property a wiring test needs. + +import { mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "../../observability/index.js"; + +vi.mock("../figma/index.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // Stands in for a keychain that never answers (the 0.3.0 boot-hang class of failure — see + // `macos-keychain.ts`'s file header): reports "unavailable" (`undefined`, falls through to the + // keyfile tier) while emitting `security.keychain.fallback` on whatever sink it was given, the + // same observable contract the real bounded reader has. + figmaKeychainReader: ( + options: { readonly sink?: { write: (event: unknown) => void } } = {}, + ): Buffer | undefined => { + options.sink?.write({ + level: "warn", + category: "security", + op: "security.keychain.fallback", + extra: { reasonKind: "ETIMEDOUT", boundedExitKind: "timeout" }, + }); + return undefined; + }, + }; +}); + +// Imported AFTER the mock declaration so `readFigmaVaultToken`/`figmaTokenStoreFor`'s internal +// `figmaKeychainReader` import (via `productionKeychainAccess`) binds to the fake above. +const { readFigmaVaultToken, figmaTokenStoreFor } = + await import("../figmaSnapshotOrchestration.js"); + +let dir: string; +let sink: BufferedServerLogSink; + +beforeEach(() => { + dir = mkdtempSync(join(realpathSync(tmpdir()), "keiko-figma-keychain-log-")); + sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + resetServerLogger(); + vi.restoreAllMocks(); +}); + +describe("readFigmaVaultToken — production keychain access wires the activity log", () => { + it("records a keychain fallback on server.log when deps.keychainAccess is omitted", () => { + // No stored vault token, so `read()` returns undefined regardless — the wiring under test is + // the log line the (mocked) keychain fallback produces on the way there, not the read result. + readFigmaVaultToken({ evidenceDir: dir, env: {}, now: new Date(0).toISOString() }); + + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.keychain.fallback" }), + ); + }); +}); + +describe("figmaTokenStoreFor — production keychain access wires the activity log", () => { + it("records a keychain fallback on server.log when deps.keychainAccess is omitted", () => { + figmaTokenStoreFor({ evidenceDir: dir, env: {} }); + + expect(sink.events).toContainEqual( + expect.objectContaining({ category: "security", op: "security.keychain.fallback" }), + ); + }); +}); diff --git a/packages/keiko-server/src/qualityIntelligence/figma/index.ts b/packages/keiko-server/src/qualityIntelligence/figma/index.ts index beec04da37..a60d9c91bf 100644 --- a/packages/keiko-server/src/qualityIntelligence/figma/index.ts +++ b/packages/keiko-server/src/qualityIntelligence/figma/index.ts @@ -79,6 +79,7 @@ export { export { createFigmaTokenStore, resolveFigmaVaultKey, + keyFromKeychain as figmaKeychainReader, NO_FIGMA_KEYCHAIN, type FigmaTokenStore, type FigmaTokenStoreDeps, diff --git a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotOrchestration.ts b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotOrchestration.ts index bb23505f23..90efbb62ba 100644 --- a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotOrchestration.ts +++ b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotOrchestration.ts @@ -30,6 +30,7 @@ import { resolveFigmaToken, resolveScopedPaginationLimits, resolveFigmaVaultKey, + figmaKeychainReader, type FigmaConnectorMetrics, type FigmaHttpPort, type FigmaKeychainAccess, @@ -44,6 +45,7 @@ import { import { QualityIntelligenceFigma } from "@oscharko-dev/keiko-quality-intelligence"; import type { EnvSource } from "@oscharko-dev/keiko-security"; import type { OutboundHttpEgressConfig } from "@oscharko-dev/keiko-model-gateway/internal/http"; +import { processServerLogSink } from "../process-log-sink.js"; const FIGMA_VAULT_SUBDIR = "figma"; const FIGMA_TOKEN_VAULT_FILE = "figma-token.vault"; @@ -96,6 +98,19 @@ const tokenVaultDir = (evidenceDir: string): string => join(evidenceDir, FIGMA_V const tokenVaultPath = (evidenceDir: string): string => join(tokenVaultDir(evidenceDir), FIGMA_TOKEN_VAULT_FILE); +// `deps.keychainAccess` is a test/CI seam (every call site in this package's tests supplies one). +// Production omits it, and this is where that gap is filled: the SAME bounded reader +// `resolveFigmaVaultKey`'s own default would use, but with the process-wide activity log +// (Wave 4a, epic #3233 §8) threaded into it, so a keychain that never answers emits +// `security.keychain.fallback` on `server.log` instead of falling through silently. Built once per +// call rather than hoisted to a constant so `processServerLogSink()` — itself cheap and resolved +// per write — always reflects the currently configured server logger. +const productionKeychainAccess = (deps: { + readonly keychainAccess?: FigmaKeychainAccess; +}): FigmaKeychainAccess => + deps.keychainAccess ?? + ((): Buffer | undefined => figmaKeychainReader({ sink: processServerLogSink() })); + /** * Read the encrypted-at-rest vault PAT (#758), or `undefined` when no vault token is stored or the * vault key cannot be resolved. Highest precedence in {@link resolveFigmaToken}. Never throws — a @@ -106,7 +121,7 @@ export const readFigmaVaultToken = (deps: GovernedSnapshotDeps): string | undefi const { key } = resolveFigmaVaultKey( deps.env, tokenVaultDir(deps.evidenceDir), - deps.keychainAccess, + productionKeychainAccess(deps), ); return createFigmaTokenStore({ key, storePath: tokenVaultPath(deps.evidenceDir) }).read(); } catch { @@ -121,7 +136,7 @@ export const figmaTokenStoreFor = ( const { key } = resolveFigmaVaultKey( deps.env, tokenVaultDir(deps.evidenceDir), - deps.keychainAccess, + productionKeychainAccess(deps), ); return createFigmaTokenStore({ key, storePath: tokenVaultPath(deps.evidenceDir) }); }; diff --git a/packages/keiko-server/src/store-fingerprints.test.ts b/packages/keiko-server/src/store-fingerprints.test.ts new file mode 100644 index 0000000000..40dd8ea911 --- /dev/null +++ b/packages/keiko-server/src/store-fingerprints.test.ts @@ -0,0 +1,201 @@ +// Wave 4a (epic #3233 §6.2/§8): `collectStoreFingerprints` unit tests. Seeds real, on-disk +// instances of all three stores (ui, local-knowledge, memory-vault) under one temp state dir +// using each store package's own create/open helpers — never a hand-rolled fixture that +// re-derives a schema this test does not own — then calls `collectStoreFingerprints` directly +// and asserts the three-valid / missing / corrupt-and-untouched outcomes this function owns. +// +// RED (before this file existed): the CLI-side `keiko-cli` package imported +// `@oscharko-dev/keiko-local-knowledge` directly to compute this exact collection, which +// `arch:check`'s ADR-0019 direction rule 7 forbids (a leaf CLI consumer must reach domain +// packages only through their public surfaces it is actually allowed — local-knowledge was never +// on that allowlist). This suite pins the collection's behaviour at the layer that now owns it. + +import { randomBytes } from "node:crypto"; +import { + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + isStoreFingerprint, + type EmbeddingModelIdentity, + type KnowledgeCapsuleId, + type StoreFingerprint, +} from "@oscharko-dev/keiko-contracts"; +import type { MemoryId, UserId } from "@oscharko-dev/keiko-contracts/memory"; +import { createMemoryVault } from "@oscharko-dev/keiko-memory-vault"; +import { + createCapsule, + openKnowledgeStore, + resolveKnowledgeStorePath, + type CreateCapsuleInput, +} from "@oscharko-dev/keiko-local-knowledge"; + +import { createNodeUiStore, UI_DB_FILENAME } from "./store/index.js"; +import { collectStoreFingerprints } from "./store-fingerprints.js"; + +const UI_PROJECT_MARKER = "unit-ui-project-marker-3233"; +const CAPSULE_MARKER = "unit-local-knowledge-capsule-marker-3233"; +const MEMORY_BODY_MARKER = "unit-memory-vault-body-marker-3233"; + +const EMBEDDING_IDENTITY: EmbeddingModelIdentity = { + provider: "openai", + modelId: "text-embedding-3-small", + vectorDimensions: 1536, + vectorMetric: "cosine", + normalization: "l2", + instructionVersion: "keiko-embedding-input-v1", + embeddingSpaceFingerprint: "keiko-embedding-space-fingerprint-v1:3233-unit", +}; + +function seedUiStore(stateDir: string): void { + const dbPath = join(stateDir, "ui", UI_DB_FILENAME); + const store = createNodeUiStore(dbPath); + const projectDir1 = mkdtempSync(join(stateDir, `${UI_PROJECT_MARKER}-1-`)); + const projectDir2 = mkdtempSync(join(stateDir, `${UI_PROJECT_MARKER}-2-`)); + store.createProject(projectDir1, UI_PROJECT_MARKER); + store.createProject(projectDir2, UI_PROJECT_MARKER); + store.close(); +} + +function capsuleInput(id: string): CreateCapsuleInput { + return { + id: id as KnowledgeCapsuleId, + displayName: CAPSULE_MARKER, + tags: [], + retrievalEffort: "default", + outputMode: "answers", + answerGroundingPolicy: "require-citations", + embeddingModelIdentity: EMBEDDING_IDENTITY, + lifecycleState: "draft", + storageReference: `${CAPSULE_MARKER}/${id}`, + }; +} + +function seedLocalKnowledgeStore(stateDir: string): void { + const dbPath = resolveKnowledgeStorePath({ runtimeStateDir: stateDir }); + const store = openKnowledgeStore({ dbPath }); + createCapsule(store, capsuleInput("cap-1")); + createCapsule(store, capsuleInput("cap-2")); + store.close(); +} + +function seedMemoryVault(stateDir: string, memoryKeyBase64: string): void { + const memoryDir = join(stateDir, "memory"); + const vault = createMemoryVault({ + memoryDir, + env: { KEIKO_MEMORY_DIR: memoryDir, KEIKO_MEMORY_KEY: memoryKeyBase64 }, + }); + const t = 1_700_000_000_000; + const memory = (id: string): Parameters[0] => ({ + id: id as MemoryId, + schemaVersion: "1", + scope: { kind: "user", userId: "u-1" as UserId }, + type: "preference", + body: MEMORY_BODY_MARKER, + provenance: { + sourceKind: "explicit-user-instruction", + capturedAt: t, + confidence: 0.9, + sensitivity: "confidential", + }, + validity: { validFrom: t }, + status: "accepted", + pinned: false, + tags: [], + createdAt: t, + updatedAt: t, + }); + vault.insertMemory(memory("m1")); + vault.insertMemory(memory("m2")); + vault.close(); +} + +describe("collectStoreFingerprints", () => { + let stateDir: string; + let memoryKeyBase64: string; + + beforeEach(() => { + // Realpath the tmpdir root: on macOS both /tmp and /var are symlinks, and the memory vault's + // own path guard refuses a path with a symlinked ancestor (mirrors keiko-server's own + // UI-db guard) — the same reason this package's own db.test.ts realpaths its tmp root. + const root = realpathSync(tmpdir()); + stateDir = mkdtempSync(join(root, "keiko-store-fp-unit-state-")); + memoryKeyBase64 = randomBytes(32).toString("base64"); + }); + + afterEach(() => { + rmSync(stateDir, { recursive: true, force: true }); + }); + + it("computes three valid fingerprints matching the seeded row counts, with nothing unavailable", async () => { + seedUiStore(stateDir); + seedLocalKnowledgeStore(stateDir); + seedMemoryVault(stateDir, memoryKeyBase64); + + const result = await collectStoreFingerprints({ + stateDir, + env: { KEIKO_MEMORY_KEY: memoryKeyBase64 }, + }); + + expect(result.unavailable).toEqual([]); + expect(result.fingerprints).toHaveLength(3); + expect(result.fingerprints.every((entry) => isStoreFingerprint(entry))).toBe(true); + + const byStore = new Map( + result.fingerprints.map((entry) => [entry.store, entry]), + ); + expect(byStore.get("ui")?.tableRowCounts.projects).toBe(2); + expect(byStore.get("local-knowledge")?.tableRowCounts.capsules).toBe(2); + expect(byStore.get("memory-vault")?.tableRowCounts.memories).toBe(2); + for (const store of ["ui", "local-knowledge", "memory-vault"]) { + expect(byStore.get(store)?.quickCheckOk).toBe(true); + } + }); + + it("reports every store as missing, and creates none of them, when none exist under stateDir", async () => { + const result = await collectStoreFingerprints({ stateDir, env: {} }); + + expect(result.fingerprints).toEqual([]); + expect(result.unavailable).toEqual( + expect.arrayContaining([ + { store: "ui", reasonKind: "missing" }, + { store: "local-knowledge", reasonKind: "missing" }, + { store: "memory-vault", reasonKind: "missing" }, + ]), + ); + expect(result.unavailable).toHaveLength(3); + }); + + // RED (before fix): computing a store's fingerprint went through that store package's mutating + // production open path, which quarantines confirmed SQLite corruption as an ordinary part of + // opening — renaming the corrupt file aside and silently creating an empty replacement. A + // diagnostic collection must never destroy the very corruption evidence an operator is trying + // to capture; instead it degrades to `quickCheckOk: false` and leaves the bytes exactly as + // found. + it("degrades a genuinely SQLite-corrupt ui store file to quickCheckOk:false, leaving its bytes untouched", async () => { + const uiDbPath = join(stateDir, "ui", UI_DB_FILENAME); + const corruptBytes = "garbage that is not a sqlite header"; + mkdirSync(dirname(uiDbPath), { recursive: true }); + writeFileSync(uiDbPath, corruptBytes); + + const result = await collectStoreFingerprints({ stateDir, env: {} }); + + expect(readFileSync(uiDbPath, "utf8")).toBe(corruptBytes); + const siblingNames = readdirSync(dirname(uiDbPath)); + expect(siblingNames.some((name) => name.includes(".corrupt."))).toBe(false); + + const uiEntry = result.fingerprints.find((entry) => entry.store === "ui"); + expect(uiEntry).toBeDefined(); + expect(uiEntry?.quickCheckOk).toBe(false); + expect(result.unavailable.some((entry) => entry.store === "ui")).toBe(false); + }); +}); diff --git a/packages/keiko-server/src/store-fingerprints.ts b/packages/keiko-server/src/store-fingerprints.ts new file mode 100644 index 0000000000..f70d6bdb11 --- /dev/null +++ b/packages/keiko-server/src/store-fingerprints.ts @@ -0,0 +1,177 @@ +// StoreFingerprint collection (Wave 4a, epic #3233 §6.2/§8) — the `keiko support export` +// manifest's per-store schema/integrity snapshot. Lives in keiko-server, not keiko-cli, because +// this package already depends on all three store packages (its own ./store/, plus +// keiko-local-knowledge and keiko-memory-vault) per ADR-0019's Target Package Topology; keiko-cli +// is a leaf consumer of the domain packages through their public surfaces only and must not +// import keiko-local-knowledge directly (ADR-0019 direction rule 7, cli boundary). `keiko support +// export` reaches this function through keiko-server's own public package surface — the same +// `loadServer()` seam keiko-cli's support.ts already uses for the rest of the manifest. +// +// Each of the three per-store attempt functions below opens its store through that package's +// GENUINELY read-only open (`openNodeUiDatabaseReadOnly` / `openKnowledgeStoreReadOnly` / +// `openMemoryDatabaseReadOnly` — `node:sqlite`'s `readOnly: true`), never the mutating production +// open path (`openNodeUiDatabase` / `openKnowledgeStore` / `openMemoryDatabase`), which runs +// migrations, a client-turn recovery UPDATE, an encryption sweep, and a corruption-quarantine +// reopen as an ordinary part of opening. A diagnostic export must not perform any of those against +// the very store an operator is trying to inspect. It then calls that store package's own +// `computeStoreFingerprint(db)` — never a new redaction or key-resolution scheme. A store whose db +// file does not exist yet is never opened at all (the mutating open path can create one, or — for +// the memory vault's keyfile tier — mint a brand-new key, exactly the write this collection must +// not cause for a subsystem the operator never touched); every other failure (corruption, a vault +// key the operator has not supplied, an unreadable path) collapses to the same closed-vocabulary +// "open-failed" reason, one bucket wide by design so this diagnostic surface never has to carry a +// store-specific error. + +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import type { StoreFingerprint } from "@oscharko-dev/keiko-contracts"; +import { + computeStoreFingerprint as computeLocalKnowledgeStoreFingerprint, + openKnowledgeStoreReadOnly, + resolveKnowledgeStorePath, +} from "@oscharko-dev/keiko-local-knowledge"; +import { + computeStoreFingerprint as computeMemoryVaultStoreFingerprint, + MEMORY_DB_FILENAME, + openMemoryDatabaseReadOnly, + resolveVaultKey, +} from "@oscharko-dev/keiko-memory-vault"; +import { + computeStoreFingerprint as computeUiStoreFingerprint, + openNodeUiDatabaseReadOnly, + UI_DB_FILENAME, +} from "./store/index.js"; + +export type StoreFingerprintUnavailableReasonKind = "missing" | "open-failed"; + +export interface StoreFingerprintUnavailableEntry { + readonly store: StoreFingerprint["store"]; + readonly reasonKind: StoreFingerprintUnavailableReasonKind; +} + +export interface CollectStoreFingerprintsInput { + readonly stateDir: string; + readonly env: Readonly>; +} + +export interface CollectStoreFingerprintsResult { + readonly fingerprints: readonly StoreFingerprint[]; + readonly unavailable: readonly StoreFingerprintUnavailableEntry[]; +} + +type StoreFingerprintOutcome = + | { readonly fingerprint: StoreFingerprint } + | { readonly unavailable: StoreFingerprintUnavailableEntry }; + +function unavailableOutcome( + store: StoreFingerprint["store"], + reasonKind: StoreFingerprintUnavailableReasonKind, +): StoreFingerprintOutcome { + return { unavailable: { store, reasonKind } }; +} + +// `keiko support export --state-dir` maps the ui store to `/ui/keiko-ui.db` and the +// memory vault to `/memory/keiko-memory.db` unless KEIKO_UI_DATA_DIR / KEIKO_MEMORY_DIR +// override them. The subdir names are duplicated here rather than imported from keiko-cli's +// state-paths.ts (`DEFAULT_UI_STATE_SUBDIR`): ADR-0019's dependency direction runs +// keiko-cli -> keiko-server, never the reverse, so this package cannot depend on keiko-cli for a +// constant — the same accepted, documented-duplication tradeoff state-paths.ts itself already +// applies in the other direction for these packages' own filenames. +const UI_STATE_SUBDIR = "ui"; +const MEMORY_STATE_SUBDIR = "memory"; + +// `node:sqlite` is synchronous end to end, so — unlike keiko-cli's lazy-loaded equivalents this +// function replaces — none of the three per-store attempts below needs `await`: the store +// packages are already static imports in this package (no dynamic `import()` to wait on). +function uiStoreFingerprintOutcome( + stateDir: string, + env: Readonly>, +): StoreFingerprintOutcome { + try { + const uiDataDir = env.KEIKO_UI_DATA_DIR ?? join(stateDir, UI_STATE_SUBDIR); + const dbPath = join(uiDataDir, UI_DB_FILENAME); + if (!existsSync(dbPath)) return unavailableOutcome("ui", "missing"); + const db = openNodeUiDatabaseReadOnly(dbPath); + try { + return { fingerprint: computeUiStoreFingerprint(db) }; + } finally { + db.close(); + } + } catch { + return unavailableOutcome("ui", "open-failed"); + } +} + +function localKnowledgeStoreFingerprintOutcome(stateDir: string): StoreFingerprintOutcome { + try { + const dbPath = resolveKnowledgeStorePath({ runtimeStateDir: stateDir }); + if (!existsSync(dbPath)) return unavailableOutcome("local-knowledge", "missing"); + const db = openKnowledgeStoreReadOnly(dbPath); + try { + return { fingerprint: computeLocalKnowledgeStoreFingerprint(db) }; + } finally { + db.close(); + } + } catch { + return unavailableOutcome("local-knowledge", "open-failed"); + } +} + +function memoryVaultStoreFingerprintOutcome( + stateDir: string, + env: Readonly>, +): StoreFingerprintOutcome { + try { + const memoryDir = env.KEIKO_MEMORY_DIR ?? join(stateDir, MEMORY_STATE_SUBDIR); + const dbPath = join(memoryDir, MEMORY_DB_FILENAME); + if (!existsSync(dbPath)) return unavailableOutcome("memory-vault", "missing"); + const resolved = resolveVaultKey(env, memoryDir); + const db = openMemoryDatabaseReadOnly(dbPath); + try { + return { fingerprint: computeMemoryVaultStoreFingerprint(db, resolved.source) }; + } finally { + db.close(); + } + } catch { + return unavailableOutcome("memory-vault", "open-failed"); + } +} + +function partitionStoreFingerprintOutcomes( + outcomes: readonly StoreFingerprintOutcome[], +): CollectStoreFingerprintsResult { + const fingerprints: StoreFingerprint[] = []; + const unavailable: StoreFingerprintUnavailableEntry[] = []; + for (const outcome of outcomes) { + if ("fingerprint" in outcome) { + fingerprints.push(outcome.fingerprint); + } else { + unavailable.push(outcome.unavailable); + } + } + return { fingerprints, unavailable }; +} + +/** + * Opens each of the ui, local-knowledge, and memory-vault stores found under `input.stateDir` + * through its owning package's genuinely read-only open, computes a redacted + * {@link StoreFingerprint} for each store that is present and openable, and reports every other + * store — never used yet from this state dir, or present but unopenable — in `unavailable` with a + * closed-vocabulary reason. Never throws: a per-store open failure degrades to an `unavailable` + * entry rather than failing the whole collection. + * + * Returns a `Promise` for parity with `keiko-cli`'s other lazily-loaded server-module seams (and + * headroom for a genuinely async store package later), even though every read here is + * synchronous today (`node:sqlite`) — an `async` declaration with no real `await` inside it would + * itself be flagged (`@typescript-eslint/require-await`), so the wrapping is explicit instead. + */ +export function collectStoreFingerprints( + input: CollectStoreFingerprintsInput, +): Promise { + const outcomes = [ + uiStoreFingerprintOutcome(input.stateDir, input.env), + localKnowledgeStoreFingerprintOutcome(input.stateDir), + memoryVaultStoreFingerprintOutcome(input.stateDir, input.env), + ]; + return Promise.resolve(partitionStoreFingerprintOutcomes(outcomes)); +} diff --git a/packages/keiko-server/src/store/db.test.ts b/packages/keiko-server/src/store/db.test.ts index f404d08da7..be03aad6af 100644 --- a/packages/keiko-server/src/store/db.test.ts +++ b/packages/keiko-server/src/store/db.test.ts @@ -14,9 +14,13 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join, dirname } from "node:path"; -import type { StoredPdfCitationPreviewCitation } from "@oscharko-dev/keiko-contracts"; +import { + isStoreFingerprint, + type StoredPdfCitationPreviewCitation, +} from "@oscharko-dev/keiko-contracts"; import { MAX_DESKTOP_CHAT_CLIENT_TURN_ID_CHARS } from "@oscharko-dev/keiko-contracts/bff-wire"; import { + buildUiStoreOverDatabase, createInMemoryUiStore, createNodeUiStore, openNodeUiDatabase, @@ -25,6 +29,10 @@ import { type GroundedAnswer, type NewChatMessage, } from "./index.js"; +// Not yet re-exported through the barrel (Wave 4a is scoped to db.ts/deps.ts) — imported directly +// from the co-located module instead, same package, no boundary crossed. +import { computeStoreFingerprint, UI_STORE_FINGERPRINT_TABLES } from "./db.js"; +import type { ServerLogEvent, ServerLogSink } from "../observability/index.js"; // Narrows an array-index access (T | undefined) to T without a non-null assertion. function must(value: T | undefined): T { @@ -1076,3 +1084,137 @@ describe("UI DB busy_timeout (issue #639)", () => { store.close(); }); }); + +// Wave 4a, epic #3233 §6.2 — the redacted, point-in-time schema/integrity snapshot embedded in +// the support bundle manifest. +describe("computeStoreFingerprint (Wave 4a, epic #3233 §6.2)", () => { + it("computes a valid, fully-populated fingerprint for a freshly migrated store", () => { + const dbPath = join(tmpDir, "fingerprint-fresh.db"); + const db = openNodeUiDatabase(dbPath); + try { + const fingerprint = computeStoreFingerprint(db); + // Validated against the real, independently-owned keiko-contracts guard rather than + // re-asserting each field by hand — the producer and the shape gate must agree. + expect(isStoreFingerprint(fingerprint)).toBe(true); + expect(fingerprint.store).toBe("ui"); + expect(fingerprint.schemaVersion).toBe(SCHEMA_VERSION); + expect(fingerprint.migrationsApplied).toEqual( + Array.from({ length: SCHEMA_VERSION }, (_unused, index) => `v${String(index + 1)}`), + ); + expect(Object.keys(fingerprint.tableRowCounts).sort()).toEqual( + [...UI_STORE_FINGERPRINT_TABLES].sort(), + ); + expect(Object.values(fingerprint.tableRowCounts).every((count) => count === 0)).toBe(true); + expect(fingerprint.quickCheckOk).toBe(true); + expect(fingerprint.encryptionMode).toBe("plaintext"); + expect(fingerprint.keySource).toBeUndefined(); + } finally { + db.close(); + } + }); + + it("counts rows actually present in a table, independently per table", () => { + const dbPath = join(tmpDir, "fingerprint-counts.db"); + const db = openNodeUiDatabase(dbPath); + try { + const store = buildUiStoreOverDatabase(db); + const projectA = mkdtempSync(join(tmpDir, "fingerprint-project-a-")); + const projectB = mkdtempSync(join(tmpDir, "fingerprint-project-b-")); + store.createProject(projectA, "Project A"); + store.createProject(projectB, "Project B"); + const fingerprint = computeStoreFingerprint(db); + expect(fingerprint.tableRowCounts.projects).toBe(2); + expect(fingerprint.tableRowCounts.chats).toBe(0); + } finally { + db.close(); + } + }); + + // Regression pin: the manifest assembler this feeds must still produce a (degraded) fingerprint + // for the very store an operator is trying to diagnose — it must never throw and abort the + // whole support-bundle export over one unreadable store. + it("never throws against a corrupted file, and returns a degraded fingerprint instead", () => { + const dbPath = join(tmpDir, "fingerprint-corrupt.db"); + writeFileSync(dbPath, Buffer.from("not a sqlite db")); + const db = new DatabaseSync(dbPath); + try { + let fingerprint: ReturnType | undefined; + expect(() => { + fingerprint = computeStoreFingerprint(db); + }).not.toThrow(); + const resolved = must(fingerprint); + expect(isStoreFingerprint(resolved)).toBe(true); + expect(resolved.quickCheckOk).toBe(false); + expect(resolved.schemaVersion).toBe(0); + expect(resolved.migrationsApplied).toEqual([]); + expect(resolved.tableRowCounts).toEqual({}); + } finally { + db.close(); + } + }); +}); + +// Wave 4a, epic #3233 §8 — a `store.opened` activity-log event once per successful open. +describe("openNodeUiDatabase — store.opened activity log (Wave 4a, epic #3233 §8)", () => { + it("emits exactly one store.opened event through the supplied sink on a successful open", () => { + const dbPath = join(tmpDir, "opened.db"); + const events: ServerLogEvent[] = []; + const sink: ServerLogSink = { + write: (event) => { + events.push(event); + }, + }; + const db = openNodeUiDatabase(dbPath, sink); + try { + expect(events).toHaveLength(1); + const event = must(events[0]); + expect(event.category).toBe("setup"); + expect(event.op).toBe("store.opened"); + expect(typeof event.durationMs).toBe("number"); + expect(event.durationMs ?? -1).toBeGreaterThanOrEqual(0); + expect(event.extra?.store).toBe("ui"); + expect(event.extra?.storeSchemaVersion).toBe(SCHEMA_VERSION); + expect(event.extra?.migrationsAppliedCount).toBe(SCHEMA_VERSION); + expect(event.extra?.quickCheckOk).toBe(true); + expect(event.extra?.encryptionMode).toBe("plaintext"); + // This store is never encrypted, so no key is ever resolved — `keySource` must be absent, + // not merely `undefined`, on the emitted event. + expect(event.extra !== undefined && "keySource" in event.extra).toBe(false); + } finally { + db.close(); + } + }); + + it("does not emit any event, and still opens normally, when no sink is supplied", () => { + const dbPath = join(tmpDir, "opened-no-sink.db"); + const db = openNodeUiDatabase(dbPath); + try { + const row = db.prepare("PRAGMA user_version").get() as + { readonly user_version?: number } | undefined; + expect(row?.user_version).toBe(SCHEMA_VERSION); + } finally { + db.close(); + } + }); + + it("still returns a working database when the supplied sink throws on write", () => { + const dbPath = join(tmpDir, "opened-sink-throws.db"); + const sink: ServerLogSink = { + write: () => { + throw new Error("boom"); + }, + }; + let db: DatabaseSync | undefined; + expect(() => { + db = openNodeUiDatabase(dbPath, sink); + }).not.toThrow(); + const opened = must(db); + try { + const row = opened.prepare("PRAGMA user_version").get() as + { readonly user_version?: number } | undefined; + expect(row?.user_version).toBe(SCHEMA_VERSION); + } finally { + opened.close(); + } + }); +}); diff --git a/packages/keiko-server/src/store/db.ts b/packages/keiko-server/src/store/db.ts index 0b38aeb5d0..296f2f0234 100644 --- a/packages/keiko-server/src/store/db.ts +++ b/packages/keiko-server/src/store/db.ts @@ -10,6 +10,14 @@ import { MAX_DESKTOP_CHAT_CLIENT_TURN_ID_CHARS, canonicalDesktopChatTurnReferenceSeed, } from "@oscharko-dev/keiko-contracts/bff-wire"; +import type { StoreFingerprint } from "@oscharko-dev/keiko-contracts"; +// Reused directly rather than re-declared: `store/db.ts` lives inside `keiko-server` itself, the +// same package that owns `ServerLogSink`/`ServerLogEvent`, so — unlike `KnowledgeLogSink` +// (`keiko-local-knowledge`) or `SecurityLogSink` (`keiko-security`), which each declare their own +// structural mirror because importing this package from BELOW it would invert ADR-0019's +// dependency direction — there is no boundary here to protect, and a third near-duplicate +// interface in the same package would be pure duplication (AGENTS.md §5). +import type { ServerLogEvent, ServerLogSink } from "../observability/index.js"; // Shared fs-hardening owner [GEN-MAINT-COUPLING-005]: the single 0o700/0o600 hardening pair. import { chmodIfPresent, @@ -43,7 +51,7 @@ import type { WorkspaceTrustRecordRow, WorkspaceTrustRecordRowInput, } from "./types.js"; -import { runMigrations } from "./schema.js"; +import { runMigrations, SCHEMA_VERSION } from "./schema.js"; import { deleteProject as sqlDeleteProject, getProject as sqlGetProject, @@ -784,6 +792,161 @@ function quarantineCorruptDb(target: string, cause?: unknown): void { ); } +// ─── StoreFingerprint (Wave 4a, epic #3233 §6.2) ─────────────────────────────────────────────── +// +// `keiko bundle export`'s manifest assembly calls this to embed a redacted, point-in-time +// snapshot of this store's schema/integrity state. Every field is a count, a closed-vocabulary +// label, or a bounded identifier — never a row, a path, a key, a secret, or free text. +// +// FIXED, closed table-name list this package already owns — enumerated explicitly from +// `schema.ts`'s own `CREATE TABLE` statements, never a dynamic `sqlite_master` walk. No migration +// through v19 has ever dropped or renamed one of these tables. +export const UI_STORE_FINGERPRINT_TABLES = [ + "projects", + "chats", + "chat_messages", + "relationships", + "relationship_lifecycle_history", + "relationship_audit_entries", + "task_workspace_instances", + "task_workspace_active_pointer", + "coding_runtime_snapshots", + "memory_autonomy_policy", + "workspace_trust_records", + "workspace_manifests", + "workspace_manifest_roots", +] as const; + +// `computeStoreFingerprint` is READ-ONLY and must never throw, even against a corrupted or +// half-written file — it is called from `keiko bundle export`, which must still produce a +// (degraded) manifest for the very store an operator is trying to diagnose. Every read below is +// therefore individually guarded and degrades in place rather than propagating. +function safeReadSchemaVersion(db: DatabaseSync): number { + try { + const row = db.prepare("PRAGMA user_version").get() as { user_version?: number } | undefined; + return typeof row?.user_version === "number" && Number.isInteger(row.user_version) + ? row.user_version + : 0; + } catch { + return 0; + } +} + +function boundedSchemaVersion(rawSchemaVersion: number): number { + if (rawSchemaVersion < 0) return 0; + // Clamped to the binary's own known ceiling: a value above it is unreadable noise from a + // corrupted header, not a real future schema this binary could ever have produced migrations for. + return Math.min(rawSchemaVersion, SCHEMA_VERSION); +} + +// This store's migrations are tracked only by `schema.ts`'s numeric `PRAGMA user_version`, not by +// named migration files, so a bounded identifier per applied version number ("v1".."vN") is this +// store's own honest rendering of "migration-group names, already tracked by the migration +// runner" — never a fabricated or borrowed name. +function migrationsAppliedFor(schemaVersion: number): readonly string[] { + return Array.from({ length: schemaVersion }, (_unused, index) => `v${String(index + 1)}`); +} + +function readQuickCheckOk(db: DatabaseSync): boolean { + try { + assertQuickCheckOk(db); + return true; + } catch { + return false; + } +} + +function readOneTableRowCount(db: DatabaseSync, table: string): number | undefined { + try { + // Table names cannot be bind parameters; `table` is always drawn from the fixed, package-owned + // `UI_STORE_FINGERPRINT_TABLES` constant above, never from caller input. + const row = db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get() as + { count?: number } | undefined; + return typeof row?.count === "number" ? row.count : undefined; + } catch { + // Absent (older schema, not yet migrated to this table) or unreadable — omit rather than fail + // the whole fingerprint over one table. + return undefined; + } +} + +function readTableRowCounts(db: DatabaseSync): Record { + const counts: Record = {}; + for (const table of UI_STORE_FINGERPRINT_TABLES) { + const count = readOneTableRowCount(db, table); + if (count !== undefined) counts[table] = count; + } + return counts; +} + +/** + * Redacted, point-in-time snapshot of this store's schema/integrity state (Wave 4a). Read-only + * and never throws, including against a corrupted or half-migrated file — a degraded fingerprint + * (e.g. `quickCheckOk: false`, an empty `tableRowCounts`) is always returned instead. + * + * This store is never encrypted at rest (content encryption in this codebase applies to Local + * Knowledge and the memory vault, not the UI store), so `encryptionMode` is always `"plaintext"` + * and `keySource` is always omitted. + */ +export function computeStoreFingerprint(db: DatabaseSync): StoreFingerprint { + const schemaVersion = boundedSchemaVersion(safeReadSchemaVersion(db)); + return { + store: "ui", + schemaVersion, + migrationsApplied: migrationsAppliedFor(schemaVersion), + tableRowCounts: readTableRowCounts(db), + quickCheckOk: readQuickCheckOk(db), + encryptionMode: "plaintext", + }; +} + +// ─── `store.opened` activity-log event (Wave 4a, epic #3233 §8) ─────────────────────────────── +// +// `openNodeUiDatabase` is where every real production caller and every test all necessarily pass +// through, so this is the one place that can honestly say the store just finished opening. A sink +// failure must never surface as a store-open failure — the real `processServerLogSink()` already +// cannot throw here (it degrades and self-reports through the process logger), so the guard below +// protects only a non-conforming sink a future caller or test might supply. +function startUiStoreOpenTimer(): () => number { + const startedAt = performance.now(); + return (): number => Math.round((performance.now() - startedAt) * 1000) / 1000; +} + +function buildUiStoreOpenedEvent(db: DatabaseSync, durationMs: number): ServerLogEvent { + const fingerprint = computeStoreFingerprint(db); + return { + category: "setup", + op: "store.opened", + durationMs, + extra: { + store: fingerprint.store, + // Named `storeSchemaVersion`, not `schemaVersion`: the latter is a RESERVED envelope field + // name on the log line itself (the log schema's own version) and would be silently dropped. + storeSchemaVersion: fingerprint.schemaVersion, + migrationsAppliedCount: fingerprint.migrationsApplied.length, + quickCheckOk: fingerprint.quickCheckOk, + encryptionMode: fingerprint.encryptionMode, + // `keySource` is omitted: this store is never encrypted, so no key is ever resolved. + }, + }; +} + +function emitUiStoreOpenedEvent(sink: ServerLogSink | undefined, event: ServerLogEvent): void { + if (sink === undefined) return; + try { + sink.write(event); + } catch { + try { + process.emitWarning("Keiko UI store activity log write failed.", { + type: "KeikoActivityLog", + code: "KEIKO_LOG_SINK_FAILED", + }); + } catch { + // The process warning channel is the last one there is; a report beyond it does not exist. + } + } +} + // Issue #639 — bound the SQLITE_BUSY window so concurrent UI/BFF writers (chat writes, // relationship writes, evidence-adjacent updates) wait for the writer lock for a short, bounded // interval instead of failing immediately. 5_000ms matches the conservative default we want for @@ -816,7 +979,14 @@ export function createInMemoryUiStore(opts?: UiStoreFactoryOptions): UiStore { // the same UI database file. The relationship V5 schema lives in this DB (schema.ts §V5); // keeping a single connection avoids WAL-coordination overhead. `createNodeUiStore` stays a // one-shot convenience for callers that do not need the underlying handle. -export function openNodeUiDatabase(dbPath: string): DatabaseSync { +// +// `sink` is optional and `KnowledgeLogSink`-shaped (Wave 4a, epic #3233 §8): when supplied, a +// single `store.opened` event is emitted once the open fully succeeds (recovery included), never +// on a path that still throws. Production wires `processServerLogSink()` in at the composition +// root (`deps.ts`); every other caller, and every existing test, keeps working unchanged with no +// sink at all. +export function openNodeUiDatabase(dbPath: string, sink?: ServerLogSink): DatabaseSync { + const elapsed = startUiStoreOpenTimer(); ensureDirHardened(dirname(dbPath)); let db = preparedDatabase(dbPath); try { @@ -839,9 +1009,22 @@ export function openNodeUiDatabase(dbPath: string): DatabaseSync { chmodIfPresent(dbPath, FILE_MODE); chmodIfPresent(`${dbPath}-wal`, FILE_MODE); chmodIfPresent(`${dbPath}-shm`, FILE_MODE); + emitUiStoreOpenedEvent(sink, buildUiStoreOpenedEvent(db, elapsed())); return db; } +// Genuinely read-only open for a diagnostic snapshot (Wave 4a, epic #3233 §6.2/§8): `node:sqlite`'s +// `readOnly` mode opens the file without ever running `PRAGMA journal_mode = WAL`, `runMigrations`, +// `sqlRecoverInterruptedClientTurns`, or the corruption-quarantine reopen loop `openNodeUiDatabase` +// runs above — every one of those is a write. A WAL-mode reader/writer elsewhere on the same file is +// unaffected: SQLite serves a read-only connection through the existing wal-index. Callers computing +// only `computeStoreFingerprint` must use this, never `openNodeUiDatabase`, so a diagnostic export +// can never flip a `client_turn_state`, apply a migration, or quarantine the very file an operator +// is trying to inspect. +export function openNodeUiDatabaseReadOnly(dbPath: string): DatabaseSync { + return new DatabaseSync(dbPath, { readOnly: true }); +} + export function buildUiStoreOverDatabase(db: DatabaseSync, opts?: UiStoreFactoryOptions): UiStore { return buildStore(db, resolveOptions(opts)); } diff --git a/packages/keiko-server/src/store/index.ts b/packages/keiko-server/src/store/index.ts index 8d3b11c93b..8554bbd2cc 100644 --- a/packages/keiko-server/src/store/index.ts +++ b/packages/keiko-server/src/store/index.ts @@ -52,9 +52,12 @@ export { export { runMigrations, SCHEMA_VERSION, UiStoreSchemaVersionError } from "./schema.js"; export { buildUiStoreOverDatabase, + computeStoreFingerprint, createInMemoryUiStore, createNodeUiStore, isProjectAvailable, openNodeUiDatabase, + openNodeUiDatabaseReadOnly, UI_DB_BUSY_TIMEOUT_MS, + UI_STORE_FINGERPRINT_TABLES, } from "./db.js"; diff --git a/packages/keiko-server/src/workspace-index-provider.ts b/packages/keiko-server/src/workspace-index-provider.ts index 8d885c66b2..2c4a45de41 100644 --- a/packages/keiko-server/src/workspace-index-provider.ts +++ b/packages/keiko-server/src/workspace-index-provider.ts @@ -12,6 +12,7 @@ import { resolveLocalVaultKey, type LocalVaultKeychainAccess, } from "@oscharko-dev/keiko-security/secret-vault"; +import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; import { emitServerDiagnostic, serverDiagnosticFromError, @@ -28,6 +29,9 @@ export interface ServerWorkspaceIndexProviderOptions { readonly env?: WorkspaceIndexEnv | undefined; readonly keychainAccess?: LocalVaultKeychainAccess | undefined; readonly diagnostics?: ServerDiagnosticSink | undefined; + // Optional activity-log seam (ADR-0019); the deps.ts composition root supplies + // `processServerLogSink()`. + readonly securityLogSink?: SecurityLogSink | undefined; } export type WorkspaceIndexProvider = (workspaceRoot: string) => WorkspaceIndex | undefined; @@ -300,6 +304,7 @@ export function createServerWorkspaceIndexProvider( keychainService: WORKSPACE_INDEX_KEYCHAIN_SERVICE, keyfileName: WORKSPACE_INDEX_KEYFILE, ...(options.keychainAccess === undefined ? {} : { keychainAccess: options.keychainAccess }), + sink: options.securityLogSink, }); const keyFingerprint = workspaceIndexKeyFingerprint(key); if (existing?.generation.active === true && existing.keyFingerprint === keyFingerprint) { From 2a5c423268c7e77ca90d999709d90ed1cd907607 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 09:27:07 +0200 Subject: [PATCH 02/19] test(coverage): cover the Wave 4a security and CLI seams and regenerate the package coverage baseline (#3239) Adds the tests that hold keiko-security and keiko-cli at their recorded floors after the Wave 4a sources landed (keychain fallback, vault shard faults, harness error classes, support export), and regenerates the package coverage baseline for the new files; no floor was lowered. Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 75 ++++++------ packages/keiko-cli/src/runner.test.ts | 5 + packages/keiko-cli/src/state-paths.test.ts | 23 ++++ packages/keiko-cli/src/support.test.ts | 25 ++++ packages/keiko-cli/src/ui.test.ts | 39 ++++++ .../keiko-security/src/errors/harness.test.ts | 72 +++++++++++ .../keiko-security/src/macos-keychain.test.ts | 61 ++++++++++ packages/keiko-security/src/redaction.test.ts | 15 +++ .../secret-vault.fs-fault-injection.test.ts | 112 ++++++++++++++++++ .../keiko-security/src/secret-vault.test.ts | 103 ++++++++++++++++ .../src/sqlite-corruption.test.ts | 27 +++++ .../src/windows-shortcuts.test.ts | 55 +++++++++ 12 files changed, 572 insertions(+), 40 deletions(-) create mode 100644 packages/keiko-security/src/errors/harness.test.ts create mode 100644 packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index 55abc03d2f..07da77a033 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,13 +8,13 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 387, - "totalLines": 4880, + "uncoveredLines": 381, + "totalLines": 4881, "coverage": { - "lines": 92.11, - "statements": 90.39, + "lines": 92.19, + "statements": 90.5, "branches": 85.19, - "functions": 93.23 + "functions": 93.7 } }, "keiko-connectors": { @@ -30,15 +30,15 @@ } }, "keiko-contracts": { - "files": 186, + "files": 187, "uncoveredFiles": 0, "uncoveredLines": 857, - "totalLines": 13852, + "totalLines": 13877, "coverage": { - "lines": 93.81, - "statements": 92.39, - "branches": 90.23, - "functions": 97.46 + "lines": 93.82, + "statements": 92.41, + "branches": 90.26, + "functions": 97.47 } }, "keiko-editor": { @@ -104,12 +104,12 @@ "keiko-local-knowledge": { "files": 128, "uncoveredFiles": 0, - "uncoveredLines": 754, - "totalLines": 9383, + "uncoveredLines": 755, + "totalLines": 9417, "coverage": { - "lines": 91.97, - "statements": 89.58, - "branches": 80.95, + "lines": 91.98, + "statements": 89.6, + "branches": 80.96, "functions": 94.31 } }, @@ -162,15 +162,15 @@ } }, "keiko-memory-vault": { - "files": 22, + "files": 23, "uncoveredFiles": 0, - "uncoveredLines": 78, - "totalLines": 953, + "uncoveredLines": 80, + "totalLines": 1032, "coverage": { - "lines": 91.82, - "statements": 90.46, - "branches": 85.55, - "functions": 91.34 + "lines": 92.25, + "statements": 90.99, + "branches": 85.98, + "functions": 91.42 } }, "keiko-model-gateway": { @@ -222,26 +222,26 @@ } }, "keiko-security": { - "files": 23, + "files": 24, "uncoveredFiles": 0, - "uncoveredLines": 11, - "totalLines": 614, + "uncoveredLines": 0, + "totalLines": 659, "coverage": { - "lines": 98.21, - "statements": 97.61, - "branches": 94.07, - "functions": 98.6 + "lines": 100, + "statements": 99.86, + "branches": 98.96, + "functions": 100 } }, "keiko-server": { - "files": 580, + "files": 581, "uncoveredFiles": 0, "uncoveredLines": 4516, - "totalLines": 56239, + "totalLines": 56320, "coverage": { - "lines": 91.97, - "statements": 89.27, - "branches": 81.88, + "lines": 91.98, + "statements": 89.28, + "branches": 81.89, "functions": 94.86 } }, @@ -342,11 +342,6 @@ "tolerance": 0, "lines": 90 }, - "packages/keiko-security/src/errors/harness.ts": { - "governance": "ratcheted", - "tolerance": 0.5, - "lines": 16.17 - }, "packages/keiko-server/src/editor/dap/dapCapsuleSupervisor.ts": { "governance": "absolute", "tolerance": 0, diff --git a/packages/keiko-cli/src/runner.test.ts b/packages/keiko-cli/src/runner.test.ts index 8e1469b40a..8248cdd523 100644 --- a/packages/keiko-cli/src/runner.test.ts +++ b/packages/keiko-cli/src/runner.test.ts @@ -163,6 +163,11 @@ describe("runCli", () => { ["verify", ["--help"], "keiko verify"], ["evaluate", ["--suite", "definitely-not-a-suite"], "unknown suite"], ["memory", [], "Usage:"], + // Both handlers are wrapped in a closure in COMMAND_HANDLERS (they thread `env`/build extra + // deps rather than being registered directly), so calling `runPromptEnhancerCli`/ + // `runSupportCli` straight from their own test files never exercises the wrapper itself. + ["prompt-enhancer", ["--help"], "keiko prompt-enhancer"], + ["support", ["--help"], "keiko support"], ] as const)("dispatches %s through the top-level command table", async (_name, rest, marker) => { const c = makeIo(); const code = await runCli([_name, ...rest], c.io); diff --git a/packages/keiko-cli/src/state-paths.test.ts b/packages/keiko-cli/src/state-paths.test.ts index 038f9d56d0..02fbc8969c 100644 --- a/packages/keiko-cli/src/state-paths.test.ts +++ b/packages/keiko-cli/src/state-paths.test.ts @@ -350,6 +350,29 @@ describe("scanRuntimeState — runtime-state manifest", () => { ); }); + it("retains a stray file dropped directly under a subtree that owns no files of its own", () => { + // `local-knowledge/` and `evidence/qi/figma-snapshots/` are classified subtrees that own + // NOTHING but their recognized child directories (`OWNS_NO_FILE`): a namespace/run-id + // directory is owned via `childSubtree`, but any plain FILE sitting directly inside them — + // which nothing in this manifest ever writes — must be retained, not silently swallowed. + const stateDir = join(makeRoot(), ".keiko"); + mkdirSync(join(stateDir, "local-knowledge", "default"), { recursive: true }); + touch(join(stateDir, "local-knowledge", "default", "capsules.db")); + touch(join(stateDir, "local-knowledge", "stray.txt")); + + const scan = scanRuntimeState(stateDir); + + expect(categoryOf(scan, "local-knowledge/default/capsules.db")).toBe("local-knowledge"); + expect(categoryOf(scan, "local-knowledge/stray.txt")).toBeUndefined(); + const retained = scan.retained.find((r) => r.relPath === "local-knowledge/stray.txt"); + expect(retained).toEqual({ + relPath: "local-knowledge/stray.txt", + absPath: join(stateDir, "local-knowledge", "stray.txt"), + reason: "unknown", + owned: false, + }); + }); + it("retains a customer file whose name only resembles a database (no prefix over-match)", () => { const stateDir = join(makeRoot(), ".keiko"); mkdirSync(stateDir, { recursive: true }); diff --git a/packages/keiko-cli/src/support.test.ts b/packages/keiko-cli/src/support.test.ts index 0bfaa96c72..0bcfb52f7d 100644 --- a/packages/keiko-cli/src/support.test.ts +++ b/packages/keiko-cli/src/support.test.ts @@ -205,6 +205,31 @@ describe("runSupportCli export", () => { expect(written).not.toContain(HEALTHY_AUDIT.stateDir); }); + it("records a log file that vanishes between discovery and read as skipped, not aborted", async () => { + // `discoverServerLogFiles` only `statSync`s each name (which succeeds on a directory too), so + // a directory sitting where `server.log` belongs passes discovery — the read step afterward + // (`readFileSync`) is what actually fails, with EISDIR. This is the same "vanished between two + // fs calls" shape the sink's own rotation/retention pruning produces, exercised deterministically + // instead of via a real race. + mkdirSync(join(stateDir, "logs", "server.log"), { recursive: true }); + + const c = makeIo(); + const code = await runSupportCli(["export", "--state-dir", stateDir], c.io, AUDIT_ENV, { + cwd: outDir, + now: () => new Date("2026-08-21T12:00:00.000Z"), + auditDeps: healthyAuditDeps(), + evidenceStore: createInMemoryEvidenceStore(), + }); + + expect(code).toBe(0); + const outPath = join(outDir, "keiko-support-2026-08-21T12-00-00.000Z.jsonl"); + const manifest: Record = JSON.parse( + readFileSync(outPath, "utf8").split("\n")[0] ?? "{}", + ) as Record; + expect(manifest.sourceLogFiles).toEqual([]); + expect(manifest.skippedLogFiles).toEqual([{ name: "server.log", errorKind: "EISDIR" }]); + }); + it("defaults stateDirSource to 'default' when neither --state-dir nor KEIKO_STATE_DIR is set", async () => { const cwdWithDefaultState = mkdtempSync(join(tmpdir(), "keiko-support-cli-default-")); mkdirSync(join(cwdWithDefaultState, ".keiko", "logs"), { recursive: true }); diff --git a/packages/keiko-cli/src/ui.test.ts b/packages/keiko-cli/src/ui.test.ts index 6cf04d25d8..761130f5bf 100644 --- a/packages/keiko-cli/src/ui.test.ts +++ b/packages/keiko-cli/src/ui.test.ts @@ -963,6 +963,45 @@ describe("runUiCli — node:sqlite re-exec guard (ADR-0013 D2)", () => { expect(child.listenerCount("exit")).toBeGreaterThanOrEqual(0); }); + // Without these forwarders, a Ctrl-C during re-exec kills the PARENT (whose own SIGINT + // listeners are the default Node behaviour: terminate) while the re-exec'd CHILD — the process + // actually doing the work — keeps running orphaned. The parent must relay the signal instead. + it("forwards SIGINT and SIGTERM to the re-exec'd child instead of leaving it orphaned", async () => { + const { io } = captureIo(); + const child = new EventEmitter() as EventEmitter & { + kill: (signal?: NodeJS.Signals) => boolean; + }; + const killedWith: (NodeJS.Signals | undefined)[] = []; + child.kill = (signal?: NodeJS.Signals): boolean => { + killedWith.push(signal); + return true; + }; + + const promise = runUiCli( + [], + io, + {}, + { + currentExecArgv: () => [], + sqliteProbe: () => false, + spawnFn: () => child as unknown as import("node:child_process").ChildProcess, + }, + ); + + // Everything up to and including `process.on("SIGINT"/"SIGTERM", ...)` inside the re-exec + // guard runs synchronously (spawnFn is called synchronously, and nothing awaits before the + // listeners are registered) — so both are already attached the instant `runUiCli` yields its + // pending promise back to this line, with no need to wait a tick first. + process.emit("SIGINT"); + process.emit("SIGTERM"); + expect(killedWith).toEqual(["SIGINT", "SIGTERM"]); + + child.emit("exit", 0, null); + expect(await promise).toBe(0); + // The forwarders must be gone once the child has exited (no listener leak). + expect(child.listenerCount("exit")).toBeGreaterThanOrEqual(0); + }); + // Regression pin (KEIKO-0443): the three "does not re-exec" tests below previously used the // invalid `["--host", "0.0.0.0"]` args, so parseUiArgsOrExit returned 2 *before* the sqlite // guard ran and neither `sqliteProbe`, the NODE_OPTIONS branch of `alreadyFlagged`, nor the diff --git a/packages/keiko-security/src/errors/harness.test.ts b/packages/keiko-security/src/errors/harness.test.ts new file mode 100644 index 0000000000..bbd336e1cb --- /dev/null +++ b/packages/keiko-security/src/errors/harness.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { + HARNESS_CODES, + HarnessError, + HarnessInternalError, + HarnessModelError, + HarnessToolError, + LimitExceededError, + toFailure, +} from "./harness.js"; + +describe("harness errors", () => { + it("redacts the message at construction", () => { + const secret = "sk-" + "u".repeat(24); + const error = new HarnessModelError(`failed with ${secret}`); + expect(error.message).not.toContain(secret); + expect(error.message).toContain("[REDACTED]"); + }); + + it("carries stable codes", () => { + expect(new HarnessModelError("m").code).toBe(HARNESS_CODES.MODEL_ERROR); + expect(new HarnessToolError("m").code).toBe(HARNESS_CODES.TOOL_ERROR); + expect(new HarnessInternalError("m").code).toBe(HARNESS_CODES.INTERNAL); + }); + + it("subclasses are HarnessError and real Error", () => { + expect(new HarnessToolError("m")).toBeInstanceOf(HarnessError); + expect(new HarnessInternalError("m")).toBeInstanceOf(Error); + }); + + describe("LimitExceededError", () => { + it("carries the caller-supplied code discriminant", () => { + const error = new LimitExceededError(HARNESS_CODES.MODEL_ERROR, "budget exceeded"); + expect(error.code).toBe(HARNESS_CODES.MODEL_ERROR); + expect(error).toBeInstanceOf(HarnessError); + }); + + it("defaults secrets to an empty list when omitted", () => { + // No third argument at all — exercises the `secrets: readonly string[] = []` default-arg + // branch, distinct from explicitly passing an empty array. + const error = new LimitExceededError(HARNESS_CODES.TOOL_ERROR, "no extra secrets here"); + expect(error.message).toBe("no extra secrets here"); + }); + + it("redacts a caller-supplied secret when one is passed", () => { + const error = new LimitExceededError(HARNESS_CODES.INTERNAL, "contains hunter2 value", [ + "hunter2", + ]); + expect(error.message).not.toContain("hunter2"); + }); + }); + + describe("toFailure", () => { + it("omits detail entirely when it is undefined", () => { + const failure = toFailure(HARNESS_CODES.MODEL_ERROR, "model call failed"); + expect(failure).toEqual({ + category: HARNESS_CODES.MODEL_ERROR, + message: "model call failed", + }); + expect(Object.keys(failure)).not.toContain("detail"); + }); + + it("carries detail when one is supplied", () => { + const failure = toFailure(HARNESS_CODES.TOOL_ERROR, "tool call failed", "raw diagnostic"); + expect(failure).toEqual({ + category: HARNESS_CODES.TOOL_ERROR, + message: "tool call failed", + detail: "raw diagnostic", + }); + }); + }); +}); diff --git a/packages/keiko-security/src/macos-keychain.test.ts b/packages/keiko-security/src/macos-keychain.test.ts index 7246d8b843..2bea2673e2 100644 --- a/packages/keiko-security/src/macos-keychain.test.ts +++ b/packages/keiko-security/src/macos-keychain.test.ts @@ -5,6 +5,7 @@ import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import type { SecurityLogEvent, SecurityLogSink } from "./log-port.js"; import { KEYCHAIN_SPAWN_TIMEOUT_MS, + emitKeychainFallback, readMacosKeychainSecret, writeMacosKeychainSecret, } from "./macos-keychain.js"; @@ -133,6 +134,18 @@ describe("readMacosKeychainSecret", () => { expect(read.kind).toBe(process.platform === "darwin" ? "absent" : "unavailable"); }); + it("resolves the default executable path itself when none is supplied, off darwin", () => { + // `resolveSpawn` computes the `options.executable ?? MACOS_SECURITY_EXECUTABLE` default + // unconditionally, before the darwin check short-circuits — so this exercises that default + // branch without ever spawning the real `/usr/bin/security` (the early non-darwin return fires + // first, exactly as the equivalent-executable HANGS-off-darwin test above proves for the + // caller-supplied path). + const started = process.hrtime.bigint(); + const read = readMacosKeychainSecret("svc", "acct", { platform: "linux" }); + expect(Number(process.hrtime.bigint() - started) / 1e6).toBeLessThan(1_000); + expect(read).toEqual({ kind: "unavailable" }); + }); + it("applies the production spawn bound when timeoutMs is omitted", () => { // The one call that really omits timeoutMs runs against the HANGING fixture on purpose: a // fast fixture would flake on a saturated runner (the incident this file hardens against), @@ -384,3 +397,51 @@ describe("readMacosKeychainSecret sink wiring", () => { }).not.toThrow(); }); }); + +// classifyBoundedExit's closed vocabulary end to end: readMacosKeychainSecret's own fixtures only +// ever reach "timeout" (HANGS) and "exit-status" (DENIES/REFUSES). The other two members — +// "signal" and "spawn-error" — describe spawn shapes `execFileSync` can produce that this suite's +// shell-script fixtures cannot fabricate on demand, so they are exercised directly through +// `emitKeychainFallback`, the exported reporter both keychain surfaces in this package share +// [GEN-MAINT-COUPLING-006]. "unknown" covers a thrown value that is not even an object. +describe("emitKeychainFallback — boundedExitKind classification", () => { + function recordingSink(): { sink: SecurityLogSink; events: SecurityLogEvent[] } { + const events: SecurityLogEvent[] = []; + return { + sink: { + write: (event): void => { + events.push(event); + }, + }, + events, + }; + } + + it("classifies a thrown non-object value as unknown", () => { + const { sink, events } = recordingSink(); + emitKeychainFallback(sink, "plain string throw", () => 0); + expect(events).toHaveLength(1); + expect(events[0]?.extra).toMatchObject({ boundedExitKind: "unknown" }); + }); + + it("classifies a thrown null as unknown", () => { + const { sink, events } = recordingSink(); + emitKeychainFallback(sink, null, () => 0); + expect(events).toHaveLength(1); + expect(events[0]?.extra).toMatchObject({ boundedExitKind: "unknown" }); + }); + + it("classifies a spawn killed by signal as signal", () => { + const { sink, events } = recordingSink(); + emitKeychainFallback(sink, { signal: "SIGKILL" }, () => 0); + expect(events).toHaveLength(1); + expect(events[0]?.extra).toMatchObject({ boundedExitKind: "signal" }); + }); + + it("classifies a spawn failure carrying a string code other than ETIMEDOUT as spawn-error", () => { + const { sink, events } = recordingSink(); + emitKeychainFallback(sink, { code: "ENOENT" }, () => 0); + expect(events).toHaveLength(1); + expect(events[0]?.extra).toMatchObject({ boundedExitKind: "spawn-error" }); + }); +}); diff --git a/packages/keiko-security/src/redaction.test.ts b/packages/keiko-security/src/redaction.test.ts index fe23f477e7..53801a1423 100644 --- a/packages/keiko-security/src/redaction.test.ts +++ b/packages/keiko-security/src/redaction.test.ts @@ -366,6 +366,21 @@ describe("credential key-name scanner", () => { it("does not flag adjacent non-secret metadata keys", () => { expect(objectContainsCredentialKey({ tokenCount: 42, secretariat: "team" })).toBe(false); }); + + it("terminates on a circular reference instead of recursing forever", () => { + // The `seen` WeakSet guard exists precisely for this shape: a self-referencing object would + // otherwise recurse without bound. No credential key anywhere in the cycle, so this must + // settle on false rather than exhausting the stack. + const cyclic: Record = { harmless: "value" }; + cyclic.self = cyclic; + expect(objectContainsCredentialKey(cyclic)).toBe(false); + }); + + it("still finds a credential key reachable before the cycle is revisited", () => { + const cyclic: Record = { client_secret: "opaque" }; + cyclic.self = cyclic; + expect(objectContainsCredentialKey(cyclic)).toBe(true); + }); }); describe("createAuditRedactor — env-value redaction", () => { diff --git a/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts b/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts new file mode 100644 index 0000000000..387ee8f95b --- /dev/null +++ b/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts @@ -0,0 +1,112 @@ +// Isolated in its own file so the `node:fs` module mock below never leaks into the rest of the +// secret-vault suite (secret-vault.test.ts): `vi.mock` is hoisted and applies to the WHOLE file's +// module graph, so keeping the blast radius to two narrow, precisely-targeted scenarios is +// deliberate. Both mocked functions pass every other call straight through to the real +// implementation, so nothing outside the exact matched path/flags combination is affected. +import { mkdtempSync, readdirSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createLocalSecretVault, + createShardedLocalSecretVault, + type LocalSecretVaultDeps, +} from "./secret-vault.js"; + +let blockedOpenDir = ""; +let blockedRenameDest = ""; + +vi.mock("node:fs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // fsyncDirectory's `try { fd = openSync(dir, "r"); ... } finally { if (fd !== undefined) ... }` + // (writeStore/writeShard's post-rename directory fsync) has a branch that is otherwise + // unreachable hermetically: `fd` stays `undefined` only when `openSync(dir, "r")` itself + // throws. Real permission tricks can't isolate that from the rest of the write — + // `ensureDirHardened` re-chmods the directory to 0700 immediately before the write, undoing + // any pre-set restriction. This scoped mock is the only hermetic way to exercise it. + openSync: (path: unknown, flags: unknown, mode?: unknown): number => { + if (path === blockedOpenDir && flags === "r") { + throw Object.assign(new Error("simulated: directory cannot be opened for fsync"), { + code: "EACCES", + }); + } + return (actual.openSync as (...args: unknown[]) => number)(path, flags, mode); + }, + // writeStore's `finally { if (existsSync(tempPath)) { try { unlinkSync(tempPath); } ... } }` + // cleanup only runs when the commit rename itself fails AFTER the temp file was written. For + // the single-file layout that is hermetically unreachable by blocking the destination with a + // real pre-existing path: `set()`/`replaceAll()`/`delete()` all call `readStore` on that exact + // path FIRST, so a blocking directory or unreadable file there makes `readStore` throw before + // `writeStore` (and its rename) is ever reached — proven by the sharded-layout version of this + // proof (secret-vault.test.ts), which works because sharded `set()` never reads its target + // first. This scoped mock fails only the rename itself, leaving the preceding read untouched. + renameSync: (oldPath: unknown, newPath: unknown): void => { + if (newPath === blockedRenameDest) { + throw Object.assign(new Error("simulated: rename destination refused"), { + code: "EACCES", + }); + } + (actual.renameSync as (...args: unknown[]) => void)(oldPath, newPath); + }, + }; +}); + +const KEY = Buffer.alloc(32, 7); +const REAL_TMPDIR = realpathSync(tmpdir()); +const dirs: string[] = []; + +afterEach(() => { + blockedOpenDir = ""; + blockedRenameDest = ""; + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); +}); + +function tempDir(prefix: string): string { + const created = realpathSync(mkdtempSync(join(REAL_TMPDIR, prefix))); + dirs.push(created); + return created; +} + +describe("fsyncDirectory — the directory itself cannot be opened for fsync", () => { + it("single-file layout: set() still commits and returns the secret when the post-rename directory fsync fails to open", () => { + const dir = tempDir("secret-vault-fsync-open-"); + blockedOpenDir = dir; + + const vault = createLocalSecretVault({ key: KEY, storePath: join(dir, "vault.enc.json") }); + expect(() => { + vault.set("cred:a", "value"); + }).not.toThrow(); + expect(vault.get("cred:a")).toBe("value"); + }); + + it("sharded layout: set() still commits and returns the secret when the post-rename directory fsync fails to open", () => { + const dir = tempDir("secret-vault-fsync-open-shard-"); + const storeDir = join(dir, "sharded"); + blockedOpenDir = storeDir; + + const vault = createShardedLocalSecretVault({ key: KEY, storeDir }); + expect(() => { + vault.set("cred:a", "value"); + }).not.toThrow(); + expect(vault.get("cred:a")).toBe("value"); + }); +}); + +describe("createLocalSecretVault — writeStore leaves no temp file behind when the commit rename fails", () => { + it("cleans up the temp file when renameSync cannot replace the store path", () => { + const dir = tempDir("secret-vault-rename-fail-"); + const storePath = join(dir, "vault.enc.json"); + const deps: LocalSecretVaultDeps = { key: KEY, storePath }; + blockedRenameDest = resolve(storePath); + + const vault = createLocalSecretVault(deps); + expect(() => { + vault.set("cred:a", "value"); + }).toThrow("simulated: rename destination refused"); + // The temp file the failed rename left behind must not survive — best-effort cleanup runs in + // writeStore's `finally`, exercising the branch where `existsSync(tempPath)` is true. + expect(readdirSync(dir).filter((n) => n.endsWith(".tmp"))).toEqual([]); + }); +}); diff --git a/packages/keiko-security/src/secret-vault.test.ts b/packages/keiko-security/src/secret-vault.test.ts index 40bddf44a1..b276fcc55f 100644 --- a/packages/keiko-security/src/secret-vault.test.ts +++ b/packages/keiko-security/src/secret-vault.test.ts @@ -188,6 +188,30 @@ describe("resolveLocalVaultKey — KEYFILE tier", () => { }); }); +describe("resolveLocalVaultKey — default keychainAccess when the option is omitted", () => { + const originalPlatform = process.platform; + + afterEach(() => { + Object.defineProperty(process, "platform", { value: originalPlatform, configurable: true }); + }); + + it("builds the default macOS keychain access itself and falls through to keyfile off darwin", () => { + // No `keychainAccess` in these options at all — computeLocalVaultKey must construct the default + // (createKeychainVaultKeyAccess-backed) access itself. Off darwin that default returns undefined + // WITHOUT spawning `security`, so this stays hermetic while still exercising the real default + // construction path rather than an injected stand-in. + Object.defineProperty(process, "platform", { value: "linux", configurable: true }); + const resolved = resolveLocalVaultKey({ + env: {}, + vaultDir: dir, + envVarName: "KEIKO_TEST_VAULT_KEY", + keychainService: "keiko-test-vault", + keyfileName: "test-vault.key", + }); + expect(resolved.source).toBe("keyfile"); + }); +}); + // gap g18: the key-source fact (which tier answered) was previously invisible even on the // ordinary, no-fallback path. `security.vault.key-resolved` closes that — it fires every time a // tier resolves, independent of whether anything went wrong. @@ -597,6 +621,56 @@ describe("createLocalSecretVault — symlink guard", () => { }); }); +describe("createLocalSecretVault — non-symlink lstat failure propagates unmodified", () => { + it("propagates a non-ENOENT lstat failure raised while walking ancestor path segments", () => { + // A path segment that is itself a plain FILE (not a directory) makes lstat on anything beneath + // it fail with ENOTDIR rather than ENOENT — the "keychain refused, not merely absent" analogue + // for the symlink-guard walk: only ENOENT is swallowed as "not a symlink", every other lstat + // failure must propagate as-is rather than being reinterpreted as the deliberate guard error. + const blockedFile = join(dir, "blocked-file"); + writeFileSync(blockedFile, "not a directory"); + const storePath = join(blockedFile, "sub", "vault.enc.json"); + const vault = vaultAt(storePath); + + let caught: unknown; + try { + vault.set("cred:a", "value"); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as NodeJS.ErrnoException).code).toBe("ENOTDIR"); + // Not the deliberate symlink-guard error: this is the raw fs failure propagating unmodified. + expect(String(caught)).not.toContain("symlinked path"); + }); +}); + +describe("createLocalSecretVault — fsyncDirectory skips the directory fsync on win32", () => { + const originalPlatform = process.platform; + + afterEach(() => { + Object.defineProperty(process, "platform", { value: originalPlatform, configurable: true }); + }); + + it("still completes a write when the platform is win32 (POSIX directory fsync is meaningless there)", () => { + Object.defineProperty(process, "platform", { value: "win32", configurable: true }); + const storePath = join(dir, "vault.enc.json"); + const vault = vaultAt(storePath); + + expect(() => { + vault.set("cred:a", "value"); + }).not.toThrow(); + expect(vault.get("cred:a")).toBe("value"); + }); +}); + +// NOTE: the analogous "leaves no temp file behind when the commit rename fails" proof for the +// SHARDED layout (below, near writeShard) blocks the target with a real pre-existing directory +// because sharded set() never reads its target before writing. The single-file layout's set() +// always reads the CURRENT store content first (readStore), so the same trick makes readStore +// itself fail first (EISDIR) — never reaching writeStore's rename at all. That version of this +// proof lives in secret-vault.fs-fault-injection.test.ts, using a scoped renameSync mock instead. + describe("createLocalSecretVault — additional isStoreFile branches", () => { it("a store file containing JSON null fails closed", () => { const storePath = join(dir, "vault.enc.json"); @@ -725,6 +799,35 @@ describe("createKeychainVaultKeyAccess", () => { expect(createKeychainVaultKeyAccess("svc", runner)()).toBeUndefined(); }); + it("regenerates the key when the stored keychain value cannot be decoded (corrupt/garbage bytes)", () => { + setPlatform("darwin"); + // Base64 of 16 bytes decodes cleanly but to the wrong length, so decodeKeyOrThrow rejects it — + // distinct from "keychain has none" (item-not-found): here the read SUCCEEDS with a value this + // vault cannot use, and the undecodable-value path must replace it exactly as generate-on-miss does. + const undecodable = Buffer.alloc(16, 2).toString("base64"); + const commands: string[][] = []; + const runner = (args: readonly string[]): string => { + commands.push([...args]); + if (args[0] === "find-generic-password") return `${undecodable}\n`; + return ""; + }; + const key = createKeychainVaultKeyAccess("svc", runner)(); + expect(key?.length).toBe(32); + expect(commands.map((c) => c[0])).toEqual(["find-generic-password", "add-generic-password"]); + }); + + it("returns undefined when the item-not-found path's own key generation fails (add error)", () => { + setPlatform("darwin"); + const commands: string[][] = []; + const runner = (args: readonly string[]): string => { + commands.push([...args]); + if (args[0] === "find-generic-password") throw itemNotFound(); + throw new Error("add-generic-password failed"); + }; + expect(createKeychainVaultKeyAccess("svc", runner)()).toBeUndefined(); + expect(commands.map((c) => c[0])).toEqual(["find-generic-password", "add-generic-password"]); + }); + // The keychain key-tier path spawns `security` through its own injectable `KeychainCommandRunner` // rather than calling `readMacosKeychainSecret`, so it needs its own proof that a read failure // (other than "no such item") reports `security.keychain.fallback` — the same shape diff --git a/packages/keiko-security/src/sqlite-corruption.test.ts b/packages/keiko-security/src/sqlite-corruption.test.ts index 12498ded4f..2d7f9b1c30 100644 --- a/packages/keiko-security/src/sqlite-corruption.test.ts +++ b/packages/keiko-security/src/sqlite-corruption.test.ts @@ -3,6 +3,7 @@ import { SqliteQuickCheckError, errorRecord, isSqliteCorruptionError, + safeName, sqliteErrorLike, sqliteErrorText, } from "./sqlite-corruption.js"; @@ -82,6 +83,32 @@ describe("shape helpers", () => { }); }); +describe("safeName", () => { + it("returns the value when the reflective read finds a string name", () => { + expect(safeName(new Error("boom"))).toBe("Error"); + expect(safeName({ name: "CustomName" })).toBe("CustomName"); + }); + + it("returns undefined when there is no name property at all", () => { + expect(safeName({})).toBeUndefined(); + }); + + it("returns undefined when name is present but not a string", () => { + expect(safeName({ name: 42 })).toBeUndefined(); + }); + + it("returns undefined instead of throwing when the reflective read itself throws", () => { + // A hostile getter on `name` must not escape this reflective read — it is exercised over + // untrusted thrown values, which may be adversarially shaped. + const hostile = { + get name(): string { + throw new Error("getter boom"); + }, + }; + expect(safeName(hostile)).toBeUndefined(); + }); +}); + // ── 0.3.0 audit item 6: the persisted quarantine diagnostic is hardened ──────── // // `errorRecord` output is written VERBATIM into `.corrupt..diagnostic.json` by three stores, diff --git a/packages/keiko-security/src/windows-shortcuts.test.ts b/packages/keiko-security/src/windows-shortcuts.test.ts index b5c1e6d80f..67a2a6a816 100644 --- a/packages/keiko-security/src/windows-shortcuts.test.ts +++ b/packages/keiko-security/src/windows-shortcuts.test.ts @@ -56,6 +56,24 @@ describe("windows shortcut fallback codec", () => { `${JSON.stringify({ schema: "keiko-windows-shortcut-v1", targetPath: "x" })}\n`, ], ["non-object", '"just a string"\n'], + [ + "non-string targetPath", + `${JSON.stringify({ + schema: "keiko-windows-shortcut-v1", + targetPath: 123, + workingDirectory: "wd", + iconPath: "ip", + })}\n`, + ], + [ + "non-string iconPath", + `${JSON.stringify({ + schema: "keiko-windows-shortcut-v1", + targetPath: "tp", + workingDirectory: "wd", + iconPath: 123, + })}\n`, + ], ])("refuses a %s fallback document", (_label, content) => { const path = join(tempRoot(), "Keiko.lnk"); writeFileSync(path, content, "utf8"); @@ -132,6 +150,14 @@ describe("definition read/write entry points on the win32 route", () => { expect(spawnFn).toHaveBeenCalledTimes(1); expect(spawnFn.mock.calls[0]?.[1]).toContain("create"); }); + + it("returns undefined instead of throwing when the underlying cscript call fails", () => { + // On the win32 route, readWindowsShortcutDefinition is a fail-closed READ: any refusal from + // runWindowsShortcutCommand (a nonzero exit here) must be swallowed, not propagated. + stubWin32(); + const spawnFn = vi.fn(() => spawnResult({ status: 1 })); + expect(readWindowsShortcutDefinition("p", {}, "test prefix", spawnFn)).toBeUndefined(); + }); }); describe("runWindowsShortcutCommand", () => { @@ -182,6 +208,35 @@ describe("runWindowsShortcutCommand", () => { expect(spawnFn.mock.calls[0]?.[0]).toBe(String.raw`C:\Windows\System32\cscript.exe`); }); + it("passes TEMP/TMP through to the script-host environment when the caller's env carries them", () => { + const spawnFn = vi.fn(() => spawnResult()); + runWindowsShortcutCommand( + "read", + "p", + DEFINITION, + { SystemRoot: String.raw`C:\Windows`, TEMP: String.raw`C:\Temp`, TMP: String.raw`C:\Tmp` }, + "test prefix", + spawnFn, + ); + expect(spawnFn.mock.calls[0]?.[2]?.env).toEqual({ + SystemRoot: String.raw`C:\Windows`, + WINDIR: String.raw`C:\Windows`, + ComSpec: String.raw`C:\Windows\System32\cmd.exe`, + TEMP: String.raw`C:\Temp`, + TMP: String.raw`C:\Tmp`, + }); + }); + + it("falls back to an empty buffer when stdout/stderr are null instead of throwing", () => { + // The WindowsShortcutSpawnFn type allows a null stdout/stderr (matches spawnSync's own return + // shape); every OTHER fixture in this file supplies a real Buffer, so the `?? Buffer.alloc(0)` + // fallback on each field is otherwise never exercised. + const spawnFn = vi.fn(() => + spawnResult({ stdout: null, stderr: null }), + ); + expect(runWindowsShortcutCommand("read", "p", DEFINITION, {}, "test prefix", spawnFn)).toBe(""); + }); + it("fails closed on a nonzero exit", () => { const spawnFn = vi.fn(() => spawnResult({ status: 1 })); expect(() => From e1fde8e83bedd49472a24dad7540fe14996397cd Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 09:56:24 +0200 Subject: [PATCH 03/19] feat(observability): HTTP/SSE lifecycle detail, diagnostic-label reduction, body-free browser diagnostic ingest (#3240) Wave 5 of #3233 (ADR-0173 D13): the request line carries the exact matched route template, query parameter names, response bytes and an aborted flag (status 0 for a request abandoned before any write); every SSE stream ends with one sse.stream.closed line (frames, bytes, duration, closed reason); accepted request bodies log http.request.body.received and six ad hoc body readers share readBoundedRequestBody; diagnostic operation labels never carry a raw path; POST /api/diagnostics/client ingests a bounded, body-free browser diagnostic with a re-validated correlation id behind a process-wide rate limit, and the browser reports fan out to it with the ApiError correlation id at ten real call sites. Co-Authored-By: Claude Fable 5 --- ...-log-v2-machine-reconstruction-contract.md | 56 +++ docs/observability/op-catalog.generated.json | 34 +- .../keiko-contracts/src/diagnostics.test.ts | 111 ++++++ packages/keiko-contracts/src/diagnostics.ts | 120 ++++++ packages/keiko-contracts/src/index.ts | 14 + .../src/bounded-request-body.test.ts | 41 +- .../keiko-server/src/bounded-request-body.ts | 40 ++ .../src/client-diagnostics-routes.test.ts | 235 +++++++++++ .../src/client-diagnostics-routes.ts | 199 +++++++++ .../keiko-server/src/diagnostics-log.test.ts | 61 +++ packages/keiko-server/src/diagnostics-log.ts | 36 +- .../keiko-server/src/gateway-readiness.ts | 41 +- packages/keiko-server/src/gitRoutes.ts | 71 ++-- .../src/http-lifecycle.e2e.test.ts | 377 ++++++++++++++++++ .../src/memory-consolidation-handlers.test.ts | 16 + .../src/memory-consolidation-handlers.ts | 48 +-- .../keiko-server/src/memory-conv-handlers.ts | 50 +-- .../keiko-server/src/memory-handlers.test.ts | 20 + packages/keiko-server/src/memory-handlers.ts | 62 +-- .../src/observability/route-template.ts | 2 + .../qualityIntelligence/modelPolicyRoutes.ts | 48 +-- .../src/relationship-handlers.test.ts | 24 ++ .../keiko-server/src/relationship-handlers.ts | 51 +-- .../src/request-cancellation.test.ts | 63 ++- .../keiko-server/src/request-cancellation.ts | 31 +- packages/keiko-server/src/routes.ts | 5 + .../src/run-handlers-sse-backpressure.test.ts | 165 ++++++++ packages/keiko-server/src/run-handlers.ts | 7 +- packages/keiko-server/src/server.test.ts | 217 +++++++++- packages/keiko-server/src/server.ts | 256 +++++++++--- packages/keiko-server/src/sse-write.test.ts | 153 ++++++- packages/keiko-server/src/sse-write.ts | 118 ++++++ packages/keiko-server/src/sse.ts | 32 +- .../src/app/atlassian-connectors/error.tsx | 3 +- .../components/desktop/AppShellBoundary.tsx | 6 +- .../desktop/hooks/useUnhandledRejectionLog.ts | 3 +- .../app/components/desktop/shellRecovery.ts | 3 +- .../widgets/cards/sharedEventSource.test.ts | 34 ++ .../widgets/cards/sharedEventSource.ts | 22 + .../useRelationshipActivityStream.test.tsx | 30 ++ .../panels/useRelationshipActivityStream.ts | 22 + .../desktop/windows/WindowBodyBoundary.tsx | 3 +- .../src/app/local-knowledge/capsule/error.tsx | 3 +- .../src/lib/client-diagnostics.test.ts | 62 ++- .../keiko-ui/src/lib/client-diagnostics.ts | 34 +- .../keiko-ui/src/lib/client-error-summary.ts | 25 ++ .../coding-workbench-event-retention.test.ts | 31 +- .../lib/coding-workbench-event-retention.ts | 22 + .../lib/coding-workbench-runtime-effects.ts | 3 +- .../lib/install-client-diagnostics.test.ts | 185 ++++++++- .../src/lib/install-client-diagnostics.ts | 178 ++++++++- .../lib/useCodingWorkbenchApprovalReview.ts | 3 +- .../lib/useCodingWorkbenchResearch.test.tsx | 43 ++ .../src/lib/useCodingWorkbenchResearch.ts | 3 +- packages/keiko-ui/src/lib/useSSE.test.tsx | 28 ++ packages/keiko-ui/src/lib/useSSE.ts | 22 + .../lib/verified-task-workspace-binding.ts | 5 +- 57 files changed, 3232 insertions(+), 345 deletions(-) create mode 100644 packages/keiko-contracts/src/diagnostics.test.ts create mode 100644 packages/keiko-contracts/src/diagnostics.ts create mode 100644 packages/keiko-server/src/client-diagnostics-routes.test.ts create mode 100644 packages/keiko-server/src/client-diagnostics-routes.ts create mode 100644 packages/keiko-server/src/http-lifecycle.e2e.test.ts create mode 100644 packages/keiko-server/src/run-handlers-sse-backpressure.test.ts diff --git a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md index 9cad737238..a9aa86beb7 100644 --- a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md +++ b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md @@ -438,6 +438,59 @@ impossible to violate, because there is no longer more than one copy to diverge. relaxation of the pin, and no future change may cite this ADR to justify re-introducing a second copy without a single source of truth. +### D13 — HTTP and SSE lifecycle detail, and a body-free browser diagnostic ingest (Wave 5, landed) + +Wave 5 closes the gap between "a request line exists" and "a request line is enough to reproduce the +request": + +- **The `request` line carries the exact matched route, not a guess.** `routeTemplate` is the + `RouteDefinition.pattern` the dispatcher actually resolved (`/api/relationships/:id`), recorded by + a per-request context the dispatcher fills and the close-time writer reads — never derived from + the raw path after the fact, where a customer id shaped like a route word would misclassify. + Unmatched and static requests fall back to `redactRoutePath`. `queryParamNames` lists the query + parameter NAMES only (deduplicated, shape-checked against a bounded identifier pattern, sorted, + capped at 16; anything dropped is counted in `queryParamDroppedCount`), never a value. + `responseBytes` is the byte count `writeJson` already computed and previously discarded. + `aborted` is computed at `close` by the shared `requestAlreadyClosed` predicate, and a request the + client abandoned before any write logs `status: 0` instead of Node's default `200` — the + predicate was corrected in the same wave so a normally ended response (which Node also marks + `destroyed`) is not mistaken for an abort. +- **Every SSE stream ends with exactly one terminal line.** `sse.stream.closed` (`frameCount`, + `bytesStreamed`, `durationMs`, `reason`) is emitted once per response on its `close` event by the + shared frame recorder every SSE writer already funnels through; `reason` is the closed + vocabulary `completed | client-disconnected | backpressure-killed | server-error`, and a write path + that destroys the socket because a write was rejected marks the stream first so a backpressure + kill is never reported as a client disconnect. `http.request.body.received` records the media + type and byte count of an accepted request body; the six ad hoc body readers that predated + `readBoundedRequestBody` were consolidated onto it so the line — and the 413 path — have one + owner. +- **Diagnostic `operation` labels never carry a raw request path.** `diagnosticLabel` reduces a + path-bearing operation label through the same route reducer the activity log uses and degrades to + the fixed `server.operation` fallback when the path cannot be templated; the two git diff handlers + now pass their route literal instead of `ctx.url.pathname` at the source. +- **The browser reports to the log, body-free.** `POST /api/diagnostics/client` accepts a + `ClientDiagnosticIngestRequest` (`keiko-contracts`) — a bounded message, `clientTs`, an optional + SSE `readyState`, a closed `kind`, and an optional `correlationId` that the server re-validates + with `isValidCorrelationId` and drops otherwise — and writes `client.diagnostic` with the message + redacted into `clientNote` (never under a `message` key) behind a process-wide token bucket + (reusing the editor's inline-completion limiter, 60 s window) that logs one + `client.diagnostic.rate-limited` line per window carrying the count of further drops it + suppressed, and answers `204` whether a report was kept or dropped. + The route's body reader returns a module-tagged outcome, not a duck-typed `RouteResult`, so a + client body shaped like `{status, body}` can never be reflected as the route's own response. On the + browser side the existing `reportClientDiagnostic` sink fans out to the console and to this route + (best-effort, throttled, never awaited); the four native `EventSource.onerror` sites report + `readyState` and a closed reason label, and every call site that catches an `ApiError` passes its + `correlationId` through a structured `meta` argument — the join that lets an agent pair a + browser-visible failure with the exact server request line it came from. Producers that + structurally have no id (native `EventSource`, message-only notices) say so in their doc comments + rather than inventing one. + +The agent-reading step this adds: **for a failed request**, read `routeTemplate`, +`queryParamNames`, `responseBytes`, `aborted` and — for a stream — the `sse.stream.closed` line's +`reason`, then look for a `client.diagnostic` line sharing the `correlationId` to learn what the +browser saw. Everything on these lines is a count, a closed label, a template, or an id. + ### D12 — Relation to prior decisions - **ADR-0010** (audit ledger and evidence manifests) established the precedent this contract @@ -502,6 +555,9 @@ rather than left implicit across the Decision section: further evidence-gap classes (stack-frame unions, gateway replay scripts, and other later-wave evidence). A warning names exactly what evidence class is missing and why, so an agent's report to a human names the actual gap instead of guessing. +8. **For a failed or slow request** (landed Wave 5), read the `request` line's `routeTemplate`, + `queryParamNames`, `responseBytes` and `aborted`, the stream's `sse.stream.closed` `reason`, and + any `client.diagnostic` line that shares the request's `correlationId` (D13). ## Consequences diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index d22b3c042f..4f53996aed 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -593,7 +593,7 @@ { "op": "", "category": "diagnostic", - "site": "packages/keiko-server/src/diagnostics-log.ts:213", + "site": "packages/keiko-server/src/diagnostics-log.ts:214", "package": "keiko-server" }, { @@ -620,6 +620,18 @@ "site": "packages/keiko-server/src/observability/server-logger.ts:170", "package": "keiko-server" }, + { + "op": "client.diagnostic", + "category": "diagnostic", + "site": "packages/keiko-server/src/client-diagnostics-routes.ts:119", + "package": "keiko-server" + }, + { + "op": "client.diagnostic.rate-limited", + "category": "diagnostic", + "site": "packages/keiko-server/src/client-diagnostics-routes.ts:101", + "package": "keiko-server" + }, { "op": "embedding.memory.failed", "category": "embedding", @@ -671,19 +683,25 @@ { "op": "http.request.body.cancelled", "category": "http", - "site": "packages/keiko-server/src/bounded-request-body.ts:88", + "site": "packages/keiko-server/src/bounded-request-body.ts:92", "package": "keiko-server" }, { "op": "http.request.body.failed", "category": "http", - "site": "packages/keiko-server/src/bounded-request-body.ts:102", + "site": "packages/keiko-server/src/bounded-request-body.ts:106", + "package": "keiko-server" + }, + { + "op": "http.request.body.received", + "category": "http", + "site": "packages/keiko-server/src/bounded-request-body.ts:142", "package": "keiko-server" }, { "op": "http.request.body.rejected", "category": "http", - "site": "packages/keiko-server/src/bounded-request-body.ts:73", + "site": "packages/keiko-server/src/bounded-request-body.ts:77", "package": "keiko-server" }, { @@ -773,7 +791,7 @@ { "op": "request", "category": "http", - "site": "packages/keiko-server/src/server.ts:323", + "site": "packages/keiko-server/src/server.ts:468", "package": "keiko-server" }, { @@ -793,6 +811,12 @@ "category": "diagnostic", "site": "packages/keiko-server/src/observability/server-log.ts:338", "package": "keiko-server" + }, + { + "op": "sse.stream.closed", + "category": "http", + "site": "packages/keiko-server/src/sse-write.ts:69", + "package": "keiko-server" } ], "violations": [] diff --git a/packages/keiko-contracts/src/diagnostics.test.ts b/packages/keiko-contracts/src/diagnostics.test.ts new file mode 100644 index 0000000000..efee2836fa --- /dev/null +++ b/packages/keiko-contracts/src/diagnostics.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; + +import { + CLIENT_DIAGNOSTIC_KINDS, + CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH, + CLIENT_DIAGNOSTIC_READY_STATES, + isClientDiagnosticIngestRequest, + isClientDiagnosticKind, +} from "./diagnostics.js"; + +function validRequest(): Record { + return { + message: "boundary caught TypeError", + clientTs: "2026-08-21T10:00:00.000Z", + }; +} + +describe("isClientDiagnosticIngestRequest", () => { + it("accepts the minimal required shape", () => { + expect(isClientDiagnosticIngestRequest(validRequest())).toBe(true); + }); + + it("accepts every optional field populated with an in-range value", () => { + for (const readyState of CLIENT_DIAGNOSTIC_READY_STATES) { + for (const kind of CLIENT_DIAGNOSTIC_KINDS) { + expect( + isClientDiagnosticIngestRequest({ + ...validRequest(), + readyState, + correlationId: "abcdefgh", + kind, + }), + ).toBe(true); + } + } + }); + + it("rejects a non-object value", () => { + expect(isClientDiagnosticIngestRequest(null)).toBe(false); + expect(isClientDiagnosticIngestRequest(undefined)).toBe(false); + expect(isClientDiagnosticIngestRequest("a string")).toBe(false); + expect(isClientDiagnosticIngestRequest(["array"])).toBe(false); + }); + + it("rejects a missing or non-string message", () => { + expect(isClientDiagnosticIngestRequest({ clientTs: validRequest().clientTs })).toBe(false); + expect(isClientDiagnosticIngestRequest({ ...validRequest(), message: 42 })).toBe(false); + }); + + it("rejects an empty message", () => { + expect(isClientDiagnosticIngestRequest({ ...validRequest(), message: "" })).toBe(false); + }); + + it(`rejects a message over ${String(CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH)} characters`, () => { + const tooLong = "a".repeat(CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH + 1); + expect(isClientDiagnosticIngestRequest({ ...validRequest(), message: tooLong })).toBe(false); + }); + + it(`accepts a message at exactly ${String(CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH)} characters`, () => { + const atLimit = "a".repeat(CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH); + expect(isClientDiagnosticIngestRequest({ ...validRequest(), message: atLimit })).toBe(true); + }); + + it("rejects a missing or malformed clientTs", () => { + expect(isClientDiagnosticIngestRequest({ message: validRequest().message })).toBe(false); + expect(isClientDiagnosticIngestRequest({ ...validRequest(), clientTs: "not-a-date" })).toBe( + false, + ); + expect(isClientDiagnosticIngestRequest({ ...validRequest(), clientTs: "2026-08-21" })).toBe( + false, + ); + }); + + it("rejects a readyState outside the closed 0|1|2 vocabulary", () => { + expect(isClientDiagnosticIngestRequest({ ...validRequest(), readyState: 3 })).toBe(false); + expect(isClientDiagnosticIngestRequest({ ...validRequest(), readyState: "1" })).toBe(false); + }); + + it("rejects a kind outside the closed vocabulary", () => { + expect(isClientDiagnosticIngestRequest({ ...validRequest(), kind: "crash" })).toBe(false); + }); + + it("rejects a correlationId that is empty or over the bounded length", () => { + expect(isClientDiagnosticIngestRequest({ ...validRequest(), correlationId: "" })).toBe(false); + expect( + isClientDiagnosticIngestRequest({ ...validRequest(), correlationId: "a".repeat(129) }), + ).toBe(false); + }); + + // This guard only asserts wire SHAPE (AGENTS.md: "reuse first" — the leaf must not duplicate + // `correlation.ts`'s alphabet policy). A shape-conforming but semantically invalid id is the + // server route's job to reject via `isValidCorrelationId`, never this guard's. + it("accepts a correlationId shape that a stricter server-side policy would still reject", () => { + expect( + isClientDiagnosticIngestRequest({ ...validRequest(), correlationId: "not valid!!" }), + ).toBe(true); + }); +}); + +describe("isClientDiagnosticKind", () => { + it("accepts every declared kind", () => { + for (const kind of CLIENT_DIAGNOSTIC_KINDS) { + expect(isClientDiagnosticKind(kind)).toBe(true); + } + }); + + it("rejects a value outside the closed vocabulary", () => { + expect(isClientDiagnosticKind("crash")).toBe(false); + expect(isClientDiagnosticKind(1)).toBe(false); + }); +}); diff --git a/packages/keiko-contracts/src/diagnostics.ts b/packages/keiko-contracts/src/diagnostics.ts new file mode 100644 index 0000000000..0e16b78742 --- /dev/null +++ b/packages/keiko-contracts/src/diagnostics.ts @@ -0,0 +1,120 @@ +// Wire contract for `POST /api/diagnostics/client` (Wave 5 of epic #3233, ADR-0173). +// +// The browser is untrusted input, so this shape — and its guard — live in the leaf contracts +// layer like every other request shape the server accepts from a client it does not control +// (ADR-0019). `correlationId` is the fatal-flaw fix all three design-panel judges independently +// flagged as missing: it is what lets an agent deterministically join a browser crash report to +// the specific failed server request it is reporting on, instead of fuzzy timestamp matching. It +// is DESIGNED to be populated from the same correlation id already threaded into every +// `ApiError`/SSE event (`packages/keiko-ui/src/lib/http.ts`), and is re-validated server-side with +// `isValidCorrelationId` before it is trusted — this guard only admits its general SHAPE (a short, +// bounded string), never the full correlation-id policy, which is server plumbing +// (`packages/keiko-server/src/correlation.ts`), not a wire concern. +// +// `install-client-diagnostics.ts` is the only place a `ClientDiagnosticIngestRequest` is built. +// `reportClientDiagnostic` (client-diagnostics.ts) takes an optional structured second argument, +// `{ correlationId?: string }`, and every call site that catches an `ApiError` (which +// `bffFetchJson`, keiko-ui's http.ts, stamps a `.correlationId` on for every non-2xx and every +// contract-validation failure) passes it through via `correlationIdOf(error)` +// (client-error-summary.ts). `install-client-diagnostics.ts` re-validates the shape client-side +// before putting it on the wire. It stays genuinely absent for the four SSE `onerror` call sites +// (sharedEventSource.ts, useSSE.ts, coding-workbench-event-retention.ts, +// useRelationshipActivityStream.ts): the native `EventSource` API exposes no response headers to +// page script, so there is no id to recover there — a hard platform limit, not a wiring gap. +// +// This guard is a promise about SHAPE only, never about the CONTENT of `message`. The browser-side +// sink's own doc comment promises an already-redacted string, but that promise is a library +// contract on the browser side — never a trust boundary the server may rely on. The server treats +// `message` as hostile regardless, and writes it under `extra.clientNote`, never `extra.message` +// (`"message"` is on `log-redaction.ts`'s `DENIED_FIELD_NAMES` and would collapse to +// `[redacted:key]` even though the value is already length-bounded here); the existing log-value +// guards (length/secret/personal/prose/path) do the actual content safety work on `clientNote`. + +// EventSource.readyState at the moment the browser observed the failure: CONNECTING (0), OPEN (1) +// or CLOSED (2). A closed vocabulary, not a raw number, so a future EventSource-shaped value can +// never smuggle an out-of-range number onto the wire. +export const CLIENT_DIAGNOSTIC_READY_STATES = [0, 1, 2] as const; +export type ClientDiagnosticReadyState = (typeof CLIENT_DIAGNOSTIC_READY_STATES)[number]; + +// What raised the diagnostic: a caught render-boundary error, an unhandled promise rejection, an +// SSE transport failure, or anything else a call site does not further classify. +export const CLIENT_DIAGNOSTIC_KINDS = [ + "boundary", + "unhandled-rejection", + "sse-error", + "other", +] as const; +export type ClientDiagnosticKind = (typeof CLIENT_DIAGNOSTIC_KINDS)[number]; + +// The browser-side sink already bounds a diagnostic message to this length (client-diagnostics.ts, +// `reportClientDiagnostic`); the server enforces the SAME bound independently rather than trusting +// the browser's own promise, per this module's header. +export const CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH = 200; + +const CORRELATION_ID_MAX_LENGTH = 128; +const ISO_INSTANT_MAX_LENGTH = 40; + +// Deliberately less strict than `correlation.ts`'s SAFE_CORRELATION_ID: this file only asserts the +// wire SHAPE (a short, non-empty string) so the leaf never has to import server plumbing. The +// server re-validates with `isValidCorrelationId` before trusting the value for anything. +const ISO_INSTANT_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/; + +export interface ClientDiagnosticIngestRequest { + readonly message: string; + readonly clientTs: string; + readonly readyState?: ClientDiagnosticReadyState | undefined; + readonly correlationId?: string | undefined; + readonly kind?: ClientDiagnosticKind | undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isBoundedString(value: unknown, maxLength: number): value is string { + return typeof value === "string" && value.length > 0 && value.length <= maxLength; +} + +function isIsoInstant(value: unknown): value is string { + return ( + isBoundedString(value, ISO_INSTANT_MAX_LENGTH) && + ISO_INSTANT_PATTERN.test(value) && + !Number.isNaN(Date.parse(value)) + ); +} + +const CLIENT_DIAGNOSTIC_READY_STATE_SET: ReadonlySet = new Set( + CLIENT_DIAGNOSTIC_READY_STATES, +); + +function isClientDiagnosticReadyState(value: unknown): value is ClientDiagnosticReadyState { + return typeof value === "number" && CLIENT_DIAGNOSTIC_READY_STATE_SET.has(value); +} + +const CLIENT_DIAGNOSTIC_KIND_SET: ReadonlySet = new Set(CLIENT_DIAGNOSTIC_KINDS); + +export function isClientDiagnosticKind(value: unknown): value is ClientDiagnosticKind { + return typeof value === "string" && CLIENT_DIAGNOSTIC_KIND_SET.has(value); +} + +function isCorrelationIdShape(value: unknown): value is string { + return isBoundedString(value, CORRELATION_ID_MAX_LENGTH); +} + +// A value present under an optional key must still conform to `guard`; absent is always accepted. +function isOptional(value: unknown, guard: (candidate: unknown) => boolean): boolean { + return value === undefined || guard(value); +} + +export function isClientDiagnosticIngestRequest( + value: unknown, +): value is ClientDiagnosticIngestRequest { + if (!isRecord(value)) return false; + const { message, clientTs, readyState, correlationId, kind } = value; + if (!isBoundedString(message, CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH)) return false; + if (!isIsoInstant(clientTs)) return false; + if (!isOptional(readyState, isClientDiagnosticReadyState)) return false; + if (!isOptional(correlationId, isCorrelationIdShape)) return false; + if (!isOptional(kind, isClientDiagnosticKind)) return false; + return true; +} diff --git a/packages/keiko-contracts/src/index.ts b/packages/keiko-contracts/src/index.ts index 8d39e0bf33..c054ba186f 100644 --- a/packages/keiko-contracts/src/index.ts +++ b/packages/keiko-contracts/src/index.ts @@ -4768,3 +4768,17 @@ export { PROMPT_CANDIDATE_RANKING_EXPECTED_ORDER, PROMPT_CANDIDATE_RANKING_FIXTURE, } from "./prompt-enhancer-ranking-fixture.js"; + +// ─── Client diagnostics ingest wire contract (Wave 5 of epic #3233) ───────────── +export { + CLIENT_DIAGNOSTIC_KINDS, + CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH, + CLIENT_DIAGNOSTIC_READY_STATES, + isClientDiagnosticIngestRequest, + isClientDiagnosticKind, +} from "./diagnostics.js"; +export type { + ClientDiagnosticIngestRequest, + ClientDiagnosticKind, + ClientDiagnosticReadyState, +} from "./diagnostics.js"; diff --git a/packages/keiko-server/src/bounded-request-body.test.ts b/packages/keiko-server/src/bounded-request-body.test.ts index 3c7ecd717d..b25efe2f8d 100644 --- a/packages/keiko-server/src/bounded-request-body.test.ts +++ b/packages/keiko-server/src/bounded-request-body.test.ts @@ -306,8 +306,8 @@ describe("bounded request body activity log", () => { ]); }); - it("writes nothing when the body stays inside the limit", async () => { - const sink = captureServerLog("debug"); + it("writes nothing at info when the body stays inside the limit", async () => { + const sink = captureServerLog("info"); await expect( readBoundedRequestBody(asRequest(Readable.from([Buffer.from("hello")])), 5), @@ -316,6 +316,43 @@ describe("bounded request body activity log", () => { expect(sink.events).toEqual([]); }); + it("logs one debug success line with the reduced media type and byte count", async () => { + const sink = captureServerLog("debug"); + const stream = new PassThrough(); + const req = asRequest(stream); + Object.defineProperty(req, "headers", { + configurable: true, + value: { "content-type": "application/json; charset=utf-8" }, + }); + const outcome = readBoundedRequestBody(req, 128_000, undefined, "req-ok-01"); + + stream.end(Buffer.from("hello")); + + await expect(outcome).resolves.toBe("hello"); + expect(sink.events).toEqual([ + { + level: "debug", + category: "http", + op: "http.request.body.received", + correlationId: "req-ok-01", + durationMs: undefined, + status: undefined, + errorKind: undefined, + extra: { contentType: "application/json", receivedBytes: 5 }, + }, + ]); + }); + + it("falls back to the closed 'unspecified' label when no Content-Type header was sent", async () => { + const sink = captureServerLog("debug"); + const req = asRequest(Readable.from([Buffer.from("hi")])); + Object.defineProperty(req, "headers", { configurable: true, value: {} }); + + await expect(readBoundedRequestBody(req, 128_000)).resolves.toBe("hi"); + + expect(sink.events[0]?.extra).toEqual({ contentType: "unspecified", receivedBytes: 2 }); + }); + it("logs one rejection even when a late data event arrives after the read settled", async () => { const sink = captureServerLog("info"); const stream = new PassThrough(); diff --git a/packages/keiko-server/src/bounded-request-body.ts b/packages/keiko-server/src/bounded-request-body.ts index aa6f18ae64..61e01962d8 100644 --- a/packages/keiko-server/src/bounded-request-body.ts +++ b/packages/keiko-server/src/bounded-request-body.ts @@ -2,6 +2,10 @@ import type { IncomingMessage } from "node:http"; import { errorKindOf, getServerLogger } from "./observability/index.js"; +// A raw Node header value: absent, a single value, or (for a repeated header) several. Shared by +// every helper below that reads `Content-Type` off a request, so the union is spelled once. +type ContentTypeHeaderValue = string | string[] | undefined; + export class RequestBodyTooLargeError extends Error { public constructor() { super("request body too large"); @@ -106,6 +110,41 @@ function logBodyFailed( }); } +// `IncomingMessage.headers` is typed as always present, but several test doubles across the +// codebase construct a stream cast to `IncomingMessage` without setting it — read defensively so a +// caller that never populated it does not crash the very read it is trying to observe. +function safeContentTypeHeader(req: IncomingMessage): ContentTypeHeaderValue { + const headers = req as { headers?: IncomingMessage["headers"] }; + return headers.headers?.["content-type"]; +} + +// Reduces a `Content-Type` header to its media type, discarding parameters (`; charset=utf-8`, +// `; boundary=...`) that can carry caller-chosen, unbounded text. Mirrors `isJsonRequest`'s +// reduction in `server.ts`. A media type is left readable by `log-redaction.ts`'s deep-path guard +// on purpose (it is at most one `/`, never three-plus path segments), so no further redaction is +// needed once it is isolated this way. +function mediaTypeOf(header: ContentTypeHeaderValue): string { + const value = typeof header === "string" ? header : header?.[0]; + const mediaType = value?.split(";", 1)[0]?.trim().toLowerCase(); + return mediaType === undefined || mediaType.length === 0 ? "unspecified" : mediaType; +} + +// The one success line this reader emits, at debug: per-request volume makes it unfit for info, +// but it is what closes the loop for `keiko log:analyze` — the rejected/cancelled/failed paths +// above already say what went wrong; this says what a body that just worked actually looked like. +function logBodyReceived( + correlationId: string | undefined, + contentType: ContentTypeHeaderValue, + receivedBytes: number, +): void { + getServerLogger().debug(() => ({ + category: "http" as const, + op: "http.request.body.received", + ...(correlationId === undefined ? {} : { correlationId }), + extra: { contentType: mediaTypeOf(contentType), receivedBytes }, + })); +} + class BoundedRequestBodyReader { private readonly chunks: Buffer[] = []; private total = 0; @@ -173,6 +212,7 @@ class BoundedRequestBodyReader { if (this.settled) return; this.settled = true; this.cleanup(); + logBodyReceived(this.correlationId, safeContentTypeHeader(this.req), this.total); this.resolve(Buffer.concat(this.chunks).toString("utf8")); }; diff --git a/packages/keiko-server/src/client-diagnostics-routes.test.ts b/packages/keiko-server/src/client-diagnostics-routes.test.ts new file mode 100644 index 0000000000..9f7c188626 --- /dev/null +++ b/packages/keiko-server/src/client-diagnostics-routes.test.ts @@ -0,0 +1,235 @@ +import { IncomingMessage, ServerResponse } from "node:http"; +import { Socket } from "node:net"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + handleClientDiagnosticIngest, + resetClientDiagnosticsIngestStateForTests, +} from "./client-diagnostics-routes.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, + type ServerLogEvent, +} from "./observability/index.js"; +import type { RouteContext } from "./routes.js"; + +const CORRELATION_ID = "diagnostics-route-test"; +const CLIENT_TS = "2026-08-21T10:00:00.000Z"; + +function request(rawBody: string): IncomingMessage { + const req = new IncomingMessage(new Socket()); + req.push(rawBody); + req.push(null); + return req; +} + +function context( + rawBody: string, + correlationId: string | undefined = CORRELATION_ID, +): RouteContext { + const req = request(rawBody); + return { + req, + res: new ServerResponse(req), + params: {}, + url: new URL("http://localhost/api/diagnostics/client"), + correlationId, + }; +} + +function captureServerLog(): BufferedServerLogSink { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "debug" })); + return sink; +} + +// The bounded-body reader this route shares with every other route also writes its own +// `http.request.body.received`/`.rejected` lines, independent of whether this route's OWN report +// is accepted. Every assertion below is scoped to this route's own op, never the raw sink, so it +// stays correct regardless of what else the shared body reader logs. +function clientDiagnosticEvents(sink: BufferedServerLogSink): readonly ServerLogEvent[] { + return sink.events.filter((event) => event.op === "client.diagnostic"); +} + +function clientDiagnosticLine(sink: BufferedServerLogSink): Record { + const index = sink.events.findIndex((event) => event.op === "client.diagnostic"); + expect(index).toBeGreaterThanOrEqual(0); + const line = sink.lines()[index]; + expect(line).toBeDefined(); + return JSON.parse(line ?? "{}") as Record; +} + +describe("POST /api/diagnostics/client", () => { + beforeEach(() => { + resetClientDiagnosticsIngestStateForTests(); + }); + + afterEach(() => { + resetServerLogger(); + resetClientDiagnosticsIngestStateForTests(); + }); + + // The FATAL-FLAW FIX (all three design-panel judges independently flagged it): the field is + // exactly what lets an agent join a browser crash report to the specific failed server request + // it describes — the ORIGINAL request's correlation id, never this POST's own. + it("accepts a well-formed report, always with 204, and round-trips a valid correlationId", async () => { + const sink = captureServerLog(); + const body = JSON.stringify({ + message: "boundary caught TypeError", + clientTs: CLIENT_TS, + readyState: 2, + correlationId: "original-request-correlation-id", + kind: "boundary", + }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result).toEqual({ status: 204, body: null }); + const events = clientDiagnosticEvents(sink); + expect(events).toHaveLength(1); + const [event] = events; + expect(event?.category).toBe("diagnostic"); + expect(event?.correlationId).toBe("original-request-correlation-id"); + expect(event?.errorKind).toBe("boundary"); + }); + + // FATAL-FLAW FIX #2 (graft from the reuse-maximal design): `"message"` is on + // `log-redaction.ts`'s DENIED_FIELD_NAMES and would collapse to `[redacted:key]` even though the + // value is already bounded — the wire field named `message` must never reach the log line under + // that same name. + it("projects the wire field literally named 'message' onto extra.clientNote, never extra.message", async () => { + const sink = captureServerLog(); + const body = JSON.stringify({ message: "boundary caught TypeError", clientTs: CLIENT_TS }); + + await handleClientDiagnosticIngest(context(body)); + + const line = clientDiagnosticLine(sink); + expect(line).not.toHaveProperty("message"); + expect(line.clientNote).toBe("boundary caught TypeError"); + }); + + it("drops an invalid correlationId instead of rejecting the whole report", async () => { + const sink = captureServerLog(); + // Fails `isValidCorrelationId`'s alphabet (spaces and `!` are not in [A-Za-z0-9._-]), but is a + // conforming wire STRING, so the contract guard alone must not be trusted for this field. + const body = JSON.stringify({ + message: "unhandled rejection", + clientTs: CLIENT_TS, + correlationId: "not valid!!", + }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result).toEqual({ status: 204, body: null }); + expect(clientDiagnosticEvents(sink)[0]?.correlationId).toBeUndefined(); + }); + + it("rejects an oversized body with 413 and never reaches the logger", async () => { + const sink = captureServerLog(); + // Comfortably over MAX_CLIENT_DIAGNOSTIC_BODY_BYTES (4096) in raw bytes, so the bounded reader + // itself rejects the body before JSON parsing or shape validation ever runs. + const body = JSON.stringify({ message: "x".repeat(5_000), clientTs: CLIENT_TS }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result.status).toBe(413); + expect(clientDiagnosticEvents(sink)).toEqual([]); + }); + + it("rejects a message over the 200-character wire bound with 400", async () => { + const sink = captureServerLog(); + const body = JSON.stringify({ message: "y".repeat(201), clientTs: CLIENT_TS }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result.status).toBe(400); + expect(clientDiagnosticEvents(sink)).toEqual([]); + }); + + it("rejects malformed JSON with 400", async () => { + const sink = captureServerLog(); + + const result = await handleClientDiagnosticIngest(context("{not json")); + + expect(result.status).toBe(400); + expect(clientDiagnosticEvents(sink)).toEqual([]); + }); + + it("rejects a body missing the required clientTs field with 400", async () => { + const sink = captureServerLog(); + const body = JSON.stringify({ message: "no timestamp" }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result.status).toBe(400); + expect(clientDiagnosticEvents(sink)).toEqual([]); + }); + + it("redacts a hostile message carrying an email address", async () => { + const sink = captureServerLog(); + const body = JSON.stringify({ + message: "contact jane.doe@example.com for help", + clientTs: CLIENT_TS, + }); + + await handleClientDiagnosticIngest(context(body)); + + expect(clientDiagnosticLine(sink).clientNote).toBe("[redacted:personal]"); + }); + + it("redacts a hostile message carrying an API-key-shaped secret", async () => { + const sink = captureServerLog(); + // The secret pattern is anchored at the start of the value, so the message must BEGIN with a + // recognised key prefix rather than merely contain one. + const body = JSON.stringify({ message: `sk-ant-${"a".repeat(40)}`, clientTs: CLIENT_TS }); + + await handleClientDiagnosticIngest(context(body)); + + expect(clientDiagnosticLine(sink).clientNote).toBe("[redacted:secret]"); + }); + + // Regression for the confusable-shape trust-boundary finding: a parsed JSON body that happens to + // look like this route's own internal `RouteResult` sentinel (a numeric `status` plus a `body` + // key) must never be echoed back verbatim, and must still go through shape validation, the rate + // limiter, and the logger like any other malformed report. + it("never reflects a client body shaped like {status, body} as the route's own response", async () => { + const sink = captureServerLog(); + const body = JSON.stringify({ status: 200, body: { secret: "attacker-controlled" } }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result).not.toEqual({ status: 200, body: { secret: "attacker-controlled" } }); + expect(result.status).toBe(400); + expect(clientDiagnosticEvents(sink)).toEqual([]); + }); + + it("never forges an out-of-range status code from a client body shaped like a RouteResult", async () => { + const body = JSON.stringify({ status: 599, body: "arbitrary" }); + + const result = await handleClientDiagnosticIngest(context(body)); + + expect(result.status).toBe(400); + }); + + it("drops the 61st report within the same rolling minute, still answering 204", async () => { + const sink = captureServerLog(); + const results: number[] = []; + for (let index = 0; index < 61; index += 1) { + const body = JSON.stringify({ + message: `report number ${String(index)}`, + clientTs: CLIENT_TS, + }); + const result = await handleClientDiagnosticIngest(context(body)); + results.push(result.status); + } + + expect(results.every((status) => status === 204)).toBe(true); + expect(sink.events.filter((event) => event.op === "client.diagnostic")).toHaveLength(60); + expect( + sink.events.filter((event) => event.op === "client.diagnostic.rate-limited"), + ).toHaveLength(1); + }); +}); diff --git a/packages/keiko-server/src/client-diagnostics-routes.ts b/packages/keiko-server/src/client-diagnostics-routes.ts new file mode 100644 index 0000000000..6fbbd7aed2 --- /dev/null +++ b/packages/keiko-server/src/client-diagnostics-routes.ts @@ -0,0 +1,199 @@ +// `POST /api/diagnostics/client` (Wave 5 of epic #3233, ADR-0173). +// +// The browser-side sink (`packages/keiko-ui/src/lib/client-diagnostics.ts`) buffers an already +// bounded, already-redacted-by-convention diagnostic string, but that promise is a LIBRARY +// contract on the browser side — never a trust boundary the server may rely on. This route treats +// every field as hostile input and re-validates it independently: +// +// * shape and length come from `isClientDiagnosticIngestRequest` (keiko-contracts, the leaf); +// * `correlationId` is only ever trusted after `isValidCorrelationId` — the same server-side +// policy every other correlation id on this server goes through (`correlation.ts`) — so a +// browser cannot inject an arbitrary join key onto another request's timeline; +// * the message text is written under `extra.clientNote`, NEVER `extra.message`: `"message"` is +// on `log-redaction.ts`'s `DENIED_FIELD_NAMES` and would collapse to `[redacted:key]` even +// though the value is already length-bounded by the wire guard. `clientNote` normalises to +// `"clientnote"`, which is not denied, so the existing per-value guards (length/secret/ +// personal/prose/path) do the actual content-safety work on the value itself — the same +// "structural, not caller discipline" principle `log-redaction.ts`'s own header states. No new +// redaction path is written here; the existing choke point is applied a second time, +// server-side, exactly as it is for every other logged field. +// +// Rate limiting reuses `createInlineCompletionRateLimiter` (the editor's existing token-bucket +// primitive — AGENTS.md §5 forbids a second one) as a single, process-wide bucket: a flapping tab +// or a hostile page must not be able to grow the activity log without bound. The response is 204 +// whether a report was accepted or dropped by the limiter — the limit itself is never disclosed to +// the browser — and a dropped report is still counted, mirroring `server-log.ts`'s own +// `reportServerLogFailure` throttle-and-count-suppressed shape: the first drop after a quiet window +// is logged immediately, later drops in the same window are counted silently, and the count is +// flushed on the next window's first drop. + +import type { IncomingMessage } from "node:http"; + +import { + isClientDiagnosticIngestRequest, + type ClientDiagnosticIngestRequest, +} from "@oscharko-dev/keiko-contracts"; + +import { + RequestBodyCancelledError, + RequestBodyTooLargeError, + readBoundedRequestBody, +} from "./bounded-request-body.js"; +import { isValidCorrelationId } from "./correlation.js"; +import { + createInlineCompletionRateLimiter, + type InlineCompletionRateLimiter, +} from "./editor/inlineCompletionRateLimiter.js"; +import { getServerLogger } from "./observability/index.js"; +import { errorBody, type RouteContext, type RouteResult } from "./routes.js"; + +// A diagnostic report is a handful of short fields — generous relative to the wire guard's own +// 200-character message cap, never a channel for an attached body. +const MAX_CLIENT_DIAGNOSTIC_BODY_BYTES = 4_096; + +// Process-wide, not per-connection (this endpoint identifies no session): 60 accepted reports per +// rolling minute is generous for genuine crash/error reporting and bounds a flooding or hostile +// page. `minIntervalMs: 0` disables the limiter's own burst/cooldown gate, so only the sliding +// window cap below applies. +const CLIENT_DIAGNOSTIC_RATE_LIMIT_KEY = "client-diagnostics"; +let rateLimiter: InlineCompletionRateLimiter = createInlineCompletionRateLimiter({ + minIntervalMs: 0, + maxPerWindow: 60, + windowMs: 60_000, +}); + +const DROP_NOTICE_WINDOW_MS = 60_000; + +interface DropNoticeState { + readonly lastAt: number | null; + readonly suppressed: number; +} + +let dropNotice: DropNoticeState = { lastAt: null, suppressed: 0 }; + +/** Test-only: puts the shared rate limiter and drop-notice counter back to a clean start. */ +export function resetClientDiagnosticsIngestStateForTests(): void { + rateLimiter = createInlineCompletionRateLimiter({ + minIntervalMs: 0, + maxPerWindow: 60, + windowMs: 60_000, + }); + dropNotice = { lastAt: null, suppressed: 0 }; +} + +function dropNoticeThrottled(now: number): boolean { + if (dropNotice.lastAt === null) return false; + const elapsed = now - dropNotice.lastAt; + return elapsed >= 0 && elapsed < DROP_NOTICE_WINDOW_MS; +} + +// Reports a rate-limited drop exactly once per window, carrying how many further drops that same +// window suppressed — never the report content, which was never admitted past the limiter. +function noticeRateLimitedDrop(now: number): void { + if (dropNoticeThrottled(now)) { + dropNotice = { lastAt: dropNotice.lastAt, suppressed: dropNotice.suppressed + 1 }; + return; + } + const suppressed = dropNotice.suppressed; + dropNotice = { lastAt: now, suppressed: 0 }; + getServerLogger().warn({ + category: "diagnostic", + op: "client.diagnostic.rate-limited", + ...(suppressed > 0 ? { extra: { suppressedDrops: suppressed } } : {}), + }); +} + +// Projects the validated request onto the activity log. `message` is admitted only as +// `extra.clientNote` (see module header); `readyState`/`kind` ride along as bounded, closed-shape +// fields the value guards pass through unchanged. +function logClientDiagnostic(request: ClientDiagnosticIngestRequest): void { + const extra: Record = { clientNote: request.message }; + if (request.readyState !== undefined) extra.readyState = request.readyState; + if (request.kind !== undefined) extra.clientKind = request.kind; + const correlationId = + request.correlationId !== undefined && isValidCorrelationId(request.correlationId) + ? request.correlationId + : undefined; + getServerLogger().warn({ + category: "diagnostic", + op: "client.diagnostic", + ...(correlationId === undefined ? {} : { correlationId }), + ...(request.kind === undefined ? {} : { errorKind: request.kind }), + extra, + }); +} + +// Discriminates a rejected read (already a fully-formed `RouteResult`) from a successfully parsed +// body, instead of duck-typing the parsed value's shape. Attacker-controlled JSON can legally +// contain a numeric `status` field and a `body` key (e.g. `{"status":200,"body":{...}}`), which +// would collide with a shape test and let the client's own parsed JSON be returned verbatim as this +// route's HTTP response — bypassing `isClientDiagnosticIngestRequest`, the rate limiter, and the +// logger entirely. A tagged union makes that collision structurally impossible: the tag is set by +// this module, never derived from the parsed value. +type BodyReadOutcome = + | { readonly kind: "rejected"; readonly result: RouteResult } + | { readonly kind: "parsed"; readonly value: unknown }; + +function badRequest(message: string, correlationId: string | undefined): RouteResult { + return { status: 400, body: errorBody("BAD_REQUEST", message, correlationId) }; +} + +async function readClientDiagnosticBody( + req: IncomingMessage, + correlationId: string | undefined, +): Promise { + let raw: string; + try { + raw = await readBoundedRequestBody( + req, + MAX_CLIENT_DIAGNOSTIC_BODY_BYTES, + undefined, + correlationId, + ); + } catch (error) { + if (error instanceof RequestBodyTooLargeError) { + return { + kind: "rejected", + result: { + status: 413, + body: errorBody( + "PAYLOAD_TOO_LARGE", + "Request body exceeds the size limit.", + correlationId, + ), + }, + }; + } + if (error instanceof RequestBodyCancelledError) { + return { + kind: "rejected", + result: { status: 499, body: errorBody("REQUEST_CANCELLED", "Request was cancelled.") }, + }; + } + throw error; + } + try { + return { kind: "parsed", value: JSON.parse(raw) as unknown }; + } catch { + return { + kind: "rejected", + result: badRequest("Request body is not valid JSON.", correlationId), + }; + } +} + +export async function handleClientDiagnosticIngest(ctx: RouteContext): Promise { + const outcome = await readClientDiagnosticBody(ctx.req, ctx.correlationId); + if (outcome.kind === "rejected") return outcome.result; + const parsed = outcome.value; + if (!isClientDiagnosticIngestRequest(parsed)) { + return badRequest("Request body is not a valid diagnostic report.", ctx.correlationId); + } + const now = Date.now(); + if (!rateLimiter.tryAcquire(CLIENT_DIAGNOSTIC_RATE_LIMIT_KEY, now)) { + noticeRateLimitedDrop(now); + return { status: 204, body: null }; + } + logClientDiagnostic(parsed); + return { status: 204, body: null }; +} diff --git a/packages/keiko-server/src/diagnostics-log.test.ts b/packages/keiko-server/src/diagnostics-log.test.ts index 6c92d91ff0..cbfefd3082 100644 --- a/packages/keiko-server/src/diagnostics-log.test.ts +++ b/packages/keiko-server/src/diagnostics-log.test.ts @@ -272,6 +272,67 @@ describe("emitServerDiagnostic (RB-6)", () => { expect(JSON.stringify(record)).not.toContain(labelMarker); }); + // ADR-0173 D3-adjacent follow-up (#3235 review, Wave 5): an `operation` label built from a live + // request path (`GET ${ctx.url.pathname}`-shaped) could carry a customer-chosen route segment + // verbatim onto the record. `diagnosticLabel` now reduces the path portion of an operation label + // through the SAME `redactRoutePath` reducer the HTTP request line uses, rather than accepting + // any string that merely matches the shape regex. + describe("operation labels never carry a raw request path (review follow-up, #3235)", () => { + it("reduces a path with a customer-named segment to its route template", () => { + const customerMarker = "fixture-customer-named-repository"; + const record = serverDiagnosticFromError({ + correlationId: "cid-path-template", + operation: `GET /api/git/repositories/${customerMarker}/status`, + source: "unit", + error: new Error("placeholder"), + redact: identity, + now: () => 0, + }); + + expect(record.operation).toBe("GET /api/git/repositories/{id}/status"); + expect(JSON.stringify(record)).not.toContain(customerMarker); + }); + + it("leaves an operation label with no path component unchanged", () => { + const record = serverDiagnosticFromError({ + correlationId: "cid-no-path", + operation: "chat.stream", + source: "unit", + error: new Error("placeholder"), + redact: identity, + now: () => 0, + }); + + expect(record.operation).toBe("chat.stream"); + }); + + it("falls back to server.operation for a traversal-shaped path", () => { + const record = serverDiagnosticFromError({ + correlationId: "cid-traversal", + operation: "GET /api/git/../../etc/passwd", + source: "unit", + error: new Error("placeholder"), + redact: identity, + now: () => 0, + }); + + expect(record.operation).toBe("server.operation"); + }); + + it("keeps a fully-literal route path unchanged (every segment is a declared route word)", () => { + const record = serverDiagnosticFromError({ + correlationId: "cid-literal-path", + operation: "POST /api/desktop/chat/stream", + source: "unit", + error: new Error("placeholder"), + redact: identity, + now: () => 0, + }); + + expect(record.operation).toBe("POST /api/desktop/chat/stream"); + }); + }); + it("routes the record to the provided sink", () => { const records: ServerDiagnosticRecord[] = []; const record = serverDiagnosticFromError({ diff --git a/packages/keiko-server/src/diagnostics-log.ts b/packages/keiko-server/src/diagnostics-log.ts index bc97799131..c67e138fb5 100644 --- a/packages/keiko-server/src/diagnostics-log.ts +++ b/packages/keiko-server/src/diagnostics-log.ts @@ -6,6 +6,7 @@ import { safeProperty, } from "./observability/error-classification.js"; import { closeReasonVocabulary } from "./observability/log-redaction.js"; +import { redactRoutePath } from "./observability/route-template.js"; import { createFileServerLogSink } from "./observability/server-log.js"; import { causeChain, keikoStackFrames } from "./observability/stack-frames.js"; @@ -354,16 +355,41 @@ function compatibilitySummary( } } +// An operation label shaped like `"GET /api/foo/bar"` (a method word, a space, then a path +// starting with `/`) carries a live request path in its trailing word, and a path segment can be +// a customer-chosen identifier — a project name, a repository, a capsule id — exactly the raw +// content this record's own contract (counts, hashes, closed-vocabulary labels; never customer +// content) forbids. `redactRoutePath` (route-template.ts) is the reducer the HTTP request line +// itself already uses for the same purpose; calling it here reuses that one reduction instead of +// writing a second one, so there is exactly one place that decides what a route "looks like" once +// redacted. `SOURCE_LABEL_SHAPE` never admits a `/`, so this only ever runs for `fallback === +// "server.operation"`. A label with no path component (e.g. `"chat.stream"`) is returned as-is. +function pathReducedOperationLabel(value: string): string | undefined { + const spaceIndex = value.indexOf(" "); + const prefix = spaceIndex === -1 ? "" : value.slice(0, spaceIndex + 1); + const path = spaceIndex === -1 ? value : value.slice(spaceIndex + 1); + if (!path.startsWith("/")) return value; + const template = redactRoutePath(path, MAX_DIAGNOSTIC_LABEL_LENGTH - prefix.length); + return template === undefined ? undefined : `${prefix}${template}`; +} + function diagnosticLabel( value: unknown, shape: RegExp, fallback: "server.operation" | "server.diagnostic", ): string { - return typeof value === "string" && - value.length <= MAX_DIAGNOSTIC_LABEL_LENGTH && - shape.test(value) - ? value - : fallback; + if ( + typeof value !== "string" || + value.length > MAX_DIAGNOSTIC_LABEL_LENGTH || + !shape.test(value) + ) { + return fallback; + } + if (fallback !== "server.operation") return value; + // A path this server does not serve (an unknown route) or one wearing traversal segments + // (`redactRoutePath` fails closed on both) is not partially echoed — the whole label degrades + // to the fallback, same fail-closed direction `redactRoutePath` itself documents. + return pathReducedOperationLabel(value) ?? fallback; } // Emits a diagnostic record through the provided sink (falling back to the default stderr sink). diff --git a/packages/keiko-server/src/gateway-readiness.ts b/packages/keiko-server/src/gateway-readiness.ts index f8244b715f..66bfef1864 100644 --- a/packages/keiko-server/src/gateway-readiness.ts +++ b/packages/keiko-server/src/gateway-readiness.ts @@ -28,6 +28,7 @@ import { newCorrelationId } from "./correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError } from "./diagnostics-log.js"; import { rerankSelection } from "./grounded-rerank-facade.js"; import type { RouteContext, RouteResult } from "./routes.js"; +import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; const DEFAULT_PROBES: readonly GatewayReadinessProbeName[] = [ "chat", @@ -84,33 +85,6 @@ interface ProviderSelection { readonly capability: ModelCapability | undefined; } -class ReadinessBodyTooLargeError extends Error { - constructor() { - super("Readiness request body exceeds the size limit."); - } -} - -function readBody(req: IncomingMessage): Promise { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let bytes = 0; - req.on("data", (chunk: Buffer | string) => { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf8"); - bytes += buffer.byteLength; - if (bytes > MAX_BODY_BYTES) { - reject(new ReadinessBodyTooLargeError()); - req.destroy(); - return; - } - chunks.push(buffer); - }); - req.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - req.on("error", reject); - }); -} - function error(code: string, message: string, status = 400): RouteResult { return { status, body: { error: { code, message } } }; } @@ -123,12 +97,17 @@ function isProbeName(value: unknown): value is GatewayReadinessProbeName { return typeof value === "string" && ALL_PROBES.has(value as GatewayReadinessProbeName); } -async function readJsonBody(req: IncomingMessage): Promise { +// Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the cap above is +// unchanged, only the ad hoc listener wiring is gone. +async function readJsonBody( + req: IncomingMessage, + correlationId?: string, +): Promise { let raw: string; try { - raw = await readBody(req); + raw = await readBoundedRequestBody(req, MAX_BODY_BYTES, undefined, correlationId); } catch (bodyError) { - if (bodyError instanceof ReadinessBodyTooLargeError) { + if (bodyError instanceof RequestBodyTooLargeError) { return error("PAYLOAD_TOO_LARGE", "Readiness request body exceeds the size limit.", 413); } return error("BAD_REQUEST", "The readiness request body could not be read."); @@ -1438,7 +1417,7 @@ export async function handleGatewayReadiness( ctx: RouteContext, deps: UiHandlerDeps, ): Promise { - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if ("status" in body) return body; const report = await runGatewayReadiness(body.parsed, deps, ctx.correlationId); if ("status" in report) return report; diff --git a/packages/keiko-server/src/gitRoutes.ts b/packages/keiko-server/src/gitRoutes.ts index 55f343a5ee..e1127f062f 100644 --- a/packages/keiko-server/src/gitRoutes.ts +++ b/packages/keiko-server/src/gitRoutes.ts @@ -1223,6 +1223,7 @@ export async function handleGitDiff( "GIT_DIFF_FAILED", "Git diff is unavailable for this folder.", "The bounded diff read was unavailable.", + GIT_DIFF_ROUTE_TEMPLATE, ), }; } @@ -1249,6 +1250,7 @@ export async function handleGitDiff( }; return { status: 200, body: redacted(deps, body) }; }, + GIT_DIFF_ROUTE_TEMPLATE, ); } @@ -1285,6 +1287,16 @@ class GitRouteReadError extends Error { } } +// The declared route templates `gitReadErrorBody` reports as `operation` — literal constants, not +// derived from any live request, so the two routes sharing `runGitDiffHandler` can never be +// confused for one another and neither can ever carry a request-supplied segment. +const GIT_DIFF_ROUTE_TEMPLATE = "/api/git/diff"; +const GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE = "/api/git/diff/structured"; + +// `routeTemplate` is the DECLARED route pattern the caller is answering for (e.g. +// `"/api/git/diff"`), never `ctx.url.pathname` — the live request path is not read here, so a +// route registered without path parameters can never leak one, and a future dynamic segment on +// one of these routes could not smuggle a customer-chosen value through this diagnostic either. function gitReadErrorBody( ctx: RouteContext, deps: UiHandlerDeps, @@ -1292,13 +1304,14 @@ function gitReadErrorBody( code: string, message: string, summary: ServerDiagnosticSummary, + routeTemplate: string, ): ReturnType { const correlationId = ctx.correlationId ?? randomUUID(); emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ correlationId, - operation: `GET ${ctx.url.pathname}`, + operation: `GET ${routeTemplate}`, source: "git-routes", error, summary, @@ -1312,6 +1325,7 @@ async function runGitDiffHandler( ctx: RouteContext, deps: UiHandlerDeps, work: () => Promise, + routeTemplate: string, ): Promise { try { return await work(); @@ -1326,6 +1340,7 @@ async function runGitDiffHandler( "GIT_DIFF_FAILED", "Git diff is unavailable for this folder.", "The bounded diff read was unavailable.", + routeTemplate, ), }; } @@ -1340,6 +1355,7 @@ async function runGitDiffHandler( error.code, error.message, "The bounded diff read was unavailable.", + routeTemplate, ) : errorBody(error.code, error.message), }; @@ -1374,30 +1390,39 @@ export async function handleGitStructuredDiff( deps: UiHandlerDeps, rawOptions?: GitRouteOptions, ): Promise { - return runGitDiffHandler(ctx, deps, async () => { - const options = optionsWithDefaults(rawOptions ?? deps.gitRouteOptions); - const scope = parseStructuredScope(ctx.url.searchParams.get("scope")); - const path = validatePath(ctx.url.searchParams.get("path")); - const repo = await resolveRepository(ctx, deps, options); - if ("available" in repo) { - return { status: 200, body: redacted(deps, unavailableStructuredDiff(scope)) }; - } - if (path !== undefined) await assertContainedGitPath(repo, path); - const result = await runDiff( - repo, - options, - scope === "staged", - path, - GIT_EDITOR_DIFF_MAX_BYTES, - ); - if (result.exitCode !== 0) { - if (isUnavailableReadFailure(result)) { + return runGitDiffHandler( + ctx, + deps, + async () => { + const options = optionsWithDefaults(rawOptions ?? deps.gitRouteOptions); + const scope = parseStructuredScope(ctx.url.searchParams.get("scope")); + const path = validatePath(ctx.url.searchParams.get("path")); + const repo = await resolveRepository(ctx, deps, options); + if ("available" in repo) { return { status: 200, body: redacted(deps, unavailableStructuredDiff(scope)) }; } - return correlatedGitError(ctx, "GIT_DIFF_FAILED", "Git diff is unavailable for this folder."); - } - return { status: 200, body: redacted(deps, structuredDiffBody(scope, repo, result)) }; - }); + if (path !== undefined) await assertContainedGitPath(repo, path); + const result = await runDiff( + repo, + options, + scope === "staged", + path, + GIT_EDITOR_DIFF_MAX_BYTES, + ); + if (result.exitCode !== 0) { + if (isUnavailableReadFailure(result)) { + return { status: 200, body: redacted(deps, unavailableStructuredDiff(scope)) }; + } + return correlatedGitError( + ctx, + "GIT_DIFF_FAILED", + "Git diff is unavailable for this folder.", + ); + } + return { status: 200, body: redacted(deps, structuredDiffBody(scope, repo, result)) }; + }, + GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE, + ); } function parseBlameRequest(ctx: RouteContext): GitEditorBlameRequest { diff --git a/packages/keiko-server/src/http-lifecycle.e2e.test.ts b/packages/keiko-server/src/http-lifecycle.e2e.test.ts new file mode 100644 index 0000000000..a7a5bb86c0 --- /dev/null +++ b/packages/keiko-server/src/http-lifecycle.e2e.test.ts @@ -0,0 +1,377 @@ +// Wave 5 ACCEPTANCE TEST (epic #3233, ADR-0173 SS9/SS11) — end-to-end proof, against the REAL +// `createUiServer` (a genuine bound loopback socket, real route dispatch, real SSE framing), that +// this wave's four deliverables actually work together: +// +// (a) a client that disconnects mid-response yields an `op: "request"` line with `aborted: true`, +// the matched route TEMPLATE (never a raw-path guess), the query-parameter NAMES, and a +// `responseBytes` field; +// (b) an SSE stream that emits exactly 3 frames yields one `sse.stream.closed` line whose +// `frameCount` is 3 and whose `bytesStreamed` equals the bytes the server actually wrote; +// (c) `POST /api/diagnostics/client` round-trips a body-supplied `correlationId` into a log line +// carrying the message under `extra.clientNote` — never the denylisted `extra.message` — and +// enforces its own size/shape bounds (413 oversized, 400 malformed); +// (d) a diagnostic raised from a git route whose QUERY carries a customer-named value (a git +// working-tree path) never leaks that value — the diagnostic's `operation` is the DECLARED +// route template, `"GET /api/git/diff"`, not anything derived from the live request. +// +// Every other Wave 5 file (`server.test.ts`, `client-diagnostics-routes.test.ts`) already covers +// its own unit/near-integration surface with fake req/res doubles or a direct handler call. This +// file is deliberately different: one real `net.Server`, real sockets, a real client that actually +// aborts a real TCP connection — the shape none of those narrower suites can prove. +// +// If an assertion below cannot pass because the wiring it depends on is missing, it is left exactly +// as written (never loosened) so a real gap fails loudly instead of a green suite hiding it. + +import { randomUUID } from "node:crypto"; +import type { Server } from "node:http"; +import { tmpdir } from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; + +import { buildCspHeader } from "./csp.js"; +import { resetClientDiagnosticsIngestStateForTests } from "./client-diagnostics-routes.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; +import { + buildRedactor, + createInMemoryUiStore, + createRunRegistry, + QueueEventSink, + type StreamEvent, + type UiHandlerDeps, +} from "./index.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, + type ServerLogEvent, +} from "./observability/index.js"; +import { UI_HOST } from "./server.js"; +import { readyMessage } from "./sse.js"; +import { closeUiTestServer, startUiTestServer } from "./ui-test-server/_support.js"; + +// `staticRoot` only has to exist: every request in this file targets an `/api/...` route, so +// `serveStatic` is never reached and nothing is ever read from this directory. +const staticRoot = tmpdir(); + +function baseUrl(port: number): string { + return `http://${UI_HOST}:${String(port)}`; +} + +function silentDiagnostics(): ServerDiagnosticSink { + return { record: () => undefined }; +} + +// The minimal, fully-wired `UiHandlerDeps` every scenario below shares — a fresh in-memory store +// and run registry per test so no test can observe another's state. `diagnostics` defaults to a +// silent sink (the shipped default writes to stderr on every record, which would just be noise +// here); scenario (d) overrides it with a capturing one. +function minimalHandlerDeps(diagnostics?: ServerDiagnosticSink): UiHandlerDeps { + return { + config: undefined, + configPresent: false, + evidenceStore: { put: () => "", list: () => [], get: () => undefined, delete: () => undefined }, + env: {}, + redactor: buildRedactor({}), + diagnostics: diagnostics ?? silentDiagnostics(), + registry: createRunRegistry(), + modelPortFactory: () => undefined, + store: createInMemoryUiStore(), + }; +} + +interface TestServerHandle { + readonly server: Server; + readonly port: number; + readonly sink: BufferedServerLogSink; +} + +// Starts a real `createUiServer` on an ephemeral loopback port with a fresh buffered log sink wired +// as BOTH `activityLog` (what `logRequestOnClose`/`server.ts` write to directly) and the process-wide +// `ServerLogger` (what `sse-write.ts`'s `sse.stream.closed` and `client-diagnostics-routes.ts`'s +// `client.diagnostic` go through via `getServerLogger()`) — the SAME underlying array, so one sink +// observes every line this wave's mechanisms produce, regardless of which of the two paths wrote it. +async function startTestServer(handlerDeps: UiHandlerDeps): Promise { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "debug" })); + const started = await startUiTestServer({ + staticRoot, + csp: buildCspHeader([]), + handlerDeps, + activityLog: sink, + }); + return { server: started.server, port: started.port, sink }; +} + +// Polls the buffered sink for the first RAW event matching `predicate`. Real-socket teardown and +// the `res.on("close", ...)` handlers that emit these lines are asynchronous, so the line is not +// necessarily present the instant the client-side call resolves. +async function waitForEvent( + sink: BufferedServerLogSink, + predicate: (event: ServerLogEvent) => boolean, + timeoutMs = 2000, +): Promise { + const deadline = Date.now() + timeoutMs; + let found = sink.events.find(predicate); + while (found === undefined) { + if (Date.now() > deadline) { + throw new Error("timed out waiting for a matching activity log event"); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + found = sink.events.find(predicate); + } + return found; +} + +function isHttpRequestLine(event: ServerLogEvent): boolean { + return event.category === "http" && event.op === "request"; +} + +afterEach(() => { + // Both the process-wide logger slot and the client-diagnostics rate limiter/drop-notice counter + // are module-level state shared across every test in this file (and, for the logger, across the + // whole worker) — reset unconditionally so one test's setup or a mid-test throw can never leak + // into the next test's assertions. + resetServerLogger(); + resetClientDiagnosticsIngestStateForTests(); +}); + +describe("(a) client disconnect mid-response — real socket abort", () => { + it("logs aborted:true with the matched route template, query-parameter names, and responseBytes", async () => { + const handlerDeps = minimalHandlerDeps(); + const runId = randomUUID(); + // A running (never-terminated) sink keeps `/api/runs/:runId/events` open indefinitely once the + // `ready` frame is sent — exactly what lets a real client abort mid-stream instead of racing a + // response that would complete before the abort ever reaches the server. + handlerDeps.registry.register({ + runId, + fingerprint: "fp-mid-response-abort", + modelId: "test-model", + sink: new QueueEventSink(), + cancel: () => undefined, + }); + let started: TestServerHandle | undefined; + try { + started = await startTestServer(handlerDeps); + const controller = new AbortController(); + const response = await fetch( + `${baseUrl(started.port)}/api/runs/${runId}/events?foo=1&bar=2`, + { signal: controller.signal }, + ); + const reader = response.body?.getReader(); + if (reader === undefined) { + throw new Error("expected a readable SSE response body"); + } + // Read the `ready` frame first: this proves the response genuinely started (headers sent, + // bytes flowing) before the client tears the connection down, i.e. a true mid-response abort + // rather than a pre-response cancel. + await reader.read(); + controller.abort(); + await reader.cancel().catch(() => undefined); + + const event = await waitForEvent(started.sink, isHttpRequestLine); + expect(event.extra?.aborted).toBe(true); + expect(event.extra?.routeTemplate).toBe("/api/runs/:runId/events"); + expect(event.extra?.queryParamNames).toEqual(["bar", "foo"]); + // A STREAMING route never calls `writeJson` (the only place `responseBytes` is computed), so + // the field is present and reports the documented default rather than being silently absent. + expect(event.extra?.responseBytes).toBe(0); + } finally { + if (started !== undefined) await closeUiTestServer(started.server); + handlerDeps.store.close(); + } + }); +}); + +describe("(b) SSE terminal line — real frame/byte counters", () => { + it("logs sse.stream.closed with frameCount 3 and bytesStreamed equal to the bytes actually written", async () => { + const handlerDeps = minimalHandlerDeps(); + const runId = randomUUID(); + const eventSink = new QueueEventSink(); + for (let seq = 0; seq < 3; seq += 1) { + const event: StreamEvent = { + schemaVersion: "1", + runId, + fingerprint: "fp-three-frames", + seq, + ts: Date.now(), + type: "workflow:progress", + }; + eventSink.emit(event); + } + // Terminating the sink BEFORE any writer attaches means the real route (`openSseStream`) + // replays exactly these 3 buffered events on connect, then ends the response itself — a + // deterministic, real end-to-end SSE close with no reliance on wall-clock timing. + eventSink.closeAll(); + handlerDeps.registry.register({ + runId, + fingerprint: "fp-three-frames", + modelId: "test-model", + sink: eventSink, + cancel: () => undefined, + }); + let started: TestServerHandle | undefined; + try { + started = await startTestServer(handlerDeps); + const response = await fetch(`${baseUrl(started.port)}/api/runs/${runId}/events`); + const text = await response.text(); + + const event = await waitForEvent( + started.sink, + (candidate) => candidate.op === "sse.stream.closed", + ); + expect(event.extra?.frameCount).toBe(3); + + // The response is exactly [3 counted event frames][1 uncounted `ready` frame], in that order + // (verified from `openSseStream`: buffer replay happens before the `ready` write). Deriving + // the expected byte count from the REAL bytes the server sent — rather than recomputing the + // SSE framing formula independently — means this assertion can never drift from whatever the + // production frame format actually is. + const totalBytes = Buffer.byteLength(text, "utf8"); + const readyBytes = Buffer.byteLength(readyMessage(), "utf8"); + expect(event.extra?.bytesStreamed).toBe(totalBytes - readyBytes); + } finally { + if (started !== undefined) await closeUiTestServer(started.server); + handlerDeps.store.close(); + } + }); +}); + +describe("(c) POST /api/diagnostics/client — real ingest route", () => { + it("round-trips a body-supplied correlationId under extra.clientNote, with no message key on the formatted line", async () => { + const handlerDeps = minimalHandlerDeps(); + const correlationId = randomUUID(); + // Short and low-punctuation on purpose: `log-redaction.ts`'s generic prose guard collapses any + // value with more than 3 spaces to `[redacted:shape]` regardless of field name — a real + // constraint on the CONTENT this endpoint can log verbatim, orthogonal to the field-NAME fix + // this test proves. A value this shape survives formatting unredacted, so the assertions below + // can check the formatted line's exact content, not just its key names. + const message = "sse-error readyState=2"; + let started: TestServerHandle | undefined; + try { + started = await startTestServer(handlerDeps); + const response = await fetch(`${baseUrl(started.port)}/api/diagnostics/client`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Keiko-CSRF": "1" }, + body: JSON.stringify({ + message, + clientTs: new Date().toISOString(), + readyState: 2, + correlationId, + kind: "sse-error", + }), + }); + expect(response.status).toBe(204); + + const rawEvent = await waitForEvent( + started.sink, + (candidate) => candidate.op === "client.diagnostic", + ); + // The raw (pre-format) event: proves the route's own code writes the field under the name + // `clientNote`, never `message`. + expect(rawEvent.extra?.clientNote).toBe(message); + expect(rawEvent.correlationId).toBe(correlationId); + + // The FORMATTED line — what actually reaches disk in production — after redaction. If the + // route had used `extra.message` instead, `message` would be DENIED and this same line would + // carry `"message":"[redacted:key]"`; because it used `clientNote`, no `message` key exists at + // all and the content survives intact. + const lineIndex = started.sink.events.indexOf(rawEvent); + const formattedLine = started.sink.lines()[lineIndex]; + if (formattedLine === undefined) { + throw new Error("expected a formatted log line for the captured event"); + } + const parsed = JSON.parse(formattedLine) as Record; + expect(parsed.clientNote).toBe(message); + expect(parsed.correlationId).toBe(correlationId); + expect("message" in parsed).toBe(false); + } finally { + if (started !== undefined) await closeUiTestServer(started.server); + handlerDeps.store.close(); + } + }); + + it("rejects an oversized body with 413", async () => { + const handlerDeps = minimalHandlerDeps(); + let started: TestServerHandle | undefined; + try { + started = await startTestServer(handlerDeps); + const response = await fetch(`${baseUrl(started.port)}/api/diagnostics/client`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Keiko-CSRF": "1" }, + // Content need not be valid JSON: the bounded-body reader rejects on byte count while + // streaming, before any JSON parsing is attempted. + body: "a".repeat(10_000), + }); + expect(response.status).toBe(413); + } finally { + if (started !== undefined) await closeUiTestServer(started.server); + handlerDeps.store.close(); + } + }); + + it("rejects a malformed (schema-invalid) body with 400", async () => { + const handlerDeps = minimalHandlerDeps(); + let started: TestServerHandle | undefined; + try { + started = await startTestServer(handlerDeps); + const response = await fetch(`${baseUrl(started.port)}/api/diagnostics/client`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Keiko-CSRF": "1" }, + // Valid JSON, but missing the required `clientTs` field — fails + // `isClientDiagnosticIngestRequest`, not JSON.parse. + body: JSON.stringify({ message: "missing clientTs" }), + }); + expect(response.status).toBe(400); + } finally { + if (started !== undefined) await closeUiTestServer(started.server); + handlerDeps.store.close(); + } + }); +}); + +// NOTE on this scenario's fails-before status: verified by temporarily reverting +// `gitRoutes.ts` to its pre-Wave-5 `dev` content and re-running this file — this assertion +// PASSED against that baseline too. Pre-Wave-5, `gitReadErrorBody`'s `operation` was built from +// `` `GET ${ctx.url.pathname}` ``, and `URL.pathname` never includes the query string in either +// version — so for the two CURRENTLY declared, segment-free `/api/git/*` routes, the raw-pathname +// read and the new literal `GIT_DIFF_ROUTE_TEMPLATE` constant are byte-identical for every request +// that can actually reach this handler; there is no live request that makes them diverge. The +// Wave-5 change is real (confirmed by diff: a per-call-site literal constant replaces the +// `ctx.url.pathname` read) and the invariant this test locks in is real and worth pinning, but it +// is forward-looking defense-in-depth — the code's own comment says so ("a future dynamic segment +// on one of these routes could not smuggle a customer-chosen value through this diagnostic +// either") — not a fix to a currently reachable leak. Keeping this test as a regression PIN rather +// than removing it: it is what stops a future git route with a real `:param` segment from silently +// regressing back to reading `ctx.url.pathname`. +describe("(d) git route diagnostic — declared template, never the raw query value", () => { + it('carries operation "GET /api/git/diff" and never the customer-named path from the query string', async () => { + const records: ServerDiagnosticRecord[] = []; + const handlerDeps = minimalHandlerDeps({ record: (record) => records.push(record) }); + // An absolute path: `isRootRelativeFileIdentifier` rejects it (BAD_PATH) before the handler + // ever resolves a repository, so no real git repository is needed for this test at all. The + // segment itself is the "customer-named" value the diagnostic must never echo. + const customerSegment = "customer-secret-acme-corp"; + let started: TestServerHandle | undefined; + try { + started = await startTestServer(handlerDeps); + const response = await fetch( + `${baseUrl(started.port)}/api/git/diff?path=${encodeURIComponent(`/${customerSegment}/config.json`)}`, + ); + expect(response.status).toBe(400); + const body = (await response.json()) as { readonly error: { readonly code: string } }; + expect(body.error.code).toBe("BAD_PATH"); + + expect(records).toHaveLength(1); + const [record] = records; + if (record === undefined) { + throw new Error("expected exactly one diagnostic record"); + } + expect(record.operation).toBe("GET /api/git/diff"); + expect(JSON.stringify(record)).not.toContain(customerSegment); + } finally { + if (started !== undefined) await closeUiTestServer(started.server); + handlerDeps.store.close(); + } + }); +}); diff --git a/packages/keiko-server/src/memory-consolidation-handlers.test.ts b/packages/keiko-server/src/memory-consolidation-handlers.test.ts index 7a72df6022..08cb9d84b1 100644 --- a/packages/keiko-server/src/memory-consolidation-handlers.test.ts +++ b/packages/keiko-server/src/memory-consolidation-handlers.test.ts @@ -165,6 +165,22 @@ describe("memory consolidation job handlers", () => { expect(result.status).toBe(503); }); + // #2902 w5-sse-counters: readJsonBody now consolidates onto the shared readBoundedRequestBody, + // so an oversized body must still yield the shared reader's own 413 rejection shape. + it("rejects an oversized body using the shared bounded-body reader", async () => { + const deps = makeDeps({ memoryVault: makeVault() }); + + const result = await handleCreateConsolidationJob( + makeCtx("/api/memory/consolidation/jobs", { + scopes: [{ kind: "user", userId: "u-1" }], + settings: { notes: "x".repeat(70_000) }, + }), + deps, + ); + + expect(result.status).toBe(413); + }); + it("registers a queued job and then skips when no memories match", async () => { const vault = makeVault(); const deps = makeDeps({ memoryVault: vault }); diff --git a/packages/keiko-server/src/memory-consolidation-handlers.ts b/packages/keiko-server/src/memory-consolidation-handlers.ts index eb8f0dd75b..51f6b7b9c7 100644 --- a/packages/keiko-server/src/memory-consolidation-handlers.ts +++ b/packages/keiko-server/src/memory-consolidation-handlers.ts @@ -46,6 +46,7 @@ import type { ConsolidationJobSettings, } from "./memory-consolidation-registry.js"; import { enrichReviewItemsWithAdvisory } from "./memory-conflict-advisory.js"; +import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; const MAX_BODY_BYTES = 64_000; const DEFAULT_JACCARD_THRESHOLD = 0.85; @@ -61,13 +62,6 @@ const DEFAULT_CONSOLIDATION_STATUSES: readonly MemoryStatus[] = [ "conflicted", ]; -class BodyTooLargeError extends Error { - public constructor() { - super("request body too large"); - this.name = "BodyTooLargeError"; - } -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -76,37 +70,17 @@ function isRouteResult(value: unknown): value is RouteResult { return isRecord(value) && typeof value.status === "number"; } -function readBody(req: IncomingMessage): Promise { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let total = 0; - let capped = false; - req.on("data", (chunk: Buffer) => { - total += chunk.length; - if (total > MAX_BODY_BYTES) { - if (!capped) { - capped = true; - chunks.length = 0; - reject(new BodyTooLargeError()); - req.resume(); - } - return; - } - chunks.push(chunk); - }); - req.on("end", () => { - if (!capped) resolve(Buffer.concat(chunks).toString("utf8")); - }); - req.on("error", reject); - }); -} - -async function readJsonBody(req: IncomingMessage): Promise | RouteResult> { +// Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the cap above is +// unchanged, only the ad hoc listener wiring is gone. +async function readJsonBody( + req: IncomingMessage, + correlationId?: string, +): Promise | RouteResult> { let raw: string; try { - raw = await readBody(req); + raw = await readBoundedRequestBody(req, MAX_BODY_BYTES, undefined, correlationId); } catch (error) { - if (error instanceof BodyTooLargeError) { + if (error instanceof RequestBodyTooLargeError) { return { status: 413, body: errorBody("PAYLOAD_TOO_LARGE", "Request body too large.") }; } throw error; @@ -709,7 +683,7 @@ export async function handleCreateConsolidationJob( if (isRouteResult(vault)) return vault; const registry = resolveJobRegistry(deps); if (isRouteResult(registry)) return registry; - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseCreateInput(body); if (isRouteResult(input)) return input; @@ -1019,7 +993,7 @@ export async function handleApplyConsolidationReviewItem( if (previous !== undefined) return previous; const inputs = findApplyInputs(route.record, route.itemId); if (isRouteResult(inputs)) return inputs; - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const latest = route.registry.get(route.jobId); const concurrent = latest === undefined ? undefined : previousApplication(latest, route.itemId); diff --git a/packages/keiko-server/src/memory-conv-handlers.ts b/packages/keiko-server/src/memory-conv-handlers.ts index 16b4758f32..e41e40e7df 100644 --- a/packages/keiko-server/src/memory-conv-handlers.ts +++ b/packages/keiko-server/src/memory-conv-handlers.ts @@ -61,6 +61,7 @@ import { recordMemoryAudit } from "./memory-audit-handler.js"; import { recordAutoAcceptedMemoryCaptureDecision } from "./memory-capture-audit.js"; import { buildMemoryRecordFromProposal } from "./memory-record-builders.js"; import { persistCapturedMemory } from "./memory-capture-persistence.js"; +import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; import { enforcePersistableMemoryOutcome, FORGOTTEN_MEMORY_SUPPRESSION_REASON, @@ -84,50 +85,23 @@ import { const MAX_BODY_BYTES = 64_000; -// ─── Body reading (mirrors memory-handlers.ts pattern) ──────────────────────── - -class BodyTooLargeError extends Error { - public constructor() { - super("request body too large"); - this.name = "BodyTooLargeError"; - } -} +// ─── Body reading ────────────────────────────────────────────────────────────── +// Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the cap below is +// unchanged, only the ad hoc listener wiring is gone. function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function readBody(req: IncomingMessage): Promise { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let total = 0; - let capped = false; - req.on("data", (chunk: Buffer) => { - total += chunk.length; - if (total > MAX_BODY_BYTES) { - if (!capped) { - capped = true; - chunks.length = 0; - reject(new BodyTooLargeError()); - req.resume(); - } - return; - } - chunks.push(chunk); - }); - req.on("end", () => { - if (!capped) resolve(Buffer.concat(chunks).toString("utf8")); - }); - req.on("error", reject); - }); -} - -async function readJsonBody(req: IncomingMessage): Promise | RouteResult> { +async function readJsonBody( + req: IncomingMessage, + correlationId?: string, +): Promise | RouteResult> { let raw: string; try { - raw = await readBody(req); + raw = await readBoundedRequestBody(req, MAX_BODY_BYTES, undefined, correlationId); } catch (err) { - if (err instanceof BodyTooLargeError) { + if (err instanceof RequestBodyTooLargeError) { return { status: 413, body: errorBody("PAYLOAD_TOO_LARGE", "Request body too large.") }; } throw err; @@ -336,7 +310,7 @@ export async function handleMemoryRetrieveContext( ): Promise { const vault = resolveVault(deps); if (isRouteResult(vault)) return vault; - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseContextInput(body); if (isRouteResult(input)) return input; @@ -452,7 +426,7 @@ export async function handleMemoryCaptureFromConversation( ): Promise { const vault = resolveVault(deps); if (isRouteResult(vault)) return vault; - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseCaptureInput(body); if (isRouteResult(input)) return input; diff --git a/packages/keiko-server/src/memory-handlers.test.ts b/packages/keiko-server/src/memory-handlers.test.ts index 4b9c9ed376..0caa832343 100644 --- a/packages/keiko-server/src/memory-handlers.test.ts +++ b/packages/keiko-server/src/memory-handlers.test.ts @@ -895,6 +895,26 @@ describe("memory handlers", () => { expect(vault.getEmbedding(memoryId("memory-edit-2"))).toBeUndefined(); }); + // #2902 w5-sse-counters: readJsonBody now consolidates onto the shared readBoundedRequestBody, + // so an oversized body must still yield the shared reader's own 413 rejection shape. + it("rejects an oversized body using the shared bounded-body reader", async () => { + const vault = makeVault(); + + const result = await handleCorrectMemory( + makeCtx( + "/api/memory/memory-oversize/correct", + { body: "x".repeat(70_000) }, + { id: "memory-oversize" }, + ), + makeDeps({ memoryVault: vault }), + ); + + expect(result.status).toBe(413); + expect(asJson(result)).toEqual({ + error: { code: "PAYLOAD_TOO_LARGE", message: "Request body too large." }, + }); + }); + it("creates a correction proposal with a provenance-preserving supersession edge", async () => { const vault = makeVault(); const evidenceStore = createInMemoryEvidenceStore(); diff --git a/packages/keiko-server/src/memory-handlers.ts b/packages/keiko-server/src/memory-handlers.ts index 337da9cde7..e6a64821c9 100644 --- a/packages/keiko-server/src/memory-handlers.ts +++ b/packages/keiko-server/src/memory-handlers.ts @@ -79,6 +79,7 @@ import { type MemoryCaptureDecision, } from "./memory-capture-projection.js"; import { refreshMemoryEmbeddingAfterBodyEdit } from "./memory-embedding.js"; +import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; // ─── Constants ──────────────────────────────────────────────────────────────── @@ -243,45 +244,18 @@ function parseScope(raw: unknown): MemoryScope | RouteResult { } // ─── Body reading ────────────────────────────────────────────────────────────── +// Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the caps below are +// unchanged, only the ad hoc listener wiring is gone. -class BodyTooLargeError extends Error { - public constructor() { - super("request body too large"); - this.name = "BodyTooLargeError"; - } -} - -function readBody(req: IncomingMessage): Promise { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let total = 0; - let capped = false; - req.on("data", (chunk: Buffer) => { - total += chunk.length; - if (total > MAX_MEMORY_BODY_BYTES) { - if (!capped) { - capped = true; - chunks.length = 0; - reject(new BodyTooLargeError()); - req.resume(); - } - return; - } - chunks.push(chunk); - }); - req.on("end", () => { - if (!capped) resolve(Buffer.concat(chunks).toString("utf8")); - }); - req.on("error", reject); - }); -} - -async function readJsonBody(req: IncomingMessage): Promise | RouteResult> { +async function readJsonBody( + req: IncomingMessage, + correlationId?: string, +): Promise | RouteResult> { let raw: string; try { - raw = await readBody(req); + raw = await readBoundedRequestBody(req, MAX_MEMORY_BODY_BYTES, undefined, correlationId); } catch (err) { - if (err instanceof BodyTooLargeError) { + if (err instanceof RequestBodyTooLargeError) { return { status: 413, body: errorBody("PAYLOAD_TOO_LARGE", "Request body too large.") }; } throw err; @@ -887,7 +861,7 @@ function memoryIdFromParams(ctx: RouteContext): MemoryId | RouteResult { } async function readEditRouteInput(ctx: RouteContext): Promise { - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; return parseEditInput(body); } @@ -1020,7 +994,7 @@ export async function handleArchiveMemory( return { status: 400, body: errorBody("BAD_REQUEST", "Memory id is required.") }; } - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const rawReason = typeof body.reason === "string" ? body.reason : undefined; @@ -1338,7 +1312,7 @@ export async function handleForgetMemory( return { status: 400, body: errorBody("BAD_REQUEST", "Memory id is required.") }; } - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseDestructiveInput(body); @@ -1367,7 +1341,7 @@ export async function handleForgetMemories( const vault = resolveVault(deps); if (isRouteResult(vault)) return vault; - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseForgetSelectionInput(body); @@ -1399,7 +1373,7 @@ export async function handleDeleteMemory( return { status: 400, body: errorBody("BAD_REQUEST", "Memory id is required.") }; } - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseDestructiveInput(body); @@ -1644,7 +1618,7 @@ export async function handleResolveMemoryConflict( const vault = resolveVault(deps); if (isRouteResult(vault)) return vault; - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseConflictResolutionInput(body); @@ -1779,7 +1753,7 @@ export async function handleCorrectMemory( return { status: 400, body: errorBody("BAD_REQUEST", "Memory id is required.") }; } - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const input = parseCorrectInput(body); @@ -2009,7 +1983,7 @@ export async function handleAcceptMemoryProposal( if (id === undefined || id.length === 0) { return { status: 400, body: errorBody("BAD_REQUEST", "Memory id is required.") }; } - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const bodyOverride = parseAcceptBody(body, id as MemoryId, deps); if (isRouteResult(bodyOverride)) return bodyOverride; @@ -2066,7 +2040,7 @@ export async function handleRejectMemoryProposal( return { status: 400, body: errorBody("BAD_REQUEST", "Memory id is required.") }; } - const body = await readJsonBody(ctx.req); + const body = await readJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(body)) return body; const { reason } = parseRejectInput(body); diff --git a/packages/keiko-server/src/observability/route-template.ts b/packages/keiko-server/src/observability/route-template.ts index 65d098de90..34cd6120e1 100644 --- a/packages/keiko-server/src/observability/route-template.ts +++ b/packages/keiko-server/src/observability/route-template.ts @@ -99,6 +99,7 @@ export const API_ROUTE_LITERAL_SEGMENTS: ReadonlySet = new Set([ "check", "citation-preview", "cleanup", + "client", "clone", "code", "codex-subscription", @@ -125,6 +126,7 @@ export const API_ROUTE_LITERAL_SEGMENTS: ReadonlySet = new Set([ "delete", "dependencies", "desktop", + "diagnostics", "diff", "directories", "docs-browser", diff --git a/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts b/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts index e41673e3be..adb22ddfdd 100644 --- a/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts +++ b/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts @@ -33,6 +33,7 @@ import type { import type { RouteContext, RouteDefinition, RouteResult } from "../routes.js"; import type { UiHandlerDeps } from "../deps.js"; import { currentGateway, currentGatewayConfig } from "../deps.js"; +import { readBoundedRequestBody, RequestBodyTooLargeError } from "../bounded-request-body.js"; import { normaliseQiModelPolicy, recommendQiModelPolicy, @@ -73,13 +74,6 @@ export class QiModelPolicyError extends Error { } } -class BodyTooLargeError extends Error { - constructor() { - super("QI model-policy request body is too large"); - this.name = "BodyTooLargeError"; - } -} - export function resolveQiPolicyPath(evidenceDir: string): string { return join(dirname(evidenceDir), QI_POLICY_DIR, QI_POLICY_FILE); } @@ -152,31 +146,6 @@ function configuredModels(deps: UiHandlerDeps): readonly ModelCapability[] { return config === undefined ? [] : listConfiguredCapabilities(config); } -function readBody(req: IncomingMessage): Promise { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - let total = 0; - let capped = false; - req.on("data", (chunk: Buffer) => { - total += chunk.length; - if (total > MAX_POLICY_BODY_BYTES) { - if (!capped) { - capped = true; - chunks.length = 0; - reject(new BodyTooLargeError()); - req.resume(); - } - return; - } - chunks.push(chunk); - }); - req.on("end", () => { - if (!capped) resolve(Buffer.concat(chunks).toString("utf8")); - }); - req.on("error", reject); - }); -} - function isObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -195,12 +164,17 @@ function parsePolicyValue(raw: unknown): QualityIntelligenceModelPolicy | undefi }); } -async function parseJsonBody(req: IncomingMessage): Promise | RouteResult> { +// Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the cap above is +// unchanged, only the ad hoc listener wiring is gone. +async function parseJsonBody( + req: IncomingMessage, + correlationId?: string, +): Promise | RouteResult> { let raw: string; try { - raw = await readBody(req); + raw = await readBoundedRequestBody(req, MAX_POLICY_BODY_BYTES, undefined, correlationId); } catch (error) { - return error instanceof BodyTooLargeError + return error instanceof RequestBodyTooLargeError ? errorResult(413, "QI_BAD_MODEL_POLICY", "Request body is too large.") : errorResult(400, "QI_BAD_MODEL_POLICY", "Could not read request body."); } @@ -497,7 +471,7 @@ export async function handlePutQiModelPolicy( if (deps.evidenceDir === undefined) { return errorResult(500, "QI_NO_EVIDENCE_DIR", "The evidence directory is not configured."); } - const parsed = await parseJsonBody(ctx.req); + const parsed = await parseJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(parsed)) return parsed; const policy = parsePolicyValue(parsed.modelPolicy ?? parsed.policy ?? parsed); if (policy === undefined) { @@ -547,7 +521,7 @@ export async function handlePreflightQiModelPolicy( ctx: RouteContext, deps: UiHandlerDeps, ): Promise { - const parsed = await parseJsonBody(ctx.req); + const parsed = await parseJsonBody(ctx.req, ctx.correlationId); if (isRouteResult(parsed)) return parsed; const modelPolicy = parsePolicyValue(parsed.modelPolicy); if (parsed.modelPolicy !== undefined && modelPolicy === undefined) { diff --git a/packages/keiko-server/src/relationship-handlers.test.ts b/packages/keiko-server/src/relationship-handlers.test.ts index 78e90fc4d6..7b9014553c 100644 --- a/packages/keiko-server/src/relationship-handlers.test.ts +++ b/packages/keiko-server/src/relationship-handlers.test.ts @@ -45,6 +45,9 @@ interface FakeReq extends EventEmitter { headers: Record; url: string; method: string; + // The shared bounded-body reader calls `resume()` to drain an oversized/cancelled body + // (#2902 w5-sse-counters); a bare EventEmitter has no such method. + resume(): void; } function makeReq(opts: { @@ -57,6 +60,7 @@ function makeReq(opts: { e.headers = opts.headers ?? {}; e.url = opts.url ?? "/"; e.method = opts.method ?? "GET"; + e.resume = (): void => undefined; // Defer body emission to next tick so consumer can attach `data`/`end` listeners. process.nextTick(() => { if (opts.body !== undefined) { @@ -480,6 +484,26 @@ describe("POST /api/relationships (create + validate-before-persist)", () => { expect((res.body as { error: { code: string } }).error.code).toBe("relationship/bad-request"); }); + // #2902 w5-sse-counters: readJsonBody now consolidates onto the shared readBoundedRequestBody, + // so an oversized body must still yield this handler's own 413 shape (16 KiB cap unchanged). + it("rejects an oversized body using the shared bounded-body reader", async () => { + const store = freshStore(); + const { redactor } = trackingRedactor(); + const deps = buildDeps("ws-a", store, redactor); + const req = makeReq({ + method: "POST", + url: "/api/relationships/validate", + body: "x".repeat(17 * 1024), + }); + + const res = await handleRelationshipValidate(makeCtx(req), deps); + + expect(res.status).toBe(413); + expect((res.body as { error: { code: string } }).error.code).toBe( + "relationship/payload-too-large", + ); + }); + it("replays an identical body via cached idempotency record", async () => { const store = freshStore(); const { redactor } = trackingRedactor(); diff --git a/packages/keiko-server/src/relationship-handlers.ts b/packages/keiko-server/src/relationship-handlers.ts index c4ac79e430..984114efb6 100644 --- a/packages/keiko-server/src/relationship-handlers.ts +++ b/packages/keiko-server/src/relationship-handlers.ts @@ -74,6 +74,7 @@ import type { import { UiStoreError } from "./store/errors.js"; import { SSE_HEADERS } from "./sse.js"; import { writeOrDestroy } from "./sse-write.js"; +import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; import { subscribeActivityBroadcast, type ActivityFrame, @@ -313,38 +314,24 @@ class HandlerError extends Error { } } -function readBody(req: IncomingMessage): Promise { - return new Promise((resolveBody, rejectBody) => { - const chunks: Buffer[] = []; - let total = 0; - let capped = false; - req.on("data", (chunk: Buffer) => { - total += chunk.length; - if (total > MAX_BODY_BYTES) { - if (!capped) { - capped = true; - chunks.length = 0; - rejectBody( - new HandlerError(413, "relationship/payload-too-large", "Body exceeds 16 KiB."), - ); - req.resume(); - } - return; - } - chunks.push(chunk); - }); - req.on("end", () => { - if (!capped) resolveBody(Buffer.concat(chunks).toString("utf8")); - }); - req.on("error", rejectBody); - }); -} - -async function readJsonBody(req: IncomingMessage): Promise<{ +// Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the 16 KiB cap above is +// unchanged, only the ad hoc listener wiring is gone. +async function readJsonBody( + req: IncomingMessage, + correlationId?: string, +): Promise<{ readonly raw: string; readonly value: Record; }> { - const raw = await readBody(req); + let raw: string; + try { + raw = await readBoundedRequestBody(req, MAX_BODY_BYTES, undefined, correlationId); + } catch (bodyReadError) { + if (bodyReadError instanceof RequestBodyTooLargeError) { + throw new HandlerError(413, "relationship/payload-too-large", "Body exceeds 16 KiB."); + } + throw bodyReadError; + } let parsed: unknown; try { parsed = JSON.parse(raw); @@ -1157,7 +1144,7 @@ async function handleValidateImpl(ctx: RouteContext, deps: UiHandlerDeps): Promi async () => { const relationship = readRelationshipDeps(deps); const workspaceId = scope(ctx.req, relationship); - const { value: body } = await readJsonBody(ctx.req); + const { value: body } = await readJsonBody(ctx.req, ctx.correlationId); assertOnlyKeys(body, ["schemaVersion", "proposal"], "body"); assertSchemaVersion(body); const proposal = parseProposal(body, workspaceId); @@ -1296,7 +1283,7 @@ async function handleCreateImpl(ctx: RouteContext, deps: UiHandlerDeps): Promise const relationship = readRelationshipDeps(deps); const workspaceId = scope(ctx.req, relationship); const idempotencyHeader = requireIdempotencyKey(ctx.req); - const { raw, value: body } = await readJsonBody(ctx.req); + const { raw, value: body } = await readJsonBody(ctx.req, ctx.correlationId); assertOnlyKeys(body, ["schemaVersion", "proposal"], "body"); assertSchemaVersion(body); const proposal = parseProposal(body, workspaceId); @@ -1598,7 +1585,7 @@ async function performPatchPreflight( const id = requireRelationshipId(ctx.params.id); const ifMatch = requireIfMatch(ctx.req); requireIdempotencyKey(ctx.req); - const { value: body } = await readJsonBody(ctx.req); + const { value: body } = await readJsonBody(ctx.req, ctx.correlationId); assertOnlyKeys(body, ["schemaVersion", "transition", "reconnect"], "body"); assertSchemaVersion(body); const transition = body.transition; diff --git a/packages/keiko-server/src/request-cancellation.test.ts b/packages/keiko-server/src/request-cancellation.test.ts index 4977daed1f..0918df6a40 100644 --- a/packages/keiko-server/src/request-cancellation.test.ts +++ b/packages/keiko-server/src/request-cancellation.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from "node:events"; import type { IncomingMessage, ServerResponse } from "node:http"; import { describe, expect, it } from "vitest"; -import { createRequestCancellation } from "./request-cancellation.js"; +import { createRequestCancellation, requestAlreadyClosed } from "./request-cancellation.js"; import type { RouteContext } from "./routes.js"; interface RequestDouble extends EventEmitter { @@ -123,3 +123,64 @@ describe("request cancellation", () => { cancellation.dispose(); }); }); + +// `requestAlreadyClosed` itself, exported so `server.ts`'s activity log http-request line can +// reuse it at response `close` time — a moment `createRequestCancellation` never evaluates it at +// (it only calls the predicate once, synchronously, before any response activity). +describe("requestAlreadyClosed", () => { + it("accepts the minimal req/res pair directly, without a full RouteContext", () => { + const req = Object.assign(new EventEmitter(), { complete: true, destroyed: false }); + const res = Object.assign(new EventEmitter(), { + closed: false, + destroyed: false, + writableEnded: true, + }); + + expect( + requestAlreadyClosed({ + req: req as unknown as IncomingMessage, + res: res as unknown as ServerResponse, + }), + ).toBe(false); + }); + + // Verified against a real `http.Server` (not only these fake doubles): Node marks a + // `ServerResponse` `destroyed` once its stream is torn down after a fully successful `res.end()` + // too, not only on an abrupt client disconnect. Every EXISTING caller (`createRequestCancellation`, + // above) only ever evaluated this predicate before any response activity, where `writableEnded` is + // always still `false` — so a `destroyed`-without-`writableEnded` shape never arose there and this + // gap was invisible. The activity log's http-request line calls it AFTER the response may have + // completed, which is exactly the shape this pins: `destroyed` alone must never mean "aborted" + // once the response actually ended. + it("does not treat a normally completed response (destroyed AND writableEnded) as already closed", () => { + const req = Object.assign(new EventEmitter(), { complete: true, destroyed: false }); + const res = Object.assign(new EventEmitter(), { + closed: true, + destroyed: true, + writableEnded: true, + }); + + expect( + requestAlreadyClosed({ + req: req as unknown as IncomingMessage, + res: res as unknown as ServerResponse, + }), + ).toBe(false); + }); + + it("still treats a destroyed response that never finished as closed", () => { + const req = Object.assign(new EventEmitter(), { complete: true, destroyed: false }); + const res = Object.assign(new EventEmitter(), { + closed: false, + destroyed: true, + writableEnded: false, + }); + + expect( + requestAlreadyClosed({ + req: req as unknown as IncomingMessage, + res: res as unknown as ServerResponse, + }), + ).toBe(true); + }); +}); diff --git a/packages/keiko-server/src/request-cancellation.ts b/packages/keiko-server/src/request-cancellation.ts index 66134e3b88..dd543c40f5 100644 --- a/packages/keiko-server/src/request-cancellation.ts +++ b/packages/keiko-server/src/request-cancellation.ts @@ -1,3 +1,4 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; import type { RouteContext } from "./routes.js"; export interface RequestCancellation { @@ -6,9 +7,33 @@ export interface RequestCancellation { readonly dispose: () => void; } -function requestAlreadyClosed(ctx: RouteContext): boolean { - const responseClosed = ctx.res.destroyed || (ctx.res.closed && !ctx.res.writableEnded); - const requestAborted = ctx.req.destroyed && !ctx.req.complete; +// The minimal req/res pair `requestAlreadyClosed` actually reads. A full `RouteContext` satisfies +// this structurally (its `params`/`url`/`correlationId` just come along for the ride), and so does +// the raw pair the top-level HTTP callback in `server.ts` holds before any route has matched — the +// moment the activity log's http-request line needs this same predicate, at response `close`. +export interface CancellableRequestTarget { + readonly req: IncomingMessage; + readonly res: ServerResponse; +} + +// True once either side of the connection is gone: the request socket was destroyed before its +// body finished arriving, or the response socket closed without this server ever marking the +// response ended (a client abort, not a normal completion). Exported so the activity log's +// http-request line (`server.ts`) can reuse this exact predicate instead of re-reading `req`/`res` +// state a second way: `res.statusCode` defaults to 200 from construction regardless of whether +// anything was ever written, so the line needs this same "did the connection actually complete" +// signal to avoid reporting a dropped connection as a successful one. +// +// `!target.res.writableEnded` guards BOTH disjuncts, not just `closed`: verified against a real +// `http.Server` (not only the fake req/res doubles below), Node marks a `ServerResponse` `destroyed` +// once its stream is torn down AFTER a fully successful `res.end()` too — `destroyed` alone does not +// mean "aborted". Only every caller of this function today (`createRequestCancellation`) happens to +// call it before any response activity, where `writableEnded` is always still `false`, so this guard +// was a no-op for that caller and the bug was invisible until a second caller (the activity log's +// http-request line, at response `close`, i.e. potentially AFTER a normal completion) exercised it. +export function requestAlreadyClosed(target: CancellableRequestTarget): boolean { + const responseClosed = !target.res.writableEnded && (target.res.destroyed || target.res.closed); + const requestAborted = target.req.destroyed && !target.req.complete; return requestAborted || responseClosed; } diff --git a/packages/keiko-server/src/routes.ts b/packages/keiko-server/src/routes.ts index ec3b6a83a0..edbcbc322c 100644 --- a/packages/keiko-server/src/routes.ts +++ b/packages/keiko-server/src/routes.ts @@ -375,6 +375,7 @@ import { GIT_DELIVERY_PR_ROUTE_GROUP } from "./gitDelivery/prRoutes.js"; import { GIT_DELIVERY_MERGE_ROUTE_GROUP } from "./gitDelivery/mergeRoutes.js"; import { GIT_DELIVERY_SYNC_ROUTE_GROUP } from "./gitDelivery/syncRoutes.js"; import { GIT_AGENT_OPERATION_ROUTE_GROUP } from "./gitDelivery/agentOperationsRoutes.js"; +import { handleClientDiagnosticIngest } from "./client-diagnostics-routes.js"; export interface ApiError { readonly error: { @@ -1498,6 +1499,10 @@ export const API_ROUTES: readonly RouteDefinition[] = [ // authorization, the server deployment ceiling, and connector-scope grants. Context stays // untrusted-labeled and evidence content-free; upstream failures answer as an opaque 502. ...CODING_CONTEXT_ROUTE_GROUP, + // Browser-side diagnostic ingest (Wave 5 of epic #3233): a bounded, redaction-checked crash/error + // report from the UI's client-diagnostics sink, joined to the server request it describes via + // `correlationId`. See client-diagnostics-routes.ts for the trust boundary this route enforces. + { method: "POST", pattern: "/api/diagnostics/client", handler: handleClientDiagnosticIngest }, ]; interface PreparedRoute { diff --git a/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts b/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts new file mode 100644 index 0000000000..be3a1e9547 --- /dev/null +++ b/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts @@ -0,0 +1,165 @@ +// Regression (#2902 w5-sse-counters review finding): `run-handlers.ts`'s two SSE writers +// (`openSseStream`'s writer, reached from `handleRunEvents` — GET /api/runs/:runId/events — and +// `aggregateRunWriter`, reached from `handleAllRunEvents` — GET /api/runs/events) each call +// `res.destroy()` directly on a rejected write instead of going through `writeOrDestroy`/ +// `writeOrDestroyLegacy`. Neither called `markSseStreamBackpressureKilled` before destroying, so a +// real slow-client backpressure kill on either stream was indistinguishable in the terminal +// `sse.stream.closed` line from an ordinary client disconnect (`reason: "client-disconnected"` +// instead of `"backpressure-killed"`) — exactly the distinction #2902's SSE terminal-line mechanism +// exists to preserve. Kept in its own file (never a describe block appended to run-handlers.test.ts) +// because that suite's `server`/`afterEach` lifecycle is scoped to a real bound HTTP server every +// test in it starts; these tests call the two handlers directly against hand-built doubles and never +// bind a socket, so sharing that lifecycle would be a foreign, non-hermetic dependency. + +import { afterEach, describe, expect, it } from "vitest"; + +import { buildRedactor, createRunRegistry, handleRunEvents, QueueEventSink } from "./index.js"; +import { handleAllRunEvents } from "./run-handlers.js"; +import type { RouteContext } from "./routes.js"; +import type { UiHandlerDeps } from "./deps.js"; +import { createInMemoryUiStore } from "./store/index.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; + +// A minimal `ServerResponse`-shaped double (mirrors `sse-write.test.ts`'s `listenableFakeRes`): +// `write` always rejects, so the very first frame trips the writer's own destroy path. `on` records +// listeners so the test can fire "close" deterministically, exactly as the terminal-line mechanism +// does on a real socket teardown. +function rejectingFakeRes(): { res: RouteContext["res"]; fireClose: () => void } { + // A real `ServerResponse` is an EventEmitter: `.on("close", ...)` may legitimately be called more + // than once (this route does — once for the stream's own frame/byte tracking, once for its + // sink-detach cleanup) and every registered listener fires on a real "close" event. A single-slot + // map here would silently drop all but the last registration and misreport this regression as + // fixed regardless of whether the product code is correct. + const listeners = new Map void)[]>(); + const res = { + writableEnded: false, + destroyed: false, + write: (): boolean => false, + writeHead: (): void => undefined, + end: (): void => undefined, + destroy: (): void => undefined, + on: (event: string, handler: () => void): void => { + const existing = listeners.get(event) ?? []; + existing.push(handler); + listeners.set(event, existing); + }, + } as unknown as RouteContext["res"]; + return { + res, + fireClose: (): void => { + for (const handler of listeners.get("close") ?? []) handler(); + }, + }; +} + +function fakeReq(): RouteContext["req"] { + return { headers: {}, on: (): void => undefined } as unknown as RouteContext["req"]; +} + +function captureServerLog(): BufferedServerLogSink { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + return sink; +} + +function minimalDeps(registry: ReturnType): UiHandlerDeps { + return { + config: undefined, + configPresent: false, + evidenceStore: { put: () => "", list: () => [], get: () => undefined, delete: () => undefined }, + env: {}, + redactor: buildRedactor({}), + registry, + modelPortFactory: () => undefined, + store: createInMemoryUiStore(), + }; +} + +function terminalReason(sink: BufferedServerLogSink): unknown { + const closedLine = sink.events.find((event) => event.op === "sse.stream.closed"); + return (closedLine?.extra as { reason?: unknown } | undefined)?.reason; +} + +afterEach(() => { + resetServerLogger(); +}); + +describe("SSE writers report reason=backpressure-killed, not client-disconnected", () => { + it("handleRunEvents (openSseStream) marks backpressure before destroying on a rejected replay write", () => { + const sink = captureServerLog(); + const registry = createRunRegistry(); + const eventSink = new QueueEventSink(); + eventSink.emit({ + schemaVersion: "1", + runId: "run-bp-1", + fingerprint: "fp-bp-1", + seq: 0, + ts: Date.now(), + type: "workflow:progress", + }); + registry.register({ + runId: "run-bp-1", + fingerprint: "fp-bp-1", + modelId: "test-model", + sink: eventSink, + cancel: () => undefined, + }); + const deps = minimalDeps(registry); + const { res, fireClose } = rejectingFakeRes(); + const ctx: RouteContext = { + req: fakeReq(), + res, + params: { runId: "run-bp-1" }, + url: new URL("http://localhost/api/runs/run-bp-1/events"), + }; + + handleRunEvents(ctx, deps); + fireClose(); + + expect(terminalReason(sink)).toBe("backpressure-killed"); + deps.store.close(); + }); + + it("handleAllRunEvents (aggregateRunWriter) marks backpressure before destroying on a rejected fan-out write", () => { + const sink = captureServerLog(); + const registry = createRunRegistry(); + const eventSink = new QueueEventSink(); + registry.register({ + runId: "run-bp-2", + fingerprint: "fp-bp-2", + modelId: "test-model", + sink: eventSink, + cancel: () => undefined, + }); + const deps = minimalDeps(registry); + const { res, fireClose } = rejectingFakeRes(); + const ctx: RouteContext = { + req: fakeReq(), + res, + params: {}, + url: new URL("http://localhost/api/runs/events"), + }; + + handleAllRunEvents(ctx, deps); + // The aggregate writer attaches to newly-registered runs on emit; the rejected write this test + // targets only happens once a live event is actually fanned out. + eventSink.emit({ + schemaVersion: "1", + runId: "run-bp-2", + fingerprint: "fp-bp-2", + seq: 0, + ts: Date.now(), + type: "workflow:progress", + }); + fireClose(); + + expect(terminalReason(sink)).toBe("backpressure-killed"); + deps.store.close(); + }); +}); diff --git a/packages/keiko-server/src/run-handlers.ts b/packages/keiko-server/src/run-handlers.ts index 22fcd95873..9532456fa7 100644 --- a/packages/keiko-server/src/run-handlers.ts +++ b/packages/keiko-server/src/run-handlers.ts @@ -13,6 +13,7 @@ import type { RunRequest, RunVoiceOrigin } from "./run-request.js"; import { startRun, applyRun, type EngineContext } from "./run-engine.js"; import { ActiveRunLimitError, type AppliableSnapshot, type RunRecord } from "./runs.js"; import { SSE_HEADERS, writeMessageEvent, readyMessage, startSseHeartbeat } from "./sse.js"; +import { markSseStreamBackpressureKilled } from "./sse-write.js"; import type { SseWriter, StreamEvent } from "./sink.js"; import type { RouteContext, RouteResult, HandlerOutcome } from "./routes.js"; import { errorBody, STREAMING } from "./routes.js"; @@ -486,7 +487,10 @@ function aggregateRunWriter( write: (event: StreamEvent): boolean => { if (!agentRecordSessionMatches(record, ctx, deps)) return false; const accepted = writeMessageEvent(ctx.res, event, deps.redactor); - if (!accepted) ctx.res.destroy(); + if (!accepted) { + markSseStreamBackpressureKilled(ctx.res); + ctx.res.destroy(); + } return accepted; }, // A single run reaching terminal must not close the aggregate desktop stream. @@ -518,6 +522,7 @@ function openSseStream( write: (event: StreamEvent): boolean => { const accepted = writeMessageEvent(res, event, redactor); if (!accepted) { + markSseStreamBackpressureKilled(res); res.destroy(); } return accepted; diff --git a/packages/keiko-server/src/server.test.ts b/packages/keiko-server/src/server.test.ts index ffd8f74cc8..2e7b8f055f 100644 --- a/packages/keiko-server/src/server.test.ts +++ b/packages/keiko-server/src/server.test.ts @@ -1,9 +1,10 @@ import { mkdtemp, writeFile, rm, mkdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { EventEmitter } from "node:events"; import { request } from "node:http"; import type { AddressInfo } from "node:net"; -import type { Server } from "node:http"; +import type { IncomingMessage, Server, ServerResponse } from "node:http"; import { gunzipSync } from "node:zlib"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { @@ -18,11 +19,12 @@ import { createRunRegistry, type UiHandlerDeps, } from "./index.js"; -import { createUiServer, UI_HOST } from "./server.js"; +import { createUiServer, logRequestOnClose, UI_HOST, type RequestLogContext } from "./server.js"; import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; import { buildCspHeader } from "./csp.js"; import { resetWorkspaceStateForTests } from "./workspace-state-handlers.js"; import type { EditorHotExitStore } from "./editor/hotExitStore.js"; +import { createBufferedServerLogSink, type ServerLogEvent } from "./observability/index.js"; let server: Server; let staticRoot: string; @@ -1098,3 +1100,214 @@ describe("top-level route-error catch (GEN-TEST-MISSING-008, RB-6)", () => { // catch — so a mid-stream case cannot be constructed without adding a production test hook, which // is out of scope for this test-only change. The headers-not-sent branch above is fully covered. }); + +// Wave 5 (w5-http-request-enrichment, #3233): `logRequestOnClose` reads the matched route pattern, +// query-parameter NAMES, the response byte count, and an `aborted` flag from a per-request context +// `dispatchApi`/`serveStatic`/`writeJson` populate as the request is actually resolved, instead of +// the pre-existing lossy raw-path re-derivation and Node's un-set-by-default `res.statusCode`. Every +// field asserted here did not exist on the pre-fix http-request line at all, so each assertion below +// fails before this wave's `server.ts` change and passes after it. +describe("activity log: http-request line enrichment (Wave 5, w5-http-request-enrichment)", () => { + async function waitForActivityLogEvent( + sink: ReturnType, + timeoutMs = 2000, + ): Promise { + const deadline = Date.now() + timeoutMs; + while (sink.events.length === 0) { + if (Date.now() > deadline) { + throw new Error("timed out waiting for an activity log event"); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const [event] = sink.events; + if (event === undefined) throw new Error("unreachable: length checked above"); + return event; + } + + async function startWithActivityLog(): Promise> { + const sink = createBufferedServerLogSink(); + await closeServer(); + server = createUiServer({ staticRoot, csp: buildCspHeader([]), port, activityLog: sink }); + await new Promise((res) => server.listen(port, UI_HOST, res)); + return sink; + } + + it("uses the exact matched RouteDefinition pattern, not a raw-path guess that can misfire on a customer id shaped like a route word", async () => { + const sink = await startWithActivityLog(); + // "explain" is itself a literal segment of a DIFFERENT route (/api/relationships/:id/explain), + // so a relationship id that happens to be literally "explain" is exactly the case the old + // raw-path auto-template heuristic could misclassify as that literal segment instead of a + // customer-supplied :id. The real match is unambiguous: only /api/relationships/:id (3 + // segments) matches this 3-segment path at all. + await fetchRaw("/api/relationships/explain"); + const event = await waitForActivityLogEvent(sink); + + expect(event.category).toBe("http"); + expect(event.op).toBe("request"); + expect(event.extra?.routeTemplate).toBe("/api/relationships/:id"); + }); + + it("falls back to the route-template reduction of the raw path for an unmatched request", async () => { + const sink = await startWithActivityLog(); + await fetchRaw("/api/this-route-does-not-exist"); + const event = await waitForActivityLogEvent(sink); + + expect(event.status).toBe(404); + expect(event.extra?.routeTemplate).toBe("/api/{id}"); + }); + + it("collects distinct query-parameter NAMES only, sorted, and never a value", async () => { + const sink = await startWithActivityLog(); + await fetchRaw("/api/health?foo=1&bar=2"); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.queryParamNames).toEqual(["bar", "foo"]); + const serialized = JSON.stringify(event.extra); + expect(serialized).not.toContain("=1"); + expect(serialized).not.toContain("=2"); + }); + + it("drops an over-length query-parameter name and counts it instead of logging it", async () => { + const sink = await startWithActivityLog(); + const longName = "n".repeat(200); + await fetchRaw(`/api/health?${longName}=1&normal=2`); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.queryParamNames).toEqual(["normal"]); + expect(event.extra?.queryParamDroppedCount).toBe(1); + expect(JSON.stringify(event.extra)).not.toContain(longName); + }); + + it("records writeJson's own computed response byte count", async () => { + const sink = await startWithActivityLog(); + const res = await fetchRaw("/api/health"); + const expectedBytes = Buffer.byteLength(res.text, "utf8"); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.responseBytes).toBe(expectedBytes); + expect(event.extra?.responseBytes).toBeGreaterThan(0); + }); + + it("reports aborted:false and the real status for a normally completed request", async () => { + const sink = await startWithActivityLog(); + await fetchRaw("/api/health"); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.aborted).toBe(false); + expect(event.status).toBe(200); + }); +}); + +// Direct unit coverage of `logRequestOnClose` itself, using the same fake req/res EventEmitter +// doubles `request-cancellation.test.ts` already uses for `requestAlreadyClosed` — deterministic, +// and independent of real socket teardown timing (which real-HTTP `close` timing cannot reliably +// force into the "aborted before any write" shape below). +describe("logRequestOnClose", () => { + interface RequestDouble extends EventEmitter { + complete: boolean; + destroyed: boolean; + url?: string; + method?: string; + } + + interface ResponseDouble extends EventEmitter { + closed: boolean; + destroyed: boolean; + writableEnded: boolean; + headersSent: boolean; + statusCode: number; + } + + function doubles(): { readonly req: RequestDouble; readonly res: ResponseDouble } { + const req = Object.assign(new EventEmitter(), { + complete: false, + destroyed: false, + url: "/api/health", + method: "GET", + }); + const res = Object.assign(new EventEmitter(), { + closed: false, + destroyed: false, + writableEnded: false, + headersSent: false, + statusCode: 200, + }); + return { req, res }; + } + + it("logs aborted:true and status 0 when the connection is gone and nothing was ever written", () => { + const { req, res } = doubles(); + const sink = createBufferedServerLogSink(); + const context: RequestLogContext = {}; + logRequestOnClose( + req as unknown as IncomingMessage, + res as unknown as ServerResponse, + "corr-aborted", + sink, + context, + ); + + // The client vanished mid-handler: the response socket is destroyed, headers were never sent, + // so `res.statusCode` is still Node's construction-time default of 200. + res.destroyed = true; + res.emit("close"); + + expect(sink.events).toHaveLength(1); + const [event] = sink.events; + expect(event?.extra?.aborted).toBe(true); + expect(event?.status).toBe(0); + }); + + it("does not report status 0 once headers were actually sent, even if the connection later drops", () => { + const { req, res } = doubles(); + const sink = createBufferedServerLogSink(); + const context: RequestLogContext = {}; + logRequestOnClose( + req as unknown as IncomingMessage, + res as unknown as ServerResponse, + "corr-partial", + sink, + context, + ); + + res.headersSent = true; + res.statusCode = 200; + res.destroyed = true; + res.emit("close"); + + const [event] = sink.events; + expect(event?.extra?.aborted).toBe(true); + expect(event?.status).toBe(200); + }); + + it("surfaces routeTemplate, queryParamNames, queryParamDroppedCount and responseBytes from the request context", () => { + const { req, res } = doubles(); + const sink = createBufferedServerLogSink(); + const context: RequestLogContext = { + routeTemplate: "/api/memory/:id", + queryParamNames: ["bar", "foo"], + queryParamDroppedCount: 2, + responseBytes: 42, + }; + logRequestOnClose( + req as unknown as IncomingMessage, + res as unknown as ServerResponse, + "corr-context", + sink, + context, + ); + + res.headersSent = true; + res.writableEnded = true; + res.emit("close"); + + const [event] = sink.events; + expect(event?.extra).toMatchObject({ + routeTemplate: "/api/memory/:id", + queryParamNames: ["bar", "foo"], + queryParamDroppedCount: 2, + responseBytes: 42, + aborted: false, + }); + }); +}); diff --git a/packages/keiko-server/src/server.ts b/packages/keiko-server/src/server.ts index 7d7d8e401a..d34b77f93f 100644 --- a/packages/keiko-server/src/server.ts +++ b/packages/keiko-server/src/server.ts @@ -8,6 +8,7 @@ import { Readable } from "node:stream"; import { createGzip } from "node:zlib"; import { applySecurityHeaders } from "./headers.js"; import { isAllowedHost } from "./host-check.js"; +import { requestAlreadyClosed } from "./request-cancellation.js"; import { resolveContainedPath, serveFile } from "./static.js"; import { errorBody, @@ -20,7 +21,12 @@ import { type RouteContext, } from "./routes.js"; import { buildRedactor, type UiHandlerDeps } from "./deps.js"; -import { nullServerLogSink, type ServerLogSink } from "./observability/server-log.js"; +import { + MAX_LOG_STRING_LENGTH, + nullServerLogSink, + redactRoutePath, + type ServerLogSink, +} from "./observability/server-log.js"; import { CORRELATION_RESPONSE_HEADER, resolveCorrelationId } from "./correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError } from "./diagnostics-log.js"; import { isVoiceDictationCapable, isVoiceRealtimeCapable } from "./read-handlers.js"; @@ -37,6 +43,17 @@ export { UI_HOST }; export { DEFAULT_UI_PORT } from "@oscharko-dev/keiko-contracts"; const CSP_CACHE_TTL_MS = 1000; const JSON_GZIP_MIN_BYTES = 1024; +// The http-request line's query-param-NAME field admits only a bounded identifier shape — the +// same bar `log-redaction.ts`'s own field-name guard holds a field NAME to, applied here to a +// query PARAM name before it ever reaches `extra`. Restated rather than imported: importing +// `log-redaction.ts`'s private constant here would reach across the redaction choke point for a +// shape both files already independently agree is "a bounded identifier", +// `^[A-Za-z_][A-Za-z0-9_.-]{0,63}$`. +const QUERY_PARAM_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_.-]{0,63}$/; +// Mirrors the bounded-array discipline every other array field in this schema already has +// (`MAX_LOG_ARRAY_LENGTH`, `keikoStackFrames`' own frame cap): a request with more distinct query +// parameter names than this is truncated, not silently grown without limit. +const MAX_QUERY_PARAM_NAMES = 16; const cspCache = new WeakMap< UiServerDeps, { readonly value: string; readonly expiresAt: number } @@ -62,6 +79,21 @@ export interface UiServerDeps { readonly activityLog?: ServerLogSink | undefined; } +// Per-request scratch space the http-request line reads at response `close`, populated as the +// request is actually resolved: `handle` fills in the query-parameter fields as soon as the URL is +// parsed, `dispatchApi`/`serveStatic` fill in `routeTemplate` once the route (or its absence) is +// known, and `writeJson` fills in `responseBytes` the one time it actually serialises a body. A +// fresh object is created once per incoming request and threaded explicitly through the same call +// chain `correlationId` already travels — plain per-request state, not a keyed side-table. +// Exported for `server.test.ts` only (not part of the package's public entry point, mirroring how +// `request-cancellation.ts` exports its own req/res-close shapes for the same reason). +export interface RequestLogContext { + routeTemplate?: string; + queryParamNames?: readonly string[]; + queryParamDroppedCount?: number; + responseBytes?: number; +} + function acceptsGzip(acceptEncoding: string | readonly string[] | undefined): boolean { const value = typeof acceptEncoding === "string" ? acceptEncoding : (acceptEncoding ?? []).join(","); @@ -77,16 +109,19 @@ function writeJson( status: number, body: unknown, headers: Readonly> = {}, + context?: RequestLogContext, ): void { res.statusCode = status; for (const [key, value] of Object.entries(headers)) { res.setHeader(key, typeof value === "string" ? value : [...value]); } if (status === 204 || status === 304) { + if (context !== undefined) context.responseBytes = 0; res.end(); return; } const payload = Buffer.from(JSON.stringify(body), "utf8"); + if (context !== undefined) context.responseBytes = payload.byteLength; res.setHeader("Content-Type", "application/json; charset=utf-8"); if (payload.byteLength >= JSON_GZIP_MIN_BYTES && acceptsGzip(req.headers["accept-encoding"])) { res.setHeader("Content-Encoding", "gzip"); @@ -110,17 +145,30 @@ function hasCsrfHeader(req: IncomingMessage): boolean { return value === "1"; } -function rejectUnsupportedMediaType(req: IncomingMessage, res: ServerResponse): void { +function rejectUnsupportedMediaType( + req: IncomingMessage, + res: ServerResponse, + context: RequestLogContext, +): void { writeJson( req, res, 415, errorBody("UNSUPPORTED_MEDIA_TYPE", "State-changing API requests must use JSON."), + {}, + context, ); } -function rejectCsrf(req: IncomingMessage, res: ServerResponse): void { - writeJson(req, res, 403, errorBody("FORBIDDEN_CSRF", "Missing state-changing request guard.")); +function rejectCsrf(req: IncomingMessage, res: ServerResponse, context: RequestLogContext): void { + writeJson( + req, + res, + 403, + errorBody("FORBIDDEN_CSRF", "Missing state-changing request guard."), + {}, + context, + ); } // A minimal default deps object so a 3-arg server can still serve the deps-bound read routes (e.g. @@ -154,21 +202,31 @@ function rejectIfInvalidStateChange( res: ServerResponse, method: string, pathname: string, + context: RequestLogContext, ): boolean { if (!isJsonRequest(req)) { - rejectUnsupportedMediaType(req, res); + rejectUnsupportedMediaType(req, res, context); return true; } if (isCsrfExemptStateChange(method, pathname)) { return false; } if (!hasCsrfHeader(req)) { - rejectCsrf(req, res); + rejectCsrf(req, res, context); return true; } return false; } +// The route-template reduction of the raw path, for a request `dispatchApi` never resolved to a +// declared route (404/405) or that never entered API dispatch at all (a static asset). A matched +// API request gets the exact `RouteDefinition.pattern` instead (`dispatchApi` sets it directly) — +// the two no longer come from the same lossy, independently-re-derived heuristic. +function setFallbackRouteTemplate(context: RequestLogContext, pathname: string): void { + const template = redactRoutePath(pathname, MAX_LOG_STRING_LENGTH); + if (template !== undefined) context.routeTemplate = template; +} + async function dispatchApi( handlerDeps: UiHandlerDeps, req: IncomingMessage, @@ -176,17 +234,24 @@ async function dispatchApi( method: string, url: URL, correlationId: string, + context: RequestLogContext, ): Promise { const match = matchRoute(method, url.pathname); if (match === undefined) { - writeJson(req, res, 404, notFoundBody()); + setFallbackRouteTemplate(context, url.pathname); + writeJson(req, res, 404, notFoundBody(), {}, context); return; } if (match === "method-not-allowed") { - writeJson(req, res, 405, methodNotAllowedBody()); + setFallbackRouteTemplate(context, url.pathname); + writeJson(req, res, 405, methodNotAllowedBody(), {}, context); return; } - if (isStateChangingMethod(method) && rejectIfInvalidStateChange(req, res, method, url.pathname)) { + context.routeTemplate = match.definition.pattern; + const invalidStateChange = + isStateChangingMethod(method) && + rejectIfInvalidStateChange(req, res, method, url.pathname, context); + if (invalidStateChange) { return; } const ctx: RouteContext = { req, res, params: match.params, url, correlationId }; @@ -194,7 +259,7 @@ async function dispatchApi( if (outcome === STREAMING) { return; } - writeJson(req, res, outcome.status, outcome.body, outcome.headers); + writeJson(req, res, outcome.status, outcome.body, outcome.headers, context); } function resolveStaticTargets(pathname: string): readonly string[] { @@ -212,7 +277,9 @@ async function serveStatic( res: ServerResponse, staticRoot: string, pathname: string, + context: RequestLogContext, ): Promise { + setFallbackRouteTemplate(context, pathname); const targets = resolveStaticTargets(pathname); for (const target of targets) { const resolved = resolveContainedPath(staticRoot, target); @@ -227,12 +294,23 @@ async function serveStatic( if (await serveFile(res, indexPath, req.headers["accept-encoding"])) { return; } - writeJson(req, res, 404, errorBody("NOT_FOUND", "The requested resource was not found.")); + writeJson( + req, + res, + 404, + errorBody("NOT_FOUND", "The requested resource was not found."), + {}, + context, + ); } -function rejectForbiddenHost(req: IncomingMessage, res: ServerResponse): void { +function rejectForbiddenHost( + req: IncomingMessage, + res: ServerResponse, + context: RequestLogContext, +): void { const body: ApiError = errorBody("FORBIDDEN_HOST", "Request host is not the local interface."); - writeJson(req, res, 403, body); + writeJson(req, res, 403, body, {}, context); } async function resolveCsp(deps: UiServerDeps): Promise { @@ -252,14 +330,40 @@ async function resolveCsp(deps: UiServerDeps): Promise { } } +// Names only — values never leave this function. Deduplicated (a repeated key collapses to one +// name), sorted for a stable line, and capped the same way every other bounded array field in this +// schema is: a name beyond the cap, and any name that is not a bounded identifier (the same shape +// a log field name itself must have), is dropped and COUNTED rather than silently grown or +// silently truncated with no trace. +function computeQueryParamFields(url: URL, context: RequestLogContext): void { + const names = new Set(url.searchParams.keys()); + const kept: string[] = []; + let dropped = 0; + for (const name of names) { + if (QUERY_PARAM_NAME_PATTERN.test(name)) { + kept.push(name); + } else { + dropped += 1; + } + } + kept.sort((a, b) => a.localeCompare(b)); + if (kept.length > MAX_QUERY_PARAM_NAMES) { + dropped += kept.length - MAX_QUERY_PARAM_NAMES; + } + context.queryParamNames = kept.slice(0, MAX_QUERY_PARAM_NAMES); + if (dropped > 0) context.queryParamDroppedCount = dropped; +} + async function handle( deps: UiServerDeps, handlerDeps: UiHandlerDeps, req: IncomingMessage, res: ServerResponse, correlationId: string, + context: RequestLogContext, ): Promise { const url = new URL(req.url ?? "/", `http://${UI_HOST}`); + computeQueryParamFields(url, context); const apiPath = isApiPath(url.pathname); // Issue #495/#497 — scope the Permissions-Policy microphone directive to deployments that advertise // speech-to-text dictation OR full-realtime voice (whose WebRTC capture track also needs the mic); @@ -268,15 +372,15 @@ async function handle( allowMicrophone: isVoiceDictationCapable(handlerDeps) || isVoiceRealtimeCapable(handlerDeps), }); if (!isAllowedHost(req, deps.port)) { - rejectForbiddenHost(req, res); + rejectForbiddenHost(req, res, context); return; } const method = (req.method ?? "GET").toUpperCase(); if (apiPath) { - await dispatchApi(handlerDeps, req, res, method, url, correlationId); + await dispatchApi(handlerDeps, req, res, method, url, correlationId, context); return; } - await serveStatic(req, res, deps.staticRoot, url.pathname); + await serveStatic(req, res, deps.staticRoot, url.pathname, context); } function createVoicePlanes( @@ -304,31 +408,108 @@ function createVoicePlanes( // ADR-0101) re-opens the upgrade for the single loopback voice control path `/api/voice/control`, and // ONLY when the deployment is full-realtime voice capable; every other upgrade keeps the hard reject. +// Content-free by construction: every value here is a count, a bounded label, or a route +// TEMPLATE — never a raw query value, a header or a body. `path` keeps its pre-existing behaviour +// (reduced generically by `log-redaction.ts`'s own path guard); `routeTemplate` is the field this +// wave adds so a reader learns WHICH declared route actually matched, sourced from the real match +// `dispatchApi` resolved rather than re-derived independently from the same raw path. +function buildHttpRequestExtra( + method: string, + path: string, + aborted: boolean, + context: RequestLogContext, +): Record { + return { + method, + path, + routeTemplate: context.routeTemplate, + queryParamNames: context.queryParamNames ?? [], + queryParamDroppedCount: context.queryParamDroppedCount, + responseBytes: context.responseBytes ?? 0, + aborted, + }; +} + // Every incoming HTTP request emits ONE structured line in the activity log on close, with the // correlation id echoed in the response header. Extracted from the createServer callback so the -// top-level handler stays under the 50-line ceiling; the payload is intentionally content-free -// (method, path, status, duration). -function logRequestOnClose( +// top-level handler stays under the 50-line ceiling. `context` is the SAME object `handle`'s call +// chain mutates as the request is actually resolved (query-parameter fields at URL-parse time, the +// matched route pattern once `dispatchApi`/`serveStatic` know it, the response byte count the one +// time `writeJson` actually serialises a body) — read here only once the response has finished or +// the connection has gone away, so every mutation above is guaranteed to have already happened. +// +// `aborted` reuses the SAME `requestAlreadyClosed` predicate `request-cancellation.ts` already uses +// to decide whether an in-flight handler's `AbortSignal` should fire — not a second, independent +// read of `req`/`res` state invented for this line. When the connection is gone AND no response was +// ever actually sent (`!res.headersSent`), `status` is logged as 0 instead of Node's own default +// `res.statusCode` of 200: that default holds from construction regardless of whether anything was +// ever written, so echoing it here for an aborted, header-less response would misreport a dropped +// connection as a successful one. +// +// Exported for `server.test.ts` only, so the close-time behaviour (aborted mid-response before any +// write, a normally completed request) can be driven with the same fake req/res EventEmitter +// doubles `request-cancellation.test.ts` already uses for `requestAlreadyClosed` itself, instead of +// depending on real socket teardown timing. +export function logRequestOnClose( req: IncomingMessage, res: ServerResponse, correlationId: string, activityLog: ServerLogSink, + context: RequestLogContext, ): void { const startedAt = Date.now(); const requestUrl = req.url ?? ""; const method = req.method ?? "GET"; res.on("close", () => { + const aborted = requestAlreadyClosed({ req, res }); + const status = aborted && !res.headersSent ? 0 : res.statusCode; activityLog.write({ category: "http", op: "request", correlationId, - status: res.statusCode, + status, durationMs: Date.now() - startedAt, - extra: { method, path: requestUrl.split("?")[0] }, + extra: buildHttpRequestExtra(method, requestUrl.split("?")[0] ?? "", aborted, context), }); }); } +// The cause is no longer discarded: it is routed — REDACTED — to the operator diagnostic sink, +// keyed by the correlation id, and the id is folded into the opaque 500 body so a user-reported +// failure can be tied back to exactly one server-side record (GEN-OBS-DIAGNOSTICS-901). Split out +// of `createUiServer`'s callback so that closure stays under the line-count ceiling. +function reportTopLevelFailure( + req: IncomingMessage, + res: ServerResponse, + correlationId: string, + handlerDeps: UiHandlerDeps, + context: RequestLogContext, + error: unknown, +): void { + emitServerDiagnostic( + handlerDeps.diagnostics, + serverDiagnosticFromError({ + correlationId, + operation: "server.request", + source: "server.top-level-catch", + error, + redact: (message) => String(handlerDeps.redactor(message)), + }), + ); + if (!res.headersSent) { + writeJson( + req, + res, + 500, + errorBody("INTERNAL", "An unexpected error occurred.", correlationId), + {}, + context, + ); + } else { + res.end(); + } +} + export function createUiServer(deps: UiServerDeps): Server { const handlerDeps = deps.handlerDeps ?? fallbackDeps(); const { voiceControl, liveDictation } = createVoicePlanes(deps.port, handlerDeps); @@ -336,32 +517,13 @@ export function createUiServer(deps: UiServerDeps): Server { const server = createServer((req, res) => { const correlationId = resolveCorrelationId(req); res.setHeader(CORRELATION_RESPONSE_HEADER, correlationId); - logRequestOnClose(req, res, correlationId, activityLog); - void handle(deps, handlerDeps, req, res, correlationId).catch((error: unknown) => { - // The cause is no longer discarded: it is routed — REDACTED — to the operator diagnostic sink, - // keyed by the correlation id, and the id is folded into the opaque 500 body so a user-reported - // failure can be tied back to exactly one server-side record (GEN-OBS-DIAGNOSTICS-901). - emitServerDiagnostic( - handlerDeps.diagnostics, - serverDiagnosticFromError({ - correlationId, - operation: "server.request", - source: "server.top-level-catch", - error, - redact: (message) => String(handlerDeps.redactor(message)), - }), - ); - if (!res.headersSent) { - writeJson( - req, - res, - 500, - errorBody("INTERNAL", "An unexpected error occurred.", correlationId), - ); - } else { - res.end(); - } - }); + const requestLogContext: RequestLogContext = {}; + logRequestOnClose(req, res, correlationId, activityLog, requestLogContext); + void handle(deps, handlerDeps, req, res, correlationId, requestLogContext).catch( + (error: unknown) => { + reportTopLevelFailure(req, res, correlationId, handlerDeps, requestLogContext, error); + }, + ); }); server.on("upgrade", (req, socket, head) => { if (voiceControl.handleUpgrade(req, socket, head)) { diff --git a/packages/keiko-server/src/sse-write.test.ts b/packages/keiko-server/src/sse-write.test.ts index dafaa2f780..641c86a2b9 100644 --- a/packages/keiko-server/src/sse-write.test.ts +++ b/packages/keiko-server/src/sse-write.test.ts @@ -2,9 +2,17 @@ // exactly once BEFORE the socket is destroyed, so a slow-client termination is not silently relabeled // as a user cancel. The signal must carry only non-secret counts (no body bytes) and an observer throw // must never break the protective abort+destroy path. -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { ServerResponse } from "node:http"; +import { mockResponse } from "./_support.js"; import { writeOrDestroy, type SseBackpressureSignal } from "./sse-write.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; function fakeRes(writeReturns: boolean): { res: ServerResponse; @@ -93,4 +101,147 @@ describe("writeOrDestroy backpressure signal (GEN-PERF-CHAT-006)", () => { expect(controller.signal.aborted).toBe(true); expect(destroy).toHaveBeenCalledTimes(1); }); + + it("never throws when res.on is not implemented (a minimal write/destroy-only double)", () => { + const { res } = fakeRes(true); + const controller = new AbortController(); + + expect(() => writeOrDestroy(res, "frame", controller)).not.toThrow(); + }); +}); + +// The terminal `sse.stream.closed` line (#2902 w5-sse-counters): a per-response frame/byte counter +// closed over the SAME write path every SSE route already funnels through, surfaced exactly once +// when the stream reaches its terminal `close` event. +describe("sse.stream.closed terminal line", () => { + function captureServerLog(): BufferedServerLogSink { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + return sink; + } + + // A hand-rolled EventEmitter-shaped fake (mirrors sse.test.ts's own convention) so a test can fire + // "close" deterministically without waiting on a real stream's autoDestroy timing. + function listenableFakeRes(writeReturns: boolean): { + res: ServerResponse; + fireClose: () => void; + } { + const listeners = new Map void>(); + const res = { + writableEnded: false, + write: vi.fn().mockReturnValue(writeReturns), + destroy: vi.fn(), + on: (event: string, handler: () => void) => { + listeners.set(event, handler); + }, + } as unknown as ServerResponse; + return { res, fireClose: () => listeners.get("close")?.() }; + } + + afterEach(() => { + resetServerLogger(); + }); + + it("counts every frame written through writeOrDestroy and reports reason=completed on a real stream end", async () => { + const sink = captureServerLog(); + const { res } = mockResponse(); + const controller = new AbortController(); + const frames = ["event: a\ndata: {}\n\n", "event: b\ndata: {}\n\n", "event: c\ndata: {}\n\n"]; + + for (const frame of frames) writeOrDestroy(res, frame, controller); + const closed = new Promise((resolve) => res.once("close", resolve)); + res.end(); + await closed; + + expect(sink.events).toHaveLength(1); + const [event] = sink.events; + expect(event).toMatchObject({ level: "info", category: "http", op: "sse.stream.closed" }); + expect(typeof event?.durationMs).toBe("number"); + expect(event?.extra).toEqual({ + frameCount: 3, + bytesStreamed: frames.reduce((sum, f) => sum + Buffer.byteLength(f, "utf8"), 0), + reason: "completed", + }); + }); + + it("reports reason=client-disconnected when the socket closes without the producer ending it", async () => { + const sink = captureServerLog(); + const { res } = mockResponse(); + const controller = new AbortController(); + + writeOrDestroy(res, "event: a\ndata: {}\n\n", controller); + const closed = new Promise((resolve) => res.once("close", resolve)); + res.destroy(); + await closed; + + expect(sink.events[0]?.extra).toMatchObject({ reason: "client-disconnected" }); + }); + + it("reports reason=backpressure-killed and threads the supplied correlation id", () => { + const sink = captureServerLog(); + const { res, fireClose } = listenableFakeRes(false); + const controller = new AbortController(); + + writeOrDestroy(res, "event: a\ndata: {}\n\n", controller, undefined, "corr-sse-1"); + fireClose(); + + expect(sink.events).toEqual([ + { + level: "info", + category: "http", + op: "sse.stream.closed", + correlationId: "corr-sse-1", + durationMs: expect.any(Number) as number, + status: undefined, + errorKind: undefined, + extra: { + frameCount: 1, + bytesStreamed: Buffer.byteLength("event: a\ndata: {}\n\n"), + reason: "backpressure-killed", + }, + }, + ]); + }); + + it("emits the terminal line exactly once even when close fires more than once", () => { + const sink = captureServerLog(); + const { res, fireClose } = listenableFakeRes(true); + const controller = new AbortController(); + + writeOrDestroy(res, "frame", controller); + fireClose(); + fireClose(); + + expect(sink.events).toHaveLength(1); + }); + + it("reports reason=server-error when the response emits an error before closing", () => { + const sink = captureServerLog(); + const listeners = new Map void>(); + const res = { + writableEnded: false, + write: vi.fn().mockReturnValue(true), + destroy: vi.fn(), + on: (event: string, handler: () => void) => { + listeners.set(event, handler); + }, + } as unknown as ServerResponse; + const controller = new AbortController(); + + writeOrDestroy(res, "frame", controller); + listeners.get("error")?.(); + listeners.get("close")?.(); + + expect(sink.events[0]?.extra).toMatchObject({ reason: "server-error" }); + }); + + it("never tracks or logs a response that does not implement .on", () => { + const sink = captureServerLog(); + const { res } = fakeRes(true); + const controller = new AbortController(); + + writeOrDestroy(res, "frame", controller); + + expect(sink.events).toEqual([]); + }); }); diff --git a/packages/keiko-server/src/sse-write.ts b/packages/keiko-server/src/sse-write.ts index f356844e58..6fefad4196 100644 --- a/packages/keiko-server/src/sse-write.ts +++ b/packages/keiko-server/src/sse-write.ts @@ -14,6 +14,7 @@ import { randomUUID } from "node:crypto"; import type { ServerResponse } from "node:http"; import { emitServerDiagnostic, type ServerDiagnosticSink } from "./diagnostics-log.js"; +import { getServerLogger } from "./observability/index.js"; /** * Backpressure signal (GEN-PERF-CHAT-006). Emitted exactly once when a write is rejected because the @@ -26,6 +27,116 @@ export interface SseBackpressureSignal { readonly accepted: false; } +type SseStreamCloseReason = + "completed" | "client-disconnected" | "backpressure-killed" | "server-error"; + +interface SseStreamCounterState { + frameCount: number; + bytesStreamed: number; + readonly startedAt: number; + backpressureKilled: boolean; + serverErrored: boolean; + emitted: boolean; + correlationId: string | undefined; +} + +// Per-response frame/byte counters (#2902 w5-sse-counters), keyed by the live `ServerResponse` so +// every SSE write path in the server — `writeOrDestroy` here, plus `sse.ts`'s legacy write path, +// `writeEvent`/`writeMessageEvent` and the heartbeat — shares ONE count per stream without any of +// them needing a threaded "deps" object. A `WeakMap` means a stream that never gets tracked (a +// minimal unit-test double with no `.on`) costs nothing and is never retained past `res`'s own +// lifetime. +const sseStreamCounters = new WeakMap(); + +function sseStreamReason(res: ServerResponse, state: SseStreamCounterState): SseStreamCloseReason { + if (state.backpressureKilled) return "backpressure-killed"; + if (state.serverErrored) return "server-error"; + // A stream whose producer called `res.end()` and had it fully flush is "completed"; one whose + // socket closed without that — the ordinary shape of a client navigating away or losing network — + // is "client-disconnected". This is the same distinction `logRequestOnClose` does not need to make + // (a JSON response always ends itself before `close`) but a long-lived SSE stream does. + return res.writableEnded ? "completed" : "client-disconnected"; +} + +// Emitted exactly once per stream (guarded by `state.emitted`), on `res`'s terminal `close` event — +// the one event every SSE stream reaches exactly once, whether it finished normally, was killed for +// backpressure, or the client simply disconnected. +function emitSseStreamClosed(res: ServerResponse, state: SseStreamCounterState): void { + if (state.emitted) return; + state.emitted = true; + getServerLogger().info({ + category: "http", + op: "sse.stream.closed", + ...(state.correlationId === undefined ? {} : { correlationId: state.correlationId }), + durationMs: Date.now() - state.startedAt, + extra: { + frameCount: state.frameCount, + bytesStreamed: state.bytesStreamed, + reason: sseStreamReason(res, state), + }, + }); +} + +// Lazily creates and attaches the terminal-line listeners the first time a frame is recorded for +// `res`; the `WeakMap` guard means `close`/`error` are attached exactly once per stream regardless +// of how many frames it writes. A response double that does not implement `.on` (several SSE route +// suites construct a bare `{ write, destroy }` fake) is left untracked rather than throwing — such a +// fake never reaches a real `close` event either, so there is nothing correct to count. +function sseStreamState(res: ServerResponse): SseStreamCounterState | undefined { + const existing = sseStreamCounters.get(res); + if (existing !== undefined) return existing; + if (typeof res.on !== "function") return undefined; + const state: SseStreamCounterState = { + frameCount: 0, + bytesStreamed: 0, + startedAt: Date.now(), + backpressureKilled: false, + serverErrored: false, + emitted: false, + correlationId: undefined, + }; + sseStreamCounters.set(res, state); + res.on("close", () => { + emitSseStreamClosed(res, state); + }); + res.on("error", () => { + state.serverErrored = true; + }); + return state; +} + +/** + * Records one SSE frame write against `res`'s per-stream counter (#2902 w5-sse-counters). Closed + * over the SAME write path every SSE route already funnels through — `writeOrDestroy` below, plus + * `sse.ts`'s legacy write path, `writeEvent`, `writeMessageEvent` and the heartbeat — so no route + * handler has to opt in. Never throws: observability must never break a write. + */ +export function recordSseStreamFrame( + res: ServerResponse, + frame: string, + correlationId?: string, +): void { + const state = sseStreamState(res); + if (state === undefined) return; + state.frameCount += 1; + state.bytesStreamed += Buffer.byteLength(frame, "utf8"); + if (correlationId !== undefined && state.correlationId === undefined) { + state.correlationId = correlationId; + } +} + +/** + * Marks `res`'s stream as ended-by-backpressure so the terminal line reports `"backpressure-killed"` + * rather than the generic `"client-disconnected"`. Called only from a write path that is about to + * `destroy()` the socket as a direct, deterministic consequence of the rejected write — never from + * `writeEvent`/`writeMessageEvent`, where a caller-observed `false` does not by itself mean the + * stream is being torn down. + */ +export function markSseStreamBackpressureKilled(res: ServerResponse): void { + const state = sseStreamCounters.get(res); + if (state !== undefined) state.backpressureKilled = true; +} + /** * Writes `frame` to `res`. When `res.write` returns false (TCP send-buffer full / slow client), * aborts `controller` (stops the upstream producer) and destroys the socket. @@ -35,6 +146,10 @@ export interface SseBackpressureSignal { * backpressure kill rather than silently relabeling it as a user cancel. The callback is wrapped in a * try/catch so an observer throw can never propagate into the write loop. * + * `correlationId` is optional (most SSE routes never received a `RouteContext`, mirroring + * `readBoundedRequestBody`'s own optional correlation id) and, when supplied, is attached to this + * stream's terminal `sse.stream.closed` line. + * * Returns the raw boolean from `res.write` so callers can short-circuit if needed. */ export function writeOrDestroy( @@ -42,9 +157,12 @@ export function writeOrDestroy( frame: string, controller: AbortController, onBackpressure?: (signal: SseBackpressureSignal) => void, + correlationId?: string, ): boolean { + recordSseStreamFrame(res, frame, correlationId); const accepted = res.write(frame); if (!accepted) { + markSseStreamBackpressureKilled(res); if (onBackpressure !== undefined) { try { onBackpressure({ frameBytes: Buffer.byteLength(frame, "utf8"), accepted: false }); diff --git a/packages/keiko-server/src/sse.ts b/packages/keiko-server/src/sse.ts index 3fd5b6cfdd..0299971bc0 100644 --- a/packages/keiko-server/src/sse.ts +++ b/packages/keiko-server/src/sse.ts @@ -9,7 +9,12 @@ import type { ServerResponse } from "node:http"; import type { StreamEvent } from "./sink.js"; import type { Redactor } from "./deps.js"; import { redactedEventJson } from "./sse-frame-cache.js"; -import { writeOrDestroy, type SseBackpressureSignal } from "./sse-write.js"; +import { + markSseStreamBackpressureKilled, + recordSseStreamFrame, + writeOrDestroy, + type SseBackpressureSignal, +} from "./sse-write.js"; /** * Optional protective wiring for the heartbeat. A heartbeat can be the FIRST write rejected on an @@ -20,11 +25,18 @@ import { writeOrDestroy, type SseBackpressureSignal } from "./sse-write.js"; export interface SseHeartbeatBackpressure { readonly controller: AbortController; readonly onBackpressure?: ((signal: SseBackpressureSignal) => void) | undefined; + // Attached to this stream's terminal `sse.stream.closed` line (#2902 w5-sse-counters) when the + // caller already holds the request-scoped correlation id. + readonly correlationId?: string | undefined; } function writeOrDestroyLegacy(res: ServerResponse, frame: string): boolean { + recordSseStreamFrame(res, frame); const accepted = res.write(frame); - if (!accepted) res.destroy(); + if (!accepted) { + markSseStreamBackpressureKilled(res); + res.destroy(); + } return accepted; } @@ -47,7 +59,13 @@ export function startSseHeartbeat( const writeFrame = (frame: string): boolean => backpressure === undefined ? writeOrDestroyLegacy(res, frame) - : writeOrDestroy(res, frame, backpressure.controller, backpressure.onBackpressure); + : writeOrDestroy( + res, + frame, + backpressure.controller, + backpressure.onBackpressure, + backpressure.correlationId, + ); const timer = setInterval(() => { if (res.destroyed || res.writableEnded) return; if (!writeFrame(": keep-alive\n\n")) { @@ -94,7 +112,9 @@ export function readyMessage(): string { // Writes one framed event to the response stream. Returns Node's backpressure signal so the caller can // detach a slow client instead of letting the HTTP response buffer grow without bound. export function writeEvent(res: ServerResponse, event: StreamEvent, redactor: Redactor): boolean { - return res.write(frameEvent(event, redactor)); + const frame = frameEvent(event, redactor); + recordSseStreamFrame(res, frame); + return res.write(frame); } export function writeMessageEvent( @@ -102,5 +122,7 @@ export function writeMessageEvent( event: StreamEvent, redactor: Redactor, ): boolean { - return res.write(frameMessageEvent(event, redactor)); + const frame = frameMessageEvent(event, redactor); + recordSseStreamFrame(res, frame); + return res.write(frame); } diff --git a/packages/keiko-ui/src/app/atlassian-connectors/error.tsx b/packages/keiko-ui/src/app/atlassian-connectors/error.tsx index a27f043ec5..9b8b3f3827 100644 --- a/packages/keiko-ui/src/app/atlassian-connectors/error.tsx +++ b/packages/keiko-ui/src/app/atlassian-connectors/error.tsx @@ -11,7 +11,7 @@ import { useEffect, type ReactNode } from "react"; import { useTranslate } from "@/lib/i18n"; -import { clientErrorSummary } from "@/lib/client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "@/lib/client-error-summary"; import { reportClientDiagnostic } from "@/lib/client-diagnostics"; export default function AtlassianConnectorsRouteError({ @@ -28,6 +28,7 @@ export default function AtlassianConnectorsRouteError({ // stay diagnosable from the console even though the UI only shows the recovery surface. reportClientDiagnostic( `[keiko] atlassian-connectors route crashed: ${clientErrorSummary(error)}`, + { correlationId: correlationIdOf(error) }, ); }, [error]); diff --git a/packages/keiko-ui/src/app/components/desktop/AppShellBoundary.tsx b/packages/keiko-ui/src/app/components/desktop/AppShellBoundary.tsx index 8c9cd2cc69..10c65ddfff 100644 --- a/packages/keiko-ui/src/app/components/desktop/AppShellBoundary.tsx +++ b/packages/keiko-ui/src/app/components/desktop/AppShellBoundary.tsx @@ -14,7 +14,7 @@ import { Component, type ReactNode } from "react"; import { useTranslate, type I18nTranslate } from "@/lib/i18n"; import { resetPersistedShortcutOverrides } from "./shellRecovery"; import styles from "./AppShellBoundary.module.css"; -import { clientErrorSummary } from "@/lib/client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "@/lib/client-error-summary"; import { reportClientDiagnostic } from "@/lib/client-diagnostics"; interface AppShellBoundaryProps { @@ -45,7 +45,9 @@ class InnerAppShellBoundary extends Component { diff --git a/packages/keiko-ui/src/app/components/desktop/hooks/useUnhandledRejectionLog.ts b/packages/keiko-ui/src/app/components/desktop/hooks/useUnhandledRejectionLog.ts index 8c572d9637..e316ec5b4b 100644 --- a/packages/keiko-ui/src/app/components/desktop/hooks/useUnhandledRejectionLog.ts +++ b/packages/keiko-ui/src/app/components/desktop/hooks/useUnhandledRejectionLog.ts @@ -11,7 +11,7 @@ import { useEffect } from "react"; import { reportClientDiagnostic } from "@/lib/client-diagnostics"; -import { clientErrorSummary } from "@/lib/client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "@/lib/client-error-summary"; const MAX_LOGGED_REJECTIONS = 5; @@ -23,6 +23,7 @@ export function useUnhandledRejectionLog(): void { logged += 1; reportClientDiagnostic( `[keiko] unhandled promise rejection: ${clientErrorSummary(event.reason)}`, + { correlationId: correlationIdOf(event.reason) }, ); }; window.addEventListener("unhandledrejection", onRejection); diff --git a/packages/keiko-ui/src/app/components/desktop/shellRecovery.ts b/packages/keiko-ui/src/app/components/desktop/shellRecovery.ts index 93e792190c..8df8381d38 100644 --- a/packages/keiko-ui/src/app/components/desktop/shellRecovery.ts +++ b/packages/keiko-ui/src/app/components/desktop/shellRecovery.ts @@ -12,7 +12,7 @@ import { type EditorM11SettingsSnapshot, } from "@oscharko-dev/keiko-contracts"; import { reportClientDiagnostic } from "@/lib/client-diagnostics"; -import { clientErrorSummary } from "@/lib/client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "@/lib/client-error-summary"; const RESET_SCOPES: readonly EditorM11SettingScope[] = ["user", "workspace"]; @@ -55,6 +55,7 @@ async function clearOverridesInScope( } catch (error) { reportClientDiagnostic( `shell-recovery: ${scope} layer reset failed (${clientErrorSummary(error)})`, + { correlationId: correlationIdOf(error) }, ); return null; } diff --git a/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.test.ts b/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.test.ts index c7a15b6c6a..0fc3dbac90 100644 --- a/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.test.ts +++ b/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.test.ts @@ -1,4 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { + resetClientDiagnosticWriter, + setClientDiagnosticWriter, +} from "../../../../../lib/client-diagnostics"; import { resetSharedEventSourcesForTests, sharedEventSourceGeneration, @@ -13,6 +17,9 @@ class FakeEventSource { onopen: (() => void) | null = null; onerror: (() => void) | null = null; closed = false; + // Real EventSource is CLOSED (2) by the time `onerror` typically fires for a fatal failure; tests + // that care about a different observed state override this before triggering onerror. + readyState = 2; constructor(url: string) { this.url = url; @@ -42,6 +49,7 @@ afterEach(() => { FakeEventSource.instances = []; FakeEventSource.immediateEventType = undefined; vi.unstubAllGlobals(); + resetClientDiagnosticWriter(); }); describe("subscribeSharedEventSource", () => { @@ -224,4 +232,30 @@ describe("subscribeSharedEventSource", () => { unsubscribeBackground(); unsubscribeEssential(); }); + + // Wave 5 of epic #3233 (g6): every EventSource.onerror handler reports a client diagnostic + // carrying the observed readyState and a closed reason label. + it("reports a client diagnostic with readyState and a reason label on stream error", () => { + vi.useFakeTimers(); + vi.stubGlobal("EventSource", FakeEventSource); + const reported: string[] = []; + setClientDiagnosticWriter((message) => reported.push(message)); + + const unsubscribe = subscribeSharedEventSource( + "/api/editor/debug/events?workspaceId=workspace-1", + ["editor-debug:output"], + () => {}, + ); + const first = FakeEventSource.instances[0]; + if (first === undefined) throw new Error("Expected stream."); + first.readyState = 0; + + first.onerror?.(); + + expect(reported).toEqual([ + "[keiko] shared-event-source sse stream error (kind=sse-error, readyState=0, reason=connecting)", + ]); + unsubscribe(); + vi.useRealTimers(); + }); }); diff --git a/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts b/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts index 19a7023f1c..1cbb265ea4 100644 --- a/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts +++ b/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts @@ -6,6 +6,27 @@ import { subscribeBrowserStreamCapacity, } from "../../../../../lib/browser-stream-capacity"; import { secureRandomInt } from "../../../../../lib/secure-random"; +import { reportClientDiagnostic } from "../../../../../lib/client-diagnostics"; + +// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, +// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from +// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's +// diagnostic transport as a side effect at import time (by design — see its own header), and none of +// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own +// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against +// all four call sites so the two ends cannot silently drift apart. +type SseStreamCloseReason = "connecting" | "closed" | "unknown"; + +function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { + if (readyState === 0) return "connecting"; + if (readyState === 2) return "closed"; + return "unknown"; +} + +function sseStreamErrorDiagnostic(readyState: number | undefined): string { + const readyStateText = readyState === undefined ? "unknown" : String(readyState); + return `[keiko] shared-event-source sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; +} type SharedEventListener = (event: MessageEvent) => void; @@ -140,6 +161,7 @@ function openEntrySource(entry: SharedEventSourceEntry): void { entry.reconnectAttempts = 0; }; source.onerror = () => { + reportClientDiagnostic(sseStreamErrorDiagnostic(source.readyState)); closeEntrySource(entry); scheduleReconnect(entry); }; diff --git a/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.test.tsx b/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.test.tsx index 767eb4f5db..c6558f8805 100644 --- a/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.test.tsx +++ b/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.test.tsx @@ -21,6 +21,10 @@ import { useRelationshipActivityStream, N_VISIBLE } from "./useRelationshipActiv import { RelationshipEdgeBadge, ACTIVITY_VISUALS } from "./RelationshipEdgeBadge"; import type { RelationshipActivityState } from "@oscharko-dev/keiko-contracts"; import { RELATIONSHIP_ACTIVITY_STATES } from "@oscharko-dev/keiko-contracts"; +import { + resetClientDiagnosticWriter, + setClientDiagnosticWriter, +} from "../../../../../lib/client-diagnostics"; expect.extend(toHaveNoViolations); @@ -36,6 +40,9 @@ class FakeEventSource { private readonly listeners: Map = new Map(); public onmessage: MessageHandler | null = null; public onerror: (() => void) | null = null; + // Real EventSource is CLOSED (2) by the time `onerror` typically fires for a fatal failure; + // tests that care about a different observed state override this before triggering onerror. + public readyState = 2; constructor(_url: string) { FakeEventSource.last = this; @@ -161,6 +168,7 @@ afterEach(() => { vi.unstubAllGlobals(); vi.clearAllTimers(); vi.useRealTimers(); + resetClientDiagnosticWriter(); }); // ─── Tests ───────────────────────────────────────────────────────────────────── @@ -524,6 +532,28 @@ describe("useRelationshipActivityStream", () => { expect(fetchSpy).not.toHaveBeenCalled(); }); + // Wave 5 of epic #3233 (g6): every EventSource.onerror handler reports a client diagnostic + // carrying the observed readyState and a closed reason label. + it("reports a client diagnostic with readyState and a reason label on stream error", () => { + vi.useFakeTimers(); + const reported: string[] = []; + setClientDiagnosticWriter((message) => reported.push(message)); + + render( undefined} />); + expect(FakeEventSource.last).not.toBeNull(); + const source = FakeEventSource.last; + if (source === null) throw new Error("Expected stream."); + source.readyState = 0; + + act(() => { + source.onerror?.(); + }); + + expect(reported).toEqual([ + "[keiko] relationship-activity sse stream error (kind=sse-error, readyState=0, reason=connecting)", + ]); + }); + it("pauses expiry and reconnect while the page is hidden, then resumes on visibility", async () => { vi.useFakeTimers(); let captured: ReturnType | null = null; diff --git a/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts b/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts index 083fb7a162..724bdcce3e 100644 --- a/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts +++ b/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts @@ -28,9 +28,30 @@ import { useCallback, useEffect, useRef, useState } from "react"; import type { RelationshipActivityState } from "@oscharko-dev/keiko-contracts"; import { RELATIONSHIP_FORBIDDEN_METADATA_KEY_SUBSTRINGS } from "@oscharko-dev/keiko-contracts"; +import { reportClientDiagnostic } from "../../../../../lib/client-diagnostics"; import { createSameOriginApiEventSource } from "../../../../../lib/safe-event-source"; import { secureRandomInt } from "../../../../../lib/secure-random"; +// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, +// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from +// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's +// diagnostic transport as a side effect at import time (by design — see its own header), and none of +// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own +// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against +// all four call sites so the two ends cannot silently drift apart. +type SseStreamCloseReason = "connecting" | "closed" | "unknown"; + +function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { + if (readyState === 0) return "connecting"; + if (readyState === 2) return "closed"; + return "unknown"; +} + +function sseStreamErrorDiagnostic(readyState: number | undefined): string { + const readyStateText = readyState === undefined ? "unknown" : String(readyState); + return `[keiko] relationship-activity sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; +} + // ─── Constants ───────────────────────────────────────────────────────────────── /** Max concurrent animated badges (activity-state.md §5.3). */ @@ -477,6 +498,7 @@ export function useRelationshipActivityStream( es.onerror = (): void => { if (closed) return; + reportClientDiagnostic(sseStreamErrorDiagnostic(es?.readyState)); closeStream(); scheduleReconnect(); }; diff --git a/packages/keiko-ui/src/app/components/desktop/windows/WindowBodyBoundary.tsx b/packages/keiko-ui/src/app/components/desktop/windows/WindowBodyBoundary.tsx index 6d838cfd93..159cbf30d4 100644 --- a/packages/keiko-ui/src/app/components/desktop/windows/WindowBodyBoundary.tsx +++ b/packages/keiko-ui/src/app/components/desktop/windows/WindowBodyBoundary.tsx @@ -16,7 +16,7 @@ import { Component, type ReactNode } from "react"; import { useTranslate, type I18nTranslate } from "@/lib/i18n"; import { reportClientDiagnostic } from "@/lib/client-diagnostics"; -import { clientErrorSummary } from "@/lib/client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "@/lib/client-error-summary"; interface WindowBodyBoundaryProps { // Window TYPE (registry key), not the user-supplied title: the type is the only value @@ -48,6 +48,7 @@ class InnerWindowBodyBoundary extends Component< // be diagnosable from the console, keyed by window type so it can be attributed. reportClientDiagnostic( `[keiko] window body crashed: ${this.props.windowType}: ${clientErrorSummary(error)}`, + { correlationId: correlationIdOf(error) }, ); } diff --git a/packages/keiko-ui/src/app/local-knowledge/capsule/error.tsx b/packages/keiko-ui/src/app/local-knowledge/capsule/error.tsx index 32a457c0b7..2a4cd84053 100644 --- a/packages/keiko-ui/src/app/local-knowledge/capsule/error.tsx +++ b/packages/keiko-ui/src/app/local-knowledge/capsule/error.tsx @@ -11,7 +11,7 @@ import { useEffect, type ReactNode } from "react"; import { useLocalKnowledgeTranslate as useTranslate } from "../local-knowledge-i18n"; -import { clientErrorSummary } from "@/lib/client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "@/lib/client-error-summary"; import { reportClientDiagnostic } from "@/lib/client-diagnostics"; export default function CapsuleDetailRouteError({ @@ -28,6 +28,7 @@ export default function CapsuleDetailRouteError({ // stay diagnosable from the console even though the UI only shows the recovery surface. reportClientDiagnostic( `[keiko] local-knowledge capsule route crashed: ${clientErrorSummary(error)}`, + { correlationId: correlationIdOf(error) }, ); }, [error]); diff --git a/packages/keiko-ui/src/lib/client-diagnostics.test.ts b/packages/keiko-ui/src/lib/client-diagnostics.test.ts index 43e2d8a700..61564220a8 100644 --- a/packages/keiko-ui/src/lib/client-diagnostics.test.ts +++ b/packages/keiko-ui/src/lib/client-diagnostics.test.ts @@ -6,8 +6,9 @@ import { reportClientDiagnostic, resetClientDiagnosticWriter, setClientDiagnosticWriter, + type ClientDiagnosticMeta, } from "./client-diagnostics"; -import { clientErrorSummary } from "./client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "./client-error-summary"; // The shared setup installs the application's console transport for every test, which is what the // rest of the suite should exercise. These cases are about the sink BEFORE any transport exists, so @@ -83,6 +84,65 @@ describe("reportClientDiagnostic", () => { expect(written).toEqual([]); }); + + // Wave 5 follow-up (epic #3233): the second argument is how a call site that holds an ApiError + // hands its correlation id to a transport, without folding it into the message string. + it("passes a supplied correlationId through to the installed writer", () => { + const received: (ClientDiagnosticMeta | undefined)[] = []; + setClientDiagnosticWriter((_message, meta) => received.push(meta)); + + reportClientDiagnostic("desktop shell crashed", { correlationId: "req-abc12345" }); + + expect(received).toEqual([{ correlationId: "req-abc12345" }]); + }); + + it("passes undefined meta through unchanged when the caller has no correlation id", () => { + const received: (ClientDiagnosticMeta | undefined)[] = []; + setClientDiagnosticWriter((_message, meta) => received.push(meta)); + + reportClientDiagnostic("workspace-state: local persistence parse failed"); + + expect(received).toEqual([undefined]); + }); + + // The pre-transport buffer must replay each record's OWN meta, not lose it or leak a later + // record's id onto an earlier one — the same in-order guarantee the plain-message buffering test + // above already covers for `message`. + it("replays buffered diagnostics with their original correlationId, each kept separate", () => { + reportClientDiagnostic("boot: first", { correlationId: "req-boot-000001" }); + reportClientDiagnostic("boot: second"); + reportClientDiagnostic("boot: third", { correlationId: "req-boot-000003" }); + + const received: (ClientDiagnosticMeta | undefined)[] = []; + setClientDiagnosticWriter((_message, meta) => received.push(meta)); + + expect(received).toEqual([ + { correlationId: "req-boot-000001" }, + undefined, + { correlationId: "req-boot-000003" }, + ]); + }); +}); + +describe("correlationIdOf", () => { + it("recovers a string correlationId from any error-shaped object that carries one", () => { + const apiErrorShaped = Object.assign(new Error("boom"), { correlationId: "req-xyz12345" }); + + expect(correlationIdOf(apiErrorShaped)).toBe("req-xyz12345"); + }); + + it("returns undefined for an error with no correlationId, or a non-error thrown value", () => { + expect(correlationIdOf(new Error("boom"))).toBeUndefined(); + expect(correlationIdOf("plain string throw")).toBeUndefined(); + expect(correlationIdOf(undefined)).toBeUndefined(); + expect(correlationIdOf(null)).toBeUndefined(); + }); + + it("ignores a non-string correlationId field rather than passing it through unchecked", () => { + const malformed = Object.assign(new Error("boom"), { correlationId: 12345 }); + + expect(correlationIdOf(malformed)).toBeUndefined(); + }); }); describe("clientErrorSummary", () => { diff --git a/packages/keiko-ui/src/lib/client-diagnostics.ts b/packages/keiko-ui/src/lib/client-diagnostics.ts index 5816a7aa31..172aaac120 100644 --- a/packages/keiko-ui/src/lib/client-diagnostics.ts +++ b/packages/keiko-ui/src/lib/client-diagnostics.ts @@ -19,18 +19,33 @@ // `unknown` or an `Error`: a raw error carries a stack with absolute paths and a message Keiko does // not control, and a diagnostic surface is what users screenshot into bug reports. Callers that hold // an error pass `clientErrorSummary(error)`, which yields its class and nothing else. +// +// The optional second argument is metadata ABOUT the report, not content: currently just the +// correlation id of the server request the diagnostic describes (see `correlationIdOf` in +// client-error-summary.ts), when the caller has one. It rides alongside `message` rather than being +// folded into it so a transport can use it structurally (e.g. as a real wire field) instead of every +// caller re-deriving a string convention a transport then has to parse back out. + +export interface ClientDiagnosticMeta { + readonly correlationId?: string | undefined; +} -export type ClientDiagnosticWriter = (message: string) => void; +export type ClientDiagnosticWriter = (message: string, meta?: ClientDiagnosticMeta) => void; + +interface PendingDiagnostic { + readonly message: string; + readonly meta?: ClientDiagnosticMeta | undefined; +} // Bounded on purpose: a failing poll loop can raise a diagnostic every tick while the BFF restarts, // and an unbounded pre-transport buffer would grow without limit in exactly that case. The oldest // records are dropped first — a storm's later entries describe the same fault as its first. const PENDING_LIMIT = 100; -const pending: string[] = []; +const pending: PendingDiagnostic[] = []; -function bufferUntilTransportArrives(message: string): void { - pending.push(message); +function bufferUntilTransportArrives(message: string, meta?: ClientDiagnosticMeta): void { + pending.push({ message, meta }); if (pending.length > PENDING_LIMIT) pending.shift(); } @@ -40,10 +55,13 @@ let writer: ClientDiagnosticWriter = bufferUntilTransportArrives; * Report a bounded, already-redacted operator diagnostic. * * `message` must contain only counts, statuses, closed identifiers and error classes — never a raw - * error, a file path, a URL with a query string, or anything the user typed. + * error, a file path, a URL with a query string, or anything the user typed. `meta.correlationId`, + * when supplied, must be the ORIGINAL failed request's id (e.g. a caught `ApiError`'s + * `.correlationId`) — never this report's own; a transport re-validates its shape independently + * before trusting it for anything (never assume a caller-supplied value is well-formed). */ -export function reportClientDiagnostic(message: string): void { - writer(message); +export function reportClientDiagnostic(message: string, meta?: ClientDiagnosticMeta): void { + writer(message, meta); } /** @@ -56,7 +74,7 @@ export function setClientDiagnosticWriter(next: ClientDiagnosticWriter): void { writer = next; if (pending.length === 0) return; const buffered = pending.splice(0, pending.length); - for (const message of buffered) next(message); + for (const record of buffered) next(record.message, record.meta); } /** Restore the buffering default and discard anything held. Tests use this; product code does not. */ diff --git a/packages/keiko-ui/src/lib/client-error-summary.ts b/packages/keiko-ui/src/lib/client-error-summary.ts index 013aa061f1..5426329ae1 100644 --- a/packages/keiko-ui/src/lib/client-error-summary.ts +++ b/packages/keiko-ui/src/lib/client-error-summary.ts @@ -26,3 +26,28 @@ export function clientErrorSummary(error: unknown): string { // A thrown non-Error still has a useful shape without quoting its content. return typeof error; } + +function hasStringCorrelationId(value: unknown): value is { correlationId: string } { + return ( + typeof value === "object" && + value !== null && + "correlationId" in value && + typeof (value as { correlationId?: unknown }).correlationId === "string" + ); +} + +/** + * The originating request's correlation id, when the caught error carries one — currently + * `ApiError` (api.ts), which `bffFetchJson` (http.ts) stamps on every non-2xx and every contract + * validation failure. Duck-typed rather than an `instanceof ApiError` check so any future error + * class that exposes the same field (e.g. a widened `StreamingUnavailableError`) is picked up here + * too, without this module importing api.ts. + * + * Undefined for a native thrown value, a boundary-caught render error with no id, or any failure + * that never went through `bffFetchJson` — most notably `EventSource.onerror`: the native + * EventSource API exposes no response headers to page script, so no producer downstream of one can + * ever recover a correlation id from it. + */ +export function correlationIdOf(error: unknown): string | undefined { + return hasStringCorrelationId(error) ? error.correlationId : undefined; +} diff --git a/packages/keiko-ui/src/lib/coding-workbench-event-retention.test.ts b/packages/keiko-ui/src/lib/coding-workbench-event-retention.test.ts index 39a16f0624..99cee6d675 100644 --- a/packages/keiko-ui/src/lib/coding-workbench-event-retention.test.ts +++ b/packages/keiko-ui/src/lib/coding-workbench-event-retention.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { CodingWorkbenchRuntimeSseEvent } from "@oscharko-dev/keiko-contracts"; +import { resetClientDiagnosticWriter, setClientDiagnosticWriter } from "./client-diagnostics"; import { CODING_WORKBENCH_EVENT_RETENTION_LIMIT, CODING_WORKBENCH_OBSERVATION_BATCH_MS, @@ -8,6 +9,10 @@ import { retainCodingWorkbenchRuntimeEvents, } from "./coding-workbench-event-retention"; +afterEach(() => { + resetClientDiagnosticWriter(); +}); + function event( sequence: number, overrides: Partial = {}, @@ -123,6 +128,27 @@ describe("Coding Workbench event retention", () => { expect(onError).not.toHaveBeenCalled(); }); + // Wave 5 of epic #3233 (g6): every EventSource.onerror handler reports a client diagnostic + // carrying the observed readyState and a closed reason label. + it("reports a client diagnostic with readyState and a reason label on stream error", () => { + const reported: string[] = []; + setClientDiagnosticWriter((message) => reported.push(message)); + const source = new FakeEventSource(); + source.readyState = 0; + const session = createCodingWorkbenchRuntimeStreamSession( + "run-1", + { onOpen: vi.fn(), onEvents: vi.fn(), onError: vi.fn(), onReset: vi.fn() }, + { createEventSource: () => source as unknown as EventSource }, + ); + + source.onerror?.(new Event("error")); + + expect(reported).toEqual([ + "[keiko] coding-workbench-runtime sse stream error (kind=sse-error, readyState=0, reason=connecting)", + ]); + session.close(); + }); + it("delivers observations before a later state event without bypassing the batch", () => { vi.useFakeTimers(); try { @@ -267,6 +293,9 @@ class FakeEventSource { public onopen: ((event: Event) => void) | null = null; public onerror: ((event: Event) => void) | null = null; public readonly close = vi.fn(); + // Real EventSource is CLOSED (2) by the time `onerror` typically fires for a fatal failure; + // tests that care about a different observed state override this before triggering onerror. + public readyState = 2; private readonly listeners = new Map(); public addEventListener(type: string, listener: EventListener): void { diff --git a/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts b/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts index e07b13728f..eef6ccafe6 100644 --- a/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts +++ b/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts @@ -10,11 +10,32 @@ import { parseCodingWorkbenchRuntimeEvent, } from "./coding-workbench-runtime-api"; import { reserveInteractiveBrowserStreamCapacity } from "./browser-stream-capacity"; +import { reportClientDiagnostic } from "./client-diagnostics"; export const CODING_WORKBENCH_EVENT_RETENTION_LIMIT = 500; export const CODING_WORKBENCH_OBSERVATION_BATCH_MS = 100; export const CODING_WORKBENCH_EVENT_STREAM_STALE_MS = 35_000; +// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, +// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from +// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's +// diagnostic transport as a side effect at import time (by design — see its own header), and none of +// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own +// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against +// all four call sites so the two ends cannot silently drift apart. +type SseStreamCloseReason = "connecting" | "closed" | "unknown"; + +function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { + if (readyState === 0) return "connecting"; + if (readyState === 2) return "closed"; + return "unknown"; +} + +function sseStreamErrorDiagnostic(readyState: number | undefined): string { + const readyStateText = readyState === undefined ? "unknown" : String(readyState); + return `[keiko] coding-workbench-runtime sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; +} + const TERMINAL_STATES = new Set([ "succeeded", "failed", @@ -124,6 +145,7 @@ class RuntimeEventStreamSession implements CodingWorkbenchRuntimeStreamSession { }; source.onerror = (): void => { if (this.source !== source) return; + reportClientDiagnostic(sseStreamErrorDiagnostic(source.readyState)); this.handlers.onError(new Error("The runtime event stream is reconnecting.")); }; const receive: EventListener = (event) => { diff --git a/packages/keiko-ui/src/lib/coding-workbench-runtime-effects.ts b/packages/keiko-ui/src/lib/coding-workbench-runtime-effects.ts index 5ef6cf1c96..d2af13c0ef 100644 --- a/packages/keiko-ui/src/lib/coding-workbench-runtime-effects.ts +++ b/packages/keiko-ui/src/lib/coding-workbench-runtime-effects.ts @@ -8,7 +8,7 @@ import type { CodingWorkbenchRuntimeState, CodingWorkbenchRuntimeStateAction, } from "./coding-workbench-live-state"; -import { clientErrorSummary } from "./client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "./client-error-summary"; import { reportClientDiagnostic } from "./client-diagnostics"; type RuntimeDispatch = Dispatch; @@ -73,6 +73,7 @@ export function useCodingWorkbenchPairingEffect(dispatch: RuntimeDispatch): void // `unknown`, while the underlying failure remains diagnosable. reportClientDiagnostic( `[keiko] coding workbench pairing discovery failed: ${clientErrorSummary(error)}`, + { correlationId: correlationIdOf(error) }, ); if (!cancelled) dispatch({ kind: "pairing-set", pairing: "unknown" }); }, diff --git a/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts b/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts index 44d94a064e..9c7d063806 100644 --- a/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts +++ b/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts @@ -3,15 +3,34 @@ // that it does — and that it does nothing else — lives where a reviewer looks for it. import { afterEach, describe, expect, it, vi } from "vitest"; -import { writeToBrowserConsole } from "./install-client-diagnostics"; +import { + clientDiagnosticPostFailureCount, + clientDiagnosticPostThrottledCount, + fanOutClientDiagnostic, + resetClientDiagnosticPostStateForTests, + writeToBrowserConsole, +} from "./install-client-diagnostics"; import { reportClientDiagnostic, resetClientDiagnosticWriter, setClientDiagnosticWriter, } from "./client-diagnostics"; +function jsonResponse(status = 204): Response { + return new Response(null, { status }); +} + +function lastPostedBody(fetchMock: ReturnType): Record { + const calls = fetchMock.mock.calls; + const lastCall = calls[calls.length - 1] as [string, RequestInit]; + return JSON.parse(lastCall[1].body as string) as Record; +} + afterEach(() => { resetClientDiagnosticWriter(); + resetClientDiagnosticPostStateForTests(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); }); describe("writeToBrowserConsole", () => { @@ -41,3 +60,167 @@ describe("writeToBrowserConsole", () => { consoleWarn.mockRestore(); }); }); + +// The second transport (Wave 5 of epic #3233, g6): a best-effort POST to +// `POST /api/diagnostics/client`, fanned out alongside the console so neither call site regresses +// when the other is added. +describe("fanOutClientDiagnostic", () => { + it("writes to the console and posts the same message to the server", async () => { + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + + fanOutClientDiagnostic("workspace-state: pull failed (network error)"); + + expect(consoleWarn).toHaveBeenCalledWith("workspace-state: pull failed (network error)"); + expect(fetchMock).toHaveBeenCalledOnce(); + const [path, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(path).toBe("/api/diagnostics/client"); + expect(init.method).toBe("POST"); + expect(init.keepalive).toBe(true); + const body = lastPostedBody(fetchMock); + expect(body["message"]).toBe("workspace-state: pull failed (network error)"); + expect(typeof body["clientTs"]).toBe("string"); + // A plain diagnostic never invented by this module carries no readyState/kind: only the four + // SSE call sites' exact convention (below) does. + expect(body).not.toHaveProperty("readyState"); + expect(body).not.toHaveProperty("kind"); + consoleWarn.mockRestore(); + }); + + it("recovers readyState and kind from the shared SSE onerror message convention", () => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + fanOutClientDiagnostic( + "[keiko] shared-event-source sse stream error (kind=sse-error, readyState=0, reason=connecting)", + ); + + const body = lastPostedBody(fetchMock); + expect(body["readyState"]).toBe(0); + expect(body["kind"]).toBe("sse-error"); + }); + + // Pins the exact convention each of the four SSE-consuming modules independently formats + // (sharedEventSource.ts, useSSE.ts, coding-workbench-event-retention.ts, + // useRelationshipActivityStream.ts) — the two ends are not import-linked (see this module's own + // header), so this is what catches the format drifting apart. + it.each([ + [ + "[keiko] shared-event-source sse stream error (kind=sse-error, readyState=2, reason=closed)", + 2, + ], + ["[keiko] run-events sse stream error (kind=sse-error, readyState=0, reason=connecting)", 0], + [ + "[keiko] coding-workbench-runtime sse stream error (kind=sse-error, readyState=1, reason=unknown)", + 1, + ], + [ + "[keiko] relationship-activity sse stream error (kind=sse-error, readyState=2, reason=closed)", + 2, + ], + ])("parses %s", (message, expectedReadyState) => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + fanOutClientDiagnostic(message); + + const body = lastPostedBody(fetchMock); + expect(body["readyState"]).toBe(expectedReadyState); + expect(body["kind"]).toBe("sse-error"); + }); + + it("counts a rejected POST without throwing back into the call site", async () => { + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new TypeError("network error"))); + + expect(() => fanOutClientDiagnostic("boot: gateway probe failed")).not.toThrow(); + + await vi.waitFor(() => { + expect(clientDiagnosticPostFailureCount()).toBe(1); + }); + }); + + it("drops the 21st POST within a rolling minute and counts it, without dropping the console write", () => { + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + + for (let index = 0; index < 21; index += 1) { + fanOutClientDiagnostic(`tick ${String(index)}`); + } + + expect(fetchMock).toHaveBeenCalledTimes(20); + expect(consoleWarn).toHaveBeenCalledTimes(21); + expect(clientDiagnosticPostThrottledCount()).toBe(1); + }); +}); + +// The fatal-flaw fix (Wave 5 follow-up, epic #3233): `correlationId` is what lets an agent join a +// browser diagnostic to the specific failed server request it describes. These cases pin +// `clientDiagnosticPostBody`'s shape validation (exercised only through the public +// `fanOutClientDiagnostic` entry point, matching every other case in this file). +describe("fanOutClientDiagnostic correlationId handling", () => { + it("puts a shape-valid correlationId on the wire body when the caller supplies one", () => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + fanOutClientDiagnostic("[keiko] app shell crashed: TypeError", { + correlationId: "original-request-id-01", + }); + + const body = lastPostedBody(fetchMock); + expect(body["correlationId"]).toBe("original-request-id-01"); + }); + + it("omits correlationId from the wire body when the caller supplies none", () => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + fanOutClientDiagnostic("[keiko] app shell crashed: TypeError"); + + expect(lastPostedBody(fetchMock)).not.toHaveProperty("correlationId"); + }); + + // Out-of-shape ids are dropped silently — never thrown, and never sent — rather than trusted + // as-is: this file is upstream of the server's OWN independent re-validation + // (client-diagnostics-routes.ts), so a malformed id here would just be redundant, not unsafe: this + // is defense in depth on the sending side, catching the mistake as close to its source as + // possible. + it.each([ + ["too short (7 chars, one under the 8-char floor)", "a".repeat(7)], + ["too long (129 chars, one over the 128-char ceiling)", "a".repeat(129)], + ["contains a raw CRLF", "abcdef\r\nghij"], + ["contains a space", "not a valid id"], + ["contains a disallowed symbol", "req-id-!!!"], + ["is empty", ""], + ])("drops an out-of-shape correlationId: %s", (_label, correlationId) => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + fanOutClientDiagnostic("[keiko] app shell crashed: TypeError", { correlationId }); + + expect(lastPostedBody(fetchMock)).not.toHaveProperty("correlationId"); + }); + + it("still carries readyState/kind from the SSE convention alongside a valid correlationId", () => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse()); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + + fanOutClientDiagnostic( + "[keiko] shared-event-source sse stream error (kind=sse-error, readyState=0, reason=connecting)", + { correlationId: "sse-req-0000001" }, + ); + + const body = lastPostedBody(fetchMock); + expect(body["correlationId"]).toBe("sse-req-0000001"); + expect(body["readyState"]).toBe(0); + expect(body["kind"]).toBe("sse-error"); + }); +}); diff --git a/packages/keiko-ui/src/lib/install-client-diagnostics.ts b/packages/keiko-ui/src/lib/install-client-diagnostics.ts index 8bb26ddd99..f5039b5a5d 100644 --- a/packages/keiko-ui/src/lib/install-client-diagnostics.ts +++ b/packages/keiko-ui/src/lib/install-client-diagnostics.ts @@ -1,22 +1,47 @@ "use client"; -// Where THIS application delivers client diagnostics (0.3.0 release audit, #2802). +// Where THIS application delivers client diagnostics (0.3.0 release audit, #2802; second +// transport added Wave 5 of epic #3233 / ADR-0173, g6). // // `client-diagnostics.ts` owns the contract — what a diagnostic is, and that it is already redacted. // It deliberately owns no transport, so that the one place a browser console is written to is a // module whose whole purpose is choosing the transport, rather than a line buried in the library // every call site imports. // -// The console is the right destination for this product today: Keiko is local-first, there is no -// client->server diagnostics ingest, and adding one would mean a new subsystem and a new trust -// boundary for client-supplied text (AGENTS.md §5). Swapping it later — an operator panel, a support -// bundle, an opted-in endpoint — is an edit to this file and nothing else. -// // Importing this module installs the transport as a side effect, at module scope rather than in an // effect, so diagnostics raised during hydration or an early boot crash are delivered too. Whatever // the sink buffered before this point is flushed by `setClientDiagnosticWriter`. +// +// FAN-OUT, NOT A REPLACEMENT: the console remains the first transport (a developer watching devtools +// still sees every diagnostic even when the network call below is slow, throttled, or fails) and a +// best-effort POST to `POST /api/diagnostics/client` (packages/keiko-server/src/ +// client-diagnostics-routes.ts) is added alongside it, so the same already-redacted, already-bounded +// string also reaches the server's activity log — the machine-reconstruction surface the rest of +// epic #3233 builds. The wire contract (`packages/keiko-contracts/src/diagnostics.ts`) treats the +// browser as untrusted input regardless of what this module sends; nothing here is a second place +// that does redaction, it only forwards the SAME string `reportClientDiagnostic` callers already +// bounded and redacted by convention. +// +// `correlationId` (Wave 5, wired in a follow-up to this wave): `client-error-summary.ts`'s +// `correlationIdOf(error)` recovers the originating request's id from any caught `ApiError` (the +// class `bffFetchJson`, http.ts, stamps a `.correlationId` on for every non-2xx and every contract +// validation failure — api.ts ~line 195, http.ts ~lines 126-129/148-149). Every `reportClientDiagnostic` +// call site that catches such an error passes it through `meta.correlationId`, and +// `clientDiagnosticPostBody` below puts it on the wire once it re-validates the shape client-side +// (defense in depth — the server, `client-diagnostics-routes.ts`, re-validates it again +// independently before trusting it for anything). It stays genuinely absent for the four SSE +// `onerror` call sites (sharedEventSource.ts, useSSE.ts, coding-workbench-event-retention.ts, +// useRelationshipActivityStream.ts): the native `EventSource` API exposes no response headers to +// page script, so there is no id to recover at that call site, ever — not a gap, a hard platform +// limit. -import { setClientDiagnosticWriter } from "./client-diagnostics"; +import { + CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH, + type ClientDiagnosticIngestRequest, + type ClientDiagnosticReadyState, +} from "@oscharko-dev/keiko-contracts"; +import { type ClientDiagnosticMeta, setClientDiagnosticWriter } from "./client-diagnostics"; +import { bffFetchJson } from "./http"; function writeToBrowserConsole(message: string): void { // The single sanctioned console access in keiko-ui production code. Everything above this line is @@ -25,6 +50,141 @@ function writeToBrowserConsole(message: string): void { if (typeof console !== "undefined" && typeof console.warn === "function") console.warn(message); } -setClientDiagnosticWriter(writeToBrowserConsole); +// Mirrors the server's SAFE_CORRELATION_ID predicate (packages/keiko-server/src/correlation.ts). +// keiko-ui may only depend on the server through the shared contract types (AGENTS.md §4), never on +// a server module directly, so this file re-derives the same alphabet+length shape as its own, +// client-side copy rather than importing one — the same layering `diagnostics.ts` (keiko-contracts) +// documents for its own, deliberately looser, wire-shape guard. +const CLIENT_CORRELATION_ID_PATTERN = /^[A-Za-z0-9._-]{8,128}$/; + +// Drops silently (never throws) on a caller-supplied id that is not shape-valid: a malformed id is +// still a successful diagnostic report, just without a join key attached. +function validCorrelationId(correlationId: string | undefined): string | undefined { + return correlationId !== undefined && CLIENT_CORRELATION_ID_PATTERN.test(correlationId) + ? correlationId + : undefined; +} + +// The four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, +// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) each format their +// `EventSource.onerror` diagnostic with this EXACT substring so this module can recover the +// structured `readyState`/`kind` wire fields from the plain string `reportClientDiagnostic` takes — +// without those four modules importing this one (which would pull its module-scope +// `setClientDiagnosticWriter` side effect into their own unit tests). The four call sites duplicate +// a four-line pure formatter rather than share this module for exactly that reason; this pattern is +// the other half of that contract and is pinned against all four by this file's own test. +const SSE_DIAGNOSTIC_MESSAGE_PATTERN = + /kind=sse-error, readyState=([0-2]), reason=(?:connecting|closed|unknown)/; + +function parsedSseReadyState(digit: string): ClientDiagnosticReadyState { + if (digit === "0") return 0; + if (digit === "2") return 2; + return 1; +} + +// Builds the wire body for one already-bounded diagnostic message. `clientTs` is stamped at send +// time (not at the original `reportClientDiagnostic` call), which is close enough for an operator +// diagnostic and avoids threading a timestamp through the sink's string-only contract. +// `exactOptionalPropertyTypes` is honoured because `ClientDiagnosticIngestRequest`'s optional fields +// are all typed `T | undefined` (keiko-contracts), so assigning `undefined` outright is legal — and +// `JSON.stringify` drops an `undefined`-valued key from the wire body regardless, so an absent +// correlation id never reaches the request at all. +function clientDiagnosticPostBody( + message: string, + correlationId: string | undefined, +): ClientDiagnosticIngestRequest { + const bounded = + message.length > CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH + ? message.slice(0, CLIENT_DIAGNOSTIC_MESSAGE_MAX_LENGTH) + : message; + const clientTs = new Date().toISOString(); + const validId = validCorrelationId(correlationId); + const sseMatch = SSE_DIAGNOSTIC_MESSAGE_PATTERN.exec(message); + if (sseMatch === null) return { message: bounded, clientTs, correlationId: validId }; + const readyStateDigit = sseMatch[1]; + if (readyStateDigit === undefined) return { message: bounded, clientTs, correlationId: validId }; + return { + message: bounded, + clientTs, + correlationId: validId, + readyState: parsedSseReadyState(readyStateDigit), + kind: "sse-error", + }; +} + +// Process-wide (module-scope), not per-diagnostic: a flapping stream or a hostile page must not be +// able to grow the activity log without bound. The server independently rate-limits the same route +// (client-diagnostics-routes.ts); this is defense in depth on the sending side, so a burst never +// leaves the tab at all. +const CLIENT_DIAGNOSTIC_POST_LIMIT_PER_WINDOW = 20; +const CLIENT_DIAGNOSTIC_POST_WINDOW_MS = 60_000; + +let postWindowStartedAtMs = 0; +let postCountInWindow = 0; +let postFailureCount = 0; +let postThrottledCount = 0; + +function admittedByClientPostRateLimit(nowMs: number): boolean { + if (nowMs - postWindowStartedAtMs >= CLIENT_DIAGNOSTIC_POST_WINDOW_MS) { + postWindowStartedAtMs = nowMs; + postCountInWindow = 0; + } + if (postCountInWindow >= CLIENT_DIAGNOSTIC_POST_LIMIT_PER_WINDOW) return false; + postCountInWindow += 1; + return true; +} + +/** Test-only: number of best-effort diagnostic POSTs that failed (network error or non-2xx). */ +export function clientDiagnosticPostFailureCount(): number { + return postFailureCount; +} + +/** Test-only: number of diagnostic POSTs dropped by the client-side rate limit. */ +export function clientDiagnosticPostThrottledCount(): number { + return postThrottledCount; +} + +/** Test-only: put the POST transport's rate limiter and failure/drop counters back to a clean start. */ +export function resetClientDiagnosticPostStateForTests(): void { + postWindowStartedAtMs = 0; + postCountInWindow = 0; + postFailureCount = 0; + postThrottledCount = 0; +} + +// Best-effort POST to the server activity log. Never awaited by a call site and never lets a +// rejected fetch (or a non-2xx `ApiError` `bffFetchJson` throws) reach back into +// `reportClientDiagnostic`'s caller — the same best-effort discipline `writeToBrowserConsole` +// already has, just with a `.catch` standing in for that function's `typeof` guards. A failure is +// counted, never logged to console (AGENTS.md §6: this module is the one sanctioned console site, +// and re-entering it from a diagnostics-transport failure risks a loop under a flapping connection). +function postClientDiagnosticToServer(message: string, meta?: ClientDiagnosticMeta): void { + if (!admittedByClientPostRateLimit(Date.now())) { + postThrottledCount += 1; + return; + } + try { + const body = clientDiagnosticPostBody(message, meta?.correlationId); + void bffFetchJson("/api/diagnostics/client", { + method: "POST", + body: JSON.stringify(body), + keepalive: true, + }).catch(() => { + postFailureCount += 1; + }); + } catch { + postFailureCount += 1; + } +} + +// The fan-out composite: every diagnostic reaches both transports. Console first, so a developer +// watching devtools sees it even when the POST below is throttled or fails. `meta` only ever +// affects the POST body — the console transport stays the plain, undecorated message it always was. +function fanOutClientDiagnostic(message: string, meta?: ClientDiagnosticMeta): void { + writeToBrowserConsole(message); + postClientDiagnosticToServer(message, meta); +} + +setClientDiagnosticWriter(fanOutClientDiagnostic); -export { writeToBrowserConsole }; +export { writeToBrowserConsole, fanOutClientDiagnostic }; diff --git a/packages/keiko-ui/src/lib/useCodingWorkbenchApprovalReview.ts b/packages/keiko-ui/src/lib/useCodingWorkbenchApprovalReview.ts index 12568af29d..4ff8f8595d 100644 --- a/packages/keiko-ui/src/lib/useCodingWorkbenchApprovalReview.ts +++ b/packages/keiko-ui/src/lib/useCodingWorkbenchApprovalReview.ts @@ -5,7 +5,7 @@ import type { CodingWorkbenchRuntimePendingApprovalReview } from "@oscharko-dev/ import { codingAppSessionPairingSettled } from "./coding-app-session-client"; import { getCodingWorkbenchRuntimeApprovalReview } from "./coding-workbench-runtime-api"; -import { clientErrorSummary } from "./client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "./client-error-summary"; import { reportClientDiagnostic } from "./client-diagnostics"; export type CodingWorkbenchApprovalReviewStatus = "idle" | "loading" | "ready" | "unavailable"; @@ -76,6 +76,7 @@ function startApprovalReviewSync( // content-free "unavailable", but the underlying refresh failure remains diagnosable. reportClientDiagnostic( `[keiko] approval review channel refresh failed: ${clientErrorSummary(error)}`, + { correlationId: correlationIdOf(error) }, ); publish(scopeState(runId, permissionRequestId, UNAVAILABLE)); } diff --git a/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.test.tsx b/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.test.tsx index 883609ce81..30125f2684 100644 --- a/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.test.tsx +++ b/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.test.tsx @@ -9,6 +9,12 @@ import { useCodingWorkbenchResearch, type UseCodingWorkbenchResearchInput, } from "./useCodingWorkbenchResearch"; +import { ApiError } from "./api"; +import { resetClientDiagnosticWriter, setClientDiagnosticWriter } from "./client-diagnostics"; +import { + fanOutClientDiagnostic, + resetClientDiagnosticPostStateForTests, +} from "./install-client-diagnostics"; const getResearchMock = vi.hoisted(() => vi.fn()); const pairingSettledMock = vi.hoisted(() => vi.fn()); @@ -126,6 +132,43 @@ describe("useCodingWorkbenchResearch", () => { expect(result.current).toEqual({ status: "unavailable", ask: null, grant: null }); }); + // Wave 5 follow-up (epic #3233) — the fatal-flaw fix: a real caught `ApiError` must carry its + // correlation id all the way onto the `POST /api/diagnostics/client` wire body, through + // `reportClientDiagnostic`'s structured second argument and `install-client-diagnostics.ts`'s + // transport, not just through a mock standing in for either. Installing the REAL transport (not a + // spy on `reportClientDiagnostic`) is what makes this end-to-end: it fails if the wiring line in + // `useCodingWorkbenchResearch.ts`'s catch block is ever removed, same as it would fail if + // `clientDiagnosticPostBody` stopped forwarding a valid id. + describe("correlationId wiring to the diagnostic ingest POST", () => { + afterEach(() => { + resetClientDiagnosticWriter(); + resetClientDiagnosticPostStateForTests(); + vi.unstubAllGlobals(); + }); + + it("carries the failed request's correlationId onto the diagnostic ingest POST body", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal("fetch", fetchMock); + setClientDiagnosticWriter(fanOutClientDiagnostic); + const failure = new ApiError("INTERNAL", "research channel unavailable", 502); + failure.correlationId = "req-research-000001"; + getResearchMock.mockRejectedValue(failure); + + const { result } = renderHook(() => useCodingWorkbenchResearch(RUN)); + + await waitFor(() => { + expect(result.current.status).toBe("unavailable"); + }); + await waitFor(() => { + expect(fetchMock).toHaveBeenCalled(); + }); + const [path, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(path).toBe("/api/diagnostics/client"); + const body = JSON.parse(init.body as string) as Record; + expect(body["correlationId"]).toBe("req-research-000001"); + }); + }); + it("re-reads when a new ask or runtime revision replaces current research truth", async () => { getResearchMock.mockResolvedValue(active()); diff --git a/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.ts b/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.ts index 33f9e24b80..e4ea5efb37 100644 --- a/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.ts +++ b/packages/keiko-ui/src/lib/useCodingWorkbenchResearch.ts @@ -10,7 +10,7 @@ import type { import { codingAppSessionPairingSettled } from "./coding-app-session-client"; import { getCodingWorkbenchRuntimeResearch } from "./coding-workbench-runtime-api"; import { reportClientDiagnostic } from "./client-diagnostics"; -import { clientErrorSummary } from "./client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "./client-error-summary"; export type CodingWorkbenchResearchStatus = "idle" | "loading" | "ready" | "unavailable"; @@ -89,6 +89,7 @@ function startResearchSync( // content-free "unavailable", but the underlying refresh failure remains diagnosable. reportClientDiagnostic( `[keiko] research channel refresh failed: ${clientErrorSummary(error)}`, + { correlationId: correlationIdOf(error) }, ); publish( scopeResearchStateFromInput(input, { diff --git a/packages/keiko-ui/src/lib/useSSE.test.tsx b/packages/keiko-ui/src/lib/useSSE.test.tsx index a31ea0b3b4..a97ff80a0e 100644 --- a/packages/keiko-ui/src/lib/useSSE.test.tsx +++ b/packages/keiko-ui/src/lib/useSSE.test.tsx @@ -1,5 +1,6 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { resetClientDiagnosticWriter, setClientDiagnosticWriter } from "./client-diagnostics"; import { useSSE } from "./useSSE"; class FakeEventSource { @@ -10,6 +11,9 @@ class FakeEventSource { onerror: ((event: Event) => void) | null = null; onmessage: ((event: MessageEvent) => void) | null = null; readonly close = vi.fn(); + // Real EventSource is CLOSED (2) by the time `onerror` typically fires for a fatal failure; + // tests that care about a different observed state override this before triggering onerror. + readyState = 2; private readonly listeners = new Map(); constructor(url: string) { @@ -61,6 +65,7 @@ describe("useSSE", () => { vi.unstubAllGlobals(); vi.restoreAllMocks(); vi.useRealTimers(); + resetClientDiagnosticWriter(); }); it("opens encoded run streams, recovers after transient errors, ignores malformed frames, and closes on terminal events", async () => { @@ -225,4 +230,27 @@ describe("useSSE", () => { expect(view.result.current.events[0]?.seq).toBe(20); expect(view.result.current.events[499]?.seq).toBe(519); }); + + // Wave 5 of epic #3233 (g6): every EventSource.onerror handler reports a client diagnostic + // carrying the observed readyState and a closed reason label. + it("reports a client diagnostic with readyState and a reason label on stream error", () => { + vi.stubGlobal("EventSource", FakeEventSource); + const reported: string[] = []; + setClientDiagnosticWriter((message) => reported.push(message)); + const view = renderHook(({ runId }: { runId: string | null }) => useSSE(runId), { + initialProps: { runId: "run 1" }, + }); + const source = FakeEventSource.instances[0]; + if (source === undefined) throw new Error("Expected stream."); + source.readyState = 2; + + act(() => { + source.onerror?.(new Event("error")); + }); + + expect(reported).toEqual([ + "[keiko] run-events sse stream error (kind=sse-error, readyState=2, reason=closed)", + ]); + view.unmount(); + }); }); diff --git a/packages/keiko-ui/src/lib/useSSE.ts b/packages/keiko-ui/src/lib/useSSE.ts index 3ea1a27996..f42389bde4 100644 --- a/packages/keiko-ui/src/lib/useSSE.ts +++ b/packages/keiko-ui/src/lib/useSSE.ts @@ -6,6 +6,7 @@ */ import { useEffect, useRef, useState } from "react"; +import { reportClientDiagnostic } from "./client-diagnostics"; import { createSameOriginApiEventSource } from "./safe-event-source"; import { secureRandomInt } from "./secure-random"; import { TERMINAL_EVENT_TYPES, type HarnessEvent, type SseStatus } from "./types"; @@ -15,6 +16,26 @@ const RECONNECT_INITIAL_DELAY_MS = 1000; const RECONNECT_MAX_DELAY_MS = 30000; const RUN_EVENTS_URL = "/api/runs/events"; +// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, +// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from +// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's +// diagnostic transport as a side effect at import time (by design — see its own header), and none of +// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own +// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against +// all four call sites so the two ends cannot silently drift apart. +type SseStreamCloseReason = "connecting" | "closed" | "unknown"; + +function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { + if (readyState === 0) return "connecting"; + if (readyState === 2) return "closed"; + return "unknown"; +} + +function sseStreamErrorDiagnostic(readyState: number | undefined): string { + const readyStateText = readyState === undefined ? "unknown" : String(readyState); + return `[keiko] run-events sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; +} + export interface UseSSEResult { events: HarnessEvent[]; status: SseStatus; @@ -139,6 +160,7 @@ function openSharedEventSource(): void { }); sharedEventSource.onerror = () => { + reportClientDiagnostic(sseStreamErrorDiagnostic(sharedEventSource?.readyState)); notifyAll("error", "Stream disconnected. Attempting to reconnect…"); closeSharedEventSource(); scheduleReconnect(); diff --git a/packages/keiko-ui/src/lib/verified-task-workspace-binding.ts b/packages/keiko-ui/src/lib/verified-task-workspace-binding.ts index 775b3c3320..1a42e81057 100644 --- a/packages/keiko-ui/src/lib/verified-task-workspace-binding.ts +++ b/packages/keiko-ui/src/lib/verified-task-workspace-binding.ts @@ -13,7 +13,7 @@ import { type ActiveWorkspaceView, } from "./task-workspace-api"; import { isWorkspaceFailureClass, type WorkspaceFailureClass } from "@oscharko-dev/keiko-contracts"; -import { clientErrorSummary } from "./client-error-summary"; +import { clientErrorSummary, correlationIdOf } from "./client-error-summary"; import { reportClientDiagnostic } from "./client-diagnostics"; export type VerifiedTaskWorkspaceBindFailureReason = "branch-conflict"; @@ -41,6 +41,9 @@ export interface VerifiedTaskWorkspaceBindInput { function warnBindStage(stage: string, error: unknown): void { reportClientDiagnostic( `[keiko] task workspace bind ${stage} failed: ${clientErrorSummary(error)}`, + { + correlationId: correlationIdOf(error), + }, ); } From d7ae2193947cd90c530bc6099f4c72295656a097 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 10:56:35 +0200 Subject: [PATCH 04/19] =?UTF-8?q?fix(observability):=20Wave=204a=20audit?= =?UTF-8?q?=20repairs=20=E2=80=94=20read-only=20vault=20key=20resolution?= =?UTF-8?q?=20for=20fingerprints,=20busy=5Ftimeout=20on=20read-only=20open?= =?UTF-8?q?s=20(#3239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - collectStoreFingerprints resolves the vault key through the new resolveVaultKeyReadOnly (env -> read-only keychain find -> undefined): a support export can no longer mint a keyfile or store a keychain secret. - The three read-only opens set their store's busy_timeout so a fingerprint under WAL contention waits instead of reporting open-failed. - keiko support export prints a progress line before fingerprint collection. - Op catalog regenerated. Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 2 +- packages/keiko-cli/src/support.test.ts | 20 +++++++++ packages/keiko-cli/src/support.ts | 11 +++++ .../keiko-local-knowledge/src/store.test.ts | 18 ++++++++ packages/keiko-local-knowledge/src/store.ts | 8 +++- .../keiko-memory-vault/src/cipher.test.ts | 42 ++++++++++++++++++- packages/keiko-memory-vault/src/cipher.ts | 37 ++++++++++++++++ packages/keiko-memory-vault/src/db.test.ts | 23 ++++++++++ packages/keiko-memory-vault/src/db.ts | 16 ++++++- packages/keiko-memory-vault/src/index.ts | 2 + .../src/store-fingerprints.test.ts | 25 +++++++++++ .../keiko-server/src/store-fingerprints.ts | 8 +++- packages/keiko-server/src/store/db.test.ts | 21 ++++++++++ packages/keiko-server/src/store/db.ts | 11 ++++- 14 files changed, 236 insertions(+), 8 deletions(-) diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index 483f001454..e36db271ac 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -341,7 +341,7 @@ { "op": "memory-vault.store.quarantined", "category": "diagnostic", - "site": "packages/keiko-memory-vault/src/db.ts:121", + "site": "packages/keiko-memory-vault/src/db.ts:127", "package": "keiko-memory-vault" }, { diff --git a/packages/keiko-cli/src/support.test.ts b/packages/keiko-cli/src/support.test.ts index 0bcfb52f7d..3467e4043e 100644 --- a/packages/keiko-cli/src/support.test.ts +++ b/packages/keiko-cli/src/support.test.ts @@ -493,6 +493,26 @@ describe("runSupportCli export", () => { expect.not.arrayContaining([{ store: "ui", reasonKind: "open-failed" }]), ); }); + + // Finding 1 (minor): store fingerprint collection runs a synchronous full-DB quick_check plus + // per-table row counts with nothing printed while it runs, so a slow run against a large + // local-knowledge index looks hung to the operator. A stderr progress line before the call + // fixes that. RED (before fix): no such line was ever written to stderr. + it("prints a stderr progress line before computing store fingerprints", async () => { + const c = makeIo(); + const outPath = join(outDir, "progress-line.jsonl"); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", outPath], + c.io, + AUDIT_ENV, + { auditDeps: healthyAuditDeps(), evidenceStore: createInMemoryEvidenceStore() }, + ); + + expect(code).toBe(0); + expect(c.err()).toContain( + "keiko support export: computing store fingerprints (may take a while on a large local-knowledge index)...", + ); + }); }); describe("runSupportCli analyze", () => { diff --git a/packages/keiko-cli/src/support.ts b/packages/keiko-cli/src/support.ts index 6993ad8ff2..a85bf79949 100644 --- a/packages/keiko-cli/src/support.ts +++ b/packages/keiko-cli/src/support.ts @@ -268,6 +268,16 @@ function writeBundleOrExitCode(outPath: string, contents: string, io: CliIo): nu } } +// Finding 1 (minor): store fingerprint collection runs a synchronous full-DB `quick_check` plus +// a row count per table for each store, which can take a while against a large local-knowledge +// index with nothing printed while it runs. This progress line keeps a slow run from looking +// hung, without changing the collection's synchronous, untimed behavior itself. +function reportStoreFingerprintProgress(io: CliIo): void { + io.err( + "keiko support export: computing store fingerprints (may take a while on a large local-knowledge index)...\n", + ); +} + function reportAuditFailure(error: unknown, io: CliIo): number { if (error instanceof AuditLoadError) { io.err( @@ -312,6 +322,7 @@ async function runSupportExport( // Deferred until after the audit's own fail-closed check: opening three real stores is real // I/O, wasted if the export is about to be refused anyway. + reportStoreFingerprintProgress(io); const storeFingerprintCollection = await server.collectStoreFingerprints({ stateDir, env }); const generatedAtDate = now(); const manifest = buildSupportBundleManifest({ diff --git a/packages/keiko-local-knowledge/src/store.test.ts b/packages/keiko-local-knowledge/src/store.test.ts index 4c3e255139..0294405e0a 100644 --- a/packages/keiko-local-knowledge/src/store.test.ts +++ b/packages/keiko-local-knowledge/src/store.test.ts @@ -27,6 +27,7 @@ import { computeStoreFingerprint, LK_STORE_BUSY_TIMEOUT_MS, openKnowledgeStore, + openKnowledgeStoreReadOnly, type KnowledgeStoreKeyProvider, } from "./store.js"; import { STORE_CONTENT_ENCRYPTION_TEST_CONSTANTS } from "./store-content-encryption.js"; @@ -943,3 +944,20 @@ describe("computeStoreFingerprint", () => { } }); }); + +describe("openKnowledgeStoreReadOnly (Finding 2 — busy_timeout on the read-only diagnostic open)", () => { + it("sets the active PRAGMA busy_timeout to LK_STORE_BUSY_TIMEOUT_MS, not node:sqlite's default of 0", () => { + const dbPath = join(tmp, "capsules.db"); + openKnowledgeStore({ dbPath }).close(); + + const db = openKnowledgeStoreReadOnly(dbPath); + try { + const rows = db.prepare("PRAGMA busy_timeout").all() as unknown as readonly { + timeout: number; + }[]; + expect(rows[0]?.timeout).toBe(LK_STORE_BUSY_TIMEOUT_MS); + } finally { + db.close(); + } + }); +}); diff --git a/packages/keiko-local-knowledge/src/store.ts b/packages/keiko-local-knowledge/src/store.ts index e32799cefa..2920671306 100644 --- a/packages/keiko-local-knowledge/src/store.ts +++ b/packages/keiko-local-knowledge/src/store.ts @@ -545,7 +545,13 @@ export function openKnowledgeStore(opts: OpenKnowledgeStoreOptions): KnowledgeSt // a store. `computeStoreFingerprint` below needs only `PRAGMA user_version`/`quick_check` and fixed // `SELECT COUNT(*)` reads, none of which need write access. export function openKnowledgeStoreReadOnly(dbPath: string): DatabaseSync { - return new DatabaseSync(dbPath, { readOnly: true }); + const db = new DatabaseSync(dbPath, { readOnly: true }); + // A short busy_timeout (Finding 2) so a reader opened with no wait bound does not spuriously + // report the store `open-failed` on an immediate SQLITE_BUSY from a concurrent WAL checkpoint + // or schema-changing transaction on a live production server. Connection-local PRAGMA: no write, + // does not throw on a `readOnly: true` handle, so the read-only guarantee above is unaffected. + db.exec(`PRAGMA busy_timeout = ${String(LK_STORE_BUSY_TIMEOUT_MS)}`); + return db; } // Reuses the SAME source list and comparison `runMigrations` already applies (`version` vs the diff --git a/packages/keiko-memory-vault/src/cipher.test.ts b/packages/keiko-memory-vault/src/cipher.test.ts index 323a36113d..875341d86d 100644 --- a/packages/keiko-memory-vault/src/cipher.test.ts +++ b/packages/keiko-memory-vault/src/cipher.test.ts @@ -1,5 +1,13 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { chmodSync, mkdtempSync, rmSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { + chmodSync, + mkdtempSync, + readdirSync, + rmSync, + readFileSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { randomBytes } from "node:crypto"; @@ -8,6 +16,7 @@ import { keyFromKeychain, NO_KEYCHAIN, resolveVaultKey, + resolveVaultKeyReadOnly, } from "./cipher.js"; import { MemoryStorageError } from "./errors.js"; @@ -168,6 +177,37 @@ describe("resolveVaultKey — keychain tier", () => { }); }); +describe("resolveVaultKeyReadOnly (Finding 0 — read-only diagnostic export seam)", () => { + it("never writes a keyfile when the env and keychain tiers both miss, and reports no key", () => { + const resolved = resolveVaultKeyReadOnly({}, () => undefined); + expect(resolved.key).toBeUndefined(); + expect(resolved.source).toBeUndefined(); + // The RED assertion: `resolveVaultKey({}, dir, NO_KEYCHAIN)` against this same empty dir + // would call `keyFromKeyfile`, which mints and writes `vault.key`. The read-only resolver + // must leave the directory exactly as it found it — no file of any name appears. + expect(readdirSync(dir)).toEqual([]); + }); + + it("still honors the env tier, and never touches the keychain when the env key is present", () => { + const raw = randomBytes(32); + let keychainCalls = 0; + const resolved = resolveVaultKeyReadOnly({ KEIKO_MEMORY_KEY: raw.toString("base64") }, () => { + keychainCalls += 1; + return undefined; + }); + expect(resolved.source).toBe("env"); + expect(resolved.key?.equals(raw)).toBe(true); + expect(keychainCalls).toBe(0); + }); + + it("reports the keychain tier when a read-only lookup finds a stored key, without minting one", () => { + const stored = randomBytes(32); + const resolved = resolveVaultKeyReadOnly({}, () => stored); + expect(resolved.source).toBe("keychain"); + expect(resolved.key?.equals(stored)).toBe(true); + }); +}); + describe("createMemoryContentCipher", () => { it("seals and opens a string bound to the resolved key", () => { const cipher = createMemoryContentCipher(randomBytes(32)); diff --git a/packages/keiko-memory-vault/src/cipher.ts b/packages/keiko-memory-vault/src/cipher.ts index d94d538379..3e4879c27c 100644 --- a/packages/keiko-memory-vault/src/cipher.ts +++ b/packages/keiko-memory-vault/src/cipher.ts @@ -96,6 +96,21 @@ function generateKeychainKey(account: string, options: MacosKeychainOptions): Bu : undefined; } +// Read-only keychain lookup for the diagnostic export seam (Wave 4a, epic #3233 §6.2): a plain +// find, never `generateKeychainKey`'s find-or-mint-and-store. A miss (including `unavailable`) is +// reported as "this tier has nothing to offer", not "mint one" — the export path this feeds must +// never write a new secret to the OS keychain any more than it may write a keyfile. +export function keyFromKeychainReadOnly(options: MacosKeychainOptions = {}): Buffer | undefined { + const account = userInfo().username; + const read = readMacosKeychainSecret(KEYCHAIN_SERVICE, account, options); + if (read.kind !== "found") return undefined; + try { + return decodeKeyOrThrow(read.secret, "Keychain key"); + } catch { + return undefined; + } +} + function keyFromKeyfile(memoryDir: string): Buffer { ensureDirHardened(memoryDir); const keyfile = join(memoryDir, KEYFILE_NAME); @@ -123,6 +138,28 @@ export function resolveVaultKey( // Test/CI seam: an explicit "no keychain" reader so callers can force the keyfile tier. export const NO_KEYCHAIN: KeychainAccess = () => undefined; +export interface ResolvedVaultKeyReadOnly { + readonly key: Buffer | undefined; + readonly source: VaultKeySource | undefined; +} + +// Read-only counterpart to `resolveVaultKey` for the diagnostic export seam (Wave 4a, epic #3233 +// §6.2/Finding 0): tries env, then a plain keychain lookup, and — unlike `resolveVaultKey` — never +// falls through to the keyfile tier, because that tier's only miss behaviour is minting and +// persisting a brand-new key to the customer's state directory. No `memoryDir` parameter: the +// whole point of this function is that it never touches the filesystem, so it must not carry a +// parameter that invites a future keyfile branch. +export function resolveVaultKeyReadOnly( + env: Readonly>, + keychainAccess: KeychainAccess = keyFromKeychainReadOnly, +): ResolvedVaultKeyReadOnly { + const fromEnv = keyFromEnv(env); + if (fromEnv !== undefined) return { key: fromEnv, source: "env" }; + const fromKeychain = keychainAccess(); + if (fromKeychain !== undefined) return { key: fromKeychain, source: "keychain" }; + return { key: undefined, source: undefined }; +} + export function createMemoryContentCipher(key: Buffer): MemoryContentCipher { return { sealString: (plaintext: string): string => sealString(key, plaintext), diff --git a/packages/keiko-memory-vault/src/db.test.ts b/packages/keiko-memory-vault/src/db.test.ts index 7b5759973c..1cac66d8db 100644 --- a/packages/keiko-memory-vault/src/db.test.ts +++ b/packages/keiko-memory-vault/src/db.test.ts @@ -17,6 +17,7 @@ import { chmodIfPresent, computeStoreFingerprint, openMemoryDatabase, + openMemoryDatabaseReadOnly, quarantineCorruptDb, } from "./db.js"; import { MEMORY_VAULT_SCHEMA_VERSION } from "./schema.js"; @@ -364,6 +365,28 @@ describe("computeStoreFingerprint", () => { }); }); +describe("openMemoryDatabaseReadOnly (Finding 2 — busy_timeout on the read-only diagnostic open)", () => { + // RED (before fix): `node:sqlite`'s default busy_timeout is 0, so a reader started against a + // live production server can receive an immediate SQLITE_BUSY from a concurrent WAL checkpoint + // and spuriously report the vault `open-failed`, exactly the moment `keiko support export` + // needs the fingerprint to work. + it("sets the active PRAGMA busy_timeout, matching the production open path", () => { + const dir = freshDir(); + const dbPath = join(dir, "keiko-memory.db"); + openMemoryDatabase(dbPath, TEST_CIPHER).close(); + + const db = openMemoryDatabaseReadOnly(dbPath); + try { + const rows = db.prepare("PRAGMA busy_timeout").all() as unknown as readonly { + timeout: number; + }[]; + expect(rows[0]?.timeout).toBe(5000); + } finally { + db.close(); + } + }); +}); + describe("chmodIfPresent", () => { it("is a no-op for non-existent paths", () => { const dir = freshDir(); diff --git a/packages/keiko-memory-vault/src/db.ts b/packages/keiko-memory-vault/src/db.ts index 436801fde6..cea307f540 100644 --- a/packages/keiko-memory-vault/src/db.ts +++ b/packages/keiko-memory-vault/src/db.ts @@ -32,10 +32,16 @@ import { export { chmodIfPresent, ensureDirHardened }; +// Issue #639's busy_timeout bound, shared by the production open path (`preparedDatabase` below) +// and the read-only diagnostic open (`openMemoryDatabaseReadOnly`, Finding 2) so both connections +// wait the same short, bounded interval for a concurrent writer's lock instead of failing +// immediately with SQLITE_BUSY. +const MEMORY_VAULT_BUSY_TIMEOUT_MS = 5_000; + export function preparedDatabase(target: string): DatabaseSync { const db = new DatabaseSync(target); db.exec("PRAGMA foreign_keys = ON"); - db.exec("PRAGMA busy_timeout = 5000"); + db.exec(`PRAGMA busy_timeout = ${String(MEMORY_VAULT_BUSY_TIMEOUT_MS)}`); return db; } @@ -172,7 +178,13 @@ export function openMemoryDatabase( // `PRAGMA user_version`/`quick_check` and fixed `SELECT COUNT(*)` reads, none of which need write // access. export function openMemoryDatabaseReadOnly(dbPath: string): DatabaseSync { - return new DatabaseSync(dbPath, { readOnly: true }); + const db = new DatabaseSync(dbPath, { readOnly: true }); + // A short busy_timeout (Finding 2) so a reader opened with no wait bound does not spuriously + // report the store `open-failed` on an immediate SQLITE_BUSY from a concurrent WAL checkpoint + // or schema-changing transaction on a live production server. Connection-local PRAGMA: no write, + // does not throw on a `readOnly: true` handle, so the read-only guarantee above is unaffected. + db.exec(`PRAGMA busy_timeout = ${String(MEMORY_VAULT_BUSY_TIMEOUT_MS)}`); + return db; } // ─── computeStoreFingerprint (Wave 4a, epic #3233 §6.2) ──────────────────────────────────────── diff --git a/packages/keiko-memory-vault/src/index.ts b/packages/keiko-memory-vault/src/index.ts index 71d3f69470..31a99a73c5 100644 --- a/packages/keiko-memory-vault/src/index.ts +++ b/packages/keiko-memory-vault/src/index.ts @@ -30,7 +30,9 @@ export { memoryBodySuppressionHash } from "./body-fingerprint.js"; export { createMemoryContentCipher, resolveVaultKey, + resolveVaultKeyReadOnly, type MemoryContentCipher, + type ResolvedVaultKeyReadOnly, type VaultKeySource, } from "./cipher.js"; export { computeStoreFingerprint, openMemoryDatabase, openMemoryDatabaseReadOnly } from "./db.js"; diff --git a/packages/keiko-server/src/store-fingerprints.test.ts b/packages/keiko-server/src/store-fingerprints.test.ts index 40dd8ea911..fa3638ce1b 100644 --- a/packages/keiko-server/src/store-fingerprints.test.ts +++ b/packages/keiko-server/src/store-fingerprints.test.ts @@ -12,6 +12,7 @@ import { randomBytes } from "node:crypto"; import { + existsSync, mkdirSync, mkdtempSync, readdirSync, @@ -198,4 +199,28 @@ describe("collectStoreFingerprints", () => { expect(uiEntry?.quickCheckOk).toBe(false); expect(result.unavailable.some((entry) => entry.store === "ui")).toBe(false); }); + + // Finding 0 (blocker): a memory-vault db that exists with KEIKO_MEMORY_KEY unset must never + // mint and persist a brand-new `vault.key` into the state dir as a side effect of a read-only + // diagnostic collection — regardless of what the OS keychain tier answers on the machine + // running the test (it may legitimately hold a real "keiko-memory-vault" entry on a dev + // machine that has run the product). RED (before fix): `memoryVaultStoreFingerprintOutcome` + // called the mutating `resolveVaultKey`, which falls through to `keyFromKeyfile` whenever the + // keychain tier misses and writes the keyfile; the deterministic, keychain-independent proof + // that this can never happen again lives in cipher.test.ts's + // "resolveVaultKeyReadOnly (Finding 0 …)" suite, which injects a fake keychain reader. This + // test additionally pins the invariant at the layer `keiko support export` actually calls. + it("never mints a vault.key as a side effect of fingerprinting a memory vault whose key is unresolved", async () => { + seedMemoryVault(stateDir, memoryKeyBase64); + + const result = await collectStoreFingerprints({ stateDir, env: {} }); + + expect(existsSync(join(stateDir, "memory", "vault.key"))).toBe(false); + // The store is still fingerprinted, never reported open-failed just because the key could + // not be resolved, and `keySource` is never "keyfile" — the one value that could only be + // produced by minting and persisting a new key. + const memoryEntry = result.fingerprints.find((entry) => entry.store === "memory-vault"); + expect(memoryEntry).toBeDefined(); + expect(memoryEntry?.keySource).not.toBe("keyfile"); + }); }); diff --git a/packages/keiko-server/src/store-fingerprints.ts b/packages/keiko-server/src/store-fingerprints.ts index f70d6bdb11..6822832f8e 100644 --- a/packages/keiko-server/src/store-fingerprints.ts +++ b/packages/keiko-server/src/store-fingerprints.ts @@ -34,7 +34,7 @@ import { computeStoreFingerprint as computeMemoryVaultStoreFingerprint, MEMORY_DB_FILENAME, openMemoryDatabaseReadOnly, - resolveVaultKey, + resolveVaultKeyReadOnly, } from "@oscharko-dev/keiko-memory-vault"; import { computeStoreFingerprint as computeUiStoreFingerprint, @@ -125,7 +125,11 @@ function memoryVaultStoreFingerprintOutcome( const memoryDir = env.KEIKO_MEMORY_DIR ?? join(stateDir, MEMORY_STATE_SUBDIR); const dbPath = join(memoryDir, MEMORY_DB_FILENAME); if (!existsSync(dbPath)) return unavailableOutcome("memory-vault", "missing"); - const resolved = resolveVaultKey(env, memoryDir); + // `resolveVaultKeyReadOnly` (never the mutating `resolveVaultKey`) — a db that exists but + // answers to neither the env key nor the OS keychain must degrade `keySource` to `undefined` + // (computeMemoryVaultStoreFingerprint tolerates that), never mint and persist a brand-new + // keyfile into the very state dir this diagnostic export is trying to inspect (Finding 0). + const resolved = resolveVaultKeyReadOnly(env); const db = openMemoryDatabaseReadOnly(dbPath); try { return { fingerprint: computeMemoryVaultStoreFingerprint(db, resolved.source) }; diff --git a/packages/keiko-server/src/store/db.test.ts b/packages/keiko-server/src/store/db.test.ts index be03aad6af..88a81084d4 100644 --- a/packages/keiko-server/src/store/db.test.ts +++ b/packages/keiko-server/src/store/db.test.ts @@ -24,6 +24,7 @@ import { createInMemoryUiStore, createNodeUiStore, openNodeUiDatabase, + openNodeUiDatabaseReadOnly, SCHEMA_VERSION, UI_DB_BUSY_TIMEOUT_MS, type GroundedAnswer, @@ -1083,6 +1084,26 @@ describe("UI DB busy_timeout (issue #639)", () => { expect(store.listProjects()).toEqual([]); store.close(); }); + + // Finding 2: the read-only diagnostic open (`keiko support export`'s fingerprint collection) + // must set the same busy_timeout as the production open, so a reader started against a live + // production server does not spuriously report the store `open-failed` on an immediate + // SQLITE_BUSY from a concurrent WAL checkpoint. RED (before fix): `node:sqlite`'s default + // busy_timeout is 0, so this assertion fails against the un-pragma'd read-only open. + it("sets the active PRAGMA busy_timeout on the read-only node UI database open", () => { + const dbPath = join(tmpDir, "busy-readonly.db"); + openNodeUiDatabase(dbPath).close(); + + const db = openNodeUiDatabaseReadOnly(dbPath); + try { + const rows = db.prepare("PRAGMA busy_timeout").all() as unknown as readonly { + timeout: number; + }[]; + expect(rows[0]?.timeout).toBe(UI_DB_BUSY_TIMEOUT_MS); + } finally { + db.close(); + } + }); }); // Wave 4a, epic #3233 §6.2 — the redacted, point-in-time schema/integrity snapshot embedded in diff --git a/packages/keiko-server/src/store/db.ts b/packages/keiko-server/src/store/db.ts index 296f2f0234..09006492d2 100644 --- a/packages/keiko-server/src/store/db.ts +++ b/packages/keiko-server/src/store/db.ts @@ -1022,7 +1022,16 @@ export function openNodeUiDatabase(dbPath: string, sink?: ServerLogSink): Databa // can never flip a `client_turn_state`, apply a migration, or quarantine the very file an operator // is trying to inspect. export function openNodeUiDatabaseReadOnly(dbPath: string): DatabaseSync { - return new DatabaseSync(dbPath, { readOnly: true }); + const db = new DatabaseSync(dbPath, { readOnly: true }); + // Issue #639's busy_timeout applies here too (Finding 2): without it, a reader opened with no + // wait bound can receive an immediate SQLITE_BUSY from a concurrent WAL checkpoint or + // schema-changing transaction on a live production server — exactly the moment `keiko support + // export` needs the fingerprint to work — and spuriously report the store `open-failed` for a + // purely transient reason. This is a connection-local PRAGMA; it performs no write and does not + // throw on a `readOnly: true` handle, so it does not affect the genuinely-read-only guarantee + // documented above. + db.exec(`PRAGMA busy_timeout = ${String(UI_DB_BUSY_TIMEOUT_MS)}`); + return db; } export function buildUiStoreOverDatabase(db: DatabaseSync, opts?: UiStoreFactoryOptions): UiStore { From a533fb24fb796445376c9dd909ab66a66ed8cf69 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 10:56:31 +0200 Subject: [PATCH 05/19] =?UTF-8?q?fix(observability):=20Wave=205=20audit=20?= =?UTF-8?q?repairs=20=E2=80=94=20SSE=20terminal=20line=20carries=20the=20r?= =?UTF-8?q?equest=20id,=20bounded=20query-name=20scan,=20one=20SSE=20diagn?= =?UTF-8?q?ostic=20helper,=20shared=20JSON=20body=20reader=20(#3240)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sse.stream.closed carries the request's correlationId at every real SSE write path (terminal, browser, chat stream, relationship fan-out/events). - Query-parameter names are bounded before sorting and scanned only after the allowed-host check. - The client correlation-id pattern is pinned to the server's by a drift test. - sseStreamErrorDiagnostic lives once in client-diagnostics.ts; the three memory handlers share readJsonRequestBody; Set fan-out iterates the live set. Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 2 +- .../src/bounded-request-body.test.ts | 57 ++++++++++++ .../keiko-server/src/bounded-request-body.ts | 46 ++++++++++ .../keiko-server/src/browser-routes.test.ts | 52 +++++++++++ packages/keiko-server/src/browser.ts | 9 +- .../src/chat-stream-handlers.test.ts | 59 +++++++++++++ .../keiko-server/src/chat-stream-handlers.ts | 11 ++- .../src/memory-consolidation-handlers.ts | 29 ++----- .../keiko-server/src/memory-conv-handlers.ts | 29 ++----- packages/keiko-server/src/memory-handlers.ts | 29 ++----- .../relationship-activity-broadcast.test.ts | 87 +++++++++++++++++++ .../src/relationship-activity-broadcast.ts | 21 +++-- .../src/relationship-handlers.test.ts | 83 +++++++++++++++++- .../keiko-server/src/relationship-handlers.ts | 48 +++++++--- packages/keiko-server/src/server.test.ts | 52 ++++++++++- packages/keiko-server/src/server.ts | 26 ++++-- .../keiko-server/src/terminal-routes.test.ts | 46 ++++++++++ packages/keiko-server/src/terminal-routes.ts | 16 +++- .../widgets/cards/sharedEventSource.ts | 27 ++---- .../panels/useRelationshipActivityStream.ts | 27 ++---- .../keiko-ui/src/lib/client-diagnostics.ts | 20 +++++ .../lib/coding-workbench-event-retention.ts | 26 +----- packages/keiko-ui/src/lib/useSSE.ts | 24 +---- .../correlation-id-pattern-drift.test.mjs | 52 +++++++++++ 24 files changed, 692 insertions(+), 186 deletions(-) create mode 100644 scripts/__tests__/correlation-id-pattern-drift.test.mjs diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index 4f53996aed..589ddd171c 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -791,7 +791,7 @@ { "op": "request", "category": "http", - "site": "packages/keiko-server/src/server.ts:468", + "site": "packages/keiko-server/src/server.ts:478", "package": "keiko-server" }, { diff --git a/packages/keiko-server/src/bounded-request-body.test.ts b/packages/keiko-server/src/bounded-request-body.test.ts index b25efe2f8d..219d03dab5 100644 --- a/packages/keiko-server/src/bounded-request-body.test.ts +++ b/packages/keiko-server/src/bounded-request-body.test.ts @@ -3,6 +3,7 @@ import { PassThrough, Readable } from "node:stream"; import { afterEach, describe, expect, it, vi } from "vitest"; import { readBoundedRequestBody, + readJsonRequestBody, RequestBodyCancelledError, RequestBodyTooLargeError, } from "./bounded-request-body.js"; @@ -369,3 +370,59 @@ describe("bounded request body activity log", () => { stream.end(); }); }); + +// #2902 audit finding 3: memory-handlers.ts, memory-conv-handlers.ts and +// memory-consolidation-handlers.ts each hand-rolled a byte-identical "bounded read, then +// parse+validate as a JSON object" wrapper on top of `readBoundedRequestBody`. `readJsonRequestBody` +// is now the one owner of that wrapper layer; each caller keeps its own max-bytes constant and +// becomes a one-line delegate to this function. +describe("readJsonRequestBody", () => { + it("returns 413 PAYLOAD_TOO_LARGE for an oversized body", async () => { + const req = asRequest(Readable.from([Buffer.from("this body is too long")])); + + const result = await readJsonRequestBody(req, 4); + + expect(result).toEqual({ + status: 413, + body: { error: { code: "PAYLOAD_TOO_LARGE", message: "Request body too large." } }, + }); + }); + + it("returns 400 BAD_REQUEST for malformed JSON", async () => { + const req = asRequest(Readable.from([Buffer.from("{not json")])); + + const result = await readJsonRequestBody(req, 128_000); + + expect(result).toEqual({ + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body is not valid JSON." } }, + }); + }); + + it("returns 400 BAD_REQUEST for valid JSON that is not an object (an array)", async () => { + const req = asRequest(Readable.from([Buffer.from("[1,2,3]")])); + + const result = await readJsonRequestBody(req, 128_000); + + expect(result).toEqual({ + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body must be a JSON object." } }, + }); + }); + + it("returns the parsed record for a valid JSON object body", async () => { + const req = asRequest(Readable.from([Buffer.from('{"projectId":"p-1","cwd":"/tmp"}')])); + + const result = await readJsonRequestBody(req, 128_000); + + expect(result).toEqual({ projectId: "p-1", cwd: "/tmp" }); + }); + + it("treats an empty body as an empty object", async () => { + const req = asRequest(Readable.from([])); + + const result = await readJsonRequestBody(req, 128_000); + + expect(result).toEqual({}); + }); +}); diff --git a/packages/keiko-server/src/bounded-request-body.ts b/packages/keiko-server/src/bounded-request-body.ts index 61e01962d8..6cf0ee8fb7 100644 --- a/packages/keiko-server/src/bounded-request-body.ts +++ b/packages/keiko-server/src/bounded-request-body.ts @@ -240,3 +240,49 @@ export function readBoundedRequestBody( new BoundedRequestBodyReader(req, maxBytes, signal, correlationId, resolve, reject).start(); }); } + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +// #2902 audit finding 3: memory-handlers.ts, memory-conv-handlers.ts and memory-consolidation- +// handlers.ts each hand-rolled a byte-identical "read a bounded body, then parse+validate it as a +// JSON object" wrapper on top of `readBoundedRequestBody` — differing only in a catch-variable +// name and which (identically-valued, 64_000) max-bytes constant they read. This is the ONE owner +// for that wrapper layer; callers keep their own max-bytes constant (there is no reason to force +// them to share one, only the logic), pass it in, and get back either the parsed JSON object or the +// RouteResult (413/400) their handler should return as-is. +export async function readJsonRequestBody( + req: IncomingMessage, + maxBytes: number, + correlationId?: string, +): Promise | { readonly status: number; readonly body: unknown }> { + let raw: string; + try { + raw = await readBoundedRequestBody(req, maxBytes, undefined, correlationId); + } catch (error) { + if (error instanceof RequestBodyTooLargeError) { + return { + status: 413, + body: { error: { code: "PAYLOAD_TOO_LARGE", message: "Request body too large." } }, + }; + } + throw error; + } + let parsed: unknown; + try { + parsed = raw.length === 0 ? {} : JSON.parse(raw); + } catch { + return { + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body is not valid JSON." } }, + }; + } + if (!isPlainRecord(parsed)) { + return { + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body must be a JSON object." } }, + }; + } + return parsed; +} diff --git a/packages/keiko-server/src/browser-routes.test.ts b/packages/keiko-server/src/browser-routes.test.ts index 6658f4be5f..1a04c37e3e 100644 --- a/packages/keiko-server/src/browser-routes.test.ts +++ b/packages/keiko-server/src/browser-routes.test.ts @@ -16,6 +16,12 @@ import { EventEmitter } from "node:events"; import type { ServerResponse } from "node:http"; import { openBrowserSseStream } from "./browser.js"; import type { SseBackpressureSignal } from "./sse-write.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, +} from "./observability/index.js"; import { BrowserToolError, type BrowserEventEmitter, @@ -821,3 +827,49 @@ describe("openBrowserSseStream backpressure (KEIKO-0142)", () => { expect(fake.writes).toHaveLength(writesAfterClose); }); }); + +// Finding 0 (#2902 audit): the request-scoped correlationId never reached the terminal +// `sse.stream.closed` line — openBrowserSseStream had no parameter to receive it. The ready frame +// (not the heartbeat, whose own write is deferred to its interval timer) is the actual first write +// on the stream, so that is where correlationId must be threaded. +describe("openBrowserSseStream correlationId threading (#2902 audit finding 0)", () => { + afterEach(() => { + resetServerLogger(); + }); + + it("attaches the supplied correlationId to the sse.stream.closed terminal line", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const fake = makeFakeSseRes(); + const manager = new FakeBrowserSessionManager(); + + openBrowserSseStream( + fake.res, + manager, + "session-corr", + (value) => value, + undefined, + "corr-browser-1", + ); + fake.emitClose(); + + expect(sink.events).toHaveLength(1); + expect(sink.events[0]).toMatchObject({ + op: "sse.stream.closed", + correlationId: "corr-browser-1", + }); + }); + + it("omits correlationId from the terminal line when none is supplied (unchanged behavior)", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const fake = makeFakeSseRes(); + const manager = new FakeBrowserSessionManager(); + + openBrowserSseStream(fake.res, manager, "session-no-corr", (value) => value); + fake.emitClose(); + + expect(sink.events).toHaveLength(1); + expect(sink.events[0]?.correlationId).toBeUndefined(); + }); +}); diff --git a/packages/keiko-server/src/browser.ts b/packages/keiko-server/src/browser.ts index 156a662b57..fe29d1a230 100644 --- a/packages/keiko-server/src/browser.ts +++ b/packages/keiko-server/src/browser.ts @@ -266,6 +266,7 @@ export function handleBrowserEvents(ctx: RouteContext, deps: UiHandlerDeps): Han sessionId, deps.redactor, sseBackpressureReporter(deps, "browser"), + ctx.correlationId, ); ctx.req.on("close", () => { ctx.res.end(); @@ -282,6 +283,7 @@ export function openBrowserSseStream( sessionId: string, redactor: UiHandlerDeps["redactor"], onBackpressure?: (signal: SseBackpressureSignal) => void, + correlationId?: string, ): void { res.writeHead(200, SSE_HEADERS); // Per-connection abort: a slow-client backpressure kill (writeOrDestroy) aborts this controller, @@ -290,9 +292,14 @@ export function openBrowserSseStream( // subscribe() returns synchronously and events fire only asynchronously afterward, so no event // (hence no abort) can occur before `unsubscribe` is assigned. const controller = new AbortController(); + // correlationId (#2902 w5-sse-counters) is threaded to every write path below so whichever one + // runs first attaches it: sse-write.ts's per-stream state is set-once-wins. The heartbeat's own + // write is deferred to its interval timer, so the ready frame just below is the actual first + // write in practice — it also carries correlationId for that reason. startSseHeartbeat(res, undefined, undefined, { controller, ...(onBackpressure === undefined ? {} : { onBackpressure }), + ...(correlationId === undefined ? {} : { correlationId }), }); let seq = 0; const unsubscribe = manager.subscribe(sessionId, (event) => { @@ -313,7 +320,7 @@ export function openBrowserSseStream( // The ready frame goes through the same protective path: a client that is already not draining // must abort and unsubscribe here too, rather than leaving the subscription live until some // later event happens to trip writeOrDestroy. - writeOrDestroy(res, readyMessage(), controller, onBackpressure); + writeOrDestroy(res, readyMessage(), controller, onBackpressure, correlationId); res.on("close", () => { stop(); }); diff --git a/packages/keiko-server/src/chat-stream-handlers.test.ts b/packages/keiko-server/src/chat-stream-handlers.test.ts index fb2c02ac2e..357b45a7b8 100644 --- a/packages/keiko-server/src/chat-stream-handlers.test.ts +++ b/packages/keiko-server/src/chat-stream-handlers.test.ts @@ -32,6 +32,12 @@ import type { ConversationMemoryRuntimeContext } from "./memory-conversation-con import { composeDiscussionDirectiveBlock } from "./discussion-prompt.js"; import { STREAMING, type RouteContext } from "./routes.js"; import { buildRedactor, createRunRegistry, type UiHandlerDeps } from "./index.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, +} from "./observability/index.js"; import type { RuntimeGatewayConfig } from "./deps.js"; import { createInMemoryUiStore, type UiStore } from "./store/index.js"; import type { ModelPort } from "@oscharko-dev/keiko-harness"; @@ -3281,3 +3287,56 @@ describe("concurrent chat stream bulkhead", () => { expect(parseSse(second.writes).some((record) => record.event === "done")).toBe(true); }); }); + +// Finding 0 (#2902 audit): the request-scoped correlationId never reached the terminal +// `sse.stream.closed` line on the desktop chat stream. The heartbeat's own write is deferred to +// its interval timer, so in practice the first model token — written via streamConversation's +// writeOrDestroy call — is the write that actually attaches it first. +describe("desktop chat SSE stream correlationId threading (#2902 audit finding 0)", () => { + afterEach(() => { + resetServerLogger(); + }); + + it("attaches the supplied correlationId to the sse.stream.closed terminal line", async () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const chatId = seedChat(); + const captured = captureResWithEvents(); + const { model } = streamingModel("answer"); + const ctx: RouteContext = { + ...routeContext( + makeReq({ chatId, projectPath: projectDir, modelId: CHAT_MODEL, content: "hello" }), + captured.res, + ), + correlationId: "corr-chat-stream-1", + }; + + await handleSendDesktopChatStream(ctx, deps(model)); + captured.emitClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(1); + expect(closed[0]?.correlationId).toBe("corr-chat-stream-1"); + }); + + it("omits correlationId from the terminal line when none is supplied (unchanged behavior)", async () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const chatId = seedChat(); + const captured = captureResWithEvents(); + const { model } = streamingModel("answer"); + + await handleSendDesktopChatStream( + routeContext( + makeReq({ chatId, projectPath: projectDir, modelId: CHAT_MODEL, content: "hello" }), + captured.res, + ), + deps(model), + ); + captured.emitClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(1); + expect(closed[0]?.correlationId).toBeUndefined(); + }); +}); diff --git a/packages/keiko-server/src/chat-stream-handlers.ts b/packages/keiko-server/src/chat-stream-handlers.ts index f13dcca567..1e6a8a1bc9 100644 --- a/packages/keiko-server/src/chat-stream-handlers.ts +++ b/packages/keiko-server/src/chat-stream-handlers.ts @@ -167,6 +167,7 @@ async function streamConversation( () => { termination.backpressure = true; }, + ctx.correlationId, ); if (requestIsAborted(controller.signal)) return undefined; } else { @@ -501,7 +502,15 @@ async function executeAdmittedDesktopChatStream( try { ctx.res.writeHead(200, SSE_HEADERS); markStreamStarted(); - stopHeartbeat = startSseHeartbeat(ctx.res); + // correlationId (#2902 w5-sse-counters) is threaded here AND into streamConversation's + // per-token writeOrDestroy call: whichever write actually happens first attaches it to the + // terminal `sse.stream.closed` line (sse-write.ts's per-stream state is set-once-wins). The + // heartbeat's own write is deferred to its interval timer, so in practice the first model + // token — not the heartbeat — is usually the write that sets it. + stopHeartbeat = startSseHeartbeat(ctx.res, undefined, undefined, { + controller, + ...(ctx.correlationId === undefined ? {} : { correlationId: ctx.correlationId }), + }); await streamAndPersist(ctx, deps, turn, controller); } catch (error) { writeStreamFailure(ctx, deps, turn.prepared.request, controller, error); diff --git a/packages/keiko-server/src/memory-consolidation-handlers.ts b/packages/keiko-server/src/memory-consolidation-handlers.ts index 51f6b7b9c7..163107798b 100644 --- a/packages/keiko-server/src/memory-consolidation-handlers.ts +++ b/packages/keiko-server/src/memory-consolidation-handlers.ts @@ -46,7 +46,7 @@ import type { ConsolidationJobSettings, } from "./memory-consolidation-registry.js"; import { enrichReviewItemsWithAdvisory } from "./memory-conflict-advisory.js"; -import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; +import { readJsonRequestBody } from "./bounded-request-body.js"; const MAX_BODY_BYTES = 64_000; const DEFAULT_JACCARD_THRESHOLD = 0.85; @@ -71,30 +71,15 @@ function isRouteResult(value: unknown): value is RouteResult { } // Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the cap above is -// unchanged, only the ad hoc listener wiring is gone. -async function readJsonBody( +// unchanged, only the ad hoc listener wiring is gone. The read-parse-validate wrapper itself is +// also consolidated (#2902 audit finding 3): `readJsonRequestBody` (bounded-request-body.ts) is +// the one owner of "bounded read, then parse+validate as a JSON object", previously hand-rolled +// identically in this file, memory-handlers.ts and memory-conv-handlers.ts. +function readJsonBody( req: IncomingMessage, correlationId?: string, ): Promise | RouteResult> { - let raw: string; - try { - raw = await readBoundedRequestBody(req, MAX_BODY_BYTES, undefined, correlationId); - } catch (error) { - if (error instanceof RequestBodyTooLargeError) { - return { status: 413, body: errorBody("PAYLOAD_TOO_LARGE", "Request body too large.") }; - } - throw error; - } - let parsed: unknown; - try { - parsed = raw.length === 0 ? {} : JSON.parse(raw); - } catch { - return { status: 400, body: errorBody("BAD_REQUEST", "Request body is not valid JSON.") }; - } - if (!isRecord(parsed)) { - return { status: 400, body: errorBody("BAD_REQUEST", "Request body must be a JSON object.") }; - } - return parsed; + return readJsonRequestBody(req, MAX_BODY_BYTES, correlationId); } function resolveVault(deps: UiHandlerDeps): MemoryVaultStore | RouteResult { diff --git a/packages/keiko-server/src/memory-conv-handlers.ts b/packages/keiko-server/src/memory-conv-handlers.ts index e41e40e7df..441c284643 100644 --- a/packages/keiko-server/src/memory-conv-handlers.ts +++ b/packages/keiko-server/src/memory-conv-handlers.ts @@ -61,7 +61,7 @@ import { recordMemoryAudit } from "./memory-audit-handler.js"; import { recordAutoAcceptedMemoryCaptureDecision } from "./memory-capture-audit.js"; import { buildMemoryRecordFromProposal } from "./memory-record-builders.js"; import { persistCapturedMemory } from "./memory-capture-persistence.js"; -import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; +import { readJsonRequestBody } from "./bounded-request-body.js"; import { enforcePersistableMemoryOutcome, FORGOTTEN_MEMORY_SUPPRESSION_REASON, @@ -87,35 +87,20 @@ const MAX_BODY_BYTES = 64_000; // ─── Body reading ────────────────────────────────────────────────────────────── // Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the cap below is -// unchanged, only the ad hoc listener wiring is gone. +// unchanged, only the ad hoc listener wiring is gone. The read-parse-validate wrapper itself is +// also consolidated (#2902 audit finding 3): `readJsonRequestBody` (bounded-request-body.ts) is +// the one owner of "bounded read, then parse+validate as a JSON object", previously hand-rolled +// identically in this file, memory-handlers.ts and memory-consolidation-handlers.ts. function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -async function readJsonBody( +function readJsonBody( req: IncomingMessage, correlationId?: string, ): Promise | RouteResult> { - let raw: string; - try { - raw = await readBoundedRequestBody(req, MAX_BODY_BYTES, undefined, correlationId); - } catch (err) { - if (err instanceof RequestBodyTooLargeError) { - return { status: 413, body: errorBody("PAYLOAD_TOO_LARGE", "Request body too large.") }; - } - throw err; - } - let parsed: unknown; - try { - parsed = raw.length === 0 ? {} : JSON.parse(raw); - } catch { - return { status: 400, body: errorBody("BAD_REQUEST", "Request body is not valid JSON.") }; - } - if (!isRecord(parsed)) { - return { status: 400, body: errorBody("BAD_REQUEST", "Request body must be a JSON object.") }; - } - return parsed; + return readJsonRequestBody(req, MAX_BODY_BYTES, correlationId); } function isRouteResult(value: unknown): value is RouteResult { diff --git a/packages/keiko-server/src/memory-handlers.ts b/packages/keiko-server/src/memory-handlers.ts index e6a64821c9..a7fb0afdc2 100644 --- a/packages/keiko-server/src/memory-handlers.ts +++ b/packages/keiko-server/src/memory-handlers.ts @@ -79,7 +79,7 @@ import { type MemoryCaptureDecision, } from "./memory-capture-projection.js"; import { refreshMemoryEmbeddingAfterBodyEdit } from "./memory-embedding.js"; -import { readBoundedRequestBody, RequestBodyTooLargeError } from "./bounded-request-body.js"; +import { readJsonRequestBody } from "./bounded-request-body.js"; // ─── Constants ──────────────────────────────────────────────────────────────── @@ -245,31 +245,16 @@ function parseScope(raw: unknown): MemoryScope | RouteResult { // ─── Body reading ────────────────────────────────────────────────────────────── // Consolidated onto the shared bounded reader (#2902 w5-sse-counters) — the caps below are -// unchanged, only the ad hoc listener wiring is gone. +// unchanged, only the ad hoc listener wiring is gone. The read-parse-validate wrapper itself is +// also consolidated (#2902 audit finding 3): `readJsonRequestBody` (bounded-request-body.ts) is +// the one owner of "bounded read, then parse+validate as a JSON object", previously hand-rolled +// identically in this file, memory-conv-handlers.ts and memory-consolidation-handlers.ts. -async function readJsonBody( +function readJsonBody( req: IncomingMessage, correlationId?: string, ): Promise | RouteResult> { - let raw: string; - try { - raw = await readBoundedRequestBody(req, MAX_MEMORY_BODY_BYTES, undefined, correlationId); - } catch (err) { - if (err instanceof RequestBodyTooLargeError) { - return { status: 413, body: errorBody("PAYLOAD_TOO_LARGE", "Request body too large.") }; - } - throw err; - } - let parsed: unknown; - try { - parsed = raw.length === 0 ? {} : JSON.parse(raw); - } catch { - return { status: 400, body: errorBody("BAD_REQUEST", "Request body is not valid JSON.") }; - } - if (!isRecord(parsed)) { - return { status: 400, body: errorBody("BAD_REQUEST", "Request body must be a JSON object.") }; - } - return parsed; + return readJsonRequestBody(req, MAX_MEMORY_BODY_BYTES, correlationId); } function isRouteResult(v: unknown): v is RouteResult { diff --git a/packages/keiko-server/src/relationship-activity-broadcast.test.ts b/packages/keiko-server/src/relationship-activity-broadcast.test.ts index cc0c6e28a0..bf16e064c5 100644 --- a/packages/keiko-server/src/relationship-activity-broadcast.test.ts +++ b/packages/keiko-server/src/relationship-activity-broadcast.test.ts @@ -6,12 +6,19 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { ServerResponse } from "node:http"; +import { EventEmitter } from "node:events"; import { subscribeActivityBroadcast, _activeBroadcastCountForTests, type ActivityFrame, type ActivitySubscriber, } from "./relationship-activity-broadcast.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, +} from "./observability/index.js"; const REFRESH_MS = 5_000; const PING_MS = 30_000; @@ -207,3 +214,83 @@ describe("subscribeActivityBroadcast", () => { expect(_activeBroadcastCountForTests(scopeB)).toBe(0); }); }); + +// Finding 0 (#2902 audit): a fanned-out subscriber's OWN request correlationId never reached its +// own stream's terminal `sse.stream.closed` line — ActivitySubscriber had no field to carry it and +// writeTo() (the shared fan-out write path) never passed one to writeOrDestroy. +describe("ActivitySubscriber correlationId threading (#2902 audit finding 0)", () => { + function listenableFakeRes(): { res: ServerResponse; fireClose: () => void } { + const emitter = new EventEmitter(); + const res = { + writableEnded: false, + write: () => true, + destroy: (): void => undefined, + on: (event: string, handler: (...args: unknown[]) => void) => { + emitter.on(event, handler); + }, + } as unknown as ServerResponse; + return { res, fireClose: () => emitter.emit("close") }; + } + + afterEach(() => { + resetServerLogger(); + }); + + it("attaches the joining subscriber's own correlationId to its sse.stream.closed line", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const scope = {}; + const source = { + collectFrames: (): readonly ActivityFrame[] => [frame("snap-1", 1)], + refreshMs: REFRESH_MS, + pingMs: PING_MS, + }; + const { res, fireClose } = listenableFakeRes(); + const subscriber: ActivitySubscriber = { + res, + controller: new AbortController(), + correlationId: "corr-relact-1", + }; + + subscribeActivityBroadcast(scope, "ws-corr", subscriber, source); + fireClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(1); + expect(closed[0]?.correlationId).toBe("corr-relact-1"); + }); + + it("two concurrent subscribers on the same broadcaster each keep their own correlationId", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const scope = {}; + const source = { + collectFrames: (): readonly ActivityFrame[] => [frame("snap-1", 1)], + refreshMs: REFRESH_MS, + pingMs: PING_MS, + }; + const first = listenableFakeRes(); + const second = listenableFakeRes(); + subscribeActivityBroadcast( + scope, + "ws-corr-2", + { res: first.res, controller: new AbortController(), correlationId: "corr-first" }, + source, + ); + subscribeActivityBroadcast( + scope, + "ws-corr-2", + { res: second.res, controller: new AbortController(), correlationId: "corr-second" }, + source, + ); + first.fireClose(); + second.fireClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(2); + expect(closed.map((event) => event.correlationId).sort()).toEqual([ + "corr-first", + "corr-second", + ]); + }); +}); diff --git a/packages/keiko-server/src/relationship-activity-broadcast.ts b/packages/keiko-server/src/relationship-activity-broadcast.ts index 0e14f99a1e..f403342798 100644 --- a/packages/keiko-server/src/relationship-activity-broadcast.ts +++ b/packages/keiko-server/src/relationship-activity-broadcast.ts @@ -35,6 +35,9 @@ export interface ActivitySubscriber { readonly res: ServerResponse; readonly controller: AbortController; readonly onBackpressure?: (signal: SseBackpressureSignal) => void; + // The subscriber's OWN request-scoped correlation id (#2902 w5-sse-counters), attached to its + // own fanned-out stream — never the broadcaster's, since one broadcaster serves many requests. + readonly correlationId?: string; } interface Broadcaster { @@ -58,7 +61,13 @@ function writeTo(subscriber: ActivitySubscriber, frame: string): void { // A subscriber whose socket died mid-fan-out is skipped; its abort listener has // already detached it and writing to a destroyed response would throw. if (subscriber.controller.signal.aborted) return; - writeOrDestroy(subscriber.res, frame, subscriber.controller, subscriber.onBackpressure); + writeOrDestroy( + subscriber.res, + frame, + subscriber.controller, + subscriber.onBackpressure, + subscriber.correlationId, + ); } function runTick(broadcaster: Broadcaster): void { @@ -66,15 +75,17 @@ function runTick(broadcaster: Broadcaster): void { for (const { id, frame } of broadcaster.source.collectFrames()) { nextFrames.set(id, frame); if (broadcaster.lastFrames.get(id) !== frame) { - // Copy before fan-out: a backpressure kill aborts→detaches mid-iteration. - for (const subscriber of [...broadcaster.subscribers]) writeTo(subscriber, frame); + // Iterating the live Set is safe: a backpressure kill that aborts→detaches mid-iteration + // deletes from the Set, and a deleted, not-yet-visited entry is simply skipped — exactly the + // subscriber that must not be written to again. + for (const subscriber of broadcaster.subscribers) writeTo(subscriber, frame); } } broadcaster.lastFrames = nextFrames; } function runPing(broadcaster: Broadcaster): void { - for (const subscriber of [...broadcaster.subscribers]) writeTo(subscriber, PING_FRAME); + for (const subscriber of broadcaster.subscribers) writeTo(subscriber, PING_FRAME); } function ensureBroadcaster( @@ -125,7 +136,7 @@ function emitJoinState(broadcaster: Broadcaster, joiner: ActivitySubscriber): vo for (const { id, frame } of broadcaster.source.collectFrames()) { nextFrames.set(id, frame); const changed = broadcaster.lastFrames.get(id) !== frame; - for (const subscriber of [...broadcaster.subscribers]) { + for (const subscriber of broadcaster.subscribers) { if (subscriber === joiner || changed) writeTo(subscriber, frame); } } diff --git a/packages/keiko-server/src/relationship-handlers.test.ts b/packages/keiko-server/src/relationship-handlers.test.ts index 7b9014553c..7201f0dc36 100644 --- a/packages/keiko-server/src/relationship-handlers.test.ts +++ b/packages/keiko-server/src/relationship-handlers.test.ts @@ -9,7 +9,7 @@ // - redactor invocation (single call site) // - happy paths for the read routes -import { describe, expect, it, beforeEach, vi } from "vitest"; +import { afterEach, describe, expect, it, beforeEach, vi } from "vitest"; import { DatabaseSync } from "node:sqlite"; import { EventEmitter } from "node:events"; import { promises as fs, realpathSync } from "node:fs"; @@ -40,6 +40,12 @@ import { buildUiHandlerDeps, type UiHandlerDeps } from "./deps.js"; import type { RouteContext, RouteResult } from "./routes.js"; import { STREAMING } from "./routes.js"; import { createRunRegistry } from "./runs.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, +} from "./observability/index.js"; interface FakeReq extends EventEmitter { headers: Record; @@ -1523,6 +1529,81 @@ describe("GET /api/relationships/:id/dependencies + impact + health + explain + }); }); +// Finding 0 (#2902 audit): the request-scoped correlationId never reached the terminal +// `sse.stream.closed` line on /api/relationships/events. On an idle workspace (no activity +// snapshots), the `retry: … \n: connected` frame written directly in handleRelationshipEvents is +// the ONLY write on the stream until the 30s ping, so correlationId must be attached there. +describe("GET /api/relationships/events correlationId threading (#2902 audit finding 0)", () => { + afterEach(() => { + resetServerLogger(); + }); + + // Like makeCtx, but the response double also implements `on("close", …)` so the shared SSE + // write path (sse-write.ts) tracks and emits the per-stream terminal line, and carries a + // request-scoped correlationId the way the real server.ts request entry point does. + function makeListenableCtx(req: FakeReq, correlationId?: string): RouteContext { + const url = new URL(`http://localhost${req.url}`); + const emitter = new EventEmitter(); + const res = { + writeHead: (): void => undefined, + flushHeaders: (): void => undefined, + write: () => true, + end: (): void => undefined, + destroy: (): void => undefined, + writableEnded: false, + on: (event: string, handler: (...args: unknown[]) => void) => { + emitter.on(event, handler); + }, + _fireClose: () => emitter.emit("close"), + } as unknown as ServerResponse; + return { + req: req as unknown as IncomingMessage, + res, + params: {}, + url, + ...(correlationId === undefined ? {} : { correlationId }), + }; + } + + it("attaches the supplied correlationId to the sse.stream.closed terminal line", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const store = freshStore(); + const { redactor } = trackingRedactor(); + const deps = buildDeps("ws-corr-idle", store, redactor); + const req = makeReq({ method: "GET", url: "/api/relationships/events" }); + const ctx = makeListenableCtx(req, "corr-relhandlers-1"); + + const result = handleRelationshipEvents(ctx, deps); + expect(result).toBe(STREAMING); + (ctx.res as unknown as { _fireClose: () => void })._fireClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(1); + expect(closed[0]?.correlationId).toBe("corr-relhandlers-1"); + req.emit("close"); + }); + + it("omits correlationId from the terminal line when none is supplied (unchanged behavior)", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const store = freshStore(); + const { redactor } = trackingRedactor(); + const deps = buildDeps("ws-corr-idle-2", store, redactor); + const req = makeReq({ method: "GET", url: "/api/relationships/events" }); + const ctx = makeListenableCtx(req); + + const result = handleRelationshipEvents(ctx, deps); + expect(result).toBe(STREAMING); + (ctx.res as unknown as { _fireClose: () => void })._fireClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(1); + expect(closed[0]?.correlationId).toBeUndefined(); + req.emit("close"); + }); +}); + describe("POST /api/relationships/validate (preview)", () => { it("returns decision.allowed=true for a valid proposal", async () => { const store = freshStore(); diff --git a/packages/keiko-server/src/relationship-handlers.ts b/packages/keiko-server/src/relationship-handlers.ts index 984114efb6..1f7028157a 100644 --- a/packages/keiko-server/src/relationship-handlers.ts +++ b/packages/keiko-server/src/relationship-handlers.ts @@ -1964,16 +1964,45 @@ function handleEventsImpl(ctx: RouteContext, deps: UiHandlerDeps): RouteResult | // `retry:` reconnect directive plus an SSE comment (`:` lines are ignored by EventSource), so it // carries no relationship payload and never trips the activity allowlist. res.flushHeaders(); - // GEN-PERF-RELACT-001 — refresh/ping timers, snapshot sweeps, and frame serialization - // are shared per workspace via the broadcaster instead of duplicated per connection, - // and every write reacts to backpressure (writeOrDestroy) so a non-draining client is - // destroyed instead of growing the response buffer without bound. + const unsubscribe = openRelationshipEventsStream(ctx, deps, res, relationship, workspaceId); + ctx.req.on("close", () => { + unsubscribe(); + res.end(); + }); + return STREAMING; +} + +// GEN-PERF-RELACT-001 — refresh/ping timers, snapshot sweeps, and frame serialization are shared +// per workspace via the broadcaster instead of duplicated per connection, and every write reacts +// to backpressure (writeOrDestroy) so a non-draining client is destroyed instead of growing the +// response buffer without bound. `ctx.correlationId` (#2902 w5-sse-counters) is attached to the +// "connected" frame below — the actual first write on the stream, ahead of +// subscribeActivityBroadcast's own fanned-out writes — so sse-write.ts's set-once-wins per-stream +// state captures it immediately, AND to the subscriber itself so a later fanned-out write (e.g. a +// tick or ping) on a stream that never got an early frame still carries it. +function openRelationshipEventsStream( + ctx: RouteContext, + deps: UiHandlerDeps, + res: ServerResponse, + relationship: RelationshipHandlerDeps, + workspaceId: string, +): () => void { const controller = new AbortController(); - writeOrDestroy(res, `retry: ${String(ACTIVITY_SSE_RETRY_MS)}\n: connected\n\n`, controller); - const unsubscribe = subscribeActivityBroadcast( + writeOrDestroy( + res, + `retry: ${String(ACTIVITY_SSE_RETRY_MS)}\n: connected\n\n`, + controller, + undefined, + ctx.correlationId, + ); + return subscribeActivityBroadcast( deps, workspaceId, - { res, controller }, + { + res, + controller, + ...(ctx.correlationId === undefined ? {} : { correlationId: ctx.correlationId }), + }, { collectFrames: (): readonly ActivityFrame[] => collectActivitySnapshots(deps, relationship, workspaceId).map((snapshot) => ({ @@ -1984,11 +2013,6 @@ function handleEventsImpl(ctx: RouteContext, deps: UiHandlerDeps): RouteResult | pingMs: ACTIVITY_SSE_PING_MS, }, ); - ctx.req.on("close", () => { - unsubscribe(); - res.end(); - }); - return STREAMING; } // ─── Exposed handler bindings ───────────────────────────────────────────────── diff --git a/packages/keiko-server/src/server.test.ts b/packages/keiko-server/src/server.test.ts index 2e7b8f055f..55e4025613 100644 --- a/packages/keiko-server/src/server.test.ts +++ b/packages/keiko-server/src/server.test.ts @@ -6,7 +6,7 @@ import { request } from "node:http"; import type { AddressInfo } from "node:net"; import type { IncomingMessage, Server, ServerResponse } from "node:http"; import { gunzipSync } from "node:zlib"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { GatewayConfig, GatewayRequest, @@ -1196,6 +1196,56 @@ describe("activity log: http-request line enrichment (Wave 5, w5-http-request-en expect(event.extra?.aborted).toBe(false); expect(event.status).toBe(200); }); + + // #2902 audit finding 1: computeQueryParamFields used to sort the FULL kept array before + // slicing it to the 16-name cap (MAX_QUERY_PARAM_NAMES, mirrored here — not exported), so a + // client sending far more than 16 conforming query-param names paid an unbounded, locale-aware + // O(n log n) sort. The fix bounds the collection loop at the cap BEFORE sorting. + it("caps the collected set at 16 names before sorting, regardless of how many the client sends", async () => { + const sink = await startWithActivityLog(); + const names = Array.from({ length: 20 }, (_, i) => `p${String(19 - i).padStart(2, "0")}`); + const query = names.map((n) => `${n}=1`).join("&"); + const sortSpy = vi.spyOn(Array.prototype, "sort"); + let event: ServerLogEvent; + let queryNameSorts: string[][]; + try { + await fetchRaw(`/api/health?${query}`); + event = await waitForActivityLogEvent(sink); + // `mock.contexts` captures the `this` receiver of every sort() call — i.e. the array that + // was actually sorted. None of them may be the `p\d\d`-shaped query-name collection at more + // than the 16-name cap: that is the direct proof the bound runs BEFORE the sort, not after. + // Read BEFORE mockRestore(), which also clears the call/context history (mockRestore does + // everything mockReset() does, plus restoring the original implementation). + queryNameSorts = sortSpy.mock.contexts.filter( + (receiver): receiver is string[] => + Array.isArray(receiver) && + receiver.every((v) => typeof v === "string" && /^p\d\d$/.test(v)), + ); + } finally { + sortSpy.mockRestore(); + } + + expect(event.extra?.queryParamNames).toHaveLength(16); + expect(event.extra?.queryParamDroppedCount).toBe(4); + expect(queryNameSorts.length).toBeGreaterThan(0); + for (const receiver of queryNameSorts) { + expect(receiver.length).toBeLessThanOrEqual(16); + } + }); + + // #2902 audit finding 1: computeQueryParamFields was called unconditionally before the + // trust-boundary host check, so a request that fails isAllowedHost still paid the full + // query-string scan even though its result was discarded. It now runs only after the host + // check passes, so a FORBIDDEN_HOST request's own activity-log line carries no query names. + it("carries no query-param names on a request that fails the host check", async () => { + const sink = await startWithActivityLog(); + const res = await rawRequestWithHost("/api/health?foo=1&bar=2", "evil.example.com"); + expect(res.status).toBe(403); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.queryParamNames).toEqual([]); + expect(event.extra?.queryParamDroppedCount).toBeUndefined(); + }); }); // Direct unit coverage of `logRequestOnClose` itself, using the same fake req/res EventEmitter diff --git a/packages/keiko-server/src/server.ts b/packages/keiko-server/src/server.ts index d34b77f93f..1588a99991 100644 --- a/packages/keiko-server/src/server.ts +++ b/packages/keiko-server/src/server.ts @@ -335,22 +335,30 @@ async function resolveCsp(deps: UiServerDeps): Promise { // schema is: a name beyond the cap, and any name that is not a bounded identifier (the same shape // a log field name itself must have), is dropped and COUNTED rather than silently grown or // silently truncated with no trace. +// +// #2902 audit finding 1: the collection loop is bounded at MAX_QUERY_PARAM_NAMES BEFORE the sort +// runs, rather than sorting the full kept set and slicing afterward — a client fully controls the +// query string, and an unbounded sort (locale-aware, O(n log n)) over an attacker-supplied name +// count is a synchronous, amplifiable CPU cost on the request hot path. The remaining per-name work +// (Set membership + regex test) stays a cheap O(n) pass, the same order as the WHATWG URL parsing +// that already runs unconditionally for routing on every request. function computeQueryParamFields(url: URL, context: RequestLogContext): void { - const names = new Set(url.searchParams.keys()); + const seen = new Set(); const kept: string[] = []; let dropped = 0; - for (const name of names) { - if (QUERY_PARAM_NAME_PATTERN.test(name)) { + for (const name of url.searchParams.keys()) { + if (seen.has(name)) continue; + seen.add(name); + if (!QUERY_PARAM_NAME_PATTERN.test(name)) { + dropped += 1; + } else if (kept.length < MAX_QUERY_PARAM_NAMES) { kept.push(name); } else { dropped += 1; } } kept.sort((a, b) => a.localeCompare(b)); - if (kept.length > MAX_QUERY_PARAM_NAMES) { - dropped += kept.length - MAX_QUERY_PARAM_NAMES; - } - context.queryParamNames = kept.slice(0, MAX_QUERY_PARAM_NAMES); + context.queryParamNames = kept; if (dropped > 0) context.queryParamDroppedCount = dropped; } @@ -363,7 +371,6 @@ async function handle( context: RequestLogContext, ): Promise { const url = new URL(req.url ?? "/", `http://${UI_HOST}`); - computeQueryParamFields(url, context); const apiPath = isApiPath(url.pathname); // Issue #495/#497 — scope the Permissions-Policy microphone directive to deployments that advertise // speech-to-text dictation OR full-realtime voice (whose WebRTC capture track also needs the mic); @@ -375,6 +382,9 @@ async function handle( rejectForbiddenHost(req, res, context); return; } + // #2902 audit finding 1: computed only once the trust-boundary host check has passed (AGENTS.md + // "validate before you process") — a FORBIDDEN_HOST request never pays the query-string scan. + computeQueryParamFields(url, context); const method = (req.method ?? "GET").toUpperCase(); if (apiPath) { await dispatchApi(handlerDeps, req, res, method, url, correlationId, context); diff --git a/packages/keiko-server/src/terminal-routes.test.ts b/packages/keiko-server/src/terminal-routes.test.ts index 506de94db7..23ec15258d 100644 --- a/packages/keiko-server/src/terminal-routes.test.ts +++ b/packages/keiko-server/src/terminal-routes.test.ts @@ -24,6 +24,12 @@ import { type TerminalExecutionManager, type TerminalExecutionResult, } from "./index.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, +} from "./observability/index.js"; interface FakeOptions { readonly executeShouldThrow?: TerminalToolError; @@ -644,3 +650,43 @@ describe("openTerminalSseStream backpressure (KEIKO-0142)", () => { expect(fake.writes.length).toBeGreaterThan(1); }); }); + +// Finding 0 (#2902 audit): the request-scoped correlationId never reached the terminal +// `sse.stream.closed` line because openTerminalSseStream had no parameter to receive it, even +// though handleTerminalEvents' RouteContext carries one. The heartbeat is the first write on +// every stream (sse-write.ts's per-stream state is set-once-wins), so threading it through the +// heartbeat's backpressure object is sufficient for the whole stream's terminal line. +describe("openTerminalSseStream correlationId threading (#2902 audit finding 0)", () => { + afterEach(() => { + resetServerLogger(); + }); + + it("attaches the supplied correlationId to the sse.stream.closed terminal line", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const fake = makeFakeSseRes(); + const manager = new FakeTerminalExecutionManager(); + + openTerminalSseStream(fake.res, manager, (value) => value, undefined, "corr-terminal-1"); + fake.emitClose(); + + expect(sink.events).toHaveLength(1); + expect(sink.events[0]).toMatchObject({ + op: "sse.stream.closed", + correlationId: "corr-terminal-1", + }); + }); + + it("omits correlationId from the terminal line when none is supplied (unchanged behavior)", () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const fake = makeFakeSseRes(); + const manager = new FakeTerminalExecutionManager(); + + openTerminalSseStream(fake.res, manager, (value) => value); + fake.emitClose(); + + expect(sink.events).toHaveLength(1); + expect(sink.events[0]?.correlationId).toBeUndefined(); + }); +}); diff --git a/packages/keiko-server/src/terminal-routes.ts b/packages/keiko-server/src/terminal-routes.ts index a153b709b7..96e870b23c 100644 --- a/packages/keiko-server/src/terminal-routes.ts +++ b/packages/keiko-server/src/terminal-routes.ts @@ -227,7 +227,13 @@ export function handleDeleteTerminalExecution(ctx: RouteContext, deps: UiHandler export function handleTerminalEvents(ctx: RouteContext, deps: UiHandlerDeps): HandlerOutcome { const guard = requireTerminal(deps); if (isRouteResult(guard)) return guard; - openTerminalSseStream(ctx.res, guard, deps.redactor, sseBackpressureReporter(deps, "terminal")); + openTerminalSseStream( + ctx.res, + guard, + deps.redactor, + sseBackpressureReporter(deps, "terminal"), + ctx.correlationId, + ); ctx.req.on("close", () => { ctx.res.end(); }); @@ -242,6 +248,7 @@ export function openTerminalSseStream( manager: TerminalExecutionManager, redactor: UiHandlerDeps["redactor"], onBackpressure?: (signal: SseBackpressureSignal) => void, + correlationId?: string, ): void { res.writeHead(200, SSE_HEADERS); // Per-connection abort: a slow-client backpressure kill (writeOrDestroy) aborts this controller, @@ -250,9 +257,14 @@ export function openTerminalSseStream( // subscribe() returns synchronously and events fire only asynchronously afterward, so no event // (hence no abort) can occur before `unsubscribe` is assigned. const controller = new AbortController(); + // correlationId (#2902 w5-sse-counters) is threaded to every write path below so whichever one + // runs first attaches it: sse-write.ts's per-stream state is set-once-wins. The heartbeat's own + // write is deferred to its interval timer, so the ready frame just below is the actual first + // write in practice — it also carries correlationId for that reason. startSseHeartbeat(res, undefined, undefined, { controller, ...(onBackpressure === undefined ? {} : { onBackpressure }), + ...(correlationId === undefined ? {} : { correlationId }), }); let seq = 0; const unsubscribe = manager.subscribe((event) => { @@ -269,7 +281,7 @@ export function openTerminalSseStream( // The ready frame goes through the same protective path: a client that is already not draining // must abort and unsubscribe here too, rather than leaving the subscription live until some // later event happens to trip writeOrDestroy. - writeOrDestroy(res, readyMessage(), controller, onBackpressure); + writeOrDestroy(res, readyMessage(), controller, onBackpressure, correlationId); res.on("close", () => { stop(); }); diff --git a/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts b/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts index 1cbb265ea4..42e7aa3da7 100644 --- a/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts +++ b/packages/keiko-ui/src/app/components/desktop/widgets/cards/sharedEventSource.ts @@ -6,27 +6,10 @@ import { subscribeBrowserStreamCapacity, } from "../../../../../lib/browser-stream-capacity"; import { secureRandomInt } from "../../../../../lib/secure-random"; -import { reportClientDiagnostic } from "../../../../../lib/client-diagnostics"; - -// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, -// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from -// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's -// diagnostic transport as a side effect at import time (by design — see its own header), and none of -// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own -// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against -// all four call sites so the two ends cannot silently drift apart. -type SseStreamCloseReason = "connecting" | "closed" | "unknown"; - -function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { - if (readyState === 0) return "connecting"; - if (readyState === 2) return "closed"; - return "unknown"; -} - -function sseStreamErrorDiagnostic(readyState: number | undefined): string { - const readyStateText = readyState === undefined ? "unknown" : String(readyState); - return `[keiko] shared-event-source sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; -} +import { + reportClientDiagnostic, + sseStreamErrorDiagnostic, +} from "../../../../../lib/client-diagnostics"; type SharedEventListener = (event: MessageEvent) => void; @@ -161,7 +144,7 @@ function openEntrySource(entry: SharedEventSourceEntry): void { entry.reconnectAttempts = 0; }; source.onerror = () => { - reportClientDiagnostic(sseStreamErrorDiagnostic(source.readyState)); + reportClientDiagnostic(sseStreamErrorDiagnostic("shared-event-source", source.readyState)); closeEntrySource(entry); scheduleReconnect(entry); }; diff --git a/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts b/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts index 724bdcce3e..fdd1ef2e40 100644 --- a/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts +++ b/packages/keiko-ui/src/app/components/desktop/widgets/panels/useRelationshipActivityStream.ts @@ -28,30 +28,13 @@ import { useCallback, useEffect, useRef, useState } from "react"; import type { RelationshipActivityState } from "@oscharko-dev/keiko-contracts"; import { RELATIONSHIP_FORBIDDEN_METADATA_KEY_SUBSTRINGS } from "@oscharko-dev/keiko-contracts"; -import { reportClientDiagnostic } from "../../../../../lib/client-diagnostics"; +import { + reportClientDiagnostic, + sseStreamErrorDiagnostic, +} from "../../../../../lib/client-diagnostics"; import { createSameOriginApiEventSource } from "../../../../../lib/safe-event-source"; import { secureRandomInt } from "../../../../../lib/secure-random"; -// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, -// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from -// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's -// diagnostic transport as a side effect at import time (by design — see its own header), and none of -// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own -// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against -// all four call sites so the two ends cannot silently drift apart. -type SseStreamCloseReason = "connecting" | "closed" | "unknown"; - -function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { - if (readyState === 0) return "connecting"; - if (readyState === 2) return "closed"; - return "unknown"; -} - -function sseStreamErrorDiagnostic(readyState: number | undefined): string { - const readyStateText = readyState === undefined ? "unknown" : String(readyState); - return `[keiko] relationship-activity sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; -} - // ─── Constants ───────────────────────────────────────────────────────────────── /** Max concurrent animated badges (activity-state.md §5.3). */ @@ -498,7 +481,7 @@ export function useRelationshipActivityStream( es.onerror = (): void => { if (closed) return; - reportClientDiagnostic(sseStreamErrorDiagnostic(es?.readyState)); + reportClientDiagnostic(sseStreamErrorDiagnostic("relationship-activity", es?.readyState)); closeStream(); scheduleReconnect(); }; diff --git a/packages/keiko-ui/src/lib/client-diagnostics.ts b/packages/keiko-ui/src/lib/client-diagnostics.ts index 172aaac120..ba50021b3d 100644 --- a/packages/keiko-ui/src/lib/client-diagnostics.ts +++ b/packages/keiko-ui/src/lib/client-diagnostics.ts @@ -82,3 +82,23 @@ export function resetClientDiagnosticWriter(): void { writer = bufferUntilTransportArrives; pending.length = 0; } + +type SseStreamCloseReason = "connecting" | "closed" | "unknown"; + +function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { + if (readyState === 0) return "connecting"; + if (readyState === 2) return "closed"; + return "unknown"; +} + +/** + * The one text convention an `EventSource.onerror` site reports through `reportClientDiagnostic`. + * `install-client-diagnostics.ts` owns the matching parser (`SSE_DIAGNOSTIC_MESSAGE_PATTERN`) and + * pins this exact shape in its test; keeping the producer here — in the leaf every SSE consumer + * already imports — means one copy of the convention instead of one per consumer. `stream` is a + * fixed, code-owned label naming the consumer (never user content). + */ +export function sseStreamErrorDiagnostic(stream: string, readyState: number | undefined): string { + const readyStateText = readyState === undefined ? "unknown" : String(readyState); + return `[keiko] ${stream} sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; // i18n-exempt: developer diagnostic for the activity log, never rendered to a person +} diff --git a/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts b/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts index eef6ccafe6..be5236b351 100644 --- a/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts +++ b/packages/keiko-ui/src/lib/coding-workbench-event-retention.ts @@ -10,32 +10,12 @@ import { parseCodingWorkbenchRuntimeEvent, } from "./coding-workbench-runtime-api"; import { reserveInteractiveBrowserStreamCapacity } from "./browser-stream-capacity"; -import { reportClientDiagnostic } from "./client-diagnostics"; +import { reportClientDiagnostic, sseStreamErrorDiagnostic } from "./client-diagnostics"; export const CODING_WORKBENCH_EVENT_RETENTION_LIMIT = 500; export const CODING_WORKBENCH_OBSERVATION_BATCH_MS = 100; export const CODING_WORKBENCH_EVENT_STREAM_STALE_MS = 35_000; -// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, -// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from -// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's -// diagnostic transport as a side effect at import time (by design — see its own header), and none of -// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own -// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against -// all four call sites so the two ends cannot silently drift apart. -type SseStreamCloseReason = "connecting" | "closed" | "unknown"; - -function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { - if (readyState === 0) return "connecting"; - if (readyState === 2) return "closed"; - return "unknown"; -} - -function sseStreamErrorDiagnostic(readyState: number | undefined): string { - const readyStateText = readyState === undefined ? "unknown" : String(readyState); - return `[keiko] coding-workbench-runtime sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; -} - const TERMINAL_STATES = new Set([ "succeeded", "failed", @@ -145,7 +125,9 @@ class RuntimeEventStreamSession implements CodingWorkbenchRuntimeStreamSession { }; source.onerror = (): void => { if (this.source !== source) return; - reportClientDiagnostic(sseStreamErrorDiagnostic(source.readyState)); + reportClientDiagnostic( + sseStreamErrorDiagnostic("coding-workbench-runtime", source.readyState), + ); this.handlers.onError(new Error("The runtime event stream is reconnecting.")); }; const receive: EventListener = (event) => { diff --git a/packages/keiko-ui/src/lib/useSSE.ts b/packages/keiko-ui/src/lib/useSSE.ts index f42389bde4..5f6992f71b 100644 --- a/packages/keiko-ui/src/lib/useSSE.ts +++ b/packages/keiko-ui/src/lib/useSSE.ts @@ -6,7 +6,7 @@ */ import { useEffect, useRef, useState } from "react"; -import { reportClientDiagnostic } from "./client-diagnostics"; +import { reportClientDiagnostic, sseStreamErrorDiagnostic } from "./client-diagnostics"; import { createSameOriginApiEventSource } from "./safe-event-source"; import { secureRandomInt } from "./secure-random"; import { TERMINAL_EVENT_TYPES, type HarnessEvent, type SseStatus } from "./types"; @@ -16,26 +16,6 @@ const RECONNECT_INITIAL_DELAY_MS = 1000; const RECONNECT_MAX_DELAY_MS = 30000; const RUN_EVENTS_URL = "/api/runs/events"; -// Duplicated identically across the four SSE-consuming modules (sharedEventSource.ts, useSSE.ts, -// coding-workbench-event-retention.ts, useRelationshipActivityStream.ts) rather than imported from -// `install-client-diagnostics.ts`, which owns the matching parser: that module installs the app's -// diagnostic transport as a side effect at import time (by design — see its own header), and none of -// these four modules' unit tests may pull that side effect (a real `fetch` attempt) into their own -// module graph. `install-client-diagnostics.ts`'s own test pins the exact convention below against -// all four call sites so the two ends cannot silently drift apart. -type SseStreamCloseReason = "connecting" | "closed" | "unknown"; - -function sseStreamCloseReason(readyState: number | undefined): SseStreamCloseReason { - if (readyState === 0) return "connecting"; - if (readyState === 2) return "closed"; - return "unknown"; -} - -function sseStreamErrorDiagnostic(readyState: number | undefined): string { - const readyStateText = readyState === undefined ? "unknown" : String(readyState); - return `[keiko] run-events sse stream error (kind=sse-error, readyState=${readyStateText}, reason=${sseStreamCloseReason(readyState)})`; -} - export interface UseSSEResult { events: HarnessEvent[]; status: SseStatus; @@ -160,7 +140,7 @@ function openSharedEventSource(): void { }); sharedEventSource.onerror = () => { - reportClientDiagnostic(sseStreamErrorDiagnostic(sharedEventSource?.readyState)); + reportClientDiagnostic(sseStreamErrorDiagnostic("run-events", sharedEventSource?.readyState)); notifyAll("error", "Stream disconnected. Attempting to reconnect…"); closeSharedEventSource(); scheduleReconnect(); diff --git a/scripts/__tests__/correlation-id-pattern-drift.test.mjs b/scripts/__tests__/correlation-id-pattern-drift.test.mjs new file mode 100644 index 0000000000..9d934754e5 --- /dev/null +++ b/scripts/__tests__/correlation-id-pattern-drift.test.mjs @@ -0,0 +1,52 @@ +// #2902 audit finding 2: `install-client-diagnostics.ts` (keiko-ui) hand-copies the server's +// `SAFE_CORRELATION_ID` regex byte-for-byte as its own `CLIENT_CORRELATION_ID_PATTERN` constant, +// because keiko-ui may only depend on the server through the shared contract types (AGENTS.md §4), +// never on a server module directly — so it cannot import the server's declaration the way +// ERROR_KIND_PATTERN was consolidated into `keiko-contracts` (ADR-0173 D11). Relocating this +// pattern into the leaf would conflict with the deliberate, already-documented layering decision in +// `packages/keiko-contracts/src/diagnostics.ts` (the wire-shape guard there intentionally stays +// LOOSER than SAFE_CORRELATION_ID so the leaf never imports server policy) — `install-client- +// diagnostics.ts`'s own header cites that file as precedent for keeping its own copy. So unlike +// ERROR_KIND_PATTERN, this is not consolidated into one declaration; it stays two independent, +// differently-scoped declarations by design (server policy vs. client-side best-effort pre-filter). +// +// What WAS missing is drift protection: no test anywhere referenced either constant name, so +// either literal could be edited alone (e.g. widening the UI copy's length bound) and the full +// suite would stay green — nothing would catch the divergence. This is the weaker of the two +// techniques already used in this repo (a two-file byte-for-byte diff, not a single-declaration +// repo-wide scan) because there legitimately are two declarations here; a single-source-of-truth +// assertion would be the wrong pin to write for this pair. + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = resolve(import.meta.dirname, "..", ".."); +const SERVER_FILE = "packages/keiko-server/src/correlation.ts"; +const CLIENT_FILE = "packages/keiko-ui/src/lib/install-client-diagnostics.ts"; + +// Extracts the regex literal assigned to `constantName` from `source`, e.g. given +// `const SAFE_CORRELATION_ID = /^[A-Za-z0-9._-]{8,128}$/;` and `"SAFE_CORRELATION_ID"`, returns +// `"/^[A-Za-z0-9._-]{8,128}$/"` — the literal text, not a compiled RegExp, so the comparison below +// is character-for-character and trips on a widened character class or changed length bound even +// if the two patterns would still accept/reject the same handful of test strings. +function extractRegexLiteral(source, constantName) { + const pattern = new RegExp(`\\b${constantName}\\s*=\\s*(\\/[^\\n]*\\/)`); + const match = pattern.exec(source); + if (match === null) { + throw new Error(`could not find a declaration of ${constantName} in the given source`); + } + return match[1]; +} + +describe("client/server correlation-id pattern drift (#2902 audit finding 2)", () => { + it("keeps CLIENT_CORRELATION_ID_PATTERN character-for-character identical to SAFE_CORRELATION_ID", () => { + const serverSource = readFileSync(resolve(REPO_ROOT, SERVER_FILE), "utf8"); + const clientSource = readFileSync(resolve(REPO_ROOT, CLIENT_FILE), "utf8"); + + const serverLiteral = extractRegexLiteral(serverSource, "SAFE_CORRELATION_ID"); + const clientLiteral = extractRegexLiteral(clientSource, "CLIENT_CORRELATION_ID_PATTERN"); + + expect(clientLiteral).toBe(serverLiteral); + }); +}); From d1cf8f45d44df9cbf16111ac0e827f4b9c642d44 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 11:21:02 +0200 Subject: [PATCH 06/19] test(coverage): regenerate the package coverage baseline for the Wave 5 sources (#3240) Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index 55abc03d2f..120346a882 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,7 +8,7 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 387, + "uncoveredLines": 385, "totalLines": 4880, "coverage": { "lines": 92.11, @@ -30,14 +30,14 @@ } }, "keiko-contracts": { - "files": 186, + "files": 187, "uncoveredFiles": 0, "uncoveredLines": 857, - "totalLines": 13852, + "totalLines": 13875, "coverage": { - "lines": 93.81, - "statements": 92.39, - "branches": 90.23, + "lines": 93.82, + "statements": 92.41, + "branches": 90.25, "functions": 97.46 } }, @@ -234,15 +234,15 @@ } }, "keiko-server": { - "files": 580, + "files": 581, "uncoveredFiles": 0, - "uncoveredLines": 4516, - "totalLines": 56239, + "uncoveredLines": 4485, + "totalLines": 56247, "coverage": { - "lines": 91.97, - "statements": 89.27, - "branches": 81.88, - "functions": 94.86 + "lines": 92.03, + "statements": 89.32, + "branches": 81.95, + "functions": 94.9 } }, "keiko-tools": { @@ -260,13 +260,13 @@ "keiko-ui": { "files": 418, "uncoveredFiles": 3, - "uncoveredLines": 2969, - "totalLines": 39424, + "uncoveredLines": 2962, + "totalLines": 39477, "coverage": { - "lines": 92.47, - "statements": 89.52, - "branches": 81.8, - "functions": 91.28 + "lines": 92.5, + "statements": 89.54, + "branches": 81.82, + "functions": 91.29 } }, "keiko-verification": { From 2164b6e9e4b620f4a3ed9969e0537eefc70dc246 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 11:38:27 +0200 Subject: [PATCH 07/19] test(coverage): regenerate the package coverage baseline after the audit repairs and the dev merge (#3239) Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 79 ++++++++++++-------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index f81af9a6b0..d7ed06f3df 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,13 +8,13 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 387, - "totalLines": 4880, + "uncoveredLines": 383, + "totalLines": 4883, "coverage": { - "lines": 92.11, - "statements": 90.39, + "lines": 92.16, + "statements": 90.45, "branches": 85.19, - "functions": 93.23 + "functions": 93.71 } }, "keiko-connectors": { @@ -30,15 +30,15 @@ } }, "keiko-contracts": { - "files": 186, + "files": 187, "uncoveredFiles": 0, "uncoveredLines": 857, - "totalLines": 13852, + "totalLines": 13877, "coverage": { - "lines": 93.81, - "statements": 92.39, - "branches": 90.23, - "functions": 97.46 + "lines": 93.82, + "statements": 92.41, + "branches": 90.26, + "functions": 97.47 } }, "keiko-editor": { @@ -105,12 +105,12 @@ "files": 128, "uncoveredFiles": 0, "uncoveredLines": 754, - "totalLines": 9383, + "totalLines": 9419, "coverage": { - "lines": 91.97, - "statements": 89.58, - "branches": 80.95, - "functions": 94.31 + "lines": 91.99, + "statements": 89.62, + "branches": 80.96, + "functions": 94.34 } }, "keiko-memory-capture": { @@ -162,15 +162,15 @@ } }, "keiko-memory-vault": { - "files": 22, + "files": 23, "uncoveredFiles": 0, - "uncoveredLines": 78, - "totalLines": 953, + "uncoveredLines": 85, + "totalLines": 1046, "coverage": { - "lines": 91.82, - "statements": 90.46, - "branches": 85.55, - "functions": 91.34 + "lines": 91.87, + "statements": 90.6, + "branches": 85.64, + "functions": 91.48 } }, "keiko-model-gateway": { @@ -222,27 +222,27 @@ } }, "keiko-security": { - "files": 23, + "files": 24, "uncoveredFiles": 0, - "uncoveredLines": 11, - "totalLines": 616, + "uncoveredLines": 0, + "totalLines": 661, "coverage": { - "lines": 98.21, - "statements": 97.62, - "branches": 94.08, - "functions": 98.6 + "lines": 100, + "statements": 99.86, + "branches": 98.96, + "functions": 100 } }, "keiko-server": { - "files": 580, + "files": 581, "uncoveredFiles": 0, "uncoveredLines": 4516, - "totalLines": 56260, + "totalLines": 56343, "coverage": { "lines": 91.98, - "statements": 89.27, - "branches": 81.89, - "functions": 94.86 + "statements": 89.28, + "branches": 81.9, + "functions": 94.87 } }, "keiko-tools": { @@ -261,10 +261,10 @@ "files": 419, "uncoveredFiles": 3, "uncoveredLines": 2967, - "totalLines": 39452, + "totalLines": 39455, "coverage": { "lines": 92.48, - "statements": 89.52, + "statements": 89.53, "branches": 81.8, "functions": 91.28 } @@ -342,11 +342,6 @@ "tolerance": 0, "lines": 90 }, - "packages/keiko-security/src/errors/harness.ts": { - "governance": "ratcheted", - "tolerance": 0.5, - "lines": 16.17 - }, "packages/keiko-server/src/editor/dap/dapCapsuleSupervisor.ts": { "governance": "absolute", "tolerance": 0, From c29216e43ac29003ebec6ef7fac9c174fe1b666a Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 11:47:44 +0200 Subject: [PATCH 08/19] fix(observability): reviewer-audit repairs for the merged logging waves (#3233) - The stderr diagnostic line uses the same allowlist projection as the activity-log line instead of serializing the raw record. - Memory-retrieval degradation signals and the Figma snapshot internal failure route through emitServerDiagnostic instead of console.*. - A directly launched keiko ui sets the real KEIKO_STATE_DIR so a crash still writes its process.fatal line. - The support bundle records budgetExceeded and exports only the tail of an oversized current log file through a bounded reader (currentFileTailTruncated), never the whole file into memory. - The op-catalog generator resolves a call's object-argument category instead of attributing 29 entries to unknown; catalog and coverage baseline regenerated with no floor lowered. Co-Authored-By: Claude Fable 5 --- ...-log-v2-machine-reconstruction-contract.md | 6 +- docs/observability/op-catalog.generated.json | 44 ++--- docs/qa/package-coverage-baseline.json | 14 +- packages/keiko-cli/src/support-export.test.ts | 177 ++++++++++++++++++ packages/keiko-cli/src/support-export.ts | 157 ++++++++++++++-- packages/keiko-cli/src/support.test.ts | 48 +++++ packages/keiko-cli/src/support.ts | 17 +- packages/keiko-cli/src/ui.test.ts | 20 ++ packages/keiko-cli/src/ui.ts | 12 ++ .../src/diagnostics-log.activity-log.test.ts | 41 ++++ packages/keiko-server/src/diagnostics-log.ts | 22 ++- packages/keiko-server/src/grounded-qa.test.ts | 16 +- .../src/memory-retrieval-signals.test.ts | 112 ++++++++--- .../src/memory-retrieval-signals.ts | 69 ++++--- .../figmaSnapshotRoutes.test.ts | 20 +- .../figmaSnapshotRoutes.ts | 20 +- scripts/__tests__/op-catalog-drift.test.mjs | 13 ++ scripts/generate-op-catalog.mjs | 85 ++++++++- 18 files changed, 776 insertions(+), 117 deletions(-) diff --git a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md index 9cad737238..e7139d6b40 100644 --- a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md +++ b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md @@ -337,7 +337,11 @@ against a `stateDir` whose history predates that fix. **Size bounds.** Capped at the sink's own retention window, further capped by an overall byte ceiling; files are dropped oldest-first when the ceiling is exceeded, and every drop is recorded in -the manifest's `truncatedLogFiles` — never silent. +the manifest's `truncatedLogFiles` — never silent. The current (never-dropped) file is not exempt +from the ceiling: when it alone still exceeds the residual budget, only its tail is exported — +the newest bytes, advanced to the next line boundary so the first exported line is always +complete — read with a bounded reader rather than loading the whole oversized file, and recorded +in the manifest's `currentFileTailTruncated` (name and dropped-byte count only, never a path). ### D9 — CLI surface: `keiko support export` and `keiko support analyze` diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index d22b3c042f..191a8a1458 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -148,7 +148,7 @@ }, { "op": "embedding.preflight.identity-rejected", - "category": "unknown", + "category": "embedding", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2692", "package": "keiko-local-knowledge" }, @@ -160,103 +160,103 @@ }, { "op": "indexing.chunking.failed", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1425", "package": "keiko-local-knowledge" }, { "op": "indexing.chunking.failed", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1763", "package": "keiko-local-knowledge" }, { "op": "indexing.chunking.failed", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:967", "package": "keiko-local-knowledge" }, { "op": "indexing.discovery.limit-reached", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2868", "package": "keiko-local-knowledge" }, { "op": "indexing.discovery.scope-error", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2216", "package": "keiko-local-knowledge" }, { "op": "indexing.document.chunked", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1302", "package": "keiko-local-knowledge" }, { "op": "indexing.document.embedded", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1498", "package": "keiko-local-knowledge" }, { "op": "indexing.document.embedding-started", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1054", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extracted", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1298", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extracted", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1985", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extraction-failed", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1094", "package": "keiko-local-knowledge" }, { "op": "indexing.document.extraction-started", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2202", "package": "keiko-local-knowledge" }, { "op": "indexing.document.failed", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1467", "package": "keiko-local-knowledge" }, { "op": "indexing.document.skipped", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1067", "package": "keiko-local-knowledge" }, { "op": "indexing.document.skipped", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1116", "package": "keiko-local-knowledge" }, { "op": "indexing.document.skipped", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:1988", "package": "keiko-local-knowledge" }, { "op": "indexing.job.finished", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:3193", "package": "keiko-local-knowledge" }, @@ -268,19 +268,19 @@ }, { "op": "indexing.job.started", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2946", "package": "keiko-local-knowledge" }, { "op": "indexing.source.completed", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2325", "package": "keiko-local-knowledge" }, { "op": "indexing.source.started", - "category": "unknown", + "category": "indexing", "site": "packages/keiko-local-knowledge/src/indexing/orchestrator.ts:2307", "package": "keiko-local-knowledge" }, @@ -593,7 +593,7 @@ { "op": "", "category": "diagnostic", - "site": "packages/keiko-server/src/diagnostics-log.ts:213", + "site": "packages/keiko-server/src/diagnostics-log.ts:231", "package": "keiko-server" }, { diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index 55abc03d2f..6141ffbde4 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,12 +8,12 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 387, - "totalLines": 4880, + "uncoveredLines": 385, + "totalLines": 4915, "coverage": { - "lines": 92.11, - "statements": 90.39, - "branches": 85.19, + "lines": 92.17, + "statements": 90.44, + "branches": 85.23, "functions": 93.23 } }, @@ -236,8 +236,8 @@ "keiko-server": { "files": 580, "uncoveredFiles": 0, - "uncoveredLines": 4516, - "totalLines": 56239, + "uncoveredLines": 4515, + "totalLines": 56242, "coverage": { "lines": 91.97, "statements": 89.27, diff --git a/packages/keiko-cli/src/support-export.test.ts b/packages/keiko-cli/src/support-export.test.ts index c97eb664d6..9e8421df9a 100644 --- a/packages/keiko-cli/src/support-export.test.ts +++ b/packages/keiko-cli/src/support-export.test.ts @@ -19,6 +19,18 @@ import { type LogFileInfo, } from "./support-export.js"; +// A fixed-width line so a chosen `--max-bytes` cuts it at a known, deterministic byte offset: +// `{"seq":000}` is exactly 11 bytes for every index in [0, 999], so a file of `count` such lines +// (each followed by "\n") is exactly `count * 12` bytes, and any byte offset within it can be +// reasoned about without measuring the file after the fact. +function fixedWidthLine(index: number): string { + return `{"seq":${String(index).padStart(3, "0")}}`; +} + +function fixedWidthLogText(count: number): string { + return `${Array.from({ length: count }, (_, i) => fixedWidthLine(i)).join("\n")}\n`; +} + const HEALTHY_AUDIT: AuditResult = { ok: true, stateDir: "/tmp/example/.keiko", @@ -44,6 +56,8 @@ function baseManifestInput( stateDirSource: "default", sourceLogFiles: [], truncatedLogFiles: [], + currentFileTailTruncated: undefined, + budgetExceeded: false, skippedLogFiles: [], auditSummary: HEALTHY_AUDIT, evidenceIndexCount: 0, @@ -125,6 +139,7 @@ describe("selectLogFilesWithinBudget", () => { expect(selection.kept).toEqual(files); expect(selection.truncatedLogFiles).toEqual([]); + expect(selection.budgetExceeded).toBe(false); }); it("drops the oldest files first, recording their names, never truncating the current file", () => { @@ -143,6 +158,9 @@ describe("selectLogFilesWithinBudget", () => { "server-2026-08-20.log", ]); expect(selection.kept.map((f) => f.name)).toEqual(["server.log"]); + // The residual `kept` total (server.log's 10 bytes) is back under the 15-byte budget once the + // oldest files were dropped, so the budget WAS honoured in the end. + expect(selection.budgetExceeded).toBe(false); }); it("never drops the last remaining file even if it alone exceeds the budget", () => { @@ -152,6 +170,22 @@ describe("selectLogFilesWithinBudget", () => { expect(selection.kept).toEqual(files); expect(selection.truncatedLogFiles).toEqual([]); + // Nothing was left to drop, yet the surviving file alone (1000 bytes) is still over the + // 10-byte budget: `truncatedLogFiles` alone would read identically to "everything fit" + // without this flag. This is the SIZE-only, pre-tail signal — see `readKeptFiles`'s own + // `budgetExceeded` below the current file's tail is what decides the manifest's final value. + expect(selection.budgetExceeded).toBe(true); + // Tells `readKeptFiles` to give this file's own tail reader the full 10-byte budget, since + // nothing else in `kept` shares it. + expect(selection.currentFileTailBudgetBytes).toBe(10); + }); + + it("leaves currentFileTailBudgetBytes undefined when every kept file fits in full", () => { + const files = [fileInfo("server-2026-08-19.log", 10), fileInfo("server.log", 10)]; + + const selection = selectLogFilesWithinBudget(files, 100); + + expect(selection.currentFileTailBudgetBytes).toBeUndefined(); }); it("the default budget is 50MB", () => { @@ -267,6 +301,120 @@ describe("readKeptFiles", () => { }); }); +// Regression for #2902 PR review, follow-up finding: `selectLogFilesWithinBudget`'s "never drop +// the last file" rule kept a single oversized current `server.log` in full, exceeding +// `--max-bytes`, and `readVerbatimLogLines` read it with a whole-file `readFileSync`. These tests +// exercise the fix's contract directly on `readKeptFiles`: only the LAST kept file's tail is read +// (bounded, never the whole file), the kept content always starts on a complete line, and the +// manifest-facing `budgetExceeded` reflects whether that tail read actually rescued the export. +describe("readKeptFiles — current-file tail truncation", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "keiko-support-export-tail-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("reads only the current file's tail when it alone exceeds the budget, starting on a complete line and staying within budget", () => { + const lineCount = 20; + const text = fixedWidthLogText(lineCount); // 12 bytes/line (11 + "\n") = 240 bytes total + const path = join(dir, CURRENT_LOG_FILE_NAME); + writeFileSync(path, text); + const sizeBytes = Buffer.byteLength(text, "utf8"); + const tailBudgetBytes = 30; // < sizeBytes; cuts mid-line, so the boundary advance is exercised + + const result = readKeptFiles( + [{ name: CURRENT_LOG_FILE_NAME, path, sizeBytes }], + tailBudgetBytes, + ); + + // The first kept line is one of the file's own complete lines, never a partial JSON fragment. + expect(result.contentLines.length).toBeGreaterThan(0); + expect(result.contentLines[0]).toMatch(/^\{"seq":\d{3}\}$/); + // Every kept line is the file's tail, in original (oldest-first) order. + expect(result.contentLines).toEqual([fixedWidthLine(18), fixedWidthLine(19)]); + const keptBytes = Buffer.byteLength(`${result.contentLines.join("\n")}\n`, "utf8"); + expect(keptBytes).toBeLessThanOrEqual(tailBudgetBytes); + // (a) the manifest fact is set, name only, with the exact dropped-byte count. + expect(result.currentFileTailTruncated).toEqual({ + name: CURRENT_LOG_FILE_NAME, + droppedBytes: sizeBytes - keptBytes, + }); + // (b) the tail strategy rescued the export, so the manifest must not claim the budget failed. + expect(result.budgetExceeded).toBe(false); + expect(result.skippedLogFiles).toEqual([]); + }); + + it("keeps every older file's read in full and only tail-reads the current (last) file", () => { + const rotatedPath = join(dir, "server-2026-08-19.log"); + const currentPath = join(dir, CURRENT_LOG_FILE_NAME); + writeFileSync(rotatedPath, '{"seq":"old"}\n'); + const currentText = fixedWidthLogText(20); + writeFileSync(currentPath, currentText); + const currentSizeBytes = Buffer.byteLength(currentText, "utf8"); + + const result = readKeptFiles( + [ + { name: "server-2026-08-19.log", path: rotatedPath, sizeBytes: 0 }, + { name: CURRENT_LOG_FILE_NAME, path: currentPath, sizeBytes: currentSizeBytes }, + ], + 30, + ); + + expect(result.contentLines[0]).toBe('{"seq":"old"}'); + expect(result.contentLines.slice(1)).toEqual([fixedWidthLine(18), fixedWidthLine(19)]); + expect(result.currentFileTailTruncated?.name).toBe(CURRENT_LOG_FILE_NAME); + expect(result.budgetExceeded).toBe(false); + }); + + // (c) A budget smaller than a single line — here, a file that is one giant line with no + // newline anywhere at all, so no byte offset within it can ever start a complete line. + it("keeps an empty tail and reports budgetExceeded when the budget is smaller than one line", () => { + const path = join(dir, CURRENT_LOG_FILE_NAME); + const text = `{"seq":"${"x".repeat(1_000)}"}`; // one line, no trailing newline anywhere + writeFileSync(path, text); + const sizeBytes = Buffer.byteLength(text, "utf8"); + + const result = readKeptFiles([{ name: CURRENT_LOG_FILE_NAME, path, sizeBytes }], 5); + + expect(result.contentLines).toEqual([]); + expect(result.currentFileTailTruncated).toEqual({ + name: CURRENT_LOG_FILE_NAME, + droppedBytes: sizeBytes, + }); + expect(result.budgetExceeded).toBe(true); + }); + + it("never attempts a tail read, and never sets budgetExceeded, when currentFileTailBudgetBytes is undefined", () => { + const path = join(dir, CURRENT_LOG_FILE_NAME); + writeFileSync(path, fixedWidthLogText(5)); + + const result = readKeptFiles([{ name: CURRENT_LOG_FILE_NAME, path, sizeBytes: 0 }]); + + expect(result.contentLines).toHaveLength(5); + expect(result.currentFileTailTruncated).toBeUndefined(); + expect(result.budgetExceeded).toBe(false); + }); + + // Same vanish-before-read race `readVerbatimLogLinesOrSkip` already guards against, exercised on + // the bounded tail-reader path instead: the file selected for a tail read can still disappear + // before `openSync` runs. + it("skips the current file, recording its name and error kind, when it vanishes before the tail read", () => { + const path = join(dir, CURRENT_LOG_FILE_NAME); + writeFileSync(path, fixedWidthLogText(5)); + rmSync(path); + + const result = readKeptFiles([{ name: CURRENT_LOG_FILE_NAME, path, sizeBytes: 1_000 }], 30); + + expect(result.contentLines).toEqual([]); + expect(result.currentFileTailTruncated).toBeUndefined(); + expect(result.skippedLogFiles).toEqual([{ name: CURRENT_LOG_FILE_NAME, errorKind: "ENOENT" }]); + }); +}); + describe("describeErrorKind", () => { it("reports the fs error's code when it has one, never the message or a path", () => { const error = Object.assign(new Error("ENOENT: no such file or directory, open '/secret'"), { @@ -316,6 +464,7 @@ describe("buildSupportBundleManifest", () => { redactionAttested: true, sourceLogFiles: ["server-2026-08-20.log", "server.log"], truncatedLogFiles: ["server-2026-08-18.log"], + budgetExceeded: false, skippedLogFiles: [{ name: "server-2026-08-17.log", errorKind: "ENOENT" }], sectionsExcluded: [], auditSummary: REDACTED_HEALTHY_AUDIT, @@ -347,6 +496,34 @@ describe("buildSupportBundleManifest", () => { ); }); + // `budgetExceeded` propagates verbatim from `ManifestInput` — the caller (`support.ts`) is the + // one place that resolves it from `readKeptFiles`'s post-tail result, so by the time it reaches + // this function it is already the authoritative value: true only when even a tail read of the + // current file could not keep a single complete line (see `ReadKeptFilesResult.budgetExceeded`). + it("propagates budgetExceeded from ManifestInput", () => { + expect( + buildSupportBundleManifest(baseManifestInput({ budgetExceeded: true })).budgetExceeded, + ).toBe(true); + expect( + buildSupportBundleManifest(baseManifestInput({ budgetExceeded: false })).budgetExceeded, + ).toBe(false); + }); + + // `currentFileTailTruncated` propagates verbatim from `ManifestInput`, same as + // `truncatedLogFiles` — the distinct, machine-readable fact that the current file's tail (not + // its whole content) was exported, naming only the file and the byte count cut, never a path. + it("propagates currentFileTailTruncated from ManifestInput", () => { + const fact = { name: CURRENT_LOG_FILE_NAME, droppedBytes: 123 }; + expect( + buildSupportBundleManifest(baseManifestInput({ currentFileTailTruncated: fact })) + .currentFileTailTruncated, + ).toEqual(fact); + expect( + buildSupportBundleManifest(baseManifestInput({ currentFileTailTruncated: undefined })) + .currentFileTailTruncated, + ).toBeUndefined(); + }); + it("always leaves sectionsExcluded empty in this minimal version", () => { expect(buildSupportBundleManifest(baseManifestInput()).sectionsExcluded).toEqual([]); }); diff --git a/packages/keiko-cli/src/support-export.ts b/packages/keiko-cli/src/support-export.ts index 9e1bee47cd..0397c807ef 100644 --- a/packages/keiko-cli/src/support-export.ts +++ b/packages/keiko-cli/src/support-export.ts @@ -14,10 +14,17 @@ // audit/evidence subsystems; this file owns everything that can be exercised without touching // argv, process.*, or another package's runtime. -import { readFileSync, readdirSync, statSync } from "node:fs"; +import { closeSync, openSync, readFileSync, readSync, readdirSync, statSync } from "node:fs"; import { join } from "node:path"; import type { AuditResult } from "./audit.js"; +// The one byte that ends a log line in this format (server-log.ts's own file sink writes ASCII +// "\n" between JSON objects, never inside one — any literal 0x0A byte in the file is therefore a +// real line boundary, never a raw newline embedded in a JSON string field, which is always escaped +// as the two-character sequence "\\n" at write time). Reading a tail region and splitting on this +// byte is exactly as safe as `readVerbatimLogLines`'s existing full-file `split("\n")`. +const NEWLINE_BYTE = 0x0a; + export const CURRENT_LOG_FILE_NAME = "server.log"; // The sink's own DEFAULT_LOG_RETENTION_DAYS (7) keeps at most a week of rotated files on disk, so @@ -138,13 +145,26 @@ export function discoverServerLogFiles(logsDir: string): LogFileDiscovery { export interface LogFileSelection { readonly kept: readonly LogFileInfo[]; readonly truncatedLogFiles: readonly string[]; + // Set when the current (never-dropped) file alone still exceeds the residual budget once every + // droppable file has been dropped: the byte budget `readKeptFiles` should give that file's own + // tail reader instead of reading it whole. `undefined` when every kept file already fits + // `maxBytes` read in full — nothing needs a tail read. + readonly currentFileTailBudgetBytes: number | undefined; + // True when the surviving `kept` files, read in FULL, would still total more than `maxBytes` — + // i.e. exactly when `currentFileTailBudgetBytes` is set (by construction, only the single + // never-dropped file can be left once this is true). This is the SIZE-only, pre-tail signal; + // `readKeptFiles`'s own `budgetExceeded` is the authoritative, post-tail value the manifest uses, + // since a tail read can still bring the export back under budget. + readonly budgetExceeded: boolean; } // Drops the OLDEST files first when the combined size would exceed maxBytes, and never drops the // last (current) file: dropping the most recent evidence instead of the least recent, or dropping -// silently, would defeat the whole point of naming what was truncated. maxBytes bounds the sum of -// the copied log-file bytes only — the one manifest line is a small, roughly-constant addition on -// top of that budget, not subtracted from it. +// silently, would defeat the whole point of naming what was truncated. When the current file alone +// still exceeds `maxBytes` after every droppable file is gone, it is not read in full either — +// `currentFileTailBudgetBytes` tells `readKeptFiles` to keep only that file's tail instead. maxBytes +// bounds the sum of the copied log-file bytes only — the one manifest line is a small, +// roughly-constant addition on top of that budget, not subtracted from it. export function selectLogFilesWithinBudget( files: readonly LogFileInfo[], maxBytes: number, @@ -158,7 +178,9 @@ export function selectLogFilesWithinBudget( truncatedLogFiles.push(dropped.name); total -= dropped.sizeBytes; } - return { kept, truncatedLogFiles }; + const budgetExceeded = total > maxBytes; + const currentFileTailBudgetBytes = budgetExceeded && kept.length === 1 ? maxBytes : undefined; + return { kept, truncatedLogFiles, currentFileTailBudgetBytes, budgetExceeded }; } // Splits raw file bytes into lines, dropping only the single empty artifact a trailing newline @@ -187,10 +209,74 @@ function readVerbatimLogLinesOrSkip( } } +// One current (never-dropped) file whose full content did not fit `--max-bytes`, so only its tail +// (the newest bytes) was exported instead of the whole file. `droppedBytes` counts that file's own +// leading bytes that were cut to make the tail fit — name only, never the file's absolute path +// (AGENTS.md §7). Distinct from `truncatedLogFiles`: those files were dropped WHOLE for the size +// budget; this one file was kept, just not in full. +export interface CurrentFileTailTruncated { + readonly name: string; + readonly droppedBytes: number; +} + +interface TailReadOutcome { + readonly lines: readonly string[]; + readonly droppedBytes: number; +} + +// Reads only the last `tailBudgetBytes` bytes of `path` via a bounded `openSync`/`readSync` pair — +// never `readFileSync`'ing the whole (potentially oversized) file — then advances past the first +// newline inside that region so the kept content always starts on a complete line, never a partial +// JSON line. When the region contains no newline at all (the budget is smaller than a single +// line, or the region's only newline is the file's own final byte), nothing can be kept safely and +// `lines` is empty with `droppedBytes` equal to the whole file size. +function readTailLines(path: string, sizeBytes: number, tailBudgetBytes: number): TailReadOutcome { + const regionLength = Math.max(0, Math.min(tailBudgetBytes, sizeBytes)); + const regionStart = sizeBytes - regionLength; + const buffer = Buffer.alloc(regionLength); + const fd = openSync(path, "r"); + try { + if (regionLength > 0) readSync(fd, buffer, 0, regionLength, regionStart); + } finally { + closeSync(fd); + } + const newlineIndex = buffer.indexOf(NEWLINE_BYTE); + if (newlineIndex === -1) return { lines: [], droppedBytes: sizeBytes }; + const droppedBytes = regionStart + newlineIndex + 1; + const keptText = buffer.toString("utf8", newlineIndex + 1); + if (keptText.length === 0) return { lines: [], droppedBytes }; + const lines = keptText.split("\n"); + if (lines.at(-1) === "") lines.pop(); + return { lines, droppedBytes }; +} + +// Same vanish-before-read race as `readVerbatimLogLinesOrSkip`, for the bounded tail reader. +function readTailLinesOrSkip( + path: string, + sizeBytes: number, + tailBudgetBytes: number, +): TailReadOutcome | { readonly skip: string } { + try { + return readTailLines(path, sizeBytes, tailBudgetBytes); + } catch (error) { + return { skip: describeErrorKind(error) }; + } +} + export interface ReadKeptFilesResult { readonly contentLines: readonly string[]; // Relative names only, from `LogFileInfo.name` — never the absolute `LogFileInfo.path`. readonly skippedLogFiles: readonly SkippedLogFile[]; + // Set when the current file's tail was read instead of its full content — see + // `CurrentFileTailTruncated`. `undefined` when no tail read was attempted at all. + readonly currentFileTailTruncated: CurrentFileTailTruncated | undefined; + // True only when a tail read WAS attempted and could not keep even one complete line inside the + // budget (e.g. the budget is smaller than one line) — the one case a tail read cannot rescue. + // False whenever no tail read was attempted, and false when the tail read kept >=1 line: the + // export is within budget in both of those cases. This is the manifest's authoritative, + // post-tail `budgetExceeded` value — see `LogFileSelection.budgetExceeded` for the earlier, + // size-only signal this one supersedes. + readonly budgetExceeded: boolean; } // Reads every kept log file's bytes verbatim, in file order, tolerating the same @@ -200,18 +286,51 @@ export interface ReadKeptFilesResult { // `contentLines.push(...fileLines)`: a spread of a large array as call arguments can throw // `RangeError: Maximum call stack size exceeded` (observed at ~262k elements on Node v24), and a // single oversized rotated log file must not abort the whole export. -export function readKeptFiles(keptFiles: readonly LogFileInfo[]): ReadKeptFilesResult { +// +// `currentFileTailBudgetBytes` (from `LogFileSelection`) applies only to the LAST file in +// `keptFiles` — by construction the one file `selectLogFilesWithinBudget` never drops — and only +// that file is read with the bounded tail reader instead of `readVerbatimLogLines`'s whole-file +// read; every other kept file is read in full exactly as before. +export function readKeptFiles( + keptFiles: readonly LogFileInfo[], + currentFileTailBudgetBytes?: number, +): ReadKeptFilesResult { const contentLines: string[] = []; const skippedLogFiles: SkippedLogFile[] = []; - for (const file of keptFiles) { - const lookup = readVerbatimLogLinesOrSkip(file.path); - if ("lines" in lookup) { - for (const line of lookup.lines) contentLines.push(line); - } else { + let currentFileTailTruncated: CurrentFileTailTruncated | undefined; + let budgetExceeded = false; + const lastIndex = keptFiles.length - 1; + + for (const [index, file] of keptFiles.entries()) { + const tailBudget = index === lastIndex ? currentFileTailBudgetBytes : undefined; + const lookup = readKeptFileLines(file, tailBudget); + if ("skip" in lookup) { skippedLogFiles.push({ name: file.name, errorKind: lookup.skip }); + continue; + } + for (const line of lookup.lines) contentLines.push(line); + if (lookup.tail !== undefined) { + currentFileTailTruncated = lookup.tail; + budgetExceeded = lookup.lines.length === 0; } } - return { contentLines, skippedLogFiles }; + return { contentLines, skippedLogFiles, currentFileTailTruncated, budgetExceeded }; +} + +type KeptFileLookup = + | { readonly lines: readonly string[]; readonly tail: CurrentFileTailTruncated | undefined } + | { readonly skip: SkippedLogFile["errorKind"] }; + +// One kept file's lines: the bounded tail when a tail budget applies (only ever the last, never- +// dropped file), the whole file otherwise. `tail` is set exactly when the tail reader ran. +function readKeptFileLines(file: LogFileInfo, tailBudgetBytes: number | undefined): KeptFileLookup { + if (tailBudgetBytes === undefined) { + const whole = readVerbatimLogLinesOrSkip(file.path); + return "skip" in whole ? whole : { lines: whole.lines, tail: undefined }; + } + const tail = readTailLinesOrSkip(file.path, file.sizeBytes, tailBudgetBytes); + if ("skip" in tail) return tail; + return { lines: tail.lines, tail: { name: file.name, droppedBytes: tail.droppedBytes } }; } // What the manifest is allowed to say about the audit: everything EXCEPT `stateDir`. `AuditResult` @@ -243,6 +362,16 @@ export interface SupportBundleManifest { readonly redactionAttested: true; readonly sourceLogFiles: readonly string[]; readonly truncatedLogFiles: readonly string[]; + // Set when the current (never-dropped) log file's full content did not fit `--max-bytes`, so + // only its tail (the newest bytes, advanced to the next line boundary so the first exported line + // is always complete) was exported instead of the whole file. `undefined` when every log file + // was exported in full. See `CurrentFileTailTruncated` and `ReadKeptFilesResult`. + readonly currentFileTailTruncated: CurrentFileTailTruncated | undefined; + // True only when even a tail read of the current file could not keep a single complete line + // inside `--max-bytes` (e.g. the budget is smaller than one line) — the one case the tail + // strategy above cannot rescue. False whenever the export (in full, or via a successful tail + // read) fits `--max-bytes`. See `ReadKeptFilesResult.budgetExceeded`. + readonly budgetExceeded: boolean; // Files a directory listing named but that had vanished (the sink's own rotation/retention // pruning) by the time this export tried to size or read them — named (never the files' // absolute paths) alongside the fs error kind that caused the skip. Distinct from @@ -277,6 +406,8 @@ export interface ManifestInput { readonly stateDirSource: "default" | "env-override"; readonly sourceLogFiles: readonly string[]; readonly truncatedLogFiles: readonly string[]; + readonly currentFileTailTruncated: CurrentFileTailTruncated | undefined; + readonly budgetExceeded: boolean; readonly skippedLogFiles: readonly SkippedLogFile[]; readonly auditSummary: AuditResult; readonly evidenceIndexCount: number; @@ -297,6 +428,8 @@ export function buildSupportBundleManifest(input: ManifestInput): SupportBundleM redactionAttested: true, sourceLogFiles: input.sourceLogFiles, truncatedLogFiles: input.truncatedLogFiles, + currentFileTailTruncated: input.currentFileTailTruncated, + budgetExceeded: input.budgetExceeded, skippedLogFiles: input.skippedLogFiles, sectionsExcluded: [], auditSummary: redactedAuditSummary(input.auditSummary), diff --git a/packages/keiko-cli/src/support.test.ts b/packages/keiko-cli/src/support.test.ts index 70a605b2b6..0dde2d744d 100644 --- a/packages/keiko-cli/src/support.test.ts +++ b/packages/keiko-cli/src/support.test.ts @@ -12,6 +12,7 @@ import { SERVER_LOG_SCHEMA_VERSION } from "@oscharko-dev/keiko-server"; import type { AuditResult } from "./audit.js"; import type { CliIo } from "./runner.js"; import { parseSupportArgs, runSupportCli, type SupportCliDeps } from "./support.js"; +import { CURRENT_LOG_FILE_NAME } from "./support-export.js"; function makeIo(): { io: CliIo; out: () => string; err: () => string } { const outChunks: string[] = []; @@ -261,6 +262,53 @@ describe("runSupportCli export", () => { expect(manifest.sourceLogFiles).toEqual(["server.log"]); }); + // Regression for #2902 PR review, follow-up finding: a single oversized CURRENT server.log was + // exported in full (never dropped, per the rule above), exceeding --max-bytes outright, and read + // via a whole-file readFileSync. Combines both effects of a tiny budget in one export: an older + // rotated file is dropped whole (still recorded in truncatedLogFiles, per (d) above) AND the + // current file alone still exceeds what's left of the budget, so only its tail is exported. + it("drops older files first and ALSO tail-truncates the current file when both are needed to fit --max-bytes", async () => { + const rotatedLine = JSON.stringify({ + ts: "2026-08-18T00:00:00.000Z", + category: "http", + op: "old", + }); + writeFileSync(join(stateDir, "logs", "server-2026-08-18.log"), `${rotatedLine}\n`.repeat(50)); + // 20 fixed-width lines (11 bytes + "\n" = 12 bytes each, 240 bytes total) so the tail cut lands + // at a byte offset that can be reasoned about exactly, the same fixture shape + // support-export.test.ts uses for the same scenario. + const currentLine = (i: number): string => `{"seq":${String(i).padStart(3, "0")}}`; + const currentText = `${Array.from({ length: 20 }, (_, i) => currentLine(i)).join("\n")}\n`; + writeFileSync(join(stateDir, "logs", CURRENT_LOG_FILE_NAME), currentText); + + const c = makeIo(); + const code = await runSupportCli( + ["export", "--state-dir", stateDir, "--out", join(outDir, "tail.jsonl"), "--max-bytes", "50"], + c.io, + AUDIT_ENV, + { auditDeps: healthyAuditDeps(), evidenceStore: createInMemoryEvidenceStore() }, + ); + + expect(code).toBe(0); + const written = readFileSync(join(outDir, "tail.jsonl"), "utf8"); + const [manifestLine, ...logLines] = written.trimEnd().split("\n"); + const manifest: Record = JSON.parse(manifestLine ?? "{}") as Record< + string, + unknown + >; + + expect(manifest.truncatedLogFiles).toEqual(["server-2026-08-18.log"]); + expect(manifest.sourceLogFiles).toEqual([CURRENT_LOG_FILE_NAME]); + expect(manifest.currentFileTailTruncated).toEqual({ + name: CURRENT_LOG_FILE_NAME, + droppedBytes: 192, + }); + // The tail strategy brought the export back within budget, so budgetExceeded must be false. + expect(manifest.budgetExceeded).toBe(false); + // Every surviving line is one of the file's own complete lines — the newest ones, in order. + expect(logLines).toEqual([currentLine(16), currentLine(17), currentLine(18), currentLine(19)]); + }); + it("fails closed with exit 1 when the local-state audit cannot run, and writes no bundle", async () => { const c = makeIo(); const code = await runSupportCli( diff --git a/packages/keiko-cli/src/support.ts b/packages/keiko-cli/src/support.ts index f5b760d802..27ed1c6e42 100644 --- a/packages/keiko-cli/src/support.ts +++ b/packages/keiko-cli/src/support.ts @@ -38,6 +38,7 @@ import { readKeptFiles, selectLogFilesWithinBudget, serializeBundleLines, + type CurrentFileTailTruncated, type SkippedLogFile, } from "./support-export.js"; @@ -50,7 +51,8 @@ evidence-index count, exactly which log files were copied) followed by every lin /logs/server*.log, copied byte-for-byte. Default --out is ./keiko-support-.jsonl (colons replaced with '-'); default --max-bytes is 50MB — the oldest log files are dropped first when the cap would be exceeded, and always named in the -manifest's truncatedLogFiles. +manifest's truncatedLogFiles. The current log file is never dropped; if it alone still exceeds +the cap, only its tail is exported instead, named in the manifest's currentFileTailTruncated. analyze reads FILE (a support bundle or a raw server.log — auto-detected), groups its lines by correlationId, and prints one reconstructed timeline per id. Each process lifetime is ordered by @@ -222,6 +224,8 @@ interface LogContent { readonly contentLines: readonly string[]; readonly sourceLogFiles: readonly string[]; readonly truncatedLogFiles: readonly string[]; + readonly currentFileTailTruncated: CurrentFileTailTruncated | undefined; + readonly budgetExceeded: boolean; readonly skippedLogFiles: readonly SkippedLogFile[]; } @@ -230,11 +234,14 @@ interface LogContent { // (support-export.ts's `discoverServerLogFiles`, between `readdirSync` and `statSync`, and // `readKeptFiles`, between selection and the actual read): `sourceLogFiles` names only the files // that actually contributed content; `skippedLogFiles` names every file that vanished at either -// boundary, by name only, never by its absolute path. +// boundary, by name only, never by its absolute path. `budgetExceeded` and +// `currentFileTailTruncated` come from `readKeptFiles`, not `selection`: only the read step knows +// whether a tail read of the current file actually managed to keep a complete line, which is what +// decides whether the size budget was, in the end, honoured. function collectLogContent(logsDir: string, maxBytes: number): LogContent { const discovery = discoverServerLogFiles(logsDir); const selection = selectLogFilesWithinBudget(discovery.files, maxBytes); - const read = readKeptFiles(selection.kept); + const read = readKeptFiles(selection.kept, selection.currentFileTailBudgetBytes); const readSkipped = new Set(read.skippedLogFiles.map((skipped) => skipped.name)); const sourceLogFiles = selection.kept .map((file) => file.name) @@ -243,6 +250,8 @@ function collectLogContent(logsDir: string, maxBytes: number): LogContent { contentLines: read.contentLines, sourceLogFiles, truncatedLogFiles: selection.truncatedLogFiles, + currentFileTailTruncated: read.currentFileTailTruncated, + budgetExceeded: read.budgetExceeded, skippedLogFiles: [...discovery.skippedLogFiles, ...read.skippedLogFiles], }; } @@ -316,6 +325,8 @@ async function runSupportExport( stateDirSource, sourceLogFiles: logContent.sourceLogFiles, truncatedLogFiles: logContent.truncatedLogFiles, + currentFileTailTruncated: logContent.currentFileTailTruncated, + budgetExceeded: logContent.budgetExceeded, skippedLogFiles: logContent.skippedLogFiles, auditSummary, evidenceIndexCount, diff --git a/packages/keiko-cli/src/ui.test.ts b/packages/keiko-cli/src/ui.test.ts index 6cf04d25d8..5d1629c82b 100644 --- a/packages/keiko-cli/src/ui.test.ts +++ b/packages/keiko-cli/src/ui.test.ts @@ -213,6 +213,26 @@ describe("runUiCli", () => { expect(err.join("")).toContain("UI database path must be absolute"); }); + // process-guards.ts's fatal-crash handler reads `process.env.KEIKO_STATE_DIR` directly (it has + // no other seam into a real, non-injected launch). This drives the REAL (non-injected) launch + // path — `deps.createServer` is left undefined — but stops before any socket binds by forcing + // the same early `UiStoreError` bail as "fails fast when --ui-db is relative" above, so the + // real process.env mutation is observable without ever starting a real server. + it("sets the real process.env.KEIKO_STATE_DIR on a direct (non-injected) launch before any crash could occur", async () => { + const { io } = captureIo(); + const cwd = await mkdtemp(join(REAL_TMPDIR, "keiko-ui-cli-real-launch-state-env-")); + vi.stubEnv("KEIKO_STATE_DIR", ""); + try { + expect(process.env.KEIKO_STATE_DIR).toBe(""); + const code = await runUiCli(["--ui-db", ".keiko/ui.db"], io, {}, { staticRoot, cwd }); + expect(code).toBe(2); + expect(process.env.KEIKO_STATE_DIR).toBe(join(cwd, ".keiko")); + } finally { + vi.unstubAllEnvs(); + await rm(cwd, { recursive: true, force: true }); + } + }); + it("fails fast when --ui-db is inside the current workspace", async () => { const { io, err } = captureIo(); const nested = join(process.cwd(), ".keiko-test-ui", "ui.db"); diff --git a/packages/keiko-cli/src/ui.ts b/packages/keiko-cli/src/ui.ts index 1ce19ae438..76326d1b36 100644 --- a/packages/keiko-cli/src/ui.ts +++ b/packages/keiko-cli/src/ui.ts @@ -965,6 +965,18 @@ async function launchUiFromDeps( deps: UiCliDeps, ): Promise { const stateDir = resolveRuntimeStateDir(cwd, effectiveEnv); + // process-guards.ts's fatal-crash handler is installed before any CLI parsing happens and reads + // the REAL process.env directly (it cannot receive this value any other way) -- without this + // assignment, a direct `keiko ui` launch (no --state-dir / KEIKO_STATE_DIR from the operator) + // never has process.env.KEIKO_STATE_DIR set, so a crash inside startUiServer/the real server + // factory below writes only the generic stderr line and silently drops the structured + // process.fatal record. Mirrors what lifecycle.ts's spawnUiProcess already gives the CHILD + // `keiko ui` process for free. Gated on the same "real launch" condition `startUiServer` + // computes (`deps.createServer === undefined`) so injected-server unit tests never mutate the + // real process.env. + if (deps.createServer === undefined) { + process.env.KEIKO_STATE_DIR = stateDir; + } // Captured from the ORIGINAL env, before `withDefaultLocalRuntimeStateEnv` unconditionally sets // `KEIKO_STATE_DIR` on the derived copy below — otherwise every launch would read back as // `env-override` regardless of what the operator actually configured. diff --git a/packages/keiko-server/src/diagnostics-log.activity-log.test.ts b/packages/keiko-server/src/diagnostics-log.activity-log.test.ts index a4c306d2d2..e7fffce31c 100644 --- a/packages/keiko-server/src/diagnostics-log.activity-log.test.ts +++ b/packages/keiko-server/src/diagnostics-log.activity-log.test.ts @@ -89,6 +89,47 @@ describe("diagnostic records on the activity log", () => { expect(JSON.stringify(line)).not.toContain("JaneDoe1985"); }); + it("redacts the stderr line the same way as the activity-log line", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const record = { + correlationId: "req-1f2e3d", + timestamp: "2026-08-21T00:00:00.000Z", + operation: "chat.stream", + source: "server.top-level-catch", + errorClass: "GatewayError", + message: "Provider verification failed without exposing upstream response details.", + code: "GATEWAY_ERROR", + notes: "JaneDoe1985", + } as ServerDiagnosticRecord & { readonly notes: string }; + + defaultServerDiagnosticSink.record(record); + const stderrLine = errorSpy.mock.calls[0]?.[0] as string; + + // The undeclared, caller-supplied field must never reach stderr, exactly as it never reaches + // the activity-log file. + expect(stderrLine).not.toContain("JaneDoe1985"); + // The allowlisted summary still makes it through, projected the same way the file line + // projects it. + expect(stderrLine).toContain( + "Provider verification failed without exposing upstream response details.", + ); + + // A message NOT in the closed vocabulary must not appear verbatim on stderr either. + errorSpy.mockClear(); + const foreignRecord: ServerDiagnosticRecord = { + correlationId: "req-foreign", + timestamp: "2026-08-21T00:00:00.000Z", + operation: "chat.stream", + source: "server.top-level-catch", + errorClass: "GatewayError", + message: "not-in-the-closed-vocabulary", + }; + defaultServerDiagnosticSink.record(foreignRecord); + const foreignStderrLine = errorSpy.mock.calls[0]?.[0] as string; + expect(foreignStderrLine).not.toContain("not-in-the-closed-vocabulary"); + }); + // ADR-0173 D3/D11 (g2, g29): a thrown Error carries its stack and its allowlisted summary all // the way to the persisted line, through `serverDiagnosticFromError` → `defaultServerDiagnosticSink` // → the file sink's redaction pass — not merely through `describeError` in isolation. diff --git a/packages/keiko-server/src/diagnostics-log.ts b/packages/keiko-server/src/diagnostics-log.ts index bc97799131..a2702c9bee 100644 --- a/packages/keiko-server/src/diagnostics-log.ts +++ b/packages/keiko-server/src/diagnostics-log.ts @@ -82,6 +82,11 @@ export interface ServerDiagnosticRecord { // explicitly asserted), `dropped` were removed (Keiko had only inferred the role). readonly unverifiedEmbeddingModelCount?: number | undefined; readonly droppedEmbeddingModelCount?: number | undefined; + // How many candidate memories the conversation-retrieval semantic reranker skipped for a stored + // embedding whose identity did not match the query's, and how many candidates were in play when + // it did. Bounded counts only; never a memory id, model id, or embedding vector. + readonly semanticSkippedCount?: number | undefined; + readonly semanticCandidateCount?: number | undefined; // Keiko-code stack frames the error passed through, nearest-to-throw-site first, reduced by // `keikoStackFrames` (ADR-0173 D3) to their dist/src-anchored form — never an absolute path, never // a `node_modules`/`node:internal` frame. Capped at 8; absent when the error carried no `.stack` @@ -102,8 +107,19 @@ export interface ServerDiagnosticSink { // keep observing stderr, while the shipped server writes a file the operator can read directly. export const defaultServerDiagnosticSink: ServerDiagnosticSink = { record(record: ServerDiagnosticRecord): void { + // The stderr line must carry the SAME redaction guarantee as the activity-log file line: the + // raw caller-supplied `record` is never serialised directly. `diagnosticActivityLogFields` + // already projects it onto allowlisted, bounded fields for the file sink (ADR-0173 D11 g29); + // reused here rather than a second ad-hoc redaction so both channels share one contract. + const safeLine = { + correlationId: record.correlationId, + timestamp: record.timestamp, + operation: record.operation, + errorClass: record.errorClass, + ...diagnosticActivityLogFields(record), + }; // eslint-disable-next-line no-console - console.error(`[keiko-server:diagnostic] ${JSON.stringify(record)}`); + console.error(`[keiko-server:diagnostic] ${JSON.stringify(safeLine)}`); appendDiagnosticToActivityLog(record); }, }; @@ -161,6 +177,8 @@ function diagnosticActivityLogFields(record: ServerDiagnosticRecord): Record @@ -273,6 +291,8 @@ const SERVER_DIAGNOSTIC_SUMMARIES = [ "Debug production service composition failed.", "Managed task-workspace boundary materialization failed.", "Evidence retention deleted manifests.", + "Semantic memory retrieval skipped incompatible embeddings.", + "Semantic memory retrieval was disabled for this turn.", ] as const; export type ServerDiagnosticSummary = (typeof SERVER_DIAGNOSTIC_SUMMARIES)[number]; diff --git a/packages/keiko-server/src/grounded-qa.test.ts b/packages/keiko-server/src/grounded-qa.test.ts index 1794288925..4053f8f5ea 100644 --- a/packages/keiko-server/src/grounded-qa.test.ts +++ b/packages/keiko-server/src/grounded-qa.test.ts @@ -2601,8 +2601,13 @@ describe("handleGroundedAsk", () => { expect( (result.body as GroundedAnswer & { readonly memory?: unknown }).memory, ).toBeUndefined(); - expect(diagnostics).toHaveLength(1); - expect(diagnostics[0]).toMatchObject({ + // Two records: the semantic-retrieval signal (now a diagnostic, never console.warn — audit of + // #3233) and the enrichment failure this test is about. + expect(diagnostics.map((record) => record.operation)).toEqual([ + "memory.retrieval.semantic-disabled", + "grounded.memory", + ]); + expect(diagnostics[1]).toMatchObject({ operation: "grounded.memory", source: "grounded-qa.attach-memory", message: "grounded-memory-enrichment-failed", @@ -2742,7 +2747,12 @@ describe("handleGroundedAsk", () => { expect(result.status).toBe(200); expect((result.body as GroundedAnswer).content).toContain("Dark mode"); expect(store.listMessages(chatId)).toHaveLength(2); - expect(diagnostics).toMatchObject([ + // The semantic-retrieval signal precedes the capture failure (audit of #3233). + expect(diagnostics.map((record) => record.operation)).toEqual([ + "memory.retrieval.semantic-disabled", + "grounded.memory", + ]); + expect(diagnostics.slice(1)).toMatchObject([ { operation: "grounded.memory", source: "grounded-qa.attach-memory", diff --git a/packages/keiko-server/src/memory-retrieval-signals.test.ts b/packages/keiko-server/src/memory-retrieval-signals.test.ts index 5e29f86b88..7a32f6d622 100644 --- a/packages/keiko-server/src/memory-retrieval-signals.test.ts +++ b/packages/keiko-server/src/memory-retrieval-signals.test.ts @@ -13,6 +13,7 @@ import type { } from "@oscharko-dev/keiko-memory-vault"; import type { UiHandlerDeps } from "./deps.js"; +import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; const { embedMock, observations, searchMock } = vi.hoisted(() => ({ embedMock: vi.fn(), @@ -111,6 +112,20 @@ function collectSignals( }); } +// A recording diagnostics sink, so a test can assert on the redaction-safe record a degraded +// semantic-retrieval path emits instead of on a console.warn call (#2902 O-F4). +function depsWithDiagnostics(): { + readonly deps: UiHandlerDeps; + readonly calls: ServerDiagnosticRecord[]; +} { + const calls: ServerDiagnosticRecord[] = []; + const deps = { + env: {}, + diagnostics: { record: (record: ServerDiagnosticRecord) => calls.push(record) }, + } as unknown as UiHandlerDeps; + return { deps, calls }; +} + beforeEach(() => { observations.length = 0; embedMock.mockReset(); @@ -215,25 +230,31 @@ describe("buildConversationRetrievalSignals", () => { () => [metadata(invalidNonFinite, 300), metadata(invalidZero, 200), metadata(valid, 100)], embeddings, ); - const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined); - - try { - const signals = await collectSignals(vault); - - expect(observations[0]?.ids).toEqual(["valid"]); - expect([...(signals.semanticById?.keys() ?? [])]).toEqual([valid]); - expect(warning).toHaveBeenCalledWith( - "memory semantic scores skipped for incompatible embeddings", - { - reason: "identity-mismatch", - skipped: 2, - candidates: 3, - queryModelId: IDENTITY.modelId, - }, - ); - } finally { - warning.mockRestore(); - } + const { deps, calls } = depsWithDiagnostics(); + + const signals = await buildConversationRetrievalSignals( + deps, + vault, + "memory query", + [SCOPE], + NOW_MS, + { allowed: true, reason: "allowed" }, + ); + + expect(observations[0]?.ids).toEqual(["valid"]); + expect([...(signals.semanticById?.keys() ?? [])]).toEqual([valid]); + // Degraded semantic retrieval reaches the redaction-safe operator diagnostic sink (so it + // lands in server.log and a support bundle), never only a console.warn nobody captures. + const skipped = calls.find( + (record) => record.operation === "memory.retrieval.semantic-skipped", + ); + expect(skipped).toMatchObject({ + code: "identity-mismatch", + semanticSkippedCount: 2, + semanticCandidateCount: 3, + }); + // Never the raw model id — only bounded counts and the closed-vocabulary reason code. + expect(JSON.stringify(skipped)).not.toContain(IDENTITY.modelId); }); it("reports the vector-index failure reason without misclassifying it as an identity mismatch", async () => { @@ -243,19 +264,48 @@ describe("buildConversationRetrievalSignals", () => { ]); const vault = vaultFor(() => [metadata(alpha, 100)], embeddings); searchMock.mockResolvedValueOnce({ ok: false, reason: "runtime-integrity-failed" }); - const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const { deps, calls } = depsWithDiagnostics(); - try { - const signals = await collectSignals(vault); + const signals = await buildConversationRetrievalSignals( + deps, + vault, + "memory query", + [SCOPE], + NOW_MS, + { allowed: true, reason: "allowed" }, + ); - expect(signals.semanticById).toBeUndefined(); - expect(warning).toHaveBeenCalledWith("memory semantic scores disabled", { - reason: "vector-index-failed", - vectorIndexReason: "runtime-integrity-failed", - candidates: 1, - }); - } finally { - warning.mockRestore(); - } + expect(signals.semanticById).toBeUndefined(); + const disabled = calls.find( + (record) => record.operation === "memory.retrieval.semantic-disabled", + ); + expect(disabled).toMatchObject({ + code: "vector-index-failed:runtime-integrity-failed", + semanticCandidateCount: 1, + }); + }); + + it("reports a degraded conversation-memory recall diagnostic, not a knowledge-store citation one", async () => { + // Regression pin (#2902 Finding 1 refinement): this module feeds chat/BFF conversation-memory + // recall, not the local-knowledge citation-grounding pipeline. Its diagnostics must never be + // mistaken for a local-knowledge-store event. + const alpha = memoryId("alpha"); + const embeddings = new Map([ + [alpha, embedding(alpha, new Float32Array([1, 0]))], + ]); + const vault = vaultFor(() => [metadata(alpha, 100)], embeddings); + searchMock.mockResolvedValueOnce({ ok: false, reason: "runtime-integrity-failed" }); + const { deps, calls } = depsWithDiagnostics(); + + await buildConversationRetrievalSignals(deps, vault, "memory query", [SCOPE], NOW_MS, { + allowed: true, + reason: "allowed", + }); + + const disabled = calls.find( + (record) => record.operation === "memory.retrieval.semantic-disabled", + ); + expect(disabled?.source).toBe("memory-retrieval-signals"); + expect(disabled?.source).not.toMatch(/local-knowledge/iu); }); }); diff --git a/packages/keiko-server/src/memory-retrieval-signals.ts b/packages/keiko-server/src/memory-retrieval-signals.ts index 76174af641..50daf13247 100644 --- a/packages/keiko-server/src/memory-retrieval-signals.ts +++ b/packages/keiko-server/src/memory-retrieval-signals.ts @@ -15,7 +15,7 @@ // embedding model => semanticById undefined (lexical fallback); empty access history => strengthById // empty (the ranker zeroes its weight). -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { isIP } from "node:net"; import type { MemoryId, MemoryScope } from "@oscharko-dev/keiko-contracts/memory"; @@ -51,6 +51,7 @@ import { } from "@oscharko-dev/keiko-memory-capture"; import { currentGatewayConfig, type UiHandlerDeps } from "./deps.js"; +import { emitServerDiagnostic } from "./diagnostics-log.js"; import { configuredEmbeddingProviders } from "./local-knowledge-handlers.js"; import { embedMemoryText, memoryEmbeddingCalibrationFor } from "./memory-embedding.js"; @@ -266,15 +267,22 @@ function compatibleMemoryEntries( return { entries, skipped }; } -function warnSkippedIncompatible(skipped: number, candidates: number, queryModelId: string): void { +// Content-free operator diagnostic (O-F4 / #2902): routes through the single redaction-safe +// server diagnostic sink (diagnostics-log.ts) instead of console.* directly, mirroring +// emitSalienceDiagnostic in memory-salience.ts. Never the query text, a memory body, or a model +// id — only bounded counts and the closed-vocabulary skip reason. +function warnSkippedIncompatible(deps: UiHandlerDeps, skipped: number, candidates: number): void { if (skipped === 0) return; - // Safe diagnostic: counts and model ids only, never query text or memory bodies. - // eslint-disable-next-line no-console - console.warn("memory semantic scores skipped for incompatible embeddings", { - reason: "identity-mismatch", - skipped, - candidates, - queryModelId, + emitServerDiagnostic(deps.diagnostics, { + correlationId: randomUUID(), + timestamp: new Date().toISOString(), + operation: "memory.retrieval.semantic-skipped", + source: "memory-retrieval-signals.semanticScoresFrom", + errorClass: "SemanticRetrievalSkipped", + message: "Semantic memory retrieval skipped incompatible embeddings.", + code: "identity-mismatch", + semanticSkippedCount: skipped, + semanticCandidateCount: candidates, }); } @@ -396,7 +404,7 @@ async function semanticScoresFrom( throwIfAborted(signal); const queryEmbedding = await embedMemoryText(deps, queryText, "query"); if (queryEmbedding === null) { - warnSemanticRetrievalDisabled("no-embedder", { candidates: candidateIds.length }); + warnSemanticRetrievalDisabled(deps, "no-embedder", candidateIds.length); return undefined; } throwIfAborted(signal); @@ -407,7 +415,7 @@ async function semanticScoresFrom( candidateIds, embeddings, ); - warnSkippedIncompatible(compatible.skipped, candidateIds.length, queryEmbedding.modelId); + warnSkippedIncompatible(deps, compatible.skipped, candidateIds.length); if (compatible.entries.length === 0) return new Map(); throwIfAborted(signal); const result = await createMemoryVectorIndexPort(queryIdentity, compatible.entries).search({ @@ -419,23 +427,39 @@ async function semanticScoresFrom( candidateIds, }); if (!result.ok) { - warnSemanticRetrievalDisabled("vector-index-failed", { - vectorIndexReason: result.diagnostics.reason ?? "unknown", - candidates: compatible.entries.length, - }); + warnSemanticRetrievalDisabled( + deps, + "vector-index-failed", + compatible.entries.length, + result.diagnostics.reason ?? "unknown", + ); return undefined; } return new Map(result.candidates.map((candidate) => [candidate.id as MemoryId, candidate.score])); } +// Content-free operator diagnostic (O-F4 / #2902): see warnSkippedIncompatible above for why this +// routes through emitServerDiagnostic rather than console.warn. `vectorIndexReason` is the vector +// index port's own contract-guaranteed content-free reason code (VectorIndexDiagnostics.reason, +// "never a body, a path, an endpoint, or a query string") — safe to compose onto `code` alongside +// the already-closed `reason` union. function warnSemanticRetrievalDisabled( + deps: UiHandlerDeps, reason: SemanticRetrievalGateReason, - extra: Record = {}, + candidates: number, + vectorIndexReason?: string, ): void { if (reason === "allowed") return; - // Safe diagnostic: reason and counts only, never query text, memory bodies, endpoints, or keys. - // eslint-disable-next-line no-console - console.warn("memory semantic scores disabled", { reason, ...extra }); + emitServerDiagnostic(deps.diagnostics, { + correlationId: randomUUID(), + timestamp: new Date().toISOString(), + operation: "memory.retrieval.semantic-disabled", + source: "memory-retrieval-signals", + errorClass: "SemanticRetrievalDisabled", + message: "Semantic memory retrieval was disabled for this turn.", + code: vectorIndexReason === undefined ? reason : `${reason}:${vectorIndexReason}`, + semanticCandidateCount: candidates, + }); } function hasNonZeroMagnitude(vector: Float32Array): boolean { @@ -511,14 +535,15 @@ function collectEmbeddingSignals( } function warnSemanticGate( + deps: UiHandlerDeps, semanticGate: SemanticRetrievalGate, embeddings: ReadonlyMap, candidateCount: number, ): void { if (semanticGate.allowed && embeddings.size === 0) { - warnSemanticRetrievalDisabled("no-embeddings", { candidates: candidateCount }); + warnSemanticRetrievalDisabled(deps, "no-embeddings", candidateCount); } else if (!semanticGate.allowed && semanticGate.reason !== "no-query") { - warnSemanticRetrievalDisabled(semanticGate.reason, { candidates: candidateCount }); + warnSemanticRetrievalDisabled(deps, semanticGate.reason, candidateCount); } } @@ -552,7 +577,7 @@ export async function buildConversationRetrievalSignals( : new Map(); throwIfAborted(signal); const embeddingSignals = collectEmbeddingSignals(deps, embeddings, signal); - warnSemanticGate(semanticGate, embeddings, candidateIds.length); + warnSemanticGate(deps, semanticGate, embeddings, candidateIds.length); const semanticById = shouldComputeSemanticScores(semanticGate, queryText, embeddings) ? await semanticScoresFrom(deps, queryText, candidateIds, embeddings, signal) : undefined; diff --git a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.test.ts b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.test.ts index ef9e3f26cd..ebdfd00180 100644 --- a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.test.ts +++ b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.test.ts @@ -31,6 +31,7 @@ import { Readable } from "node:stream"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createNodeFigmaSnapshotStore } from "@oscharko-dev/keiko-evidence"; import { buildCspHeader } from "../csp.js"; +import type { ServerDiagnosticRecord } from "../diagnostics-log.js"; import { buildRedactor, createInMemoryUiStore, type UiHandlerDeps } from "../index.js"; import { createRunRegistry } from "../runs.js"; import { UI_HOST } from "../server.js"; @@ -381,22 +382,27 @@ describe("POST /api/figma/snapshots — code→status matrix", () => { const spy = vi.spyOn(orchModule, "governedSnapshotBuild"); // A non-coded build error whose message embeds the secret PAT (defence-in-depth redaction). spy.mockRejectedValueOnce(new TypeError(`render parse failed token=${TOKEN}`)); - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const diagnosticCalls: ServerDiagnosticRecord[] = []; + const deps: UiHandlerDeps = { + ...makeDeps(evidenceDir, { FIGMA_ACCESS_TOKEN: TOKEN }), + diagnostics: { record: (record) => diagnosticCalls.push(record) }, + }; try { const result = await handleFigmaTriggerSnapshot( makeCtx(JSON.stringify({ boardLink: BOARD_LINK, acknowledgeReadOnly: false })), - makeDeps(evidenceDir, { FIGMA_ACCESS_TOKEN: TOKEN }), + deps, ); expect(result.status).toBe(500); expect((result.body as { error: { code: string } }).error.code).toBe("FIGMA_INTERNAL"); - expect(errorSpy).toHaveBeenCalled(); - const logged = errorSpy.mock.calls.flat().map(String).join(" "); - expect(logged).toContain("TypeError"); // the cause class is surfaced for diagnosis… - expect(logged).not.toContain(TOKEN); // …but the secret PAT is redacted out + // The cause reaches the redaction-safe operator diagnostic sink, not raw console.error. + expect(diagnosticCalls.length).toBeGreaterThan(0); + const record = diagnosticCalls[0]; + expect(record?.errorClass).toBe("TypeError"); // the cause class is surfaced for diagnosis… + expect(record?.correlationId).toMatch(/^[0-9a-f-]{36}$/u); + expect(JSON.stringify(record)).not.toContain(TOKEN); // …but the secret PAT is redacted out } finally { spy.mockRestore(); - errorSpy.mockRestore(); } }); diff --git a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.ts b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.ts index db233e03b6..f64110d8b3 100644 --- a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.ts +++ b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotRoutes.ts @@ -57,6 +57,7 @@ import { } from "../deps.js"; import { redact } from "@oscharko-dev/keiko-security"; import type { EnvSource } from "@oscharko-dev/keiko-security"; +import { emitServerDiagnostic } from "../diagnostics-log.js"; import { appendFigmaConnectorAudit, parseFigmaTarget, @@ -774,16 +775,21 @@ function figmaRequestTimeoutMsFromEnv(env: EnvSource): number { // the coded body is content-free, so on its own an operator cannot tell a transient render-body // malformation from a filesystem failure from a genuine bug. Log the redacted cause (class + message, // secrets scrubbed) so the incident is diagnosable without ever leaking a token or provider body. -// Matches the redacted-console.error convention (memory-salience.ts). Only fires for FIGMA_INTERNAL — -// expected coded errors (consent/auth/rate-limit) stay quiet (they are already audited). +// Routes through the single redaction-safe server diagnostic sink (diagnostics-log.ts), the same +// pattern memory-salience.ts's emitSalienceDiagnostic uses — never console.* directly, so the record +// lands in server.log with a correlationId and is visible to a support bundle. Only fires for +// FIGMA_INTERNAL — expected coded errors (consent/auth/rate-limit) stay quiet (already audited). function logFigmaInternal(stage: string, err: unknown, deps: UiHandlerDeps): void { const name = err instanceof Error ? err.constructor.name : typeof err; const message = err instanceof Error ? err.message : String(err); - // eslint-disable-next-line no-console - console.error( - `figma snapshot-build failed (${stage}): ${name}`, - redact(message, currentRedactionSecrets(deps)), - ); + emitServerDiagnostic(deps.diagnostics, { + correlationId: randomUUID(), + timestamp: new Date().toISOString(), + operation: "figma.snapshotBuild", + source: `figmaSnapshotRoutes.logFigmaInternal.${stage}`, + errorClass: name, + message: redact(message, currentRedactionSecrets(deps)), + }); } // Map a thrown error from the governed build to a coded route result: a coded connector error maps to diff --git a/scripts/__tests__/op-catalog-drift.test.mjs b/scripts/__tests__/op-catalog-drift.test.mjs index f4070c92e6..860eccb1b6 100644 --- a/scripts/__tests__/op-catalog-drift.test.mjs +++ b/scripts/__tests__/op-catalog-drift.test.mjs @@ -58,6 +58,19 @@ describe("op catalog drift", () => { expect(checkedIn.generatedBy).toBe("scripts/generate-op-catalog.mjs"); }); + // #2902 W5: orchestrator.ts's logIndexing/logEmbeddingRun/logDocument hardcode `category` inside + // their OWN body rather than the caller's object literal, so tier 1 (findSiblingCategory) never + // finds a sibling `category:` at these call sites, and tier 3 (fileCategoryBinding) backs off + // because the file binds two distinct categories. Before OBJECT_ARG_CATEGORY_FUNCTIONS, both ops + // below resolved to "unknown" even though the runtime always stamps a deterministic category for + // them. Driven through the real generator entry point, not a re-derivation of its category rules. + it("attributes the deterministic category to an op:-only call site of a checked-in object-arg category function", () => { + const catalog = generateOpCatalog(repoRoot); + const byOp = (op) => catalog.entries.find((entry) => entry.op === op); + expect(byOp("indexing.document.failed")?.category).toBe("indexing"); + expect(byOp("embedding.preflight.identity-rejected")?.category).toBe("embedding"); + }); + // Proves the drift gate actually fails closed: mutating a COPY of the checked-in catalog must // make it stop matching what the generator produces right now. Without this, a future change // that made `generateOpCatalog` just return the parsed checked-in file (or made the "matches the diff --git a/scripts/generate-op-catalog.mjs b/scripts/generate-op-catalog.mjs index fec229c930..9221a9ab68 100644 --- a/scripts/generate-op-catalog.mjs +++ b/scripts/generate-op-catalog.mjs @@ -141,6 +141,33 @@ const POSITIONAL_OP_HELPERS = [ { name: "adoptPreflightIdentity", argIndex: 2, category: "embedding" }, ]; +// Functions that receive the whole log-event object literal as an argument and hardcode their own +// `category` before forwarding it to the sink (orchestrator.ts's logIndexing/logEmbeddingRun/ +// logDocument, #2902 W5). Unlike POSITIONAL_OP_HELPERS, `op` here is a NAMED PROPERTY inside the +// object-literal argument, not a bare positional string — tier 1 (findSiblingCategory) never finds +// a sibling `category:` at these call sites because the category lives inside the callee's body, +// not the caller's object literal, and tier 3 (fileCategoryBinding) backs off for this file because +// it binds two distinct categories ("indexing" and "embedding") depending on which of these three +// functions is called. `file` scopes every entry, exactly like POSITIONAL_OP_HELPERS' scoped +// entries, so a same-named helper anywhere else is never misattributed. +const OBJECT_ARG_CATEGORY_FUNCTIONS = [ + { + name: "logIndexing", + category: "indexing", + file: "packages/keiko-local-knowledge/src/indexing/orchestrator.ts", + }, + { + name: "logEmbeddingRun", + category: "embedding", + file: "packages/keiko-local-knowledge/src/indexing/orchestrator.ts", + }, + { + name: "logDocument", + category: "indexing", + file: "packages/keiko-local-knowledge/src/indexing/orchestrator.ts", + }, +]; + function packageNameFromRoot(root) { const match = /^packages\/([^/]+)\/src$/.exec(root); if (match?.[1] === undefined) throw new Error(`Unexpected scanned root shape: ${root}`); @@ -373,6 +400,61 @@ function findSiblingCategory(lines, opLine) { return categoryAbove(lines, opLine, opIndent) ?? categoryBelow(lines, opLine); } +// The identifier that ends `text` (ignoring trailing whitespace), found by a backward scan rather +// than an end-anchored regex — `([\w$]*)\s*$` backtracks super-linearly on long lines (Sonar S8786). +function trailingIdentifier(text) { + let end = text.length; + while (end > 0 && /\s/.test(text[end - 1])) end -= 1; + let start = end; + while (start > 0 && /[\w$]/.test(text[start - 1])) start -= 1; + if (start === end || /\d/.test(text[start])) return undefined; + return text.slice(start, end); +} + +// Walks backward from `fromIndex` (exclusive) tracking bracket depth, and returns the index of the +// nearest UNMATCHED opening bracket — the bracket that encloses `fromIndex` one level up. Every +// closing bracket seen first increments `depth` (one more matching opener is now owed before we are +// back to the enclosing level); every opening bracket either satisfies one of those or, at depth 0, +// IS the answer. Mirrors `scanBalanced`'s forward depth bookkeeping, run in reverse, so the object +// literal an `op:` property lives in — and, one level further out, the call it is an argument +// to — can be found without a full AST. Like every other extractor in this file, this does not +// track string/template spans on the way back; on this codebase's Prettier-formatted, one-property- +// per-line source that risk is the same one `findSiblingCategory`'s line scan already accepts. +function enclosingOpenBracketIndex(source, fromIndex) { + let depth = 0; + for (let i = fromIndex - 1; i >= 0; i -= 1) { + const ch = source[i]; + if (CLOSE_BRACKETS.has(ch)) { + depth += 1; + } else if (OPEN_BRACKETS.has(ch)) { + if (depth === 0) return i; + depth -= 1; + } + } + return -1; +} + +// Tier 2.5 (see OBJECT_ARG_CATEGORY_FUNCTIONS' header comment): resolves the category for an +// `op:` property whose enclosing object literal is itself an argument to one of those checked-in +// functions — a shape tiers 1 and 3 cannot see, since the category lives inside the callee's body, +// never near the call site. Finds the object literal's own opening `{` first (must be a `{`, not +// some other enclosing bracket — a `[`/`(` there means `op:` is not sitting in a plain object- +// literal argument), then the call's opening `(` one level further out, then reads the identifier +// immediately before it. Returns undefined — never a guess — the moment any of those structural +// expectations fails, exactly like `fileCategoryBinding`'s own "no single safe answer" contract. +function objectArgCategory(source, colonEnd, relPath) { + const braceIndex = enclosingOpenBracketIndex(source, colonEnd); + if (braceIndex === -1 || source[braceIndex] !== "{") return undefined; + const parenIndex = enclosingOpenBracketIndex(source, braceIndex); + if (parenIndex === -1 || source[parenIndex] !== "(") return undefined; + const name = trailingIdentifier(source.slice(0, parenIndex)); + if (name === undefined) return undefined; + const match = OBJECT_ARG_CATEGORY_FUNCTIONS.find( + (entry) => entry.name === name && entry.file === relPath, + ); + return match?.category; +} + // Blanks `//` line comments and `/* … */` block comments in `source` — replaces every comment // character with a space, character for character, while every OTHER character (including every // newline, whether inside a comment or not) passes through unchanged. Unlike removing comment @@ -555,7 +637,8 @@ function opPropertyEntries(source, lines, offsets, constMap, relPath, colonEnd) const value = rawValue.trim(); if (closesOverDeclaration(stopChar) || isTypeAnnotationValue(value, constMap)) return []; const opLine = lineNumberAt(offsets, colonEnd); - const category = findSiblingCategory(lines, opLine) ?? "unknown"; + const category = + findSiblingCategory(lines, opLine) ?? objectArgCategory(source, colonEnd, relPath) ?? "unknown"; const literals = resolveLiteralValues(value, constMap); if (literals === null) return [siteEntry("", category, relPath, opLine)]; return literals.map((literal) => siteEntry(literal, category, relPath, opLine)); From 61456d06619c65b0e63d7099228724a9f205ae63 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 12:00:57 +0200 Subject: [PATCH 09/19] test(coverage): regenerate the package coverage baseline after merging dev (#3240) Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index f81af9a6b0..a33542c3b3 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -30,14 +30,14 @@ } }, "keiko-contracts": { - "files": 186, + "files": 187, "uncoveredFiles": 0, "uncoveredLines": 857, - "totalLines": 13852, + "totalLines": 13875, "coverage": { - "lines": 93.81, - "statements": 92.39, - "branches": 90.23, + "lines": 93.82, + "statements": 92.41, + "branches": 90.25, "functions": 97.46 } }, @@ -234,15 +234,15 @@ } }, "keiko-server": { - "files": 580, + "files": 581, "uncoveredFiles": 0, - "uncoveredLines": 4516, - "totalLines": 56260, + "uncoveredLines": 4485, + "totalLines": 56268, "coverage": { - "lines": 91.98, - "statements": 89.27, - "branches": 81.89, - "functions": 94.86 + "lines": 92.03, + "statements": 89.33, + "branches": 81.95, + "functions": 94.9 } }, "keiko-tools": { @@ -260,13 +260,13 @@ "keiko-ui": { "files": 419, "uncoveredFiles": 3, - "uncoveredLines": 2967, - "totalLines": 39452, + "uncoveredLines": 2962, + "totalLines": 39508, "coverage": { - "lines": 92.48, - "statements": 89.52, - "branches": 81.8, - "functions": 91.28 + "lines": 92.5, + "statements": 89.55, + "branches": 81.82, + "functions": 91.3 } }, "keiko-verification": { From 28683def3da7a2517867d43d1a57354f1e261fd7 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 12:24:21 +0200 Subject: [PATCH 10/19] test(coverage): regenerate the package coverage baseline after the audit repairs and the dev merge (#3233) Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index f81af9a6b0..022a21ae5a 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,12 +8,12 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 387, - "totalLines": 4880, + "uncoveredLines": 385, + "totalLines": 4916, "coverage": { - "lines": 92.11, - "statements": 90.39, - "branches": 85.19, + "lines": 92.17, + "statements": 90.44, + "branches": 85.23, "functions": 93.23 } }, @@ -237,7 +237,7 @@ "files": 580, "uncoveredFiles": 0, "uncoveredLines": 4516, - "totalLines": 56260, + "totalLines": 56263, "coverage": { "lines": 91.98, "statements": 89.27, @@ -261,10 +261,10 @@ "files": 419, "uncoveredFiles": 3, "uncoveredLines": 2967, - "totalLines": 39452, + "totalLines": 39455, "coverage": { "lines": 92.48, - "statements": 89.52, + "statements": 89.53, "branches": 81.8, "functions": 91.28 } From 42acac5bbb9a12ee5c5bb277e6d3f344c1000942 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 12:25:15 +0200 Subject: [PATCH 11/19] feat(observability): thread the correlation id through every model-call, WebSocket and diagnostic site (Wave 3b of #3233) (#3237) Every remaining request-constructing model.call / callStream / gateway.chat site passes logContext: { correlationId } from the request, run or job in scope; WebSocket sessions resolve one id per connection at upgrade; diagnostic sites that minted a fresh id per failure thread the request/run/job id or mint once per operation and set parentCorrelationId; the buffered chat path and grounded QA share emitGatewayErrorDiagnostic with the streaming path. A correlation id that is honestly unknown is the shape-valid sentinel UNKNOWN_CORRELATION_ID instead of the 7-character "unknown", which the diagnostic writer's sanitizer rewrote to its invalid-id marker. Op catalog and package coverage baseline regenerated; no floor lowered. Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 6 + docs/qa/package-coverage-baseline.json | 20 +- .../src/atlassian/syncRoutes.test.ts | 34 + .../keiko-server/src/atlassian/syncRoutes.ts | 13 +- .../src/atlassian/writeActionRoutes.test.ts | 27 + .../src/atlassian/writeActionRoutes.ts | 5 +- .../keiko-server/src/browser-routes.test.ts | 49 +- packages/keiko-server/src/browser.ts | 4 +- .../src/chat-compaction-evidence.test.ts | 7 + .../src/chat-compaction-evidence.ts | 11 +- .../src/chat-compaction-model-summary.test.ts | 66 ++ .../src/chat-compaction-model-summary.ts | 47 +- .../keiko-server/src/chat-handlers.test.ts | 347 ++++++++- packages/keiko-server/src/chat-handlers.ts | 197 ++++- .../src/chat-stream-handlers.test.ts | 30 +- .../keiko-server/src/chat-stream-handlers.ts | 94 ++- .../codingContextRoutes.test.ts | 18 + .../src/coding-context/codingContextRoutes.ts | 4 +- .../src/coding-sidecar-gateway.test.ts | 80 ++ .../src/coding-sidecar-gateway.ts | 22 +- .../src/correlation-threading.e2e.test.ts | 683 ++++++++++++++++++ packages/keiko-server/src/correlation.ts | 10 + .../src/desktop-chat-handlers.test.ts | 79 +- .../src/editor/completionRoutes.ts | 8 +- .../src/editor/inlineCompletionRoutes.ts | 8 +- .../localHistory/localHistoryCapture.test.ts | 30 + .../localHistory/localHistoryCapture.ts | 10 +- .../src/editor/patchApplyRoutes.test.ts | 30 + .../src/editor/patchApplyRoutes.ts | 13 +- packages/keiko-server/src/files.test.ts | 89 +++ packages/keiko-server/src/files.ts | 30 +- .../src/gateway-error-diagnostic.test.ts | 78 ++ .../src/gateway-error-diagnostic.ts | 48 ++ .../keiko-server/src/gateway-setup.test.ts | 34 + packages/keiko-server/src/gateway-setup.ts | 41 +- .../mutationEvidenceLedger.test.ts | 66 ++ .../src/gitDelivery/mutationEvidenceLedger.ts | 25 +- .../src/gitDelivery/syncEvidence.test.ts | 5 + .../src/gitDelivery/syncEvidence.ts | 17 +- .../src/grounded-entailment-judge.test.ts | 23 +- .../src/grounded-entailment-judge.ts | 5 +- .../src/grounded-entailment-stage.ts | 2 +- .../src/grounded-qa-hybrid.test.ts | 151 +++- .../keiko-server/src/grounded-qa-hybrid.ts | 7 +- .../src/grounded-qa-multi-source.test.ts | 56 +- .../src/grounded-qa-multi-source.ts | 2 + packages/keiko-server/src/grounded-qa.test.ts | 91 ++- packages/keiko-server/src/grounded-qa.ts | 162 ++++- .../src/local-knowledge-grounded-qa.ts | 44 +- .../src/memory-audit-handler.test.ts | 40 + .../keiko-server/src/memory-audit-handler.ts | 18 +- .../src/memory-conflict-advisory.test.ts | 33 + .../src/memory-conflict-advisory.ts | 84 ++- .../src/memory-maintenance-handlers.test.ts | 87 ++- .../src/memory-maintenance-handlers.ts | 39 +- .../keiko-server/src/memory-salience.test.ts | 79 +- packages/keiko-server/src/memory-salience.ts | 107 ++- .../__tests__/figmaSnapshotAdapter.test.ts | 42 ++ .../__tests__/generationPort.test.ts | 13 + .../__tests__/judgePort.test.ts | 19 + .../figmaSnapshotAdapter.ts | 7 +- .../src/qualityIntelligence/generationPort.ts | 11 +- .../src/qualityIntelligence/handoffRoutes.ts | 2 +- .../src/qualityIntelligence/judgePort.ts | 7 +- .../qualityIntelligence/modelPolicyRoutes.ts | 98 ++- .../src/qualityIntelligence/reCheckRoutes.ts | 15 +- .../src/qualityIntelligence/runExecution.ts | 25 +- .../src/qualityIntelligence/runRoutes.ts | 2 +- packages/keiko-server/src/run-engine.test.ts | 74 ++ packages/keiko-server/src/run-engine.ts | 60 +- .../keiko-server/src/run-handlers.test.ts | 79 ++ packages/keiko-server/src/run-handlers.ts | 9 +- packages/keiko-server/src/sse-write.test.ts | 39 +- packages/keiko-server/src/sse-write.ts | 9 +- .../keiko-server/src/terminal-routes.test.ts | 32 +- packages/keiko-server/src/terminal-routes.ts | 9 +- .../src/update-remediation-routes.test.ts | 20 + .../src/update-remediation-routes.ts | 4 +- .../src/update-remediation.test.ts | 60 ++ .../keiko-server/src/update-remediation.ts | 61 +- .../keiko-server/src/voice-control-ws.test.ts | 243 +++++++ packages/keiko-server/src/voice-handlers.ts | 3 +- .../keiko-server/src/voice-live-dictation.ts | 16 +- .../keiko-server/src/voice-realtime.test.ts | 45 +- packages/keiko-server/src/voice-realtime.ts | 24 +- .../src/workspace-index-provider.test.ts | 43 ++ .../src/workspace-index-provider.ts | 40 +- 87 files changed, 4135 insertions(+), 391 deletions(-) create mode 100644 packages/keiko-server/src/correlation-threading.e2e.test.ts create mode 100644 packages/keiko-server/src/gateway-error-diagnostic.test.ts create mode 100644 packages/keiko-server/src/gateway-error-diagnostic.ts diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index 2d550fcaa8..8d58ec37ce 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -626,6 +626,12 @@ "site": "packages/keiko-server/src/observability/server-logger.ts:180", "package": "keiko-server" }, + { + "op": "chat.turn.started", + "category": "gateway", + "site": "packages/keiko-server/src/chat-handlers.ts:1687", + "package": "keiko-server" + }, { "op": "embedding.memory.failed", "category": "embedding", diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index f81af9a6b0..3c30a706ab 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,7 +8,7 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 387, + "uncoveredLines": 385, "totalLines": 4880, "coverage": { "lines": 92.11, @@ -234,15 +234,15 @@ } }, "keiko-server": { - "files": 580, + "files": 581, "uncoveredFiles": 0, - "uncoveredLines": 4516, - "totalLines": 56260, + "uncoveredLines": 4503, + "totalLines": 56309, "coverage": { - "lines": 91.98, - "statements": 89.27, - "branches": 81.89, - "functions": 94.86 + "lines": 92, + "statements": 89.3, + "branches": 81.91, + "functions": 94.91 } }, "keiko-tools": { @@ -261,10 +261,10 @@ "files": 419, "uncoveredFiles": 3, "uncoveredLines": 2967, - "totalLines": 39452, + "totalLines": 39455, "coverage": { "lines": 92.48, - "statements": 89.52, + "statements": 89.53, "branches": 81.8, "functions": 91.28 } diff --git a/packages/keiko-server/src/atlassian/syncRoutes.test.ts b/packages/keiko-server/src/atlassian/syncRoutes.test.ts index 0b18cc43aa..e5db41a78a 100644 --- a/packages/keiko-server/src/atlassian/syncRoutes.test.ts +++ b/packages/keiko-server/src/atlassian/syncRoutes.test.ts @@ -685,3 +685,37 @@ describe("Confluence sync — request validation fail-closed", () => { ); }); }); + +describe("Confluence sync — governed (agent-initiated) start correlation", () => { + it("threads the request's own correlation id into an authority-denied governed start instead of minting one", async () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts) and is already + // in scope in handleStartAtlassianConnectorSync — the governed denial record must reuse it, + // not a disconnected randomUUID(). An authority referencing a runId the server-side registry + // never issued denies fast (authority-invalid) without needing a real envelope setup. + const port = createInMemoryConfluenceFixture({ baseUrl: BASE_URL, spaces: [] }); + const { deps, credential } = depsFor(port); + const ctx = { + ...ctxFor( + "POST", + { authRef: credential.authRef }, + { + spaceKeys: ["ENG"], + authority: { + runId: "unregistered-agent-run", + envelopeDigest: "0".repeat(64), + workspaceRoot: "/nonexistent/workspace", + }, + }, + ), + correlationId: "req-governed-thread-01", + }; + + const result = await handleStartAtlassianConnectorSync(ctx, deps); + + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ + disposition: "denied", + correlationId: "req-governed-thread-01", + }); + }); +}); diff --git a/packages/keiko-server/src/atlassian/syncRoutes.ts b/packages/keiko-server/src/atlassian/syncRoutes.ts index ae41a7ff7d..d62e19f5d0 100644 --- a/packages/keiko-server/src/atlassian/syncRoutes.ts +++ b/packages/keiko-server/src/atlassian/syncRoutes.ts @@ -384,10 +384,10 @@ async function startSyncGoverned( credential: AtlassianCredentialMetadata, body: StartSyncBody, authority: AtlassianActionAuthorityContext, + correlationId: string, ): Promise { const actionType = SYNC_ACTION_TYPE_FOR_PROVIDER[credential.provider]; const connectorId = connectorIdForAuthRef(credential.authRef); - const correlationId = randomUUID(); const targetRef = syncScopeTargetRef(body); const outcome = decideGovernedAtlassianAction(actionType, authority, deps); const denied = (reasonCode: AtlassianConnectorActivityReasonCode): RouteResult => @@ -428,7 +428,16 @@ export function handleStartAtlassianConnectorSync( const credential = requireAtlassianCredential(ctx, guard); const body = validateStartSyncBody(await readJsonObject(ctx.req), credential.provider); if (body.authority !== undefined) { - return startSyncGoverned(deps, guard, credential, body, body.authority); + // Threads the request's own correlation id (ADR-0173 D5 / g12) into the governed-start + // denial/pending-approval/allowed records instead of a disconnected mint. + return startSyncGoverned( + deps, + guard, + credential, + body, + body.authority, + ctx.correlationId ?? randomUUID(), + ); } // Direct human-triggered start: human-approved by construction (ADR-0129; ADR-0128 D5) — // recorded as `allowed` + `human-initiated` on the run's activity record. diff --git a/packages/keiko-server/src/atlassian/writeActionRoutes.test.ts b/packages/keiko-server/src/atlassian/writeActionRoutes.test.ts index 12b935497b..cb35ede758 100644 --- a/packages/keiko-server/src/atlassian/writeActionRoutes.test.ts +++ b/packages/keiko-server/src/atlassian/writeActionRoutes.test.ts @@ -1185,3 +1185,30 @@ describe("write-action route — clear-field validation (KEIKO-0319)", () => { expect(result.status).toBe(400); }); }); + +describe("write-action route — governed action correlation", () => { + it("threads the request's own correlation id into a policy-denied response instead of minting one", async () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts) and is already + // in scope in handleExecuteAtlassianConnectorAction — the governed-action denial record must + // reuse it, not a disconnected randomUUID(). An envelope with no write scope denies fast + // (policy-denied) without needing a provider round-trip. + const guard = guardWith({ count: 0, requests: [] }); + const deniedAuthority = registerEnvelope("autonomous-delivery", []); + const result = (await handleExecuteAtlassianConnectorAction( + { + ...ctx( + { action: ACTION_REQUESTS["transition-issue"], authority: deniedAuthority }, + { authRef: JIRA_AUTH_REF }, + ), + correlationId: "req-write-thread-01", + }, + deps(guard, "autonomous-delivery"), + )) as { status: number; body: Record }; + + expect(result.status).toBe(200); + expect(result.body).toMatchObject({ + disposition: "denied", + correlationId: "req-write-thread-01", + }); + }); +}); diff --git a/packages/keiko-server/src/atlassian/writeActionRoutes.ts b/packages/keiko-server/src/atlassian/writeActionRoutes.ts index ce159dd898..54dfd480f2 100644 --- a/packages/keiko-server/src/atlassian/writeActionRoutes.ts +++ b/packages/keiko-server/src/atlassian/writeActionRoutes.ts @@ -690,9 +690,9 @@ function governedActionResult( credential: AtlassianCredentialMetadata, authority: AtlassianActionAuthorityContext, plan: GovernedActionPlan, + correlationId: string, ): Promise | RouteResult { const connectorId = connectorIdForAuthRef(credential.authRef); - const correlationId = randomUUID(); const denied = (reasonCode: AtlassianConnectorActivityReasonCode): RouteResult => deniedAtlassianActionResult({ connectorId, @@ -785,11 +785,14 @@ export function handleExecuteAtlassianConnectorAction( throw invalid("authority must carry runId, envelopeDigest, and workspaceRoot"); } const input = validateGovernedActionInput(body.action, credential.provider); + // Threads the request's own correlation id (ADR-0173 D5 / g12) into the governed-action + // denial/pending-approval/allowed records instead of a disconnected mint. return governedActionResult( deps, credential, authority, actionPlanFor(guard, credential, input), + ctx.correlationId ?? randomUUID(), ); }); } diff --git a/packages/keiko-server/src/browser-routes.test.ts b/packages/keiko-server/src/browser-routes.test.ts index 6658f4be5f..d0570fee37 100644 --- a/packages/keiko-server/src/browser-routes.test.ts +++ b/packages/keiko-server/src/browser-routes.test.ts @@ -14,8 +14,10 @@ import { createRunRegistry } from "./runs.js"; import { createUiServer, UI_HOST } from "./server.js"; import { EventEmitter } from "node:events"; import type { ServerResponse } from "node:http"; -import { openBrowserSseStream } from "./browser.js"; +import { handleBrowserEvents, openBrowserSseStream } from "./browser.js"; import type { SseBackpressureSignal } from "./sse-write.js"; +import type { RouteContext } from "./routes.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; import { BrowserToolError, type BrowserEventEmitter, @@ -821,3 +823,48 @@ describe("openBrowserSseStream backpressure (KEIKO-0142)", () => { expect(fake.writes).toHaveLength(writesAfterClose); }); }); + +describe("handleBrowserEvents backpressure correlation (ADR-0173 D5 / g12)", () => { + it("threads the request's own correlation id into the backpressure diagnostic instead of minting one", () => { + const fake = makeFakeSseRes(); + fake.writeReturns = false; // rejects the ready frame -> immediate backpressure kill. + const manager = new FakeBrowserSessionManager(); + manager.opened.push("session-thread"); + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (entry) => { + records.push(entry); + }, + }; + const baseDeps: UiHandlerDeps = { + config: undefined, + configPresent: false, + evidenceStore: { + put: (): string => "", + list: (): readonly string[] => [], + get: (): undefined => undefined, + delete: (): undefined => undefined, + }, + env: process.env, + redactor: buildRedactor({}), + registry: createRunRegistry(), + modelPortFactory: (): undefined => undefined, + store: createInMemoryUiStore(), + browser: manager, + diagnostics, + }; + const ctx: RouteContext = { + req: { on: (): void => undefined } as unknown as RouteContext["req"], + res: fake.res, + params: { sessionId: "session-thread" }, + url: new URL("http://127.0.0.1/api/browser/sessions/session-thread/events"), + correlationId: "req-browser-thread-01", + }; + + handleBrowserEvents(ctx, baseDeps); + + expect(records).toHaveLength(1); + expect(records[0]?.source).toBe("sse.browser.backpressure"); + expect(records[0]?.correlationId).toBe("req-browser-thread-01"); + }); +}); diff --git a/packages/keiko-server/src/browser.ts b/packages/keiko-server/src/browser.ts index 156a662b57..5cfc6fbd60 100644 --- a/packages/keiko-server/src/browser.ts +++ b/packages/keiko-server/src/browser.ts @@ -260,12 +260,14 @@ export function handleBrowserEvents(ctx: RouteContext, deps: UiHandlerDeps): Han if (!guard.hasSession(sessionId)) { return { status: 404, body: errorBody("SESSION_NOT_FOUND", "Browser session not found.") }; } + // Threads the request's own correlation id (ADR-0173 D5 / g12) so a later backpressure kill + // joins back to the request that opened this stream instead of a disconnected mint. openBrowserSseStream( ctx.res, guard, sessionId, deps.redactor, - sseBackpressureReporter(deps, "browser"), + sseBackpressureReporter(deps, "browser", ctx.correlationId), ); ctx.req.on("close", () => { ctx.res.end(); diff --git a/packages/keiko-server/src/chat-compaction-evidence.test.ts b/packages/keiko-server/src/chat-compaction-evidence.test.ts index bac6cea3f7..da216504b5 100644 --- a/packages/keiko-server/src/chat-compaction-evidence.test.ts +++ b/packages/keiko-server/src/chat-compaction-evidence.test.ts @@ -425,6 +425,13 @@ describe("chat compaction evidence wiring (ADR-0057 D3)", () => { expect(records[0]?.message).toBe("Audit or evidence persistence failed."); expect(records[0]?.errorClass).toMatch(/^[A-Z][A-Za-z0-9]*$/); expect(records[0]?.correlationId).toMatch(/^[A-Za-z0-9._-]{8,128}$/); + // ADR-0173 D5 / g12: the failure's correlationId is THIS attempt's own runId (same + // derivation the successful-persist tests above pin), not a disconnected `randomUUID()` — + // an operator can join the failure back to the compaction attempt it belongs to. Before the + // fix this was a random UUID (with dashes) and never matched the runId shape below. + expect(records[0]?.correlationId).toBe( + `chat-${sha256Hex("chat-compaction-diagnostic").slice(0, 16)}-t4`, + ); expect(JSON.stringify(records)).not.toContain(SECRET); expect(consoleWarn).not.toHaveBeenCalled(); } finally { diff --git a/packages/keiko-server/src/chat-compaction-evidence.ts b/packages/keiko-server/src/chat-compaction-evidence.ts index 8bfc04ce2e..b413e3f107 100644 --- a/packages/keiko-server/src/chat-compaction-evidence.ts +++ b/packages/keiko-server/src/chat-compaction-evidence.ts @@ -11,6 +11,7 @@ import { resolveCostClass } from "@oscharko-dev/keiko-model-gateway"; import { sha256Hex } from "@oscharko-dev/keiko-security"; import type { ContextCompactionRecord } from "@oscharko-dev/keiko-contracts"; import { randomUUID } from "node:crypto"; +import { isValidCorrelationId } from "./correlation.js"; import type { UiHandlerDeps } from "./deps.js"; import { currentAuditRedactString, currentRedactionSecrets } from "./deps.js"; import { @@ -49,11 +50,16 @@ export function persistChatCompactionEvidence( return; } const record = input.compaction; + // Computed before the try so a persistence failure can still report under it (ADR-0173 D5 / + // g12): this run's own runId already ties the diagnostic back to the SAME compaction evidence + // attempt an operator would otherwise have to guess at from a disconnected mint. + let runId: string | undefined; try { const chatIdHash = sha256Hex(input.chatId); + runId = compactionRunId(chatIdHash, input.messageCount); persistCompactionEvidence( { - runId: compactionRunId(chatIdHash, input.messageCount), + runId, modelId: input.modelId, records: [record], startedAt: input.startedAt, @@ -75,10 +81,11 @@ export function persistChatCompactionEvidence( // Best-effort stays best-effort — the send is unaffected — but the failure is no longer a // `console.warn` carrying the raw error object on a channel production never overrode. It goes to // the server's single redacted diagnostic sink so a compaction-evidence gap is observable. + const correlationId = runId !== undefined && isValidCorrelationId(runId) ? runId : randomUUID(); emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "chat.compaction.evidence", source: "chat-compaction-evidence", error, diff --git a/packages/keiko-server/src/chat-compaction-model-summary.test.ts b/packages/keiko-server/src/chat-compaction-model-summary.test.ts index a61fcf6353..9e4d859465 100644 --- a/packages/keiko-server/src/chat-compaction-model-summary.test.ts +++ b/packages/keiko-server/src/chat-compaction-model-summary.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; import { CONTEXT_ENGINEERING_SCHEMA_VERSION, @@ -13,12 +14,14 @@ import { import { sha256Hex } from "@oscharko-dev/keiko-security"; import { createDefaultChatCapability, + type GatewayCallRequest, type GatewayConfig, type GatewayRequest, type NormalizedResponse, } from "@oscharko-dev/keiko-model-gateway"; import type { ModelPort } from "@oscharko-dev/keiko-harness"; import type { UiHandlerDeps } from "./deps.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; import type { ChatMessage } from "./store/index.js"; import { enrichChatCompactionWithModelSummary } from "./chat-compaction-model-summary.js"; @@ -113,6 +116,7 @@ function deps( store: EvidenceStore, model: ModelPort | undefined, supportsResponseFormat = true, + diagnostics?: ServerDiagnosticSink, ): UiHandlerDeps { return { config: gatewayConfig(supportsResponseFormat), @@ -121,6 +125,7 @@ function deps( env: {}, redactor, modelPortFactory: () => model, + diagnostics, } as unknown as UiHandlerDeps; } @@ -198,6 +203,14 @@ function neverResolvingModel(): ModelPort { }; } +function rejectingModel(): ModelPort { + return { + call(): Promise { + return Promise.reject(new Error("summary model transport failed")); + }, + }; +} + function defaultEnrichmentInput( messageCount = 2, ): Parameters[1] { @@ -260,6 +273,21 @@ describe("enrichChatCompactionWithModelSummary", () => { expectStructuredSummaryPersisted(persisted); }); + // ADR-0173 D5: this best-effort background summarization has no live HTTP request in scope, so + // the chat's own (internally-minted, opaque) id is the stable correlation key stamped into the + // model's GatewayCallRequest.logContext. + it("stamps the chat id into the model gateway call's logContext", async () => { + const store = createInMemoryEvidenceStore(); + const calls: GatewayRequest[] = []; + await enrichChatCompactionWithModelSummary( + deps(store, structuredSummaryModel(calls)), + defaultEnrichmentInput(), + ); + + const request = requireFirstRequest(calls); + expect((request as GatewayCallRequest).logContext?.correlationId).toBe(CHAT_ID); + }); + it("keeps a safe legacy text fallback when the model lacks response-format support", async () => { const store = createInMemoryEvidenceStore(); const calls: GatewayRequest[] = []; @@ -403,6 +431,44 @@ describe("enrichChatCompactionWithModelSummary", () => { expect(persisted.content).toBe(""); }); + // ADR-0173 D5 g25 — a scheduled-enrichment call failure used to reach only a bare `console.warn` + // (see the source-grep pin below); background summarization has no live REQUEST correlation id + // in scope, so the chat's own id — already the stable job key `callModelWithTimeout` labels its + // own call with — is the join key this diagnostic carries instead. + it("routes a model-call failure through the diagnostic sink, keyed by chatId", async () => { + const store = createInMemoryEvidenceStore(); + const events: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (record): void => { + events.push(record); + }, + }; + + await enrichChatCompactionWithModelSummary( + deps(store, rejectingModel(), true, diagnostics), + defaultEnrichmentInput(9), + ); + + expect(events).toHaveLength(1); + const [event] = events; + if (event === undefined) throw new Error("expected a diagnostic record"); + expect(event.correlationId).toBe(CHAT_ID); + expect(event.operation).toBe("chat.compaction.summary"); + expect(event.source).toBe("chat.compaction.model-summary"); + expect(event.errorClass).toBe("Error"); + // The diagnostic is additive: the turn still gets a usable fallback summary either way. + const persisted = requireModelSummary(store, 9); + expect(persisted.failureReason).toBe("model-unavailable"); + }); + + it("no longer logs a scheduled-enrichment failure through console.warn", () => { + const source = readFileSync( + new URL("./chat-compaction-model-summary.ts", import.meta.url), + "utf8", + ); + expect(source).not.toContain("console.warn("); + }); + it("accepts a structured summary needing no safety redaction despite cosmetic whitespace", async () => { const store = createInMemoryEvidenceStore(); const calls: GatewayRequest[] = []; diff --git a/packages/keiko-server/src/chat-compaction-model-summary.ts b/packages/keiko-server/src/chat-compaction-model-summary.ts index bc255bf907..8da2fe82fa 100644 --- a/packages/keiko-server/src/chat-compaction-model-summary.ts +++ b/packages/keiko-server/src/chat-compaction-model-summary.ts @@ -23,6 +23,7 @@ import { persistChatCompactionEvidence, type ChatCompactionEvidenceInput, } from "./chat-compaction-evidence.js"; +import { emitServerDiagnostic, serverDiagnosticFromError } from "./diagnostics-log.js"; const MODEL_SUMMARY_TIMEOUT_MS = 15_000; const MAX_SOURCE_TURNS = 16; @@ -154,7 +155,7 @@ export async function enrichChatCompactionWithModelSummary( const modelSummary = model === undefined ? failureModelSummary(record, input.modelId, "unavailable", "model-unavailable") - : await buildModelSummary(model, deps.redactor, input, record, prompt, responseMode); + : await buildModelSummary(model, deps, input, record, prompt, responseMode); if (modelSummary !== undefined) { persistChatCompactionEvidence(deps, { ...input, @@ -162,22 +163,31 @@ export async function enrichChatCompactionWithModelSummary( }); } } catch (error) { - logSummaryFailure(error); + logSummaryFailure(deps, input.chatId, error); } } async function buildModelSummary( model: ModelPort, - redactor: Redactor, + deps: UiHandlerDeps, input: ChatCompactionModelSummaryInput, record: ContextCompactionRecord, prompt: string, responseMode: ModelSummaryResponseMode, ): Promise { - const result = await callModelWithTimeout(model, input.modelId, prompt, responseMode); + // Background best-effort summarization has no live request correlation id in scope; the chat's + // own id is the stable job key an operator greps by, mirroring the `jobId`-as-correlationId + // convention background jobs elsewhere in the BFF already use (ADR-0173 D5). + const result = await callModelWithTimeout( + model, + input.modelId, + prompt, + responseMode, + input.chatId, + ); return result.kind === "response" - ? modelSummaryFromResponse(record, input.modelId, result.response, redactor, responseMode) - : modelSummaryFromCallFailure(record, input.modelId, result); + ? modelSummaryFromResponse(record, input.modelId, result.response, deps.redactor, responseMode) + : modelSummaryFromCallFailure(deps, input.chatId, record, input.modelId, result); } function modelSummaryFromResponse( @@ -222,6 +232,8 @@ function modelSummaryFromPayload( } function modelSummaryFromCallFailure( + deps: UiHandlerDeps, + correlationId: string, record: ContextCompactionRecord, modelId: string, result: Exclude, @@ -229,7 +241,7 @@ function modelSummaryFromCallFailure( if (result.kind === "timed-out") { return failureModelSummary(record, modelId, "timed-out", "timed-out"); } - logSummaryFailure(result.error); + logSummaryFailure(deps, correlationId, result.error); return failureModelSummary(record, modelId, "unavailable", "model-unavailable"); } @@ -238,6 +250,7 @@ async function callModelWithTimeout( modelId: string, prompt: string, responseMode: ModelSummaryResponseMode, + correlationId: string, ): Promise { const controller = new AbortController(); let timer: ReturnType | undefined; @@ -263,6 +276,7 @@ async function callModelWithTimeout( ...(responseMode === "structured" ? { responseFormat: MODEL_SUMMARY_RESPONSE_FORMAT } : {}), + logContext: { correlationId }, }, controller.signal, ), @@ -663,10 +677,19 @@ function failureModelSummary( }); } -function logSummaryFailure(error: unknown): void { - // eslint-disable-next-line no-console - console.warn( - "chat-compaction-model-summary: enrichment failed (best-effort, send unaffected)", - error, +// Replaces a bare `console.warn` (ADR-0173 D5 g25): a best-effort background enrichment failure +// (send unaffected — the compaction record itself already persisted) is still an operator-visible +// event, not a silent one. `correlationId` is the chat id (see `buildModelSummary` above): this +// background job has no live request id in scope, so the chat's own id is the stable join key. +function logSummaryFailure(deps: UiHandlerDeps, correlationId: string, error: unknown): void { + emitServerDiagnostic( + deps.diagnostics, + serverDiagnosticFromError({ + correlationId, + operation: "chat.compaction.summary", + source: "chat.compaction.model-summary", + error, + redact: (message) => String(deps.redactor(message)), + }), ); } diff --git a/packages/keiko-server/src/chat-handlers.test.ts b/packages/keiko-server/src/chat-handlers.test.ts index 2d2ca3dec0..dd2a02e845 100644 --- a/packages/keiko-server/src/chat-handlers.test.ts +++ b/packages/keiko-server/src/chat-handlers.test.ts @@ -1,19 +1,34 @@ -import { mkdirSync, mkdtempSync, realpathSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync } from "node:fs"; import type { IncomingMessage, ServerResponse } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Readable } from "node:stream"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { MAX_DESKTOP_CHAT_CLIENT_TURN_ID_CHARS } from "@oscharko-dev/keiko-contracts/bff-wire"; -import { parseGatewayConfig } from "@oscharko-dev/keiko-model-gateway"; import { + parseGatewayConfig, + type GatewayConfig, + type NormalizedResponse, +} from "@oscharko-dev/keiko-model-gateway"; +import type { ModelPort } from "@oscharko-dev/keiko-harness"; +import { + chatTurnShapeFields, handleCreateDesktopChat, handleSendDesktopChat, parseClientTurnId, parseExpectedGroundingScopeIdentity, } from "./chat-handlers.js"; -import { buildUiHandlerDeps, type UiHandlerDeps } from "./deps.js"; +import { buildRedactor, buildUiHandlerDeps, type UiHandlerDeps } from "./deps.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, +} from "./observability/index.js"; import type { RouteContext } from "./routes.js"; +import { createRunRegistry } from "./runs.js"; +import { createInMemoryUiStore } from "./store/index.js"; const VALID_GROUNDING_SCOPE_IDENTITY = `gsi-v1:${"a".repeat(64)}`; const INVALID_CLIENT_TURN_ID = { @@ -324,3 +339,327 @@ describe("desktop chat production gateway reuse", () => { } }); }); + +// ADR-0173 D5 g9 — `chatTurnShapeFields` is the exact production formula `chat.turn.started` +// logs; these tests derive their expectations by calling it directly rather than restating the +// counting logic as a second copy that could drift from it (AGENTS.md §7). +describe("chatTurnShapeFields", () => { + it("counts a 3-message turn by role and totals a single image attachment", () => { + const messages = [{ role: "system" }, { role: "user" }, { role: "assistant" }]; + const attachments = [ + { kind: "image" as const, mimeType: "image/png", sizeBytes: 40_000 }, + { kind: "document" as const, mimeType: "application/pdf", sizeBytes: 12_000 }, + ]; + + expect(chatTurnShapeFields(messages, attachments)).toEqual({ + messageCount: 3, + roleCounts: { system: 1, user: 1, assistant: 1, tool: 0 }, + toolCount: 0, + imageAttachmentCount: 1, + imageAttachmentBytes: 40_000, + }); + }); + + it("sums bytes across multiple image attachments and ignores an unrecognised role", () => { + const messages = [{ role: "system" }, { role: "tool" }, { role: "unknown-role" }]; + const attachments = [ + { kind: "image" as const, mimeType: "image/png", sizeBytes: 1_000 }, + { kind: "image" as const, mimeType: "image/jpeg", sizeBytes: 2_500 }, + ]; + + const fields = chatTurnShapeFields(messages, attachments); + expect(fields.messageCount).toBe(3); + expect(fields.roleCounts).toEqual({ system: 1, user: 0, assistant: 0, tool: 1 }); + expect(fields.toolCount).toBe(1); + expect(fields.imageAttachmentCount).toBe(2); + expect(fields.imageAttachmentBytes).toBe(3_500); + }); + + it("returns zeroed counts for an empty turn", () => { + expect(chatTurnShapeFields([], [])).toEqual({ + messageCount: 0, + roleCounts: { system: 0, user: 0, assistant: 0, tool: 0 }, + toolCount: 0, + imageAttachmentCount: 0, + imageAttachmentBytes: 0, + }); + }); +}); + +function turnShapeGatewayConfig(modelId: string): GatewayConfig { + return { + // `listConfiguredCapabilities` cross-references `capabilities` against `providers` by + // `modelId` (`model-selection.ts`) — a capability with no matching provider entry is + // filtered out of the registry, so `modelCapabilityRegistry` would see this model as + // "not chat-capable" without one, even though this test never dials out to it. + providers: [ + { + modelId, + baseUrl: "https://provider.example.invalid/v1", + apiKey: "unused-test-key", + timeoutMs: 5_000, + maxRetries: 0, + retryBaseDelayMs: 1, + }, + ], + circuitBreaker: { failureThreshold: 5, cooldownMs: 1_000, halfOpenProbes: 1 }, + capabilities: [ + { + id: modelId, + kind: "chat", + contextWindow: 64_000, + maxOutputTokens: 4_096, + toolCalling: false, + structuredOutput: false, + streaming: true, + supportsImageInput: false, + supportsDocumentInput: false, + workflowEligible: false, + costClass: "medium", + latencyClass: "standard", + throughputHint: "test", + preferredUseCases: [], + knownLimitations: [], + }, + ], + }; +} + +function turnShapeModel(): ModelPort { + return { + call(request): Promise { + return Promise.resolve({ + modelId: request.modelId, + content: "Hallo!", + finishReason: "stop", + toolCalls: [], + structuredOutput: null, + usage: { + requestId: "turn-shape-test", + promptTokens: 3, + completionTokens: 2, + latencyMs: 5, + costClass: "low", + }, + }); + }, + }; +} + +describe("chat.turn.started", () => { + afterEach(() => { + resetServerLogger(); + }); + + it("logs messageCount/roleCounts once per turn, keyed to the request correlation id", async () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const modelId = "turn-shape-chat"; + const root = mkdtempSync(join(realpathSync(tmpdir()), "keiko-chat-turn-shape-")); + const projectPath = join(root, "repo"); + try { + mkdirSync(projectPath); + const store = createInMemoryUiStore(); + store.createProject(projectPath, "repo"); + const chat = store.createChat(projectPath, "Turn shape", modelId); + const deps: UiHandlerDeps = { + config: turnShapeGatewayConfig(modelId), + configPresent: true, + evidenceStore: { + put: () => "", + list: () => [], + get: () => undefined, + delete: () => undefined, + }, + env: {}, + redactor: buildRedactor({}), + registry: createRunRegistry(), + modelPortFactory: () => turnShapeModel(), + store, + }; + // A prior completed turn — sent through the SAME production path, not hand-seeded into the + // store — so the second turn's assembled prompt carries 4 messages: the fixed system + // prompt, the prior user+assistant pair, and the current user message being sent. The exact + // "3-message" formula case is covered precisely by chatTurnShapeFields above; this send only + // needs a real, non-trivial shape to prove the wiring counts what the assembled prompt + // actually contains, not a restated expectation. + const priorResult = await handleSendDesktopChat( + { + ...requestContext({ + chatId: chat.id, + projectPath, + modelId, + content: "What is on the roadmap?", + }), + correlationId: "turn-shape-prior", + }, + deps, + ); + if (priorResult.status !== 200) { + throw new Error( + `expected the seeded prior turn to succeed: ${JSON.stringify(priorResult)}`, + ); + } + const result = await handleSendDesktopChat( + { + ...requestContext({ chatId: chat.id, projectPath, modelId, content: "Hello there" }), + correlationId: "turn-shape-correlation-1", + }, + deps, + ); + expect(result.status).toBe(200); + const events = sink.events.filter( + (event) => + event.op === "chat.turn.started" && event.correlationId === "turn-shape-correlation-1", + ); + expect(events).toHaveLength(1); + const [event] = events; + if (event === undefined) throw new Error("expected a chat.turn.started event"); + expect(event.category).toBe("gateway"); + expect(event.extra).toEqual({ + messageCount: 4, + roleCounts: { system: 1, user: 2, assistant: 1, tool: 0 }, + toolCount: 0, + imageAttachmentCount: 0, + imageAttachmentBytes: 0, + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +// ADR-0173 D5 g25 — the buffered `/api/desktop/chat` path used to map a GatewayError straight to +// an HTTP body with no operator diagnostic at all, unlike the SSE `/api/desktop/chat/stream` path +// (`chat-stream-handlers.test.ts` pins that side). This is the fails-before proof for the shared +// symmetry fix: before it, `events` below stayed empty on a RateLimitError. +describe("desktopChatErrorResult gateway diagnostic symmetry", () => { + it("emits the same diagnostic shape as the streaming path on a RateLimitError", async () => { + const fixture = await createGatewayBreakerFixture(); + try { + const events: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (record): void => { + events.push(record); + }, + }; + const deps: UiHandlerDeps = { ...fixture.deps, diagnostics }; + const fetchSpy = vi.fn(() => + Promise.resolve( + new Response(JSON.stringify({ error: { message: "slow down" } }), { + status: 429, + headers: { "content-type": "application/json", "retry-after": "2" }, + }), + ), + ); + vi.stubGlobal("fetch", fetchSpy); + + const result = await handleSendDesktopChat( + { + ...requestContext({ + chatId: fixture.chatId, + projectPath: fixture.projectPath, + modelId: "breaker-chat", + content: "please respond", + }), + correlationId: "rate-limit-correlation-1", + }, + deps, + ); + + expect(gatewayErrorCode(result)).toBe("GATEWAY_RATE_LIMIT"); + expect(result.status).toBe(503); + expect(events).toHaveLength(1); + const [event] = events; + if (event === undefined) throw new Error("expected a diagnostic record"); + expect(event.correlationId).toBe("rate-limit-correlation-1"); + expect(event.operation).toBe("POST /api/desktop/chat"); + expect(event.source).toBe("chat.send"); + expect(event.errorClass).toBe("RateLimitError"); + } finally { + vi.unstubAllGlobals(); + await disposeGatewayBreakerFixture(fixture); + } + }); +}); + +// ADR-0173 D5 g25 — scheduling wrapper around `enrichChatCompactionWithModelSummary` +// (`chat-handlers.ts`'s own `logCompactionSummaryFailure`, mirroring `recordPostCommitMemoryFailure` +// at chat-handlers.ts:1444-1452). `enrichChatCompactionWithModelSummary` already absorbs every +// failure it can reach internally (see chat-compaction-model-summary.test.ts), so this wrapper's +// own catch is exercised here by mocking that import — the only way to reach it without relying on +// an internal implementation detail of the mocked module. +describe("logCompactionSummaryFailure", () => { + afterEach(() => { + vi.doUnmock("./chat-compaction-model-summary.js"); + vi.resetModules(); + }); + + it("routes a scheduled-enrichment rejection through the diagnostic sink instead of console.warn", async () => { + // `chat-handlers.js` (and its static import of `chat-compaction-model-summary.js`) is already + // cached from this file's top-level imports; the cache must be cleared BEFORE re-importing, + // or the fresh import below would resolve to the same already-loaded, unmocked module graph. + vi.resetModules(); + vi.doMock("./chat-compaction-model-summary.js", () => ({ + enrichChatCompactionWithModelSummary: (): Promise => + Promise.reject(new Error("scheduled enrichment blew up")), + })); + const { recordChatCompaction } = await import("./chat-handlers.js"); + const events: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (record): void => { + events.push(record); + }, + }; + const deps = { + evidenceStore: { + put: () => "", + list: () => [], + get: () => undefined, + delete: () => undefined, + }, + env: {}, + redactor: (value: unknown): unknown => value, + diagnostics, + } as unknown as UiHandlerDeps; + + recordChatCompaction(deps, { + compaction: { + laneId: "history-summary", + reason: "exceeded effective input budget", + itemsBefore: 2, + itemsAfter: 1, + tokensBefore: 100, + tokensAfter: 20, + preservedFacts: [], + decisions: [], + }, + request: { chatId: "chat-scheduling-failure-1" }, + modelId: "scheduling-failure-model", + messageCount: 1, + startedAt: Date.now(), + historyPrefix: [], + correlationId: "scheduling-failure-correlation-1", + } as never); + + // The failure surfaces via a detached `setImmediate` + a rejected promise's `.catch`; give + // both a turn of the event loop to run before asserting. + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const scheduled = events.filter( + (event) => event.operation === "chat.compaction.summary.scheduled", + ); + expect(scheduled).toHaveLength(1); + const [event] = scheduled; + if (event === undefined) throw new Error("expected a diagnostic record"); + expect(event.correlationId).toBe("scheduling-failure-correlation-1"); + expect(event.source).toBe("chat.compaction.model-summary"); + expect(event.errorClass).toBe("Error"); + }); + + it("no longer logs a scheduled-enrichment failure through console.warn", () => { + const source = readFileSync(new URL("./chat-handlers.ts", import.meta.url), "utf8"); + expect(source).not.toContain("console.warn("); + }); +}); diff --git a/packages/keiko-server/src/chat-handlers.ts b/packages/keiko-server/src/chat-handlers.ts index 6d4a12bf03..8078d52ded 100644 --- a/packages/keiko-server/src/chat-handlers.ts +++ b/packages/keiko-server/src/chat-handlers.ts @@ -127,7 +127,14 @@ import { embedAndStoreMemory } from "./memory-embedding.js"; import { recordMemoryAudit } from "./memory-audit-handler.js"; import { recordAutoAcceptedMemoryCaptureDecision } from "./memory-capture-audit.js"; import { scheduleMemorySalienceCapture } from "./memory-salience.js"; -import { contentFreeErrorClass, emitServerDiagnostic } from "./diagnostics-log.js"; +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; +import { + contentFreeErrorClass, + emitServerDiagnostic, + serverDiagnosticFromError, +} from "./diagnostics-log.js"; +import { emitGatewayErrorDiagnostic } from "./gateway-error-diagnostic.js"; +import { getServerLogger } from "./observability/index.js"; import { assertUsableAssistantContent, isLegacyEmptyAssistantPlaceholder, @@ -356,12 +363,31 @@ function gatewayErrorStatus(error: GatewayError): number { return 502; } -function gatewayErrorResult(error: GatewayError, deps: UiHandlerDeps): RouteResult { +// ADR-0173 D5 g25 — every GatewayError this path maps to a response also reaches the redacted +// operator diagnostic sink, the same symmetry `chat-stream-handlers.ts`'s SSE path already had. +// `emitDiagnostic` defaults on for the normal (response-returning) callers below and is turned off +// by the ONE caller that already emitted its own broader diagnostic for this exact error a moment +// earlier and calls back in purely to reuse the code/message mapping (`chat-stream-handlers.ts`'s +// `errorEvent`) — without it that caller would double-log the same failure. +function gatewayErrorResult( + error: GatewayError, + deps: UiHandlerDeps, + correlationId: string | undefined, + emitDiagnostic: boolean, +): RouteResult { + if (emitDiagnostic) { + emitGatewayErrorDiagnostic(deps, error, correlationId, "POST /api/desktop/chat", "chat.send"); + } const status = gatewayErrorStatus(error); return { status, body: errorBody(error.code, redactErrorMessage(error.message, deps)) }; } -export function desktopChatErrorResult(error: unknown, deps: UiHandlerDeps): RouteResult { +export function desktopChatErrorResult( + error: unknown, + deps: UiHandlerDeps, + correlationId?: string, + emitDiagnostic = true, +): RouteResult { if (error instanceof ConversationAttachmentStoreError) { return { status: 409, @@ -369,7 +395,7 @@ export function desktopChatErrorResult(error: unknown, deps: UiHandlerDeps): Rou }; } if (error instanceof GatewayError) { - return gatewayErrorResult(error, deps); + return gatewayErrorResult(error, deps, correlationId, emitDiagnostic); } if (error instanceof UiStoreError) { return { @@ -1135,11 +1161,23 @@ export function maybeRunChatAutoMaintenance( vault: MemoryVaultStore, state: AutoMaintenanceState = memoryMaintenanceCursor, nowMs: number = Date.now(), + // The triggering chat request's own correlation id, when known (ADR-0173 D5 / g12). This pass + // is genuinely background-originated (opportunistic, rate-limited, may run well after the turn + // that triggered it), so it mints its own id below rather than reusing the request's outright — + // but a known request id still rides as `parentCorrelationId` so an operator can join this + // pass's diagnostics back to the request that opportunistically triggered it. + requestCorrelationId?: string, ): void { if (deps.env.KEIKO_MEMORY_AUTO_MAINTAIN === "0") return; if (!isMaintenanceDue(state.lastRunAtMs, nowMs)) return; + // Minted ONCE here, at the start of this maintenance pass, rather than inside each helper's own + // catch block: the retention-policy read, the autonomy-mode read, and the maintenance sweep + // itself are three separate failure points of the SAME pass, and used to mint three disconnected + // ids — making it impossible for an operator to tell they came from one invocation (ADR-0173 D5 + // / g12). + const correlationId = randomUUID(); const multipliers = memorySemanticizationMultipliers(deps.env); - const retention = resolveMemoryRetentionPolicy(deps); + const retention = resolveMemoryRetentionPolicy(deps, correlationId); // A malformed retention setting disables only the retention phase. The resolver already emits a // diagnostic; promotion, consolidation, supersession, and fade must keep running so one invalid // optional setting cannot silently suspend all pre-existing vault maintenance. @@ -1147,17 +1185,20 @@ export function maybeRunChatAutoMaintenance( maybeRunAutoMaintenance(vault, memoryMaintenanceAuditSink(deps), state, { nowMs, enabled: true, - autonomyMode: resolveMaintenanceAutonomyMode(deps), + autonomyMode: resolveMaintenanceAutonomyMode(deps, correlationId), ...(multipliers !== undefined ? { decayHalfLifeMultiplierByType: multipliers } : {}), ...(retentionPolicy !== undefined ? { retentionPolicy } : {}), onFailure: (error): void => { emitServerDiagnostic(deps.diagnostics, { - correlationId: randomUUID(), + correlationId, timestamp: new Date(Date.now()).toISOString(), operation: "chat.memory.auto-maintenance", source: "chat.memory.maintenance", errorClass: contentFreeErrorClass(error), message: "chat-memory-auto-maintenance-failed", + ...(requestCorrelationId === undefined + ? {} + : { parentCorrelationId: requestCorrelationId }), }); }, }); @@ -1584,6 +1625,71 @@ export function buildGatewayAssembly( return selected; } +// ADR-0173 D5 g9 — the INPUT shape of a chat turn, never its content: how many messages the +// assembled prompt carries (split by role) and how many image attachments rode along (count + +// bytes). Logged once, at the point the assembled prompt and the parsed attachments are both +// already in hand, so an agent reconstructing a defect from the activity log can tell "a +// 40-message context with two images" from "a bare one-line question" without ever seeing a +// token of either. Deliberately NOT the speculative JSON shape-skeleton feature (positional +// locator tuples over the request/response bodies) — that stays a documented forward guardrail, +// not built here (final-design.md Decisions Log D14). +const CHAT_TURN_ROLES = ["system", "user", "assistant", "tool"] as const; +type ChatTurnRole = (typeof CHAT_TURN_ROLES)[number]; +const CHAT_TURN_ROLE_SET: ReadonlySet = new Set(CHAT_TURN_ROLES); + +export interface ChatTurnShapeFields { + readonly messageCount: number; + readonly roleCounts: Readonly>; + // Denormalized copy of roleCounts.tool: a scalar an agent can grep for directly, without + // descending into the nested extra.roleCounts object. + readonly toolCount: number; + readonly imageAttachmentCount: number; + readonly imageAttachmentBytes: number; +} + +// Exported so its co-located test derives its expectations by calling this exact production +// formula (AGENTS.md §7) rather than restating the counting logic as a second copy that could +// drift from it. +export function chatTurnShapeFields( + messages: readonly { readonly role: string }[], + attachments: readonly ConversationAttachment[], +): ChatTurnShapeFields { + const roleCounts: Record = { system: 0, user: 0, assistant: 0, tool: 0 }; + for (const message of messages) { + if (CHAT_TURN_ROLE_SET.has(message.role)) { + roleCounts[message.role as ChatTurnRole] += 1; + } + } + let imageAttachmentCount = 0; + let imageAttachmentBytes = 0; + for (const attachment of attachments) { + if (attachment.kind === "image") { + imageAttachmentCount += 1; + imageAttachmentBytes += attachment.sizeBytes; + } + } + return { + messageCount: messages.length, + roleCounts, + toolCount: roleCounts.tool, + imageAttachmentCount, + imageAttachmentBytes, + }; +} + +function logChatTurnStarted( + correlationId: string | undefined, + messages: readonly { readonly role: string }[], + attachments: readonly ConversationAttachment[], +): void { + getServerLogger().info({ + category: "gateway", + op: "chat.turn.started", + correlationId, + extra: { ...chatTurnShapeFields(messages, attachments) }, + }); +} + export interface ChatCompactionTurn { readonly compaction: ConversationCompactionOutcome["compaction"]; readonly request: SendDesktopChatRequest; @@ -1591,6 +1697,10 @@ export interface ChatCompactionTurn { readonly messageCount: number; readonly startedAt: number; readonly historyPrefix: readonly ChatMessage[]; + // ADR-0173 D5 g25 — the request's correlation id, carried through so a scheduled-enrichment + // failure (logged well after the response left, from inside a detached setImmediate) still + // joins back to the request that triggered it instead of standing alone in the activity log. + readonly correlationId: string | undefined; } // ADR-0057 D3: best-effort persist of the turn's compaction record AFTER the response completes. @@ -1607,13 +1717,14 @@ export function recordChatCompaction(deps: UiHandlerDeps, turn: ChatCompactionTu finishedAt: Date.now(), } satisfies ChatCompactionEvidenceInput; persistChatCompactionEvidence(deps, input); - scheduleCompactionModelSummary(deps, input, turn.historyPrefix); + scheduleCompactionModelSummary(deps, input, turn.historyPrefix, turn.correlationId); } function scheduleCompactionModelSummary( deps: UiHandlerDeps, input: ChatCompactionEvidenceInput, historyPrefix: readonly ChatMessage[], + correlationId: string | undefined, ): void { if ( input.compaction === undefined || @@ -1625,7 +1736,7 @@ function scheduleCompactionModelSummary( const handle = setImmediate(() => { void enrichChatCompactionWithModelSummary(deps, { ...input, historyPrefix }) .catch((error: unknown) => { - logCompactionSummaryFailure(error); + logCompactionSummaryFailure(deps, correlationId, error); }) .finally(() => { pendingCompactionSummaries -= 1; @@ -1634,9 +1745,25 @@ function scheduleCompactionModelSummary( handle.unref(); } -function logCompactionSummaryFailure(error: unknown): void { - // eslint-disable-next-line no-console - console.warn("chat-compaction-model-summary: scheduled enrichment failed", error); +// Replaces a bare `console.warn` (ADR-0173 D5 g25): the scheduled enrichment runs detached from +// the request/response cycle, so its own internal try/catch (`enrichChatCompactionWithModelSummary`) +// already routes the ordinary failure paths to a diagnostic — this outer catch only fires for a +// failure that escapes THAT guard, and must not go back to being invisible. +function logCompactionSummaryFailure( + deps: UiHandlerDeps, + correlationId: string | undefined, + error: unknown, +): void { + emitServerDiagnostic( + deps.diagnostics, + serverDiagnosticFromError({ + correlationId: correlationId ?? UNKNOWN_CORRELATION_ID, + operation: "chat.compaction.summary.scheduled", + source: "chat.compaction.model-summary", + error, + redact: (message) => String(deps.redactor(message)), + }), + ); } function buildRegenerateGatewayAssembly( @@ -1796,6 +1923,7 @@ async function resolveBufferedMemory( prepared: PreparedDesktopChatSend, admitted: AdmittedTurnHandle, abortSignal: AbortSignal, + correlationId: string | undefined, ): Promise { const { request, memoryContext } = prepared; let memory: ConversationMemoryResultWire; @@ -1807,7 +1935,9 @@ async function resolveBufferedMemory( } catch (error) { const cancelled = requestSignalAborted(abortSignal); settleRejectedDesktopChatTurn(deps, prepared, admitted, cancelled ? "cancelled" : "failed"); - return cancelled ? requestCancelledResult() : desktopChatErrorResult(error, deps); + return cancelled + ? requestCancelledResult() + : desktopChatErrorResult(error, deps, correlationId); } // Cancellation that lands during retrieval must be settled HERE, before assembly and the // provider call — this is still pre-provider, so a legacy row is discarded rather than @@ -1854,6 +1984,7 @@ async function persistModelChatTurn( deps: UiHandlerDeps, prepared: PreparedDesktopChatSend, abortSignal: AbortSignal, + correlationId: string | undefined, ): Promise { const { request } = prepared; // ADR-0057 D3: pin the pre-user-message count BEFORE createUserMessage stores the turn, so the @@ -1867,11 +1998,14 @@ async function persistModelChatTurn( abortSignal, messageCountBeforeTurn, startedAt, + correlationId, ); } catch (error) { const cancelled = requestSignalAborted(abortSignal); failDesktopChatTurn(deps, request, cancelled ? "cancelled" : "failed"); - return cancelled ? requestCancelledResult() : desktopChatErrorResult(error, deps); + return cancelled + ? requestCancelledResult() + : desktopChatErrorResult(error, deps, correlationId); } } @@ -1881,6 +2015,7 @@ async function executeBufferedModelTurn( abortSignal: AbortSignal, messageCountBeforeTurn: number, startedAt: number, + correlationId: string | undefined, ): Promise { const { request, modelId } = prepared; const outcome = admitBufferedModelTurn(deps, prepared); @@ -1888,21 +2023,22 @@ async function executeBufferedModelTurn( const { admitted, executionAdmission } = outcome; const { userMessage } = admitted; const gatewayTurn = captureGatewayTurnSnapshot(deps, request, userMessage); - const memory = await resolveBufferedMemory(deps, prepared, admitted, abortSignal); + const memory = await resolveBufferedMemory(deps, prepared, admitted, abortSignal, correlationId); if (isRouteResult(memory)) return memory; - const assembly = assemblyWithConversationImages( - deps, - request, - modelId, - buildGatewayAssembly(deps, request, memory, modelId, gatewayTurn), - ); + const baseAssembly = buildGatewayAssembly(deps, request, memory, modelId, gatewayTurn); + // Logged from the base assembly, BEFORE image content parts are spliced in: image delivery can + // still fail its own (unrelated) authority/session check below, and this shape evidence must + // exist either way. Splicing only augments the final message's contentParts, never message + // count or role — so the counted shape is identical from either assembly. + logChatTurnStarted(correlationId, baseAssembly.messages, request.attachments); + const assembly = assemblyWithConversationImages(deps, request, modelId, baseAssembly); const model = bufferedModelAtProviderBoundary(deps, modelId, executionAdmission); if (isRouteResult(model)) { settleRejectedDesktopChatTurn(deps, prepared, admitted); return model; } const response = await model.call( - { modelId, messages: assembly.messages, stream: false }, + { modelId, messages: assembly.messages, stream: false, logContext: { correlationId } }, abortSignal, ); const cancelledAfterCall = bufferedTurnCancellationResult(deps, prepared, abortSignal); @@ -1918,6 +2054,7 @@ async function executeBufferedModelTurn( messageCount: messageCountBeforeTurn, startedAt, historyPrefix: gatewayHistoryPrefix(gatewayTurn), + correlationId, }, ); } @@ -1929,6 +2066,7 @@ interface BufferedCompactionContext { readonly messageCount: number; readonly startedAt: number; readonly historyPrefix: readonly ChatMessage[]; + readonly correlationId: string | undefined; } async function finalizeAndRecordBufferedTurn( @@ -1948,6 +2086,7 @@ async function finalizeAndRecordBufferedTurn( messageCount: compaction.messageCount, startedAt: compaction.startedAt, historyPrefix: compaction.historyPrefix, + correlationId: compaction.correlationId, }); } return finalized; @@ -2298,7 +2437,7 @@ export async function handleSendDesktopChat( () => { const current = validateCurrentDesktopChatSend(parsed, deps); if (isRouteResult(current)) return current; - return persistModelChatTurn(deps, current, cancellation.signal); + return persistModelChatTurn(deps, current, cancellation.signal, ctx.correlationId); }, ); return result === CHAT_TURN_WAIT_CANCELLED ? requestCancelledResult() : result; @@ -2683,6 +2822,7 @@ async function persistRegeneratedChatTurn( deps: UiHandlerDeps, prepared: PreparedDesktopChatRegenerate, signal: AbortSignal, + correlationId: string | undefined, ): Promise { const { chat, modelId, memoryRequest, executionAdmission } = prepared; try { @@ -2698,7 +2838,10 @@ async function persistRegeneratedChatTurn( if (model === undefined) { return { status: 400, body: errorBody("NO_MODEL", "No model provider is configured.") }; } - const response = await model.call({ modelId, messages, stream: false }, signal); + const response = await model.call( + { modelId, messages, stream: false, logContext: { correlationId } }, + signal, + ); if (requestSignalAborted(signal)) return requestCancelledResult(); const redactedContent = deps.redactor(response.content) as string; assertUsableAssistantContent(redactedContent, modelId); @@ -2720,7 +2863,9 @@ async function persistRegeneratedChatTurn( }, }; } catch (error) { - return signal.aborted ? requestCancelledResult() : desktopChatErrorResult(error, deps); + return signal.aborted + ? requestCancelledResult() + : desktopChatErrorResult(error, deps, correlationId); } } @@ -2741,7 +2886,7 @@ export async function handleRegenerateDesktopChat( const current = prepareDesktopChatRegenerateRequest(prepared.request, deps); return isRouteResult(current) ? current - : persistRegeneratedChatTurn(deps, current, cancellation.signal); + : persistRegeneratedChatTurn(deps, current, cancellation.signal, ctx.correlationId); }, ); return result === CHAT_TURN_WAIT_CANCELLED ? requestCancelledResult() : result; diff --git a/packages/keiko-server/src/chat-stream-handlers.test.ts b/packages/keiko-server/src/chat-stream-handlers.test.ts index fb2c02ac2e..6bba03c8b9 100644 --- a/packages/keiko-server/src/chat-stream-handlers.test.ts +++ b/packages/keiko-server/src/chat-stream-handlers.test.ts @@ -36,6 +36,7 @@ import type { RuntimeGatewayConfig } from "./deps.js"; import { createInMemoryUiStore, type UiStore } from "./store/index.js"; import type { ModelPort } from "@oscharko-dev/keiko-harness"; import type { + GatewayCallRequest, GatewayConfig, GatewayRequest, GatewayStreamChunk, @@ -284,7 +285,7 @@ function deferred(): { interface StreamingModel { readonly model: ModelPort; - readonly recorded: { request: GatewayRequest | undefined }; + readonly recorded: { request: GatewayCallRequest | undefined }; readonly calls: { count: number }; } @@ -292,13 +293,13 @@ interface StreamingModel { // terminal done chunk. `onFirstDelta` (used by the cancel test) runs after the first delta is yielded // so the test can abort the controller deterministically before the done chunk arrives. function streamingModel(content: string, onFirstDelta?: () => void): StreamingModel { - const recorded: { request: GatewayRequest | undefined } = { request: undefined }; + const recorded: { request: GatewayCallRequest | undefined } = { request: undefined }; const calls = { count: 0 }; const model: ModelPort = { call(): Promise { return Promise.resolve(normalizedResponse(content)); }, - async *callStream(request: GatewayRequest): AsyncGenerator { + async *callStream(request: GatewayCallRequest): AsyncGenerator { calls.count += 1; recorded.request = request; yield { type: "delta", token: "hi" }; @@ -2842,6 +2843,29 @@ describe("desktop chat SSE streaming handler", () => { expect(JSON.stringify(record)).not.toContain("boom-unexpected-mid-stream"); }); + // ADR-0173 D5: the streaming call site (streamAndPersist) must stamp the request's correlation id + // into GatewayCallRequest.logContext, mirroring the buffered path, so a gateway retry line for a + // streamed turn joins the same trail as the rest of the request. + it("threads the request correlation id into the streaming model gateway call's logContext", async () => { + const chatId = seedChat(); + const streaming = streamingModel("hi"); + const res = captureRes(); + const ctx: RouteContext = { + ...routeContext( + makeReq({ chatId, projectPath: projectDir, modelId: CHAT_MODEL, content: "hello" }), + res.res, + ), + correlationId: "cid-stream-logcontext-000001", + }; + + await handleSendDesktopChatStream(ctx, deps(streaming.model)); + + expect(streaming.calls.count).toBe(1); + expect(streaming.recorded.request?.logContext?.correlationId).toBe( + "cid-stream-logcontext-000001", + ); + }); + it("persists the user message but NO assistant message when the stream is cancelled", async () => { const chatId = seedChat(); // captureResWithEvents is required here so the res.on("close") listener registered by diff --git a/packages/keiko-server/src/chat-stream-handlers.ts b/packages/keiko-server/src/chat-stream-handlers.ts index f13dcca567..99ab4434f2 100644 --- a/packages/keiko-server/src/chat-stream-handlers.ts +++ b/packages/keiko-server/src/chat-stream-handlers.ts @@ -17,7 +17,8 @@ import { type RouteContext, type RouteResult, } from "./routes.js"; -import { emitServerDiagnostic, serverDiagnosticFromError } from "./diagnostics-log.js"; +import { emitGatewayErrorDiagnostic } from "./gateway-error-diagnostic.js"; +import type { ConversationCompactionOutcome } from "./conversation-compaction.js"; import type { UiHandlerDeps } from "./deps.js"; import type { ChatMessage } from "./store/index.js"; import { ensureOnDemandConversationReadiness } from "./gateway-readiness.js"; @@ -116,15 +117,12 @@ function reportStreamIteratorCleanupFailure( deps: UiHandlerDeps, error: unknown, ): void { - emitServerDiagnostic( - deps.diagnostics, - serverDiagnosticFromError({ - correlationId: ctx.correlationId ?? "unknown", - operation: "POST /api/desktop/chat/stream", - source: "chat.stream.iterator-cleanup", - error, - redact: (message) => String(deps.redactor(message)), - }), + emitGatewayErrorDiagnostic( + deps, + error, + ctx.correlationId, + "POST /api/desktop/chat/stream", + "chat.stream.iterator-cleanup", ); } @@ -264,15 +262,12 @@ function errorEvent( deps: UiHandlerDeps, correlationId: string | undefined, ): { code: string; message: string; correlationId?: string } { - emitServerDiagnostic( - deps.diagnostics, - serverDiagnosticFromError({ - correlationId: correlationId ?? "unknown", - operation: "POST /api/desktop/chat/stream", - source: "chat.stream", - error, - redact: (message) => String(deps.redactor(message)), - }), + emitGatewayErrorDiagnostic( + deps, + error, + correlationId, + "POST /api/desktop/chat/stream", + "chat.stream", ); const withId = (payload: { code: string; @@ -284,7 +279,9 @@ function errorEvent( } => (correlationId === undefined ? payload : { ...payload, correlationId }); let result; try { - result = desktopChatErrorResult(error, deps); + // emitDiagnostic: false — the diagnostic for this exact error was already emitted above; this + // call is reused purely for its redacted code/message mapping (#154), not as a second response. + result = desktopChatErrorResult(error, deps, correlationId, false); } catch { return withId({ code: "INTERNAL", message: "An unexpected error occurred." }); } @@ -314,7 +311,7 @@ async function streamAndPersist( admitted: AdmittedDesktopChatStream, controller: AbortController, ): Promise { - const { prepared, callStream, userMessage, gatewayTurn, messageCountBeforeTurn } = admitted; + const { prepared, callStream, userMessage, gatewayTurn } = admitted; const { request, modelId, memoryContext } = prepared; const startedAt = Date.now(); const memory = admitted.memory ?? (await resolveMemory(deps, request, memoryContext)); @@ -328,7 +325,10 @@ async function streamAndPersist( modelId, buildGatewayAssembly(deps, request, memory, modelId, gatewayTurn), ); - const stream = callStream({ modelId, messages: assembly.messages }, controller.signal); + const stream = callStream( + { modelId, messages: assembly.messages, logContext: { correlationId: ctx.correlationId } }, + controller.signal, + ); const termination: StreamTermination = { backpressure: false }; const turn = await streamConversation(ctx, deps, stream, controller, termination); if (turn === undefined || requestIsAborted(controller.signal)) { @@ -349,13 +349,28 @@ async function streamAndPersist( failCancelledStreamTurn(ctx, deps, request, true); return; } + finalizeStreamedTurn(ctx, deps, payload, assembly.compaction, admitted, startedAt); +} + +// Split out of streamAndPersist to keep it within the line budget: records the compaction evidence +// for this turn and writes the terminal SSE `done` frame the client is waiting on. +function finalizeStreamedTurn( + ctx: RouteContext, + deps: UiHandlerDeps, + payload: DesktopChatSendResponse, + compaction: ConversationCompactionOutcome["compaction"], + admitted: AdmittedDesktopChatStream, + startedAt: number, +): void { + const { prepared, gatewayTurn, messageCountBeforeTurn } = admitted; recordChatCompaction(deps, { - compaction: assembly.compaction, - request, - modelId, + compaction, + request: prepared.request, + modelId: prepared.modelId, messageCount: messageCountBeforeTurn, startedAt, historyPrefix: gatewayHistoryPrefix(gatewayTurn), + correlationId: ctx.correlationId, }); writeTerminalFrame(ctx, sseMessage({ event: "done", data: payload })); } @@ -371,7 +386,11 @@ export async function handleSendDesktopChatStream( if (activeChatStreams >= maxActiveChatStreams()) { return { status: 429, - body: errorBody("TOO_MANY_STREAMS", "Too many concurrent chat streams; retry buffered."), + body: errorBody( + "TOO_MANY_STREAMS", + "Too many concurrent chat streams; retry buffered.", + ctx.correlationId, + ), }; } activeChatStreams += 1; @@ -397,10 +416,14 @@ type DesktopChatStreamPreparation = | { readonly kind: "outcome"; readonly outcome: HandlerOutcome } | { readonly kind: "replay"; readonly response: DesktopChatSendResponse }; -function streamingUnsupportedOutcome(): RouteResult { +function streamingUnsupportedOutcome(correlationId: string | undefined): RouteResult { return { status: 400, - body: errorBody("STREAMING_UNSUPPORTED", "Streaming is not available for this model."), + body: errorBody( + "STREAMING_UNSUPPORTED", + "Streaming is not available for this model.", + correlationId, + ), }; } @@ -453,6 +476,7 @@ function resolveDesktopChatStreamCall( prepared: PreparedDesktopChatSend, executionAdmission: DesktopChatExecutionAdmission, deps: UiHandlerDeps, + correlationId: string | undefined, ): StreamCall | RouteResult { const invalidExecution = validateDesktopChatProviderBoundary( prepared.modelId, @@ -462,7 +486,7 @@ function resolveDesktopChatStreamCall( if (invalidExecution !== undefined) return invalidExecution; const model = deps.modelPortFactory(prepared.modelId); return model?.callStream === undefined - ? streamingUnsupportedOutcome() + ? streamingUnsupportedOutcome(correlationId) : model.callStream.bind(model); } @@ -520,6 +544,7 @@ interface DesktopChatStreamExecutionPreflight { function preflightDesktopChatStreamExecution( prepared: PreparedDesktopChatSend, deps: UiHandlerDeps, + correlationId: string | undefined, ): DesktopChatStreamExecutionPreflight | RouteResult { const legacyExecutionAdmission = prepared.request.clientTurnId === undefined @@ -539,7 +564,7 @@ function preflightDesktopChatStreamExecution( const probed = legacyExecutionAdmission === undefined ? undefined - : resolveDesktopChatStreamCall(prepared, legacyExecutionAdmission, deps); + : resolveDesktopChatStreamCall(prepared, legacyExecutionAdmission, deps, correlationId); if (probed !== undefined && typeof probed !== "function") return probed; return { legacyExecutionAdmission, @@ -556,6 +581,7 @@ async function prepareDesktopChatProviderStream( legacyCall: StreamCall | undefined, controller: AbortController, admitted: AdmittedTurnHandle, + correlationId: string | undefined, ): Promise | RouteResult> { let memory: AdmittedDesktopChatStream["memory"]; try { @@ -573,13 +599,14 @@ async function prepareDesktopChatProviderStream( if (cancelled) { return { status: 499, body: errorBody("REQUEST_CANCELLED", "Request was cancelled.") }; } - return desktopChatErrorResult(error, deps); + return desktopChatErrorResult(error, deps, correlationId); } if (requestIsAborted(controller.signal)) { settleRejectedDesktopChatTurn(deps, prepared, admitted, "cancelled"); return { status: 499, body: errorBody("REQUEST_CANCELLED", "Request was cancelled.") }; } - const callStream = legacyCall ?? resolveDesktopChatStreamCall(prepared, executionAdmission, deps); + const callStream = + legacyCall ?? resolveDesktopChatStreamCall(prepared, executionAdmission, deps, correlationId); if (typeof callStream === "function") return { callStream, memory }; settleRejectedDesktopChatTurn(deps, prepared, admitted); return callStream; @@ -594,7 +621,7 @@ async function runAdmittedDesktopChatStream( ): Promise { const prepared = validateCurrentDesktopChatSend(start.parsed, deps); if ("status" in prepared) return prepared; - const preflight = preflightDesktopChatStreamExecution(prepared, deps); + const preflight = preflightDesktopChatStreamExecution(prepared, deps, ctx.correlationId); if ("status" in preflight) return preflight; const messageCountBeforeTurn = deps.store.countMessages(prepared.request.chatId); const admission = admitDesktopChatTurn(deps, prepared); @@ -613,6 +640,7 @@ async function runAdmittedDesktopChatStream( preflight.legacyCall, controller, admission, + ctx.correlationId, ); if ("status" in provider) return provider; const gatewayTurn = captureGatewayTurnSnapshot(deps, prepared.request, admission.userMessage); diff --git a/packages/keiko-server/src/coding-context/codingContextRoutes.test.ts b/packages/keiko-server/src/coding-context/codingContextRoutes.test.ts index 685344edcc..aa6635c99e 100644 --- a/packages/keiko-server/src/coding-context/codingContextRoutes.test.ts +++ b/packages/keiko-server/src/coding-context/codingContextRoutes.test.ts @@ -551,4 +551,22 @@ describe("coding context pack route", () => { expect(typeof error.correlationId).toBe("string"); expect(JSON.stringify(result.body)).not.toContain("secret endpoint detail"); }); + + it("threads the request's own correlation id into the upstream-failure response instead of minting one", async () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts) and is already + // in scope here — the failure record and the response body must reuse it, not a disconnected + // randomUUID(). Before the fix the response correlationId never matched ctx.correlationId. + const failingPort: GitHubCodeContextApiPort = { + readJson: () => Promise.reject(new Error("secret endpoint detail must not leak")), + }; + const ctx = { ...ctxFor(packRequest()), correlationId: "req-thread-0123456789" }; + const result = await handleCodingContextPack( + ctx, + depsFor({ codingContextGitHubPort: failingPort }), + ); + + expect(result.status).toBe(502); + const error = bodyOf(result).error as Record; + expect(error.correlationId).toBe("req-thread-0123456789"); + }); }); diff --git a/packages/keiko-server/src/coding-context/codingContextRoutes.ts b/packages/keiko-server/src/coding-context/codingContextRoutes.ts index 182674b190..aaa2acfcf0 100644 --- a/packages/keiko-server/src/coding-context/codingContextRoutes.ts +++ b/packages/keiko-server/src/coding-context/codingContextRoutes.ts @@ -331,7 +331,9 @@ export async function handleCodingContextPack( } catch (error) { // Port failures stay opaque: content-free code + correlation id only. The // connector layer never places endpoints, credentials, or bodies on errors. - const correlationId = randomUUID(); + // Threads the request's own correlation id (ADR-0173 D5 / g12) rather than minting a + // disconnected one — this IS the request whose failure is being reported. + const correlationId = ctx.correlationId ?? randomUUID(); emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ diff --git a/packages/keiko-server/src/coding-sidecar-gateway.test.ts b/packages/keiko-server/src/coding-sidecar-gateway.test.ts index b091201125..f9b8b1544c 100644 --- a/packages/keiko-server/src/coding-sidecar-gateway.test.ts +++ b/packages/keiko-server/src/coding-sidecar-gateway.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { ProviderError, resolveCodingSafeSidecarGatewayProfile, + type GatewayCallRequest, type GatewayConfig, type GatewayRequest, type GatewayStreamChunk, @@ -13,6 +14,7 @@ import { type NormalizedResponse, } from "@oscharko-dev/keiko-model-gateway"; import { buildRedactor, type UiHandlerDeps } from "./deps.js"; +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; import { createOpenCodeGatewayReadinessRegistry, @@ -1629,6 +1631,51 @@ describe("coding-sidecar gateway", () => { expect(JSON.stringify(streamRecords)).not.toContain("upstream reset"); }); + // Regression: a mid-stream failure with no request correlation id in scope used to fall back to + // the bare literal `"unknown"` (7 characters), which fails `isValidCorrelationId`'s 8-character + // floor and was silently rewritten by `emitServerDiagnostic`'s sanitizer to the "hostile value" + // marker `"invalid-correlation-id"` — misreporting an honestly-absent id as a malformed one. The + // fallback is now the shape-valid sentinel `UNKNOWN_CORRELATION_ID`, which survives the sanitizer + // unchanged. This test fails against the old bare-`"unknown"` fallback. + it("falls back the stream-failure correlation id to the unknown-id sentinel, never the invalid-id marker", async () => { + const diagnostics = { record: vi.fn<(record: ServerDiagnosticRecord) => void>() }; + const stream = async function* (): AsyncGenerator { + await Promise.resolve(); + yield { type: "delta", token: "partial" }; + throw Object.assign(new Error("upstream reset"), { code: "GATEWAY_TRANSPORT" }); + }; + const response = mockResponse({ captureBody: true }); + const context: RouteContext = { + ...authenticatedContext({ + model: "coding", + stream: true, + messages: [{ role: "user", content: "mid-stream failure, no correlation id" }], + tools: modelVisibleTools(), + }), + res: response.res, + }; + const deps = { + ...runtimeGatewayDeps( + () => ({ ok: true, binding: { runId: "run-stream-failure-no-corr" } }), + undefined, + createOpenCodeGatewayReadinessRegistry(), + (): (() => AsyncIterable) => (): AsyncIterable => + stream(), + ), + diagnostics, + } as UiHandlerDeps; + + const result = await handleCodingSidecarGatewayChatCompletions(context, deps); + + expect(result).toBe(STREAMING); + const streamRecords = diagnostics.record.mock.calls + .map(([entry]) => entry) + .filter((entry) => entry.source === "coding-sidecar-gateway.stream"); + expect(streamRecords).toHaveLength(1); + expect(streamRecords[0]?.correlationId).toBe(UNKNOWN_CORRELATION_ID); + expect(streamRecords[0]?.correlationId).not.toBe("invalid-correlation-id"); + }); + it("counts only each new UTF-8 stream delta instead of re-encoding accumulated output", async () => { const firstToken = "gateway-delta-one-α"; const secondToken = "gateway-delta-two-β"; @@ -2176,6 +2223,39 @@ describe("coding-sidecar gateway", () => { expect(JSON.stringify(result.body)).not.toContain("provider-secret"); }); + // ADR-0173 D5: the buffered chat completion request built for the gateway must carry the HTTP + // request's correlation id in GatewayCallRequest.logContext, so a gateway retry/circuit-breaker + // line for this call joins the same trail as the sidecar request that triggered it. + it("threads the request correlation id into the Gateway double's GatewayCallRequest.logContext", async () => { + const seenRequests: GatewayCallRequest[] = []; + const deps = depsValue( + configValue(provider(), capability()), + ( + _config: GatewayConfig, + modelId: string, + ): ((request: GatewayCallRequest) => Promise) => { + return (request: GatewayCallRequest): Promise => { + seenRequests.push(request); + return Promise.resolve(assistantResponse(modelId)); + }; + }, + ); + const context: RouteContext = { + ...routeContext({ + model: "azure-coding-model", + messages: [{ role: "user", content: "continue" }], + }), + correlationId: "sidecar-corr-logcontext-0001", + }; + + const result = await handleCodingSidecarGatewayChatCompletions(context, deps); + + assertRouteResult(result); + expect(result.status).toBe(200); + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0]?.logContext?.correlationId).toBe("sidecar-corr-logcontext-0001"); + }); + it("returns BAD_REQUEST for malformed OpenAI-compatible tools", async () => { const deps = depsValue(configValue(provider(), capability())); const result = await handleCodingSidecarGatewayChatCompletions( diff --git a/packages/keiko-server/src/coding-sidecar-gateway.ts b/packages/keiko-server/src/coding-sidecar-gateway.ts index 3d9b99b10a..5a76f46fe9 100644 --- a/packages/keiko-server/src/coding-sidecar-gateway.ts +++ b/packages/keiko-server/src/coding-sidecar-gateway.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { resolveCodingSafeSidecarGatewayProfile, type Gateway, + type GatewayCallRequest, type GatewayConfig, type GatewayRequest, type GatewayStreamChunk, @@ -24,6 +25,7 @@ import { } from "./deps.js"; import { OPENCODE_RUNTIME_MODEL_ALIAS } from "./coding-runtime/opencodeLaunchProfile.js"; import { hasExactOpenCodeVisibleToolContract } from "./coding-runtime/opencodeToolSchemas.js"; +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError } from "./diagnostics-log.js"; import { readJsonObject } from "./files.js"; import { STREAMING, errorBody, type RouteContext, type RouteResult } from "./routes.js"; @@ -356,7 +358,8 @@ function buildChatRequest( modelAlias: string, cancellationSignal: AbortSignal, maxOutputTokens: number, -): GatewayRequest { + correlationId: string | undefined, +): GatewayCallRequest { return { modelId: modelAlias, messages: parsed.messages, @@ -365,6 +368,7 @@ function buildChatRequest( ...(parsed.top_p === undefined ? {} : { topP: parsed.top_p }), cancellationSignal, maxOutputTokens, + logContext: { correlationId }, }; } @@ -642,7 +646,7 @@ function emitGatewayFailureDiagnostic( emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ - correlationId: ctx.correlationId ?? "unknown", + correlationId: ctx.correlationId ?? UNKNOWN_CORRELATION_ID, operation: CODING_SIDECAR_GATEWAY_ROUTE, source: "coding-sidecar-gateway.chat", error, @@ -668,7 +672,7 @@ function emitGatewayStreamFailureDiagnostic( emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ - correlationId: ctx.correlationId ?? "unknown", + correlationId: ctx.correlationId ?? UNKNOWN_CORRELATION_ID, operation: CODING_SIDECAR_GATEWAY_ROUTE, source: "coding-sidecar-gateway.stream", error, @@ -742,7 +746,7 @@ function emitGatewayToolContractDiagnostic( if (tools === undefined) code = "CODING_GATEWAY_TOOL_CONTRACT_MISSING"; else if (tools.length === 0) code = "CODING_GATEWAY_TOOL_CONTRACT_EMPTY"; emitServerDiagnostic(deps.diagnostics, { - correlationId: ctx.correlationId ?? "unknown", + correlationId: ctx.correlationId ?? UNKNOWN_CORRELATION_ID, timestamp: new Date(Date.now()).toISOString(), operation: CODING_SIDECAR_GATEWAY_ROUTE, source: "coding-sidecar-gateway.tool-contract", @@ -781,7 +785,7 @@ function noteToolAdoptionGap( if (!hasToolAdoptionGapFingerprint(messages)) return; if (gatewayReadinessRegistry(deps)?.noteAdoptionGapDiagnosed(runId) === false) return; emitServerDiagnostic(deps.diagnostics, { - correlationId: ctx.correlationId ?? "unknown", + correlationId: ctx.correlationId ?? UNKNOWN_CORRELATION_ID, timestamp: new Date(Date.now()).toISOString(), operation: CODING_SIDECAR_GATEWAY_ROUTE, source: "coding-sidecar-gateway.tool-adoption", @@ -914,7 +918,13 @@ async function executeGatewayChat( ): Promise { const { modelAlias, maxOutputTokens, upstreamStreamingSupported } = delivery; const cancellation = gatewayRequestCancellation(ctx, deps, binding.config, modelAlias, runId); - const request = buildChatRequest(parsed, modelAlias, cancellation.signal, maxOutputTokens); + const request = buildChatRequest( + parsed, + modelAlias, + cancellation.signal, + maxOutputTokens, + ctx.correlationId, + ); let bufferedStream: BufferedOpenAiStreamSession | undefined; try { if (parsed.stream && upstreamStreamingSupported) { diff --git a/packages/keiko-server/src/correlation-threading.e2e.test.ts b/packages/keiko-server/src/correlation-threading.e2e.test.ts new file mode 100644 index 0000000000..423b5d84e9 --- /dev/null +++ b/packages/keiko-server/src/correlation-threading.e2e.test.ts @@ -0,0 +1,683 @@ +// Wave 3 ACCEPTANCE TEST (epic #3233, ADR-0173 D5, final-design.md §7) — proves, against the REAL +// `createUiServer` route handler (never a bare handler function call), that ONE correlation id +// threads end to end: from an inbound HTTP/WS header, through the BFF, through the model gateway's +// own activity-log lines, and back onto the response — and that a forced provider failure's +// redacted diagnostic record and the gateway's own retry line carry that id plus the provider- +// detail fields g26 added. Where an assertion cannot pass because the wiring g26/g9 promises is +// not (yet) built, the assertion is left exactly as specified — never weakened — so a fix makes it +// go green rather than a rewrite making it agree with the gap. +// +// Doubles used throughout are either a hand-rolled fake `ModelPort` (a bare object implementing the +// port) or a REAL `Gateway` instance wired with a fake `ProviderAdapter` / `createScriptedGatewayFetch` +// (ADR-0173 §7.3) — never a mocked HTTP layer — so the gateway's own logging code actually runs. + +import { mkdirSync, mkdtempSync, rmSync, realpathSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { WebSocket } from "ws"; + +import { GatewayModelPort, type ModelPort } from "@oscharko-dev/keiko-harness"; +import { + Gateway, + createScriptedGatewayClock, + createScriptedGatewayFetch, + type GatewayConfig, + type GatewayReplayScriptEntry, + type ModelProviderConfig, + type NormalizedResponse, + type ProviderAdapter, + type RealtimeNegotiationOutcome, +} from "@oscharko-dev/keiko-model-gateway"; +import { RateLimitError } from "@oscharko-dev/keiko-security/errors/gateway"; + +import { buildRedactor, createRunRegistry, type UiHandlerDeps } from "./index.js"; +import { createInMemoryUiStore, type UiStore } from "./store/index.js"; +import { createUiServer, UI_HOST } from "./server.js"; +import { buildCspHeader } from "./csp.js"; +import { CORRELATION_HEADER } from "./correlation.js"; +import { VOICE_LIVE_TRANSCRIBE_PATH } from "./voice-live-dictation.js"; +import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, + type ServerLogEvent, +} from "./observability/index.js"; +import { closeUiTestServer, startUiTestServer } from "./ui-test-server/_support.js"; + +const OFFER_SDP = + "v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\nm=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=sendonly\r\n"; + +// ─── Shared config/double builders ────────────────────────────────────────────── + +function bffGatewayConfig(modelId: string, supportsImageInput = false): GatewayConfig { + return { + providers: [ + { + modelId, + baseUrl: "https://bff-provider.example.invalid/v1", + apiKey: "unused-bff-test-key", + timeoutMs: 5_000, + maxRetries: 0, + retryBaseDelayMs: 1, + }, + ], + circuitBreaker: { failureThreshold: 5, cooldownMs: 1_000, halfOpenProbes: 1 }, + capabilities: [ + { + id: modelId, + kind: "chat", + contextWindow: 64_000, + maxOutputTokens: 4_096, + toolCalling: true, + structuredOutput: true, + streaming: true, + supportsImageInput, + supportsDocumentInput: false, + workflowEligible: false, + costClass: "medium", + latencyClass: "standard", + throughputHint: "test", + preferredUseCases: [], + knownLimitations: [], + }, + ], + }; +} + +function gatewayProvider(overrides: Partial = {}): ModelProviderConfig { + return { + modelId: "gw-model", + baseUrl: "https://provider.example.invalid/v1", + apiKey: "gw-example-secret-token", + timeoutMs: 30_000, + maxRetries: 0, + retryBaseDelayMs: 1, + ...overrides, + }; +} + +function gatewayLevelConfig(providers: readonly ModelProviderConfig[]): GatewayConfig { + return { + providers: [...providers], + circuitBreaker: { failureThreshold: 5, cooldownMs: 1_000, halfOpenProbes: 1 }, + }; +} + +function okResponse(modelId: string, content: string): NormalizedResponse { + return { + modelId, + content, + finishReason: "stop", + toolCalls: [], + structuredOutput: null, + usage: { + requestId: "gw-test-request", + promptTokens: 3, + completionTokens: 2, + latencyMs: 4, + costClass: "low", + }, + }; +} + +function fakeAdapter(impl: ProviderAdapter["call"]): ProviderAdapter { + return { call: impl }; +} + +function okChatBody(content: string): unknown { + return { + choices: [{ message: { role: "assistant", content }, finish_reason: "stop" }], + usage: { prompt_tokens: 3, completion_tokens: 1 }, + }; +} + +function minimalDeps(overrides: Partial): UiHandlerDeps { + return { + config: undefined, + configPresent: false, + evidenceStore: { put: () => "", list: () => [], get: () => undefined, delete: () => undefined }, + env: {}, + redactor: buildRedactor({}), + registry: createRunRegistry(), + modelPortFactory: () => undefined, + store: createInMemoryUiStore(), + ...overrides, + }; +} + +// Seeds exactly ONE prior turn that contributes exactly ONE gateway message: a legacy (no +// client_turn_id) user/assistant pair whose assistant half is the exact legacy-empty-response +// placeholder `usableGatewayMessages` (conversation-gateway.ts) drops via +// `isLegacyEmptyAssistantPlaceholder`. The store's scan layer (messages.ts's +// `scanLegacyGatewayRow`) still counts the pair as ONE eligible history unit, but only the user +// half survives into the assembled prompt — giving a real, production-path-derived odd total +// (system + priorUser + currentUser = 3) instead of guessing at internal counting rules. +function seedLegacyPriorTurn(store: UiStore, chatId: string): void { + const seededAt = Date.now() - 60_000; + const base = { + chatId, + runId: undefined, + workflowId: undefined, + workflowStatus: undefined, + shortResult: undefined, + taskType: undefined, + }; + store.createMessage({ ...base, role: "user", content: "Earlier question", timestamp: seededAt }); + store.createMessage({ + ...base, + role: "assistant", + content: "The model returned an empty response.", + timestamp: seededAt + 1_000, + }); +} + +// ─── Server lifecycle ─────────────────────────────────────────────────────────── + +let activeServer: Server | undefined; +const tempDirs: string[] = []; + +afterEach(async () => { + resetServerLogger(); + if (activeServer !== undefined) { + await closeUiTestServer(activeServer); + activeServer = undefined; + } + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function tempProjectDir(prefix: string): string { + const root = mkdtempSync(join(realpathSync(tmpdir()), prefix)); + tempDirs.push(root); + const projectPath = join(root, "repo"); + mkdirSync(projectPath); + return projectPath; +} + +async function boot( + handlerDeps: UiHandlerDeps, + activityLog?: BufferedServerLogSink, +): Promise { + const staticRoot = mkdtempSync(join(realpathSync(tmpdir()), "keiko-correlation-static-")); + tempDirs.push(staticRoot); + const started = await startUiTestServer({ + staticRoot, + csp: buildCspHeader([]), + handlerDeps, + ...(activityLog === undefined ? {} : { activityLog }), + }); + activeServer = started.server; + return started.port; +} + +function baseUrl(port: number): string { + return `http://${UI_HOST}:${String(port)}`; +} + +function jsonHeaders(correlationId: string): Record { + return { + "Content-Type": "application/json", + "X-Keiko-CSRF": "1", + [CORRELATION_HEADER]: correlationId, + }; +} + +// `startUiTestServer` (used by `boot` above) binds an ephemeral port by mutating its +// `UiServerDeps.port` field AFTER `createUiServer` has already run — fine for the HTTP path, +// which re-reads `deps.port` on every request, but NOT for the voice planes: `createUiServer` +// captures `deps.port` BY VALUE, once, into `createVoicePlanes(deps.port, handlerDeps)` at +// construction time, so a WS upgrade's own `isAllowedHost` check would keep comparing against the +// port-0 snapshot forever and hard-reject every upgrade with a 404. Mirrors +// voice-control-ws.test.ts's own `boot()`: probe an ephemeral port, close that throwaway server, +// then construct the REAL server with the correct port already baked in. +async function bootForVoice(handlerDeps: UiHandlerDeps): Promise { + const staticRoot = mkdtempSync(join(realpathSync(tmpdir()), "keiko-correlation-voice-static-")); + tempDirs.push(staticRoot); + const csp = buildCspHeader([]); + const probe = createUiServer({ staticRoot, csp, port: 0, handlerDeps }); + const port = await new Promise((resolve) => { + probe.listen(0, UI_HOST, () => { + resolve((probe.address() as AddressInfo).port); + }); + }); + await new Promise((resolve) => { + probe.close(() => { + resolve(); + }); + }); + const listening = createUiServer({ staticRoot, csp, port, handlerDeps }); + activeServer = listening; + await new Promise((resolve) => { + listening.listen(port, UI_HOST, resolve); + }); + return port; +} + +// Polls the buffered sink instead of assuming synchronous availability: `gateway.*`/`chat.turn.*` +// lines are written before the response leaves, but the `http`/`request` line is written from +// `res.on("close")`, which can fire a tick after `fetch()`'s promise settles (AGENTS.md §9 — await +// a condition instead of sleeping). +async function waitForEvent( + sink: BufferedServerLogSink, + predicate: (event: ServerLogEvent) => boolean, + timeoutMs = 2_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const found = sink.events.find(predicate); + if (found !== undefined) { + return found; + } + if (Date.now() > deadline) { + throw new Error("timed out waiting for the expected activity log event"); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} + +// ─── WS helpers (mirrors voice-control-ws.test.ts's proven `connect`/`expectOpen`) ───────────── + +interface WsClient { + readonly opened: boolean; + readonly ws?: WebSocket; + readonly next?: () => Promise>; +} + +function connectWs( + port: number, + options: { readonly path: string; readonly headers?: Record }, +): Promise { + const headers: Record = { + Origin: `http://${UI_HOST}:${String(port)}`, + ...(options.headers ?? {}), + }; + return new Promise((resolve) => { + const ws = new WebSocket(`ws://${UI_HOST}:${String(port)}${options.path}`, { headers }); + const queue: Record[] = []; + const waiters: ((message: Record) => void)[] = []; + ws.on("message", (data: Buffer) => { + const message = JSON.parse(data.toString("utf8")) as Record; + const waiter = waiters.shift(); + if (waiter !== undefined) { + waiter(message); + } else { + queue.push(message); + } + }); + const next = (): Promise> => { + const queued = queue.shift(); + if (queued !== undefined) { + return Promise.resolve(queued); + } + return new Promise((resolveMessage) => waiters.push(resolveMessage)); + }; + ws.once("open", () => { + resolve({ opened: true, ws, next }); + }); + ws.once("unexpected-response", () => { + ws.terminate(); + resolve({ opened: false }); + }); + ws.once("error", () => { + resolve({ opened: false }); + }); + }); +} + +interface OpenWsClient { + readonly ws: WebSocket; + readonly next: () => Promise>; +} + +function expectOpen(client: WsClient): OpenWsClient { + if (!client.opened || client.ws === undefined || client.next === undefined) { + throw new Error("expected the WebSocket upgrade to be accepted"); + } + return { ws: client.ws, next: client.next }; +} + +function voiceRealtimeConfig(): GatewayConfig { + return { + providers: [ + { + modelId: "keiko-realtime-e2e", + baseUrl: "https://realtime.example.invalid", + apiKey: "rt-e2e-secret-token-1234567890", + timeoutMs: 1_000, + maxRetries: 0, + retryBaseDelayMs: 10, + }, + ], + circuitBreaker: { failureThreshold: 5, cooldownMs: 1_000, halfOpenProbes: 1 }, + capabilities: [ + { + id: "keiko-realtime-e2e", + kind: "voice", + contextWindow: 0, + maxOutputTokens: 0, + toolCalling: false, + structuredOutput: false, + streaming: false, + supportsImageInput: false, + supportsDocumentInput: false, + supportsSpeechInput: true, + supportsRealtimeVoice: true, + realtimeTranscriptionModel: "configured-realtime-transcription", + voiceProviderLocality: "azure-foundry", + workflowEligible: false, + costClass: "low", + latencyClass: "fast", + throughputHint: "azure foundry realtime", + preferredUseCases: ["Conversation"], + knownLimitations: [], + }, + ], + }; +} + +function liveSessionCreate(): string { + return JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-e2e-1", + seq: 0, + direction: "client-to-host", + kind: "session.create", + idempotencyKey: "idem-e2e-1", + requestedProfile: "full-realtime", + negotiationMode: "proxied-sdp", + }); +} + +function offerFrame(seq: number): string { + return JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-e2e-1", + seq, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }); +} + +// ─── (1) desktop chat turn — one correlation id end to end (ADR-0173 D5 §7.1, §7.4) ───────────── + +describe("desktop chat turn — one correlation id end to end", () => { + it("threads a client-supplied X-Keiko-Correlation-Id onto the http line, gateway.chat.completed, and the response header", async () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + + const modelId = "correlation-thread-model"; + const projectPath = tempProjectDir("keiko-correlation-thread-"); + const store = createInMemoryUiStore(); + store.createProject(projectPath, "repo"); + const chat = store.createChat(projectPath, "Thread", modelId); + + const gateway = new Gateway(gatewayLevelConfig([gatewayProvider({ modelId })]), { + adapter: fakeAdapter(() => Promise.resolve(okResponse(modelId, "hello there"))), + clock: createScriptedGatewayClock(), + log: sink, + }); + const modelPort: ModelPort = new GatewayModelPort(gateway); + + const port = await boot( + minimalDeps({ + config: bffGatewayConfig(modelId), + configPresent: true, + store, + modelPortFactory: () => modelPort, + }), + sink, + ); + + const correlationId = "e2e-correlation-thread-0001"; + const res = await fetch(`${baseUrl(port)}/api/desktop/chat`, { + method: "POST", + headers: jsonHeaders(correlationId), + body: JSON.stringify({ chatId: chat.id, projectPath, modelId, content: "Hello" }), + }); + + expect(res.status).toBe(200); + expect(res.headers.get("x-keiko-correlation-id")).toBe(correlationId); + + const httpLine = await waitForEvent( + sink, + (event) => + event.category === "http" && + event.op === "request" && + event.correlationId === correlationId, + ); + expect(httpLine.op).toBe("request"); + + const completed = await waitForEvent(sink, (event) => event.op === "gateway.chat.completed"); + expect(completed.correlationId).toBe(correlationId); + }); +}); + +// ─── (2a) forced RateLimitError — diagnostic record field wiring (ADR-0173 D5 g26) ────────────── + +describe("forced RateLimitError — diagnostic record field wiring", () => { + it("keeps the request's correlation id on the diagnostic record and carries retryAfterMs/httpStatus", async () => { + const modelId = "rate-limit-diagnostic-model"; + const projectPath = tempProjectDir("keiko-correlation-ratelimit-"); + const store = createInMemoryUiStore(); + store.createProject(projectPath, "repo"); + const chat = store.createChat(projectPath, "Rate limited", modelId); + + const diagnostics: ServerDiagnosticRecord[] = []; + const rateLimitedModel: ModelPort = { + call: () => Promise.reject(new RateLimitError("provider rate limited", 4_000)), + }; + + const port = await boot( + minimalDeps({ + config: bffGatewayConfig(modelId), + configPresent: true, + store, + modelPortFactory: () => rateLimitedModel, + diagnostics: { record: (record): void => void diagnostics.push(record) }, + }), + ); + + const correlationId = "e2e-correlation-ratelimit-0001"; + const res = await fetch(`${baseUrl(port)}/api/desktop/chat`, { + method: "POST", + headers: jsonHeaders(correlationId), + body: JSON.stringify({ + chatId: chat.id, + projectPath, + modelId, + content: "Trigger a rate limit", + }), + }); + + expect(res.status).toBe(503); + expect(diagnostics).toHaveLength(1); + const [record] = diagnostics; + expect(record).toBeDefined(); + expect(record?.correlationId).toBe(correlationId); + + // g26 (final-design.md §2, ServerDiagnosticRecord v2): httpStatus/retryAfterMs are derived + // through `describeError` → `providerErrorDetail()` (keiko-model-gateway/resilience.ts), the + // SAME instanceof-based derivation `gateway.retry.*` lines use. `httpStatus` is read off BOTH + // `ProviderError` and `RateLimitError`: a rate-limited call is always HTTP 429 by definition, + // so this diagnostic record carries httpStatus=429 alongside retryAfterMs — a replay-script + // consumer (`GatewayReplayAttempt.httpStatus`) never has to infer the status from + // `errorKind === GATEWAY_RATE_LIMIT`. + const detail = record as unknown as { + readonly httpStatus?: number; + readonly retryAfterMs?: number; + }; + expect(detail.retryAfterMs).toBe(4_000); + expect(detail.httpStatus).toBe(429); + }); +}); + +// ─── (2b) scripted 429-then-200 — gateway.retry.scheduled field wiring (ADR-0173 D5 g26, §7.3) ── + +describe("scripted 429-then-200 retry — gateway.retry.scheduled field wiring", () => { + it("recovers via the real retry path and carries the request's correlation id + httpStatus=429 onto gateway.retry.scheduled", async () => { + const modelId = "retry-recovery-model"; + const projectPath = tempProjectDir("keiko-correlation-retry-"); + const store = createInMemoryUiStore(); + store.createProject(projectPath, "repo"); + const chat = store.createChat(projectPath, "Retry recovery", modelId); + + const sharedClock = createScriptedGatewayClock(); + const script: readonly GatewayReplayScriptEntry[] = [ + { + status: 429, + headers: { "retry-after": "1" }, + bodyJson: { error: { message: "slow down" } }, + latencyMs: 0, + }, + { status: 200, bodyJson: okChatBody("recovered"), latencyMs: 0 }, + ]; + const fetchImpl = createScriptedGatewayFetch(script, sharedClock); + const sink = createBufferedServerLogSink(); + const gateway = new Gateway( + gatewayLevelConfig([gatewayProvider({ modelId, maxRetries: 1, retryBaseDelayMs: 1 })]), + { clock: sharedClock, fetchImpl, log: sink, random: (): number => 0.5 }, + ); + const modelPort: ModelPort = new GatewayModelPort(gateway); + + const port = await boot( + minimalDeps({ + config: bffGatewayConfig(modelId), + configPresent: true, + store, + modelPortFactory: () => modelPort, + }), + ); + + const correlationId = "e2e-correlation-retry-0001"; + const res = await fetch(`${baseUrl(port)}/api/desktop/chat`, { + method: "POST", + headers: jsonHeaders(correlationId), + body: JSON.stringify({ chatId: chat.id, projectPath, modelId, content: "Please recover" }), + }); + + expect(res.status).toBe(200); + const scheduled = await waitForEvent(sink, (event) => event.op === "gateway.retry.scheduled"); + expect(scheduled.correlationId).toBe(correlationId); + // g26: resilience.ts's providerErrorDetail() reads httpStatus off a RateLimitError instance + // too, deliberately — the 429 response mapped here to RateLimitError + // (packages/keiko-model-gateway/src/openai-adapter.ts, response.status === 429) is always HTTP + // 429 by definition, so the scheduled retry line carries httpStatus=429 alongside the + // provider-supplied retryAfterMs, instead of forcing a consumer to infer the status from + // errorKind === GATEWAY_RATE_LIMIT. + expect(scheduled.extra?.httpStatus).toBe(429); + expect(scheduled.extra?.retryAfterMs).toBe(1_000); + }); +}); + +// ─── (3) chat.turn.started shape fields (ADR-0173 D5 g9) ──────────────────────────────────────── + +describe("chat.turn.started shape fields — 3-message turn with one image attachment", () => { + it("logs messageCount=3, imageAttachmentCount=1, keyed to the request correlation id", async () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + + const modelId = "turn-shape-image-model"; + const projectPath = tempProjectDir("keiko-correlation-turnshape-"); + const store = createInMemoryUiStore(); + store.createProject(projectPath, "repo"); + const chat = store.createChat(projectPath, "Turn shape", modelId); + seedLegacyPriorTurn(store, chat.id); + + const gateway = new Gateway(gatewayLevelConfig([gatewayProvider({ modelId })]), { + adapter: fakeAdapter(() => Promise.resolve(okResponse(modelId, "looked at it"))), + clock: createScriptedGatewayClock(), + log: sink, + }); + const modelPort: ModelPort = new GatewayModelPort(gateway); + + const port = await boot( + minimalDeps({ + config: bffGatewayConfig(modelId, true), + configPresent: true, + store, + modelPortFactory: () => modelPort, + }), + sink, + ); + + const correlationId = "e2e-correlation-turnshape-0001"; + // `chat.turn.started` is logged from the BASE assembly, before image content parts are + // spliced in (chat-handlers.ts's own comment on `logChatTurnStarted`'s call site) — so it is + // written even though this send goes on to be refused at the UNRELATED image-delivery- + // authority gate a few lines later (no `attachmentAuthority`/`attachmentIntent` supplied here; + // that gate is a distinct security check this test does not exercise). The response status is + // therefore deliberately not asserted here — only the shape fields the gateway op carries. + await fetch(`${baseUrl(port)}/api/desktop/chat`, { + method: "POST", + headers: jsonHeaders(correlationId), + body: JSON.stringify({ + chatId: chat.id, + projectPath, + modelId, + content: "Look at this", + attachments: [{ kind: "image", mimeType: "image/png", sizeBytes: 1_024 }], + }), + }); + + const started = await waitForEvent( + sink, + (event) => event.op === "chat.turn.started" && event.correlationId === correlationId, + ); + expect(started.category).toBe("gateway"); + expect(started.extra).toMatchObject({ messageCount: 3, imageAttachmentCount: 1 }); + }); +}); + +// ─── (4) voice live-dictation WS upgrade — one correlation id across two diagnostics (g15) ────── + +describe("voice live-dictation WS upgrade — one correlation id across two diagnostics", () => { + it("carries the client-supplied X-Keiko-Correlation-Id onto two successive negotiation-failure diagnostics", async () => { + const diagnostics: ServerDiagnosticRecord[] = []; + const port = await bootForVoice( + minimalDeps({ + config: voiceRealtimeConfig(), + configPresent: true, + diagnostics: { record: (record): void => void diagnostics.push(record) }, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }), + ); + + const correlationId = "e2e-correlation-voice-0001"; + const { ws: socket, next } = expectOpen( + await connectWs(port, { + path: VOICE_LIVE_TRANSCRIBE_PATH, + headers: { [CORRELATION_HEADER]: correlationId }, + }), + ); + + socket.send(liveSessionCreate()); + await next(); // session.created + await next(); // capability.offer + + socket.send(offerFrame(1)); + await next(); // media.track.state negotiating + const firstFailure = await next(); + await next(); // media.track.state ended + + socket.send(offerFrame(2)); + await next(); // media.track.state negotiating + const secondFailure = await next(); + await next(); // media.track.state ended + + expect(firstFailure.correlationId).toBe(correlationId); + expect(secondFailure.correlationId).toBe(correlationId); + expect(diagnostics).toHaveLength(2); + expect(diagnostics[0]?.correlationId).toBe(correlationId); + expect(diagnostics[1]?.correlationId).toBe(correlationId); + socket.close(); + }); +}); diff --git a/packages/keiko-server/src/correlation.ts b/packages/keiko-server/src/correlation.ts index 5ee8ea2f6d..593429f2a9 100644 --- a/packages/keiko-server/src/correlation.ts +++ b/packages/keiko-server/src/correlation.ts @@ -29,6 +29,16 @@ export function newCorrelationId(): string { return randomUUID(); } +// A fixed, shape-valid stand-in for "no correlation id was known at this call site" — as opposed to +// a hostile or malformed one (see `diagnostics-log.ts`'s `INVALID_CORRELATION_ID_MARKER`, which +// covers that case). A `ServerDiagnosticRecord.correlationId` is required, so a caller with none in +// scope needs SOME value rather than an omission; several call sites used the bare literal +// `"unknown"` for this, but at 7 characters it always fails `isValidCorrelationId` itself and was +// silently rewritten to the sanitizer's own marker — making an honestly-absent id indistinguishable +// from a hostile one. This constant already satisfies the shape, so it survives the sanitizer and +// keeps its own distinct meaning. +export const UNKNOWN_CORRELATION_ID = "unknown-correlation-id"; + // Resolves the correlation id for a request: reuse a well-formed client-supplied id (UI -> server // continuity) or mint a fresh one. Never throws. export function resolveCorrelationId(req: IncomingMessage): string { diff --git a/packages/keiko-server/src/desktop-chat-handlers.test.ts b/packages/keiko-server/src/desktop-chat-handlers.test.ts index e7877d0236..c67536516c 100644 --- a/packages/keiko-server/src/desktop-chat-handlers.test.ts +++ b/packages/keiko-server/src/desktop-chat-handlers.test.ts @@ -14,8 +14,8 @@ import { createInMemoryUiStore, type UiStore } from "./store/index.js"; import { startUiTestServer } from "./ui-test-server/_support.js"; import type { ModelPort } from "@oscharko-dev/keiko-harness"; import type { + GatewayCallRequest, GatewayConfig, - GatewayRequest, NormalizedResponse, } from "@oscharko-dev/keiko-model-gateway"; import { createMemoryVault, type MemoryVaultStore } from "@oscharko-dev/keiko-memory-vault"; @@ -41,7 +41,7 @@ let staticRoot: string; let tmp: string; let projectDir: string; let store: UiStore; -let seenRequests: GatewayRequest[]; +let seenRequests: GatewayCallRequest[]; function fakeModel(content: string): ModelPort { return { @@ -678,6 +678,35 @@ describe("desktop chat routes", () => { expect(persistedRoles).toEqual(expect.arrayContaining(["user", "assistant"])); }); + // ADR-0173 D5 (BFF -> gateway correlation threading): the client-supplied request correlation id + // must reach the Gateway double's GatewayCallRequest.logContext, not just the response header, so + // a gateway retry/circuit-breaker line for this call joins the same trail as the HTTP request. + it("threads the request correlation id into the model gateway call's logContext", async () => { + const createRes = await fetch(`${base()}/api/desktop/chats`, { + method: "POST", + headers: POST_JSON_HEADERS, + body: JSON.stringify({ projectPath: projectDir, modelId: CHAT_MODEL }), + }); + const created = (await createRes.json()) as { chat: { id: string } }; + const correlationId = "test-correlation-id-send-0001"; + + const sendRes = await fetch(`${base()}/api/desktop/chat`, { + method: "POST", + headers: { ...POST_JSON_HEADERS, "X-Keiko-Correlation-Id": correlationId }, + body: JSON.stringify({ + chatId: created.chat.id, + projectPath: projectDir, + modelId: CHAT_MODEL, + content: "Say hello with correlation", + }), + }); + + expect(sendRes.status).toBe(200); + expect(sendRes.headers.get("X-Keiko-Correlation-Id")).toBe(correlationId); + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0]?.logContext?.correlationId).toBe(correlationId); + }); + it("admits a long canonical final atomically and rejects content beyond the UTF-8 hard cap", async () => { const createRes = await fetch(`${base()}/api/desktop/chats`, { method: "POST", @@ -1365,6 +1394,52 @@ describe("desktop chat routes", () => { expect(store.listMessages(chat.id).map((message) => message.id)).toContain(assistant.id); }); + // ADR-0173 D5: the regenerate path builds a fresh model.call site distinct from the send path + // (chat-handlers.ts persistRegeneratedChatTurn) — it must thread the request correlation id too. + it("threads the request correlation id into the regenerate model gateway call", async () => { + await restartWithDeps(deps(fakeModel("regenerated with correlation"))); + const chat = store.createChat(projectDir, "regen correlation", CHAT_MODEL); + store.createMessage({ + chatId: chat.id, + role: "user", + content: "original question", + timestamp: 1, + runId: undefined, + workflowId: undefined, + workflowStatus: undefined, + shortResult: undefined, + taskType: undefined, + }); + const assistant = store.createMessage({ + chatId: chat.id, + role: "assistant", + content: "stale answer", + timestamp: 2, + runId: undefined, + workflowId: undefined, + workflowStatus: undefined, + shortResult: undefined, + taskType: undefined, + }); + const correlationId = "test-correlation-id-regen-0001"; + + const res = await fetch(`${base()}/api/desktop/chat/regenerate`, { + method: "POST", + headers: { ...POST_JSON_HEADERS, "X-Keiko-Correlation-Id": correlationId }, + body: JSON.stringify({ + chatId: chat.id, + projectPath: projectDir, + modelId: CHAT_MODEL, + assistantMessageId: assistant.id, + }), + }); + + expect(res.status).toBe(200); + expect(res.headers.get("X-Keiko-Correlation-Id")).toBe(correlationId); + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0]?.logContext?.correlationId).toBe(correlationId); + }); + it("rejects regeneration of a closed chat without model work or message mutation", async () => { const chat = store.createChat(projectDir, "closed regeneration", CHAT_MODEL); store.createMessage({ diff --git a/packages/keiko-server/src/editor/completionRoutes.ts b/packages/keiko-server/src/editor/completionRoutes.ts index b8d8066485..6d7691efee 100644 --- a/packages/keiko-server/src/editor/completionRoutes.ts +++ b/packages/keiko-server/src/editor/completionRoutes.ts @@ -99,7 +99,10 @@ export interface EditorCompletionRouteOptions { } // Default chat seam: route the elected model through the Model Gateway, server-side only. -function defaultChatFactoryFor(deps: UiHandlerDeps): CompletionChatFactory { +function defaultChatFactoryFor( + deps: UiHandlerDeps, + correlationId: string | undefined, +): CompletionChatFactory { return (_config, modelId): ModelChatFn => { const gateway = currentGateway(deps); if (gateway === undefined) throw new TypeError("Model gateway is unavailable."); @@ -111,6 +114,7 @@ function defaultChatFactoryFor(deps: UiHandlerDeps): CompletionChatFactory { { role: "user", content: chatRequest.user }, ], cancellationSignal: chatSignal, + logContext: { correlationId }, }); return { content: response.content, usage: response.usage }; }; @@ -649,7 +653,7 @@ export async function handleEditorCompletion( root.realRoot, signal, deps, - options.chatFactory ?? defaultChatFactoryFor(deps), + options.chatFactory ?? defaultChatFactoryFor(deps, ctx.correlationId), options.tokenBudget ?? sharedEditorModelTokenBudget, options.now ?? Date.now, ); diff --git a/packages/keiko-server/src/editor/inlineCompletionRoutes.ts b/packages/keiko-server/src/editor/inlineCompletionRoutes.ts index cd77c93765..976f93cdd8 100644 --- a/packages/keiko-server/src/editor/inlineCompletionRoutes.ts +++ b/packages/keiko-server/src/editor/inlineCompletionRoutes.ts @@ -111,7 +111,10 @@ export interface EditorInlineCompletionRouteOptions { const sharedRateLimiter: InlineCompletionRateLimiter = createInlineCompletionRateLimiter(); // Default chat seam: route the elected model through the Model Gateway, server-side only. -function defaultChatFactoryFor(deps: UiHandlerDeps): InlineCompletionChatFactory { +function defaultChatFactoryFor( + deps: UiHandlerDeps, + correlationId: string | undefined, +): InlineCompletionChatFactory { return (_config, modelId): ModelChatFn => { const gateway = currentGateway(deps); if (gateway === undefined) throw new TypeError("Model gateway is unavailable."); @@ -123,6 +126,7 @@ function defaultChatFactoryFor(deps: UiHandlerDeps): InlineCompletionChatFactory { role: "user", content: chatRequest.user }, ], cancellationSignal: chatSignal, + logContext: { correlationId }, }); return { content: response.content, usage: response.usage }; }; @@ -504,7 +508,7 @@ async function runInlineModelTier( deps, selection, modelId, - chatFactory: options.chatFactory ?? defaultChatFactoryFor(deps), + chatFactory: options.chatFactory ?? defaultChatFactoryFor(deps, correlationId), config, nowMs: now(), tokenBudget, diff --git a/packages/keiko-server/src/editor/localHistory/localHistoryCapture.test.ts b/packages/keiko-server/src/editor/localHistory/localHistoryCapture.test.ts index 0805c4f0a3..87502d4116 100644 --- a/packages/keiko-server/src/editor/localHistory/localHistoryCapture.test.ts +++ b/packages/keiko-server/src/editor/localHistory/localHistoryCapture.test.ts @@ -109,4 +109,34 @@ describe("captureEditorLocalHistorySafely", () => { expect(JSON.stringify(diagnostics)).not.toContain(secretValue); expect(JSON.stringify(diagnostics)).not.toContain(secretContent); }); + + // ADR-0173 D5 / g12: a capture failure that happens inside a request must carry THAT request's + // own correlation id, not a disconnected `local-history-` mint, so an operator can join it + // back to the rest of the request's trail. Before the fix, `correlationId` threading did not + // exist on this function at all, so this assertion fails against the pre-fix signature (the + // extra field was silently ignored and a fresh `local-history-` id was always minted instead). + it("threads the caller's own correlation id into a capture failure instead of minting one", () => { + const diagnostics: ServerDiagnosticRecord[] = []; + const secretValue = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + const secretContent = `AWS_SECRET_ACCESS_KEY=${secretValue}\n`; + const requestCorrelationId = "req-abc12345"; + + const result = captureEditorLocalHistorySafely({ + deps: deps(diagnostics), + realRoot: root, + relativePath: "src/app.ts", + absolutePath: join(root, "src", "app.ts"), + content: secretContent, + origin: "user-save", + nowMs: 3_000, + correlationId: requestCorrelationId, + }); + + expect(result).toMatchObject({ + status: "suppressed", + correlationId: requestCorrelationId, + }); + expect(diagnostics).toHaveLength(1); + expect(diagnostics[0]?.correlationId).toBe(requestCorrelationId); + }); }); diff --git a/packages/keiko-server/src/editor/localHistory/localHistoryCapture.ts b/packages/keiko-server/src/editor/localHistory/localHistoryCapture.ts index 1cffa45681..814d602175 100644 --- a/packages/keiko-server/src/editor/localHistory/localHistoryCapture.ts +++ b/packages/keiko-server/src/editor/localHistory/localHistoryCapture.ts @@ -66,8 +66,12 @@ export function emitEditorLocalHistoryCaptureFailure( origin: EditorLocalHistoryOrigin, error: unknown, nowMs = Date.now(), + // Threads the request's own correlation id (ADR-0173 D5 / g12) when the caller has one in + // scope, so this failure — and the client-visible protection payload it returns the id on — + // joins the SAME id as the rest of the request's trail instead of a disconnected mint. + requestCorrelationId?: string, ): string { - const correlationId = `local-history-${randomUUID()}`; + const correlationId = requestCorrelationId ?? `local-history-${randomUUID()}`; emitServerDiagnostic(deps.diagnostics, { correlationId, timestamp: new Date(nowMs).toISOString(), @@ -123,6 +127,8 @@ export function captureEditorLocalHistorySafely(input: { readonly content: string; readonly origin: EditorLocalHistoryOrigin; readonly nowMs?: number | undefined; + // The request's own correlation id (ADR-0173 D5 / g12), when the caller has one in scope. + readonly correlationId?: string | undefined; }): EditorLocalHistoryCaptureProtection { if (input.deps.editorLocalHistoryStore === undefined) { const error = new EditorLocalHistoryError( @@ -135,6 +141,7 @@ export function captureEditorLocalHistorySafely(input: { input.origin, error, input.nowMs, + input.correlationId, ); return degradedProtection(error, correlationId); } @@ -155,6 +162,7 @@ export function captureEditorLocalHistorySafely(input: { input.origin, error, input.nowMs, + input.correlationId, ); return protectionForCaptureFailure(error, correlationId); } diff --git a/packages/keiko-server/src/editor/patchApplyRoutes.test.ts b/packages/keiko-server/src/editor/patchApplyRoutes.test.ts index 8adfef0c86..59805b2a48 100644 --- a/packages/keiko-server/src/editor/patchApplyRoutes.test.ts +++ b/packages/keiko-server/src/editor/patchApplyRoutes.test.ts @@ -281,6 +281,36 @@ describe("POST /api/editor/patch-apply — explicit decision (AC1)", () => { expect(evidenceStore.list()).toHaveLength(2); }); + it("threads the request's own correlation id into a local-history capture failure instead of minting one", async () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts) and is already + // in scope in handleEditorPatchApply — a local-history capture failure during the apply must + // reuse it via ApplyContext.correlationId, not a disconnected `local-history-` mint. + const diagnostics: { correlationId?: unknown }[] = []; + const throwingHistoryStore = { + capture: (): never => { + throw new Error("history capture backend unavailable"); + }, + } as unknown as EditorLocalHistoryStore; + const ctx = { ...postContext(body()), correlationId: "req-patch-apply-thread-01" }; + + const result = await handleEditorPatchApply( + ctx, + { + ...deps({ env: ENABLED, editorLocalHistoryStore: throwingHistoryStore }), + diagnostics: { + record: (record: { correlationId?: unknown }): void => { + diagnostics.push(record); + }, + }, + }, + options(), + ); + + expect(wire(result).status).toBe("applied"); + expect(diagnostics).toHaveLength(1); + expect(diagnostics[0]?.correlationId).toBe("req-patch-apply-thread-01"); + }); + it("records a reject decision and mutates nothing", async () => { const result = await handleEditorPatchApply( postContext(body({ decision: "reject" })), diff --git a/packages/keiko-server/src/editor/patchApplyRoutes.ts b/packages/keiko-server/src/editor/patchApplyRoutes.ts index 289f0342ae..99eefa5261 100644 --- a/packages/keiko-server/src/editor/patchApplyRoutes.ts +++ b/packages/keiko-server/src/editor/patchApplyRoutes.ts @@ -214,7 +214,13 @@ function captureAppliedHistory(ctx: ApplyContext, result: PatchApplyResult): voi try { content = readFileSync(absolutePath, "utf8"); } catch (error) { - emitEditorLocalHistoryCaptureFailure(ctx.deps, "agent-apply", error, ctx.nowMs); + emitEditorLocalHistoryCaptureFailure( + ctx.deps, + "agent-apply", + error, + ctx.nowMs, + ctx.correlationId, + ); continue; } captureEditorLocalHistorySafely({ @@ -225,6 +231,7 @@ function captureAppliedHistory(ctx: ApplyContext, result: PatchApplyResult): voi content, origin: "agent-apply", nowMs: ctx.nowMs, + correlationId: ctx.correlationId, }); } } @@ -255,6 +262,9 @@ interface ApplyContext { readonly signal: AbortSignal; readonly nowMs: number; readonly options: EditorPatchApplyRouteOptions; + // The request's own correlation id (ADR-0173 D5 / g12), threaded down so a local-history + // capture failure joins the SAME id as the rest of this request's trail. + readonly correlationId: string | undefined; } async function runVerificationPhase( @@ -493,6 +503,7 @@ export async function handleEditorPatchApply( signal, nowMs, options, + correlationId: ctx.correlationId, }); return { status: 200, body: deps.redactor(response) }; }); diff --git a/packages/keiko-server/src/files.test.ts b/packages/keiko-server/src/files.test.ts index 0efe76f87b..f80c1fbad4 100644 --- a/packages/keiko-server/src/files.test.ts +++ b/packages/keiko-server/src/files.test.ts @@ -1020,6 +1020,51 @@ describe("desktop files browser", () => { expect(JSON.stringify(diagnostics)).not.toContain("saved despite history failure"); }); + it("threads the request's own correlation id into a local-history capture failure instead of minting one", async () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts) and is already + // in scope in writeFilesContentRoute — the capture failure must reuse it, not a disconnected + // one. Before the fix the diagnostic and response always carried a fresh mint regardless of + // ctx.correlationId. + const diagnostics: ServerDiagnosticRecord[] = []; + const failingVault: LocalSecretVault = { + get: () => undefined, + set: (): never => { + throw new Error("capture-secret-marker-2"); + }, + replaceAll: () => undefined, + delete: () => undefined, + has: () => false, + list: () => [], + }; + const failingHistory = createEditorLocalHistoryStore({ + stateDir: join(root, ".history-test-state-correlation"), + env: {}, + vaultFactory: () => failingVault, + }); + const ctx = { + ...patchContentContext({ root, path: "src/app.ts", content: "threaded id\n" }), + correlationId: "req-files-save-thread-01", + }; + + const result = await handleFilesContent(ctx, { + store, + redactor: buildRedactor({}), + editorLocalHistoryStore: failingHistory, + diagnostics: { + record: (record: ServerDiagnosticRecord): void => { + diagnostics.push(record); + }, + }, + } as unknown as UiHandlerDeps); + + expect(result.status).toBe(200); + expect(diagnostics).toHaveLength(1); + expect(diagnostics[0]?.correlationId).toBe("req-files-save-thread-01"); + expect(result.body).toMatchObject({ + localHistoryProtection: { correlationId: "req-files-save-thread-01" }, + }); + }); + it("fails closed for an unregistered root while preserving the save and original diagnostic", async () => { const arbitrary = await realpath(await mkdtemp(join(tmpdir(), "keiko-files-unregistered-"))); extraRoot = arbitrary; @@ -1610,6 +1655,50 @@ describe("desktop files mutations (create / rename / delete)", () => { ).toHaveLength(1); }); + it("sets parentCorrelationId to the spawning rename request's own id on a skipped migration", async () => { + // ADR-0173 D5 / g12: reKeyRenamedBreakpoints runs detached (never awaited by the rename + // response), so it mints its own correlationId for this background operation — but the + // rename request's own ctx.correlationId, when known, must still ride as parentCorrelationId + // so an operator can join this diagnostic back to the request that spawned it. Before the fix + // there was no parentCorrelationId field on the emitted record at all. + const renameInstrumentation = vi.fn().mockResolvedValue(undefined); + const diagnostics: ServerDiagnosticRecord[] = []; + const breakpoints = { + snapshot: (): { readonly ok: false; readonly reason: string } => ({ + ok: false, + reason: "state_unavailable", + }), + }; + const ctx = { + ...patchContentContext({ root, path: "src/app.ts", newPath: "src/renamed.ts" }), + correlationId: "req-rename-thread-01", + }; + + const result = await handleFilesRename(ctx, { + store, + redactor: buildRedactor({}), + dapDebug: { + breakpoints, + renameInstrumentation, + diagnosticSink: { + record: (record: ServerDiagnosticRecord): void => { + diagnostics.push(record); + }, + }, + }, + } as unknown as UiHandlerDeps); + + expect(result).toMatchObject({ status: 200, body: { path: "src/renamed.ts" } }); + const skipped = diagnostics.filter((record) => + record.operation.endsWith("breakpoint-migration-skipped"), + ); + expect(skipped).toHaveLength(1); + expect(skipped[0]?.parentCorrelationId).toBe("req-rename-thread-01"); + // The operation's OWN correlationId stays a fresh, disconnected mint (it is not the request's + // id) — only parentCorrelationId links it back. + expect(skipped[0]?.correlationId).not.toBe("req-rename-thread-01"); + }); + it("delegates every fileId under a renamed directory in one call (KEIKO-0179)", async () => { await writeFile(join(root, "src", "lib.ts"), "export const b = 2;\n"); const debugStateDir = await realpath(await mkdtemp(join(tmpdir(), "keiko-files-mut-dap-dir-"))); diff --git a/packages/keiko-server/src/files.ts b/packages/keiko-server/src/files.ts index 08bd1295f8..757249cdc5 100644 --- a/packages/keiko-server/src/files.ts +++ b/packages/keiko-server/src/files.ts @@ -2537,6 +2537,7 @@ async function readFilesContentRoute(ctx: RouteContext, deps: UiHandlerDeps): Pr function createPreRestoreCapture( deps: UiHandlerDeps, target: ResolvedTarget, + correlationId: string | undefined, ): (content: string) => NonNullable { return (content) => captureEditorLocalHistorySafely({ @@ -2546,6 +2547,7 @@ function createPreRestoreCapture( absolutePath: target.path, content, origin: "pre-restore", + correlationId, }); } @@ -2553,6 +2555,7 @@ function captureNormalFileSave( deps: UiHandlerDeps, target: ResolvedTarget, fields: FilesWriteFields, + correlationId: string | undefined, ): FilesContentWireResponse["localHistoryProtection"] { if (fields.historyOrigin !== undefined) return undefined; return captureEditorLocalHistorySafely({ @@ -2562,6 +2565,7 @@ function captureNormalFileSave( absolutePath: target.path, content: fields.content, origin: "user-save", + correlationId, }); } @@ -2599,10 +2603,12 @@ async function writeFilesContentRoute( typeof body.expectedModifiedAt === "number" ? body.expectedModifiedAt : undefined, baseVersion, beforeWrite: - fields.historyOrigin === "pre-restore" ? createPreRestoreCapture(deps, target) : undefined, + fields.historyOrigin === "pre-restore" + ? createPreRestoreCapture(deps, target, ctx.correlationId) + : undefined, }); notifyHostLspWorkspaceFileChanged(target.realRoot, target.path); - const localHistoryProtection = captureNormalFileSave(deps, target, fields); + const localHistoryProtection = captureNormalFileSave(deps, target, fields, ctx.correlationId); return { status: 200, body: localHistoryProtection === undefined ? response : { ...response, localHistoryProtection }, @@ -2718,6 +2724,7 @@ async function reKeyRenamedBreakpoints( realRoot: string, previousPath: string, nextPath: string, + requestCorrelationId: string | undefined, ): Promise { const service = deps.dapDebug; if (service === undefined) return; @@ -2727,9 +2734,11 @@ async function reKeyRenamedBreakpoints( // identity-inspection failure) used to skip the whole migration silently, bypassing the // service-side rejection diagnostic entirely. The rename still must not fail — but the skipped // migration has to be observable, mirroring the service's own redacted, body-free convention. - emitServerDiagnostic( - service.diagnosticSink, - serverDiagnosticFromError({ + // This runs detached from the rename response (never awaited by the caller), so it mints its + // own id rather than reusing the request's — but `parentCorrelationId` (ADR-0173 D5 / g12) + // still joins it back to the request that spawned it when that id is known. + emitServerDiagnostic(service.diagnosticSink, { + ...serverDiagnosticFromError({ correlationId: `files-rename-${randomUUID()}`, operation: "files.rename.breakpoint-migration-skipped", source: "files.rename", @@ -2738,7 +2747,8 @@ async function reKeyRenamedBreakpoints( "Breakpoint migration for a rename was skipped: the instrumentation snapshot is " + "unavailable; breakpoints remain under the old path.", }), - ); + ...(requestCorrelationId === undefined ? {} : { parentCorrelationId: requestCorrelationId }), + }); return; } const renames = affectedRenamedFileIds(snapshot, previousPath, nextPath); @@ -2783,7 +2793,13 @@ export async function handleFilesRename( // turning a long-completed filesystem rename into a UI timeout. renameInstrumentation's // contract is that it never rejects (failures degrade to redacted diagnostics), so nothing is // silently lost by detaching. - void reKeyRenamedBreakpoints(deps, resolvedRoot.realRoot, result.previousPath, result.path); + void reKeyRenamedBreakpoints( + deps, + resolvedRoot.realRoot, + result.previousPath, + result.path, + ctx.correlationId, + ); } return { status: 200, body: result }; }); diff --git a/packages/keiko-server/src/gateway-error-diagnostic.test.ts b/packages/keiko-server/src/gateway-error-diagnostic.test.ts new file mode 100644 index 0000000000..de6367fc68 --- /dev/null +++ b/packages/keiko-server/src/gateway-error-diagnostic.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; +import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; +import { + emitGatewayErrorDiagnostic, + type GatewayErrorDiagnosticDeps, +} from "./gateway-error-diagnostic.js"; + +function capturingDeps(): { + readonly deps: GatewayErrorDiagnosticDeps; + readonly events: ServerDiagnosticRecord[]; +} { + const events: ServerDiagnosticRecord[] = []; + return { + events, + deps: { + diagnostics: { + record: (record): void => { + events.push(record); + }, + }, + redactor: (value: unknown): unknown => value, + }, + }; +} + +describe("emitGatewayErrorDiagnostic", () => { + it("emits one record through the sink with the given operation/source", () => { + const { deps, events } = capturingDeps(); + + emitGatewayErrorDiagnostic( + deps, + new Error("boom"), + "correlation-9", + "POST /api/example", + "example.source", + ); + + expect(events).toHaveLength(1); + const [event] = events; + if (event === undefined) throw new Error("expected a diagnostic record"); + expect(event.correlationId).toBe("correlation-9"); + expect(event.operation).toBe("POST /api/example"); + expect(event.source).toBe("example.source"); + expect(event.errorClass).toBe("Error"); + }); + + it("falls back the correlation id to the fixed unknown-id sentinel when none is known", () => { + const { deps, events } = capturingDeps(); + + emitGatewayErrorDiagnostic( + deps, + new Error("boom"), + undefined, + "POST /api/example", + "example.source", + ); + + expect(events[0]?.correlationId).toBe(UNKNOWN_CORRELATION_ID); + }); + + it("never throws when no diagnostics sink is configured", () => { + const deps: GatewayErrorDiagnosticDeps = { + diagnostics: undefined, + redactor: (value: unknown): unknown => value, + }; + + expect(() => { + emitGatewayErrorDiagnostic( + deps, + new Error("boom"), + "correlation-1", + "POST /api/example", + "example.source", + ); + }).not.toThrow(); + }); +}); diff --git a/packages/keiko-server/src/gateway-error-diagnostic.ts b/packages/keiko-server/src/gateway-error-diagnostic.ts new file mode 100644 index 0000000000..98314b46d1 --- /dev/null +++ b/packages/keiko-server/src/gateway-error-diagnostic.ts @@ -0,0 +1,48 @@ +// Shared GatewayError → operator-diagnostic wiring (ADR-0173 D5 g25/g27). +// +// Before this module, `chat-stream-handlers.ts`'s `errorEvent()` was the ONLY place a GatewayError +// reaching the desktop chat surface got routed to the redacted operator diagnostic sink before the +// response left — the buffered `/api/desktop/chat` path (`chat-handlers.ts`) and grounded Q&A +// (`grounded-qa.ts`) mapped the SAME error class straight to an HTTP body with no diagnostic call +// at all, so a mid-request gateway failure was traceable only when it happened to arrive over SSE. +// This file extracts that one block so every caller emits the identical record shape. +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; +import { + emitServerDiagnostic, + serverDiagnosticFromError, + type ServerDiagnosticSink, +} from "./diagnostics-log.js"; +import type { Redactor } from "./deps.js"; + +// A structural subset of `UiHandlerDeps` — the two fields this helper actually reads — so callers +// never need to import the full `UiHandlerDeps` type just to satisfy this one function. +export interface GatewayErrorDiagnosticDeps { + readonly diagnostics?: ServerDiagnosticSink | undefined; + readonly redactor: Redactor; +} + +// `correlationId` falls back to `UNKNOWN_CORRELATION_ID` rather than being omitted: +// `ServerDiagnosticRecord` requires one (a diagnostic nobody can join to a request is far less +// useful than one honestly marked unjoinable), matching the fallback `chat-stream-handlers.ts`'s +// `errorEvent()` already used before this extraction — every caller inherits the same behaviour, +// not a stricter or looser one. The fixed sentinel (rather than the bare literal `"unknown"`) is +// deliberate: it is shape-valid, so it survives `emitServerDiagnostic`'s sanitizer unchanged instead +// of being rewritten to the sanitizer's own "hostile value" marker. +export function emitGatewayErrorDiagnostic( + deps: GatewayErrorDiagnosticDeps, + error: unknown, + correlationId: string | undefined, + operation: string, + source: string, +): void { + emitServerDiagnostic( + deps.diagnostics, + serverDiagnosticFromError({ + correlationId: correlationId ?? UNKNOWN_CORRELATION_ID, + operation, + source, + error, + redact: (message) => String(deps.redactor(message)), + }), + ); +} diff --git a/packages/keiko-server/src/gateway-setup.test.ts b/packages/keiko-server/src/gateway-setup.test.ts index 0d86ad811c..efefa44a04 100644 --- a/packages/keiko-server/src/gateway-setup.test.ts +++ b/packages/keiko-server/src/gateway-setup.test.ts @@ -5872,6 +5872,40 @@ describe("handleGatewaySetup", () => { deps.store.close(); }); + // ADR-0173 D5 g12: the discovery-truncation diagnostic must join the SAME trace as the rest of + // this setup attempt (e.g. the gateway.chat probe lines), not mint a disconnected id of its own + // — otherwise an operator cannot tell which setup attempt a truncation diagnostic belongs to. + it("threads the request's correlation id onto the discovery-truncation diagnostic (g12)", async () => { + const uiDir = await tempDir("keiko-gw-ui-discovery-truncated-corr-"); + const evidenceDir = await tempDir("keiko-gw-ev-discovery-truncated-corr-"); + const diagnostics: ServerDiagnosticRecord[] = []; + const oversized = Array.from({ length: MAX_DISCOVERED_MODELS + 5 }, (_unused, index) => ({ + id: `discovered-model-${String(index)}`, + })); + const deps = buildUiHandlerDeps({ + configPath: undefined, + evidenceDir, + env: { ...VAULT_ENV }, + uiDbPath: join(uiDir, "keiko-ui.db"), + gatewayModelDiscovery: () => Promise.resolve(parseModelDiscovery({ data: oversized })), + gatewayEmbeddingProbe: PASSTHROUGH_EMBEDDING_PROBE, + gatewaySetupTester: (_config, modelIds) => Promise.resolve([...modelIds]), + diagnostics: { record: (record): void => void diagnostics.push(record) }, + }); + + await handleGatewaySetup( + ctx( + { baseUrl: "https://llm-gateway.example.com", apiKey: "example-secret-token" }, + "corr-discovery-truncation-g12", + ), + deps, + ); + + const truncation = diagnostics.find((record) => record.code === "GATEWAY_DISCOVERY_TRUNCATED"); + expect(truncation?.correlationId).toBe("corr-discovery-truncation-g12"); + deps.store.close(); + }); + it("does not emit the truncation diagnostic when discovery fits the cap (KEIKO-0325)", async () => { const uiDir = await tempDir("keiko-gw-ui-discovery-fits-"); const evidenceDir = await tempDir("keiko-gw-ev-discovery-fits-"); diff --git a/packages/keiko-server/src/gateway-setup.ts b/packages/keiko-server/src/gateway-setup.ts index feac26bb05..662f33d2dc 100644 --- a/packages/keiko-server/src/gateway-setup.ts +++ b/packages/keiko-server/src/gateway-setup.ts @@ -77,6 +77,7 @@ import type { VerifiedModelCapabilityFields, } from "./deps.js"; import { currentGatewayConfig, currentGatewayEgressConfig } from "./deps.js"; +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError, @@ -1807,6 +1808,7 @@ export async function smokeTestCandidates( async function defaultGatewaySetupTester( config: GatewayConfig, candidateModelIds: readonly string[], + correlationId: string | undefined, ): Promise { // Wired to the process activity log: first-run setup is where an operator's endpoint is wrong // in a way no UI message can name (a proxy that blocks CONNECT, a provider that answers 404 for @@ -1821,6 +1823,7 @@ async function defaultGatewaySetupTester( { role: "system", content: CONVERSATION_SYSTEM_PROMPT }, { role: "user", content: "Reply with exactly: OK" }, ], + logContext: { correlationId }, }); }, SETUP_SMOKE_CONCURRENCY, @@ -1828,7 +1831,10 @@ async function defaultGatewaySetupTester( const responseFormatModelIds = await passingCandidates( testedModelIds, async (modelId) => { - const response = await gateway.chat(buildQiJudgePreflightRequest(modelId)); + const response = await gateway.chat({ + ...buildQiJudgePreflightRequest(modelId), + logContext: { correlationId }, + }); if (tryParseJudgeVerdict(response.content) === null) { throw new Error("response format unsupported"); } @@ -1913,8 +1919,18 @@ function gatewayEmbeddingProbe(deps: UiHandlerDeps): GatewayEmbeddingProbe { return deps.gatewayEmbeddingProbe ?? defaultGatewayEmbeddingProbe; } -function gatewaySetupTester(deps: UiHandlerDeps): GatewaySetupTester { - return deps.gatewaySetupTester ?? defaultGatewaySetupTester; +// The seam type (UiHandlerDeps["gatewaySetupTester"]) is a fixed 2-arg shape shared by every +// test override, so the request-scoped correlation id is closed over here rather than added as a +// 3rd seam parameter — the override contract stays untouched while the real tester still stamps +// GatewayCallRequest.logContext (ADR-0173 D5). +function gatewaySetupTester( + deps: UiHandlerDeps, + correlationId: string | undefined, +): GatewaySetupTester { + const override = deps.gatewaySetupTester; + if (override !== undefined) return override; + return (config, candidateModelIds) => + defaultGatewaySetupTester(config, candidateModelIds, correlationId); } const FIGMA_ME_ENDPOINT = "https://api.figma.com/v1/me"; @@ -4136,7 +4152,7 @@ function reportSetupVerificationFailure( emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ - correlationId: correlationId ?? "unknown", + correlationId: correlationId ?? UNKNOWN_CORRELATION_ID, operation: "POST /api/gateway/setup", source, error, @@ -4194,6 +4210,11 @@ interface SetupVerificationInput { readonly current: GatewayConfig | undefined; /** Operator diagnostic sink; used to surface discovery truncation (KEIKO-0325). */ readonly diagnostics?: ServerDiagnosticSink | undefined; + // The request's own correlation id (ADR-0173 D5 g12), threaded through so a discovery-truncation + // or unusable-models diagnostic for THIS setup attempt joins the same trace as the gateway.chat + // probe lines `verifySetupCandidate` triggers, instead of minting a disconnected id. Falls back + // to a fresh mint only when the request genuinely carried none. + readonly correlationId: string | undefined; } interface SetupCandidateModels { @@ -4616,11 +4637,12 @@ function candidateProbeOptions( // construction — a count and a code, never a model id or an endpoint. function reportDiscoveryTruncation( diagnostics: ServerDiagnosticSink | undefined, + correlationId: string | undefined, candidateModels: SetupCandidateModels, ): void { if (candidateModels.truncated !== true) return; emitServerDiagnostic(diagnostics, { - correlationId: randomUUID(), + correlationId: correlationId ?? randomUUID(), timestamp: new Date().toISOString(), operation: "POST /api/gateway/setup", source: "gateway-setup.discovery", @@ -4705,6 +4727,7 @@ async function admitEmbeddingCandidates( // channel stays free of gateway inventory. function reportUnusableDiscoveredModels( diagnostics: ServerDiagnosticSink | undefined, + correlationId: string | undefined, unsupported: readonly GatewayUnsupportedDiscoveredModel[], admission: EmbeddingAdmission, ): void { @@ -4712,7 +4735,7 @@ function reportUnusableDiscoveredModels( const dropped = admission.droppedUnverified.length; if (unsupported.length === 0 && retained === 0 && dropped === 0) return; emitServerDiagnostic(diagnostics, { - correlationId: randomUUID(), + correlationId: correlationId ?? randomUUID(), timestamp: new Date().toISOString(), operation: "POST /api/gateway/setup", source: "gateway-setup.discovery", @@ -4735,7 +4758,7 @@ async function verifySetupCandidate(input: SetupVerificationInput): Promise 0 ? DEPLOYMENT_SMOKE_TIMEOUT_MS @@ -4754,6 +4777,7 @@ async function verifySetupCandidate(input: SetupVerificationInput): Promise { const seams: SetupSeams = { - tester: gatewaySetupTester(deps), + tester: gatewaySetupTester(deps, request.correlationId), embeddingProbe: gatewayEmbeddingProbe(deps), discovery: deps.gatewayModelDiscovery ?? defaultGatewayModelDiscovery, }; diff --git a/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.test.ts b/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.test.ts index 9c6d3299f9..52ed2e5192 100644 --- a/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.test.ts +++ b/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.test.ts @@ -204,4 +204,70 @@ describe("recordGitDeliveryMutationEvidence — fail-closed + never throws", () consoleError.mockRestore(); } }); + + // ADR-0173 D5 / g12: a persistence-failure diagnostic used to mint a fresh, disconnected + // `randomUUID()` even though the record already carries the mutation's own deterministic + // `correlation.actionId` (`defaultGitDeliveryActionId`'s output shape, reused here). Fails + // before the fix — the diagnostic's correlationId would be an unrelated fresh UUID instead of + // the record's own action id. + it("threads the record's own action id as the diagnostic correlation id when it is validly shaped", () => { + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (entry) => { + records.push(entry); + }, + }; + const actionId = "gde-action-deadbeefcafefeed01234567"; + const throwingStore: EvidenceStore = { + put: (): string => { + throw Object.assign(new Error("ledger write failed"), { code: "ENOSPC" }); + }, + get: (): string | undefined => undefined, + list: (): readonly string[] => [], + delete: (): void => { + /* no-op */ + }, + }; + + recordGitDeliveryMutationEvidence( + { evidenceStore: throwingStore, redactString, diagnostics }, + record({ correlation: { workflowRunIdHash: "a".repeat(64), actionId } }), + ); + + expect(records).toHaveLength(1); + expect(records[0]?.correlationId).toBe(actionId); + }); + + // The `record()` fixture's default `actionId` ("act-1") is only 5 characters — shorter than + // `isValidCorrelationId`'s 8-character floor — so this pins the fallback branch: an actionId + // that is not validly shaped as a correlation id must never reach the diagnostic verbatim, and + // the existing shape-only pin above (line ~200) still passes because the fallback is a fresh, + // validly-shaped UUID, not the unshaped actionId itself. + it("falls back to a fresh id when the record's action id is not validly shaped", () => { + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (entry) => { + records.push(entry); + }, + }; + const throwingStore: EvidenceStore = { + put: (): string => { + throw new Error("ledger write failed"); + }, + get: (): string | undefined => undefined, + list: (): readonly string[] => [], + delete: (): void => { + /* no-op */ + }, + }; + + recordGitDeliveryMutationEvidence( + { evidenceStore: throwingStore, redactString, diagnostics }, + record(), + ); + + expect(records).toHaveLength(1); + expect(records[0]?.correlationId).not.toBe("act-1"); + expect(records[0]?.correlationId).toMatch(/^[A-Za-z0-9._-]{8,128}$/); + }); }); diff --git a/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.ts b/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.ts index cdaff760de..2d292fe54c 100644 --- a/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.ts +++ b/packages/keiko-server/src/gitDelivery/mutationEvidenceLedger.ts @@ -26,6 +26,7 @@ import { import { deepRedactStrings } from "@oscharko-dev/keiko-evidence"; import type { EvidenceStore } from "@oscharko-dev/keiko-evidence"; import { randomUUID } from "node:crypto"; +import { isValidCorrelationId } from "../correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError, @@ -66,7 +67,11 @@ export interface RecordGitDeliveryEvidenceOptions { // The previous default was `console.error("…", error)` — the raw error object on a channel no // production assembly overrode, so a failed governed-mutation evidence write was invisible AND could // carry the very content this ledger redacts. Routed through the single redacted diagnostic sink. -function reportPersistFailure(options: RecordGitDeliveryEvidenceOptions, error: unknown): void { +function reportPersistFailure( + options: RecordGitDeliveryEvidenceOptions, + correlationId: string, + error: unknown, +): void { if (options.onPersistError !== undefined) { options.onPersistError(error); return; @@ -74,7 +79,7 @@ function reportPersistFailure(options: RecordGitDeliveryEvidenceOptions, error: emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "gitDelivery.mutationEvidence.persist", source: "gitDelivery.mutationEvidenceLedger", error, @@ -84,6 +89,20 @@ function reportPersistFailure(options: RecordGitDeliveryEvidenceOptions, error: ); } +// The record's own `correlation.actionId` (ADR-0173 D5 / g12) is the run already in scope at the +// moment this evidence was built — a deterministic, content-free id (`defaultGitDeliveryActionId` +// hashes the command; `gitDelivery/mergeExecution.ts` and siblings mint it once per governed +// mutation attempt). Reusing it instead of a fresh `randomUUID()` lets an operator join a failed +// persistence diagnostic back to the SAME mutation attempt's other evidence. Re-validated here +// against `isValidCorrelationId`'s shape rather than trusted blindly: `actionId` is typed as a +// plain `string` on the wire contract, so a producer that has not adopted the deterministic helper +// (or a test fixture) can still hand this a value that is not safe to use as a correlation id — +// that case falls back to a fresh mint, exactly like the pre-fix behaviour. +function evidenceCorrelationId(record: GitDeliveryEvidenceRecord): string { + const { actionId } = record.correlation; + return isValidCorrelationId(actionId) ? actionId : randomUUID(); +} + function isPlainObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -157,6 +176,6 @@ export function recordGitDeliveryMutationEvidence( try { appendRecord(options.evidenceStore, runId, safe, cap); } catch (error) { - reportPersistFailure(options, error); + reportPersistFailure(options, evidenceCorrelationId(record), error); } } diff --git a/packages/keiko-server/src/gitDelivery/syncEvidence.test.ts b/packages/keiko-server/src/gitDelivery/syncEvidence.test.ts index 11e3634817..aa8a381e65 100644 --- a/packages/keiko-server/src/gitDelivery/syncEvidence.test.ts +++ b/packages/keiko-server/src/gitDelivery/syncEvidence.test.ts @@ -250,6 +250,11 @@ describe("recordGitSyncEvidence — best-effort and fail-closed", () => { expect(records[0]?.source).toBe("gitDelivery.syncEvidence"); expect(records[0]?.code).toBe("ENOSPC"); expect(records[0]?.correlationId).toMatch(/^[A-Za-z0-9._-]{8,128}$/); + // ADR-0173 D5 / g12: the failure's correlationId is the SAME date-bucket runId the write + // itself targeted (from the real producer, not a re-derived formula), not a disconnected + // `randomUUID()` — an operator can join the failure back to the bucket it belongs to. Before + // the fix this was a random UUID and never equalled the bucket runId. + expect(records[0]?.correlationId).toBe(gitSyncEvidenceRunIdFor(AT)); expect(JSON.stringify(records)).not.toContain(secret); expect(consoleError).not.toHaveBeenCalled(); } finally { diff --git a/packages/keiko-server/src/gitDelivery/syncEvidence.ts b/packages/keiko-server/src/gitDelivery/syncEvidence.ts index 73ed65212b..940f2c5118 100644 --- a/packages/keiko-server/src/gitDelivery/syncEvidence.ts +++ b/packages/keiko-server/src/gitDelivery/syncEvidence.ts @@ -16,6 +16,7 @@ import { deepRedactStrings } from "@oscharko-dev/keiko-evidence"; import type { EvidenceStore } from "@oscharko-dev/keiko-evidence"; import { sha256Hex } from "@oscharko-dev/keiko-security"; import { randomUUID } from "node:crypto"; +import { isValidCorrelationId } from "../correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError, @@ -78,15 +79,25 @@ export interface RecordGitSyncEvidenceOptions { // The previous default was `console.error("…", error)` — the raw error object on a channel no // production assembly overrode, so a failed evidence write was invisible AND could carry the very // content this ledger redacts. Routed through the server's single redacted diagnostic sink instead. -function reportPersistFailure(options: RecordGitSyncEvidenceOptions, error: unknown): void { +function reportPersistFailure( + options: RecordGitSyncEvidenceOptions, + runId: string, + error: unknown, +): void { if (options.onPersistError !== undefined) { options.onPersistError(error); return; } + // Reuses the date-bucket runId the write itself targeted (ADR-0173 D5 / g12, mirroring + // mutationEvidenceLedger.ts's `evidenceCorrelationId`) rather than a disconnected fresh mint, so + // an operator can join this diagnostic back to the SAME bucket's other evidence. Re-validated + // against `isValidCorrelationId` even though this module derives the shape itself: a future + // change to the runId format must not silently become an unshaped correlation id. + const correlationId = isValidCorrelationId(runId) ? runId : randomUUID(); emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "gitDelivery.syncEvidence.persist", source: "gitDelivery.syncEvidence", error, @@ -165,6 +176,6 @@ export function recordGitSyncEvidence( try { appendRecord(options.evidenceStore, runId, safe, cap); } catch (error) { - reportPersistFailure(options, error); + reportPersistFailure(options, runId, error); } } diff --git a/packages/keiko-server/src/grounded-entailment-judge.test.ts b/packages/keiko-server/src/grounded-entailment-judge.test.ts index 7dc00edb5e..a046f24c9c 100644 --- a/packages/keiko-server/src/grounded-entailment-judge.test.ts +++ b/packages/keiko-server/src/grounded-entailment-judge.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import type { - GatewayRequest, + GatewayCallRequest, ModelCapability, NormalizedResponse, } from "@oscharko-dev/keiko-model-gateway"; @@ -32,10 +32,10 @@ function throwingPort(): ModelPort { }; } -function respondingPort(content: string): { port: ModelPort; calls: GatewayRequest[] } { - const calls: GatewayRequest[] = []; +function respondingPort(content: string): { port: ModelPort; calls: GatewayCallRequest[] } { + const calls: GatewayCallRequest[] = []; const port: ModelPort = { - call: (request: GatewayRequest): Promise => { + call: (request: GatewayCallRequest): Promise => { calls.push(request); return Promise.resolve({ content, @@ -176,6 +176,21 @@ describe("createGatewayEntailmentJudge", () => { ); }); + // ADR-0173 D5: the entailment stage's own turn-scoped correlation id must reach the judge's + // model.call so a gateway retry/circuit-breaker line for this second-pass verification joins + // the same trail as the answer it is checking. + it("stamps the supplied correlation id into the judge's GatewayCallRequest.logContext", async () => { + const supported = respondingPort('{"verdict":"supported"}'); + const judge = createGatewayEntailmentJudge( + depsWith(supported.port), + MODEL_ID, + "cid-entailment-judge-000001", + ); + await judge?.judge({ claimText: "30 days", excerptText: "retention: 30 days" }); + expect(supported.calls).toHaveLength(1); + expect(supported.calls[0]?.logContext?.correlationId).toBe("cid-entailment-judge-000001"); + }); + it("fails closed to unavailable when the gateway throws (never supported, never throws)", async () => { const judge = createGatewayEntailmentJudge(depsWith(throwingPort()), MODEL_ID); expect(judge).toBeDefined(); diff --git a/packages/keiko-server/src/grounded-entailment-judge.ts b/packages/keiko-server/src/grounded-entailment-judge.ts index c1eaea6387..f2a3fa0791 100644 --- a/packages/keiko-server/src/grounded-entailment-judge.ts +++ b/packages/keiko-server/src/grounded-entailment-judge.ts @@ -19,6 +19,7 @@ import { findCapability, findConfiguredCapability, type ChatMessage, + type GatewayCallRequest, type GatewayRequest, type ModelCapability, } from "@oscharko-dev/keiko-model-gateway"; @@ -171,6 +172,7 @@ function isModelCompatible(capability: ModelCapability | undefined): boolean { export function createGatewayEntailmentJudge( deps: UiHandlerDeps, modelId: string, + correlationId?: string, ): EntailmentJudge | undefined { const capability = capabilityFor(deps, modelId); if (!isModelCompatible(capability)) return undefined; @@ -183,13 +185,14 @@ export function createGatewayEntailmentJudge( signal?: AbortSignal, ): Promise => { const cancellation = MgQI.composeCancellationSignal(profile.timeoutMsHint, signal); - const request: GatewayRequest = { + const request: GatewayCallRequest = { modelId, messages: buildEntailmentPrompt(input), stream: false, cancellationSignal: cancellation.signal, temperature: 0, responseFormat: buildEntailmentResponseFormat(), + logContext: { correlationId }, }; try { const response = await model.call(request, cancellation.signal); diff --git a/packages/keiko-server/src/grounded-entailment-stage.ts b/packages/keiko-server/src/grounded-entailment-stage.ts index 7c39d36e55..933462094f 100644 --- a/packages/keiko-server/src/grounded-entailment-stage.ts +++ b/packages/keiko-server/src/grounded-entailment-stage.ts @@ -176,7 +176,7 @@ export function createEntailmentStage( ...observability, correlationId: observability.correlationId ?? randomUUID(), }; - const judge = createGatewayEntailmentJudge(deps, modelId); + const judge = createGatewayEntailmentJudge(deps, modelId, correlated.correlationId); if (judge === undefined) { // KEIKO-0359: report WHY the stage is inert. Going inert used to be completely silent, so a // model whose capability metadata Gateway Setup never enriched looked identical to a model diff --git a/packages/keiko-server/src/grounded-qa-hybrid.test.ts b/packages/keiko-server/src/grounded-qa-hybrid.test.ts index 1abdbf062b..fb770dce14 100644 --- a/packages/keiko-server/src/grounded-qa-hybrid.test.ts +++ b/packages/keiko-server/src/grounded-qa-hybrid.test.ts @@ -54,7 +54,9 @@ import { GROUNDED_NO_EVIDENCE_ANSWER } from "./grounded-faithfulness.js"; import type { ModelPort } from "@oscharko-dev/keiko-harness"; import { parseGatewayConfig, + type GatewayCallRequest, type ModelCapability, + type NormalizedResponse, type RerankOutcome, } from "@oscharko-dev/keiko-model-gateway"; import type { GroundedRetriever } from "./grounded-qa-multi-source.js"; @@ -62,11 +64,13 @@ import { EmbeddingAdapterError, connectorQuery, connectorRetrievalTopK, + createHybridAnswerer, estimateConnectorExcerptBytes, hashString32, runHybridGroundedAsk, type ConnectorRetrieve, } from "./grounded-qa-hybrid.js"; +import { normalizeGroundedAnswerPayload } from "./grounded-answer.js"; import { createInMemoryUiStore, type UiStore } from "./store/index.js"; import type { UiHandlerDeps } from "./deps.js"; import { buildRedactor, createRunRegistry } from "./index.js"; @@ -2064,7 +2068,7 @@ describe("AC5 routing — single connector must route to handleLocalKnowledgeGro { modelId: CHAT_MODEL, baseUrl: "https://provider.example/v1", - apiKey: "test-api-key-1234567890", + apiKey: "example-secret-token", timeoutMs: 30_000, maxRetries: 0, retryBaseDelayMs: 500, @@ -2072,7 +2076,7 @@ describe("AC5 routing — single connector must route to handleLocalKnowledgeGro { modelId: embeddingModelId, baseUrl: "https://provider.example/v1", - apiKey: "test-api-key-1234567890", + apiKey: "example-secret-token", timeoutMs: 30_000, maxRetries: 0, retryBaseDelayMs: 500, @@ -2124,6 +2128,100 @@ describe("AC5 routing — single connector must route to handleLocalKnowledgeGro const lkAnswer = asLocalKnowledge(answer); expect(lkAnswer.contextPack.kind).toBe("local-knowledge"); }); + + // ADR-0173 D5: local-knowledge-grounded-qa.ts's StoreBackedAnswerGenerator.generate is the real + // model.call site this dispatch path reaches (no seam bypasses it here, unlike the hybrid tests + // above) — it must stamp the request's correlation id into GatewayCallRequest.logContext. + it("threads the request correlation id into the local-knowledge answerer's model gateway call", async () => { + const { capsuleId: capId } = await seedReadyCapsule("Solo Docs Correlation"); + const chatId = makeHybridChat([], [{ kind: "capsule", capsuleId: capId, connectedAtMs: NOW }]); + const hybrid: HybridSeam = { answer: throwingHybridAnswerer() }; + const embeddingModelId = "text-embedding-3-small"; + const adapter = scriptedAdapter(); + const seenRequests: GatewayCallRequest[] = []; + const recordingModelPort: ModelPort = { + call: (request): Promise => { + seenRequests.push(request); + return Promise.resolve({ + modelId: CHAT_MODEL, + content: "Local knowledge answer [1].", + finishReason: "stop" as const, + toolCalls: [], + structuredOutput: null, + usage: { + requestId: "lk-logcontext-test", + promptTokens: 10, + completionTokens: 5, + latencyMs: 5, + costClass: "medium" as const, + }, + }); + }, + }; + const configuredDeps: UiHandlerDeps = { + ...hybridDeps({ localKnowledgeEmbeddingRequest: adapter.request }), + config: { + providers: [ + { + modelId: CHAT_MODEL, + baseUrl: "https://provider.example/v1", + apiKey: "example-secret-token", + timeoutMs: 30_000, + maxRetries: 0, + retryBaseDelayMs: 500, + }, + { + modelId: embeddingModelId, + baseUrl: "https://provider.example/v1", + apiKey: "example-secret-token", + timeoutMs: 30_000, + maxRetries: 0, + retryBaseDelayMs: 500, + }, + ], + circuitBreaker: { failureThreshold: 5, cooldownMs: 30_000, halfOpenProbes: 2 }, + capabilities: [ + { + id: CHAT_MODEL, + kind: "chat", + contextWindow: 64_000, + maxOutputTokens: 4_096, + toolCalling: true, + structuredOutput: true, + streaming: true, + supportsImageInput: false, + supportsDocumentInput: false, + workflowEligible: false, + costClass: "medium", + latencyClass: "standard", + throughputHint: "test", + preferredUseCases: [], + knownLimitations: [], + }, + ], + }, + configPresent: true, + modelPortFactory: () => recordingModelPort, + }; + const requestCtx: RouteContext = { + ...routeCtx(JSON.stringify({ chatId, content: "Solo question", modelId: CHAT_MODEL })), + correlationId: "cid-local-knowledge-000001", + }; + + const result = await handleGroundedAsk( + requestCtx, + configuredDeps, + undefined, + undefined, + hybrid, + ); + + expect(result.status, JSON.stringify(result.body)).toBe(200); + expect(seenRequests.length).toBeGreaterThan(0); + for (const request of seenRequests) { + expect(request.logContext?.correlationId).toBe("cid-local-knowledge-000001"); + } + }); }); // ─── Case 4c: Configured model reranker over hybrid candidates ──────────────── @@ -3298,3 +3396,52 @@ describe("hybrid entailment forwards the retrieved folder packs (KEIKO-0237)", ( expect(observedCapsulesPerCall).toEqual([1]); }); }); + +// ─── Correlation threading (ADR-0173 D5) ────────────────────────────────────── +// +// Every hybrid dispatch test above injects `HybridSeam.answer` (a bare (system, user) => string +// function), which never touches the real Gateway-backed answerer this file's production code +// builds via `resolveHybridAnswerer` -> `createHybridAnswerer`. That builder is the actual +// model.call site fixed here, so it is unit-tested directly against a fake ModelPort that records +// the request it receives. +describe("createHybridAnswerer correlation threading", () => { + it("stamps the caller's correlation id into the Gateway double's GatewayCallRequest.logContext", async () => { + const seenRequests: GatewayCallRequest[] = []; + const recordingModel: ModelPort = { + call(request): Promise { + seenRequests.push(request); + return Promise.resolve({ + modelId: request.modelId, + content: "hybrid answer", + finishReason: "stop", + toolCalls: [], + structuredOutput: null, + usage: { + requestId: "hybrid-answerer-test", + promptTokens: 3, + completionTokens: 2, + latencyMs: 1, + costClass: "medium", + }, + }); + }, + }; + + const answerer = createHybridAnswerer( + recordingModel, + CHAT_MODEL, + new AbortController().signal, + "cid-hybrid-answerer-000001", + ); + // `HybridAnswerer`'s declared return type is `Promise` (a + // `string | GroundedAnswerResult` union), even though `createHybridAnswerer`'s own + // implementation always resolves the object branch — `normalizeGroundedAnswerPayload` is the + // SAME narrowing every production caller already applies to a `HybridAnswerer` result + // (grounded-qa-hybrid.ts, grounded-orchestrator.ts), not a test-only cast. + const result = normalizeGroundedAnswerPayload(await answerer("system prompt", "user prompt")); + + expect(result.content).toBe("hybrid answer"); + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0]?.logContext?.correlationId).toBe("cid-hybrid-answerer-000001"); + }); +}); diff --git a/packages/keiko-server/src/grounded-qa-hybrid.ts b/packages/keiko-server/src/grounded-qa-hybrid.ts index 75ee95ca6d..22d8eacffd 100644 --- a/packages/keiko-server/src/grounded-qa-hybrid.ts +++ b/packages/keiko-server/src/grounded-qa-hybrid.ts @@ -185,6 +185,9 @@ export interface HybridGroundedAskCtx { readonly deps: UiHandlerDeps; readonly signal: AbortSignal; readonly readinessAdmission?: ConversationReadinessAdmission | undefined; + // ADR-0173 D5: the request-scoped correlation id, threaded from PreparedGroundedAsk into the + // GatewayCallRequest.logContext the hybrid answerer stamps onto its model.call. + readonly correlationId?: string | undefined; readonly folderRetriever?: FolderRetriever; readonly connectorRetrieve?: ConnectorRetrieve; readonly answer?: HybridAnswerer; @@ -811,6 +814,7 @@ export function createHybridAnswerer( model: ModelPort, modelId: string, signal: AbortSignal, + correlationId: string | undefined, ): HybridAnswerer { return async (system, user): Promise => { ensureNotCancelled(signal); @@ -822,6 +826,7 @@ export function createHybridAnswerer( { role: "user", content: user }, ], stream: false, + logContext: { correlationId }, }, signal, ); @@ -1654,7 +1659,7 @@ function resolveHybridAnswerer(ctx: HybridGroundedAskCtx): ResolvedAnswerer | Ro readinessAdmission, ctx.deps, ); - return { answer: createHybridAnswerer(model, ctx.modelId, ctx.signal) }; + return { answer: createHybridAnswerer(model, ctx.modelId, ctx.signal, ctx.correlationId) }; } async function noEvidenceAssistant( diff --git a/packages/keiko-server/src/grounded-qa-multi-source.test.ts b/packages/keiko-server/src/grounded-qa-multi-source.test.ts index 04164b4d51..c779360765 100644 --- a/packages/keiko-server/src/grounded-qa-multi-source.test.ts +++ b/packages/keiko-server/src/grounded-qa-multi-source.test.ts @@ -41,6 +41,7 @@ import { buildLabeledAnswerCitations, buildConnectedScopes, buildMultiSourceGatewayMessages, + createMultiSourceAnswerer, mergeContextPackSummaries, runMultiSourceAsk, sourceLabels, @@ -50,6 +51,7 @@ import { type MultiSourceAnswerer, } from "./grounded-qa-multi-source.js"; import { buildGroundedAnswerContextPackSummary } from "@oscharko-dev/keiko-contracts/bff-wire"; +import { normalizeGroundedAnswerPayload } from "./grounded-answer.js"; import { attachContextBudgetDiagnostics } from "./grounded-context-diagnostics.js"; import { createInMemoryUiStore, type Chat, type UiStore } from "./store/index.js"; import type { UiHandlerDeps } from "./deps.js"; @@ -57,7 +59,12 @@ import { buildRedactor, createRunRegistry } from "./index.js"; import type { RouteContext } from "./routes.js"; import type { OrchestratorInput, OrchestratorOutput } from "./grounded-orchestrator.js"; import { RepoSearchUnsupportedFileError } from "@oscharko-dev/keiko-workspace"; -import { ContextOverflowError } from "@oscharko-dev/keiko-model-gateway"; +import { + ContextOverflowError, + type GatewayCallRequest, + type NormalizedResponse, +} from "@oscharko-dev/keiko-model-gateway"; +import type { ModelPort } from "@oscharko-dev/keiko-harness"; const NOW = 1_700_000_000_000; const CHAT_MODEL = "example-chat-model"; @@ -1635,3 +1642,50 @@ describe("multi-source entailment forwards the retrieved packs (KEIKO-0237)", () expect(observedCapsulesPerCall).toEqual([0]); }); }); + +// ─── Correlation threading (ADR-0173 D5) ────────────────────────────────────── +// +// createMultiSourceAnswerer is the real model.call site the tests above bypass via an injected +// MultiSourceSeam.answerer; unit-test it directly against a fake ModelPort that records the request. +describe("createMultiSourceAnswerer correlation threading", () => { + it("stamps the caller's correlation id into the Gateway double's GatewayCallRequest.logContext", async () => { + const seenRequests: GatewayCallRequest[] = []; + const recordingModel: ModelPort = { + call(request): Promise { + seenRequests.push(request); + return Promise.resolve({ + modelId: request.modelId, + content: "multi-source answer", + finishReason: "stop", + toolCalls: [], + structuredOutput: null, + usage: { + requestId: "multi-source-answerer-test", + promptTokens: 3, + completionTokens: 2, + latencyMs: 1, + costClass: "medium", + }, + }); + }, + }; + + const answerer = createMultiSourceAnswerer( + recordingModel, + "example-chat-model", + buildRedactor({}), + new AbortController().signal, + "cid-multi-source-answerer-000001", + ); + // `MultiSourceAnswerer`'s declared return type is `Promise` (a + // `string | GroundedAnswerResult` union), even though `createMultiSourceAnswerer`'s own + // implementation always resolves the object branch — `normalizeGroundedAnswerPayload` is the + // SAME narrowing every production caller already applies to this result + // (grounded-qa-multi-source.ts, grounded-orchestrator.ts), not a test-only cast. + const result = normalizeGroundedAnswerPayload(await answerer("What is alpha?", [])); + + expect(result.content).toBe("multi-source answer"); + expect(seenRequests).toHaveLength(1); + expect(seenRequests[0]?.logContext?.correlationId).toBe("cid-multi-source-answerer-000001"); + }); +}); diff --git a/packages/keiko-server/src/grounded-qa-multi-source.ts b/packages/keiko-server/src/grounded-qa-multi-source.ts index 9ad0d0d5d5..c8679ad22b 100644 --- a/packages/keiko-server/src/grounded-qa-multi-source.ts +++ b/packages/keiko-server/src/grounded-qa-multi-source.ts @@ -560,6 +560,7 @@ export function createMultiSourceAnswerer( modelId: string, redactor: Redactor, signal: AbortSignal, + correlationId: string | undefined, ): MultiSourceAnswerer { return async (question, labeledPacks): Promise => { ensureNotCancelled(signal); @@ -568,6 +569,7 @@ export function createMultiSourceAnswerer( modelId, messages: buildMultiSourceGatewayMessages(question, labeledPacks, redactor), stream: false, + logContext: { correlationId }, }, signal, ); diff --git a/packages/keiko-server/src/grounded-qa.test.ts b/packages/keiko-server/src/grounded-qa.test.ts index 1794288925..fb9bebe0af 100644 --- a/packages/keiko-server/src/grounded-qa.test.ts +++ b/packages/keiko-server/src/grounded-qa.test.ts @@ -34,6 +34,7 @@ import { buildGroundedGatewayMessages, groundedPromptInputTokensForCapability, handleGroundedAsk, + mappedGatewayError, modelWindowAwareBudget, modelInputPromptByteLimit, promptByteLength, @@ -52,6 +53,8 @@ import { createInMemoryEvidenceStore, loadEvidence } from "@oscharko-dev/keiko-e import { CancelledError, ContextOverflowError, + RateLimitError, + type GatewayCallRequest, type GatewayConfig, type GatewayRequest, type NormalizedResponse, @@ -70,7 +73,7 @@ import { RepoSearchInvalidQueryError } from "@oscharko-dev/keiko-workspace"; import { createMemoryVault, type MemoryVaultStore } from "@oscharko-dev/keiko-memory-vault"; import type { MemoryId } from "@oscharko-dev/keiko-contracts/memory"; import type { MemoryUserId } from "@oscharko-dev/keiko-contracts"; -import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; import { handleSendDesktopChat } from "./chat-handlers.js"; import { canonicalChatTurnGroundingScopeIdentity, @@ -1720,6 +1723,36 @@ describe("handleGroundedAsk", () => { ); }); + // ADR-0173 D5: the folder single-source answerer must stamp the request's correlation id into + // GatewayCallRequest.logContext so a gateway retry/circuit-breaker line for this call joins the + // same trail as the HTTP request that triggered it. + it("threads the request correlation id into the Model Gateway call's logContext", async () => { + const { chatId, projectPath } = await setupChatWithScope(); + seedScopedRepo(projectPath); + const seenRequests: GatewayRequest[] = []; + const requestCtx: RouteContext = { + ...ctx( + JSON.stringify({ + chatId, + content: GROUNDED_FIXTURE_QUESTION, + modelId: CHAT_MODEL, + }), + ), + correlationId: "cid-grounded-folder-000001", + }; + + const result = await handleGroundedAsk( + requestCtx, + deps(fakeModel("Grounded answer [src/foo.ts:1-3]", seenRequests)), + ); + + expect(result.status, JSON.stringify(result.body)).toBe(200); + expect(seenRequests).toHaveLength(1); + expect( + (firstGatewayRequest(seenRequests) as GatewayCallRequest).logContext?.correlationId, + ).toBe("cid-grounded-folder-000001"); + }); + it("production path includes an explicitly connected single file when the question has no lexical hit", async () => { const project = store.createProject(tmp, "demo"); mkdirSync(join(project.path, "src/pages"), { recursive: true }); @@ -3345,3 +3378,59 @@ describe("handleGroundedAsk", () => { expect(answer.uncertainty[0]?.kind).toBe("budget-clipped"); }); }); + +// ADR-0173 D5 g25/g27 — mirrors the buffered desktop chat path's own symmetry fix +// (chat-handlers.test.ts's "desktopChatErrorResult gateway diagnostic symmetry"): grounded Q&A used +// to map a GatewayError straight to a response with no operator diagnostic at all. +describe("mappedGatewayError diagnostic symmetry", () => { + function diagnosticDeps(diagnostics: ServerDiagnosticSink): UiHandlerDeps { + return { + env: {}, + config: undefined, + redactor: (value: unknown): unknown => value, + diagnostics, + } as unknown as UiHandlerDeps; + } + + it("emits an operator diagnostic for a RateLimitError, keyed to the given correlation id", () => { + const events: ServerDiagnosticRecord[] = []; + const deps = diagnosticDeps({ + record: (record): void => { + events.push(record); + }, + }); + + const result = mappedGatewayError( + new RateLimitError("provider rate limited", 1_500), + deps, + "grounded-correlation-1", + ); + + expect(result?.status).toBe(503); + expect(events).toHaveLength(1); + const [event] = events; + if (event === undefined) throw new Error("expected a diagnostic record"); + expect(event.correlationId).toBe("grounded-correlation-1"); + expect(event.operation).toBe("POST /api/chats/messages/grounded"); + expect(event.source).toBe("grounded.qa"); + expect(event.errorClass).toBe("RateLimitError"); + }); + + it("does not diagnose an intentional cancellation", () => { + const events: ServerDiagnosticRecord[] = []; + const deps = diagnosticDeps({ + record: (record): void => { + events.push(record); + }, + }); + + const result = mappedGatewayError( + new CancelledError("grounded request cancelled"), + deps, + "grounded-correlation-2", + ); + + expect(result?.status).toBe(499); + expect(events).toHaveLength(0); + }); +}); diff --git a/packages/keiko-server/src/grounded-qa.ts b/packages/keiko-server/src/grounded-qa.ts index a5c8dc958c..3338e640d0 100644 --- a/packages/keiko-server/src/grounded-qa.ts +++ b/packages/keiko-server/src/grounded-qa.ts @@ -140,6 +140,7 @@ import { evidenceRetentionDiagnosticObserver, emitServerDiagnostic, } from "./diagnostics-log.js"; +import { emitGatewayErrorDiagnostic } from "./gateway-error-diagnostic.js"; import { buildAnswerCitations as projectAnswerCitations, buildPackCitations, @@ -201,19 +202,39 @@ export function gatewayErrorStatus(error: GatewayError): number { return 502; } -function gatewayErrorResult(error: GatewayError, deps: UiHandlerDeps): RouteResult { +// ADR-0173 D5 g25/g27 — mirrors the buffered desktop chat path (`chat-handlers.ts`): a GatewayError +// mapped straight to an HTTP response used to leave no trace in the operator diagnostic sink, unlike +// the SSE chat path, which already routed the same error class through it. A cancellation is the +// caller's own choice, not a failure, so it is excluded — matching the buffered path's convention +// of never diagnosing an intentional cancel. +function gatewayErrorResult( + error: GatewayError, + deps: UiHandlerDeps, + correlationId: string | undefined, +): RouteResult { if (error instanceof CancelledError) { return { status: 499, body: errorBody(error.code, "Grounded request was cancelled.") }; } + emitGatewayErrorDiagnostic( + deps, + error, + correlationId, + "POST /api/chats/messages/grounded", + "grounded.qa", + ); const status = gatewayErrorStatus(error); const message = redact(error.message, currentRedactionSecrets(deps)); return { status, body: errorBody(error.code, message) }; } -export function mappedGatewayError(error: unknown, deps: UiHandlerDeps): RouteResult | undefined { +export function mappedGatewayError( + error: unknown, + deps: UiHandlerDeps, + correlationId?: string, +): RouteResult | undefined { return ( mappedConversationReadinessError(error) ?? - (error instanceof GatewayError ? gatewayErrorResult(error, deps) : undefined) + (error instanceof GatewayError ? gatewayErrorResult(error, deps, correlationId) : undefined) ); } @@ -864,6 +885,7 @@ function createGatewayAnswerer( redactor: Redactor, signal: AbortSignal, modelInputTokensMax: number | undefined, + correlationId: string | undefined, ): GroundedAnswerer { return { answer: async (question, pack): Promise => { @@ -874,6 +896,7 @@ function createGatewayAnswerer( modelId, messages: buildGroundedGatewayMessages(question, pack, redactor, promptOptions), stream: false, + logContext: { correlationId }, }, signal, ); @@ -905,12 +928,70 @@ function resolveGroundedAnswerModel( return withConversationReadinessAdmission(resolvedModel, modelId, readinessAdmission, deps); } +interface DefaultRunnerContext { + readonly deps: UiHandlerDeps; + readonly modelId: string; + readonly signal: AbortSignal; + readonly contextProfile: UiHandlerDeps["contextProfile"]; + readonly model: ModelPort; + readonly modelInputTokensMax: number | undefined; + readonly entailmentStage: EntailmentStage | undefined; + readonly correlationId: string | undefined; +} + +// Split out of defaultRunner to keep it within the line budget: the actual GroundedRunner closure +// invoked once per exploration input. +function runDefaultGroundedExploration( + runnerCtx: DefaultRunnerContext, + input: OrchestratorInput, +): Promise { + const { deps, modelId, signal, contextProfile, model, modelInputTokensMax, entailmentStage } = + runnerCtx; + const nowMs = Date.now; + const budgetedInput = + input.budget === undefined + ? { ...input, budget: modelWindowAwareBudget(deps, modelId) } + : input; + const contextPackReranker = configuredContextPackRerankerFor(deps, budgetedInput.query, signal); + const semanticLease = configuredRepoSemanticSearchProviderLeaseFor( + deps, + signal, + budgetedInput.workspaceRoot, + ); + return runGroundedExploration(budgetedInput, { + answerer: createGatewayAnswerer( + model, + modelId, + deps.redactor, + signal, + modelInputTokensMax, + runnerCtx.correlationId, + ), + nowMs, + signal, + microIndex: microIndexForGroundedScope(budgetedInput.scope, nowMs), + workspaceIndexForRoot: deps.workspaceIndexForRoot, + ...(contextPackReranker === undefined ? {} : { contextPackReranker }), + ...(semanticLease.provider === undefined + ? {} + : { repoSemanticSearchProvider: semanticLease.provider }), + ...(entailmentStage === undefined ? {} : { entailmentStage }), + // ADR-0055 D1/D5 (PR4-W1): thread the provisioned profile so the diagnostics observer fires + // on the assembled pack. exactOptionalPropertyTypes — omit the key entirely when absent so + // the legacy no-profile path stays byte-identical (observer guard never sees a key). + ...(contextProfile === undefined ? {} : { contextProfile }), + }).finally(() => { + semanticLease.close(); + }); +} + function defaultRunner( deps: UiHandlerDeps, modelId: string, readinessAdmission: ConversationReadinessAdmission, signal: AbortSignal, contextProfile: UiHandlerDeps["contextProfile"], + correlationId: string | undefined, ): GroundedRunner | RouteResult { const model = resolveGroundedAnswerModel(deps, modelId, readinessAdmission); if ("status" in model) return model; @@ -925,37 +1006,18 @@ function defaultRunner( { diagnostics: deps.diagnostics }, signal, ); - return (input: OrchestratorInput): Promise => { - const nowMs = Date.now; - const budgetedInput = - input.budget === undefined - ? { ...input, budget: modelWindowAwareBudget(deps, modelId) } - : input; - const contextPackReranker = configuredContextPackRerankerFor(deps, budgetedInput.query, signal); - const semanticLease = configuredRepoSemanticSearchProviderLeaseFor( - deps, - signal, - budgetedInput.workspaceRoot, - ); - return runGroundedExploration(budgetedInput, { - answerer: createGatewayAnswerer(model, modelId, deps.redactor, signal, modelInputTokensMax), - nowMs, - signal, - microIndex: microIndexForGroundedScope(budgetedInput.scope, nowMs), - workspaceIndexForRoot: deps.workspaceIndexForRoot, - ...(contextPackReranker === undefined ? {} : { contextPackReranker }), - ...(semanticLease.provider === undefined - ? {} - : { repoSemanticSearchProvider: semanticLease.provider }), - ...(entailmentStage === undefined ? {} : { entailmentStage }), - // ADR-0055 D1/D5 (PR4-W1): thread the provisioned profile so the diagnostics observer fires - // on the assembled pack. exactOptionalPropertyTypes — omit the key entirely when absent so - // the legacy no-profile path stays byte-identical (observer guard never sees a key). - ...(contextProfile === undefined ? {} : { contextProfile }), - }).finally(() => { - semanticLease.close(); - }); + const runnerCtx: DefaultRunnerContext = { + deps, + modelId, + signal, + contextProfile, + model, + modelInputTokensMax, + entailmentStage, + correlationId, }; + return (input: OrchestratorInput): Promise => + runDefaultGroundedExploration(runnerCtx, input); } // ─── Citation projection ────────────────────────────────────────────────────── @@ -1053,6 +1115,9 @@ interface AskWorkerCtx { readonly deps: UiHandlerDeps; readonly runner: GroundedRunner; readonly signal: AbortSignal; + // ADR-0173 D5 — carried from PreparedGroundedAsk.correlationId so a GatewayError surfacing from + // the runner (or a late cancellation) reaches its operator diagnostic joined to the request. + readonly correlationId: string | undefined; } interface PreparedGroundedAsk { @@ -1067,6 +1132,10 @@ interface PreparedGroundedAsk { readonly memory?: GroundedMemoryPreparation | undefined; readonly modelId?: string | undefined; readonly readinessAdmission?: ConversationReadinessAdmission | undefined; + // ADR-0173 D5: the request-scoped correlation id (RouteContext.correlationId), carried through + // every `{ ...prepared, ... }` preparation stage so the model call at the bottom of the + // orchestrator pipeline can stamp it into GatewayCallRequest.logContext. + readonly correlationId?: string | undefined; } interface GroundedMemoryPreparation { @@ -1289,7 +1358,7 @@ async function runAsk(workerCtx: AskWorkerCtx): Promise { if (!isValidGroundedPack(output.pack)) { return internalError("Grounded answer context pack failed validation."); } - const cancelResult = ensureRouteNotCancelled(workerCtx.signal, deps); + const cancelResult = ensureRouteNotCancelled(workerCtx.signal, deps, workerCtx.correlationId); if (cancelResult !== undefined) return cancelResult; return finalizeGroundedAnswer(workerCtx, output); } @@ -1368,12 +1437,13 @@ function isRouteResult(value: unknown): value is RouteResult { function ensureRouteNotCancelled( signal: AbortSignal, deps: UiHandlerDeps, + correlationId: string | undefined, ): RouteResult | undefined { try { ensureNotCancelled(signal); return undefined; } catch (error) { - const gatewayResult = mappedGatewayError(error, deps); + const gatewayResult = mappedGatewayError(error, deps, correlationId); if (gatewayResult !== undefined) return gatewayResult; throw error; } @@ -1404,7 +1474,7 @@ async function runGroundedRunner( } const workspaceResult = mappedWorkspaceError(error); if (workspaceResult !== undefined) return workspaceResult; - const gatewayResult = mappedGatewayError(error, workerCtx.deps); + const gatewayResult = mappedGatewayError(error, workerCtx.deps, workerCtx.correlationId); if (gatewayResult !== undefined) return gatewayResult; throw error; } @@ -1428,7 +1498,7 @@ async function prepareGroundedAsk( if (parsed.kind === "err") return parsed.result; const chat = findChatById(deps, parsed.value.chatId); if (chat === undefined) return notFound("Chat not found."); - return { chat, input: parsed.value, signal }; + return { chat, input: parsed.value, signal, correlationId: ctx.correlationId }; } function resolveGroundedRunner( @@ -1437,6 +1507,7 @@ function resolveGroundedRunner( readinessAdmission: ConversationReadinessAdmission, signal: AbortSignal, runner: GroundedRunner | undefined, + correlationId: string | undefined, ): | { readonly modelId: string; @@ -1452,7 +1523,14 @@ function resolveGroundedRunner( }; } const contextProfile = currentContextProfileForModel(deps, modelId); - const builtRunner = defaultRunner(deps, modelId, readinessAdmission, signal, contextProfile); + const builtRunner = defaultRunner( + deps, + modelId, + readinessAdmission, + signal, + contextProfile, + correlationId, + ); if (typeof builtRunner !== "function") return builtRunner; return { modelId, contextProfile, runner: builtRunner }; } @@ -1474,6 +1552,7 @@ function resolveMultiSourceSeam( readinessAdmission: ConversationReadinessAdmission, signal: AbortSignal, override: MultiSourceSeam | undefined, + correlationId: string | undefined, ): MultiSourceSeam | RouteResult { if (override !== undefined) return override; const resolvedModel = deps.modelPortFactory(modelId); @@ -1488,7 +1567,7 @@ function resolveMultiSourceSeam( ); return { retriever: defaultRetriever(signal, deps), - answerer: createMultiSourceAnswerer(model, modelId, deps.redactor, signal), + answerer: createMultiSourceAnswerer(model, modelId, deps.redactor, signal, correlationId), }; } @@ -1510,6 +1589,7 @@ async function dispatchMultiSourceAsk( groundedReadinessAdmission(args), signal, seamOverride, + args.correlationId, ); if ("status" in seam) return seam; return runMultiSourceAsk({ @@ -1604,6 +1684,7 @@ async function dispatchFolderAsk( groundedReadinessAdmission(prepared), signal, runner, + prepared.correlationId, ); if ("status" in resolved) return resolved; return runAsk({ @@ -1620,6 +1701,7 @@ async function dispatchFolderAsk( deps, runner: resolved.runner, signal, + correlationId: prepared.correlationId, }); } @@ -1673,6 +1755,7 @@ async function dispatchHybridAsk( contextProfile: currentContextProfileForModel(deps, modelId), deps, signal, + correlationId: prepared.correlationId, readinessAdmission: groundedReadinessAdmission(prepared), preSkippedFolders: skippedFolders.map((s) => ({ label: s.label, @@ -1734,6 +1817,7 @@ async function dispatchPreparedGroundedAsk( deps, prepared.signal, groundedReadinessAdmission(prepared), + prepared.correlationId, ); } return dispatchHybridAsk(preparedWithCanonicalFolders, deps, skippedFolders, hybrid); diff --git a/packages/keiko-server/src/local-knowledge-grounded-qa.ts b/packages/keiko-server/src/local-knowledge-grounded-qa.ts index cde13f1c83..ef04b5692e 100644 --- a/packages/keiko-server/src/local-knowledge-grounded-qa.ts +++ b/packages/keiko-server/src/local-knowledge-grounded-qa.ts @@ -661,6 +661,7 @@ class StoreBackedAnswerGenerator implements AnswerGenerator { private readonly auditSink: ReturnType, private readonly redactExcerpt: (value: string) => string, private readonly limits: ReturnType, + private readonly correlationId: string | undefined, ) {} public async generate(input: AnswerGeneratorInput): Promise { @@ -675,6 +676,7 @@ class StoreBackedAnswerGenerator implements AnswerGenerator { this.limits, ), stream: false, + logContext: { correlationId: this.correlationId }, }, input.signal ?? new AbortController().signal, ); @@ -752,7 +754,11 @@ function uniqueQueryVariants(variants: readonly string[]): readonly string[] { return out; } -function createBroadQueryTransformer(model: ModelPort, modelId: string): QueryTransformer { +function createBroadQueryTransformer( + model: ModelPort, + modelId: string, + correlationId: string | undefined, +): QueryTransformer { return { rewrite: async ({ query, maxVariants, signal }): Promise => { try { @@ -775,6 +781,7 @@ function createBroadQueryTransformer(model: ModelPort, modelId: string): QueryTr }, ], stream: false, + logContext: { correlationId }, }, queryTransformSignal(signal), ); @@ -2213,6 +2220,7 @@ function createScopedAnswerGenerator( deps: UiHandlerDeps, env: { readonly store: KnowledgeStore }, limits: ReturnType, + correlationId: string | undefined, ): StoreBackedAnswerGenerator { return new StoreBackedAnswerGenerator( model, @@ -2221,32 +2229,45 @@ function createScopedAnswerGenerator( createSqliteAuditSink(env.store), (value: string): string => redactText(deps, value), limits, + correlationId, ); } +// The trailing three positional parameters `runScopedGroundedAnswer` used to take (signal, +// readinessAdmission, correlationId) bundled into one object so the function itself stays under +// the repository's 7-parameter ceiling (Sonar S107) as ADR-0173 D5 g9's correlationId threading +// added an 8th. Grouped together because all three travel together for the lifetime of a single +// grounded-ask attempt, unlike `chat`/`input`/`deps`/`env`/`selected`, which each name a different +// piece of state. +interface ScopedGroundedAnswerContext { + readonly signal: AbortSignal; + readonly readinessAdmission: ConversationReadinessAdmission; + readonly correlationId: string | undefined; +} + async function runScopedGroundedAnswer( chat: Chat, input: AskInput, deps: UiHandlerDeps, env: Pick, "store" | "vectorIndex">, selected: SelectedLocalKnowledgeScope, - signal: AbortSignal, - readinessAdmission: ConversationReadinessAdmission, + context: ScopedGroundedAnswerContext, ): Promise { + const { signal, readinessAdmission, correlationId } = context; const embeddingAdapter = createEmbeddingAdapter(deps); if ("status" in embeddingAdapter) return embeddingAdapter; const modelId = input.modelId ?? chat.selectedModel; const model = resolveModel(deps, modelId, readinessAdmission); if ("status" in model) return model; const limits = currentGroundingLimits(deps); - const generator = createScopedAnswerGenerator(model, modelId, deps, env, limits); + const generator = createScopedAnswerGenerator(model, modelId, deps, env, limits, correlationId); const startedAt = Date.now(); const result = await runGroundedAnswer( { retrieval: { store: env.store, embeddingAdapter, - queryTransformer: createBroadQueryTransformer(model, modelId), + queryTransformer: createBroadQueryTransformer(model, modelId, correlationId), vectorIndex: env.vectorIndex, }, answerGenerator: generator, @@ -2344,6 +2365,7 @@ export async function handleLocalKnowledgeGroundedAsk( deps: UiHandlerDeps, signal: AbortSignal, readinessAdmission?: ConversationReadinessAdmission, + correlationId?: string, ): Promise { const modelId = input.modelId ?? chat.selectedModel; const effectiveAdmission = localKnowledgeReadinessAdmission(deps, modelId, readinessAdmission); @@ -2357,15 +2379,11 @@ export async function handleLocalKnowledgeGroundedAsk( if (signal.aborted) throw new CancelledError("grounded request cancelled"); return stateFailureRoute(chat, input, deps, env, selected, stateFailure); } - const answer = await runScopedGroundedAnswer( - chat, - input, - deps, - env, - selected, + const answer = await runScopedGroundedAnswer(chat, input, deps, env, selected, { signal, - effectiveAdmission, - ); + readinessAdmission: effectiveAdmission, + correlationId, + }); if ("status" in answer) return answer; return { status: 200, body: answer }; } catch (error) { diff --git a/packages/keiko-server/src/memory-audit-handler.test.ts b/packages/keiko-server/src/memory-audit-handler.test.ts index e09ecc811b..bb0e247a3d 100644 --- a/packages/keiko-server/src/memory-audit-handler.test.ts +++ b/packages/keiko-server/src/memory-audit-handler.test.ts @@ -281,6 +281,42 @@ describe("createMemoryAuditHandler", () => { expect(errors).toHaveLength(1); }); + it("reports a bridge persistence failure with the SAME date-bucket runId as its correlationId", () => { + // ADR-0173 D5 / g12: the vault-bridge path (createMemoryAuditHandler) reports through the + // diagnostic sink, not onPersistError, so its failure must carry the SAME runId the append + // targeted rather than a disconnected `randomUUID()`. Before the fix this was a random UUID. + const throwingStore: EvidenceStore = { + put: (): string => { + throw Object.assign(new Error("disk full"), { code: "ENOSPC" }); + }, + get: (): string | undefined => undefined, + list: (): readonly string[] => [], + location: (runId: string): string => runId, + delete: (): void => undefined, + }; + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (entry) => { + records.push(entry); + }, + }; + const handler = createMemoryAuditHandler({ + evidenceStore: throwingStore, + redactString: identityRedact, + now: () => FIXED_NOW, + newEventId: makeIdFactory(), + diagnostics, + }); + + expect(() => { + handler({ kind: "memory:inserted", record: makeRecord({ status: "proposed" }) }); + }).not.toThrow(); + + expect(records).toHaveLength(1); + expect(records[0]?.source).toBe("memory-audit-handler.bridge"); + expect(records[0]?.correlationId).toBe(auditRunIdFor(FIXED_NOW)); + }); + it("preserves a corrupt audit manifest instead of resetting it", () => { const store = createInMemoryEvidenceStore(); const runId = auditRunIdFor(FIXED_NOW); @@ -733,6 +769,10 @@ describe("recordMemoryAudits", () => { expect(records[0]?.operation).toBe("memory.audit.persist"); expect(records[0]?.code).toBe("EACCES"); expect(records[0]?.correlationId).toMatch(/^[A-Za-z0-9._-]{8,128}$/); + // ADR-0173 D5 / g12: the failure's correlationId is the SAME date-bucket runId the append + // itself targeted, not a disconnected `randomUUID()` — an operator can join the failure back + // to the bucket it belongs to. Before the fix this was a random UUID. + expect(records[0]?.correlationId).toBe(auditRunIdFor(FIXED_NOW)); // Body-free: the store's path never enters the record. expect(JSON.stringify(records)).not.toContain("/Users/op/.keiko/evidence"); expect(consoleError).not.toHaveBeenCalled(); diff --git a/packages/keiko-server/src/memory-audit-handler.ts b/packages/keiko-server/src/memory-audit-handler.ts index f861828b09..14677881f3 100644 --- a/packages/keiko-server/src/memory-audit-handler.ts +++ b/packages/keiko-server/src/memory-audit-handler.ts @@ -43,6 +43,7 @@ import { createHash, randomUUID } from "node:crypto"; import type { MemoryAuditEvent, MemoryId, MemoryStatus } from "@oscharko-dev/keiko-contracts"; import type { EvidenceStore } from "@oscharko-dev/keiko-evidence"; import type { MemoryEvent } from "@oscharko-dev/keiko-memory-vault"; +import { isValidCorrelationId } from "./correlation.js"; import { buildDeletedEvent, buildInsertedEvent, @@ -91,16 +92,22 @@ interface AuditPersistFailureContext { function reportAuditPersistFailure( options: AuditPersistFailureContext, source: string, + runId: string, error: unknown, ): void { if (options.onPersistError !== undefined) { options.onPersistError(error); return; } + // Reuses the date-bucket runId the failed append targeted (ADR-0173 D5 / g12, mirroring + // gitDelivery/mutationEvidenceLedger.ts's `evidenceCorrelationId`) instead of a disconnected + // fresh mint, so an operator can join this diagnostic back to the SAME bucket's other audit + // evidence. Re-validated against `isValidCorrelationId` rather than trusted blindly. + const correlationId = isValidCorrelationId(runId) ? runId : randomUUID(); emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "memory.audit.persist", source, error, @@ -171,7 +178,7 @@ function parseExistingEvents(json: string | undefined): PersistedMemoryAuditEven try { const parsed: unknown = JSON.parse(json); if (!Array.isArray(parsed)) { - throw new Error("memory audit manifest has unexpected shape"); + throw new TypeError("memory audit manifest has unexpected shape"); } return parsed as PersistedMemoryAuditEvent[]; } catch (error) { @@ -350,12 +357,13 @@ export function createMemoryAuditHandler(options: MemoryAuditHandlerOptions): Me if (auditEvent === undefined) { return; } + const runId = auditRunIdFor(auditEvent.occurredAt); try { - appendAuditEvents(options.evidenceStore, auditRunIdFor(auditEvent.occurredAt), [ + appendAuditEvents(options.evidenceStore, runId, [ sanitizeAuditEvent(auditEvent, options.redactString), ]); } catch (error) { - reportAuditPersistFailure(options, "memory-audit-handler.bridge", error); + reportAuditPersistFailure(options, "memory-audit-handler.bridge", runId, error); } }; } @@ -484,7 +492,7 @@ export function recordMemoryAudits( if (options.required === true) { throw error; } - reportAuditPersistFailure(options, "memory-audit-handler.direct", error); + reportAuditPersistFailure(options, "memory-audit-handler.direct", runId, error); } } } diff --git a/packages/keiko-server/src/memory-conflict-advisory.test.ts b/packages/keiko-server/src/memory-conflict-advisory.test.ts index f77bbc0650..65877a9e2f 100644 --- a/packages/keiko-server/src/memory-conflict-advisory.test.ts +++ b/packages/keiko-server/src/memory-conflict-advisory.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createDefaultChatCapability, + type GatewayCallRequest, type GatewayConfig, type GatewayRequest, type NormalizedResponse, @@ -250,6 +251,38 @@ describe("enrichReviewItemsWithAdvisory — eligibility carve-out (ADR-0120 D4)" expect(calls).toHaveLength(1); }); + // ADR-0173 D5: this background consolidation job has no live HTTP request in scope, so the job's + // own id is the stable correlation key stamped into the advisory model's GatewayCallRequest.logContext. + it("stamps the job id into the advisory model gateway call's logContext", async () => { + const calls: GatewayRequest[] = []; + const winner = memoryId("mem-winner"); + const loser = memoryId("mem-loser"); + const item: ReviewItem = { + id: "rv-multi-logcontext", + reason: "multi-way-duplicate", + relatedMemoryIds: [winner, loser], + sourceMemoryIds: [winner, loser], + proposedAction: { kind: "merge", winner, losers: [loser] }, + detectedAt: NOW, + }; + const deps = baseDeps( + respondingModel(calls, () => + structuredResponse({ keep: "A", rationale: "Clear duplicate." }), + ), + [], + ); + + await enrichReviewItemsWithAdvisory( + deps, + JOB_ID, + [item], + [record(winner, "Prefers TypeScript."), record(loser, "Prefers TypeScript.")], + ); + + expect(calls).toHaveLength(1); + expect((calls[0] as GatewayCallRequest | undefined)?.logContext?.correlationId).toBe(JOB_ID); + }); + it("is NOT eligible for a potential-conflict pair whose only evidence is a clean negation flip", async () => { const calls: GatewayRequest[] = []; const older = memoryId("mem-old"); diff --git a/packages/keiko-server/src/memory-conflict-advisory.ts b/packages/keiko-server/src/memory-conflict-advisory.ts index 7d0b50859c..2943f2f91c 100644 --- a/packages/keiko-server/src/memory-conflict-advisory.ts +++ b/packages/keiko-server/src/memory-conflict-advisory.ts @@ -233,6 +233,7 @@ async function callAdvisoryModel( modelId: string, prompt: string, responseFormat: ResponseFormat, + jobId: string, ): Promise { const controller = new AbortController(); let timer: ReturnType | undefined; @@ -256,6 +257,7 @@ async function callAdvisoryModel( temperature: 0, topP: 1, responseFormat, + logContext: { correlationId: jobId }, }, controller.signal, ), @@ -406,6 +408,49 @@ function emitAdvisoryPhaseSummary( }); } +interface AdvisoryPhaseSharedContext { + readonly deps: UiHandlerDeps; + readonly jobId: string; + readonly model: ModelPort; + readonly modelId: string; + readonly memoriesById: ReadonlyMap; + readonly policy: CapturePolicyOptions; + readonly startedAt: number; +} + +// Split out of runAdvisoryPhase to keep it within the line budget: one review item's candidate +// gate, cap/budget truncation, and (sequential, ADR-0120 D8) advisory model call. `counts` is +// mutated in place — the caller owns its lifetime across the whole phase. +async function processAdvisoryReviewItem( + ctx: AdvisoryPhaseSharedContext, + item: ReviewItem, + attempted: number, + counts: AdvisoryPhaseCounts, +): Promise<{ readonly item: ReviewItem; readonly attemptedCall: boolean }> { + const candidate = prepareAdvisoryCandidate(item, ctx.memoriesById, ctx.deps.redactor, ctx.policy); + if (candidate === undefined) return { item, attemptedCall: false }; + if (attempted >= MAX_ADVISORY_CALLS_PER_JOB) { + counts.truncatedByCap += 1; + return { item, attemptedCall: false }; + } + if (Date.now() - ctx.startedAt >= ADVISORY_PHASE_BUDGET_MS) { + counts.truncatedByBudget += 1; + return { item, attemptedCall: false }; + } + const call = await callAdvisoryModel( + ctx.model, + ctx.modelId, + candidate.prompt, + advisoryResponseFormat(candidate.labels), + ctx.jobId, + ); + const outcome = advisoryOutcomeFromCall(call, candidate, ctx.deps.redactor, ctx.policy); + return { + item: applyAdvisoryOutcome(item, outcome, ctx.deps, ctx.jobId, counts), + attemptedCall: true, + }; +} + async function runAdvisoryPhase( deps: UiHandlerDeps, jobId: string, @@ -429,35 +474,20 @@ async function runAdvisoryPhase( truncatedByCap: 0, truncatedByBudget: 0, }; - const startedAt = Date.now(); + const sharedCtx: AdvisoryPhaseSharedContext = { + deps, + jobId, + model, + modelId, + memoriesById, + policy, + startedAt: Date.now(), + }; let attempted = 0; for (const item of reviewItems) { - const candidate = prepareAdvisoryCandidate(item, memoriesById, deps.redactor, policy); - if (candidate === undefined) { - enriched.push(item); - continue; - } - if (attempted >= MAX_ADVISORY_CALLS_PER_JOB) { - counts.truncatedByCap += 1; - enriched.push(item); - continue; - } - if (Date.now() - startedAt >= ADVISORY_PHASE_BUDGET_MS) { - counts.truncatedByBudget += 1; - enriched.push(item); - continue; - } - attempted += 1; - // Sequential by design (ADR-0120 D8): bounds concurrency to 1 and keeps the wall-clock - // budget check above accurate between calls, rather than a call-per-item fan-out. - const call = await callAdvisoryModel( - model, - modelId, - candidate.prompt, - advisoryResponseFormat(candidate.labels), - ); - const outcome = advisoryOutcomeFromCall(call, candidate, deps.redactor, policy); - enriched.push(applyAdvisoryOutcome(item, outcome, deps, jobId, counts)); + const result = await processAdvisoryReviewItem(sharedCtx, item, attempted, counts); + enriched.push(result.item); + if (result.attemptedCall) attempted += 1; } emitAdvisoryPhaseSummary(deps, jobId, counts); // Re-check cancellation after the (possibly multi-second) advisory window closes (ADR-0120 diff --git a/packages/keiko-server/src/memory-maintenance-handlers.test.ts b/packages/keiko-server/src/memory-maintenance-handlers.test.ts index c5be638909..3141b00f4a 100644 --- a/packages/keiko-server/src/memory-maintenance-handlers.test.ts +++ b/packages/keiko-server/src/memory-maintenance-handlers.test.ts @@ -30,13 +30,14 @@ import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; const DAY = 864e5; const RETENTION_NOW = Date.parse("2026-08-02T08:00:00.000Z"); -function makeCtx(): RouteContext { +function makeCtx(correlationId?: string): RouteContext { const socket = new Socket(); return { req: {} as RouteContext["req"], res: { socket } as unknown as RouteContext["res"], params: {}, url: new URL("http://127.0.0.1/api/memory/maintenance"), + ...(correlationId === undefined ? {} : { correlationId }), }; } @@ -343,6 +344,59 @@ describe("handleRunMaintenance", () => { expect(JSON.stringify({ result, diagnostics })).not.toContain(raw); }); + it("threads the request's own correlation id into the retention-config-invalid response instead of minting one", () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts) and is already + // in scope in handleRunMaintenance — the failure diagnostic and error body must reuse it, not + // a disconnected randomUUID(). Before the fix handleRunMaintenance discarded ctx entirely + // (bound as `_ctx`), so the response correlationId never matched ctx.correlationId. + const vault = makeVault(); + const diagnostics: ServerDiagnosticRecord[] = []; + const result = handleRunMaintenance( + makeCtx("req-maintenance-thread-01"), + makeDeps({ + memoryVault: vault, + env: { KEIKO_MEMORY_RETENTION_MAX_AGE_DAYS: "not-a-number" }, + diagnostics: { record: (record) => diagnostics.push(record) }, + }), + ); + + expect(result).toMatchObject({ + status: 500, + body: { error: { correlationId: "req-maintenance-thread-01" } }, + }); + expect(diagnostics[0]?.correlationId).toBe("req-maintenance-thread-01"); + }); + + it("threads the request's own correlation id into an autonomy-mode-read failure too, sharing it with the retention read", () => { + // ADR-0173 D5 / g12: resolveMaintenanceAutonomyMode's own default mint only fires when NO id + // is threaded in; the route always threads its one correlationId (from ctx or minted once) so + // an autonomy-mode failure never gets its own disconnected id relative to the SAME call's + // other diagnostics. + const diagnostics: ServerDiagnosticRecord[] = []; + const vault = makeVault(); + const store = createInMemoryUiStore(); + const faultyStore: UiStore = { + ...store, + readMemoryAutonomyPolicy: (): never => { + throw new Error("preference store unavailable"); + }, + }; + handleRunMaintenance( + makeCtx("req-maintenance-autonomy-thread-01"), + makeDeps({ + memoryVault: vault, + store: faultyStore, + diagnostics: { record: (record) => diagnostics.push(record) }, + }), + ); + + expect(diagnostics).toHaveLength(1); + expect(diagnostics[0]?.source).toBe( + "memory-maintenance-handlers.resolveMaintenanceAutonomyMode", + ); + expect(diagnostics[0]?.correlationId).toBe("req-maintenance-autonomy-thread-01"); + }); + it("returns a review item instead of auto-superseding a pairwise correction conflict", () => { const vault = makeVault(); const now = Date.now(); @@ -776,6 +830,37 @@ describe("maybeRunAutoMaintenance (O-V4)", () => { expect(JSON.stringify(diagnostics)).not.toContain("customer content"); }); + it("mints ONE correlation id shared by every diagnostic of a single auto-maintenance pass, not one per failure point", () => { + // ADR-0173 D5 / g12: before the fix, resolveMemoryRetentionPolicy's own catch and this + // function's onFailure catch each minted a disconnected randomUUID() — an operator could not + // tell two diagnostics from the SAME opportunistic pass apart from two diagnostics from two + // different passes. A malformed retention env (read at pass start) AND a vault fault (surfaced + // through onFailure) now both report under the SAME id. + const diagnostics: ServerDiagnosticRecord[] = []; + const faulty = { + ...makeVault(), + listMemoriesAcrossScopes: () => { + throw new Error("disk gone"); + }, + } as MemoryVaultStore; + + maybeRunChatAutoMaintenance( + makeDeps({ + env: { KEIKO_MEMORY_RETENTION_MAX_AGE_DAYS: "not-a-number" }, + diagnostics: { record: (record) => diagnostics.push(record) }, + }), + faulty, + {}, + NOW, + ); + + const sources = diagnostics.map((record) => record.source); + expect(sources).toContain("memory-maintenance-handlers.resolveMemoryRetentionPolicy"); + expect(sources).toContain("chat.memory.maintenance"); + const ids = new Set(diagnostics.map((record) => record.correlationId)); + expect(ids.size).toBe(1); + }); + it("promotes nothing when no autonomy mode is supplied (fail closed to governed-assist)", () => { const vault = makeVault(); insert(vault, { diff --git a/packages/keiko-server/src/memory-maintenance-handlers.ts b/packages/keiko-server/src/memory-maintenance-handlers.ts index 732e250730..d84b8b2aa0 100644 --- a/packages/keiko-server/src/memory-maintenance-handlers.ts +++ b/packages/keiko-server/src/memory-maintenance-handlers.ts @@ -593,14 +593,22 @@ export function memoryMaintenanceAuditSink(deps: UiHandlerDeps): MemoryAuditSink // never widen authority and must never break the caller (the chat path runs this pass), so an // unreadable policy row degrades to the most restrictive posture — no unattended acceptance — and // is reported as a content-free operator diagnostic rather than swallowed. -export function resolveMaintenanceAutonomyMode(deps: UiHandlerDeps): CodingWorkbenchMode { +// +// `correlationId` defaults to a fresh mint ONLY so a caller with no id in scope keeps compiling +// unchanged (ADR-0173 D5 / g12). The route caller (`handleRunMaintenance`) threads its own +// request id; the chat auto-maintenance caller mints ONE id for its whole pass and threads that, +// so a route request or a background pass never fragments across disconnected mints. +export function resolveMaintenanceAutonomyMode( + deps: UiHandlerDeps, + correlationId: string = randomUUID(), +): CodingWorkbenchMode { try { return resolveMemoryMaintenanceAutonomyMode(deps); } catch (error) { emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "memory.maintenance.autonomy-mode", source: "memory-maintenance-handlers.resolveMaintenanceAutonomyMode", error, @@ -611,8 +619,11 @@ export function resolveMaintenanceAutonomyMode(deps: UiHandlerDeps): CodingWorkb } } -function reportRetentionPolicyFailure(deps: UiHandlerDeps, error: unknown): string { - const correlationId = randomUUID(); +function reportRetentionPolicyFailure( + deps: UiHandlerDeps, + error: unknown, + correlationId: string = randomUUID(), +): string { emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ @@ -630,22 +641,26 @@ export type MemoryRetentionPolicyResolution = | { readonly ok: true; readonly policy: MemoryRetentionPolicy | undefined } | { readonly ok: false }; -export function resolveMemoryRetentionPolicy(deps: UiHandlerDeps): MemoryRetentionPolicyResolution { +export function resolveMemoryRetentionPolicy( + deps: UiHandlerDeps, + correlationId?: string, +): MemoryRetentionPolicyResolution { try { return { ok: true, policy: memoryRetentionPolicy(deps.env) }; } catch (error) { - reportRetentionPolicyFailure(deps, error); + reportRetentionPolicyFailure(deps, error, correlationId); return { ok: false }; } } function manualRetentionPolicy( deps: UiHandlerDeps, + correlationId: string, ): MemoryRetentionPolicy | undefined | RouteResult { try { return memoryRetentionPolicy(deps.env); } catch (error) { - const correlationId = reportRetentionPolicyFailure(deps, error); + reportRetentionPolicyFailure(deps, error, correlationId); return { status: 500, body: errorBody( @@ -657,15 +672,19 @@ function manualRetentionPolicy( } } -export function handleRunMaintenance(_ctx: RouteContext, deps: UiHandlerDeps): RouteResult { +export function handleRunMaintenance(ctx: RouteContext, deps: UiHandlerDeps): RouteResult { const vault = resolveVault(deps); if (isRouteResult(vault)) return vault; + // Minted once so a caller who supplied no id (RouteContext.correlationId is optional for test + // literals) still shares ONE id across every diagnostic this single route invocation reports, + // rather than each helper minting its own (ADR-0173 D5 / g12). + const correlationId = ctx.correlationId ?? randomUUID(); try { const multipliers = memorySemanticizationMultipliers(deps.env); - const retentionPolicy = manualRetentionPolicy(deps); + const retentionPolicy = manualRetentionPolicy(deps, correlationId); if (isRouteResult(retentionPolicy)) return retentionPolicy; const counts = runMemoryMaintenance(vault, memoryMaintenanceAuditSink(deps), { - autonomyMode: resolveMaintenanceAutonomyMode(deps), + autonomyMode: resolveMaintenanceAutonomyMode(deps, correlationId), ...(multipliers !== undefined ? { decayHalfLifeMultiplierByType: multipliers } : {}), ...(retentionPolicy !== undefined ? { retentionPolicy } : {}), }); diff --git a/packages/keiko-server/src/memory-salience.test.ts b/packages/keiko-server/src/memory-salience.test.ts index 85fd5bbd1d..f26d9596fe 100644 --- a/packages/keiko-server/src/memory-salience.test.ts +++ b/packages/keiko-server/src/memory-salience.test.ts @@ -8,7 +8,11 @@ import { join } from "node:path"; import { createMemoryVault, type MemoryVaultStore } from "@oscharko-dev/keiko-memory-vault"; import type { NormalizedResponse } from "@oscharko-dev/keiko-contracts"; import type { ModelPort } from "@oscharko-dev/keiko-harness"; -import type { GatewayConfig, GatewayRequest } from "@oscharko-dev/keiko-model-gateway"; +import type { + GatewayCallRequest, + GatewayConfig, + GatewayRequest, +} from "@oscharko-dev/keiko-model-gateway"; import type { ConversationId, MemoryRecord, @@ -317,6 +321,10 @@ describe("captureSalientFromTurn", () => { expect.objectContaining({ errorClass: "SalienceCaptureDropped", message: "voice salience capture skipped: background queue full (32/32)", + // ADR-0173 D5 / g12: this informational diagnostic used to mint its own disconnected + // `randomUUID()` instead of reusing the turn-scoped id the caller already resolved + // (`"assistant-dropped"`, the correlationId argument below). Fails before the fix. + correlationId: "assistant-dropped", }), ); @@ -356,6 +364,75 @@ describe("captureSalientFromTurn", () => { expect(countMemories(vault, ctx)).toBe(3); }); + // ADR-0173 D5 / g12: the capture-summary diagnostic used to mint its own disconnected + // `randomUUID()` instead of the turn-scoped id `captureSalientFromTurn` already resolved, so a + // successful turn's summary line could never be joined to that same turn's other diagnostics. + // Fails before the fix (the summary's correlationId would be an unrelated fresh UUID). + it("carries the turn's correlation id on the capture summary diagnostic", async () => { + const vault = makeVault(); + const diagnostics = { record: vi.fn<(record: ServerDiagnosticRecord) => void>() }; + const deps = makeDeps({ memoryVault: vault, diagnostics }); + + const actions = await captureSalientFromTurn( + deps, + { content: USER_TEXT, memory: { enabled: true } }, + context(), + "gpt-test", + "Sounds like a great project!", + "desktop", + "assistant-summary-turn", + ); + + expect(actions.length).toBeGreaterThan(0); + expect(diagnostics.record).toHaveBeenCalledWith( + expect.objectContaining({ + errorClass: "SalienceCaptureSummary", + correlationId: "assistant-summary-turn", + }), + ); + }); + + // ADR-0173 D5: the same turn-scoped correlation id must also reach the salience model.call's + // GatewayCallRequest.logContext, not only the diagnostic above, so a gateway retry line for this + // extraction joins the turn's trail. + it("stamps the turn's correlation id into the salience model gateway call's logContext", async () => { + const vault = makeVault(); + const seenRequests: GatewayCallRequest[] = []; + const recordingModel: ModelPort = { + call(request): Promise { + seenRequests.push(request); + return Promise.resolve({ + modelId: request.modelId, + content: ATLAS_FACTS, + finishReason: "stop", + toolCalls: [], + structuredOutput: null, + usage: { + requestId: "salience-logcontext-test", + promptTokens: 7, + completionTokens: 3, + latencyMs: 11, + costClass: "high", + }, + }); + }, + }; + const deps = makeDeps({ memoryVault: vault, modelPortFactory: () => recordingModel }); + + await captureSalientFromTurn( + deps, + { content: USER_TEXT, memory: { enabled: true } }, + context(), + "gpt-test", + "Sounds like a great project!", + "desktop", + "assistant-logcontext-turn", + ); + + expect(seenRequests.length).toBeGreaterThan(0); + expect(seenRequests[0]?.logContext?.correlationId).toBe("assistant-logcontext-turn"); + }); + it("keeps a failed model response body out of operator diagnostics", async () => { const bodyMarker = "fixture-salience-provider-body-marker"; const vault = makeVault(); diff --git a/packages/keiko-server/src/memory-salience.ts b/packages/keiko-server/src/memory-salience.ts index a45fac6dd3..d803875a13 100644 --- a/packages/keiko-server/src/memory-salience.ts +++ b/packages/keiko-server/src/memory-salience.ts @@ -170,6 +170,7 @@ function buildSalienceContext(context: ConversationMemoryRuntimeContext): Captur function buildCallModel( deps: UiHandlerDeps, modelId: string, + correlationId: string, ): NonNullable | null { const model = deps.modelPortFactory(modelId); if (model === undefined) { @@ -185,6 +186,7 @@ function buildCallModel( stream: false, ...(responseFormat !== undefined ? { responseFormat } : {}), ...(seed !== undefined ? { seed } : {}), + logContext: { correlationId }, }, new AbortController().signal, ); @@ -218,14 +220,19 @@ function salienceSeedFor(deps: UiHandlerDeps, modelId: string): number | undefin // redaction-safe server diagnostic sink (diagnostics-log.ts) instead of console.* directly, mirroring // the emitAdvisoryPhaseSummary pattern in memory-conflict-advisory.ts. errorClass doubles as a // machine-readable event-kind tag for non-error informational records (e.g. a capture summary). +// `correlationId` is always the turn-scoped id `captureSalientFromTurn` already resolved (or its +// own fallback default) — never minted fresh here — so an informational salience diagnostic joins +// the SAME trail as that turn's eventual failure diagnostic (`emitSalienceFailureDiagnostic`) +// instead of reporting under a disconnected, unrelated id. function emitSalienceDiagnostic( deps: UiHandlerDeps, + correlationId: string, source: string, errorClass: string, message: string, ): void { emitServerDiagnostic(deps.diagnostics, { - correlationId: randomUUID(), + correlationId, timestamp: new Date().toISOString(), operation: "memory.salience", source, @@ -238,6 +245,7 @@ function logSalienceDiagnostic( diagnostic: SalienceDiagnostic, deps: UiHandlerDeps, modelId: string, + correlationId: string, ): void { const responseFormatEnabled = salienceResponseFormatFor(deps, modelId) !== undefined; const detail = @@ -247,6 +255,7 @@ function logSalienceDiagnostic( // Safe diagnostic: model id, response-format bit, and counts only; never user text or model text. emitSalienceDiagnostic( deps, + correlationId, "memory-salience.logSalienceDiagnostic", "SalienceExtractionDiagnostic", `model=${modelId} responseFormat=${String(responseFormatEnabled)} kind=${diagnostic.kind} ${detail}`, @@ -470,9 +479,14 @@ function logSalienceCaptureFailure( ); } -function logSalienceCaptureDropped(surface: SalienceCaptureSurface, deps: UiHandlerDeps): void { +function logSalienceCaptureDropped( + surface: SalienceCaptureSurface, + deps: UiHandlerDeps, + correlationId: string, +): void { emitSalienceDiagnostic( deps, + correlationId, "memory-salience.scheduleMemorySalienceCapture", "SalienceCaptureDropped", `${surface} salience capture skipped: background queue full (${String( @@ -494,7 +508,7 @@ export function scheduleMemorySalienceCapture( return; } if (pendingSalienceCaptures >= MAX_PENDING_SALIENCE_CAPTURES) { - logSalienceCaptureDropped(surface, deps); + logSalienceCaptureDropped(surface, deps, correlationId); return; } pendingSalienceCaptures += 1; @@ -522,17 +536,29 @@ type TurnSalienceExtraction = | { readonly kind: "refused"; readonly reason: RejectionReason } | { readonly kind: "outcomes"; readonly outcomes: readonly CaptureOutcome[] }; +// The per-turn scalars `extractTurnSalienceOutcomes` and `runSalienceCapture` both thread through +// unchanged, bundled into one parameter so neither function's positional-argument count crosses +// the repository's 7-argument ceiling (Sonar S107) now that g9's correlationId threading added an +// 8th to each. `SalienceCaptureInputs` extends this with the one field `runSalienceCapture` alone +// needs (`surface`), rather than widening this shape for a field `extractTurnSalienceOutcomes` +// never reads. +interface TurnSalienceInputs { + readonly modelId: string; + readonly assistantText: string; + readonly correlationId: string; +} + async function extractTurnSalienceOutcomes( deps: UiHandlerDeps, vault: MemoryVaultStore, request: SalienceTurnRequest, context: ConversationMemoryRuntimeContext, captureContext: CaptureContext, - modelId: string, - assistantText: string, + inputs: TurnSalienceInputs, ): Promise { + const { modelId, assistantText, correlationId } = inputs; const salienceModelId = configuredSalienceModelId(deps, modelId); - const callModelMessages = buildCallModel(deps, salienceModelId); + const callModelMessages = buildCallModel(deps, salienceModelId, correlationId); if (callModelMessages === null) return { kind: "unavailable" }; const policy = memoryCapturePolicyForDeps(deps); const refusalReason = memoryTextSecretEgressRejectionReason(request.content, policy); @@ -556,7 +582,7 @@ async function extractTurnSalienceOutcomes( newMemoryId: captureContext.newMemoryId, newProposalId: captureContext.newProposalId, onDiagnostic: (diagnostic) => { - logSalienceDiagnostic(diagnostic, deps, salienceModelId); + logSalienceDiagnostic(diagnostic, deps, salienceModelId, correlationId); }, }, ); @@ -599,9 +625,11 @@ function logSalienceCaptureSummary( mode: CodingWorkbenchMode, summary: SalienceCaptureSummary, deps: UiHandlerDeps, + correlationId: string, ): void { emitSalienceDiagnostic( deps, + correlationId, "memory-salience.captureSalientFromTurn", "SalienceCaptureSummary", `mode=${mode} proposed=${String(summary.proposed)} accepted=${String(summary.accepted)} ` + @@ -655,6 +683,42 @@ function activeSalienceVault( return request.memory?.enabled === true ? deps.memoryVault : undefined; } +interface SalienceCaptureInputs extends TurnSalienceInputs { + readonly surface: SalienceCaptureSurface; +} + +// The turn-scoped work `captureSalientFromTurn` runs inside its own try/catch. Split out so that +// function's own body stays a thin boundary (resolve the vault, catch, report) — everything that +// can throw lives here instead. +async function runSalienceCapture( + deps: UiHandlerDeps, + vault: MemoryVaultStore, + request: SalienceTurnRequest, + context: ConversationMemoryRuntimeContext, + inputs: SalienceCaptureInputs, +): Promise { + const { surface, correlationId } = inputs; + const mode = resolveMemoryCaptureAutonomyMode(deps, request.memory?.mode); + const captureContext = buildSalienceContext(context); + const extraction = await extractTurnSalienceOutcomes( + deps, + vault, + request, + context, + captureContext, + inputs, + ); + if (extraction.kind === "unavailable") return []; + if (extraction.kind === "refused") { + recordTurnCaptureRefusal(deps, mode, surface, context, captureContext.nowMs, extraction.reason); + return []; + } + const { outcomes } = extraction; + const { actions, summary } = await persistSalienceActions(deps, vault, outcomes, mode, surface); + if (outcomes.length > 0) logSalienceCaptureSummary(mode, summary, deps, correlationId); + return actions; +} + // Captures salient memories from a completed chat turn. Never throws — any failure (model error, // vault error, malformed output) yields [] so the chat response is unaffected. Direct callers that // do not own a committed message id receive one capture-scoped fallback correlation id; post-commit @@ -671,33 +735,12 @@ export async function captureSalientFromTurn( const vault = activeSalienceVault(deps, request); if (vault === undefined) return []; try { - const mode = resolveMemoryCaptureAutonomyMode(deps, request.memory?.mode); - const captureContext = buildSalienceContext(context); - const extraction = await extractTurnSalienceOutcomes( - deps, - vault, - request, - context, - captureContext, + return await runSalienceCapture(deps, vault, request, context, { modelId, assistantText, - ); - if (extraction.kind === "unavailable") return []; - if (extraction.kind === "refused") { - recordTurnCaptureRefusal( - deps, - mode, - surface, - context, - captureContext.nowMs, - extraction.reason, - ); - return []; - } - const { outcomes } = extraction; - const { actions, summary } = await persistSalienceActions(deps, vault, outcomes, mode, surface); - if (outcomes.length > 0) logSalienceCaptureSummary(mode, summary, deps); - return actions; + surface, + correlationId, + }); } catch (error) { // Boundary: salience must never break the chat path. Log and continue. emitSalienceFailureDiagnostic( diff --git a/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotAdapter.test.ts b/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotAdapter.test.ts index e699622d47..39499e440f 100644 --- a/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotAdapter.test.ts +++ b/packages/keiko-server/src/qualityIntelligence/__tests__/figmaSnapshotAdapter.test.ts @@ -11,6 +11,7 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parseGatewayConfig } from "@oscharko-dev/keiko-model-gateway"; import type { + GatewayCallRequest, GatewayRequest, ModelCapability, NormalizedResponse, @@ -431,6 +432,47 @@ describe("makeFigmaVisionHintProvider", () => { } }); + // ADR-0173 D5: this vision pass has no live HTTP request in scope (a background Figma snapshot + // run), so the snapshot run id is the natural correlation key stamped into the vision model's + // GatewayCallRequest.logContext. + it("stamps the snapshot run id into the vision model gateway call's logContext", async () => { + const dir = mkdtempSync(join(tmpdir(), "qi-figma-adapter-vision-logcontext-")); + const seenRequests: GatewayRequest[] = []; + try { + const { loaded, screen } = recordVisionSnapshot(dir); + const port: ModelPort = { + call: (request) => { + seenRequests.push(request); + return Promise.resolve(normalizedResponse(JSON.stringify({ hints: [] }), "vision-low")); + }, + }; + const deps = depsWith({ + config: configWith([ + capability("vision-low", { supportsImageInput: true, supportsResponseFormat: true }), + ]), + configPresent: true, + evidenceDir: dir, + modelPortFactory: () => port, + }); + + const provider = makeFigmaVisionHintProvider(deps); + await provider({ + snapshotRunId: loaded.runId, + screenId: screen.screenId, + image: screen.image, + imageRelativePath: screen.image.relativePath, + baselineText: "Screen: Login [s1]", + }); + + expect(seenRequests).toHaveLength(1); + expect((seenRequests[0] as GatewayCallRequest | undefined)?.logContext?.correlationId).toBe( + loaded.runId, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("omits strict response format when the image model lacks structured output", async () => { const dir = mkdtempSync(join(tmpdir(), "qi-figma-adapter-vision-tolerant-")); const seenRequests: GatewayRequest[] = []; diff --git a/packages/keiko-server/src/qualityIntelligence/__tests__/generationPort.test.ts b/packages/keiko-server/src/qualityIntelligence/__tests__/generationPort.test.ts index 5c8dba1665..eda2b626ae 100644 --- a/packages/keiko-server/src/qualityIntelligence/__tests__/generationPort.test.ts +++ b/packages/keiko-server/src/qualityIntelligence/__tests__/generationPort.test.ts @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { + GatewayCallRequest, GatewayRequest, ModelCapability, NormalizedResponse, @@ -846,6 +847,18 @@ describe("createQiGenerationPort.generate — determinism-first parameters", () expect(result.modelParameters?.responseFormatEnforced).toBe(false); }); + // ADR-0173 D5: the run id supplied to createQiGenerationPort must reach the model.call so a + // gateway retry/circuit-breaker line for this generation stage joins the run's other lines. + it("stamps the supplied correlation id into the GatewayCallRequest.logContext", async () => { + const { deps, calls } = depsFor("chat-model-1"); + const port = createQiGenerationPort(deps, "chat-model-1", "cid-qi-generation-000001"); + await port.generate(args()); + expect(calls).toHaveLength(1); + expect((calls[0]?.request as GatewayCallRequest | undefined)?.logContext?.correlationId).toBe( + "cid-qi-generation-000001", + ); + }); + it("does not send a seed when the model does not advertise seeding support", async () => { const { deps, calls } = depsFor("unseeded-model"); const port = createPort(deps, { diff --git a/packages/keiko-server/src/qualityIntelligence/__tests__/judgePort.test.ts b/packages/keiko-server/src/qualityIntelligence/__tests__/judgePort.test.ts index dfb1966225..ac9fd87b4f 100644 --- a/packages/keiko-server/src/qualityIntelligence/__tests__/judgePort.test.ts +++ b/packages/keiko-server/src/qualityIntelligence/__tests__/judgePort.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { + GatewayCallRequest, GatewayRequest, ModelCapability, NormalizedResponse, @@ -683,6 +684,24 @@ describe("createQiJudgePort.judge — gateway call", () => { expect(verdict.gatewayCallCount).toBe(1); }); + // ADR-0173 D5: the caller's correlation id (a run id or an HTTP request id) must reach the + // judge's model.call so a gateway retry/circuit-breaker line for this judge stage joins the + // same trail as the run/request that triggered it. + it("stamps the supplied correlation id into the GatewayCallRequest.logContext", async () => { + const { deps, calls } = depsFor("chat-model-1", VALID_VERDICT_JSON); + const port = createQiJudgePort(deps, "chat-model-1", { + correlationId: "cid-qi-judge-000001", + }); + await port.judge({ + candidateText: "candidate text", + sourceContext: [{ atomId: "atom-1", text: "REQ-1" }], + }); + expect(calls).toHaveLength(1); + expect((calls[0]?.request as GatewayCallRequest | undefined)?.logContext?.correlationId).toBe( + "cid-qi-judge-000001", + ); + }); + it("uses stream: false in the gateway request", async () => { const { deps, calls } = depsFor("chat-model-1", VALID_VERDICT_JSON); const port = createQiJudgePort(deps, "chat-model-1"); diff --git a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotAdapter.ts b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotAdapter.ts index 4ae2b45034..d2cd8be5c2 100644 --- a/packages/keiko-server/src/qualityIntelligence/figmaSnapshotAdapter.ts +++ b/packages/keiko-server/src/qualityIntelligence/figmaSnapshotAdapter.ts @@ -22,7 +22,7 @@ import { type FigmaSnapshotImageRef, type FigmaSnapshotRecord, } from "@oscharko-dev/keiko-evidence"; -import type { GatewayRequest } from "@oscharko-dev/keiko-model-gateway"; +import type { GatewayCallRequest } from "@oscharko-dev/keiko-model-gateway"; import type { UiHandlerDeps } from "../deps.js"; import { resolveQiMultimodalSelection } from "./modelSelection.js"; @@ -228,7 +228,7 @@ function buildVisionRequest( dataUrl: string, baselineText: string, structuredOutput: boolean, -): GatewayRequest { +): GatewayCallRequest { const userText = visionUserText(request, baselineText); return { modelId, @@ -256,6 +256,9 @@ function buildVisionRequest( responseFormat: VISION_RESPONSE_FORMAT, } : {}), + // The Figma snapshot run id is the natural background-job correlation key here: this vision + // pass has no live HTTP request in scope (ADR-0173 D5, background-run case). + logContext: { correlationId: request.snapshotRunId }, }; } diff --git a/packages/keiko-server/src/qualityIntelligence/generationPort.ts b/packages/keiko-server/src/qualityIntelligence/generationPort.ts index a975937690..26e4d0db37 100644 --- a/packages/keiko-server/src/qualityIntelligence/generationPort.ts +++ b/packages/keiko-server/src/qualityIntelligence/generationPort.ts @@ -13,7 +13,7 @@ import { findConfiguredCapability, QualityIntelligenceSafeErrorException, type ChatMessage, - type GatewayRequest, + type GatewayCallRequest, type ModelCapability, } from "@oscharko-dev/keiko-model-gateway"; import { @@ -285,7 +285,8 @@ function buildGenerationRequest( useSeed: boolean, requestedSeed: number | undefined, signal: AbortSignal, -): GatewayRequest { + correlationId: string | undefined, +): GatewayCallRequest { return { modelId, messages, @@ -304,6 +305,7 @@ function buildGenerationRequest( }, } : {}), + logContext: { correlationId }, }; } @@ -336,6 +338,7 @@ function abortErrorForGeneration(reasonKind: "timeout" | "external" | "none"): E // eslint-disable-next-line max-lines-per-function function createModelGenerationPort( resolved: ResolvedGenerationModel, + correlationId: string | undefined, ): QualityIntelligenceGenerationPort { const { model, modelId, useResponseFormat, useSeed, requestedSeed } = resolved; return { @@ -354,6 +357,7 @@ function createModelGenerationPort( useSeed, requestedSeed, cancellation.signal, + correlationId, ); let removeAbortListener = (): void => { /* not attached yet */ @@ -393,10 +397,11 @@ function createModelGenerationPort( export function createQiGenerationPort( deps: UiHandlerDeps, target: QiGenerationTarget, + correlationId?: string, ): QualityIntelligenceGenerationPort { const normalized = normalizeTarget(target); if (normalized.kind === "baseline") { return createBaselineGenerationPort(); } - return createModelGenerationPort(resolveGenerationModel(deps, normalized)); + return createModelGenerationPort(resolveGenerationModel(deps, normalized), correlationId); } diff --git a/packages/keiko-server/src/qualityIntelligence/handoffRoutes.ts b/packages/keiko-server/src/qualityIntelligence/handoffRoutes.ts index ec9c3ffde3..e7e03c6cef 100644 --- a/packages/keiko-server/src/qualityIntelligence/handoffRoutes.ts +++ b/packages/keiko-server/src/qualityIntelligence/handoffRoutes.ts @@ -317,7 +317,7 @@ const startHandoffRun = (deps: UiHandlerDeps, roots: readonly string[]): string const runPromise = currentGatewayConfig(deps) === undefined ? execute() - : buildQiModelRoutingForRun(deps, {}).then((modelRouting) => execute(modelRouting)); + : buildQiModelRoutingForRun(deps, {}, runId).then((modelRouting) => execute(modelRouting)); void runPromise .then((summary) => { qiRunRegistry.complete(runId, summary.status); diff --git a/packages/keiko-server/src/qualityIntelligence/judgePort.ts b/packages/keiko-server/src/qualityIntelligence/judgePort.ts index 0d995d1139..6074e5ffb0 100644 --- a/packages/keiko-server/src/qualityIntelligence/judgePort.ts +++ b/packages/keiko-server/src/qualityIntelligence/judgePort.ts @@ -14,6 +14,7 @@ import { findConfiguredCapability, QualityIntelligenceSafeErrorException, type ChatMessage, + type GatewayCallRequest, type GatewayRequest, type ModelCapability, } from "@oscharko-dev/keiko-model-gateway"; @@ -217,6 +218,9 @@ const JUDGE_TASK_PROFILE = MgQI.getQualityIntelligenceTaskProfile("qi:judge-logi export interface QiJudgePortOptions { readonly requestedSeed?: number | undefined; + // The request/run correlation id (ADR-0173 D5), stamped into every judge model call's + // GatewayCallRequest.logContext so a judge-stage gateway line joins the run's other lines. + readonly correlationId?: string | undefined; } function isRubricDimensionName(value: string): value is TestQualityDimensionName { @@ -469,7 +473,7 @@ export function createQiJudgePort( }; } const cancellation = MgQI.composeCancellationSignal(JUDGE_TASK_PROFILE.timeoutMsHint, signal); - const request: GatewayRequest = { + const request: GatewayCallRequest = { modelId, messages, stream: false, @@ -477,6 +481,7 @@ export function createQiJudgePort( temperature: 0, ...(useSeed ? { seed: requestedSeed } : {}), responseFormat: buildQiJudgeResponseFormat(), + logContext: { correlationId: options.correlationId }, }; let removeAbortListener = (): void => { /* not attached yet */ diff --git a/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts b/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts index e41673e3be..10ee5519a1 100644 --- a/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts +++ b/packages/keiko-server/src/qualityIntelligence/modelPolicyRoutes.ts @@ -18,7 +18,7 @@ import { listConfiguredCapabilities, findConfiguredCapability, } from "@oscharko-dev/keiko-model-gateway"; -import type { GatewayRequest, ModelCapability } from "@oscharko-dev/keiko-model-gateway"; +import type { GatewayCallRequest, ModelCapability } from "@oscharko-dev/keiko-model-gateway"; import type { QualityIntelligenceModelPolicy, QualityIntelligenceModelPolicyPreflightResponse, @@ -324,9 +324,10 @@ function requestForPreflight( stage: "generate" | "judge", modelId: string, _capability: ModelCapability, -): GatewayRequest { + correlationId: string | undefined, +): GatewayCallRequest { if (stage === "judge") { - return buildQiJudgePreflightRequest(modelId); + return { ...buildQiJudgePreflightRequest(modelId), logContext: { correlationId } }; } return { modelId, @@ -340,37 +341,38 @@ function requestForPreflight( content: "Quality Intelligence preflight.", }, ], + logContext: { correlationId }, }; } -async function preflightStage( +function unavailablePreflightResult( + stage: "generate" | "judge", + modelId?: string, +): QualityIntelligenceModelPreflightStageResult { + return { + stage, + ...(modelId === undefined ? {} : { modelId }), + status: "unavailable", + category: "unavailable", + message: preflightMessage("unavailable"), + }; +} + +// Split out of preflightStage to keep it within the line budget: the actual gateway round trip +// plus its schema/transport failure classification. +async function runPreflightGatewayCall( deps: UiHandlerDeps, stage: "generate" | "judge", - modelId: string | undefined, + modelId: string, + capability: ModelCapability, + correlationId: string | undefined, ): Promise { - const config = currentGatewayConfig(deps); - if (modelId === undefined || config === undefined) { - return { - stage, - status: "unavailable", - category: "unavailable", - message: preflightMessage("unavailable"), - }; - } - const capability = findConfiguredCapability(config, modelId); - if (capability?.kind !== "chat") { - return { - stage, - modelId, - status: "unavailable", - category: "unavailable", - message: preflightMessage("unavailable"), - }; - } try { const gateway = currentGateway(deps); if (gateway === undefined) throw new TypeError("Model gateway is unavailable."); - const response = await gateway.chat(requestForPreflight(stage, modelId, capability)); + const response = await gateway.chat( + requestForPreflight(stage, modelId, capability, correlationId), + ); if (stage === "judge" && tryParseJudgeVerdict(response.content) === null) { return { stage, @@ -393,6 +395,23 @@ async function preflightStage( } } +async function preflightStage( + deps: UiHandlerDeps, + stage: "generate" | "judge", + modelId: string | undefined, + correlationId: string | undefined, +): Promise { + const config = currentGatewayConfig(deps); + if (modelId === undefined || config === undefined) { + return unavailablePreflightResult(stage); + } + const capability = findConfiguredCapability(config, modelId); + if (capability?.kind !== "chat") { + return unavailablePreflightResult(stage, modelId); + } + return runPreflightGatewayCall(deps, stage, modelId, capability, correlationId); +} + function preflightSummaryStatus( generation: QualityIntelligenceModelPreflightStageResult, judge: QualityIntelligenceModelPreflightStageResult | undefined, @@ -417,6 +436,7 @@ function summarizePreflight( export async function buildQiModelRouting( deps: UiHandlerDeps, request: Pick, + correlationId?: string, ): Promise { const requested = policyForRequest(deps, request); const resolution = resolveQiModelPolicy(deps, { ...request, modelPolicy: requested }); @@ -426,11 +446,16 @@ export async function buildQiModelRouting( "The selected Quality Intelligence model policy is invalid.", ); } - const generation = await preflightStage(deps, "generate", resolution.resolved.testDesignModelId); + const generation = await preflightStage( + deps, + "generate", + resolution.resolved.testDesignModelId, + correlationId, + ); const judge = resolution.resolved.judgeModelId === undefined - ? await preflightStage(deps, "judge", undefined) - : await preflightStage(deps, "judge", resolution.resolved.judgeModelId); + ? await preflightStage(deps, "judge", undefined, correlationId) + : await preflightStage(deps, "judge", resolution.resolved.judgeModelId, correlationId); return { policyVersion: 1, requested, @@ -466,8 +491,9 @@ function judgePreflightFailureReason(routing: QualityIntelligenceModelRouting): export async function buildQiModelRoutingForRun( deps: UiHandlerDeps, request: Pick, + correlationId?: string, ): Promise { - const routing = await buildQiModelRouting(deps, request); + const routing = await buildQiModelRouting(deps, request, correlationId); const generation = routing.preflight.generation; if (generation?.status !== "passed" && routing.resolved.testDesignModelId !== undefined) { throw new QiModelPolicyError( @@ -516,7 +542,7 @@ export async function handlePutQiModelPolicy( ); } try { - await buildQiModelRoutingForRun(deps, { modelPolicy: policy }); + await buildQiModelRoutingForRun(deps, { modelPolicy: policy }, ctx.correlationId); } catch (error) { if (error instanceof QiModelPolicyError) { return errorResult(400, error.code, error.message); @@ -558,10 +584,14 @@ export async function handlePreflightQiModelPolicy( ); } try { - const modelRouting = await buildQiModelRouting(deps, { - ...(modelPolicy !== undefined ? { modelPolicy } : {}), - ...(typeof parsed.modelId === "string" ? { modelId: parsed.modelId } : {}), - }); + const modelRouting = await buildQiModelRouting( + deps, + { + ...(modelPolicy !== undefined ? { modelPolicy } : {}), + ...(typeof parsed.modelId === "string" ? { modelId: parsed.modelId } : {}), + }, + ctx.correlationId, + ); const body: QualityIntelligenceModelPolicyPreflightResponse = { modelRouting }; return { status: 200, body }; } catch (error) { diff --git a/packages/keiko-server/src/qualityIntelligence/reCheckRoutes.ts b/packages/keiko-server/src/qualityIntelligence/reCheckRoutes.ts index a2bca527e5..c08fe172e4 100644 --- a/packages/keiko-server/src/qualityIntelligence/reCheckRoutes.ts +++ b/packages/keiko-server/src/qualityIntelligence/reCheckRoutes.ts @@ -313,8 +313,9 @@ async function parseSources(req: IncomingMessage): Promise function buildJudgePortIfAvailable( deps: UiHandlerDeps, modelId: string, + correlationId: string, ): ReturnType | undefined { - const outcome = tryCreateQiJudgePort(deps, modelId); + const outcome = tryCreateQiJudgePort(deps, modelId, { correlationId }); return outcome.available ? outcome.port : undefined; } @@ -1056,6 +1057,7 @@ function regenWorkflowDeps( evidenceStore: ReturnType, capture: (cands: readonly QiTestCaseCandidate[], generatedAt: string) => void, signal: AbortSignal, + newRunId: string, ): QualityIntelligenceModelRoutedTestDesignDeps { return { sink: { emit: () => undefined }, @@ -1066,7 +1068,7 @@ function regenWorkflowDeps( capture(cands, generatedAt); }, }, - generate: createQiGenerationPort(deps, target), + generate: createQiGenerationPort(deps, target, newRunId), // The regenerate-stale judge deliberately shares the auto-selected generation model id rather than // resolving an independent qi:judge-logic model the way the initial run does (runExecution.ts). // This is safe because the regen target comes from resolveQiTestDesignSelection(deps) with NO @@ -1077,7 +1079,9 @@ function regenWorkflowDeps( // asymmetry — an explicitly requested chat-only generation model paired with a separate // structured-output judge — cannot arise here because the regen path never carries an explicit // generation-model request. - ...(target.kind === "model" ? { judge: buildJudgePortIfAvailable(deps, target.modelId) } : {}), + ...(target.kind === "model" + ? { judge: buildJudgePortIfAvailable(deps, target.modelId, newRunId) } + : {}), }; } @@ -1101,6 +1105,7 @@ async function executeScopedWorkflow(args: { readonly atomsToRegenerate: readonly QualityIntelligenceIngestedAtom[]; readonly profile: PolicyProfile; readonly signal: AbortSignal; + readonly newRunId: string; }): Promise { const { deps, @@ -1112,6 +1117,7 @@ async function executeScopedWorkflow(args: { atomsToRegenerate, profile, signal, + newRunId, } = args; try { const summary = await runQualityIntelligenceModelRoutedTestDesign( @@ -1122,7 +1128,7 @@ async function executeScopedWorkflow(args: { provenanceRefs: ingestion.provenanceRefs, profile, }, - regenWorkflowDeps(deps, target, evidenceStore, capture, signal), + regenWorkflowDeps(deps, target, evidenceStore, capture, signal, newRunId), ); return summary.status === "succeeded" ? null @@ -1183,6 +1189,7 @@ async function runScopedEphemeral(args: { atomsToRegenerate, profile, signal, + newRunId, }); if (failure !== null) return { ok: false, result: failure }; return finalizeScopedWorkflow(evidenceStore, newRunId, generatedCandidates, generatedAt); diff --git a/packages/keiko-server/src/qualityIntelligence/runExecution.ts b/packages/keiko-server/src/qualityIntelligence/runExecution.ts index 4caaff8c8f..80047dfd66 100644 --- a/packages/keiko-server/src/qualityIntelligence/runExecution.ts +++ b/packages/keiko-server/src/qualityIntelligence/runExecution.ts @@ -112,6 +112,7 @@ function resolveExecutionStrategy( deps: UiHandlerDeps, request: QualityIntelligenceStartRunRequest, modelRouting: QualityIntelligenceModelRouting, + runId: string, ): ResolvedExecutionStrategy { const modelId = modelRouting.resolved.testDesignModelId; const config = currentGatewayConfig(deps); @@ -125,7 +126,7 @@ function resolveExecutionStrategy( // attribution contract and persist `seedUsed: null` when the model cannot apply the seed. if (modelId === undefined) { return { - generate: createQiGenerationPort(deps, { kind: "baseline" }), + generate: createQiGenerationPort(deps, { kind: "baseline" }, runId), }; } if ( @@ -136,16 +137,20 @@ function resolveExecutionStrategy( }) ) { return { - generate: createQiGenerationPort(deps, { kind: "baseline" }), + generate: createQiGenerationPort(deps, { kind: "baseline" }, runId), }; } return { modelId, - generate: createQiGenerationPort(deps, { - kind: "model", - modelId, - requestedSeed: request.seed, - }), + generate: createQiGenerationPort( + deps, + { + kind: "model", + modelId, + requestedSeed: request.seed, + }, + runId, + ), }; } @@ -281,11 +286,12 @@ async function runResolvedQi( const { deps, runId, request } = input; const requestedRouting = buildExecutionRouting(input); const { ingestion, gatewayCallCount } = await ingestForRun(input, capsuleResolver); - const { modelId, generate } = resolveExecutionStrategy(deps, request, requestedRouting); + const { modelId, generate } = resolveExecutionStrategy(deps, request, requestedRouting, runId); const resolvedJudge = resolveJudgeForModelRun( deps, requestedRouting.resolved.judgeModelId, request.seed, + runId, ); // The routing the run actually executes under carries the judge degradation, so `onAccepted`, // the persisted manifest, and the terminal `done` frame all report the same classified failure. @@ -354,9 +360,10 @@ function resolveJudgeForModelRun( deps: UiHandlerDeps, judgeModelId: string | undefined, requestedSeed: number | undefined, + correlationId: string, ): ResolvedJudge { if (judgeModelId === undefined) return {}; - const outcome = tryCreateQiJudgePort(deps, judgeModelId, { requestedSeed }); + const outcome = tryCreateQiJudgePort(deps, judgeModelId, { requestedSeed, correlationId }); return outcome.available ? { judge: outcome.port } : { stageFailureReason: outcome.reasonSummary }; diff --git a/packages/keiko-server/src/qualityIntelligence/runRoutes.ts b/packages/keiko-server/src/qualityIntelligence/runRoutes.ts index 2684eee50d..325c87f7a0 100644 --- a/packages/keiko-server/src/qualityIntelligence/runRoutes.ts +++ b/packages/keiko-server/src/qualityIntelligence/runRoutes.ts @@ -616,7 +616,7 @@ export async function handleStartQiRun( let modelRouting: QualityIntelligenceModelRouting; try { - modelRouting = await buildQiModelRoutingForRun(deps, parsed.request); + modelRouting = await buildQiModelRoutingForRun(deps, parsed.request, runId); } catch (error) { qiRunRegistry.complete(runId, "failed"); if (error instanceof QiModelPolicyError) { diff --git a/packages/keiko-server/src/run-engine.test.ts b/packages/keiko-server/src/run-engine.test.ts index 4d888ad325..57e8c61a9b 100644 --- a/packages/keiko-server/src/run-engine.test.ts +++ b/packages/keiko-server/src/run-engine.test.ts @@ -323,6 +323,35 @@ describe("probeNetworkIsolationSafely", () => { expect(probeSafely("/nonexistent/workspace")).toBe(false); }); + it("threads the dispatching run's own id into the failure diagnostic instead of a disconnected mint", async () => { + // ADR-0173 D5 / g12: dispatchWorkflow/applyRun both have a runId in scope when they call this + // probe; the probe failure diagnostic must carry THAT id (via the default stderr sink, which + // this test observes through console.error) rather than a fresh randomUUID() unrelated to the + // run whose verification enforcement it affects. + vi.resetModules(); + const actualVerification = await vi.importActual< + typeof import("./editor/verificationExecution.js") + >("./editor/verificationExecution.js"); + vi.doMock("./editor/verificationExecution.js", () => ({ + ...actualVerification, + probeNetworkIsolation: (): never => { + throw new Error("probe backend detection failed"); + }, + })); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + const { probeNetworkIsolationSafely: probeSafely } = await import("./run-engine.js"); + const runId = `run-${randomUUID()}`; + expect(probeSafely("/nonexistent/workspace", runId)).toBe(false); + expect(consoleError).toHaveBeenCalledTimes(1); + const line = consoleError.mock.calls[0]?.[0] as string; + const record = JSON.parse(line.slice(line.indexOf("{"))) as { correlationId?: unknown }; + expect(record.correlationId).toBe(runId); + } finally { + consoleError.mockRestore(); + } + }); + it.each([true, false])( "passes through the real probe's available:%s without swallowing it", async (available) => { @@ -377,4 +406,49 @@ describe("applyRun — verification egress probe threading", () => { expect(capturedDeps).toBeDefined(); expect(typeof capturedDeps?.verificationEnforcedNetworkAvailable).toBe("boolean"); }); + + it("threads the replayed run's own runId into a probe failure during apply", async () => { + // ADR-0173 D5 / g12: run-handlers.ts's gated apply path always has the run's own runId in + // scope (RunRecord.runId); a probe failure during the replayed verify stage must carry it + // rather than a disconnected randomUUID(). + vi.resetModules(); + const actualWorkflows = await vi.importActual( + "@oscharko-dev/keiko-workflows", + ); + vi.doMock("@oscharko-dev/keiko-workflows", () => ({ + ...actualWorkflows, + generateUnitTests: (): Promise => Promise.resolve({ status: "completed" }), + })); + const actualVerification = await vi.importActual< + typeof import("./editor/verificationExecution.js") + >("./editor/verificationExecution.js"); + vi.doMock("./editor/verificationExecution.js", () => ({ + ...actualVerification, + probeNetworkIsolation: (): never => { + throw new Error("probe backend detection failed"); + }, + })); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + const { applyRun: apply } = await import("./run-engine.js"); + const runId = `run-${randomUUID()}`; + + await apply( + { kind: "unit-tests", payload: { workspaceRoot }, limits: undefined }, + { call: () => Promise.reject(new Error("unused")) }, + "m", + (value) => value, + undefined, + runId, + ); + + expect(consoleError).toHaveBeenCalledTimes(1); + const line = consoleError.mock.calls[0]?.[0] as string; + const record = JSON.parse(line.slice(line.indexOf("{"))) as { correlationId?: unknown }; + expect(record.correlationId).toBe(runId); + } finally { + consoleError.mockRestore(); + vi.doUnmock("./editor/verificationExecution.js"); + } + }); }); diff --git a/packages/keiko-server/src/run-engine.ts b/packages/keiko-server/src/run-engine.ts index cf53ebda5f..94555b3986 100644 --- a/packages/keiko-server/src/run-engine.ts +++ b/packages/keiko-server/src/run-engine.ts @@ -257,12 +257,15 @@ function cancelWorkflow(controller: AbortController): (reason?: string) => void // run at all — never fail open into an unenforced network:"none" step. A failure is still recorded // through the server's single redacted diagnostic sink (no cwd, no raw error text — a content-free // error class only) so a probe that starts failing is operator-visible, not silently swallowed. -export function probeNetworkIsolationSafely(cwd: string): boolean { +export function probeNetworkIsolationSafely(cwd: string, runId?: string): boolean { try { return probeNetworkIsolation(cwd).available; } catch (error) { emitServerDiagnostic(undefined, { - correlationId: randomUUID(), + // Threads the dispatching run's own id (ADR-0173 D5 / g12) when the caller has one in + // scope, rather than a disconnected mint, so this probe failure joins the SAME run's other + // diagnostics. + correlationId: runId ?? randomUUID(), timestamp: new Date().toISOString(), operation: "workflow.network-isolation-probe", source: "run-engine.probeNetworkIsolationSafely", @@ -276,36 +279,47 @@ export function probeNetworkIsolationSafely(cwd: string): boolean { // Starts the underlying run for a workflow request: an AbortController drives cancellation (the // workflow honours deps.signal), and the BFF-owned runId is injected as the workflow idSource so the // streamed events carry the same runId the registry/SSE key on. +// Split out of dispatchWorkflow purely to keep that function's line count under the repository +// limit as the probe-correlation threading grew it; behavior is unchanged from before the split. +function dispatchWorkflowMemoryDeps( + ctx: EngineContext, + runId: string, +): { readonly memoryPort?: ReturnType } { + if (ctx.memoryVault === undefined || ctx.evidence === undefined) return {}; + return { + memoryPort: createWorkflowMemoryPort({ + vault: ctx.memoryVault, + evidenceStore: ctx.evidence.store, + runId, + redactString: ctx.memoryAuditRedactString ?? ((input: string): string => input), + ...(ctx.memoryCustomerIdentifierMatchers === undefined + ? {} + : { customerIdentifierMatchers: ctx.memoryCustomerIdentifierMatchers }), + }), + }; +} + function dispatchWorkflow(ctx: EngineContext, sink: QueueEventSink, runId: string): Dispatched { const controller = new AbortController(); const ports = governedWorkflowPorts(ctx); + // Probe THIS host for an enforcing egress backend and hand the answer to the verify stage, the + // same probe-then-enforce composition the editor verification path uses (ADR-0043 D8). Threads + // this dispatch's own runId (ADR-0173 D5 / g12) into a probe-failure diagnostic. + const verificationEnforcedNetworkAvailable = probeNetworkIsolationSafely( + workspaceRoot(ctx.request), + runId, + ); const commonDeps = { model: ports.model, ...(ports.spawn === undefined ? {} : { spawn: ports.spawn }), - // Probe THIS host for an enforcing egress backend and hand the answer to the verify stage, the - // same probe-then-enforce composition the editor verification path uses. Without it the stage - // could only ever see "no backend available" and had to choose between denying every - // network:"none" step and running model-authored code with inherited network (ADR-0043 D8). - verificationEnforcedNetworkAvailable: probeNetworkIsolationSafely(workspaceRoot(ctx.request)), + verificationEnforcedNetworkAvailable, sink, signal: controller.signal, idSource: (): string => runId, ...(ctx.request.governedHandoff === undefined ? {} : { workflowHandoff: ctx.request.governedHandoff }), - ...(ctx.memoryVault !== undefined && ctx.evidence !== undefined - ? { - memoryPort: createWorkflowMemoryPort({ - vault: ctx.memoryVault, - evidenceStore: ctx.evidence.store, - runId, - redactString: ctx.memoryAuditRedactString ?? ((input: string): string => input), - ...(ctx.memoryCustomerIdentifierMatchers === undefined - ? {} - : { customerIdentifierMatchers: ctx.memoryCustomerIdentifierMatchers }), - }), - } - : {}), + ...dispatchWorkflowMemoryDeps(ctx, runId), }; if (ctx.request.kind === "unit-tests") { const result = generateUnitTests(unitTestInput(ctx.request), commonDeps).then((report) => ({ @@ -649,6 +663,10 @@ export async function applyRun( modelId: string, redactReport: (value: unknown) => unknown, governance?: AgentRunGovernanceBinding, + // The originating run's own id (ADR-0173 D5 / g12), threaded into the re-invoked verify stage's + // network-isolation probe so a probe failure joins the SAME run's other diagnostics rather than + // a disconnected mint. + runId?: string, ): Promise { const input = isRecord(snapshot.payload) ? snapshot.payload : {}; const limitsOverride = snapshot.limits !== undefined ? { limits: snapshot.limits } : {}; @@ -672,7 +690,7 @@ export async function applyRun( // Apply replays an accepted snapshot through the same verify stage the initial dispatch used // (dispatchWorkflow above); without this, a governed apply's network:"none" steps see no probe // result and are denied even on hosts an enforcing backend IS available on (ADR-0043 D8). - verificationEnforcedNetworkAvailable: probeNetworkIsolationSafely(root), + verificationEnforcedNetworkAvailable: probeNetworkIsolationSafely(root, runId), ...(snapshot.governedHandoff === undefined ? {} : { workflowHandoff: snapshot.governedHandoff }), diff --git a/packages/keiko-server/src/run-handlers.test.ts b/packages/keiko-server/src/run-handlers.test.ts index b3cf950582..78c372aa01 100644 --- a/packages/keiko-server/src/run-handlers.test.ts +++ b/packages/keiko-server/src/run-handlers.test.ts @@ -1330,3 +1330,82 @@ describe("apply re-proves workspace authorization at the write boundary", () => expect(modelCalls).toEqual([]); }); }); + +describe("apply threads the run's own id into its verification egress probe", () => { + afterEach(() => { + vi.doUnmock("./editor/verificationExecution.js"); + vi.resetModules(); + }); + + it("passes record.runId through to the network-isolation probe failure diagnostic, not a disconnected mint", async () => { + // ADR-0173 D5 / g12: run-handlers.ts's gated apply path always has RunRecord.runId in scope; + // a network-isolation probe failure reached through the replayed verify stage must carry it. + vi.resetModules(); + const actualVerification = await vi.importActual< + typeof import("./editor/verificationExecution.js") + >("./editor/verificationExecution.js"); + vi.doMock("./editor/verificationExecution.js", () => ({ + ...actualVerification, + probeNetworkIsolation: (): never => { + throw new Error("probe backend detection failed"); + }, + })); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + const { handleApplyRun: freshHandleApplyRun } = await import("./index.js"); + const registry = createRunRegistry(); + const workspace = authorizedApplyWorkspace(); + registry.register({ + runId: "probe-thread-run", + fingerprint: "fp-probe-thread-run", + modelId: "example-chat-model", + sink: new QueueEventSink(), + cancel: (): void => undefined, + }); + registry.complete( + "probe-thread-run", + "completed", + { status: "dry-run" }, + { + kind: "unit-tests", + payload: { workspaceRoot: workspace.root, target: { kind: "file", filePath: "x.ts" } }, + limits: undefined, + }, + ); + const deps: UiHandlerDeps = { + config: undefined, + configPresent: false, + evidenceStore: createInMemoryEvidenceStore(), + env: {}, + redactor: buildRedactor({}), + registry, + store: workspace.store, + modelPortFactory: (): ModelPort => ({ + call: (): Promise => Promise.reject(new Error("test-stop")), + }), + }; + + const result = await freshHandleApplyRun( + { + req: {} as never, + res: {} as never, + params: { runId: "probe-thread-run" }, + url: new URL("http://127.0.0.1/api/runs/probe-thread-run/apply"), + }, + deps, + ); + + expect(result.status).toBe(200); + expect(consoleError).toHaveBeenCalled(); + const diagnosticCall = consoleError.mock.calls.find((call) => + String(call[0]).includes("workflow.network-isolation-probe"), + ); + expect(diagnosticCall).toBeDefined(); + const line = String(diagnosticCall?.[0]); + const record = JSON.parse(line.slice(line.indexOf("{"))) as { correlationId?: unknown }; + expect(record.correlationId).toBe("probe-thread-run"); + } finally { + consoleError.mockRestore(); + } + }); +}); diff --git a/packages/keiko-server/src/run-handlers.ts b/packages/keiko-server/src/run-handlers.ts index 22fcd95873..9e4d89074d 100644 --- a/packages/keiko-server/src/run-handlers.ts +++ b/packages/keiko-server/src/run-handlers.ts @@ -689,7 +689,14 @@ export async function handleApplyRun(ctx: RouteContext, deps: UiHandlerDeps): Pr const budgetRejection = reserveAgentRunApplyBudget(record, snapshot); if (budgetRejection !== null) return budgetRejection; record.appliable = undefined; - const report = await applyRun(snapshot, model, record.modelId, deps.redactor, record.governance); + const report = await applyRun( + snapshot, + model, + record.modelId, + deps.redactor, + record.governance, + record.runId, + ); record.applyReport = report; record.appliedAt = Date.now(); return { diff --git a/packages/keiko-server/src/sse-write.test.ts b/packages/keiko-server/src/sse-write.test.ts index dafaa2f780..eb76be3cb7 100644 --- a/packages/keiko-server/src/sse-write.test.ts +++ b/packages/keiko-server/src/sse-write.test.ts @@ -4,7 +4,12 @@ // must never break the protective abort+destroy path. import { describe, expect, it, vi } from "vitest"; import type { ServerResponse } from "node:http"; -import { writeOrDestroy, type SseBackpressureSignal } from "./sse-write.js"; +import { + sseBackpressureReporter, + writeOrDestroy, + type SseBackpressureSignal, +} from "./sse-write.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; function fakeRes(writeReturns: boolean): { res: ServerResponse; @@ -94,3 +99,35 @@ describe("writeOrDestroy backpressure signal (GEN-PERF-CHAT-006)", () => { expect(destroy).toHaveBeenCalledTimes(1); }); }); + +// ADR-0173 D5 / g12: `sseBackpressureReporter` used to mint a fresh `randomUUID()` INSIDE the +// returned closure on every backpressure signal. A caller with the stream's own request/session id +// already in scope had no way to thread it through, and — the sharper defect — two signals from +// the SAME reporter (the same SSE stream) reported under two disconnected ids. +describe("sseBackpressureReporter correlation id (ADR-0173 D5 / g12)", () => { + it("threads a caller-supplied correlation id onto the backpressure diagnostic", () => { + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { record: (record) => records.push(record) }; + const observe = sseBackpressureReporter({ diagnostics }, "terminal", "req-abc12345"); + + observe({ frameBytes: 42, accepted: false }); + + expect(records).toHaveLength(1); + expect(records[0]?.correlationId).toBe("req-abc12345"); + }); + + it("mints one id per reporter construction, shared across every signal that reporter emits", () => { + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { record: (record) => records.push(record) }; + const observeA = sseBackpressureReporter({ diagnostics }, "terminal"); + const observeB = sseBackpressureReporter({ diagnostics }, "terminal"); + + observeA({ frameBytes: 1, accepted: false }); + observeA({ frameBytes: 2, accepted: false }); + observeB({ frameBytes: 3, accepted: false }); + + expect(records).toHaveLength(3); + expect(records[0]?.correlationId).toBe(records[1]?.correlationId); + expect(records[0]?.correlationId).not.toBe(records[2]?.correlationId); + }); +}); diff --git a/packages/keiko-server/src/sse-write.ts b/packages/keiko-server/src/sse-write.ts index f356844e58..6f05dc2cef 100644 --- a/packages/keiko-server/src/sse-write.ts +++ b/packages/keiko-server/src/sse-write.ts @@ -65,14 +65,21 @@ export function writeOrDestroy( * happens either way, but nothing records WHY the stream ended. * * Carries only the frame byte count (never body bytes), so it cannot leak model tokens if logged. + * + * `correlationId` defaults to a fresh mint taken ONCE here, at reporter-construction time (i.e. at + * SSE stream setup) — not inside the returned closure, which fires at most once anyway, but + * minting eagerly lets a caller that already has the stream's own request/session id in scope + * (ADR-0173 D5 / g12) pass it straight through instead of a disconnected one being drawn if and + * only if the stream is later killed. */ export function sseBackpressureReporter( deps: { readonly diagnostics?: ServerDiagnosticSink | undefined }, stream: string, + correlationId: string = randomUUID(), ): (signal: SseBackpressureSignal) => void { return (signal: SseBackpressureSignal): void => { emitServerDiagnostic(deps.diagnostics, { - correlationId: randomUUID(), + correlationId, timestamp: new Date().toISOString(), operation: `sse.${stream}`, source: `sse.${stream}.backpressure`, diff --git a/packages/keiko-server/src/terminal-routes.test.ts b/packages/keiko-server/src/terminal-routes.test.ts index 506de94db7..c91b40bc19 100644 --- a/packages/keiko-server/src/terminal-routes.test.ts +++ b/packages/keiko-server/src/terminal-routes.test.ts @@ -14,8 +14,10 @@ import { createRunRegistry } from "./runs.js"; import { createUiServer, UI_HOST } from "./server.js"; import { EventEmitter } from "node:events"; import type { ServerResponse } from "node:http"; -import { openTerminalSseStream } from "./terminal-routes.js"; +import { handleTerminalEvents, openTerminalSseStream } from "./terminal-routes.js"; import type { SseBackpressureSignal } from "./sse-write.js"; +import type { RouteContext } from "./routes.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; import { TerminalToolError, type TerminalEventEmitter, @@ -644,3 +646,31 @@ describe("openTerminalSseStream backpressure (KEIKO-0142)", () => { expect(fake.writes.length).toBeGreaterThan(1); }); }); + +describe("handleTerminalEvents backpressure correlation (ADR-0173 D5 / g12)", () => { + it("threads the request's own correlation id into the backpressure diagnostic instead of minting one", () => { + const fake = makeFakeSseRes(); + fake.writeReturns = false; // rejects the ready frame -> immediate backpressure kill. + const manager = new FakeTerminalExecutionManager(); + const records: ServerDiagnosticRecord[] = []; + const diagnostics: ServerDiagnosticSink = { + record: (entry) => { + records.push(entry); + }, + }; + const ctx: RouteContext = { + req: { on: (): void => undefined } as unknown as RouteContext["req"], + res: fake.res, + params: {}, + url: new URL("http://127.0.0.1/api/terminal/events"), + correlationId: "req-terminal-thread-01", + }; + const routeDeps: UiHandlerDeps = { ...deps, terminal: manager, diagnostics }; + + handleTerminalEvents(ctx, routeDeps); + + expect(records).toHaveLength(1); + expect(records[0]?.source).toBe("sse.terminal.backpressure"); + expect(records[0]?.correlationId).toBe("req-terminal-thread-01"); + }); +}); diff --git a/packages/keiko-server/src/terminal-routes.ts b/packages/keiko-server/src/terminal-routes.ts index a153b709b7..2c58188122 100644 --- a/packages/keiko-server/src/terminal-routes.ts +++ b/packages/keiko-server/src/terminal-routes.ts @@ -227,7 +227,14 @@ export function handleDeleteTerminalExecution(ctx: RouteContext, deps: UiHandler export function handleTerminalEvents(ctx: RouteContext, deps: UiHandlerDeps): HandlerOutcome { const guard = requireTerminal(deps); if (isRouteResult(guard)) return guard; - openTerminalSseStream(ctx.res, guard, deps.redactor, sseBackpressureReporter(deps, "terminal")); + // Threads the request's own correlation id (ADR-0173 D5 / g12) so a later backpressure kill + // joins back to the request that opened this stream instead of a disconnected mint. + openTerminalSseStream( + ctx.res, + guard, + deps.redactor, + sseBackpressureReporter(deps, "terminal", ctx.correlationId), + ); ctx.req.on("close", () => { ctx.res.end(); }); diff --git a/packages/keiko-server/src/update-remediation-routes.test.ts b/packages/keiko-server/src/update-remediation-routes.test.ts index 085823d458..49dedac9e7 100644 --- a/packages/keiko-server/src/update-remediation-routes.test.ts +++ b/packages/keiko-server/src/update-remediation-routes.test.ts @@ -54,6 +54,7 @@ function report(): UpdateRemediationStatusReport { class FakeUpdateRemediationManager implements UpdateRemediationManager { public readonly statuses: UpdateRemediationStatusRequest[] = []; public readonly actions: UpdateRemediationActionRequest[] = []; + public readonly runActionCorrelationIds: (string | undefined)[] = []; public readonly getStatus = ( request: UpdateRemediationStatusRequest = {}, @@ -64,8 +65,10 @@ class FakeUpdateRemediationManager implements UpdateRemediationManager { public readonly runAction = ( request: UpdateRemediationActionRequest, + correlationId?: string, ): Promise => { this.actions.push(request); + this.runActionCorrelationIds.push(correlationId); return Promise.resolve({ ...report(), overallStatus: "completed", updateCanComplete: true }); }; @@ -198,4 +201,21 @@ describe("update remediation routes", () => { actionId: "local-knowledge-reindex:local-knowledge", }); }); + + it("threads the request's own correlation id into runAction instead of leaving it to mint one", async () => { + // ADR-0173 D5 / g12: ctx.correlationId is minted at request entry (server.ts, honouring a + // well-formed client-supplied X-Keiko-Correlation-Id) and was already in scope in + // handleRunUpdateRemediationAction — before the fix it was never threaded into runAction, so + // every diagnostic runAction's own implementation reports minted an id disconnected from this + // request's trail. + const requestCorrelationId = "req-update-remediation-thread-01"; + const action = await fetch(`${baseUrl()}/api/update/remediation/actions`, { + method: "POST", + headers: { ...csrfHeaders(), "X-Keiko-Correlation-Id": requestCorrelationId }, + body: JSON.stringify({ actionId: "local-knowledge-reindex:local-knowledge" }), + }); + + expect(action.status).toBe(200); + expect(updateRemediation.runActionCorrelationIds).toEqual([requestCorrelationId]); + }); }); diff --git a/packages/keiko-server/src/update-remediation-routes.ts b/packages/keiko-server/src/update-remediation-routes.ts index 5128a96c23..f90a652584 100644 --- a/packages/keiko-server/src/update-remediation-routes.ts +++ b/packages/keiko-server/src/update-remediation-routes.ts @@ -124,6 +124,8 @@ export async function handleRunUpdateRemediationAction( if (!parsed.ok) { throw new UpdateRemediationError("BAD_REQUEST", parsed.errors.join("; "), 400); } - return { status: 200, body: await guard.runAction(parsed.value) }; + // Threads the request's own correlation id (ADR-0173 D5 / g12) so every diagnostic this one + // remediation action reports stays joined under it, per runAction's own contract. + return { status: 200, body: await guard.runAction(parsed.value, ctx.correlationId) }; }); } diff --git a/packages/keiko-server/src/update-remediation.test.ts b/packages/keiko-server/src/update-remediation.test.ts index 0076d8e2a6..d03d2e40fd 100644 --- a/packages/keiko-server/src/update-remediation.test.ts +++ b/packages/keiko-server/src/update-remediation.test.ts @@ -684,6 +684,66 @@ describe("update remediation manager", () => { expect(localKnowledge.runs()).toBe(1); }); + // ADR-0173 D5 / g12: each of `recordDraftFailure`'s call sites used to mint its own + // `randomUUID()`, so a cascade of failures inside a SINGLE `runAction` call (persist fails, then + // the outcome-uncertainty fallback also fails) reported as if they were unrelated operations. + // Fails before the fix — two independent random UUIDs practically never match — and passes after, + // once every reporter reads the one id `runAction` mints at its own start. + it("shares one correlation id across every diagnostic from a single failing action run", async () => { + const diagnostics: ServerDiagnosticRecord[] = []; + const localKnowledge = fakeLocalKnowledge(); + const durable = createUpdateLocalStateManager({ stateDir: makeStateDir(), now: () => NOW }); + let stateWrites = 0; + const unreliable: UpdateLocalStateManager = { + ...durable, + writeRuntimeState: (state) => { + stateWrites += 1; + if (stateWrites >= 2) throw new Error("terminal state unavailable"); + return durable.writeRuntimeState(state); + }, + }; + const subject = createUpdateRemediationManager({ + localState: unreliable, + localKnowledge, + now: () => NOW, + diagnostics: { record: (record) => diagnostics.push(record) }, + }); + + await subject.runAction({ + actionId: "local-knowledge-reindex:local-knowledge", + targetVersion: TARGET, + impact: localKnowledgeImpact, + }); + + const sources = diagnostics.map((record) => record.source); + expect(sources).toContain("update-remediation.persistDraftStatus"); + expect(sources).toContain("update-remediation.persistOutcomeUncertainty"); + const correlationIds = new Set(diagnostics.map((record) => record.correlationId)); + expect(correlationIds.size).toBe(1); + }); + + it("threads a caller-supplied correlation id through onto every reported diagnostic", async () => { + const diagnostics: ServerDiagnosticRecord[] = []; + const subject = createUpdateRemediationManager({ + localState: createUpdateLocalStateManager({ stateDir: makeStateDir(), now: () => NOW }), + localKnowledge: throwingLocalKnowledge(), + now: () => NOW, + diagnostics: { record: (record) => diagnostics.push(record) }, + redactString: (value) => value, + }); + + await subject.runAction( + { + actionId: "local-knowledge-reindex:local-knowledge", + targetVersion: TARGET, + impact: localKnowledgeImpact, + }, + "caller-req-77", + ); + + expect(diagnostics).toContainEqual(expect.objectContaining({ correlationId: "caller-req-77" })); + }); + it("retains the live lease when neither terminal nor uncertainty state can persist", async () => { const localKnowledge = fakeLocalKnowledge(); const durable = createUpdateLocalStateManager({ stateDir: makeStateDir(), now: () => NOW }); diff --git a/packages/keiko-server/src/update-remediation.ts b/packages/keiko-server/src/update-remediation.ts index 8620226694..0d19e174aa 100644 --- a/packages/keiko-server/src/update-remediation.ts +++ b/packages/keiko-server/src/update-remediation.ts @@ -27,8 +27,14 @@ import { export interface UpdateRemediationManager { readonly getStatus: (request?: UpdateRemediationStatusRequest) => UpdateRemediationStatusReport; + // `correlationId` is optional so an existing caller keeps compiling unchanged; when the HTTP + // route layer threads its own request-scoped id through (ADR-0173 D5 / g12), every diagnostic + // this ONE action execution reports stays joined under it. Absent a caller-supplied id, one is + // minted here, once, so a cascade of failures from a single execution (persist, then audit, then + // outcome-uncertainty) still shares one id instead of each reporter minting its own. readonly runAction: ( request: UpdateRemediationActionRequest, + correlationId?: string, ) => Promise; readonly completeRestart: (targetVersion?: string) => UpdateRemediationStatusReport; readonly updateCanComplete: (targetVersion?: string) => boolean; @@ -273,15 +279,19 @@ async function executeDraft( return "failed"; } +// `correlationId` is always the ONE id minted (or supplied) at the start of the enclosing +// `runAction` call — never a fresh mint here — so every failure this single action execution +// reports, however many of the call sites below fire, stays joined under it. function recordDraftFailure( options: UpdateRemediationManagerOptions, + correlationId: string, error: unknown, source = "update-remediation.executeDraft", ): void { emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "update.remediation.execute", source, error, @@ -293,11 +303,12 @@ function recordDraftFailure( async function executeDraftStatus( options: UpdateRemediationManagerOptions, draft: ActionDraft, + correlationId: string, ): Promise { try { return await executeDraft(options, draft); } catch (error) { - recordDraftFailure(options, error); + recordDraftFailure(options, correlationId, error); return "failed"; } } @@ -307,6 +318,7 @@ function persistOutcomeUncertainty( now: () => number, request: UpdateRemediationActionRequest, draft: ActionDraft, + correlationId: string, ): boolean { try { upsertRuntimeAction({ @@ -319,7 +331,12 @@ function persistOutcomeUncertainty( }); return true; } catch (error) { - recordDraftFailure(options, error, "update-remediation.persistOutcomeUncertainty"); + recordDraftFailure( + options, + correlationId, + error, + "update-remediation.persistOutcomeUncertainty", + ); return false; } } @@ -329,11 +346,12 @@ function recordDraftAuditSafely( request: UpdateRemediationActionRequest, draft: ActionDraft, status: RuntimeRemediationStatus, + correlationId: string, ): void { try { - recordRemediationAudit(options, request, draft, status); + recordRemediationAudit(options, request, draft, status, correlationId); } catch (error) { - recordDraftFailure(options, error, "update-remediation.persistDraftAudit"); + recordDraftFailure(options, correlationId, error, "update-remediation.persistDraftAudit"); } } @@ -343,6 +361,7 @@ function persistDraftStatusSafely( request: UpdateRemediationActionRequest, draft: ActionDraft, status: RuntimeRemediationStatus, + correlationId: string, ): boolean { try { upsertRuntimeAction({ @@ -354,12 +373,12 @@ function persistDraftStatusSafely( ...(status === "failed" ? { warningCode: "remediation-execution-failed" } : {}), }); } catch (error) { - recordDraftFailure(options, error, "update-remediation.persistDraftStatus"); - const interlocked = persistOutcomeUncertainty(options, now, request, draft); - recordDraftAuditSafely(options, request, draft, status); + recordDraftFailure(options, correlationId, error, "update-remediation.persistDraftStatus"); + const interlocked = persistOutcomeUncertainty(options, now, request, draft, correlationId); + recordDraftAuditSafely(options, request, draft, status, correlationId); return interlocked; } - recordDraftAuditSafely(options, request, draft, status); + recordDraftAuditSafely(options, request, draft, status, correlationId); return true; } @@ -388,6 +407,7 @@ function recordRemediationAudit( request: UpdateRemediationActionRequest, draft: ActionDraft, status: RuntimeRemediationStatus, + correlationId: string, ): void { const result = options.localState.recordAuditEvent(remediationAuditEventType(status), { targetVersion: request.targetVersion, @@ -397,7 +417,12 @@ function recordRemediationAudit( ...(status === "failed" ? { warningCode: "remediation-execution-failed" } : {}), }); if (result.warning !== undefined) { - recordDraftFailure(options, new Error(result.warning), "update-remediation.persistDraftAudit"); + recordDraftFailure( + options, + correlationId, + new Error(result.warning), + "update-remediation.persistDraftAudit", + ); } } @@ -406,6 +431,7 @@ function deferDraft( now: () => number, request: UpdateRemediationActionRequest, draft: ActionDraft, + correlationId: string, ): void { if (!draft.canDefer) { throw new UpdateRemediationError( @@ -421,7 +447,7 @@ function deferDraft( status: "deferred", now, }); - recordRemediationAudit(options, request, draft, "deferred"); + recordRemediationAudit(options, request, draft, "deferred", correlationId); } function assertDraftOutcomeKnown( @@ -444,6 +470,7 @@ async function runDraft( runningActions: Set, request: UpdateRemediationActionRequest, draft: ActionDraft, + correlationId: string, ): Promise { if (!draft.canRun) { throw new UpdateRemediationError( @@ -469,7 +496,8 @@ async function runDraft( now, request, draft, - await executeDraftStatus(options, draft), + await executeDraftStatus(options, draft, correlationId), + correlationId, ); } finally { if (releaseAllowed) { @@ -513,14 +541,15 @@ async function runRemediationAction( now: () => number, runningActions: Set, request: UpdateRemediationActionRequest, + correlationId: string, ): Promise { const drafts = draftsForImpact(options.localState, request.impact, options.localKnowledge); const draft = findDraftOrThrow(drafts, request.actionId); assertDraftOutcomeKnown(options, draft); if (request.decision === "defer") { - deferDraft(options, now, request, draft); + deferDraft(options, now, request, draft, correlationId); } else { - await runDraft(options, now, runningActions, request, draft); + await runDraft(options, now, runningActions, request, draft, correlationId); } return statusFor(options, now, { ...request, persist: false }); } @@ -559,8 +588,8 @@ export function createUpdateRemediationManager( const runningActions = new Set(); return { getStatus: (request): UpdateRemediationStatusReport => statusFor(options, now, request), - runAction: (request): Promise => - runRemediationAction(options, now, runningActions, request), + runAction: (request, correlationId = randomUUID()): Promise => + runRemediationAction(options, now, runningActions, request, correlationId), completeRestart: (targetVersion): UpdateRemediationStatusReport => completeRestartAction(options, now, targetVersion), updateCanComplete: (targetVersion): boolean => diff --git a/packages/keiko-server/src/voice-control-ws.test.ts b/packages/keiko-server/src/voice-control-ws.test.ts index d6fd822a3d..e72f227567 100644 --- a/packages/keiko-server/src/voice-control-ws.test.ts +++ b/packages/keiko-server/src/voice-control-ws.test.ts @@ -13,6 +13,7 @@ import type { Server } from "node:http"; import { WebSocket } from "ws"; import { createUiServer, UI_HOST } from "./server.js"; import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; +import { CORRELATION_HEADER } from "./correlation.js"; import { MAX_VOICE_CONTROL_FRAME_BYTES } from "./voice-realtime.js"; import { VOICE_LIVE_TRANSCRIBE_PATH } from "./voice-live-dictation.js"; import { buildRedactor, createRunRegistry, type UiHandlerDeps } from "./index.js"; @@ -561,6 +562,136 @@ describe("WebSocket live dictation upgrade — transcription-only control plane" socket.close(); }); + // RB-6 / ADR-0173 D5 regression pin: the correlation id is resolved ONCE per WebSocket connection + // at handleUpgrade, never re-minted per failure. Before the fix, `reportNegotiationFailure` called + // `randomUUID()` on every invocation, so two failures on the SAME connection carried two unrelated + // ids; this proves they now match. + it("reuses the same correlation id across two negotiation failures on one live-dictation connection", async (): Promise => { + const diagnostics: ServerDiagnosticRecord[] = []; + const port = await boot( + depsWith({ + config: voiceConfig(true), + configPresent: true, + diagnostics: { record: (record): void => void diagnostics.push(record) }, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }), + ); + const { ws: socket, next } = expectOpen( + await connect(port, { path: VOICE_LIVE_TRANSCRIBE_PATH }), + ); + socket.send(liveSessionCreate()); + await next(); // session.created + await next(); // capability.offer + + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-live-1", + seq: 1, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const firstFailure = await next(); + await next(); // media.track.state ended + + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-live-1", + seq: 2, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const secondFailure = await next(); + await next(); // media.track.state ended + + expect(firstFailure.correlationId).toMatch(/^[0-9a-f-]{36}$/u); + expect(secondFailure.correlationId).toBe(firstFailure.correlationId); + expect(diagnostics).toHaveLength(2); + expect(diagnostics[0]?.correlationId).toBe(firstFailure.correlationId); + expect(diagnostics[1]?.correlationId).toBe(firstFailure.correlationId); + socket.close(); + }); + + it("honors a well-formed client-supplied X-Keiko-Correlation-Id on the live-dictation upgrade", async (): Promise => { + const diagnostics: ServerDiagnosticRecord[] = []; + const port = await boot( + depsWith({ + config: voiceConfig(true), + configPresent: true, + diagnostics: { record: (record): void => void diagnostics.push(record) }, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }), + ); + const { ws: socket, next } = expectOpen( + await connect(port, { + path: VOICE_LIVE_TRANSCRIBE_PATH, + headers: { [CORRELATION_HEADER]: "client-supplied-live-corr-1" }, + }), + ); + socket.send(liveSessionCreate()); + await next(); // session.created + await next(); // capability.offer + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-live-1", + seq: 1, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const failure = await next(); + expect(failure.correlationId).toBe("client-supplied-live-corr-1"); + expect(diagnostics[0]?.correlationId).toBe("client-supplied-live-corr-1"); + socket.close(); + }); + + it("replaces a malformed client-supplied X-Keiko-Correlation-Id on the live-dictation upgrade", async (): Promise => { + const port = await boot( + depsWith({ + config: voiceConfig(true), + configPresent: true, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }), + ); + const { ws: socket, next } = expectOpen( + await connect(port, { + path: VOICE_LIVE_TRANSCRIBE_PATH, + headers: { [CORRELATION_HEADER]: "short" }, + }), + ); + socket.send(liveSessionCreate()); + await next(); // session.created + await next(); // capability.offer + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-live-1", + seq: 1, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const failure = await next(); + expect(failure.correlationId).not.toBe("short"); + expect(failure.correlationId).toMatch(/^[0-9a-f-]{36}$/u); + socket.close(); + }); + it("rejects chat context and persona on the live dictation endpoint", async () => { const port = await boot(depsWith({ config: voiceConfig(true), configPresent: true })); const { ws: socket } = expectOpen(await connect(port, { path: VOICE_LIVE_TRANSCRIBE_PATH })); @@ -662,6 +793,118 @@ describe("WebSocket voice control upgrade — protocol behavior", () => { socket.close(); }); + // RB-6 / ADR-0173 D5 regression pin: the correlation id is resolved ONCE per WebSocket connection + // at handleUpgrade, never re-minted per failure — mirrors the live-dictation pin above for the + // full realtime control plane, which had no correlation-id concept at all before the fix. + it("reuses the same correlation id across two negotiation failures on one realtime control connection", async (): Promise => { + const { deps, chat } = depsWithChat({ + config: voiceConfig(true), + configPresent: true, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }); + const port = await boot(deps); + const { ws: socket, next } = expectOpen(await connect(port)); + socket.send(sessionCreate(chat.id)); + await next(); // session.created + await next(); // capability.offer + + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-int-1", + seq: 1, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const firstFailure = await next(); + await next(); // media.track.state ended + + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-int-1", + seq: 2, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const secondFailure = await next(); + await next(); // media.track.state ended + + expect(firstFailure).toMatchObject({ kind: "error", code: "negotiation-failed" }); + expect(firstFailure.correlationId).toMatch(/^[0-9a-f-]{36}$/u); + expect(secondFailure.correlationId).toBe(firstFailure.correlationId); + socket.close(); + }); + + it("honors a well-formed client-supplied X-Keiko-Correlation-Id on the realtime control upgrade", async (): Promise => { + const { deps, chat } = depsWithChat({ + config: voiceConfig(true), + configPresent: true, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }); + const port = await boot(deps); + const { ws: socket, next } = expectOpen( + await connect(port, { headers: { [CORRELATION_HEADER]: "client-supplied-rt-corr-1" } }), + ); + socket.send(sessionCreate(chat.id)); + await next(); // session.created + await next(); // capability.offer + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-int-1", + seq: 1, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const failure = await next(); + expect(failure).toMatchObject({ kind: "error", code: "negotiation-failed" }); + expect(failure.correlationId).toBe("client-supplied-rt-corr-1"); + socket.close(); + }); + + it("replaces a malformed client-supplied X-Keiko-Correlation-Id on the realtime control upgrade", async (): Promise => { + const { deps, chat } = depsWithChat({ + config: voiceConfig(true), + configPresent: true, + voiceRealtimeNegotiationRequest: (): Promise => + Promise.resolve({ ok: false, kind: "wrong-header" }), + }); + const port = await boot(deps); + const { ws: socket, next } = expectOpen( + await connect(port, { headers: { [CORRELATION_HEADER]: "short" } }), + ); + socket.send(sessionCreate(chat.id)); + await next(); // session.created + await next(); // capability.offer + socket.send( + JSON.stringify({ + protocolVersion: "1", + sessionId: "sess-int-1", + seq: 1, + direction: "client-to-host", + kind: "signal.sdp.offer", + sdp: OFFER_SDP, + }), + ); + await next(); // media.track.state negotiating + const failure = await next(); + expect(failure.correlationId).not.toBe("short"); + expect(failure.correlationId).toMatch(/^[0-9a-f-]{36}$/u); + socket.close(); + }); + it("rejects a concurrent socket for an attached idempotent session", async () => { const { deps, chat } = depsWithChat({ config: voiceConfig(true), configPresent: true }); const port = await boot(deps); diff --git a/packages/keiko-server/src/voice-handlers.ts b/packages/keiko-server/src/voice-handlers.ts index 19affb3363..b982a0b7b3 100644 --- a/packages/keiko-server/src/voice-handlers.ts +++ b/packages/keiko-server/src/voice-handlers.ts @@ -41,6 +41,7 @@ import { currentGatewayConfig, currentGatewayEgressConfig } from "./deps.js"; import { isVoiceDisabledByPolicy } from "./read-handlers.js"; import { createRequestCancellation } from "./request-cancellation.js"; import { toSpeakableText } from "./voice-speech-text.js"; +import { UNKNOWN_CORRELATION_ID } from "./correlation.js"; import { emitServerDiagnostic, serverDiagnosticFromError } from "./diagnostics-log.js"; // The decoded-audio ceiling for one dictation clip. This is the authoritative bound on the @@ -793,7 +794,7 @@ async function pipeAudioStream( emitServerDiagnostic( deps.diagnostics, serverDiagnosticFromError({ - correlationId: ctx.correlationId ?? "unknown", + correlationId: ctx.correlationId ?? UNKNOWN_CORRELATION_ID, operation: "voice.speech.stream", source: "voice.speech", error, diff --git a/packages/keiko-server/src/voice-live-dictation.ts b/packages/keiko-server/src/voice-live-dictation.ts index 5d792b5e4e..d74e9c0e92 100644 --- a/packages/keiko-server/src/voice-live-dictation.ts +++ b/packages/keiko-server/src/voice-live-dictation.ts @@ -6,7 +6,6 @@ import type { IncomingMessage } from "node:http"; import type { Duplex } from "node:stream"; -import { randomUUID } from "node:crypto"; import { WebSocketServer, type RawData, type WebSocket as WsSocket } from "ws"; import { findConfiguredCapability, @@ -29,6 +28,7 @@ import { type VoiceSessionCreateMessage, } from "@oscharko-dev/keiko-contracts"; import { isAllowedHost } from "./host-check.js"; +import { resolveCorrelationId } from "./correlation.js"; import { currentGatewayConfig, currentGatewayEgressConfig, type UiHandlerDeps } from "./deps.js"; import { isVoiceDisabledByPolicy, isVoiceRealtimeCapable } from "./read-handlers.js"; import { @@ -324,6 +324,10 @@ class VoiceLiveDictationConnection { private readonly negotiate: LiveDictationNegotiateFn, private readonly redact: (value: unknown) => unknown, private readonly diagnostics: ServerDiagnosticSink | undefined, + // Resolved ONCE at handleUpgrade (RB-6 / ADR-0173 D5): every diagnostic this connection emits + // over its whole lifetime — however many negotiation attempts a client makes — is joinable to + // the same id, instead of a fresh one per failure. + private readonly correlationId: string, ) {} start(): void { @@ -452,7 +456,7 @@ class VoiceLiveDictationConnection { } private reportNegotiationFailure(kind: RealtimeNegotiationErrorKind, thrown: unknown): void { - const correlationId = randomUUID(); + const correlationId = this.correlationId; const diagnostic = serverDiagnosticFromError({ correlationId, operation: "voice.live-dictation.negotiate", @@ -513,8 +517,11 @@ class VoiceLiveDictationPlaneImpl implements VoiceControlPlane { ) { return false; } + // Resolved ONCE per upgrade (RB-6 / ADR-0173 D5), not re-minted per diagnostic — every + // negotiation failure this connection later reports carries the same id. + const correlationId = resolveCorrelationId(req); this.wss.handleUpgrade(req, sock, head, (ws) => { - this.onConnection(ws, deps); + this.onConnection(ws, deps, correlationId); }); return true; } @@ -614,7 +621,7 @@ class VoiceLiveDictationPlaneImpl implements VoiceControlPlane { }; } - private onConnection(ws: WsSocket, deps: UiHandlerDeps): void { + private onConnection(ws: WsSocket, deps: UiHandlerDeps, correlationId: string): void { this.attachHeartbeat(ws); // KEIKO-0342: enforce the concurrent-connection cap after handleUpgrade admitted the // socket. wss.clients already includes this new one by the time onConnection runs, so @@ -652,6 +659,7 @@ class VoiceLiveDictationPlaneImpl implements VoiceControlPlane { this.buildNegotiate(deps, session.transcriptionLanguage), deps.redactor, deps.diagnostics, + correlationId, ); connection.start(); }); diff --git a/packages/keiko-server/src/voice-realtime.test.ts b/packages/keiko-server/src/voice-realtime.test.ts index 084600ce05..cd15c90045 100644 --- a/packages/keiko-server/src/voice-realtime.test.ts +++ b/packages/keiko-server/src/voice-realtime.test.ts @@ -101,6 +101,10 @@ function resolvePendingNegotiations(calls: readonly PendingNegotiation[]): void for (const call of calls) call.resolve(ok()); } +// Default connection-scoped correlation id used by tests that don't care about its exact value — +// distinct from any protocol code/kind string so an accidental field mix-up is easy to spot. +const TEST_CORRELATION_ID = "conn-correlation-id-1"; + function connect(options?: { negotiate?: ( offerSdp: string, @@ -109,6 +113,7 @@ function connect(options?: { ) => Promise; redact?: (value: unknown) => unknown; session?: TestSession; + correlationId?: string; }): { socket: FakeSocket; session: TestSession; conn: VoiceControlConnection } { const socket = new FakeSocket(); const session = options?.session ?? makeSession(); @@ -117,6 +122,7 @@ function connect(options?: { session, negotiate: options?.negotiate ?? okAsync, redact: options?.redact ?? ((value: unknown): unknown => value), + correlationId: options?.correlationId ?? TEST_CORRELATION_ID, }); return { socket, session, conn }; } @@ -261,19 +267,48 @@ describe("VoiceControlConnection proxied-SDP signaling", () => { expect(negotiate.mock.calls[0]).toHaveLength(3); }); - it("answers a negotiation failure with error negotiation-failed and an ended track", async () => { + it("answers a negotiation failure with error negotiation-failed, the connection's correlation id, and an ended track", async () => { const { socket, conn } = connect({ negotiate: (): Promise => Promise.resolve({ ok: false, kind: "transport" }), + correlationId: "negotiation-fail-corr-1", }); conn.start(false); socket.sent.length = 0; await conn.receive(clientMessage("signal.sdp.offer", 1, { sdp: OFFER_SDP })); expect(kinds(socket)).toEqual(["media.track.state", "error", "media.track.state"]); - expect((socket.sent[1] as unknown as Record).code).toBe("negotiation-failed"); + const failure = socket.sent[1] as unknown as Record; + expect(failure.code).toBe("negotiation-failed"); + expect(failure.correlationId).toBe("negotiation-fail-corr-1"); }); - it("rejects a malformed SDP offer without calling the provider", async () => { + // RB-6 / ADR-0173 D5 regression pin: the correlation id is resolved ONCE per WebSocket connection + // (at handleUpgrade, injected here as the connection's constructor option), never re-minted per + // failure. Before the fix each negotiation failure on the same connection would have carried an + // unrelated fresh id; this proves a second failure on the same connection still matches the first. + it("reuses the same connection-scoped correlation id across repeated negotiation failures", async () => { + const { socket, conn } = connect({ + negotiate: (): Promise => + Promise.resolve({ ok: false, kind: "transport" }), + correlationId: "repeated-failure-corr-1", + }); + conn.start(false); + socket.sent.length = 0; + + await conn.receive(clientMessage("signal.sdp.offer", 1, { sdp: OFFER_SDP })); + const firstFailure = socket.sent[1] as unknown as Record; + expect(firstFailure.code).toBe("negotiation-failed"); + expect(firstFailure.correlationId).toBe("repeated-failure-corr-1"); + + socket.sent.length = 0; + await conn.receive(clientMessage("signal.sdp.offer", 2, { sdp: OFFER_SDP })); + const secondFailure = socket.sent[1] as unknown as Record; + expect(secondFailure.code).toBe("negotiation-failed"); + expect(secondFailure.correlationId).toBe("repeated-failure-corr-1"); + expect(secondFailure.correlationId).toBe(firstFailure.correlationId); + }); + + it("rejects a malformed SDP offer without calling the provider or attaching a correlation id", async () => { const negotiate = vi.fn(okAsync); const { socket, conn } = connect({ negotiate }); conn.start(false); @@ -281,7 +316,9 @@ describe("VoiceControlConnection proxied-SDP signaling", () => { await conn.receive(clientMessage("signal.sdp.offer", 1, { sdp: "not-an-sdp" })); expect(negotiate).not.toHaveBeenCalled(); expect(kinds(socket)).toEqual(["error"]); - expect((socket.sent[0] as unknown as Record).code).toBe("invalid-message"); + const failure = socket.sent[0] as unknown as Record; + expect(failure.code).toBe("invalid-message"); + expect(failure).not.toHaveProperty("correlationId"); }); it.each([ diff --git a/packages/keiko-server/src/voice-realtime.ts b/packages/keiko-server/src/voice-realtime.ts index 397dcf142c..d38347667f 100644 --- a/packages/keiko-server/src/voice-realtime.ts +++ b/packages/keiko-server/src/voice-realtime.ts @@ -47,6 +47,7 @@ import { type VoiceSessionCreateMessage, } from "@oscharko-dev/keiko-contracts"; import { isAllowedHost } from "./host-check.js"; +import { resolveCorrelationId } from "./correlation.js"; import { currentGatewayConfig, currentGatewayEgressConfig, type UiHandlerDeps } from "./deps.js"; import { isVoiceDisabledByPolicy, isVoiceRealtimeCapable } from "./read-handlers.js"; @@ -294,6 +295,11 @@ export interface VoiceControlConnectionOptions { readonly session: SessionState; readonly negotiate: NegotiateFn; readonly redact: (value: unknown) => unknown; + // Resolved ONCE per WebSocket connection at handleUpgrade (RB-6 / ADR-0173 D5), never re-minted + // per failure — every diagnostic-bearing message this connection emits over its whole lifetime, + // including across a session resumed by a later reconnect's OWN new connection, is joinable to + // the id of the upgrade that produced it. + readonly correlationId: string; } // The protocol state machine for one attached control socket. Pure of WebSocket/IO concerns beyond @@ -305,6 +311,7 @@ export class VoiceControlConnection { private readonly session: SessionState; private readonly negotiate: NegotiateFn; private readonly redact: (value: unknown) => unknown; + private readonly correlationId: string; private negotiation: AbortController | undefined; private closed = false; @@ -313,6 +320,7 @@ export class VoiceControlConnection { this.session = options.session; this.negotiate = options.negotiate; this.redact = options.redact; + this.correlationId = options.correlationId; } // Re-delivers the buffered replayable events to a (re)attached client, then announces the resolved @@ -429,7 +437,9 @@ export class VoiceControlConnection { } private failNegotiation(): void { - this.emitError("negotiation-failed"); + // Same connection-scoped correlation id as every other negotiation attempt on this socket + // (RB-6 / ADR-0173 D5) — never re-minted per failure. + this.emitError("negotiation-failed", this.correlationId); this.emit({ kind: "media.track.state", track: "audio-in", state: "ended" }); } @@ -471,8 +481,8 @@ export class VoiceControlConnection { this.emit({ kind: "media.track.state", track: "audio-in", state: "live" }); } - private emitError(code: VoiceProtocolErrorCode): void { - this.emit({ kind: "error", code }); + private emitError(code: VoiceProtocolErrorCode, correlationId?: string): void { + this.emit({ kind: "error", code, ...(correlationId !== undefined ? { correlationId } : {}) }); } // Builds a sequenced host→client control message from a payload, appends it to the bounded replay @@ -700,8 +710,11 @@ class VoiceControlPlaneImpl implements VoiceControlPlane { ) { return false; } + // Resolved ONCE per upgrade (RB-6 / ADR-0173 D5): the id is scoped to this physical WebSocket + // connection, not the resumable logical session, so a later reconnect gets its own fresh id. + const correlationId = resolveCorrelationId(req); this.wss.handleUpgrade(req, sock, head, (ws) => { - this.onConnection(ws, deps); + this.onConnection(ws, deps, correlationId); }); return true; } @@ -852,7 +865,7 @@ class VoiceControlPlaneImpl implements VoiceControlPlane { } // eslint-disable-next-line max-lines-per-function -- connection lifecycle keeps heartbeat, frame limits, session start, and detach handling together. - private onConnection(ws: WsSocket, deps: UiHandlerDeps): void { + private onConnection(ws: WsSocket, deps: UiHandlerDeps, correlationId: string): void { this.attachHeartbeat(ws); const voice = resolveVoiceCapability(currentGatewayConfig(deps) ?? { providers: [] }, { policyDisabled: isVoiceDisabledByPolicy(deps.env), @@ -896,6 +909,7 @@ class VoiceControlPlaneImpl implements VoiceControlPlane { session: resolved.state, negotiate, redact: deps.redactor, + correlationId, }); connection.start(resolved.resume); }); diff --git a/packages/keiko-server/src/workspace-index-provider.test.ts b/packages/keiko-server/src/workspace-index-provider.test.ts index c63b0b1e48..9527f483be 100644 --- a/packages/keiko-server/src/workspace-index-provider.test.ts +++ b/packages/keiko-server/src/workspace-index-provider.test.ts @@ -425,4 +425,47 @@ describe("workspace index provider", () => { rmSync(runtimeStateDir, { force: true, recursive: true }); } }); + + // ADR-0173 D5 / g12: a load failure and a save failure reported for the SAME generation used to + // each mint their own disconnected `randomUUID()`, so an operator (or an agent joining lines by + // correlation id) could never tell they were evidence about the same underlying epoch. Fails + // before the fix — two independent random UUIDs never match — and passes after, once both + // reporters read the one id minted when the generation was created. + it("keeps a generation's load and save failures joined under one correlation id", async () => { + const workspaceRoot = tempDir("keiko-index-workspace-private-"); + const runtimeStateDir = tempDir("keiko-index-state-"); + const records: ServerDiagnosticRecord[] = []; + const key = Buffer.alloc(32, 19).toString("base64"); + try { + const provider = createServerWorkspaceIndexProvider({ + runtimeStateDir, + env: { KEIKO_WORKSPACE_INDEX_KEY: key }, + diagnostics: { record: (record): void => void records.push(record) }, + }); + const index = provider(workspaceRoot); + if (index === undefined) throw new Error("expected workspace index"); + const keyShape = scopeKey(workspaceRoot); + await index.saveSnapshot(keyShape, emptySnapshot()); + const snapshotDir = join(runtimeStateDir, "workspace-index"); + const snapshotName = readdirSync(snapshotDir).find((name) => name.endsWith(".json")); + if (snapshotName === undefined) throw new Error("expected encrypted snapshot"); + const snapshotPath = join(snapshotDir, snapshotName); + + writeFileSync(snapshotPath, "{corrupt", "utf8"); + await expect(index.loadSnapshot(keyShape)).resolves.toBeUndefined(); + + unlinkSync(snapshotPath); + mkdirSync(snapshotPath); + await expect(index.saveSnapshot(keyShape, emptySnapshot())).rejects.toThrow(); + + expect(records).toHaveLength(2); + expect(records[0]).toMatchObject({ operation: "workspace.index.load" }); + expect(records[1]).toMatchObject({ operation: "workspace.index.save" }); + expect(records[0]?.correlationId).toMatch(/^[0-9a-f-]{36}$/u); + expect(records[1]?.correlationId).toBe(records[0]?.correlationId); + } finally { + rmSync(workspaceRoot, { force: true, recursive: true }); + rmSync(runtimeStateDir, { force: true, recursive: true }); + } + }); }); diff --git a/packages/keiko-server/src/workspace-index-provider.ts b/packages/keiko-server/src/workspace-index-provider.ts index 8d885c66b2..92bc968074 100644 --- a/packages/keiko-server/src/workspace-index-provider.ts +++ b/packages/keiko-server/src/workspace-index-provider.ts @@ -46,6 +46,11 @@ interface CachedWorkspaceIndex { interface WorkspaceIndexGeneration { active: boolean; reported: boolean; + // One id per generation (one workspace-root + key-fingerprint epoch), minted once when the + // generation is created — never re-minted per failure. A generation's load failure, save failure + // and eventual stale-generation report are all evidence about the SAME underlying epoch, so they + // must stay joinable under one id instead of each reporter drawing its own disconnected one. + readonly correlationId: string; } interface RuntimeWorkspaceIndexGeneration { @@ -141,14 +146,19 @@ function workspaceIndexKeyFingerprint(key: Buffer): string { return createHash("sha256").update(key).digest("hex"); } +// `correlationId` is minted once by the caller, at the START of whichever operation this failure +// belongs to (the provider-lookup attempt for `reportWorkspaceIndexFailure`, or the owning +// generation for the other three) — never freshly here, so a cascade of related diagnostics about +// the SAME attempt or the SAME generation stays joinable under one id (ADR-0173 D5 / g12). function reportWorkspaceIndexFailure( options: ServerWorkspaceIndexProviderOptions, + correlationId: string, error: unknown, ): void { emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId, operation: "workspace.index.open", source: "workspace-index-provider", error, @@ -159,12 +169,13 @@ function reportWorkspaceIndexFailure( function reportWorkspaceIndexLoadFailure( options: ServerWorkspaceIndexProviderOptions, + generation: WorkspaceIndexGeneration, reason: WorkspaceIndexLoadFailureReason, ): void { emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId: generation.correlationId, operation: "workspace.index.load", source: "workspace-index-provider", error: new WorkspaceIndexSnapshotLoadError(reason), @@ -173,11 +184,14 @@ function reportWorkspaceIndexLoadFailure( ); } -function reportWorkspaceIndexSaveFailure(options: ServerWorkspaceIndexProviderOptions): void { +function reportWorkspaceIndexSaveFailure( + options: ServerWorkspaceIndexProviderOptions, + generation: WorkspaceIndexGeneration, +): void { emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId: generation.correlationId, operation: "workspace.index.save", source: "workspace-index-provider", error: new WorkspaceIndexSnapshotSaveError(), @@ -195,7 +209,7 @@ function reportStaleWorkspaceIndexGeneration( emitServerDiagnostic( options.diagnostics, serverDiagnosticFromError({ - correlationId: randomUUID(), + correlationId: generation.correlationId, operation: "workspace.index.generation", source: "workspace-index-provider", error: new WorkspaceIndexKeyRotatedError(), @@ -243,7 +257,11 @@ function activeRuntimeGeneration( return existing.generation; } if (existing !== undefined) existing.generation.active = false; - const generation: WorkspaceIndexGeneration = { active: true, reported: false }; + const generation: WorkspaceIndexGeneration = { + active: true, + reported: false, + correlationId: randomUUID(), + }; generations.set(runtimeDir, { generation, keyFingerprint }); return generation; } @@ -270,10 +288,10 @@ function createGenerationWorkspaceIndex( encryptionKey: key, isGenerationActive: (): boolean => generation.active, onLoadFailure: (failure): void => { - reportWorkspaceIndexLoadFailure(options, failure.reason); + reportWorkspaceIndexLoadFailure(options, generation, failure.reason); }, onSaveFailure: (): void => { - reportWorkspaceIndexSaveFailure(options); + reportWorkspaceIndexSaveFailure(options, generation); }, }), ); @@ -292,6 +310,10 @@ export function createServerWorkspaceIndexProvider( } const cacheKey = `${resolve(workspaceRoot)}\u0000${runtimeDir}`; const existing = indexes.get(cacheKey); + // Minted once, at the start of THIS lookup attempt, rather than inside the catch below: the + // attempt has exactly one failure point (key resolution or generation construction), so this + // is the id that failure — and only that failure — is reported under. + const correlationId = randomUUID(); try { const { key } = resolveLocalVaultKey({ env: options.env ?? process.env, @@ -319,7 +341,7 @@ export function createServerWorkspaceIndexProvider( } catch (error) { if (existing !== undefined) existing.generation.active = false; retireRuntimeGeneration(generations, runtimeDir); - reportWorkspaceIndexFailure(options, error); + reportWorkspaceIndexFailure(options, correlationId, error); return undefined; } }; From fe1e77e4f62105ae2764d71a7b8866b981dc7b0c Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 12:48:28 +0200 Subject: [PATCH 12/19] =?UTF-8?q?fix(observability):=20Wave=204a=20review?= =?UTF-8?q?=20repairs=20=E2=80=94=20fail-closed=20fingerprint=20guard,=20r?= =?UTF-8?q?ead-only=20keychain=20tier,=20body-free=20vault=20events,=20eff?= =?UTF-8?q?ective=20fingerprint=20delta=20(#3239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 16 review findings repaired with fails-before/passes-after tests, plus the producer/exporter inconsistency the acceptance test exposed: a corrupt vault is reported plaintext without a keySource, and a guard-rejected fingerprint is named in storesUnavailable (invalid-fingerprint) instead of being dropped. Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 8 +- packages/keiko-cli/src/support-export.test.ts | 21 +++ packages/keiko-cli/src/support-export.ts | 42 ++++- packages/keiko-cli/src/ui.test.ts | 19 ++- .../src/store-fingerprint.test.ts | 64 ++++++- .../keiko-contracts/src/store-fingerprint.ts | 42 ++++- .../src/repository-pod.test.ts | 79 +++++++++ .../src/repository-pod.ts | 24 ++- .../src/retrieval/usearch-ann-index.test.ts | 55 +----- .../usearch-runtime-resolved-logging.test.ts | 160 ++++++++++++++++++ .../keiko-memory-vault/src/cipher.test.ts | 48 +++++- packages/keiko-memory-vault/src/cipher.ts | 41 ++++- packages/keiko-memory-vault/src/db.test.ts | 21 +++ packages/keiko-memory-vault/src/db.ts | 17 +- packages/keiko-memory-vault/src/index.test.ts | 24 +++ packages/keiko-memory-vault/src/index.ts | 17 +- .../src/migrate-encrypt.test.ts | 22 ++- .../keiko-memory-vault/src/vault-log.test.ts | 30 +++- packages/keiko-memory-vault/src/vault-log.ts | 42 ++++- packages/keiko-memory-vault/src/vault.test.ts | 33 ++++ packages/keiko-memory-vault/src/vault.ts | 7 +- packages/keiko-security/src/log-port.test.ts | 11 +- .../secret-vault.fs-fault-injection.test.ts | 9 + .../keiko-security/src/secret-vault.test.ts | 2 +- packages/keiko-security/src/secret-vault.ts | 13 +- .../src/credentialPersistence.test.ts | 86 ++++++++++ .../keiko-server/src/credentialPersistence.ts | 21 ++- .../deps-vault-key-securitylog-wiring.test.ts | 20 ++- packages/keiko-server/src/deps.ts | 1 + ...setup-vault-key-securitylog-wiring.test.ts | 8 +- ...memory-handlers-securitylog-wiring.test.ts | 22 ++- packages/keiko-server/src/store/db.test.ts | 35 +++- packages/keiko-server/src/store/db.ts | 21 ++- 33 files changed, 941 insertions(+), 124 deletions(-) create mode 100644 packages/keiko-local-knowledge/src/retrieval/usearch-runtime-resolved-logging.test.ts create mode 100644 packages/keiko-memory-vault/src/index.test.ts create mode 100644 packages/keiko-server/src/credentialPersistence.test.ts diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index b63cb0ce17..cdf161a844 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -329,7 +329,7 @@ { "op": "memory-vault.log.sink-failed", "category": "diagnostic", - "site": "packages/keiko-memory-vault/src/vault-log.ts:140", + "site": "packages/keiko-memory-vault/src/vault-log.ts:174", "package": "keiko-memory-vault" }, { @@ -659,13 +659,13 @@ { "op": "security.vault.key-resolved", "category": "security", - "site": "packages/keiko-security/src/secret-vault.ts:272", + "site": "packages/keiko-security/src/secret-vault.ts:275", "package": "keiko-security" }, { "op": "security.vault.shard-unreadable", "category": "security", - "site": "packages/keiko-security/src/secret-vault.ts:537", + "site": "packages/keiko-security/src/secret-vault.ts:540", "package": "keiko-security" }, { @@ -875,7 +875,7 @@ { "op": "store.opened", "category": "setup", - "site": "packages/keiko-server/src/store/db.ts:919", + "site": "packages/keiko-server/src/store/db.ts:928", "package": "keiko-server" } ], diff --git a/packages/keiko-cli/src/support-export.test.ts b/packages/keiko-cli/src/support-export.test.ts index 5eb554a448..bb39bb6ea0 100644 --- a/packages/keiko-cli/src/support-export.test.ts +++ b/packages/keiko-cli/src/support-export.test.ts @@ -295,6 +295,27 @@ describe("describeErrorKind", () => { }); describe("buildSupportBundleManifest", () => { + it("names a fingerprint that fails the contract guard in storesUnavailable instead of dropping it", () => { + // A store that vanished from both lists would read as "never used" — the one claim a support + // bundle must not make about a store that exists (Wave 4a acceptance regression). + const contradictory = { + store: "memory-vault", + schemaVersion: 0, + migrationsApplied: [], + tableRowCounts: {}, + quickCheckOk: false, + encryptionMode: "plaintext", + keySource: "env", + } as unknown as StoreFingerprint; + const manifest = buildSupportBundleManifest( + baseManifestInput({ storeFingerprints: [contradictory], storesUnavailable: [] }), + ); + expect(manifest.storeFingerprints).toEqual([]); + expect(manifest.storesUnavailable).toEqual([ + { store: "memory-vault", reasonKind: "invalid-fingerprint" }, + ]); + }); + it("produces the exact manifest shape for the minimal Wave 1 bundle", () => { const manifest = buildSupportBundleManifest( baseManifestInput({ diff --git a/packages/keiko-cli/src/support-export.ts b/packages/keiko-cli/src/support-export.ts index 9768146510..4195b51a97 100644 --- a/packages/keiko-cli/src/support-export.ts +++ b/packages/keiko-cli/src/support-export.ts @@ -238,7 +238,11 @@ function redactedAuditSummary(audit: AuditResult): RedactedAuditSummary { // this state dir) or that cannot be opened (corrupt, or a vault key the operator has not supplied) // contributes no fingerprint — its name and a closed-vocabulary reason go to `storesUnavailable` // instead, never a path or the underlying error's message. -export type StoreUnavailableReasonKind = "missing" | "open-failed"; +// `invalid-fingerprint`: the collector handed the exporter an object that fails the contract's +// `isStoreFingerprint` guard. The exporter never embeds such an object — and never drops it +// silently either: a store that disappears from both manifest lists would read as "never used", +// which is the one thing a support bundle must not say about a store that exists. +export type StoreUnavailableReasonKind = "missing" | "open-failed" | "invalid-fingerprint"; export interface StoreUnavailableEntry { readonly store: StoreFingerprint["store"]; @@ -309,7 +313,39 @@ export interface ManifestInput { readonly storesUnavailable: readonly StoreUnavailableEntry[]; } +const KNOWN_STORES: ReadonlySet = new Set(["ui", "local-knowledge", "memory-vault"]); + +function knownStoreName(value: unknown): StoreFingerprint["store"] | undefined { + if (typeof value !== "object" || value === null) return undefined; + const store: unknown = Reflect.get(value, "store"); + return typeof store === "string" && KNOWN_STORES.has(store) + ? (store as StoreFingerprint["store"]) + : undefined; +} + +// Every collected fingerprint either passes the contract guard and is embedded, or is named in +// `storesUnavailable` with `invalid-fingerprint` — never silently dropped. An object that does not +// even carry a known store name cannot be attributed and is the one case that is dropped, counted +// by nothing: the collector's closed `store` union makes it unreachable from production code. +function partitionManifestFingerprints(input: ManifestInput): { + readonly fingerprints: readonly StoreFingerprint[]; + readonly unavailable: readonly StoreUnavailableEntry[]; +} { + const fingerprints: StoreFingerprint[] = []; + const unavailable: StoreUnavailableEntry[] = [...input.storesUnavailable]; + for (const candidate of input.storeFingerprints) { + if (isStoreFingerprint(candidate)) { + fingerprints.push(candidate); + continue; + } + const store = knownStoreName(candidate); + if (store !== undefined) unavailable.push({ store, reasonKind: "invalid-fingerprint" }); + } + return { fingerprints, unavailable }; +} + export function buildSupportBundleManifest(input: ManifestInput): SupportBundleManifest { + const partitioned = partitionManifestFingerprints(input); return { $section: "manifest", schemaVersion: input.schemaVersion, @@ -332,8 +368,8 @@ export function buildSupportBundleManifest(input: ManifestInput): SupportBundleM // doctrine): re-validated against the same closed structural guard the manifest's own bundle // reader would use, so a malformed fingerprint (a future producer bug, a version-skewed // dependency) is silently dropped rather than embedded in a customer-facing artifact. - storeFingerprints: input.storeFingerprints.filter(isStoreFingerprint), - storesUnavailable: input.storesUnavailable, + storeFingerprints: partitioned.fingerprints, + storesUnavailable: partitioned.unavailable, }; } diff --git a/packages/keiko-cli/src/ui.test.ts b/packages/keiko-cli/src/ui.test.ts index 761130f5bf..2b5e9f38e9 100644 --- a/packages/keiko-cli/src/ui.test.ts +++ b/packages/keiko-cli/src/ui.test.ts @@ -945,6 +945,8 @@ describe("runUiCli — node:sqlite re-exec guard (ADR-0013 D2)", () => { child.kill = (): void => { /* no-op */ }; + const sigintBefore = process.listenerCount("SIGINT"); + const sigtermBefore = process.listenerCount("SIGTERM"); queueMicrotask(() => { child.emit("error", new Error("spawn EMFILE")); }); @@ -959,8 +961,10 @@ describe("runUiCli — node:sqlite re-exec guard (ADR-0013 D2)", () => { }, ); expect(code).toBe(1); - // The signal forwarders must be gone (no listener leak after the failure). - expect(child.listenerCount("exit")).toBeGreaterThanOrEqual(0); + // The signal forwarders must be gone (no listener leak after the failure): the + // process-level SIGINT/SIGTERM listener counts must be back to their pre-call baseline. + expect(process.listenerCount("SIGINT")).toBe(sigintBefore); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore); }); // Without these forwarders, a Ctrl-C during re-exec kills the PARENT (whose own SIGINT @@ -976,6 +980,8 @@ describe("runUiCli — node:sqlite re-exec guard (ADR-0013 D2)", () => { killedWith.push(signal); return true; }; + const sigintBefore = process.listenerCount("SIGINT"); + const sigtermBefore = process.listenerCount("SIGTERM"); const promise = runUiCli( [], @@ -992,14 +998,19 @@ describe("runUiCli — node:sqlite re-exec guard (ADR-0013 D2)", () => { // guard runs synchronously (spawnFn is called synchronously, and nothing awaits before the // listeners are registered) — so both are already attached the instant `runUiCli` yields its // pending promise back to this line, with no need to wait a tick first. + // The forwarders are registered: one SIGINT and one SIGTERM listener added on `process`. + expect(process.listenerCount("SIGINT")).toBe(sigintBefore + 1); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore + 1); process.emit("SIGINT"); process.emit("SIGTERM"); expect(killedWith).toEqual(["SIGINT", "SIGTERM"]); child.emit("exit", 0, null); expect(await promise).toBe(0); - // The forwarders must be gone once the child has exited (no listener leak). - expect(child.listenerCount("exit")).toBeGreaterThanOrEqual(0); + // The forwarders must be gone once the child has exited (no listener leak): the + // process-level SIGINT/SIGTERM listener counts must be back to their pre-call baseline. + expect(process.listenerCount("SIGINT")).toBe(sigintBefore); + expect(process.listenerCount("SIGTERM")).toBe(sigtermBefore); }); // Regression pin (KEIKO-0443): the three "does not re-exec" tests below previously used the diff --git a/packages/keiko-contracts/src/store-fingerprint.test.ts b/packages/keiko-contracts/src/store-fingerprint.test.ts index dd9f55c472..1a3ca789eb 100644 --- a/packages/keiko-contracts/src/store-fingerprint.test.ts +++ b/packages/keiko-contracts/src/store-fingerprint.test.ts @@ -26,18 +26,39 @@ describe("isStoreFingerprint", () => { expect(isStoreFingerprint({ ...rest, encryptionMode: "plaintext" })).toBe(true); }); - it("accepts every closed-vocabulary store, encryptionMode, and keySource value", () => { + it("accepts every closed-vocabulary store value", () => { for (const store of ["ui", "local-knowledge", "memory-vault"] as const) { expect(isStoreFingerprint({ ...validFingerprint(), store })).toBe(true); } - for (const encryptionMode of ["plaintext", "encrypted", "migrating"] as const) { + }); + + it("accepts every closed-vocabulary encryptionMode value paired with a consistent keySource", () => { + for (const encryptionMode of ["encrypted", "migrating"] as const) { expect(isStoreFingerprint({ ...validFingerprint(), encryptionMode })).toBe(true); } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { keySource: _keySource, ...rest } = validFingerprint(); + expect(isStoreFingerprint({ ...rest, encryptionMode: "plaintext" })).toBe(true); + }); + + it("accepts every closed-vocabulary keySource value", () => { for (const keySource of ["env", "keychain", "keyfile"] as const) { expect(isStoreFingerprint({ ...validFingerprint(), keySource })).toBe(true); } }); + it("rejects a plaintext fingerprint that still carries a keySource", () => { + // A `"plaintext"` store never resolved a key, so `keySource` riding along on it is the exact + // contradictory state `encryptionMode`'s own doc comment rules out. + expect( + isStoreFingerprint({ + ...validFingerprint(), + encryptionMode: "plaintext", + keySource: "keychain", + }), + ).toBe(false); + }); + it("rejects a non-object, null, and an array", () => { expect(isStoreFingerprint(undefined)).toBe(false); expect(isStoreFingerprint(null)).toBe(false); @@ -111,4 +132,43 @@ describe("isStoreFingerprint", () => { const { store: _store, ...withoutStore } = validFingerprint(); expect(isStoreFingerprint(withoutStore)).toBe(false); }); + + it("fails closed instead of throwing when top-level key enumeration is hostile", () => { + const hostile = new Proxy( + { ...validFingerprint() }, + { + ownKeys(): never { + throw new Error("hostile ownKeys trap"); + }, + }, + ); + expect(isStoreFingerprint(hostile)).toBe(false); + }); + + it("fails closed instead of throwing when a required field getter is hostile", () => { + const hostile = new Proxy( + { ...validFingerprint() }, + { + get(target: object, prop: string | symbol, receiver: unknown): unknown { + if (prop === "store") throw new Error("hostile store getter"); + return Reflect.get(target, prop, receiver); + }, + }, + ); + expect(isStoreFingerprint(hostile)).toBe(false); + }); + + it("fails closed instead of throwing when tableRowCounts enumeration is hostile", () => { + const hostileTableRowCounts = new Proxy( + {}, + { + ownKeys(): never { + throw new Error("hostile nested ownKeys trap"); + }, + }, + ); + expect( + isStoreFingerprint({ ...validFingerprint(), tableRowCounts: hostileTableRowCounts }), + ).toBe(false); + }); }); diff --git a/packages/keiko-contracts/src/store-fingerprint.ts b/packages/keiko-contracts/src/store-fingerprint.ts index 2dc301e58e..c18e408303 100644 --- a/packages/keiko-contracts/src/store-fingerprint.ts +++ b/packages/keiko-contracts/src/store-fingerprint.ts @@ -8,7 +8,9 @@ // repository's own fixed table/migration lists — never a row, a path, a key, a secret, or free // text (ADR-0128 D6 redaction vocabulary). `isStoreFingerprint` lets the manifest assembler // refuse a malformed value before it is embedded, mirroring the fail-closed guard style already -// established in `atlassian-connectors-validation.ts`. +// established in `atlassian-connectors-validation.ts`. The guard also fails closed against a +// hostile value whose property enumeration/access throws (a proxy trap, a throwing getter), and +// rejects `encryptionMode`/`keySource` combinations that contradict each other. // // Leaf-package rule (ADR-0019 direction 1): no `@oscharko-dev/keiko-*` imports, pure functions // only, zero logic beyond the shape and its guard. @@ -32,7 +34,13 @@ export interface StoreFingerprint { /** `PRAGMA quick_check` summary — pass/fail only, never the raw check output. */ readonly quickCheckOk: boolean; readonly encryptionMode: "plaintext" | "encrypted" | "migrating"; - /** The already-computed-then-discarded key-resolution tier, when the store is encrypted. */ + /** + * The already-computed-then-discarded key-resolution tier. May be present only when + * `encryptionMode` is `"encrypted"` or `"migrating"` — a `"plaintext"` store has no key + * material to report, and `isStoreFingerprint` rejects the two fields combined that way, even + * though a store may legitimately report `"encrypted"`/`"migrating"` with `keySource` omitted + * (e.g. `local-knowledge`, whose key provider has no key-resolution-tier concept to report). + */ readonly keySource?: "env" | "keychain" | "keyfile" | undefined; } @@ -129,21 +137,41 @@ function isStoreFingerprintCore(value: Record): boolean { ); } +// `keySource` documents a key-resolution tier, which only exists once a key is in play — a +// `"plaintext"` store never resolved one, so the two fields riding together is the exact +// contradictory state `readStoreEncryptionMode`/`computeStoreFingerprint` producers must never +// emit. The reverse is NOT required: `local-knowledge` legitimately reports `"encrypted"` with +// `keySource` omitted (its key provider has no key-resolution-tier concept), so this only checks +// the one direction. +function isConsistentEncryptionKeySource(value: Record): boolean { + if (value.keySource === undefined) return true; + return value.encryptionMode === "encrypted" || value.encryptionMode === "migrating"; +} + function isStoreFingerprintEncryptionShape(value: Record): boolean { return ( typeof value.quickCheckOk === "boolean" && isStoreFingerprintEncryptionMode(value.encryptionMode) && - isOptionalStoreFingerprintKeySource(value.keySource) + isOptionalStoreFingerprintKeySource(value.keySource) && + isConsistentEncryptionKeySource(value) ); } /** * Fail-closed structural guard for {@link StoreFingerprint}: every union is checked against its * closed vocabulary, every count is a finite non-negative integer, every table/migration name is - * a bounded identifier, and no unexpected field rides through. The manifest assembler uses this - * to refuse a malformed fingerprint rather than embed it. + * a bounded identifier, `encryptionMode`/`keySource` cannot contradict each other, and no + * unexpected field rides through. The manifest assembler uses this to refuse a malformed + * fingerprint rather than embed it. Wrapped in `try`/`catch`: a hostile producer value can make + * property enumeration or access (`Object.keys`, a field getter, `Object.entries` on a nested + * object) throw via a proxy trap or a throwing accessor — this guard must fail closed (`false`), + * never propagate, so one malformed value cannot abort manifest assembly. */ export function isStoreFingerprint(value: unknown): value is StoreFingerprint { - if (!isRecord(value) || !hasKnownStoreFingerprintKeys(value)) return false; - return isStoreFingerprintCore(value) && isStoreFingerprintEncryptionShape(value); + try { + if (!isRecord(value) || !hasKnownStoreFingerprintKeys(value)) return false; + return isStoreFingerprintCore(value) && isStoreFingerprintEncryptionShape(value); + } catch { + return false; + } } diff --git a/packages/keiko-local-knowledge/src/repository-pod.test.ts b/packages/keiko-local-knowledge/src/repository-pod.test.ts index 4b3b67e355..4d52c01958 100644 --- a/packages/keiko-local-knowledge/src/repository-pod.test.ts +++ b/packages/keiko-local-knowledge/src/repository-pod.test.ts @@ -22,7 +22,9 @@ import { import type { GatewayRequest, NormalizedResponse, + OpenAIEmbeddingAdapter, OpenAIEmbeddingOutcome, + OpenAIEmbeddingRequest, } from "@oscharko-dev/keiko-model-gateway"; import { nodeWorkspaceFs } from "@oscharko-dev/keiko-workspace/internal/fs"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -574,6 +576,83 @@ describe("repository pod fingerprint-diff activity log", () => { }); }); + it("suppresses removed in the logged event too, not only in the persisted counts, when enumeration is incomplete", async () => { + // A bounded scan (maxFiles) sees only a slice of the tree, so a diff against the full prior + // baseline manufactures large, spurious "removed" entries for every previously-known file the + // bound cut off — none of which are actually gone. `counts.removedFiles` already suppresses + // this to 0; the logged `repository.fingerprint-diff.completed` event must report the same + // suppressed number, not the raw (and here, wildly wrong) delta. + createShell(); + const adapter = countingAdapter(); + await refreshRepositoryPod(indexingDeps(adapter), { runId: "fp-diff-bounded-initial" }); + const { sink, events } = recordingSink(); + + const incomplete = await refreshRepositoryPod( + indexingDeps(adapter, { + discoveryOptions: { maxDepth: 12, maxFiles: 1 }, + logSink: sink, + }), + { runId: "fp-diff-bounded-incomplete" }, + ); + + expect(incomplete.run.applied).toBe(false); + expect(incomplete.run.counts.removedFiles).toBe(0); + const line = events.find((event) => event.op === "repository.fingerprint-diff.completed"); + expect(line?.extra).toMatchObject({ removed: 0 }); + }); + + it("reports the effective delta — not the raw scan — when a discovered file fails to embed", async () => { + // `scan.fingerprints` includes every file the walk finds, including one whose embedding call + // fails afterward; `next` (the persisted baseline) deliberately withholds that path so the + // next run retries it. The reported delta must come from `next`, not from the raw scan, or a + // file that never made it into the persisted baseline gets counted as "added" anyway. + createShell(); + writeFileSync( + join(repositoryRoot, "src", "flaky.ts"), + "export const flakyEmbedMarker = 1;\n", + "utf8", + ); + const { sink, events } = recordingSink(); + const adapter: OpenAIEmbeddingAdapter = { + endpoint: "https://example.test/v1", + apiKey: "sk-test", + request: (request: OpenAIEmbeddingRequest): Promise => + Promise.resolve( + request.input.includes("flakyEmbedMarker") + ? { ok: false, kind: "unsupported-model" } + : { + ok: true, + value: { vector: VECTOR, modelId: DEFAULT_EMBEDDING.modelId }, + }, + ), + }; + + const result = await refreshRepositoryPod( + { + store, + capsuleId: CAPSULE_ID, + sourceId: SOURCE_ID, + parserRegistry: createDefaultParserRegistry(), + embeddingAdapter: adapter, + workspaceFs: nodeWorkspaceFs, + trackedPaths: TRACKED_PATHS, + logSink: sink, + }, + { runId: "fp-diff-withheld-failure" }, + ); + + expect(result.run.applied).toBe(true); + expect(documentRows().map((row) => row.document_path)).toContain("src/flaky.ts"); + // 4 files (.gitignore, app.ts, service.py, worker.go) persist into the baseline; + // flaky.ts failed to embed and is withheld, so it must NOT be counted as "added". + expect(result.run.counts.addedFiles).toBe(4); + const line = events.find((event) => event.op === "repository.fingerprint-diff.completed"); + expect(line?.extra).toMatchObject({ added: 4 }); + expect([...readRepositoryFileFingerprints(store, CAPSULE_ID, SOURCE_ID).keys()]).not.toContain( + "src/flaky.ts", + ); + }); + it("writes nothing when no logSink is supplied", async () => { createShell(); const adapter = countingAdapter(); diff --git a/packages/keiko-local-knowledge/src/repository-pod.ts b/packages/keiko-local-knowledge/src/repository-pod.ts index 87401306ab..b3dddcf689 100644 --- a/packages/keiko-local-knowledge/src/repository-pod.ts +++ b/packages/keiko-local-knowledge/src/repository-pod.ts @@ -350,6 +350,22 @@ function logFingerprintDiffCompleted( }); } +// The one delta this run reports — from `prior` to the EFFECTIVE next baseline (the +// failure-withheld set the caller is about to persist, never the raw scan), with `removed` +// suppressed to 0 whenever enumeration is incomplete. Computed once and reused for both the +// logged event and the persisted `RepositoryPodChangeCounts` so the two can never disagree with +// each other about whether something was removed. +function effectiveFingerprintDelta( + prior: ReadonlyMap, + next: readonly RepositoryFileFingerprint[], + enumerationComplete: boolean, +): FingerprintSetDelta { + const delta = diffFingerprintSets(fingerprintMap([...prior.values()]), fingerprintMap(next), { + detectMoves: false, + }); + return enumerationComplete ? delta : { ...delta, removed: 0 }; +} + function runCounts( deps: RepositoryPodDeps, runId: string, @@ -359,14 +375,12 @@ function runCounts( rejectedEntries: number, enumerationComplete: boolean, ): RepositoryPodChangeCounts { - const delta = diffFingerprintSets(fingerprintMap([...prior.values()]), fingerprintMap(next), { - detectMoves: false, - }); + const delta = effectiveFingerprintDelta(prior, next, enumerationComplete); logFingerprintDiffCompleted(deps, runId, delta); return { addedFiles: delta.added, changedFiles: delta.changed, - removedFiles: enumerationComplete ? delta.removed : 0, + removedFiles: delta.removed, unchangedFiles: delta.unchanged, failedDocuments: result.failedDocuments, rejectedEntries, @@ -473,7 +487,7 @@ export async function refreshRepositoryPod( deps, runId, prior, - scan.fingerprints, + next, drained.result, scan.rejectedEntries, enumerationComplete, diff --git a/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts b/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts index 32b8a551b4..51ab24ea0f 100644 --- a/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts +++ b/packages/keiko-local-knowledge/src/retrieval/usearch-ann-index.test.ts @@ -5,8 +5,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { EmbeddingModelIdentity } from "@oscharko-dev/keiko-contracts"; -import type { KnowledgeLogEvent, KnowledgeLogSink } from "../knowledge-log.js"; - import { __resetTargetRuntimeCacheForTests, clearUsearchAnnCacheForTests, @@ -404,51 +402,14 @@ describe("USearch ANN index", () => { expect(readsAfterWarm).toBe(readsAfterCold); }); - it("logs search.native-runtime-resolved on the cold path only, content-free", async () => { - const corpus = clusteredCorpus(64, IDENTITY); - const queryVector = corpus.entries[0]?.vector; - if (queryVector === undefined) throw new Error("test corpus must contain a query vector"); - const binary = runtimePath(); - const events: KnowledgeLogEvent[] = []; - const logSink: KnowledgeLogSink = { - write: (event): void => { - events.push(event); - }, - }; - const request = { - partition: partition(corpus.entries, "native-runtime-resolved-logging"), - queryVector, - candidateLimit: 5, - exactScanThreshold: 0, - binaryPath: binary, - logSink, - }; - __resetTargetRuntimeCacheForTests(); - - const cold = await searchUsearchAnnIndex(request); - expect(cold.ok).toBe(true); - const coldLines = events.filter((event) => event.op === "search.native-runtime-resolved"); - // resolvedIndex() and buildSearchIndex() both call targetRuntime() for the same request; - // the second call is already a warm hit against the cache the first call just populated, - // so exactly one line — not two — is written for one logical search call. - expect(coldLines).toHaveLength(1); - expect(coldLines[0]).toMatchObject({ - level: "info", - category: "search", - extra: { - targetKey: usearchRuntimeTargetKey(process.platform, process.arch), - resolved: true, - version: USEARCH_RUNTIME_MANIFEST.version, - }, - }); - // Content-free: never the resolved filesystem path or the raw SHA-256 digest. - expect(JSON.stringify(coldLines[0])).not.toContain(binary); - - events.length = 0; - const warm = await searchUsearchAnnIndex(request); - expect(warm.ok).toBe(true); - expect(events.filter((event) => event.op === "search.native-runtime-resolved")).toHaveLength(0); - }); + // The "logs search.native-runtime-resolved on the cold path only, content-free" case moved to + // ./usearch-runtime-resolved-logging.test.ts: it needs a mocked runtime-manifest approval to + // run hermetically (no host-native USearch binary, no order dependency on this file's shared + // TARGET_RUNTIME_CACHE), which would have required either a per-file vi.mock here — poisoning + // every other real-binary test in this suite — or a scoped vi.doMock reimport that was harder + // to reason about than a small dedicated file. Native-addon search correctness stays covered + // right here by every other test in this file that already exercises the real binary via + // runtimePath(). it("serializes concurrent callers through queryQueue without cross-contaminating results (KEIKO-0360)", async () => { // KEIKO-0360: coverage pin for the ADR-0164 D2 single-Worker queryQueue serialization diff --git a/packages/keiko-local-knowledge/src/retrieval/usearch-runtime-resolved-logging.test.ts b/packages/keiko-local-knowledge/src/retrieval/usearch-runtime-resolved-logging.test.ts new file mode 100644 index 0000000000..dfacc57aae --- /dev/null +++ b/packages/keiko-local-knowledge/src/retrieval/usearch-runtime-resolved-logging.test.ts @@ -0,0 +1,160 @@ +// Hermetic split-out of the "logs search.native-runtime-resolved" case that used to live in +// usearch-ann-index.test.ts (finding: KEIKO PR #3244, thread contracts-lk-4). That version +// required the real host-native USearch binary (`runtimePath()`, needing `npm run +// provision:usearch`) and mutated the module-level `TARGET_RUNTIME_CACHE` without restoring it, +// making the test both host-dependent and order-dependent. +// +// This file mocks `./usearch-runtime-manifest.js` so `usearchRuntimeApproval` accepts a small, +// fully-owned fixture file instead of the real multi-MB compiled addon. That mock would poison +// every real-binary test in usearch-ann-index.test.ts if it lived in the same file (they assert +// against the REAL approved SHA-256), which is why this is a separate file rather than a +// `vi.doMock` scoped to one `it` there — native-addon search correctness stays proven by that +// file's own tests, unaffected by this one. +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { EmbeddingModelIdentity } from "@oscharko-dev/keiko-contracts"; + +import type { KnowledgeLogEvent, KnowledgeLogSink } from "../knowledge-log.js"; + +import { + __resetTargetRuntimeCacheForTests, + clearUsearchAnnCacheForTests, + searchUsearchAnnIndex, +} from "./usearch-ann-index.js"; + +// A fixture binary the test owns outright — never the real compiled USearch addon. Its SHA-256 +// is precomputed (not re-derived from the production formula: this is INPUT data the mocked +// approval below is built to accept, not a value `verifyRuntimeAt` is expected to compute +// differently). `vi.mock` factories are hoisted above every other top-level statement, so the +// fixture constants themselves are wrapped in `vi.hoisted` to be visible inside the factory +// without a temporal-dead-zone reference error. +const { FIXTURE_BINARY_CONTENT, FIXTURE_BINARY_SHA256, FIXTURE_VERSION } = vi.hoisted(() => ({ + FIXTURE_BINARY_CONTENT: "keiko-usearch-runtime-logging-fixture", + FIXTURE_BINARY_SHA256: "075f581c1183403f7c20a432fddc5ac84e700f8e39bd518dffcf297b69b5ce08", + FIXTURE_VERSION: "usearch-runtime-fixture-1.0.0", +})); + +vi.mock("./usearch-runtime-manifest.js", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + usearchRuntimeApproval: ( + targetKey: string | undefined, + ): Readonly> | undefined => + targetKey === undefined + ? undefined + : Object.freeze({ + version: FIXTURE_VERSION, + sourceCommit: "0".repeat(40), + tarballUrl: "https://example.test/usearch-fixture.tgz", + tarballSha256: "0".repeat(64), + licenseSha256: "0".repeat(64), + archivePath: "fixture/usearch.node", + binarySha256: FIXTURE_BINARY_SHA256, + }), + }; +}); + +const IDENTITY: EmbeddingModelIdentity = { + provider: "test", + modelId: "deterministic-64", + vectorDimensions: 4, + vectorMetric: "cosine", + normalization: "l2", + instructionVersion: "v1", + embeddingSpaceFingerprint: "space-v1", +}; + +function twoEntryPartition(): { + readonly partition: Parameters[0]["partition"]; + readonly queryVector: Float32Array; +} { + const entries = [ + { id: "a", vector: Float32Array.from([1, 0, 0, 0]) }, + { id: "b", vector: Float32Array.from([0, 1, 0, 0]) }, + ]; + return { + partition: { + cacheKey: "runtime-logging-partition", + cacheGroupKey: "runtime-logging-owner", + revision: "revision-1", + identity: IDENTITY, + rowCount: entries.length, + loadEntries: () => entries, + }, + queryVector: Float32Array.from([1, 0, 0, 0]), + }; +} + +afterEach(() => { + clearUsearchAnnCacheForTests(); + __resetTargetRuntimeCacheForTests(); +}); + +describe("USearch ANN index — native-runtime-resolved logging (hermetic)", () => { + it("logs search.native-runtime-resolved on the cold path only, content-free", async () => { + const fixtureDir = mkdtempSync(join(tmpdir(), "keiko-usearch-runtime-fixture-")); + const binary = join(fixtureDir, "usearch.node"); + writeFileSync(binary, FIXTURE_BINARY_CONTENT, "utf8"); + // Prove the fixture is what it claims to be, independent of the mocked approval object — + // if this ever drifted, the "resolved: true" assertions below would silently start + // exercising the "invalid" branch instead of the success branch they are named for. + expect(createHash("sha256").update(FIXTURE_BINARY_CONTENT).digest("hex")).toBe( + FIXTURE_BINARY_SHA256, + ); + const events: KnowledgeLogEvent[] = []; + const logSink: KnowledgeLogSink = { + write: (event): void => { + events.push(event); + }, + }; + const { partition, queryVector } = twoEntryPartition(); + const request = { + partition, + queryVector, + candidateLimit: 5, + exactScanThreshold: 0, + binaryPath: binary, + logSink, + }; + __resetTargetRuntimeCacheForTests(); + + try { + // The fixture verifies successfully (its hash matches the mocked approval), so + // targetRuntime() logs "resolved: true" before the ANN worker ever starts. The worker + // then fails to load the fixture as a real native addon — expected and irrelevant here, + // since this test is about the logging boundary, not ANN search correctness (which the + // real-binary tests in usearch-ann-index.test.ts already cover). + await searchUsearchAnnIndex(request); + const coldLines = events.filter((event) => event.op === "search.native-runtime-resolved"); + // resolvedIndex() and buildSearchIndex() both call targetRuntime() for the same request; + // the second call is already a warm hit against the cache the first call just populated, + // so exactly one line — not two — is written for one logical search call. + expect(coldLines).toHaveLength(1); + expect(coldLines[0]).toMatchObject({ + level: "info", + category: "search", + extra: { + resolved: true, + version: FIXTURE_VERSION, + }, + }); + // Content-free: never the resolved filesystem path or the raw SHA-256 digest. + expect(JSON.stringify(coldLines[0])).not.toContain(binary); + expect(JSON.stringify(coldLines[0])).not.toContain(FIXTURE_BINARY_SHA256); + + events.length = 0; + await searchUsearchAnnIndex(request); + expect(events.filter((event) => event.op === "search.native-runtime-resolved")).toHaveLength( + 0, + ); + } finally { + __resetTargetRuntimeCacheForTests(); + rmSync(fixtureDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/keiko-memory-vault/src/cipher.test.ts b/packages/keiko-memory-vault/src/cipher.test.ts index 875341d86d..6e987f0afa 100644 --- a/packages/keiko-memory-vault/src/cipher.test.ts +++ b/packages/keiko-memory-vault/src/cipher.test.ts @@ -14,6 +14,7 @@ import { randomBytes } from "node:crypto"; import { createMemoryContentCipher, keyFromKeychain, + keyFromKeychainReadOnly, NO_KEYCHAIN, resolveVaultKey, resolveVaultKeyReadOnly, @@ -179,9 +180,10 @@ describe("resolveVaultKey — keychain tier", () => { describe("resolveVaultKeyReadOnly (Finding 0 — read-only diagnostic export seam)", () => { it("never writes a keyfile when the env and keychain tiers both miss, and reports no key", () => { - const resolved = resolveVaultKeyReadOnly({}, () => undefined); + const resolved = resolveVaultKeyReadOnly({}, () => ({ key: undefined, malformed: false })); expect(resolved.key).toBeUndefined(); expect(resolved.source).toBeUndefined(); + expect(resolved.keychainMalformed).toBe(false); // The RED assertion: `resolveVaultKey({}, dir, NO_KEYCHAIN)` against this same empty dir // would call `keyFromKeyfile`, which mints and writes `vault.key`. The read-only resolver // must leave the directory exactly as it found it — no file of any name appears. @@ -193,18 +195,58 @@ describe("resolveVaultKeyReadOnly (Finding 0 — read-only diagnostic export sea let keychainCalls = 0; const resolved = resolveVaultKeyReadOnly({ KEIKO_MEMORY_KEY: raw.toString("base64") }, () => { keychainCalls += 1; - return undefined; + return { key: undefined, malformed: false }; }); expect(resolved.source).toBe("env"); expect(resolved.key?.equals(raw)).toBe(true); expect(keychainCalls).toBe(0); + expect(resolved.keychainMalformed).toBe(false); }); it("reports the keychain tier when a read-only lookup finds a stored key, without minting one", () => { const stored = randomBytes(32); - const resolved = resolveVaultKeyReadOnly({}, () => stored); + const resolved = resolveVaultKeyReadOnly({}, () => ({ key: stored, malformed: false })); expect(resolved.source).toBe("keychain"); expect(resolved.key?.equals(stored)).toBe(true); + expect(resolved.keychainMalformed).toBe(false); + }); + + // Finding: Thread 5. Before this fix, a stored-but-undecodable keychain secret and an outright + // absent one both collapsed to `{ key: undefined, source: undefined }` — operationally + // indistinguishable. RED (before fix): `keychainMalformed` did not exist on + // `ResolvedVaultKeyReadOnly` at all, so this assertion could not even compile against the old + // shape; against the old boolean-less behaviour it would read `undefined`, never `true`. + it("reports keychainMalformed when a stored secret exists but fails to decode as a key", () => { + const resolved = resolveVaultKeyReadOnly({}, () => ({ key: undefined, malformed: true })); + expect(resolved.key).toBeUndefined(); + expect(resolved.source).toBeUndefined(); + expect(resolved.keychainMalformed).toBe(true); + }); + + it("end-to-end: keyFromKeychainReadOnly itself reports malformed for an undecodable stored secret", () => { + const scriptDir = mkdtempSync(join(tmpdir(), "keiko-keychain-readonly-")); + try { + const security = join(scriptDir, "security"); + writeFileSync( + security, + [ + "#!/bin/sh", + 'case "$1" in', + "find-generic-password) printf %s 'not-a-32-byte-key' ;;", + "esac", + ].join("\n"), + ); + chmodSync(security, 0o700); + + const resolved = resolveVaultKeyReadOnly({}, () => + keyFromKeychainReadOnly({ executable: security, platform: "darwin" }), + ); + expect(resolved.key).toBeUndefined(); + expect(resolved.source).toBeUndefined(); + expect(resolved.keychainMalformed).toBe(true); + } finally { + rmSync(scriptDir, { recursive: true, force: true }); + } }); }); diff --git a/packages/keiko-memory-vault/src/cipher.ts b/packages/keiko-memory-vault/src/cipher.ts index 3e4879c27c..74bfd08fab 100644 --- a/packages/keiko-memory-vault/src/cipher.ts +++ b/packages/keiko-memory-vault/src/cipher.ts @@ -96,18 +96,31 @@ function generateKeychainKey(account: string, options: MacosKeychainOptions): Bu : undefined; } +// Result of the read-only keychain lookup below. A plain `Buffer | undefined` cannot distinguish +// "no stored secret" from "a stored secret exists but failed to decode as a 32-byte key" — both +// collapsed to `undefined` — so an operator inspecting a diagnostic export's absent `keySource` +// could not tell "this tier was never used" from "this tier is broken" (coding guideline: "Errors +// must surface with enough context to diagnose"). `malformed` carries exactly that distinction and +// nothing else — no secret material, no keychain account/service name, just a boolean. +export interface ReadOnlyKeychainLookup { + readonly key: Buffer | undefined; + readonly malformed: boolean; +} + // Read-only keychain lookup for the diagnostic export seam (Wave 4a, epic #3233 §6.2): a plain // find, never `generateKeychainKey`'s find-or-mint-and-store. A miss (including `unavailable`) is // reported as "this tier has nothing to offer", not "mint one" — the export path this feeds must // never write a new secret to the OS keychain any more than it may write a keyfile. -export function keyFromKeychainReadOnly(options: MacosKeychainOptions = {}): Buffer | undefined { +export function keyFromKeychainReadOnly( + options: MacosKeychainOptions = {}, +): ReadOnlyKeychainLookup { const account = userInfo().username; const read = readMacosKeychainSecret(KEYCHAIN_SERVICE, account, options); - if (read.kind !== "found") return undefined; + if (read.kind !== "found") return { key: undefined, malformed: false }; try { - return decodeKeyOrThrow(read.secret, "Keychain key"); + return { key: decodeKeyOrThrow(read.secret, "Keychain key"), malformed: false }; } catch { - return undefined; + return { key: undefined, malformed: true }; } } @@ -141,8 +154,18 @@ export const NO_KEYCHAIN: KeychainAccess = () => undefined; export interface ResolvedVaultKeyReadOnly { readonly key: Buffer | undefined; readonly source: VaultKeySource | undefined; + // True iff the keychain tier held a stored secret that failed to decode as a 32-byte key — + // distinct from an absent `source`, which also covers "the keychain had nothing at all" and "the + // env tier resolved the key so the keychain was never consulted" (both `false`). This is a + // read-only classification: unlike `keyFromKeychain`'s mutating counterpart, this path never + // replaces or regenerates the malformed value (Finding: Thread 5). + readonly keychainMalformed: boolean; } +// Test/CI seam for the read-only diagnostic path (Finding: Thread 5): distinct from `KeychainAccess` +// because the read-only resolver reports a `malformed` classification the mutating tier does not. +export type ReadOnlyKeychainAccess = () => ReadOnlyKeychainLookup; + // Read-only counterpart to `resolveVaultKey` for the diagnostic export seam (Wave 4a, epic #3233 // §6.2/Finding 0): tries env, then a plain keychain lookup, and — unlike `resolveVaultKey` — never // falls through to the keyfile tier, because that tier's only miss behaviour is minting and @@ -151,13 +174,15 @@ export interface ResolvedVaultKeyReadOnly { // parameter that invites a future keyfile branch. export function resolveVaultKeyReadOnly( env: Readonly>, - keychainAccess: KeychainAccess = keyFromKeychainReadOnly, + keychainAccess: ReadOnlyKeychainAccess = keyFromKeychainReadOnly, ): ResolvedVaultKeyReadOnly { const fromEnv = keyFromEnv(env); - if (fromEnv !== undefined) return { key: fromEnv, source: "env" }; + if (fromEnv !== undefined) return { key: fromEnv, source: "env", keychainMalformed: false }; const fromKeychain = keychainAccess(); - if (fromKeychain !== undefined) return { key: fromKeychain, source: "keychain" }; - return { key: undefined, source: undefined }; + if (fromKeychain.key !== undefined) { + return { key: fromKeychain.key, source: "keychain", keychainMalformed: false }; + } + return { key: undefined, source: undefined, keychainMalformed: fromKeychain.malformed }; } export function createMemoryContentCipher(key: Buffer): MemoryContentCipher { diff --git a/packages/keiko-memory-vault/src/db.test.ts b/packages/keiko-memory-vault/src/db.test.ts index 1cac66d8db..e20abd9e60 100644 --- a/packages/keiko-memory-vault/src/db.test.ts +++ b/packages/keiko-memory-vault/src/db.test.ts @@ -10,6 +10,7 @@ import { statSync, writeFileSync, } from "node:fs"; +import { isStoreFingerprint } from "@oscharko-dev/keiko-contracts"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Worker } from "node:worker_threads"; @@ -296,6 +297,26 @@ describe("openMemoryDatabase — store.encryption-migrated wiring", () => { }); describe("computeStoreFingerprint", () => { + it("names no keySource for a plaintext or unreadable store, so the contract guard accepts it", () => { + // Regression (#3244 review + Wave 4a acceptance): a corrupt vault file read as schema 0 and + // was reported as `plaintext` WITH the resolved `keySource`; the contract rejects that pair, + // and the exporter then had no fingerprint and no unavailability entry for the store. + const dir = mkdtempSync(join(tmpdir(), "keiko-vault-fp-")); + const corruptPath = join(dir, "keiko-memory.db"); + writeFileSync(corruptPath, "garbage that is not a sqlite header"); + const db = new DatabaseSync(corruptPath, { readOnly: true }); + try { + const fingerprint = computeStoreFingerprint(db, "env"); + expect(fingerprint.encryptionMode).toBe("plaintext"); + expect(fingerprint.quickCheckOk).toBe(false); + expect("keySource" in fingerprint).toBe(false); + expect(isStoreFingerprint(fingerprint)).toBe(true); + } finally { + db.close(); + rmSync(dir, { force: true, recursive: true }); + } + }); + it("reports schemaVersion, table row counts, quickCheckOk, encryptionMode and keySource for a healthy vault", () => { const dir = freshDir(); const dbPath = join(dir, "keiko-memory.db"); diff --git a/packages/keiko-memory-vault/src/db.ts b/packages/keiko-memory-vault/src/db.ts index cea307f540..24e5c199d3 100644 --- a/packages/keiko-memory-vault/src/db.ts +++ b/packages/keiko-memory-vault/src/db.ts @@ -276,7 +276,12 @@ function migrationsAppliedUpTo(schemaVersion: number): readonly string[] { ); } -function fallbackStoreFingerprint(keySource: VaultKeySource | undefined): StoreFingerprint { +// A store whose schema could not be read is reported as `plaintext` with NO `keySource`: the key +// tier that was resolved for it is not evidence about bytes nobody could read, and the contract's +// `isStoreFingerprint` rejects a plaintext fingerprint that still names a key source — a rejected +// fingerprint must never silently vanish from a support bundle (see the exporter's +// `invalid-fingerprint` unavailability reason, the fail-closed backstop for exactly that). +function fallbackStoreFingerprint(): StoreFingerprint { return { store: "memory-vault", schemaVersion: 0, @@ -284,7 +289,6 @@ function fallbackStoreFingerprint(keySource: VaultKeySource | undefined): StoreF tableRowCounts: {}, quickCheckOk: false, encryptionMode: "plaintext", - keySource, }; } @@ -300,16 +304,19 @@ export function computeStoreFingerprint( ): StoreFingerprint { try { const schemaVersion = safeUserVersion(db); + const encrypted = schemaVersion >= ENCRYPTION_SCHEMA_VERSION; return { store: "memory-vault", schemaVersion, migrationsApplied: migrationsAppliedUpTo(schemaVersion), tableRowCounts: safeTableRowCounts(db), quickCheckOk: safeQuickCheckOk(db), - encryptionMode: schemaVersion >= ENCRYPTION_SCHEMA_VERSION ? "encrypted" : "plaintext", - keySource, + encryptionMode: encrypted ? "encrypted" : "plaintext", + // `keySource` describes how THIS store's key was resolved; a plaintext store has no key in + // play, so naming a tier for it would contradict the contract (`isStoreFingerprint`). + ...(encrypted && keySource !== undefined ? { keySource } : {}), }; } catch { - return fallbackStoreFingerprint(keySource); + return fallbackStoreFingerprint(); } } diff --git a/packages/keiko-memory-vault/src/index.test.ts b/packages/keiko-memory-vault/src/index.test.ts new file mode 100644 index 0000000000..c499ba1fd6 --- /dev/null +++ b/packages/keiko-memory-vault/src/index.test.ts @@ -0,0 +1,24 @@ +// Public-surface regression pins for the package barrel (Epic #204 child #206; ADR-0019 trust +// rule 7 — this file is the SOLE entry point). + +import { describe, expect, it } from "vitest"; +import * as memoryVault from "./index.js"; + +describe("public surface (Finding: Thread 6 — no write-capable vault entry points)", () => { + it("re-exports the read-only diagnostic seam used by keiko-server's fingerprint collector", () => { + expect(typeof memoryVault.resolveVaultKeyReadOnly).toBe("function"); + expect(typeof memoryVault.openMemoryDatabaseReadOnly).toBe("function"); + expect(typeof memoryVault.computeStoreFingerprint).toBe("function"); + expect(typeof memoryVault.createMemoryVault).toBe("function"); + }); + + // `resolveVaultKey` can mint and persist `vault.key`; `openMemoryDatabase` can migrate, + // re-encrypt, or quarantine-and-reopen a store. Neither is imported by any package outside this + // one (verified against the full repo before this fix), so neither belongs on the public barrel + // external callers reach through. RED (before fix): both properties were present on the + // namespace object, so both assertions below failed with `true`, not `false`. + it("does not export the mutating resolveVaultKey/openMemoryDatabase primitives", () => { + expect("resolveVaultKey" in memoryVault).toBe(false); + expect("openMemoryDatabase" in memoryVault).toBe(false); + }); +}); diff --git a/packages/keiko-memory-vault/src/index.ts b/packages/keiko-memory-vault/src/index.ts index 31a99a73c5..ee7504f1aa 100644 --- a/packages/keiko-memory-vault/src/index.ts +++ b/packages/keiko-memory-vault/src/index.ts @@ -25,17 +25,26 @@ export { memoryBodySuppressionHash } from "./body-fingerprint.js"; // Read-only diagnostic seam for `keiko bundle export` (Wave 4a, epic #3233 §6.2): the exporter // resolves the vault key and opens the store itself via `openMemoryDatabaseReadOnly` (rather than // through `createMemoryVault`, whose returned `MemoryVaultStore` intentionally exposes no `db` -// handle or `keySource`, or through the mutating `openMemoryDatabase`) so it can call -// `computeStoreFingerprint` directly without migrating, re-encrypting, or quarantining the vault. +// handle or `keySource`) so it can call `computeStoreFingerprint` directly without migrating, +// re-encrypting, or quarantining the vault. +// +// `resolveVaultKey` (mutating: can mint and persist `vault.key`) and `openMemoryDatabase` +// (mutating: can migrate, re-encrypt, or quarantine-and-reopen a store) are deliberately NOT +// re-exported here (Finding: Thread 6). No package outside this one imports either — every +// external consumer that needs a fully-opened, writable vault goes through `createMemoryVault`, +// and the diagnostic seam above uses only the read-only twins. Keeping the write-capable +// primitives internal to this package (still available to `vault.ts`/`db.ts` via a direct +// `./cipher.js` / `./db.js` import) means a future external caller cannot reach past +// `createMemoryVault`'s validated construction by importing a lower-level primitive from the +// public barrel. export { createMemoryContentCipher, - resolveVaultKey, resolveVaultKeyReadOnly, type MemoryContentCipher, type ResolvedVaultKeyReadOnly, type VaultKeySource, } from "./cipher.js"; -export { computeStoreFingerprint, openMemoryDatabase, openMemoryDatabaseReadOnly } from "./db.js"; +export { computeStoreFingerprint, openMemoryDatabaseReadOnly } from "./db.js"; export type { DeleteMemoryOptions, ListMemoriesOptions, diff --git a/packages/keiko-memory-vault/src/migrate-encrypt.test.ts b/packages/keiko-memory-vault/src/migrate-encrypt.test.ts index 6b5244e098..7a09353e38 100644 --- a/packages/keiko-memory-vault/src/migrate-encrypt.test.ts +++ b/packages/keiko-memory-vault/src/migrate-encrypt.test.ts @@ -106,8 +106,16 @@ describe("encryptExistingContent — store.encryption-migrated event", () => { // The migration itself must never fail because its OWN logging failed — the same rule // `vault-log.test.ts` proves for the seam in isolation, pinned again here at the real call site. - it("never lets a throwing sink surface as a migration failure", () => { - vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + // + // A sink that ALWAYS throws (this test's `dead` sink) also throws on `emitMemoryVaultLogEvent`'s + // own envelope-only retry, so the failure must reach the last channel, `process.emitWarning` + // (`vault-log.ts`'s `warnFailedMemoryVaultLogSink`). Asserting only `.not.toThrow()` (as this + // test used to) is satisfied just as well by a sink failure that is silently swallowed — the + // "No silent failures" guideline requires proving the report actually happened, not merely that + // the migration survived it. RED (before failure reporting existed): the `emitWarning` spy below + // was never called, so `toHaveBeenCalledTimes(1)` failed with 0 calls. + it("never lets a throwing sink surface as a migration failure, and reports the drop", () => { + const warnSpy = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); const db = openTestDb(); insertMemoryRow(db, makeRecord({ id: memId("m1") }), TEST_CIPHER); overwriteBodyWithPlaintext(db, "m1", "plaintext body"); @@ -124,6 +132,16 @@ describe("encryptExistingContent — store.encryption-migrated event", () => { readonly body: string; }; expect(TEST_CIPHER.isSealed(row.body)).toBe(true); + + expect(warnSpy).toHaveBeenCalledTimes(1); + const [message, options] = warnSpy.mock.calls[0] ?? []; + expect(message).toBe("Keiko memory-vault log sink is failing; log lines are being dropped."); + // Redacted context only: the dropped op name and a shape-gated error kind — never the sink's + // thrown message ("sink is down"), which could carry caller-supplied detail. + expect(options).toMatchObject({ type: "KeikoActivityLog", code: "KEIKO_LOG_SINK_FAILED" }); + expect((options as { detail?: string } | undefined)?.detail).toBe( + "op=store.encryption-migrated errorKind=Error", + ); db.close(); }); }); diff --git a/packages/keiko-memory-vault/src/vault-log.test.ts b/packages/keiko-memory-vault/src/vault-log.test.ts index ec8aa15964..0a88f7c2aa 100644 --- a/packages/keiko-memory-vault/src/vault-log.test.ts +++ b/packages/keiko-memory-vault/src/vault-log.test.ts @@ -32,7 +32,7 @@ describe("nullMemoryVaultLogSink", () => { it("accepts an event without throwing and returns a shared instance", () => { const sink = nullMemoryVaultLogSink(); expect(() => { - sink.write({ category: "memory", op: "test.op" }); + sink.write({ category: "memory", op: "memory-vault.store.opened" }); }).not.toThrow(); expect(nullMemoryVaultLogSink()).toBe(sink); }); @@ -211,10 +211,15 @@ describe("emitMemoryVaultLogEvent", () => { }, }; + // `secret` is embedded in the SINK's own thrown message, not in `event.extra` — this event's + // `extra` is intentionally omitted, since `MemoryVaultLogExtra`'s closed schema (Finding: + // Thread 8) has no field that could hold arbitrary free text like `secret` in the first + // place. The proof this test carries is that `memoryVaultErrorKind` never reads `.message` + // (see the `memoryVaultErrorKind` suite above), so a secret embedded in a thrown error's + // message can never reach the fallback warning either. emitMemoryVaultLogEvent(dead, { category: "memory", op: "memory-vault.store.opened", - extra: { note: secret }, }); const reported = JSON.stringify(warn.mock.calls); @@ -259,4 +264,25 @@ describe("MemoryVaultLogEvent", () => { "status", ]); }); + + // Finding: Thread 8. Before this fix `op: string` and `extra?: Readonly>` accepted anything, so a memory body or filesystem path could reach this event with + // no compile-time signal. RED (before fix): both `@ts-expect-error` directives below were + // themselves compile errors ("Unused '@ts-expect-error' directive") under `npm run typecheck`, + // because the old, open shape happily accepted an arbitrary `op` string and an arbitrary + // `extra` key — there was nothing to suppress. + it("rejects an arbitrary op and an arbitrary extra field at compile time", () => { + // @ts-expect-error — `op` is now a closed `MemoryVaultLogOp` union; a caller-chosen string is + // no longer assignable. + const badOp: MemoryVaultLogEvent = { category: "memory", op: "arbitrary.caller.chosen.op" }; + const badExtra: MemoryVaultLogEvent = { + category: "memory", + op: "memory-vault.store.opened", + // @ts-expect-error — `extra` is now a closed, body-free `MemoryVaultLogExtra` schema; an + // unlisted key (here shaped like a leaked filesystem path) is no longer assignable. + extra: { path: "/Users/someone/memory/vault.db" }, + }; + expect(badOp.category).toBe("memory"); + expect(badExtra.category).toBe("memory"); + }); }); diff --git a/packages/keiko-memory-vault/src/vault-log.ts b/packages/keiko-memory-vault/src/vault-log.ts index 60229eb17f..e038ae5e1d 100644 --- a/packages/keiko-memory-vault/src/vault-log.ts +++ b/packages/keiko-memory-vault/src/vault-log.ts @@ -24,6 +24,13 @@ // A field here carries counts, durations, and shape-gated error kinds (ADR-0128 D6: identifiers, // counts, and hashes only). A memory body, a tag, a scope coordinate, a vault key, and a // filesystem path never reach a field on this event. +// +// `op` and `extra` are CLOSED, not `string` / `Record` (Finding: Thread 8 — +// review flagged the previous open shape as a path by which a caller could send a memory body or +// filesystem path through this package boundary). Every literal below is one this package's own +// three producers (`db.ts`, `migrate-encrypt.ts`, `vault.ts`) already emit; a narrower union/ +// interface is still assignable to `ServerLogEvent`'s `op: string` / `extra?: Record`, so `processServerLogSink()` remains a valid `MemoryVaultLogSink` with no adapter. import { classifyErrorKind } from "@oscharko-dev/keiko-contracts"; @@ -31,16 +38,43 @@ export type MemoryVaultLogLevel = "debug" | "info" | "warn" | "error"; export type MemoryVaultLogCategory = "memory" | "diagnostic"; +// The closed set of ops this package ever emits. +export type MemoryVaultLogOp = + | "memory-vault.store.opened" + | "memory-vault.store.quarantined" + | "store.encryption-migrated" + | "memory-vault.log.sink-failed"; + +// A closed mirror of cipher.ts's `VaultKeySource` ("env" | "keychain" | "keyfile"), duplicated +// rather than imported: cipher.ts imports from db.ts (for keyfile hardening) and db.ts imports +// from this file (to emit its own events), so an import here would complete a +// cipher.ts -> db.ts -> vault-log.ts -> cipher.ts cycle. The same accepted, documented-duplication +// tradeoff `ERROR_KIND_PATTERN` used across three packages before ADR-0173 consolidated it. +export type MemoryVaultLogKeySource = "env" | "keychain" | "keyfile"; + +// The closed, body-free `extra` schema: every field this package's producers have ever needed — +// the retained key-resolution tier, whether a quarantine reopen succeeded, the encryption sweep's +// scope transition and row count, and the op name a failed sink dropped. Never a memory body, a +// tag, a scope coordinate, a vault key, or a filesystem path. +export interface MemoryVaultLogExtra { + readonly keySource?: MemoryVaultLogKeySource | undefined; + readonly reopened?: boolean | undefined; + readonly fromScope?: "plaintext" | undefined; + readonly toScope?: "encrypted" | undefined; + readonly rowsMigrated?: number | undefined; + readonly droppedOp?: MemoryVaultLogOp | undefined; +} + export interface MemoryVaultLogEvent { // Omitted means `info`, matching the server sink's own default. readonly level?: MemoryVaultLogLevel | undefined; readonly category: MemoryVaultLogCategory; - readonly op: string; + readonly op: MemoryVaultLogOp; readonly correlationId?: string | undefined; readonly durationMs?: number | undefined; readonly status?: number | undefined; readonly errorKind?: string | undefined; - readonly extra?: Readonly> | undefined; + readonly extra?: Readonly | undefined; } export interface MemoryVaultLogSink { @@ -127,7 +161,7 @@ export function emitMemoryVaultLogEvent( function reportFailedMemoryVaultLogSink( sink: MemoryVaultLogSink, - droppedOp: string, + droppedOp: MemoryVaultLogOp, cause: unknown, ): void { if (REPORTED_FAILED_SINKS.has(sink)) return; @@ -149,7 +183,7 @@ function reportFailedMemoryVaultLogSink( warnFailedMemoryVaultLogSink(droppedOp, errorKind); } -function warnFailedMemoryVaultLogSink(droppedOp: string, errorKind: string): void { +function warnFailedMemoryVaultLogSink(droppedOp: MemoryVaultLogOp, errorKind: string): void { try { process.emitWarning("Keiko memory-vault log sink is failing; log lines are being dropped.", { type: "KeikoActivityLog", diff --git a/packages/keiko-memory-vault/src/vault.test.ts b/packages/keiko-memory-vault/src/vault.test.ts index a8fdf09782..0f89ed329c 100644 --- a/packages/keiko-memory-vault/src/vault.test.ts +++ b/packages/keiko-memory-vault/src/vault.test.ts @@ -30,6 +30,7 @@ import { MemoryStorageError, MemoryStoragePreconditionError, MemoryStorageValidationError, + type MemoryContentCipher, type MemoryEvent, type MemoryVaultStore, } from "./index.js"; @@ -1307,4 +1308,36 @@ describe("activity-log seam: memory-vault.store.opened retains the key-resolutio v.close(); }).not.toThrow(); }); + + // Finding: store-open ordering. `createMemoryVault` used to emit `memory-vault.store.opened` + // right after `openMemoryDatabase`, BEFORE `resolveBodySuppressionKey` ran. A cipher that fails + // on its very first `sealString` call (the fresh-vault path, which mints and persists a new + // body-suppression HMAC key) makes `createMemoryVault` throw, but the previous ordering had + // already reported the open as successful by then. RED (before fix): this test's second + // assertion fails because `opened` has length 1, not 0. + it("emits no store-opened event when initialization fails after the store is opened", () => { + const dir = freshDir(); + const { sink, events } = recordingSink(); + const throwingCipher: MemoryContentCipher = { + sealString: (): string => { + throw new Error("cipher unavailable"); + }, + openString: (envelope: string): string => envelope, + sealBytes: (buf: Buffer): Buffer => buf, + openBytes: (envelope: Buffer): Buffer => envelope, + isSealed: (): boolean => false, + }; + + expect(() => { + createMemoryVault({ + memoryDir: dir, + env: { KEIKO_MEMORY_DIR: dir }, + cipher: throwingCipher, + logSink: sink, + }); + }).toThrow(); + + const opened = events.filter((event) => event.op === "memory-vault.store.opened"); + expect(opened).toHaveLength(0); + }); }); diff --git a/packages/keiko-memory-vault/src/vault.ts b/packages/keiko-memory-vault/src/vault.ts index b22d7a7726..7d8ba21d7e 100644 --- a/packages/keiko-memory-vault/src/vault.ts +++ b/packages/keiko-memory-vault/src/vault.ts @@ -1207,8 +1207,13 @@ export function createMemoryVault(options?: CreateMemoryVaultOptions): MemoryVau const { cipher, keySource } = resolveCipherWithSource(options, env, options?.securityLogSink); const elapsedMs = startMemoryVaultLogTimer(); const db = openMemoryDatabase(dbPath, cipher, options?.logSink); - emitVaultOpened(options?.logSink, keySource, elapsedMs()); + // Emitted only after BOTH remaining initialization steps succeed (Finding: store-open ordering) + // — a body-suppression-key failure or a sidecar-hardening failure must fail `createMemoryVault` + // without the activity log ever reporting the open as having succeeded. Emitting right after + // `openMemoryDatabase` (as this line used to) reported a successful open even when either + // operation below then threw and the vault never actually became usable. const bodySuppressionKey = resolveBodySuppressionKey(db, cipher); hardenVaultSidecars(dbPath); + emitVaultOpened(options?.logSink, keySource, elapsedMs()); return buildStore(db, resolveOptions(options, cipher, dbPath, bodySuppressionKey)); } diff --git a/packages/keiko-security/src/log-port.test.ts b/packages/keiko-security/src/log-port.test.ts index 26dfcb97ff..392d65d73f 100644 --- a/packages/keiko-security/src/log-port.test.ts +++ b/packages/keiko-security/src/log-port.test.ts @@ -232,9 +232,16 @@ describe("startSecurityLogTimer", () => { }); it("is driven by performance.now, so a backwards wall clock cannot go negative", () => { - vi.spyOn(Date, "now").mockReturnValue(0); + // Date.now steps BACKWARDS between the start and the read: a Date.now-based timer would + // compute a negative delta here. performance.now is mocked to a known, forward-moving pair so + // the assertion pins the exact value a performance.now-based implementation must produce — + // discriminating this from a Date.now-based one, which `toBeGreaterThanOrEqual(0)` against a + // constant-0 Date.now mock could not (that passed for either implementation). + vi.spyOn(Date, "now").mockReturnValueOnce(5000).mockReturnValueOnce(1000); + const nowSpy = vi.spyOn(performance, "now"); + nowSpy.mockReturnValueOnce(2000).mockReturnValueOnce(2017.5); const elapsed = startSecurityLogTimer(); - expect(elapsed()).toBeGreaterThanOrEqual(0); + expect(elapsed()).toBe(17.5); }); }); diff --git a/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts b/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts index 387ee8f95b..2237630b9c 100644 --- a/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts +++ b/packages/keiko-security/src/secret-vault.fs-fault-injection.test.ts @@ -15,6 +15,7 @@ import { let blockedOpenDir = ""; let blockedRenameDest = ""; +let blockedOpenSyncHits = 0; vi.mock("node:fs", async (importOriginal) => { const actual = await importOriginal(); @@ -28,6 +29,7 @@ vi.mock("node:fs", async (importOriginal) => { // any pre-set restriction. This scoped mock is the only hermetic way to exercise it. openSync: (path: unknown, flags: unknown, mode?: unknown): number => { if (path === blockedOpenDir && flags === "r") { + blockedOpenSyncHits += 1; throw Object.assign(new Error("simulated: directory cannot be opened for fsync"), { code: "EACCES", }); @@ -60,6 +62,7 @@ const dirs: string[] = []; afterEach(() => { blockedOpenDir = ""; blockedRenameDest = ""; + blockedOpenSyncHits = 0; for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); }); @@ -79,6 +82,10 @@ describe("fsyncDirectory — the directory itself cannot be opened for fsync", ( vault.set("cred:a", "value"); }).not.toThrow(); expect(vault.get("cred:a")).toBe("value"); + // Prove the injected fault actually fired: without this, a path mismatch (e.g. a realpath + // difference between `blockedOpenDir` and the directory `fsyncDirectory` actually opens) would + // silently skip the throwing branch and this test would still pass for the wrong reason. + expect(blockedOpenSyncHits).toBeGreaterThan(0); }); it("sharded layout: set() still commits and returns the secret when the post-rename directory fsync fails to open", () => { @@ -91,6 +98,8 @@ describe("fsyncDirectory — the directory itself cannot be opened for fsync", ( vault.set("cred:a", "value"); }).not.toThrow(); expect(vault.get("cred:a")).toBe("value"); + // Same discrimination as the single-file case above: the fault must have actually fired. + expect(blockedOpenSyncHits).toBeGreaterThan(0); }); }); diff --git a/packages/keiko-security/src/secret-vault.test.ts b/packages/keiko-security/src/secret-vault.test.ts index b276fcc55f..10c74b3014 100644 --- a/packages/keiko-security/src/secret-vault.test.ts +++ b/packages/keiko-security/src/secret-vault.test.ts @@ -1059,7 +1059,7 @@ describe("createShardedLocalSecretVault — CRUD parity with the single-file lay level: "warn", category: "security", op: "security.vault.shard-unreadable", - errorKind: "Error", + errorKind: "EISDIR", extra: { count: 1 }, }); expect(Object.keys(event ?? {}).sort()).toEqual([ diff --git a/packages/keiko-security/src/secret-vault.ts b/packages/keiko-security/src/secret-vault.ts index eaa2204074..0282b5f504 100644 --- a/packages/keiko-security/src/secret-vault.ts +++ b/packages/keiko-security/src/secret-vault.ts @@ -50,10 +50,13 @@ import { KEYCHAIN_SPAWN_TIMEOUT_MS, keychainItemNotFound, } from "./macos-keychain.js"; -// Independent activity-log seam (ADR-0019, w4a-security-log-port) and the hardened error-class -// classifier this package already applies to its persisted quarantine diagnostic. -import { emitSecurityLogEvent, startSecurityLogTimer, type SecurityLogSink } from "./log-port.js"; -import { hardenedErrorClass } from "./sqlite-corruption.js"; +// Independent activity-log seam (ADR-0019, w4a-security-log-port). +import { + emitSecurityLogEvent, + securityErrorKind, + startSecurityLogTimer, + type SecurityLogSink, +} from "./log-port.js"; const KEY_BYTES = 32; const STORE_VERSION = 1; @@ -535,7 +538,7 @@ function emitShardUnreadable(sink: SecurityLogSink | undefined, cause: unknown): level: "warn", category: "security", op: "security.vault.shard-unreadable", - errorKind: hardenedErrorClass(cause), + errorKind: securityErrorKind(cause), // A single unreadable file per call; never the filename (it decodes to the reference) or the // read error's message (it can carry the resolved path). extra: { count: 1 }, diff --git a/packages/keiko-server/src/credentialPersistence.test.ts b/packages/keiko-server/src/credentialPersistence.test.ts new file mode 100644 index 0000000000..f51de9d1a6 --- /dev/null +++ b/packages/keiko-server/src/credentialPersistence.test.ts @@ -0,0 +1,86 @@ +// Regression coverage for the migration failure path (#3244 review, thread 12): before this test +// existed, `migrateLocalConfigCredentials`'s catch swallowed every migration failure and returned +// `{ migrated: false }` with no operator diagnostic at all — a plaintext credential configuration +// could remain in use indefinitely and nobody would ever see why. This proves the redacted, +// correlation-keyed diagnostic is emitted instead of the failure staying silent. + +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { migrateLocalConfigCredentials } from "./credentialPersistence.js"; +import type { ServerDiagnosticRecord, ServerDiagnosticSink } from "./diagnostics-log.js"; + +const dirs: string[] = []; + +afterEach(() => { + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); +}); + +function tempConfigDir(): string { + const dir = mkdtempSync(join(tmpdir(), "credential-persistence-migrate-")); + dirs.push(dir); + return dir; +} + +function recordingDiagnosticSink(): { + sink: ServerDiagnosticSink; + records: ServerDiagnosticRecord[]; +} { + const records: ServerDiagnosticRecord[] = []; + return { + sink: { + record: (record): void => { + records.push(record); + }, + }, + records, + }; +} + +describe("migrateLocalConfigCredentials", () => { + it("does nothing and reports no diagnostic when the config file simply does not exist", () => { + const dir = tempConfigDir(); + const { sink, records } = recordingDiagnosticSink(); + + const outcome = migrateLocalConfigCredentials({ + configPath: join(dir, "keiko.config.json"), + env: {}, + evidenceDir: dir, + diagnostics: sink, + }); + + expect(outcome).toEqual({ migrated: false }); + expect(records).toHaveLength(0); + }); + + it("reports a redacted, correlation-keyed diagnostic when migration fails, instead of failing silently", () => { + const dir = tempConfigDir(); + const configPath = join(dir, "keiko.config.json"); + // Malformed JSON makes JSON.parse throw inside migrateLocalConfigCredentials's try, exercising + // the best-effort catch without needing a real vault/provider fixture. + writeFileSync(configPath, "{ not valid json", "utf8"); + const { sink, records } = recordingDiagnosticSink(); + + const outcome = migrateLocalConfigCredentials({ + configPath, + env: {}, + evidenceDir: dir, + diagnostics: sink, + }); + + expect(outcome).toEqual({ migrated: false }); + expect(records).toHaveLength(1); + const [record] = records; + expect(record).toMatchObject({ + operation: "credential.migration", + source: "credentialPersistence.migrateLocalConfigCredentials", + errorClass: "SyntaxError", + }); + expect(typeof record?.correlationId).toBe("string"); + expect(record?.correlationId.length ?? 0).toBeGreaterThan(0); + // Redacted: never the config path or the malformed file content. + expect(JSON.stringify(record)).not.toContain(configPath); + expect(JSON.stringify(record)).not.toContain("not valid json"); + }); +}); diff --git a/packages/keiko-server/src/credentialPersistence.ts b/packages/keiko-server/src/credentialPersistence.ts index bcd04d5a77..93035f7332 100644 --- a/packages/keiko-server/src/credentialPersistence.ts +++ b/packages/keiko-server/src/credentialPersistence.ts @@ -12,10 +12,13 @@ // plaintext config remains on disk and the next migration re-runs idempotently (the vault writes // overwrite), while `keiko repair` flags the lingering plaintext as an incomplete migration. +import { randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import type { EnvSource } from "@oscharko-dev/keiko-model-gateway"; import type { LocalVaultKeychainAccess } from "@oscharko-dev/keiko-security/secret-vault"; import type { SecurityLogSink } from "@oscharko-dev/keiko-security"; +import { emitServerDiagnostic, type ServerDiagnosticSink } from "./diagnostics-log.js"; +import { contentFreeErrorClass } from "./observability/error-classification.js"; import { savePrivateJson } from "./private-json.js"; import { hasPlaintextGatewayCredentials, @@ -110,6 +113,10 @@ export interface MigrateCredentialsOptions { // Optional activity-log seam (ADR-0019); the deps.ts composition root supplies // `processServerLogSink()`. readonly securityLogSink?: SecurityLogSink | undefined; + // Optional operator-diagnostic sink (ADR-0173); the deps.ts composition root supplies + // `options.diagnostics`. Falls back to `defaultServerDiagnosticSink` (via `emitServerDiagnostic`) + // when omitted, so production never goes silent even when a caller wires nothing in. + readonly diagnostics?: ServerDiagnosticSink | undefined; } export interface MigrateCredentialsOutcome { @@ -143,10 +150,22 @@ export function migrateLocalConfigCredentials( securityLogSink: options.securityLogSink, }); return { migrated: true }; - } catch { + } catch (error) { // Migration is best-effort: a fault here leaves the plaintext config in place (the atomic write // never half-replaces it), the gateway still loads via the legacy plaintext path, and `keiko // repair` reports the lingering plaintext so the user can complete the migration deterministically. + // "Best-effort" must not mean "silent": without this, a plaintext credential configuration can + // remain in use indefinitely with no operator diagnostic at all. The record is redacted + // (content-free error class, no config content, no secret material) and correlation-keyed so an + // operator can find it in the server's activity log. + emitServerDiagnostic(options.diagnostics, { + correlationId: randomUUID(), + timestamp: new Date().toISOString(), + operation: "credential.migration", + source: "credentialPersistence.migrateLocalConfigCredentials", + errorClass: contentFreeErrorClass(error), + message: "Local credential migration failed; plaintext config remains in place.", + }); return { migrated: false }; } } diff --git a/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts b/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts index 4d439bd8e6..596dd6f819 100644 --- a/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts +++ b/packages/keiko-server/src/deps-vault-key-securitylog-wiring.test.ts @@ -48,6 +48,13 @@ type ResolveLocalVaultKeyOptions = Parameters< typeof import("@oscharko-dev/keiko-security/secret-vault").resolveLocalVaultKey >[0]; +// A deterministic, valid 32-byte-base64 env-tier key so every vault under test resolves at tier 1 +// (env) and `resolveLocalVaultKey` never falls through to tier 2 (macOS Keychain, via +// `createKeychainVaultKeyAccess`'s real `/usr/bin/security` spawn). Without this, an environment +// that omits the vault's `KEIKO_*_KEY` lets the resolver reach the developer's real login keychain — +// exactly the shared, uncontrolled state AGENTS.md's "tests are hermetic" rule forbids. +const WIRING_TEST_VAULT_KEY = Buffer.alloc(32, 7).toString("base64"); + let calls: ResolveLocalVaultKeyOptions[]; vi.mock("@oscharko-dev/keiko-security/secret-vault", async (importOriginal) => { @@ -131,7 +138,7 @@ describe("buildUiHandlerDeps — editorHotExitStore wires resolveLocalVaultKey's const deps = buildUiHandlerDeps({ configPath: undefined, evidenceDir, - env: { KEIKO_UI_DATA_DIR: uiDir }, + env: { KEIKO_UI_DATA_DIR: uiDir, KEIKO_EDITOR_HOT_EXIT_KEY: WIRING_TEST_VAULT_KEY }, }); try { const snapshot = hotExitSnapshot(); @@ -155,7 +162,7 @@ describe("buildUiHandlerDeps — localKnowledgeKeyProvider wires resolveLocalVau const deps = buildUiHandlerDeps({ configPath: undefined, evidenceDir, - env: { KEIKO_UI_DATA_DIR: uiDir }, + env: { KEIKO_UI_DATA_DIR: uiDir, KEIKO_LOCAL_KNOWLEDGE_KEY: WIRING_TEST_VAULT_KEY }, }); try { present(deps.localKnowledgeKeyProvider, "localKnowledgeKeyProvider").resolveKey({ @@ -179,7 +186,7 @@ describe("buildUiHandlerDeps — workspaceIndexForRoot wires resolveLocalVaultKe const deps = buildUiHandlerDeps({ configPath: undefined, evidenceDir, - env: { KEIKO_UI_DATA_DIR: uiDir }, + env: { KEIKO_UI_DATA_DIR: uiDir, KEIKO_WORKSPACE_INDEX_KEY: WIRING_TEST_VAULT_KEY }, }); try { if (deps.workspaceIndexForRoot === undefined) { @@ -203,7 +210,10 @@ describe("buildUiHandlerDeps — atlassianConnectorCredentials wires resolveLoca const deps = buildUiHandlerDeps({ configPath: join(configDir, "keiko.config.json"), evidenceDir, - env: { KEIKO_UI_DATA_DIR: uiDir }, + env: { + KEIKO_UI_DATA_DIR: uiDir, + KEIKO_ATLASSIAN_CONNECTOR_CREDENTIALS_KEY: WIRING_TEST_VAULT_KEY, + }, }); try { if (deps.atlassianConnectorCredentials === undefined) { @@ -255,7 +265,7 @@ describe("buildUiHandlerDeps — provider-credential vault wires resolveLocalVau const deps = buildUiHandlerDeps({ configPath, evidenceDir, - env: { KEIKO_UI_DATA_DIR: uiDir }, + env: { KEIKO_UI_DATA_DIR: uiDir, KEIKO_PROVIDER_CREDENTIALS_KEY: WIRING_TEST_VAULT_KEY }, store: createInMemoryUiStore(), }); try { diff --git a/packages/keiko-server/src/deps.ts b/packages/keiko-server/src/deps.ts index 0d4679b672..43bf5ba10b 100644 --- a/packages/keiko-server/src/deps.ts +++ b/packages/keiko-server/src/deps.ts @@ -2857,6 +2857,7 @@ function loadRuntimeGatewayConfig( env: options.env, evidenceDir: resolvedEvidenceDir, securityLogSink: processServerLogSink(), + diagnostics: options.diagnostics, }); const secretResolver = createProviderSecretResolver({ configPath: effectiveConfigPath, diff --git a/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts b/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts index 8c73c83d88..38a3d909e0 100644 --- a/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts +++ b/packages/keiko-server/src/gateway-setup-vault-key-securitylog-wiring.test.ts @@ -37,6 +37,12 @@ type ResolveLocalVaultKeyOptions = Parameters< typeof import("@oscharko-dev/keiko-security/secret-vault").resolveLocalVaultKey >[0]; +// A deterministic, valid 32-byte-base64 env-tier key so the provider-credential vault resolves at +// tier 1 (env) and never falls through to tier 2 (macOS Keychain, via +// `createKeychainVaultKeyAccess`'s real `/usr/bin/security` spawn) — see the sibling +// `deps-vault-key-securitylog-wiring.test.ts` for the same fix applied to the other four vaults. +const WIRING_TEST_VAULT_KEY = Buffer.alloc(32, 7).toString("base64"); + let calls: ResolveLocalVaultKeyOptions[]; vi.mock("@oscharko-dev/keiko-security/secret-vault", async (importOriginal) => { @@ -95,7 +101,7 @@ describe("gateway-setup.ts — provider-credential vault wires resolveLocalVault const deps = buildUiHandlerDeps({ configPath: undefined, evidenceDir, - env: { KEIKO_UI_DATA_DIR: uiDir }, + env: { KEIKO_UI_DATA_DIR: uiDir, KEIKO_PROVIDER_CREDENTIALS_KEY: WIRING_TEST_VAULT_KEY }, gatewayModelDiscovery: () => Promise.resolve(["wiring-test-model"]), gatewayEmbeddingProbe: (_config, ids) => Promise.resolve(ids), gatewaySetupTester: (_config, modelIds) => diff --git a/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts b/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts index 9fa4309e89..52a530f36f 100644 --- a/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts +++ b/packages/keiko-server/src/memory-handlers-securitylog-wiring.test.ts @@ -35,9 +35,29 @@ import { type CreateMemoryVaultOptions = Parameters< typeof import("@oscharko-dev/keiko-memory-vault").createMemoryVault >[0]; +type MemoryVaultStore = import("@oscharko-dev/keiko-memory-vault").MemoryVaultStore; let captured: CreateMemoryVaultOptions; +// A real `createMemoryVault(options)` call can touch OS keychain state and initialize a real +// encrypted store — this file's job is only the composition boundary (does `createBffMemoryVault` +// pass `securityLogSink` through?), never the vault's own key resolution. A `Proxy` stands in for +// the ~30-member `MemoryVaultStore` interface without restating it member-by-member: `close` (the +// only member this test's `finally` block calls) is a real no-op, and any other member this test +// does not touch throws immediately rather than silently returning `undefined`, so an accidental +// use is a loud failure instead of a hermeticity gap. +function fakeMemoryVaultStore(): MemoryVaultStore { + const target = { close: (): void => undefined }; + return new Proxy(target, { + get(obj, prop, receiver): unknown { + if (prop in obj) return Reflect.get(obj, prop, receiver); + return (): never => { + throw new Error(`fakeMemoryVaultStore: "${String(prop)}" is not implemented`); + }; + }, + }) as unknown as MemoryVaultStore; +} + vi.mock("@oscharko-dev/keiko-memory-vault", async (importOriginal) => { const actual = await importOriginal(); return { @@ -46,7 +66,7 @@ vi.mock("@oscharko-dev/keiko-memory-vault", async (importOriginal) => { options: CreateMemoryVaultOptions, ): ReturnType => { captured = options; - return actual.createMemoryVault(options); + return fakeMemoryVaultStore(); }, }; }); diff --git a/packages/keiko-server/src/store/db.test.ts b/packages/keiko-server/src/store/db.test.ts index 88a81084d4..7155fe3f32 100644 --- a/packages/keiko-server/src/store/db.test.ts +++ b/packages/keiko-server/src/store/db.test.ts @@ -1,7 +1,7 @@ // ADR-0013 D3/D8 — db.ts: createInMemoryUiStore (tests), createNodeUiStore (real on-disk). // Asserts perms 0o700/0o600 on the dir/file (Unix), and that the DB file is NOT inside process.cwd(). -import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; import { DatabaseSync } from "node:sqlite"; import { mkdtempSync, @@ -1238,4 +1238,37 @@ describe("openNodeUiDatabase — store.opened activity log (Wave 4a, epic #3233 opened.close(); } }); + + // PR #3244 review, thread 15: `buildUiStoreOpenedEvent` used to call `computeStoreFingerprint`, + // which runs a SECOND full-database `PRAGMA quick_check` (the first already ran inside this same + // `openNodeUiDatabase` call, via `assertQuickCheckOk`) and a `COUNT(*)` scan over every one of + // `UI_STORE_FINGERPRINT_TABLES` — a cost that scales with database size, paid on every production + // server start. Neither is needed: the emitted event carries only `quickCheckOk` (a boolean) and + // `storeSchemaVersion`/`migrationsAppliedCount` (from the already-cheap `PRAGMA user_version`), + // never `tableRowCounts`. + it("emits the store.opened event without re-running quick_check or scanning any table for a row count", () => { + const dbPath = join(tmpDir, "opened-perf.db"); + const sink: ServerLogSink = { write: (): void => undefined }; + const prepareSpy = vi.spyOn(DatabaseSync.prototype, "prepare"); + let db: DatabaseSync | undefined; + try { + db = openNodeUiDatabase(dbPath, sink); + const preparedSql = prepareSpy.mock.calls.map((call) => call[0]); + const quickCheckCalls = preparedSql.filter((sql) => sql.includes("quick_check")); + const countCalls = preparedSql.filter((sql) => /count\(\*\)/iu.test(sql)); + // Exactly one: the real integrity check `openNodeUiDatabase` itself needs, not a second one + // recomputed only to build the log event. + expect(quickCheckCalls).toHaveLength(1); + // `runMigrations` legitimately issues its own single, unrelated `COUNT(*)` against + // `workspace_manifests` (`migrateLegacyProjectManifests`'s post-migration trust-record + // cleanup check) — that is not what this test guards against. What must NOT happen is + // `computeStoreFingerprint`'s `readTableRowCounts`, which queries EVERY one of + // `UI_STORE_FINGERPRINT_TABLES` (13 tables). Fewer COUNT(*) calls than that full table list + // proves the whole-database row-count scan did not run for this event. + expect(countCalls.length).toBeLessThan(UI_STORE_FINGERPRINT_TABLES.length); + } finally { + prepareSpy.mockRestore(); + db?.close(); + } + }); }); diff --git a/packages/keiko-server/src/store/db.ts b/packages/keiko-server/src/store/db.ts index 09006492d2..a5e4610413 100644 --- a/packages/keiko-server/src/store/db.ts +++ b/packages/keiko-server/src/store/db.ts @@ -912,20 +912,29 @@ function startUiStoreOpenTimer(): () => number { return (): number => Math.round((performance.now() - startedAt) * 1000) / 1000; } +// Deliberately NOT `computeStoreFingerprint(db)`: that helper also runs `readTableRowCounts` (a +// `COUNT(*)` scan over all `UI_STORE_FINGERPRINT_TABLES.length` tables — O(rows), no cached count +// in SQLite) and its own `PRAGMA quick_check` via `readQuickCheckOk`, neither of which this event +// carries (see the `extra` fields below — there is no `tableRowCounts`). `openNodeUiDatabase`, this +// function's only caller, already ran `assertQuickCheckOk(db)` earlier in the very same call +// without it throwing (a throw either propagates past this call entirely or is repaired by the +// quarantine-and-reopen branch, which re-asserts before falling through here), so `quickCheckOk` is +// already a known fact and is stated directly instead of re-scanning the whole database a second +// time on every production server start. function buildUiStoreOpenedEvent(db: DatabaseSync, durationMs: number): ServerLogEvent { - const fingerprint = computeStoreFingerprint(db); + const schemaVersion = boundedSchemaVersion(safeReadSchemaVersion(db)); return { category: "setup", op: "store.opened", durationMs, extra: { - store: fingerprint.store, + store: "ui", // Named `storeSchemaVersion`, not `schemaVersion`: the latter is a RESERVED envelope field // name on the log line itself (the log schema's own version) and would be silently dropped. - storeSchemaVersion: fingerprint.schemaVersion, - migrationsAppliedCount: fingerprint.migrationsApplied.length, - quickCheckOk: fingerprint.quickCheckOk, - encryptionMode: fingerprint.encryptionMode, + storeSchemaVersion: schemaVersion, + migrationsAppliedCount: migrationsAppliedFor(schemaVersion).length, + quickCheckOk: true, + encryptionMode: "plaintext", // `keySource` is omitted: this store is never encrypted, so no key is ever resolved. }, }; From f1605ca16b71660bffe2b6e444a7f5e770f7b73c Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 13:21:26 +0200 Subject: [PATCH 13/19] =?UTF-8?q?fix(observability):=20Wave=205=20review?= =?UTF-8?q?=20repairs=20=E2=80=94=20calendar-valid=20clientTs,=20bounded?= =?UTF-8?q?=20content-type=20label,=20pinned=20git=20route=20literals,=20S?= =?UTF-8?q?SE=20tracking=20before=20model=20iteration,=20run-stream=20corr?= =?UTF-8?q?elation=20ids,=20hermetic=20lifecycle=20tests=20(#3240)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 13 CodeRabbit findings repaired with fails-before/passes-after tests; three refuted in-thread with evidence (the package-wide setServerLogger test seam, the shared vault fixture, the injectable-logger proposal). Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 8 +- .../keiko-contracts/src/diagnostics.test.ts | 21 +++ packages/keiko-contracts/src/diagnostics.ts | 38 ++++- .../src/bounded-request-body.test.ts | 64 +++++++ .../keiko-server/src/bounded-request-body.ts | 19 ++- .../src/chat-stream-handlers.test.ts | 29 ++++ .../keiko-server/src/chat-stream-handlers.ts | 11 +- .../src/client-diagnostics-routes.ts | 17 +- packages/keiko-server/src/gitRoutes.test.ts | 30 +++- packages/keiko-server/src/gitRoutes.ts | 9 +- packages/keiko-server/src/routes.ts | 6 +- .../src/run-handlers-sse-backpressure.test.ts | 49 +++++- .../src/run-handlers-sse-correlation.test.ts | 150 +++++++++++++++++ packages/keiko-server/src/run-handlers.ts | 7 +- packages/keiko-server/src/server.test.ts | 159 +++++++++++++----- packages/keiko-server/src/server.ts | 19 ++- packages/keiko-server/src/sse.ts | 3 +- .../src/ui-test-server/_support.test.ts | 72 +++++++- .../src/ui-test-server/_support.ts | 6 + .../lib/install-client-diagnostics.test.ts | 14 +- .../src/lib/install-client-diagnostics.ts | 17 +- .../correlation-id-pattern-drift.test.mjs | 24 ++- 22 files changed, 674 insertions(+), 98 deletions(-) create mode 100644 packages/keiko-server/src/run-handlers-sse-correlation.test.ts diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index 48749f8eb7..f40fce2245 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -707,13 +707,13 @@ { "op": "client.diagnostic", "category": "diagnostic", - "site": "packages/keiko-server/src/client-diagnostics-routes.ts:119", + "site": "packages/keiko-server/src/client-diagnostics-routes.ts:122", "package": "keiko-server" }, { "op": "client.diagnostic.rate-limited", "category": "diagnostic", - "site": "packages/keiko-server/src/client-diagnostics-routes.ts:101", + "site": "packages/keiko-server/src/client-diagnostics-routes.ts:104", "package": "keiko-server" }, { @@ -779,7 +779,7 @@ { "op": "http.request.body.received", "category": "http", - "site": "packages/keiko-server/src/bounded-request-body.ts:142", + "site": "packages/keiko-server/src/bounded-request-body.ts:151", "package": "keiko-server" }, { @@ -875,7 +875,7 @@ { "op": "request", "category": "http", - "site": "packages/keiko-server/src/server.ts:478", + "site": "packages/keiko-server/src/server.ts:489", "package": "keiko-server" }, { diff --git a/packages/keiko-contracts/src/diagnostics.test.ts b/packages/keiko-contracts/src/diagnostics.test.ts index efee2836fa..1bea1697f0 100644 --- a/packages/keiko-contracts/src/diagnostics.test.ts +++ b/packages/keiko-contracts/src/diagnostics.test.ts @@ -71,6 +71,27 @@ describe("isClientDiagnosticIngestRequest", () => { ); }); + // `Date.parse` silently normalizes a calendar-invalid instant instead of rejecting it (e.g. + // `2026-02-30T10:00:00.000Z` becomes `2026-03-02T10:00:00.000Z`), so a shape-only regex plus + // `!Number.isNaN(Date.parse(...))` accepts a date that never happened on the calendar. + it("rejects a calendar-invalid clientTs that Date.parse would silently normalize", () => { + expect( + isClientDiagnosticIngestRequest({ ...validRequest(), clientTs: "2026-02-30T10:00:00.000Z" }), + ).toBe(false); + // April has 30 days; the 31st does not exist. + expect( + isClientDiagnosticIngestRequest({ ...validRequest(), clientTs: "2026-04-31T00:00:00Z" }), + ).toBe(false); + // Hour 24 does not exist as a clock value. + expect( + isClientDiagnosticIngestRequest({ ...validRequest(), clientTs: "2026-08-21T24:00:00Z" }), + ).toBe(false); + // The last valid day of February in a non-leap year. + expect( + isClientDiagnosticIngestRequest({ ...validRequest(), clientTs: "2027-02-28T00:00:00Z" }), + ).toBe(true); + }); + it("rejects a readyState outside the closed 0|1|2 vocabulary", () => { expect(isClientDiagnosticIngestRequest({ ...validRequest(), readyState: 3 })).toBe(false); expect(isClientDiagnosticIngestRequest({ ...validRequest(), readyState: "1" })).toBe(false); diff --git a/packages/keiko-contracts/src/diagnostics.ts b/packages/keiko-contracts/src/diagnostics.ts index 0e16b78742..6ee40c234a 100644 --- a/packages/keiko-contracts/src/diagnostics.ts +++ b/packages/keiko-contracts/src/diagnostics.ts @@ -57,7 +57,7 @@ const ISO_INSTANT_MAX_LENGTH = 40; // Deliberately less strict than `correlation.ts`'s SAFE_CORRELATION_ID: this file only asserts the // wire SHAPE (a short, non-empty string) so the leaf never has to import server plumbing. The // server re-validates with `isValidCorrelationId` before trusting the value for anything. -const ISO_INSTANT_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/; +const ISO_INSTANT_PATTERN = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,3})?Z$/; export interface ClientDiagnosticIngestRequest { readonly message: string; @@ -75,14 +75,40 @@ function isBoundedString(value: unknown, maxLength: number): value is string { return typeof value === "string" && value.length > 0 && value.length <= maxLength; } -function isIsoInstant(value: unknown): value is string { - return ( - isBoundedString(value, ISO_INSTANT_MAX_LENGTH) && - ISO_INSTANT_PATTERN.test(value) && - !Number.isNaN(Date.parse(value)) +// UTC component getters in the same order the capture groups appear in `ISO_INSTANT_PATTERN` +// (year, month, day, hour, minute, second) — `getUTCMonth()` is 0-based, so it is adjusted to match +// the 1-based literal the input wrote. +const ISO_INSTANT_UTC_GETTERS: readonly ((date: Date) => number)[] = [ + (date): number => date.getUTCFullYear(), + (date): number => date.getUTCMonth() + 1, + (date): number => date.getUTCDate(), + (date): number => date.getUTCHours(), + (date): number => date.getUTCMinutes(), + (date): number => date.getUTCSeconds(), +]; + +// `Date.parse` silently normalizes a calendar-invalid instant instead of rejecting it (e.g. +// `2026-02-30T10:00:00.000Z` becomes `2026-03-02T10:00:00.000Z`), so a regex-shape match plus a +// non-NaN parse is not enough on its own: reparse the accepted ms value and require every UTC +// component the input literally said to still be there. All six capture groups in +// `ISO_INSTANT_PATTERN` are mandatory (only the milliseconds fraction is optional, and it is +// non-capturing), so `match[1..6]` is always populated once `match` itself is non-null. +function isCalendarValidInstant(match: RegExpExecArray, parsedMs: number): boolean { + const components = match.slice(1, 7); + const parsed = new Date(parsedMs); + return ISO_INSTANT_UTC_GETTERS.every( + (getUtcComponent, index) => getUtcComponent(parsed) === Number(components[index]), ); } +function isIsoInstant(value: unknown): value is string { + if (!isBoundedString(value, ISO_INSTANT_MAX_LENGTH)) return false; + const match = ISO_INSTANT_PATTERN.exec(value); + if (match === null) return false; + const parsedMs = Date.parse(value); + return !Number.isNaN(parsedMs) && isCalendarValidInstant(match, parsedMs); +} + const CLIENT_DIAGNOSTIC_READY_STATE_SET: ReadonlySet = new Set( CLIENT_DIAGNOSTIC_READY_STATES, ); diff --git a/packages/keiko-server/src/bounded-request-body.test.ts b/packages/keiko-server/src/bounded-request-body.test.ts index 219d03dab5..070be8cab0 100644 --- a/packages/keiko-server/src/bounded-request-body.test.ts +++ b/packages/keiko-server/src/bounded-request-body.test.ts @@ -344,6 +344,27 @@ describe("bounded request body activity log", () => { ]); }); + it("collapses an arbitrary Content-Type subtype to the fixed 'other' label instead of logging it", async () => { + const sink = captureServerLog("debug"); + const stream = new PassThrough(); + const req = asRequest(stream); + Object.defineProperty(req, "headers", { + configurable: true, + value: { "content-type": "application/x-secret-token-abc123; charset=utf-8" }, + }); + + await expect( + (async (): Promise => { + const outcome = readBoundedRequestBody(req, 128_000, undefined, "req-hostile-ct"); + stream.end(Buffer.from("hi")); + return outcome; + })(), + ).resolves.toBe("hi"); + + expect(sink.events[0]?.extra).toEqual({ contentType: "other", receivedBytes: 2 }); + expect(JSON.stringify(sink.events)).not.toContain("secret-token-abc123"); + }); + it("falls back to the closed 'unspecified' label when no Content-Type header was sent", async () => { const sink = captureServerLog("debug"); const req = asRequest(Readable.from([Buffer.from("hi")])); @@ -425,4 +446,47 @@ describe("readJsonRequestBody", () => { expect(result).toEqual({}); }); + + // `JSON.parse` never triggers a prototype-setter for an own `"__proto__"` key — it defines it + // as a plain, enumerable, OWN data property, exactly like any other key. The parsed object's + // prototype stays `Object.prototype`, so this is not prototype pollution and the caller reads + // back exactly the object it sent, with `__proto__` as an ordinary field name. + it("returns a hostile '__proto__' key as an ordinary own property, not a polluted prototype", async () => { + const req = asRequest(Readable.from([Buffer.from('{"__proto__":{"polluted":true}}')])); + + const result = await readJsonRequestBody(req, 128_000); + + // Object.prototype itself must stay clean: an unrelated, freshly-created object never picks + // up "polluted" through its prototype chain. + expect(({} as { polluted?: unknown }).polluted).toBeUndefined(); + expect(Object.getPrototypeOf(result)).toBe(Object.prototype); + expect(Object.prototype.hasOwnProperty.call(result, "__proto__")).toBe(true); + expect((result as { __proto__: unknown }).__proto__).toEqual({ polluted: true }); + }); + + it("returns 400 BAD_REQUEST for valid JSON that is not an object (null)", async () => { + const req = asRequest(Readable.from([Buffer.from("null")])); + + const result = await readJsonRequestBody(req, 128_000); + + expect(result).toEqual({ + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body must be a JSON object." } }, + }); + }); + + it.each([ + ["a number", "42"], + ["a string", '"hello"'], + ["a boolean", "true"], + ])("returns 400 BAD_REQUEST for valid JSON that is not an object (%s)", async (_label, raw) => { + const req = asRequest(Readable.from([Buffer.from(raw)])); + + const result = await readJsonRequestBody(req, 128_000); + + expect(result).toEqual({ + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body must be a JSON object." } }, + }); + }); }); diff --git a/packages/keiko-server/src/bounded-request-body.ts b/packages/keiko-server/src/bounded-request-body.ts index 6cf0ee8fb7..65bd4cf4a0 100644 --- a/packages/keiko-server/src/bounded-request-body.ts +++ b/packages/keiko-server/src/bounded-request-body.ts @@ -118,15 +118,24 @@ function safeContentTypeHeader(req: IncomingMessage): ContentTypeHeaderValue { return headers.headers?.["content-type"]; } +// Every media type this server's body readers are ever legitimately reached with: `server.ts`'s +// `isJsonRequest` gate already rejects any state-changing request whose Content-Type is not +// exactly `application/json` with a 415 before a handler can read its body. Allowlisted rather +// than left open, because stripping `; charset=...` parameters does not make an arbitrary +// subtype safe to log — a client fully controls the whole header and can place sensitive data in +// a syntactically valid subtype (e.g. `Content-Type: application/`) that never reaches +// this gate at all. +const KNOWN_REQUEST_MEDIA_TYPES = new Set(["application/json"]); + // Reduces a `Content-Type` header to its media type, discarding parameters (`; charset=utf-8`, -// `; boundary=...`) that can carry caller-chosen, unbounded text. Mirrors `isJsonRequest`'s -// reduction in `server.ts`. A media type is left readable by `log-redaction.ts`'s deep-path guard -// on purpose (it is at most one `/`, never three-plus path segments), so no further redaction is -// needed once it is isolated this way. +// `; boundary=...`) that can carry caller-chosen, unbounded text, then maps it through the +// allowlist above. A subtype this reader has no reason to ever see collapses to the fixed label +// `"other"` rather than being retained verbatim in the diagnostic sink. function mediaTypeOf(header: ContentTypeHeaderValue): string { const value = typeof header === "string" ? header : header?.[0]; const mediaType = value?.split(";", 1)[0]?.trim().toLowerCase(); - return mediaType === undefined || mediaType.length === 0 ? "unspecified" : mediaType; + if (mediaType === undefined || mediaType.length === 0) return "unspecified"; + return KNOWN_REQUEST_MEDIA_TYPES.has(mediaType) ? mediaType : "other"; } // The one success line this reader emits, at debug: per-request volume makes it unfit for info, diff --git a/packages/keiko-server/src/chat-stream-handlers.test.ts b/packages/keiko-server/src/chat-stream-handlers.test.ts index d3d0e541ad..c462ea24c3 100644 --- a/packages/keiko-server/src/chat-stream-handlers.test.ts +++ b/packages/keiko-server/src/chat-stream-handlers.test.ts @@ -3363,4 +3363,33 @@ describe("desktop chat SSE stream correlationId threading (#2902 audit finding 0 expect(closed).toHaveLength(1); expect(closed[0]?.correlationId).toBeUndefined(); }); + + it("attaches the correlationId to sse.stream.closed even when the model throws before the first chunk", async () => { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + const chatId = seedChat(); + const captured = captureResWithEvents(); + const failing: ModelPort = { + call: () => Promise.resolve(normalizedResponse("unused")), + // eslint-disable-next-line require-yield -- fails before any chunk by design + async *callStream(): AsyncGenerator { + await Promise.resolve(); + throw new Error("upstream exploded before first token"); + }, + }; + const ctx: RouteContext = { + ...routeContext( + makeReq({ chatId, projectPath: projectDir, modelId: CHAT_MODEL, content: "hello" }), + captured.res, + ), + correlationId: "corr-chat-stream-early-failure", + }; + + await handleSendDesktopChatStream(ctx, deps(failing)); + captured.emitClose(); + + const closed = sink.events.filter((event) => event.op === "sse.stream.closed"); + expect(closed).toHaveLength(1); + expect(closed[0]?.correlationId).toBe("corr-chat-stream-early-failure"); + }); }); diff --git a/packages/keiko-server/src/chat-stream-handlers.ts b/packages/keiko-server/src/chat-stream-handlers.ts index 9079b1cfee..d8968c4cf9 100644 --- a/packages/keiko-server/src/chat-stream-handlers.ts +++ b/packages/keiko-server/src/chat-stream-handlers.ts @@ -9,7 +9,7 @@ // JSON RouteResult BEFORE any SSE header so the client can fall back to the buffered route. import { SSE_HEADERS, startSseHeartbeat } from "./sse.js"; -import { writeOrDestroy } from "./sse-write.js"; +import { recordSseStreamFrame, writeOrDestroy } from "./sse-write.js"; import { STREAMING, errorBody, @@ -180,8 +180,17 @@ async function streamConversation( // A backpressure kill destroys the socket; writing another SSE frame to it is a no-op at best and can // throw on some transports. Guard terminal writes so we never write-after-destroy nor relabel a // backpressure termination as a user cancel. +// +// Every terminal frame is recorded via `recordSseStreamFrame` BEFORE the write (#2902 audit finding +// 0 follow-up), not only the per-token `writeOrDestroy` calls inside `streamConversation`. Without +// this, a stream that errors or is cancelled before its first token (e.g. the model throws on the +// very first `callStream` iteration) never calls `recordSseStreamFrame` at all: the per-stream state +// — and the `res.on("close", …)` listener that emits the terminal `sse.stream.closed` line — is only +// created lazily on the first recorded frame, so such a stream produced no closed line and no +// correlationId, silently disappearing from the operator trail. function writeTerminalFrame(ctx: RouteContext, frame: string): void { if (ctx.res.writableEnded || ctx.res.destroyed) return; + recordSseStreamFrame(ctx.res, frame, ctx.correlationId); ctx.res.write(frame); } diff --git a/packages/keiko-server/src/client-diagnostics-routes.ts b/packages/keiko-server/src/client-diagnostics-routes.ts index 6fbbd7aed2..d46cb8dae7 100644 --- a/packages/keiko-server/src/client-diagnostics-routes.ts +++ b/packages/keiko-server/src/client-diagnostics-routes.ts @@ -56,11 +56,18 @@ const MAX_CLIENT_DIAGNOSTIC_BODY_BYTES = 4_096; // page. `minIntervalMs: 0` disables the limiter's own burst/cooldown gate, so only the sliding // window cap below applies. const CLIENT_DIAGNOSTIC_RATE_LIMIT_KEY = "client-diagnostics"; -let rateLimiter: InlineCompletionRateLimiter = createInlineCompletionRateLimiter({ + +// One declaration for production and the test reset below — duplicating these three literals let +// them drift, so the test reset silently exercised a limiter with different bounds than production. +const CLIENT_DIAGNOSTIC_RATE_LIMIT_CONFIG = { minIntervalMs: 0, maxPerWindow: 60, windowMs: 60_000, -}); +} as const; + +let rateLimiter: InlineCompletionRateLimiter = createInlineCompletionRateLimiter( + CLIENT_DIAGNOSTIC_RATE_LIMIT_CONFIG, +); const DROP_NOTICE_WINDOW_MS = 60_000; @@ -73,11 +80,7 @@ let dropNotice: DropNoticeState = { lastAt: null, suppressed: 0 }; /** Test-only: puts the shared rate limiter and drop-notice counter back to a clean start. */ export function resetClientDiagnosticsIngestStateForTests(): void { - rateLimiter = createInlineCompletionRateLimiter({ - minIntervalMs: 0, - maxPerWindow: 60, - windowMs: 60_000, - }); + rateLimiter = createInlineCompletionRateLimiter(CLIENT_DIAGNOSTIC_RATE_LIMIT_CONFIG); dropNotice = { lastAt: null, suppressed: 0 }; } diff --git a/packages/keiko-server/src/gitRoutes.test.ts b/packages/keiko-server/src/gitRoutes.test.ts index abedfa54c8..3ddd121d10 100644 --- a/packages/keiko-server/src/gitRoutes.test.ts +++ b/packages/keiko-server/src/gitRoutes.test.ts @@ -18,11 +18,13 @@ import { createRunRegistry, type UiHandlerDeps, } from "./index.js"; -import type { RouteContext } from "./routes.js"; +import { matchRoute, type RouteContext } from "./routes.js"; import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; import type { UiStore } from "./store/index.js"; import { mockRequest, mockResponse } from "./_support.js"; import { + GIT_DIFF_ROUTE_TEMPLATE, + GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE, handleGitBranches, handleGitBlame, handleGitDiff, @@ -1725,3 +1727,29 @@ describe("GET /api/git/blame", () => { }, ); }); + +// The `operation` string `gitReadErrorBody` reports for a diff failure is built from +// `GIT_DIFF_ROUTE_TEMPLATE` / `GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE` (gitRoutes.ts), a literal +// that used to be restated separately in routes.ts's own `pattern` field with nothing linking +// the two — a route rename could silently leave the diagnostic reporting the old path. Both +// constants are now imported into routes.ts as the registered `pattern`, so this pin resolves +// each one through the real router (`matchRoute`) rather than comparing the constant to itself. +describe("git diff route templates stay pinned to the registered routes", () => { + it("resolves GIT_DIFF_ROUTE_TEMPLATE to the route actually registered for GET /api/git/diff", () => { + const match = matchRoute("GET", GIT_DIFF_ROUTE_TEMPLATE); + expect(match).not.toBe("method-not-allowed"); + expect(match).not.toBeUndefined(); + expect((match as { definition: { pattern: string } }).definition.pattern).toBe( + GIT_DIFF_ROUTE_TEMPLATE, + ); + }); + + it("resolves GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE to the route actually registered for GET /api/git/diff/structured", () => { + const match = matchRoute("GET", GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE); + expect(match).not.toBe("method-not-allowed"); + expect(match).not.toBeUndefined(); + expect((match as { definition: { pattern: string } }).definition.pattern).toBe( + GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE, + ); + }); +}); diff --git a/packages/keiko-server/src/gitRoutes.ts b/packages/keiko-server/src/gitRoutes.ts index e1127f062f..61fd595260 100644 --- a/packages/keiko-server/src/gitRoutes.ts +++ b/packages/keiko-server/src/gitRoutes.ts @@ -1289,9 +1289,12 @@ class GitRouteReadError extends Error { // The declared route templates `gitReadErrorBody` reports as `operation` — literal constants, not // derived from any live request, so the two routes sharing `runGitDiffHandler` can never be -// confused for one another and neither can ever carry a request-supplied segment. -const GIT_DIFF_ROUTE_TEMPLATE = "/api/git/diff"; -const GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE = "/api/git/diff/structured"; +// confused for one another and neither can ever carry a request-supplied segment. Exported so +// `routes.ts` registers these exact strings as the route `pattern` too: one declaration used by +// both the route table and this diagnostic, so a renamed route pattern cannot leave this file +// silently reporting the old path. +export const GIT_DIFF_ROUTE_TEMPLATE = "/api/git/diff"; +export const GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE = "/api/git/diff/structured"; // `routeTemplate` is the DECLARED route pattern the caller is answering for (e.g. // `"/api/git/diff"`), never `ctx.url.pathname` — the live request path is not read here, so a diff --git a/packages/keiko-server/src/routes.ts b/packages/keiko-server/src/routes.ts index edbcbc322c..56dc4a6feb 100644 --- a/packages/keiko-server/src/routes.ts +++ b/packages/keiko-server/src/routes.ts @@ -179,6 +179,8 @@ import { handleFilesTree, } from "./files.js"; import { + GIT_DIFF_ROUTE_TEMPLATE, + GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE, handleGitBlame, handleGitBranches, handleGitDiff, @@ -597,12 +599,12 @@ export const API_ROUTES: readonly RouteDefinition[] = [ }, { method: "GET", - pattern: "/api/git/diff", + pattern: GIT_DIFF_ROUTE_TEMPLATE, handler: (ctx, deps) => handleGitDiff(ctx, deps, deps.gitRouteOptions), }, { method: "GET", - pattern: "/api/git/diff/structured", + pattern: GIT_STRUCTURED_DIFF_ROUTE_TEMPLATE, handler: (ctx, deps) => handleGitStructuredDiff(ctx, deps, deps.gitRouteOptions), }, { diff --git a/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts b/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts index be3a1e9547..34502f4795 100644 --- a/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts +++ b/packages/keiko-server/src/run-handlers-sse-backpressure.test.ts @@ -27,20 +27,28 @@ import { } from "./observability/index.js"; // A minimal `ServerResponse`-shaped double (mirrors `sse-write.test.ts`'s `listenableFakeRes`): -// `write` always rejects, so the very first frame trips the writer's own destroy path. `on` records -// listeners so the test can fire "close" deterministically, exactly as the terminal-line mechanism -// does on a real socket teardown. -function rejectingFakeRes(): { res: RouteContext["res"]; fireClose: () => void } { +// `write` records the frame it was given, then always rejects, so the very first frame trips the +// writer's own destroy path. `on` records listeners so the test can fire "close" deterministically, +// exactly as the terminal-line mechanism does on a real socket teardown. +function rejectingFakeRes(): { + res: RouteContext["res"]; + fireClose: () => void; + readonly frames: readonly string[]; +} { // A real `ServerResponse` is an EventEmitter: `.on("close", ...)` may legitimately be called more // than once (this route does — once for the stream's own frame/byte tracking, once for its // sink-detach cleanup) and every registered listener fires on a real "close" event. A single-slot // map here would silently drop all but the last registration and misreport this regression as // fixed regardless of whether the product code is correct. const listeners = new Map void)[]>(); + const frames: string[] = []; const res = { writableEnded: false, destroyed: false, - write: (): boolean => false, + write: (chunk: string): boolean => { + frames.push(chunk); + return false; + }, writeHead: (): void => undefined, end: (): void => undefined, destroy: (): void => undefined, @@ -52,12 +60,33 @@ function rejectingFakeRes(): { res: RouteContext["res"]; fireClose: () => void } } as unknown as RouteContext["res"]; return { res, + frames, fireClose: (): void => { for (const handler of listeners.get("close") ?? []) handler(); }, }; } +// The event fixtures' `ts` is not part of the behavior under test (only `reason` is asserted) — +// wall-clock `Date.now()` made them non-deterministic for no reason ("Fixtures are deterministic +// and self-contained", #2902 audit thread 10). A fixed value also lets the tests below prove the +// fixture is what actually reached the wire, rather than merely being unobserved. +const FIXED_EVENT_TS = 1_700_000_000_000; + +// The two writers order their `readyMessage()` frame (no `ts` field) and the fixture's event frame +// differently — `openSseStream` replays buffered events before the ready frame, `aggregateRunWriter` +// only fans out an event emitted after it — so this scans every recorded frame for the one carrying +// a `ts`, rather than assuming a fixed position. +function eventFrameTs(frames: readonly string[]): unknown { + for (const frame of frames) { + const dataLine = frame.split("\n").find((line) => line.startsWith("data: ")); + if (dataLine === undefined) continue; + const parsed = JSON.parse(dataLine.slice("data: ".length)) as { ts?: unknown }; + if (parsed.ts !== undefined) return parsed.ts; + } + return undefined; +} + function fakeReq(): RouteContext["req"] { return { headers: {}, on: (): void => undefined } as unknown as RouteContext["req"]; } @@ -100,7 +129,7 @@ describe("SSE writers report reason=backpressure-killed, not client-disconnected runId: "run-bp-1", fingerprint: "fp-bp-1", seq: 0, - ts: Date.now(), + ts: FIXED_EVENT_TS, type: "workflow:progress", }); registry.register({ @@ -111,7 +140,7 @@ describe("SSE writers report reason=backpressure-killed, not client-disconnected cancel: () => undefined, }); const deps = minimalDeps(registry); - const { res, fireClose } = rejectingFakeRes(); + const { res, frames, fireClose } = rejectingFakeRes(); const ctx: RouteContext = { req: fakeReq(), res, @@ -123,6 +152,7 @@ describe("SSE writers report reason=backpressure-killed, not client-disconnected fireClose(); expect(terminalReason(sink)).toBe("backpressure-killed"); + expect(eventFrameTs(frames)).toBe(FIXED_EVENT_TS); deps.store.close(); }); @@ -138,7 +168,7 @@ describe("SSE writers report reason=backpressure-killed, not client-disconnected cancel: () => undefined, }); const deps = minimalDeps(registry); - const { res, fireClose } = rejectingFakeRes(); + const { res, frames, fireClose } = rejectingFakeRes(); const ctx: RouteContext = { req: fakeReq(), res, @@ -154,12 +184,13 @@ describe("SSE writers report reason=backpressure-killed, not client-disconnected runId: "run-bp-2", fingerprint: "fp-bp-2", seq: 0, - ts: Date.now(), + ts: FIXED_EVENT_TS, type: "workflow:progress", }); fireClose(); expect(terminalReason(sink)).toBe("backpressure-killed"); + expect(eventFrameTs(frames)).toBe(FIXED_EVENT_TS); deps.store.close(); }); }); diff --git a/packages/keiko-server/src/run-handlers-sse-correlation.test.ts b/packages/keiko-server/src/run-handlers-sse-correlation.test.ts new file mode 100644 index 0000000000..e60edc8bf2 --- /dev/null +++ b/packages/keiko-server/src/run-handlers-sse-correlation.test.ts @@ -0,0 +1,150 @@ +// Regression (#2902 audit thread 11): `run-handlers.ts`'s two SSE writers — `openSseStream`'s +// writer, reached from `handleRunEvents` (GET /api/runs/:runId/events), and `aggregateRunWriter`, +// reached from `handleAllRunEvents` (GET /api/runs/events) — never threaded the request's +// `ctx.correlationId` into their `writeMessageEvent` calls, so neither route's `sse.stream.closed` +// terminal line ever carried the correlation id, unlike the desktop chat stream and the relationship +// activity broadcaster (#2902 w5-sse-counters finding 0). Kept in its own file for the same reason +// as `run-handlers-sse-backpressure.test.ts`: these tests call the two handlers directly against +// hand-built doubles and never bind a real socket, so sharing `run-handlers.test.ts`'s real bound +// HTTP server lifecycle would be a foreign, non-hermetic dependency. + +import { afterEach, describe, expect, it } from "vitest"; +import { EventEmitter } from "node:events"; + +import { buildRedactor, createRunRegistry, handleRunEvents, QueueEventSink } from "./index.js"; +import { handleAllRunEvents } from "./run-handlers.js"; +import type { RouteContext } from "./routes.js"; +import type { UiHandlerDeps } from "./deps.js"; +import { createInMemoryUiStore } from "./store/index.js"; +import { + createBufferedServerLogSink, + createServerLogger, + resetServerLogger, + setServerLogger, + type BufferedServerLogSink, +} from "./observability/index.js"; + +// A minimal, ACCEPTING `ServerResponse` double (mirrors `run-handlers-sse-backpressure.test.ts`'s +// `rejectingFakeRes`, but `write` always succeeds): every event this file emits is meant to reach +// the wire so the resulting `sse.stream.closed` line can be inspected for its correlationId. +function listenableFakeRes(): { res: RouteContext["res"]; fireClose: () => void } { + const emitter = new EventEmitter(); + const res = { + writableEnded: false, + destroyed: false, + write: (): boolean => true, + writeHead: (): void => undefined, + end: (): void => undefined, + destroy: (): void => undefined, + on: (event: string, handler: (...args: unknown[]) => void): void => { + emitter.on(event, handler); + }, + } as unknown as RouteContext["res"]; + return { res, fireClose: (): void => emitter.emit("close") }; +} + +function fakeReq(): RouteContext["req"] { + return { headers: {}, on: (): void => undefined } as unknown as RouteContext["req"]; +} + +function captureServerLog(): BufferedServerLogSink { + const sink = createBufferedServerLogSink(); + setServerLogger(createServerLogger({ sink, level: "info" })); + return sink; +} + +function minimalDeps(registry: ReturnType): UiHandlerDeps { + return { + config: undefined, + configPresent: false, + evidenceStore: { put: () => "", list: () => [], get: () => undefined, delete: () => undefined }, + env: {}, + redactor: buildRedactor({}), + registry, + modelPortFactory: () => undefined, + store: createInMemoryUiStore(), + }; +} + +function terminalCorrelationId(sink: BufferedServerLogSink): unknown { + const closedLine = sink.events.find((event) => event.op === "sse.stream.closed"); + return closedLine?.correlationId; +} + +afterEach(() => { + resetServerLogger(); +}); + +describe("run SSE writers thread the request correlationId (#2902 audit thread 11)", () => { + it("handleRunEvents (openSseStream) attaches ctx.correlationId to sse.stream.closed", () => { + const sink = captureServerLog(); + const registry = createRunRegistry(); + const eventSink = new QueueEventSink(); + registry.register({ + runId: "run-corr-1", + fingerprint: "fp-corr-1", + modelId: "test-model", + sink: eventSink, + cancel: () => undefined, + }); + const deps = minimalDeps(registry); + const { res, fireClose } = listenableFakeRes(); + const ctx: RouteContext = { + req: fakeReq(), + res, + params: { runId: "run-corr-1" }, + url: new URL("http://localhost/api/runs/run-corr-1/events"), + correlationId: "corr-run-events-1", + }; + + handleRunEvents(ctx, deps); + eventSink.emit({ + schemaVersion: "1", + runId: "run-corr-1", + fingerprint: "fp-corr-1", + seq: 0, + ts: 1_700_000_000_000, + type: "workflow:progress", + }); + fireClose(); + + expect(terminalCorrelationId(sink)).toBe("corr-run-events-1"); + deps.store.close(); + }); + + it("handleAllRunEvents (aggregateRunWriter) attaches ctx.correlationId to sse.stream.closed", () => { + const sink = captureServerLog(); + const registry = createRunRegistry(); + const eventSink = new QueueEventSink(); + registry.register({ + runId: "run-corr-2", + fingerprint: "fp-corr-2", + modelId: "test-model", + sink: eventSink, + cancel: () => undefined, + }); + const deps = minimalDeps(registry); + const { res, fireClose } = listenableFakeRes(); + const ctx: RouteContext = { + req: fakeReq(), + res, + params: {}, + url: new URL("http://localhost/api/runs/events"), + correlationId: "corr-all-run-events-1", + }; + + handleAllRunEvents(ctx, deps); + eventSink.emit({ + schemaVersion: "1", + runId: "run-corr-2", + fingerprint: "fp-corr-2", + seq: 0, + ts: 1_700_000_000_000, + type: "workflow:progress", + }); + fireClose(); + + expect(terminalCorrelationId(sink)).toBe("corr-all-run-events-1"); + deps.store.close(); + }); +}); diff --git a/packages/keiko-server/src/run-handlers.ts b/packages/keiko-server/src/run-handlers.ts index 1bc8d1e1b6..18f816c37f 100644 --- a/packages/keiko-server/src/run-handlers.ts +++ b/packages/keiko-server/src/run-handlers.ts @@ -423,7 +423,7 @@ export function handleRunEvents(ctx: RouteContext, deps: UiHandlerDeps): Handler if (!agentRecordSessionMatches(record, ctx, deps)) { return { status: 404, body: errorBody("NOT_FOUND", "Unknown run.") }; } - openSseStream(ctx.res, record, lastEventId(ctx.req), deps.redactor); + openSseStream(ctx.res, record, lastEventId(ctx.req), deps.redactor, ctx.correlationId); ctx.req.on("close", () => { ctx.res.end(); }); @@ -486,7 +486,7 @@ function aggregateRunWriter( return { write: (event: StreamEvent): boolean => { if (!agentRecordSessionMatches(record, ctx, deps)) return false; - const accepted = writeMessageEvent(ctx.res, event, deps.redactor); + const accepted = writeMessageEvent(ctx.res, event, deps.redactor, ctx.correlationId); if (!accepted) { markSseStreamBackpressureKilled(ctx.res); ctx.res.destroy(); @@ -515,12 +515,13 @@ function openSseStream( record: RunRecord, afterSeq: number, redactor: UiHandlerDeps["redactor"], + correlationId?: string, ): void { res.writeHead(200, SSE_HEADERS); startSseHeartbeat(res); const writer: SseWriter = { write: (event: StreamEvent): boolean => { - const accepted = writeMessageEvent(res, event, redactor); + const accepted = writeMessageEvent(res, event, redactor, correlationId); if (!accepted) { markSseStreamBackpressureKilled(res); res.destroy(); diff --git a/packages/keiko-server/src/server.test.ts b/packages/keiko-server/src/server.test.ts index 55e4025613..f98fe749ef 100644 --- a/packages/keiko-server/src/server.test.ts +++ b/packages/keiko-server/src/server.test.ts @@ -6,7 +6,7 @@ import { request } from "node:http"; import type { AddressInfo } from "node:net"; import type { IncomingMessage, Server, ServerResponse } from "node:http"; import { gunzipSync } from "node:zlib"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import type { GatewayConfig, GatewayRequest, @@ -19,12 +19,23 @@ import { createRunRegistry, type UiHandlerDeps, } from "./index.js"; -import { createUiServer, logRequestOnClose, UI_HOST, type RequestLogContext } from "./server.js"; +import { + computeQueryParamFields, + createUiServer, + logRequestOnClose, + MAX_QUERY_PARAM_NAMES, + UI_HOST, + type RequestLogContext, +} from "./server.js"; import type { ServerDiagnosticRecord } from "./diagnostics-log.js"; import { buildCspHeader } from "./csp.js"; import { resetWorkspaceStateForTests } from "./workspace-state-handlers.js"; import type { EditorHotExitStore } from "./editor/hotExitStore.js"; -import { createBufferedServerLogSink, type ServerLogEvent } from "./observability/index.js"; +import { + createBufferedServerLogSink, + type BufferedServerLogSink, + type ServerLogEvent, +} from "./observability/index.js"; let server: Server; let staticRoot: string; @@ -1107,25 +1118,58 @@ describe("top-level route-error catch (GEN-TEST-MISSING-008, RB-6)", () => { // the pre-existing lossy raw-path re-derivation and Node's un-set-by-default `res.statusCode`. Every // field asserted here did not exist on the pre-fix http-request line at all, so each assertion below // fails before this wave's `server.ts` change and passes after it. +interface AwaitableActivityLogSink extends BufferedServerLogSink { + readonly nextEvent: () => Promise; +} + +// Wraps `createBufferedServerLogSink` so a waiter can be notified the instant a write happens, +// instead of polling `sink.events.length` on a `setTimeout` loop — the sink is the one place that +// knows when an event actually arrived, so it is the one place that should resolve the wait. +function createAwaitableActivityLogSink(): AwaitableActivityLogSink { + const buffered = createBufferedServerLogSink(); + let onNextEvent: ((event: ServerLogEvent) => void) | undefined; + return { + ...buffered, + write(event: ServerLogEvent): void { + buffered.write(event); + const notify = onNextEvent; + onNextEvent = undefined; + notify?.(event); + }, + nextEvent(): Promise { + const [existing] = buffered.events; + if (existing !== undefined) return Promise.resolve(existing); + return new Promise((resolve) => { + onNextEvent = resolve; + }); + }, + }; +} + describe("activity log: http-request line enrichment (Wave 5, w5-http-request-enrichment)", () => { + // The 2000ms deadline is a safety net for a genuinely missing event, never the mechanism that + // detects arrival: the happy path always settles through `sink.nextEvent()` below. async function waitForActivityLogEvent( - sink: ReturnType, + sink: AwaitableActivityLogSink, timeoutMs = 2000, ): Promise { - const deadline = Date.now() + timeoutMs; - while (sink.events.length === 0) { - if (Date.now() > deadline) { - throw new Error("timed out waiting for an activity log event"); - } - await new Promise((resolve) => setTimeout(resolve, 5)); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + sink.nextEvent(), + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new Error("timed out waiting for an activity log event")); + }, timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); } - const [event] = sink.events; - if (event === undefined) throw new Error("unreachable: length checked above"); - return event; } - async function startWithActivityLog(): Promise> { - const sink = createBufferedServerLogSink(); + async function startWithActivityLog(): Promise { + const sink = createAwaitableActivityLogSink(); await closeServer(); server = createUiServer({ staticRoot, csp: buildCspHeader([]), port, activityLog: sink }); await new Promise((res) => server.listen(port, UI_HOST, res)); @@ -1167,6 +1211,19 @@ describe("activity log: http-request line enrichment (Wave 5, w5-http-request-en expect(serialized).not.toContain("=2"); }); + // #2902 audit: the kept names used to be sorted with `localeCompare`, which orders by ICU + // collation rules (case-folded, locale-dependent) rather than by code point. "Banana" sorts + // AFTER "apple" under `localeCompare` (locale-aware) but BEFORE it under a plain code-point + // compare, since the uppercase 'B' (66) is less than the lowercase 'a' (97). A field compared + // or deduplicated across hosts needs one order regardless of host ICU data/default locale. + it("sorts kept query-parameter names by code point, not by locale collation", async () => { + const sink = await startWithActivityLog(); + await fetchRaw("/api/health?Banana=1&apple=2"); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.queryParamNames).toEqual(["Banana", "apple"]); + }); + it("drops an over-length query-parameter name and counts it instead of logging it", async () => { const sink = await startWithActivityLog(); const longName = "n".repeat(200); @@ -1198,39 +1255,53 @@ describe("activity log: http-request line enrichment (Wave 5, w5-http-request-en }); // #2902 audit finding 1: computeQueryParamFields used to sort the FULL kept array before - // slicing it to the 16-name cap (MAX_QUERY_PARAM_NAMES, mirrored here — not exported), so a - // client sending far more than 16 conforming query-param names paid an unbounded, locale-aware - // O(n log n) sort. The fix bounds the collection loop at the cap BEFORE sorting. - it("caps the collected set at 16 names before sorting, regardless of how many the client sends", async () => { + // slicing it to the MAX_QUERY_PARAM_NAMES cap, so a client sending far more than the cap's worth + // of conforming query-param names paid an unbounded, locale-aware O(n log n) sort. The fix bounds + // the collection loop at the cap BEFORE sorting. + // + // Pinned by calling the exported production function directly with a constructed URL — no real + // HTTP request, no server, no spy on the shared `Array.prototype.sort` (which would leave a + // global built-in monkey-patched for the duration of the test). Insertion order is descending + // (p19, p18, …, p00): bounding the first MAX_QUERY_PARAM_NAMES encountered THEN sorting yields + // the highest-numbered names in ascending order; sorting the full set first and slicing would + // instead yield the lowest-numbered ones. The two outcomes are disjoint, so the returned content + // is direct, self-contained proof of which happened. + it("caps the collected set before sorting, regardless of how many the client sends", () => { + const total = MAX_QUERY_PARAM_NAMES + 4; + const names = Array.from( + { length: total }, + (_, i) => `p${String(total - 1 - i).padStart(2, "0")}`, + ); + const url = new URL(`http://${UI_HOST}/api/health?${names.map((n) => `${n}=1`).join("&")}`); + const context: RequestLogContext = {}; + + computeQueryParamFields(url, context); + + const expectedKept = Array.from( + { length: MAX_QUERY_PARAM_NAMES }, + (_, i) => `p${String(i + 4).padStart(2, "0")}`, + ); + expect(context.queryParamNames).toEqual(expectedKept); + expect(context.queryParamDroppedCount).toBe(4); + }); + + // Companion end-to-end check: the same bound-before-sort behaviour observed through the real + // dispatch path (route match, host check, `writeJson`), not just the unit-level function call + // above — proves the wiring between `handle()` and `computeQueryParamFields` is intact. + it("surfaces the capped, sorted names on the real activity-log line for an oversized query string", async () => { const sink = await startWithActivityLog(); - const names = Array.from({ length: 20 }, (_, i) => `p${String(19 - i).padStart(2, "0")}`); + const total = MAX_QUERY_PARAM_NAMES + 4; + const names = Array.from( + { length: total }, + (_, i) => `p${String(total - 1 - i).padStart(2, "0")}`, + ); const query = names.map((n) => `${n}=1`).join("&"); - const sortSpy = vi.spyOn(Array.prototype, "sort"); - let event: ServerLogEvent; - let queryNameSorts: string[][]; - try { - await fetchRaw(`/api/health?${query}`); - event = await waitForActivityLogEvent(sink); - // `mock.contexts` captures the `this` receiver of every sort() call — i.e. the array that - // was actually sorted. None of them may be the `p\d\d`-shaped query-name collection at more - // than the 16-name cap: that is the direct proof the bound runs BEFORE the sort, not after. - // Read BEFORE mockRestore(), which also clears the call/context history (mockRestore does - // everything mockReset() does, plus restoring the original implementation). - queryNameSorts = sortSpy.mock.contexts.filter( - (receiver): receiver is string[] => - Array.isArray(receiver) && - receiver.every((v) => typeof v === "string" && /^p\d\d$/.test(v)), - ); - } finally { - sortSpy.mockRestore(); - } - expect(event.extra?.queryParamNames).toHaveLength(16); + await fetchRaw(`/api/health?${query}`); + const event = await waitForActivityLogEvent(sink); + + expect(event.extra?.queryParamNames).toHaveLength(MAX_QUERY_PARAM_NAMES); expect(event.extra?.queryParamDroppedCount).toBe(4); - expect(queryNameSorts.length).toBeGreaterThan(0); - for (const receiver of queryNameSorts) { - expect(receiver.length).toBeLessThanOrEqual(16); - } }); // #2902 audit finding 1: computeQueryParamFields was called unconditionally before the diff --git a/packages/keiko-server/src/server.ts b/packages/keiko-server/src/server.ts index 1588a99991..983c347fff 100644 --- a/packages/keiko-server/src/server.ts +++ b/packages/keiko-server/src/server.ts @@ -52,8 +52,10 @@ const JSON_GZIP_MIN_BYTES = 1024; const QUERY_PARAM_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_.-]{0,63}$/; // Mirrors the bounded-array discipline every other array field in this schema already has // (`MAX_LOG_ARRAY_LENGTH`, `keikoStackFrames`' own frame cap): a request with more distinct query -// parameter names than this is truncated, not silently grown without limit. -const MAX_QUERY_PARAM_NAMES = 16; +// parameter names than this is truncated, not silently grown without limit. Exported for +// `server.test.ts` only, so a cap-boundary test reads the real production limit instead of +// restating it as a second, independently-drifting literal. +export const MAX_QUERY_PARAM_NAMES = 16; const cspCache = new WeakMap< UiServerDeps, { readonly value: string; readonly expiresAt: number } @@ -342,7 +344,12 @@ async function resolveCsp(deps: UiServerDeps): Promise { // count is a synchronous, amplifiable CPU cost on the request hot path. The remaining per-name work // (Set membership + regex test) stays a cheap O(n) pass, the same order as the WHATWG URL parsing // that already runs unconditionally for routing on every request. -function computeQueryParamFields(url: URL, context: RequestLogContext): void { +// +// Exported for `server.test.ts` only, so the bound-before-sort ordering can be pinned by calling +// this function directly on a constructed `URL` — no real HTTP request, no spy on the shared +// `Array.prototype.sort`, and no risk of a second call site (in the test) drifting from this one's +// actual cap. +export function computeQueryParamFields(url: URL, context: RequestLogContext): void { const seen = new Set(); const kept: string[] = []; let dropped = 0; @@ -357,7 +364,11 @@ function computeQueryParamFields(url: URL, context: RequestLogContext): void { dropped += 1; } } - kept.sort((a, b) => a.localeCompare(b)); + // A code-point sort, not `localeCompare`: the list is capped at MAX_QUERY_PARAM_NAMES (16), so + // the cost difference is immaterial, but `localeCompare` depends on the host ICU data and the + // default locale — two servers could then emit the same request with a different + // `queryParamNames` order, and this field is compared/deduplicated across hosts. + kept.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); context.queryParamNames = kept; if (dropped > 0) context.queryParamDroppedCount = dropped; } diff --git a/packages/keiko-server/src/sse.ts b/packages/keiko-server/src/sse.ts index 0299971bc0..94f8470774 100644 --- a/packages/keiko-server/src/sse.ts +++ b/packages/keiko-server/src/sse.ts @@ -121,8 +121,9 @@ export function writeMessageEvent( res: ServerResponse, event: StreamEvent, redactor: Redactor, + correlationId?: string, ): boolean { const frame = frameMessageEvent(event, redactor); - recordSseStreamFrame(res, frame); + recordSseStreamFrame(res, frame, correlationId); return res.write(frame); } diff --git a/packages/keiko-server/src/ui-test-server/_support.test.ts b/packages/keiko-server/src/ui-test-server/_support.test.ts index d40af3b967..d2bd3203bc 100644 --- a/packages/keiko-server/src/ui-test-server/_support.test.ts +++ b/packages/keiko-server/src/ui-test-server/_support.test.ts @@ -1,8 +1,16 @@ import { mkdtempSync, rmSync } from "node:fs"; +import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { buildCspHeader } from "../csp.js"; +import { + buildRedactor, + createInMemoryUiStore, + createRunRegistry, + QueueEventSink, + type UiHandlerDeps, +} from "../index.js"; import { UI_HOST } from "../server.js"; import { closeUiTestServer, startUiTestServer, type StartedUiTestServer } from "./_support.js"; @@ -16,14 +24,31 @@ afterEach(async () => { } }); -async function start(): Promise { +async function start(handlerDeps?: UiHandlerDeps): Promise { const staticRoot = mkdtempSync(join(tmpdir(), "keiko-ui-test-server-")); roots.push(staticRoot); - const started = await startUiTestServer({ staticRoot, csp: buildCspHeader([]) }); + const started = await startUiTestServer({ + staticRoot, + csp: buildCspHeader([]), + ...(handlerDeps === undefined ? {} : { handlerDeps }), + }); servers.push(started); return started; } +function minimalHandlerDeps(): UiHandlerDeps { + return { + config: undefined, + configPresent: false, + evidenceStore: { put: () => "", list: () => [], get: () => undefined, delete: () => undefined }, + env: {}, + redactor: buildRedactor({}), + registry: createRunRegistry(), + modelPortFactory: () => undefined, + store: createInMemoryUiStore(), + }; +} + describe("UI test server lifecycle", () => { it("binds once and validates the exact OS-selected loopback authority", async () => { const started = await start(); @@ -46,4 +71,47 @@ describe("UI test server lifecycle", () => { expect(second.port).not.toBe(first.port); expect((await fetch(`http://${UI_HOST}:${String(second.port)}/api/health`)).status).toBe(200); }); + + // #2902 audit thread 7: `closeUiTestServer` only called `server.close()`, which — per Node's own + // `http.Server#close` contract — waits for every already-accepted, still-open connection to end on + // its own before the callback fires. An SSE response never blocks on that when it is inert (a raw + // idle TCP connect, or a fully-drained/ended response) — Node only holds `close()` open while a + // response is genuinely mid-stream (headers sent, `res.end()` never called). A test that reads an + // SSE stream's `ready` frame and then tears down without cancelling the client read leaves exactly + // that: a still-open response the server is waiting on. Force-close is proven by racing the close + // promise against a short timer: before the fix the timer always wins because nothing ever ends + // the still-open response. + it("force-closes a still-streaming SSE connection instead of leaving teardown pending", async () => { + const runId = randomUUID(); + const handlerDeps = minimalHandlerDeps(); + handlerDeps.registry.register({ + runId, + fingerprint: "fp-support-teardown", + modelId: "test-model", + sink: new QueueEventSink(), + cancel: () => undefined, + }); + const started = await start(handlerDeps); + servers.splice(servers.indexOf(started), 1); // closed manually below, not by afterEach + + const response = await fetch( + `http://${UI_HOST}:${String(started.port)}/api/runs/${runId}/events`, + ); + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("expected a readable SSE response body"); + await reader.read(); // consume the `ready` frame so the response is genuinely streaming + + const TIMED_OUT = Symbol("timed-out"); + const outcome = await Promise.race([ + closeUiTestServer(started.server).then(() => "closed" as const), + new Promise((resolve) => { + setTimeout(() => { + resolve(TIMED_OUT); + }, 500); + }), + ]); + + expect(outcome).toBe("closed"); + await reader.cancel().catch(() => undefined); + }); }); diff --git a/packages/keiko-server/src/ui-test-server/_support.ts b/packages/keiko-server/src/ui-test-server/_support.ts index 2e82dda2e7..5b5440ac3c 100644 --- a/packages/keiko-server/src/ui-test-server/_support.ts +++ b/packages/keiko-server/src/ui-test-server/_support.ts @@ -64,6 +64,11 @@ export async function startUiTestServer( throw new Error("Unable to bind a fresh UI test server authority"); } +// `server.close()` alone waits for every already-accepted connection to end on its own before its +// callback fires (Node's documented `http.Server#close` contract). A still-streaming SSE response a +// test read from but never cancelled — or any other lingering connection — then keeps teardown +// pending indefinitely. `closeAllConnections()` force-closes them immediately after `close()` has +// stopped the server accepting new ones, so teardown always completes (#2902 audit thread 7). export function closeUiTestServer(server: Server): Promise { return new Promise((resolve, reject) => { server.close((error) => { @@ -73,5 +78,6 @@ export function closeUiTestServer(server: Server): Promise { } reject(error); }); + server.closeAllConnections(); }); } diff --git a/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts b/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts index 9c7d063806..adceaef19f 100644 --- a/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts +++ b/packages/keiko-ui/src/lib/install-client-diagnostics.test.ts @@ -132,8 +132,8 @@ describe("fanOutClientDiagnostic", () => { expect(body["kind"]).toBe("sse-error"); }); - it("counts a rejected POST without throwing back into the call site", async () => { - vi.spyOn(console, "warn").mockImplementation(() => undefined); + it("counts a rejected POST, surfaces a bounded console notice, and does not throw back into the call site", async () => { + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => undefined); vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new TypeError("network error"))); expect(() => fanOutClientDiagnostic("boot: gateway probe failed")).not.toThrow(); @@ -141,6 +141,16 @@ describe("fanOutClientDiagnostic", () => { await vi.waitFor(() => { expect(clientDiagnosticPostFailureCount()).toBe(1); }); + // Once for the diagnostic itself (console-first fan-out), once for the delivery-failure + // notice — a developer watching devtools must be able to tell the server never received it. + expect(consoleWarn).toHaveBeenCalledTimes(2); + expect(consoleWarn).toHaveBeenCalledWith("boot: gateway probe failed"); + const lastCall = consoleWarn.mock.calls[consoleWarn.mock.calls.length - 1] as [string]; + expect(lastCall[0]).toMatch(/diagnostic delivery to the server failed/i); + // Bounded and redacted: the notice never repeats the original message content or any error + // detail, so it cannot itself become a place that leaks something the sink already redacted. + expect(lastCall[0]).not.toContain("network error"); + expect(lastCall[0]).not.toContain("boot: gateway probe failed"); }); it("drops the 21st POST within a rolling minute and counts it, without dropping the console write", () => { diff --git a/packages/keiko-ui/src/lib/install-client-diagnostics.ts b/packages/keiko-ui/src/lib/install-client-diagnostics.ts index f5039b5a5d..6f2f50c44b 100644 --- a/packages/keiko-ui/src/lib/install-client-diagnostics.ts +++ b/packages/keiko-ui/src/lib/install-client-diagnostics.ts @@ -50,6 +50,14 @@ function writeToBrowserConsole(message: string): void { if (typeof console !== "undefined" && typeof console.warn === "function") console.warn(message); } +// Fixed, bounded, and content-free by construction: it never repeats the original diagnostic +// message or any error detail, so it cannot itself become a place that leaks something the sink +// already redacted. Written with `writeToBrowserConsole` directly (never `reportClientDiagnostic`) +// — going back through the sink would re-enter `fanOutClientDiagnostic` and, on a persistently +// failing transport, retry the same failing POST on every diagnostic: a reporting loop. +const DIAGNOSTIC_DELIVERY_FAILURE_NOTICE = + "[keiko] diagnostic delivery to the server failed; the diagnostic above (if any) was not recorded server-side."; + // Mirrors the server's SAFE_CORRELATION_ID predicate (packages/keiko-server/src/correlation.ts). // keiko-ui may only depend on the server through the shared contract types (AGENTS.md §4), never on // a server module directly, so this file re-derives the same alphabet+length shape as its own, @@ -156,8 +164,11 @@ export function resetClientDiagnosticPostStateForTests(): void { // rejected fetch (or a non-2xx `ApiError` `bffFetchJson` throws) reach back into // `reportClientDiagnostic`'s caller — the same best-effort discipline `writeToBrowserConsole` // already has, just with a `.catch` standing in for that function's `typeof` guards. A failure is -// counted, never logged to console (AGENTS.md §6: this module is the one sanctioned console site, -// and re-entering it from a diagnostics-transport failure risks a loop under a flapping connection). +// counted (for tests) AND surfaced to the console directly via the fixed, content-free +// `DIAGNOSTIC_DELIVERY_FAILURE_NOTICE` (AGENTS.md §7: "errors must surface with enough context to +// diagnose" — silently dropping a failed POST left a developer with no way to tell the server never +// received the diagnostic). This never calls back through `reportClientDiagnostic`, which would +// re-enter `fanOutClientDiagnostic` and risk a loop under a persistently failing transport. function postClientDiagnosticToServer(message: string, meta?: ClientDiagnosticMeta): void { if (!admittedByClientPostRateLimit(Date.now())) { postThrottledCount += 1; @@ -171,9 +182,11 @@ function postClientDiagnosticToServer(message: string, meta?: ClientDiagnosticMe keepalive: true, }).catch(() => { postFailureCount += 1; + writeToBrowserConsole(DIAGNOSTIC_DELIVERY_FAILURE_NOTICE); }); } catch { postFailureCount += 1; + writeToBrowserConsole(DIAGNOSTIC_DELIVERY_FAILURE_NOTICE); } } diff --git a/scripts/__tests__/correlation-id-pattern-drift.test.mjs b/scripts/__tests__/correlation-id-pattern-drift.test.mjs index 9d934754e5..26a63128be 100644 --- a/scripts/__tests__/correlation-id-pattern-drift.test.mjs +++ b/scripts/__tests__/correlation-id-pattern-drift.test.mjs @@ -29,9 +29,11 @@ const CLIENT_FILE = "packages/keiko-ui/src/lib/install-client-diagnostics.ts"; // `const SAFE_CORRELATION_ID = /^[A-Za-z0-9._-]{8,128}$/;` and `"SAFE_CORRELATION_ID"`, returns // `"/^[A-Za-z0-9._-]{8,128}$/"` — the literal text, not a compiled RegExp, so the comparison below // is character-for-character and trips on a widened character class or changed length bound even -// if the two patterns would still accept/reject the same handful of test strings. +// if the two patterns would still accept/reject the same handful of test strings. The trailing +// `[a-zA-Z]*` after the closing delimiter captures any flags (e.g. `i`) so a flags-only edit is +// caught too — without it, `/pattern/` and `/pattern/i` extract to the identical literal text. function extractRegexLiteral(source, constantName) { - const pattern = new RegExp(`\\b${constantName}\\s*=\\s*(\\/[^\\n]*\\/)`); + const pattern = new RegExp(`\\b${constantName}\\s*=\\s*(\\/[^\\n]*\\/[a-zA-Z]*)`); const match = pattern.exec(source); if (match === null) { throw new Error(`could not find a declaration of ${constantName} in the given source`); @@ -49,4 +51,22 @@ describe("client/server correlation-id pattern drift (#2902 audit finding 2)", ( expect(clientLiteral).toBe(serverLiteral); }); + + // Regression: `extractRegexLiteral`'s capture group used to stop at the closing `/` delimiter + // and never captured trailing flags, so a client-only change from `/pattern/` to `/pattern/i` + // (or any other flag) extracted to the identical literal text as the flag-free server pattern + // and this whole test would stay green despite the two patterns now accepting different inputs + // (case-insensitive vs. case-sensitive). The extracted literal must include the flags suffix. + it("treats a flags-only difference between two patterns as a real divergence", () => { + const source = [ + "const SAFE_CORRELATION_ID = /^[A-Za-z0-9._-]{8,128}$/;", + "const CLIENT_CORRELATION_ID_PATTERN = /^[A-Za-z0-9._-]{8,128}$/i;", + ].join("\n"); + + const serverLiteral = extractRegexLiteral(source, "SAFE_CORRELATION_ID"); + const clientLiteral = extractRegexLiteral(source, "CLIENT_CORRELATION_ID_PATTERN"); + + expect(clientLiteral).toBe("/^[A-Za-z0-9._-]{8,128}$/i"); + expect(clientLiteral).not.toBe(serverLiteral); + }); }); From 441956d2cdf1dd5a2df58f5529427ebf2232131a Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 13:56:42 +0200 Subject: [PATCH 14/19] =?UTF-8?q?fix(observability):=20integration=20follo?= =?UTF-8?q?w-ups=20=E2=80=94=20route-literal=20fixture,=20deterministic=20?= =?UTF-8?q?query-name=20order=20helper,=20export=20manifest=20helpers,=20p?= =?UTF-8?q?arameterized=20body-reader=20tests=20(#3233)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- packages/keiko-cli/src/support.ts | 63 +++++++++++++++---- .../src/bounded-request-body.test.ts | 61 +++++++++--------- .../src/gateway-error-diagnostic.test.ts | 8 +-- .../src/run-handlers-sse-correlation.test.ts | 7 ++- packages/keiko-server/src/server.ts | 8 ++- 5 files changed, 99 insertions(+), 48 deletions(-) diff --git a/packages/keiko-cli/src/support.ts b/packages/keiko-cli/src/support.ts index c801b79875..65b6579a52 100644 --- a/packages/keiko-cli/src/support.ts +++ b/packages/keiko-cli/src/support.ts @@ -303,6 +303,54 @@ function reportAuditFailure(error: unknown, io: CliIo): number { throw error; } +type ManifestInput = Parameters[0]; + +function processProvenance( + server: Awaited>, + generatedAt: Date, + stateDirSource: ManifestInput["stateDirSource"], +): Pick< + ManifestInput, + | "schemaVersion" + | "productVersion" + | "platform" + | "arch" + | "nodeVersion" + | "generatedAt" + | "installMode" + | "stateDirSource" +> { + return { + schemaVersion: server.SERVER_LOG_SCHEMA_VERSION, + productVersion: KEIKO_PRODUCT_VERSION, + platform: process.platform, + arch: process.arch, + nodeVersion: process.version, + generatedAt: generatedAt.toISOString(), + installMode: resolveExportInstallMode(server), + stateDirSource, + }; +} + +function logContentManifestFields( + logContent: LogContent, +): Pick< + ManifestInput, + | "sourceLogFiles" + | "truncatedLogFiles" + | "currentFileTailTruncated" + | "budgetExceeded" + | "skippedLogFiles" +> { + return { + sourceLogFiles: logContent.sourceLogFiles, + truncatedLogFiles: logContent.truncatedLogFiles, + currentFileTailTruncated: logContent.currentFileTailTruncated, + budgetExceeded: logContent.budgetExceeded, + skippedLogFiles: logContent.skippedLogFiles, + }; +} + async function runSupportExport( args: ExportArgs, io: CliIo, @@ -336,19 +384,8 @@ async function runSupportExport( const storeFingerprintCollection = await server.collectStoreFingerprints({ stateDir, env }); const generatedAtDate = now(); const manifest = buildSupportBundleManifest({ - schemaVersion: server.SERVER_LOG_SCHEMA_VERSION, - productVersion: KEIKO_PRODUCT_VERSION, - platform: process.platform, - arch: process.arch, - nodeVersion: process.version, - generatedAt: generatedAtDate.toISOString(), - installMode: resolveExportInstallMode(server), - stateDirSource, - sourceLogFiles: logContent.sourceLogFiles, - truncatedLogFiles: logContent.truncatedLogFiles, - currentFileTailTruncated: logContent.currentFileTailTruncated, - budgetExceeded: logContent.budgetExceeded, - skippedLogFiles: logContent.skippedLogFiles, + ...processProvenance(server, generatedAtDate, stateDirSource), + ...logContentManifestFields(logContent), auditSummary, evidenceIndexCount, storeFingerprints: storeFingerprintCollection.fingerprints, diff --git a/packages/keiko-server/src/bounded-request-body.test.ts b/packages/keiko-server/src/bounded-request-body.test.ts index 070be8cab0..c6c739f063 100644 --- a/packages/keiko-server/src/bounded-request-body.test.ts +++ b/packages/keiko-server/src/bounded-request-body.test.ts @@ -398,37 +398,40 @@ describe("bounded request body activity log", () => { // is now the one owner of that wrapper layer; each caller keeps its own max-bytes constant and // becomes a one-line delegate to this function. describe("readJsonRequestBody", () => { - it("returns 413 PAYLOAD_TOO_LARGE for an oversized body", async () => { - const req = asRequest(Readable.from([Buffer.from("this body is too long")])); - - const result = await readJsonRequestBody(req, 4); - - expect(result).toEqual({ - status: 413, - body: { error: { code: "PAYLOAD_TOO_LARGE", message: "Request body too large." } }, - }); - }); - - it("returns 400 BAD_REQUEST for malformed JSON", async () => { - const req = asRequest(Readable.from([Buffer.from("{not json")])); - - const result = await readJsonRequestBody(req, 128_000); - - expect(result).toEqual({ - status: 400, - body: { error: { code: "BAD_REQUEST", message: "Request body is not valid JSON." } }, - }); - }); + it.each([ + { + name: "413 PAYLOAD_TOO_LARGE for an oversized body", + body: "this body is too long", + maxBytes: 4, + expected: { + status: 413, + body: { error: { code: "PAYLOAD_TOO_LARGE", message: "Request body too large." } }, + }, + }, + { + name: "400 BAD_REQUEST for malformed JSON", + body: "{not json", + maxBytes: 128_000, + expected: { + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body is not valid JSON." } }, + }, + }, + { + name: "400 BAD_REQUEST for valid JSON that is not an object (an array)", + body: "[1,2,3]", + maxBytes: 128_000, + expected: { + status: 400, + body: { error: { code: "BAD_REQUEST", message: "Request body must be a JSON object." } }, + }, + }, + ])("returns $name", async ({ body, maxBytes, expected }) => { + const req = asRequest(Readable.from([Buffer.from(body)])); - it("returns 400 BAD_REQUEST for valid JSON that is not an object (an array)", async () => { - const req = asRequest(Readable.from([Buffer.from("[1,2,3]")])); + const result = await readJsonRequestBody(req, maxBytes); - const result = await readJsonRequestBody(req, 128_000); - - expect(result).toEqual({ - status: 400, - body: { error: { code: "BAD_REQUEST", message: "Request body must be a JSON object." } }, - }); + expect(result).toEqual(expected); }); it("returns the parsed record for a valid JSON object body", async () => { diff --git a/packages/keiko-server/src/gateway-error-diagnostic.test.ts b/packages/keiko-server/src/gateway-error-diagnostic.test.ts index de6367fc68..a3e81611db 100644 --- a/packages/keiko-server/src/gateway-error-diagnostic.test.ts +++ b/packages/keiko-server/src/gateway-error-diagnostic.test.ts @@ -32,7 +32,7 @@ describe("emitGatewayErrorDiagnostic", () => { deps, new Error("boom"), "correlation-9", - "POST /api/example", + "POST /api/chat", "example.source", ); @@ -40,7 +40,7 @@ describe("emitGatewayErrorDiagnostic", () => { const [event] = events; if (event === undefined) throw new Error("expected a diagnostic record"); expect(event.correlationId).toBe("correlation-9"); - expect(event.operation).toBe("POST /api/example"); + expect(event.operation).toBe("POST /api/chat"); expect(event.source).toBe("example.source"); expect(event.errorClass).toBe("Error"); }); @@ -52,7 +52,7 @@ describe("emitGatewayErrorDiagnostic", () => { deps, new Error("boom"), undefined, - "POST /api/example", + "POST /api/chat", "example.source", ); @@ -70,7 +70,7 @@ describe("emitGatewayErrorDiagnostic", () => { deps, new Error("boom"), "correlation-1", - "POST /api/example", + "POST /api/chat", "example.source", ); }).not.toThrow(); diff --git a/packages/keiko-server/src/run-handlers-sse-correlation.test.ts b/packages/keiko-server/src/run-handlers-sse-correlation.test.ts index e60edc8bf2..2b5471e852 100644 --- a/packages/keiko-server/src/run-handlers-sse-correlation.test.ts +++ b/packages/keiko-server/src/run-handlers-sse-correlation.test.ts @@ -40,7 +40,12 @@ function listenableFakeRes(): { res: RouteContext["res"]; fireClose: () => void emitter.on(event, handler); }, } as unknown as RouteContext["res"]; - return { res, fireClose: (): void => emitter.emit("close") }; + return { + res, + fireClose: (): void => { + emitter.emit("close"); + }, + }; } function fakeReq(): RouteContext["req"] { diff --git a/packages/keiko-server/src/server.ts b/packages/keiko-server/src/server.ts index 983c347fff..870e262994 100644 --- a/packages/keiko-server/src/server.ts +++ b/packages/keiko-server/src/server.ts @@ -349,6 +349,12 @@ async function resolveCsp(deps: UiServerDeps): Promise { // this function directly on a constructed `URL` — no real HTTP request, no spy on the shared // `Array.prototype.sort`, and no risk of a second call site (in the test) drifting from this one's // actual cap. +// Plain code-point order (see the comment at the call site): deterministic across hosts. +function codePointOrder(a: string, b: string): number { + if (a < b) return -1; + return a > b ? 1 : 0; +} + export function computeQueryParamFields(url: URL, context: RequestLogContext): void { const seen = new Set(); const kept: string[] = []; @@ -368,7 +374,7 @@ export function computeQueryParamFields(url: URL, context: RequestLogContext): v // the cost difference is immaterial, but `localeCompare` depends on the host ICU data and the // default locale — two servers could then emit the same request with a different // `queryParamNames` order, and this field is compared/deduplicated across hosts. - kept.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + kept.sort(codePointOrder); context.queryParamNames = kept; if (dropped > 0) context.queryParamDroppedCount = dropped; } From 2ec8d8b79d3a3e799304d5d1ef4bb38b8a32d142 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 14:29:54 +0200 Subject: [PATCH 15/19] test(coverage): regenerate the package coverage baseline for the integrated logging waves (#3233) Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 93 ++++++++++++-------------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index 3c30a706ab..f961b5b593 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -8,13 +8,13 @@ "keiko-cli": { "files": 44, "uncoveredFiles": 0, - "uncoveredLines": 385, - "totalLines": 4880, + "uncoveredLines": 383, + "totalLines": 4935, "coverage": { - "lines": 92.11, - "statements": 90.39, + "lines": 92.24, + "statements": 90.52, "branches": 85.19, - "functions": 93.23 + "functions": 93.75 } }, "keiko-connectors": { @@ -30,15 +30,15 @@ } }, "keiko-contracts": { - "files": 186, + "files": 188, "uncoveredFiles": 0, "uncoveredLines": 857, - "totalLines": 13852, + "totalLines": 13919, "coverage": { - "lines": 93.81, - "statements": 92.39, - "branches": 90.23, - "functions": 97.46 + "lines": 93.84, + "statements": 92.43, + "branches": 90.28, + "functions": 97.48 } }, "keiko-editor": { @@ -104,13 +104,13 @@ "keiko-local-knowledge": { "files": 128, "uncoveredFiles": 0, - "uncoveredLines": 754, - "totalLines": 9383, + "uncoveredLines": 752, + "totalLines": 9421, "coverage": { - "lines": 91.97, - "statements": 89.58, - "branches": 80.95, - "functions": 94.31 + "lines": 92.02, + "statements": 89.64, + "branches": 80.99, + "functions": 94.35 } }, "keiko-memory-capture": { @@ -162,15 +162,15 @@ } }, "keiko-memory-vault": { - "files": 22, + "files": 23, "uncoveredFiles": 0, - "uncoveredLines": 78, - "totalLines": 953, + "uncoveredLines": 79, + "totalLines": 1048, "coverage": { - "lines": 91.82, - "statements": 90.46, - "branches": 85.55, - "functions": 91.34 + "lines": 92.46, + "statements": 91.21, + "branches": 86.28, + "functions": 91.8 } }, "keiko-model-gateway": { @@ -222,27 +222,27 @@ } }, "keiko-security": { - "files": 23, + "files": 24, "uncoveredFiles": 0, - "uncoveredLines": 11, - "totalLines": 616, + "uncoveredLines": 0, + "totalLines": 661, "coverage": { - "lines": 98.21, - "statements": 97.62, - "branches": 94.08, - "functions": 98.6 + "lines": 100, + "statements": 99.86, + "branches": 98.96, + "functions": 100 } }, "keiko-server": { - "files": 581, + "files": 583, "uncoveredFiles": 0, - "uncoveredLines": 4503, - "totalLines": 56309, + "uncoveredLines": 4473, + "totalLines": 56410, "coverage": { - "lines": 92, - "statements": 89.3, - "branches": 81.91, - "functions": 94.91 + "lines": 92.07, + "statements": 89.37, + "branches": 82, + "functions": 94.96 } }, "keiko-tools": { @@ -260,13 +260,13 @@ "keiko-ui": { "files": 419, "uncoveredFiles": 3, - "uncoveredLines": 2967, - "totalLines": 39455, + "uncoveredLines": 2963, + "totalLines": 39511, "coverage": { - "lines": 92.48, - "statements": 89.53, - "branches": 81.8, - "functions": 91.28 + "lines": 92.5, + "statements": 89.55, + "branches": 81.82, + "functions": 91.3 } }, "keiko-verification": { @@ -342,11 +342,6 @@ "tolerance": 0, "lines": 90 }, - "packages/keiko-security/src/errors/harness.ts": { - "governance": "ratcheted", - "tolerance": 0.5, - "lines": 16.17 - }, "packages/keiko-server/src/editor/dap/dapCapsuleSupervisor.ts": { "governance": "absolute", "tolerance": 0, From 54e8dbf5d349f3eaac1591559fc96c90c98272a5 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 14:31:12 +0200 Subject: [PATCH 16/19] docs(observability): regenerate the op catalog for the integrated logging waves (#3233) Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index f40fce2245..e5a118598c 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -875,7 +875,7 @@ { "op": "request", "category": "http", - "site": "packages/keiko-server/src/server.ts:489", + "site": "packages/keiko-server/src/server.ts:495", "package": "keiko-server" }, { From 47e07045b44a388f45dfa00a03ce969c8d5542b4 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 14:46:50 +0200 Subject: [PATCH 17/19] fix(observability): measure responseBytes on the socket so gzip, static and streamed responses are counted like JSON (#3240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit writeJson alone tallied the uncompressed JSON length and every static asset logged 0 (#3247 review). The request line now reports the socket's bytesWritten delta between request arrival and response close — headers and body, compressed as sent — for every writer uniformly. Co-Authored-By: Claude Fable 5 --- ...-log-v2-machine-reconstruction-contract.md | 4 +- .../src/http-lifecycle.e2e.test.ts | 6 +- packages/keiko-server/src/server.test.ts | 24 +++- packages/keiko-server/src/server.ts | 111 ++++++++---------- 4 files changed, 71 insertions(+), 74 deletions(-) diff --git a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md index d3eb2158b3..76edade1a6 100644 --- a/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md +++ b/docs/adr/ADR-0173-server-activity-log-v2-machine-reconstruction-contract.md @@ -463,7 +463,9 @@ request": Unmatched and static requests fall back to `redactRoutePath`. `queryParamNames` lists the query parameter NAMES only (deduplicated, shape-checked against a bounded identifier pattern, sorted, capped at 16; anything dropped is counted in `queryParamDroppedCount`), never a value. - `responseBytes` is the byte count `writeJson` already computed and previously discarded. + `responseBytes` is what the response actually put on the socket — headers and body, compressed + as sent — measured as the socket's `bytesWritten` delta from request arrival to response close, + so JSON, gzip, static files and streams are counted the same way. `aborted` is computed at `close` by the shared `requestAlreadyClosed` predicate, and a request the client abandoned before any write logs `status: 0` instead of Node's default `200` — the predicate was corrected in the same wave so a normally ended response (which Node also marks diff --git a/packages/keiko-server/src/http-lifecycle.e2e.test.ts b/packages/keiko-server/src/http-lifecycle.e2e.test.ts index a7a5bb86c0..2e6346e237 100644 --- a/packages/keiko-server/src/http-lifecycle.e2e.test.ts +++ b/packages/keiko-server/src/http-lifecycle.e2e.test.ts @@ -173,9 +173,9 @@ describe("(a) client disconnect mid-response — real socket abort", () => { expect(event.extra?.aborted).toBe(true); expect(event.extra?.routeTemplate).toBe("/api/runs/:runId/events"); expect(event.extra?.queryParamNames).toEqual(["bar", "foo"]); - // A STREAMING route never calls `writeJson` (the only place `responseBytes` is computed), so - // the field is present and reports the documented default rather than being silently absent. - expect(event.extra?.responseBytes).toBe(0); + // `responseBytes` is measured on the socket, so a streaming route is counted like any other: + // the headers and the first frame the client read are on the wire before the abort. + expect(event.extra?.responseBytes).toBeGreaterThan(0); } finally { if (started !== undefined) await closeUiTestServer(started.server); handlerDeps.store.close(); diff --git a/packages/keiko-server/src/server.test.ts b/packages/keiko-server/src/server.test.ts index f98fe749ef..c1c4ee52ca 100644 --- a/packages/keiko-server/src/server.test.ts +++ b/packages/keiko-server/src/server.test.ts @@ -1235,14 +1235,25 @@ describe("activity log: http-request line enrichment (Wave 5, w5-http-request-en expect(JSON.stringify(event.extra)).not.toContain(longName); }); - it("records writeJson's own computed response byte count", async () => { + it("records the bytes the response put on the socket — at least its body", async () => { const sink = await startWithActivityLog(); const res = await fetchRaw("/api/health"); - const expectedBytes = Buffer.byteLength(res.text, "utf8"); + const bodyBytes = Buffer.byteLength(res.text, "utf8"); const event = await waitForActivityLogEvent(sink); - expect(event.extra?.responseBytes).toBe(expectedBytes); - expect(event.extra?.responseBytes).toBeGreaterThan(0); + // Headers ride along, so the wire count is never below the body the client received. + expect(event.extra?.responseBytes).toBeGreaterThanOrEqual(bodyBytes); + }); + + it("counts a static asset's bytes too, not only writeJson's (#3247 review)", async () => { + // Before: only `writeJson` tallied `responseBytes`, so every static asset logged 0. + const sink = await startWithActivityLog(); + const res = await fetchRaw("/_next/app.js"); + const bodyBytes = Buffer.byteLength(res.text, "utf8"); + const event = await waitForActivityLogEvent(sink); + + expect(bodyBytes).toBeGreaterThan(0); + expect(event.extra?.responseBytes).toBeGreaterThanOrEqual(bodyBytes); }); it("reports aborted:false and the real status for a normally completed request", async () => { @@ -1329,6 +1340,7 @@ describe("logRequestOnClose", () => { destroyed: boolean; url?: string; method?: string; + socket: { bytesWritten: number }; } interface ResponseDouble extends EventEmitter { @@ -1345,6 +1357,7 @@ describe("logRequestOnClose", () => { destroyed: false, url: "/api/health", method: "GET", + socket: { bytesWritten: 0 }, }); const res = Object.assign(new EventEmitter(), { closed: false, @@ -1408,7 +1421,6 @@ describe("logRequestOnClose", () => { routeTemplate: "/api/memory/:id", queryParamNames: ["bar", "foo"], queryParamDroppedCount: 2, - responseBytes: 42, }; logRequestOnClose( req as unknown as IncomingMessage, @@ -1418,6 +1430,8 @@ describe("logRequestOnClose", () => { context, ); + // The socket carried 42 more bytes between request arrival and response close. + req.socket.bytesWritten += 42; res.headersSent = true; res.writableEnded = true; res.emit("close"); diff --git a/packages/keiko-server/src/server.ts b/packages/keiko-server/src/server.ts index 870e262994..5c29449ddc 100644 --- a/packages/keiko-server/src/server.ts +++ b/packages/keiko-server/src/server.ts @@ -83,17 +83,18 @@ export interface UiServerDeps { // Per-request scratch space the http-request line reads at response `close`, populated as the // request is actually resolved: `handle` fills in the query-parameter fields as soon as the URL is -// parsed, `dispatchApi`/`serveStatic` fill in `routeTemplate` once the route (or its absence) is -// known, and `writeJson` fills in `responseBytes` the one time it actually serialises a body. A -// fresh object is created once per incoming request and threaded explicitly through the same call -// chain `correlationId` already travels — plain per-request state, not a keyed side-table. -// Exported for `server.test.ts` only (not part of the package's public entry point, mirroring how -// `request-cancellation.ts` exports its own req/res-close shapes for the same reason). +// parsed and `dispatchApi`/`serveStatic` fill in `routeTemplate` once the route (or its absence) +// is known. A fresh object is created once per incoming request and threaded explicitly through +// the same call chain `correlationId` already travels — plain per-request state, not a keyed +// side-table. The response size is NOT tracked here: `logRequestOnClose` measures it on the socket +// (see `responseBytes` there), so every writer — JSON, gzip, static file, SSE stream — is counted +// the same way without each one reporting. Exported for `server.test.ts` only (not part of the +// package's public entry point, mirroring how `request-cancellation.ts` exports its own +// req/res-close shapes for the same reason). export interface RequestLogContext { routeTemplate?: string; queryParamNames?: readonly string[]; queryParamDroppedCount?: number; - responseBytes?: number; } function acceptsGzip(acceptEncoding: string | readonly string[] | undefined): boolean { @@ -111,19 +112,16 @@ function writeJson( status: number, body: unknown, headers: Readonly> = {}, - context?: RequestLogContext, ): void { res.statusCode = status; for (const [key, value] of Object.entries(headers)) { res.setHeader(key, typeof value === "string" ? value : [...value]); } if (status === 204 || status === 304) { - if (context !== undefined) context.responseBytes = 0; res.end(); return; } const payload = Buffer.from(JSON.stringify(body), "utf8"); - if (context !== undefined) context.responseBytes = payload.byteLength; res.setHeader("Content-Type", "application/json; charset=utf-8"); if (payload.byteLength >= JSON_GZIP_MIN_BYTES && acceptsGzip(req.headers["accept-encoding"])) { res.setHeader("Content-Encoding", "gzip"); @@ -147,30 +145,17 @@ function hasCsrfHeader(req: IncomingMessage): boolean { return value === "1"; } -function rejectUnsupportedMediaType( - req: IncomingMessage, - res: ServerResponse, - context: RequestLogContext, -): void { +function rejectUnsupportedMediaType(req: IncomingMessage, res: ServerResponse): void { writeJson( req, res, 415, errorBody("UNSUPPORTED_MEDIA_TYPE", "State-changing API requests must use JSON."), - {}, - context, ); } -function rejectCsrf(req: IncomingMessage, res: ServerResponse, context: RequestLogContext): void { - writeJson( - req, - res, - 403, - errorBody("FORBIDDEN_CSRF", "Missing state-changing request guard."), - {}, - context, - ); +function rejectCsrf(req: IncomingMessage, res: ServerResponse): void { + writeJson(req, res, 403, errorBody("FORBIDDEN_CSRF", "Missing state-changing request guard.")); } // A minimal default deps object so a 3-arg server can still serve the deps-bound read routes (e.g. @@ -204,17 +189,16 @@ function rejectIfInvalidStateChange( res: ServerResponse, method: string, pathname: string, - context: RequestLogContext, ): boolean { if (!isJsonRequest(req)) { - rejectUnsupportedMediaType(req, res, context); + rejectUnsupportedMediaType(req, res); return true; } if (isCsrfExemptStateChange(method, pathname)) { return false; } if (!hasCsrfHeader(req)) { - rejectCsrf(req, res, context); + rejectCsrf(req, res); return true; } return false; @@ -241,18 +225,17 @@ async function dispatchApi( const match = matchRoute(method, url.pathname); if (match === undefined) { setFallbackRouteTemplate(context, url.pathname); - writeJson(req, res, 404, notFoundBody(), {}, context); + writeJson(req, res, 404, notFoundBody()); return; } if (match === "method-not-allowed") { setFallbackRouteTemplate(context, url.pathname); - writeJson(req, res, 405, methodNotAllowedBody(), {}, context); + writeJson(req, res, 405, methodNotAllowedBody()); return; } context.routeTemplate = match.definition.pattern; const invalidStateChange = - isStateChangingMethod(method) && - rejectIfInvalidStateChange(req, res, method, url.pathname, context); + isStateChangingMethod(method) && rejectIfInvalidStateChange(req, res, method, url.pathname); if (invalidStateChange) { return; } @@ -261,7 +244,7 @@ async function dispatchApi( if (outcome === STREAMING) { return; } - writeJson(req, res, outcome.status, outcome.body, outcome.headers, context); + writeJson(req, res, outcome.status, outcome.body, outcome.headers); } function resolveStaticTargets(pathname: string): readonly string[] { @@ -296,23 +279,12 @@ async function serveStatic( if (await serveFile(res, indexPath, req.headers["accept-encoding"])) { return; } - writeJson( - req, - res, - 404, - errorBody("NOT_FOUND", "The requested resource was not found."), - {}, - context, - ); + writeJson(req, res, 404, errorBody("NOT_FOUND", "The requested resource was not found.")); } -function rejectForbiddenHost( - req: IncomingMessage, - res: ServerResponse, - context: RequestLogContext, -): void { +function rejectForbiddenHost(req: IncomingMessage, res: ServerResponse): void { const body: ApiError = errorBody("FORBIDDEN_HOST", "Request host is not the local interface."); - writeJson(req, res, 403, body, {}, context); + writeJson(req, res, 403, body); } async function resolveCsp(deps: UiServerDeps): Promise { @@ -396,7 +368,7 @@ async function handle( allowMicrophone: isVoiceDictationCapable(handlerDeps) || isVoiceRealtimeCapable(handlerDeps), }); if (!isAllowedHost(req, deps.port)) { - rejectForbiddenHost(req, res, context); + rejectForbiddenHost(req, res); return; } // #2902 audit finding 1: computed only once the trust-boundary host check has passed (AGENTS.md @@ -440,20 +412,25 @@ function createVoicePlanes( // (reduced generically by `log-redaction.ts`'s own path guard); `routeTemplate` is the field this // wave adds so a reader learns WHICH declared route actually matched, sourced from the real match // `dispatchApi` resolved rather than re-derived independently from the same raw path. +interface HttpRequestOutcome { + readonly method: string; + readonly path: string; + readonly aborted: boolean; + readonly responseBytes: number; +} + function buildHttpRequestExtra( - method: string, - path: string, - aborted: boolean, + outcome: HttpRequestOutcome, context: RequestLogContext, ): Record { return { - method, - path, + method: outcome.method, + path: outcome.path, routeTemplate: context.routeTemplate, queryParamNames: context.queryParamNames ?? [], queryParamDroppedCount: context.queryParamDroppedCount, - responseBytes: context.responseBytes ?? 0, - aborted, + responseBytes: outcome.responseBytes, + aborted: outcome.aborted, }; } @@ -487,16 +464,27 @@ export function logRequestOnClose( const startedAt = Date.now(); const requestUrl = req.url ?? ""; const method = req.method ?? "GET"; + // `responseBytes` is what this response actually put on the wire — headers and body, compressed + // as sent — measured as the socket's `bytesWritten` delta between request arrival and response + // close. A kept-alive socket serves responses one after another, so the delta belongs to this + // response alone; every writer (JSON, gzip, static file, SSE stream) is counted the same way, + // which a per-writer tally could not guarantee (#3247 review: `writeJson` alone reported the + // uncompressed JSON length and every static asset read as 0). + const socketBytesAtStart = req.socket.bytesWritten; res.on("close", () => { const aborted = requestAlreadyClosed({ req, res }); const status = aborted && !res.headersSent ? 0 : res.statusCode; + const responseBytes = Math.max(0, req.socket.bytesWritten - socketBytesAtStart); activityLog.write({ category: "http", op: "request", correlationId, status, durationMs: Date.now() - startedAt, - extra: buildHttpRequestExtra(method, requestUrl.split("?")[0] ?? "", aborted, context), + extra: buildHttpRequestExtra( + { method, path: requestUrl.split("?")[0] ?? "", aborted, responseBytes }, + context, + ), }); }); } @@ -524,14 +512,7 @@ function reportTopLevelFailure( }), ); if (!res.headersSent) { - writeJson( - req, - res, - 500, - errorBody("INTERNAL", "An unexpected error occurred.", correlationId), - {}, - context, - ); + writeJson(req, res, 500, errorBody("INTERNAL", "An unexpected error occurred.", correlationId)); } else { res.end(); } From 9ec3597bd14cba060bd7de885535d63d14ae68c9 Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 14:46:50 +0200 Subject: [PATCH 18/19] docs(observability): regenerate the op catalog (#3233) Co-Authored-By: Claude Fable 5 --- docs/observability/op-catalog.generated.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/observability/op-catalog.generated.json b/docs/observability/op-catalog.generated.json index e5a118598c..a298be3cb6 100644 --- a/docs/observability/op-catalog.generated.json +++ b/docs/observability/op-catalog.generated.json @@ -875,7 +875,7 @@ { "op": "request", "category": "http", - "site": "packages/keiko-server/src/server.ts:495", + "site": "packages/keiko-server/src/server.ts:480", "package": "keiko-server" }, { From 2305ddbb5bc3fcbc2d899d31ab03ceca3325c3ae Mon Sep 17 00:00:00 2001 From: oscharko-dev Date: Sat, 22 Aug 2026 15:20:17 +0200 Subject: [PATCH 19/19] test(coverage): regenerate the package coverage baseline for the integrated logging waves (#3233) Co-Authored-By: Claude Fable 5 --- docs/qa/package-coverage-baseline.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/qa/package-coverage-baseline.json b/docs/qa/package-coverage-baseline.json index f961b5b593..6d3aa60962 100644 --- a/docs/qa/package-coverage-baseline.json +++ b/docs/qa/package-coverage-baseline.json @@ -236,7 +236,7 @@ "keiko-server": { "files": 583, "uncoveredFiles": 0, - "uncoveredLines": 4473, + "uncoveredLines": 4474, "totalLines": 56410, "coverage": { "lines": 92.07, @@ -260,12 +260,12 @@ "keiko-ui": { "files": 419, "uncoveredFiles": 3, - "uncoveredLines": 2963, + "uncoveredLines": 2965, "totalLines": 39511, "coverage": { "lines": 92.5, "statements": 89.55, - "branches": 81.82, + "branches": 81.83, "functions": 91.3 } },