diff --git a/README.md b/README.md index 7e8d00d..40f404c 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ See [Vault payments](docs/vault-payments.md) for both provider flows, safety rul - `webmcp` - List native page tools across every tab and frame in a browser, then synchronously invoke an exact opaque `tool_ref` with structured input. - `exec_command` - Run shell commands inside a browser VM. Returns decoded stdout/stderr. - `search_docs` - Search Kernel platform documentation and guides. -- `get_more_tools` - Report a structured KERNEL capability or external-integration gap after checking the available tools. Existing-tool failures, transient capacity errors, and client permission restrictions are rejected from capability-demand analytics. Accepted requests emit `mcp_capability_requested`; historical unstructured requests remain under `$mcp_missing_capability`. +- `get_more_tools` - Report a structured KERNEL capability or external-integration gap after checking the available tools. Existing-tool failures, transient capacity errors, and client permission restrictions are rejected from capability-demand analytics. Accepted requests emit `mcp_capability_requested`; clients using the previous context-only schema receive a non-recording refresh response instead of a tool error. - `submit_feedback` - Send product, bot-detection, config-registry, MCP, or documentation feedback directly to the KERNEL team without interrupting the current task. Reports include a normalized task outcome; MCP reports identify one KERNEL-owned tool. Config-registry reports connect exactly one observed outcome to the browser session, recommendation metadata and evidence, and unchanged browser and proxy settings. - `open_auth_login` - Open a secure interactive Managed Auth MCP App after user consent. Registered only for clients that declare MCP Apps support; credentials and MFA never enter MCP/model traffic. diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index da77156..fd019e5 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -1028,6 +1028,19 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { missingCapabilityTool?.inputSchema, ); + const legacyRequest = await enabled.client.callTool({ + name: "get_more_tools", + arguments: { + context: + "Reporting a capability through the previous contract while refreshing the available tool definitions.", + }, + }); + expect(legacyRequest.isError).not.toBe(true); + expect(toolResultJSON(legacyRequest)).toMatchObject({ + recorded: false, + status: "legacy_schema_refresh_required", + }); + const unavailableRequest = await disabled.client.callTool({ name: "get_more_tools", arguments: { @@ -1185,6 +1198,36 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { expect(byEvent.has("$identify")).toBe(false); }); + test("classifies rejected capability input through instrumentation", async () => { + const captured: { event?: string }[] = []; + + const result = (await simulateRequest(captured, "tools/call", { + name: "get_more_tools", + arguments: { + context: + "Reporting a malformed structured capability request to verify validation telemetry.", + gap_reason: "not_a_gap_reason", + capability_area: "browser_files", + capability: "browser filesystem upload", + requested_action: "transfer", + task_outcome: "blocked", + }, + })) as { isError?: boolean }; + + expect(result.isError).toBe(true); + const toolCall = captured.find( + ({ event }) => event === PostHogMCPAnalyticsEvent.ToolCall, + ) as { properties: Record }; + expect(toolCall.properties).toMatchObject({ + [PostHogMCPAnalyticsProperty.ToolName]: "get_more_tools", + [PostHogMCPAnalyticsProperty.IsError]: true, + [PostHogMCPAnalyticsProperty.ErrorType]: "validation", + }); + expect( + toolCall.properties[PostHogMCPAnalyticsProperty.ErrorMessage], + ).toBeUndefined(); + }); + test("captures only structured capability demand", async () => { const captured: { event?: string }[] = []; diff --git a/src/lib/mcp/analytics.ts b/src/lib/mcp/analytics.ts index 2859acf..3a366bd 100644 --- a/src/lib/mcp/analytics.ts +++ b/src/lib/mcp/analytics.ts @@ -19,6 +19,7 @@ import { registerFeedbackTool, } from "@/lib/mcp/tools/feedback"; import { + KERNEL_MISSING_CAPABILITY_TOOL_NAME, type MissingCapabilityReport, registerMissingCapabilityTool, } from "@/lib/mcp/tools/missing-capability"; @@ -360,6 +361,17 @@ export const sanitizeMcpAnalyticsEvent: BeforeSendFn = (event) => { enrichMcpAnalyticsEvent(event); if (event.event === PostHogMCPAnalyticsEvent.ToolCall) { annotateProjectParamUsage(properties); + const errorMessage = properties[PostHogMCPAnalyticsProperty.ErrorMessage]; + if ( + properties[PostHogMCPAnalyticsProperty.ToolName] === + KERNEL_MISSING_CAPABILITY_TOOL_NAME && + properties[PostHogMCPAnalyticsProperty.IsError] === true && + properties[PostHogMCPAnalyticsProperty.ErrorType] === "Error" && + typeof errorMessage === "string" && + errorMessage.includes("Input validation error") + ) { + properties[PostHogMCPAnalyticsProperty.ErrorType] = "validation"; + } } for (const key of Object.keys(properties)) { @@ -763,7 +775,13 @@ export function instrumentMcpAnalytics( return; } - const analytics = instrument(server, client, { + // Register first so analytics wraps the legacy dispatch shim and records those calls. + let analytics: McpAnalytics; + registerMissingCapabilityTool(server, (report, extra) => + captureMissingCapabilityReport(report, extra, analytics), + ); + + analytics = instrument(server, client, { // The first-class get_more_tools handler validates and captures structured demand itself. // Point the SDK's name-based interception at an unadvertised name so calls to the real // tool reach its registered schema and callback even while reportMissing is disabled. @@ -812,9 +830,6 @@ export function instrumentMcpAnalytics( beforeSend: sanitizeMcpAnalyticsEvent, }); - registerMissingCapabilityTool(server, (report, extra) => - captureMissingCapabilityReport(report, extra, analytics), - ); registerFeedbackTool(server, (feedback, extra) => captureMcpFeedback(feedback, extra, analytics), ); diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 2fabb8e..5444042 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -36,6 +36,11 @@ function captureRegistration( const resources: string[] = []; const schemas = new Map>(); const server = { + server: { + _requestHandlers: new Map([ + ["tools/call", async () => ({ content: [] })], + ]), + }, prompt() {}, resource() {}, tool(name: string, _description: string, inputSchema: object) { diff --git a/src/lib/mcp/tools/missing-capability.test.ts b/src/lib/mcp/tools/missing-capability.test.ts index c162f84..4e4abb7 100644 --- a/src/lib/mcp/tools/missing-capability.test.ts +++ b/src/lib/mcp/tools/missing-capability.test.ts @@ -76,6 +76,46 @@ describe("get_more_tools", () => { } }); + test("accepts the previous context-only contract without recording demand", async () => { + const captured: MissingCapabilityReport[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerMissingCapabilityTool(server, (report) => { + captured.push(report); + }), + {}, + ); + + try { + const legacy = await client.callTool({ + name: KERNEL_MISSING_CAPABILITY_TOOL_NAME, + arguments: { + context: + "Reporting a missing capability through the previous context-only contract while the client refreshes its tools.", + }, + }); + expect(legacy.isError).not.toBe(true); + expect(toolResultJSON(legacy)).toMatchObject({ + recorded: false, + status: "legacy_schema_refresh_required", + }); + expect(captured).toHaveLength(0); + + const partial = await client.callTool({ + name: KERNEL_MISSING_CAPABILITY_TOOL_NAME, + arguments: { + context: + "Reporting a partially structured capability request that must not fall back to the compatibility contract.", + gap_reason: "kernel_capability_missing", + }, + }); + expect(partial.isError).toBe(true); + expect(captured).toHaveLength(0); + } finally { + await close(); + } + }); + test("reports capture failures without interrupting the task", async () => { const { client, close } = await connectTestMcp( (server) => diff --git a/src/lib/mcp/tools/missing-capability.ts b/src/lib/mcp/tools/missing-capability.ts index 3082254..b1a84c3 100644 --- a/src/lib/mcp/tools/missing-capability.ts +++ b/src/lib/mcp/tools/missing-capability.ts @@ -108,28 +108,80 @@ const missingCapabilityFields = { ), }; +const structuredMissingCapabilitySchema = z.object(missingCapabilityFields); + export type MissingCapabilityReport = z.infer< - z.ZodObject + typeof structuredMissingCapabilitySchema >; export type MissingCapabilityCapture = ( report: MissingCapabilityReport, extra: unknown, ) => void | Promise; +type ToolCallRequest = { + params?: { name?: unknown; arguments?: unknown }; +}; +type ToolCallHandler = ( + request: ToolCallRequest, + extra: unknown, +) => Promise; + +function legacySchemaResponse() { + return jsonResponse({ + recorded: false, + status: "legacy_schema_refresh_required", + message: + "This client used the previous get_more_tools schema. Refresh the available tool definitions, retry with the structured fields, and continue the original task with any available workaround.", + }); +} + +function isLegacyContextOnlyCall(request: ToolCallRequest) { + if (request.params?.name !== KERNEL_MISSING_CAPABILITY_TOOL_NAME) + return false; + const args = request.params.arguments; + if (!args || typeof args !== "object" || Array.isArray(args)) return false; + const entries = Object.entries(args); + return ( + entries.length === 1 && + entries[0]?.[0] === "context" && + typeof entries[0][1] === "string" + ); +} + +function acceptLegacyContextOnlyCalls(server: McpServer) { + // The SDK validates before invoking the tool callback, so handle only the exact old + // payload here while leaving the advertised structured schema unchanged. + const handlers = ( + server.server as unknown as { + _requestHandlers: Map; + } + )._requestHandlers; + const handler = handlers.get("tools/call"); + if (!handler) throw new Error("tools/call handler is not registered"); + + handlers.set("tools/call", async (request, extra) => { + if (isLegacyContextOnlyCall(request)) return legacySchemaResponse(); + return handler(request, extra); + }); +} + export function registerMissingCapabilityTool( server: McpServer, capture?: MissingCapabilityCapture, ) { - server.tool( + server.registerTool( KERNEL_MISSING_CAPABILITY_TOOL_NAME, - "Report a capability that no available KERNEL tool can provide after checking the tool list. Classify disconnected third-party services as external integrations. Do not use this for an existing tool that failed, a transient or capacity failure, or a client-side permission restriction; use submit_feedback for an existing KERNEL tool failure. Reports never replace the original task, so continue with any available workaround.", - missingCapabilityFields, { - title: "Get more tools", - readOnlyHint: false, - destructiveHint: false, - idempotentHint: false, - openWorldHint: true, + description: + "Report a capability that no available KERNEL tool can provide after checking the tool list. Classify disconnected third-party services as external integrations. Do not use this for an existing tool that failed, a transient or capacity failure, or a client-side permission restriction; use submit_feedback for an existing KERNEL tool failure. Reports never replace the original task, so continue with any available workaround.", + inputSchema: structuredMissingCapabilitySchema, + annotations: { + title: "Get more tools", + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: true, + }, }, async (report, extra) => { const externalIntegration = @@ -188,4 +240,5 @@ export function registerMissingCapabilityTool( }); }, ); + acceptLegacyContextOnlyCalls(server); }