Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ See [Vault payments](docs/vault-payments.md) for both provider flows, safety rul
- `webmcp` - List native page tools across every tab and frame in a browser, then synchronously invoke an exact opaque `tool_ref` with structured input.
- `exec_command` - Run shell commands inside a browser VM. Returns decoded stdout/stderr.
- `search_docs` - Search Kernel platform documentation and guides.
- `get_more_tools` - Report a structured KERNEL capability or external-integration gap after checking the available tools. Existing-tool failures, transient capacity errors, and client permission restrictions are rejected from capability-demand analytics. Accepted requests emit `mcp_capability_requested`; historical unstructured requests remain under `$mcp_missing_capability`.
- `get_more_tools` - Report a structured KERNEL capability or external-integration gap after checking the available tools. Existing-tool failures, transient capacity errors, and client permission restrictions are rejected from capability-demand analytics. Accepted requests emit `mcp_capability_requested`; clients using the previous context-only schema receive a non-recording refresh response instead of a tool error.
- `submit_feedback` - Send product, bot-detection, config-registry, MCP, or documentation feedback directly to the KERNEL team without interrupting the current task. Reports include a normalized task outcome; MCP reports identify one KERNEL-owned tool. Config-registry reports connect exactly one observed outcome to the browser session, recommendation metadata and evidence, and unchanged browser and proxy settings.
- `open_auth_login` - Open a secure interactive Managed Auth MCP App after user consent. Registered only for clients that declare MCP Apps support; credentials and MFA never enter MCP/model traffic.

Expand Down
43 changes: 43 additions & 0 deletions src/lib/mcp/analytics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1028,6 +1028,19 @@ describe("instrumentMcpAnalytics (SDK integration)", () => {
missingCapabilityTool?.inputSchema,
);

const legacyRequest = await enabled.client.callTool({
name: "get_more_tools",
arguments: {
context:
"Reporting a capability through the previous contract while refreshing the available tool definitions.",
},
});
expect(legacyRequest.isError).not.toBe(true);
expect(toolResultJSON(legacyRequest)).toMatchObject({
recorded: false,
status: "legacy_schema_refresh_required",
});

const unavailableRequest = await disabled.client.callTool({
name: "get_more_tools",
arguments: {
Expand Down Expand Up @@ -1185,6 +1198,36 @@ describe("instrumentMcpAnalytics (SDK integration)", () => {
expect(byEvent.has("$identify")).toBe(false);
});

test("classifies rejected capability input through instrumentation", async () => {
const captured: { event?: string }[] = [];

const result = (await simulateRequest(captured, "tools/call", {
name: "get_more_tools",
arguments: {
context:
"Reporting a malformed structured capability request to verify validation telemetry.",
gap_reason: "not_a_gap_reason",
capability_area: "browser_files",
capability: "browser filesystem upload",
requested_action: "transfer",
task_outcome: "blocked",
},
})) as { isError?: boolean };

expect(result.isError).toBe(true);
const toolCall = captured.find(
({ event }) => event === PostHogMCPAnalyticsEvent.ToolCall,
) as { properties: Record<string, unknown> };
expect(toolCall.properties).toMatchObject({
[PostHogMCPAnalyticsProperty.ToolName]: "get_more_tools",
[PostHogMCPAnalyticsProperty.IsError]: true,
[PostHogMCPAnalyticsProperty.ErrorType]: "validation",
});
expect(
toolCall.properties[PostHogMCPAnalyticsProperty.ErrorMessage],
).toBeUndefined();
});

test("captures only structured capability demand", async () => {
const captured: { event?: string }[] = [];

Expand Down
23 changes: 19 additions & 4 deletions src/lib/mcp/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
registerFeedbackTool,
} from "@/lib/mcp/tools/feedback";
import {
KERNEL_MISSING_CAPABILITY_TOOL_NAME,
type MissingCapabilityReport,
registerMissingCapabilityTool,
} from "@/lib/mcp/tools/missing-capability";
Expand Down Expand Up @@ -360,6 +361,17 @@ export const sanitizeMcpAnalyticsEvent: BeforeSendFn = (event) => {
enrichMcpAnalyticsEvent(event);
if (event.event === PostHogMCPAnalyticsEvent.ToolCall) {
annotateProjectParamUsage(properties);
const errorMessage = properties[PostHogMCPAnalyticsProperty.ErrorMessage];
if (
properties[PostHogMCPAnalyticsProperty.ToolName] ===
KERNEL_MISSING_CAPABILITY_TOOL_NAME &&
properties[PostHogMCPAnalyticsProperty.IsError] === true &&
properties[PostHogMCPAnalyticsProperty.ErrorType] === "Error" &&
typeof errorMessage === "string" &&
errorMessage.includes("Input validation error")
) {
properties[PostHogMCPAnalyticsProperty.ErrorType] = "validation";
}
}

for (const key of Object.keys(properties)) {
Expand Down Expand Up @@ -763,7 +775,13 @@ export function instrumentMcpAnalytics(
return;
}

const analytics = instrument(server, client, {
// Register first so analytics wraps the legacy dispatch shim and records those calls.
let analytics: McpAnalytics;
registerMissingCapabilityTool(server, (report, extra) =>
captureMissingCapabilityReport(report, extra, analytics),
);

analytics = instrument(server, client, {
// The first-class get_more_tools handler validates and captures structured demand itself.
// Point the SDK's name-based interception at an unadvertised name so calls to the real
// tool reach its registered schema and callback even while reportMissing is disabled.
Expand Down Expand Up @@ -812,9 +830,6 @@ export function instrumentMcpAnalytics(
beforeSend: sanitizeMcpAnalyticsEvent,
});

registerMissingCapabilityTool(server, (report, extra) =>
captureMissingCapabilityReport(report, extra, analytics),
);
registerFeedbackTool(server, (feedback, extra) =>
captureMcpFeedback(feedback, extra, analytics),
);
Expand Down
5 changes: 5 additions & 0 deletions src/lib/mcp/register.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ function captureRegistration(
const resources: string[] = [];
const schemas = new Map<string, Record<string, unknown>>();
const server = {
server: {
_requestHandlers: new Map([
["tools/call", async () => ({ content: [] })],
]),
},
prompt() {},
resource() {},
tool(name: string, _description: string, inputSchema: object) {
Expand Down
40 changes: 40 additions & 0 deletions src/lib/mcp/tools/missing-capability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,46 @@ describe("get_more_tools", () => {
}
});

test("accepts the previous context-only contract without recording demand", async () => {
const captured: MissingCapabilityReport[] = [];
const { client, close } = await connectTestMcp(
(server) =>
registerMissingCapabilityTool(server, (report) => {
captured.push(report);
}),
{},
);

try {
const legacy = await client.callTool({
name: KERNEL_MISSING_CAPABILITY_TOOL_NAME,
arguments: {
context:
"Reporting a missing capability through the previous context-only contract while the client refreshes its tools.",
},
});
expect(legacy.isError).not.toBe(true);
expect(toolResultJSON(legacy)).toMatchObject({
recorded: false,
status: "legacy_schema_refresh_required",
});
expect(captured).toHaveLength(0);

const partial = await client.callTool({
name: KERNEL_MISSING_CAPABILITY_TOOL_NAME,
arguments: {
context:
"Reporting a partially structured capability request that must not fall back to the compatibility contract.",
gap_reason: "kernel_capability_missing",
},
});
expect(partial.isError).toBe(true);
expect(captured).toHaveLength(0);
} finally {
await close();
}
});

test("reports capture failures without interrupting the task", async () => {
const { client, close } = await connectTestMcp(
(server) =>
Expand Down
71 changes: 62 additions & 9 deletions src/lib/mcp/tools/missing-capability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,28 +108,80 @@ const missingCapabilityFields = {
),
};

const structuredMissingCapabilitySchema = z.object(missingCapabilityFields);

export type MissingCapabilityReport = z.infer<
z.ZodObject<typeof missingCapabilityFields>
typeof structuredMissingCapabilitySchema
>;
export type MissingCapabilityCapture = (
report: MissingCapabilityReport,
extra: unknown,
) => void | Promise<void>;

type ToolCallRequest = {
params?: { name?: unknown; arguments?: unknown };
};
type ToolCallHandler = (
request: ToolCallRequest,
extra: unknown,
) => Promise<unknown>;

function legacySchemaResponse() {
return jsonResponse({
recorded: false,
status: "legacy_schema_refresh_required",
message:
"This client used the previous get_more_tools schema. Refresh the available tool definitions, retry with the structured fields, and continue the original task with any available workaround.",
});
}

function isLegacyContextOnlyCall(request: ToolCallRequest) {
if (request.params?.name !== KERNEL_MISSING_CAPABILITY_TOOL_NAME)
return false;
const args = request.params.arguments;
if (!args || typeof args !== "object" || Array.isArray(args)) return false;
const entries = Object.entries(args);
return (
entries.length === 1 &&
entries[0]?.[0] === "context" &&
typeof entries[0][1] === "string"
);
}

function acceptLegacyContextOnlyCalls(server: McpServer) {
// The SDK validates before invoking the tool callback, so handle only the exact old
// payload here while leaving the advertised structured schema unchanged.
const handlers = (
server.server as unknown as {
_requestHandlers: Map<string, ToolCallHandler>;
}
)._requestHandlers;
const handler = handlers.get("tools/call");
if (!handler) throw new Error("tools/call handler is not registered");

handlers.set("tools/call", async (request, extra) => {
if (isLegacyContextOnlyCall(request)) return legacySchemaResponse();
return handler(request, extra);
});
}

export function registerMissingCapabilityTool(
server: McpServer,
capture?: MissingCapabilityCapture,
) {
server.tool(
server.registerTool(
KERNEL_MISSING_CAPABILITY_TOOL_NAME,
"Report a capability that no available KERNEL tool can provide after checking the tool list. Classify disconnected third-party services as external integrations. Do not use this for an existing tool that failed, a transient or capacity failure, or a client-side permission restriction; use submit_feedback for an existing KERNEL tool failure. Reports never replace the original task, so continue with any available workaround.",
missingCapabilityFields,
{
title: "Get more tools",
readOnlyHint: false,
destructiveHint: false,
idempotentHint: false,
openWorldHint: true,
description:
"Report a capability that no available KERNEL tool can provide after checking the tool list. Classify disconnected third-party services as external integrations. Do not use this for an existing tool that failed, a transient or capacity failure, or a client-side permission restriction; use submit_feedback for an existing KERNEL tool failure. Reports never replace the original task, so continue with any available workaround.",
inputSchema: structuredMissingCapabilitySchema,
annotations: {
title: "Get more tools",
readOnlyHint: false,
destructiveHint: false,
idempotentHint: false,
openWorldHint: true,
},
},
async (report, extra) => {
const externalIntegration =
Expand Down Expand Up @@ -188,4 +240,5 @@ export function registerMissingCapabilityTool(
});
},
);
acceptLegacyContextOnlyCalls(server);
}
Loading