Add bounded progressive MCP discovery

packages/gatekeeper-mcp-portal/README.md

@@ -28,7 +28,7 @@ every system the organization has connected in one click.
 pinning different tools of one upstream server share both the name and the endpoint, so the scope is
 the only thing that distinguishes them.
 
-The session API — a typed method per tool, plus `callTool`, `getActionResult`, and `listTools` — is
+The session API — a typed method per described tool, plus `callTool`, `getActionResult`, and `listTools` — is
 the same as [`gatekeeper-mcp`](../gatekeeper-mcp/README.md#what-it-provides).
 
 Scoping to one server also shrinks what the agent reads: a 57-tool portal generates 57 tool

packages/gatekeeper-mcp-portal/src/portal.ts

@@ -26,6 +26,7 @@ import {
 import type { ToolCatalog } from "@gadgets/mcp-shared/client";
 import {
   classifyTool,
+  MAX_TOOLS_PER_SERVER,
   type ServerTrust,
 } from "@gadgets/mcp-shared/tools";
 import { bindingNameFragment, hostOf } from "@gadgets/mcp-shared/util";
@@ -397,7 +398,7 @@ class McpServerConfiguratorUI extends RpcTarget implements McpServerConfigurator
   // form treats both as having nothing to grant rather than telling them apart.
   async listServerOptions(): Promise<ConfiguratorUIOption[]> {
     const { tools, truncated } = await this.#tools();
-    if (!looksLikePortal(tools, truncated)) return [];
+    if (!looksLikePortal(tools, { truncated, cap: MAX_TOOLS_PER_SERVER })) return [];
 
     const servers = reconcilePortalServers(
       await this.#fetchPortalServers(), tools, truncated);
@@ -437,7 +438,7 @@ class McpServerConfiguratorUI extends RpcTarget implements McpServerConfigurator
     const { tools, truncated } = await this.#tools();
     requireCompleteCatalogForToolSelection(truncated);
     const scope: ToolScope = serverId ? { serverId } : {};
-    const isPortal = looksLikePortal(tools, truncated);
+    const isPortal = looksLikePortal(tools, { truncated, cap: MAX_TOOLS_PER_SERVER });
 
     return tools
       .filter(tool => scopeAllows(scope, tool.name, isPortal))
@@ -517,9 +518,9 @@ export class McpGatekeeperImpl
       ? `${scope.tools.length} named MCP tool${scope.tools.length === 1 ? "" : "s"} on ` +
         `${label} \u2014 ${counts}. Other tools are refused.`
       : scope.serverId
-      ? `All ${tools.length} MCP tool${plural} of the ` +
-        `${this.ctx.props.scopeServerName ?? scope.serverId} server on ` +
-        `${this.ctx.props.serverName} \u2014 ${counts}. Other servers on it are refused.`
+      ? `All tools of the ${this.ctx.props.scopeServerName ?? scope.serverId} server on ` +
+        `${this.ctx.props.serverName}; ${tools.length} definition${plural} shown here ` +
+        `(${counts}). Other servers on it are refused.`
       : `All ${tools.length} MCP tool${plural} on ${label} \u2014 ${counts}.`;
 
     return {

packages/gatekeeper-mcp/README.md

@@ -52,12 +52,18 @@ await env.MCP_LINEAR.callTool("search_issues", { query: "state:open" });
 
 Each generated method is a one-line delegate to `callTool`, so the scope check, approval queue, and
 observation record stay in one place. Tools whose names the RPC layer cannot deliver (`then`, `map`,
-`dup`), names that are not identifiers (`2fa`), and both sides of a case collision (`list_issues`
-and `listIssues`) get no method and remain callable through `callTool`.
-
-The agent discovers tools statically: `describeGatekeeper()` sends it the binding name and the full
-generated `.d.ts`, where each method carries the tool's own description as JSDoc and states whether
-calling it needs approval. `listTools()` exists for runtime enumeration but is rarely needed.
+`dup`), names that collide with session methods (`listTools`, `callTool`, `getActionResult`), names
+that are not identifiers (`2fa`), and both
+sides of a case collision (`list_issues` and `listIssues`) get no method and remain callable through
+`callTool`.
+
+The agent normally discovers tools statically: `describeGatekeeper()` sends it the binding name and
+the bounded generated `.d.ts`, where each method carries the tool's own description as JSDoc and
+states whether calling it needs approval. A server can publish more definitions than that bounded
+catalog holds: `listTools({ search })` returns up to 20 compact summaries from a bounded scan of the
+grant, limited to 5,000 tools / 4 MiB; `listTools({ name })` fetches one exact granted definition and
+schema under the same bound, and `callTool()` invokes it by wire name.
+`listTools()` remains the list of definitions currently described in the generated surface.
 
 See `src/types.d.ts` in `@gadgets/mcp-shared` for the base session API.
 

packages/gatekeeper-mcp/src/mcp.ts

@@ -2,7 +2,7 @@
 //
 // Every call is either an observation or an approval-gated action, `readOnlyHint` decides which,
 // writes are queued rather than performed inline, and per-tool TypeScript is generated from the
-// server's schemas so Code Mode works.
+// server's schemas so Gadget code gets typed methods.
 //
 // The endpoint is whatever a user typed, so annotations never earn auto-approval here and a Gadget
 // bound to it is owner-only. See `sharing-policy.ts` and the README.
@@ -26,6 +26,7 @@ import {
 import type { ToolCatalog } from "@gadgets/mcp-shared/client";
 import {
   classifyTool,
+  MAX_TOOLS_PER_SERVER,
   type ServerTrust,
 } from "@gadgets/mcp-shared/tools";
 import { bindingNameFragment, hostOf } from "@gadgets/mcp-shared/util";
@@ -34,7 +35,7 @@ import { generateSessionTypes, sessionTypeName } from "@gadgets/mcp-shared/schem
 import { McpAccountBase, type ConnectedServer, type ConnectOutcome }
   from "@gadgets/mcp-shared/account";
 import { generateNonce } from "@gadgets/mcp-shared/connect-nonce";
-import { fetchTools, type ConnectionAccount } from "@gadgets/mcp-shared/connection";
+import { fetchTools, withClient, type ConnectionAccount } from "@gadgets/mcp-shared/connection";
 import { McpSessionBase } from "@gadgets/mcp-shared/session";
 import { McpFacetBase } from "@gadgets/mcp-shared/facet";
 import { looksLikePortal } from "@gadgets/mcp-shared/portal";
@@ -282,8 +283,21 @@ export class GatekeeperUserImpl
         `do. Connect this endpoint through the MCP Server Portals connector instead.`);
     }
     if (scope.tools !== undefined) {
+      const selected = new Set(scope.tools);
       validateToolScopeAgainstCatalog(
-        scope, await fetchTools(this.env, this.#account(), server.endpoint));
+        scope,
+        selected.size === 0
+          ? { tools: [], truncated: false }
+          : await withClient(
+            this.env,
+            this.#account(),
+            server.endpoint,
+            client => client.listMatchingToolIndex(
+              selected.size,
+              tool => selected.has(tool.name),
+            ),
+          ),
+      );
     }
 
     const props: McpGatekeeperImplProps = {
@@ -354,7 +368,10 @@ class McpServerConfiguratorUI extends RpcTarget implements McpServerConfigurator
   async listToolOptions(): Promise<ConfiguratorUIOption[]> {
     const { tools, truncated } = await this.#tools();
     requireCompleteCatalogForToolSelection(truncated);
-    const isPortal = looksLikePortal(tools, truncated);
+    // `fetchTools` lists with the ordinary catalog cap, so that is the cap reaching it would be
+    // evidence of. Unlike the portal connector, this form refuses a truncated catalog outright
+    // rather than surveying past it, so `truncated` is already known to be false here.
+    const isPortal = looksLikePortal(tools, { truncated, cap: MAX_TOOLS_PER_SERVER });
 
     return tools
       .filter(tool => scopeAllows({}, tool.name, isPortal))
@@ -437,7 +454,8 @@ export class McpGatekeeperImpl
     const snippet = scope.tools
       ? `${scope.tools.length} named MCP tool${scope.tools.length === 1 ? "" : "s"} on ` +
         `${serverName} \u2014 ${counts}. Other tools are refused.`
-      : `All ${tools.length} MCP tool${plural} on ${serverName} \u2014 ${counts}.`;
+      : `All tools on ${serverName}; ${tools.length} tool definition${plural} shown here ` +
+        `(${counts}).`;
 
     return {
       url: this.resourceUrl,

packages/mcp-shared/README.md

@@ -21,15 +21,16 @@ does so through a named hook (`staticToken`, `mintAccount`), not a private copy.
 | `client` | Bounded Streamable HTTP transport (`initialize`, `tools/list`, `tools/call`) using official MCP wire types |
 | `oauth` | Small adapter around the official MCP client's OAuth errors and token revocation gap |
 | `tools` | The trust boundary: read/action classification, auto-approval eligibility, approval prompts, catalog fingerprinting |
-| `schema-to-ts` | JSON Schema to TypeScript, one typed method per tool plus `callTool` overloads |
+| `schema-to-ts` | JSON Schema to TypeScript, strict `callTool` overloads plus progressive discovery |
 | `session-methods` | Installs those methods at runtime, so the generated types are not a fiction |
+| `tool-search` | The one query matcher every catalog search uses, so a query cannot mean two things |
 | `portal` | Gateway detection, tool-name to upstream-server mapping, server listing |
 | `scope` | The resource-URL scope grammar, and the check every call passes through |
 | `endpoint` | Validation and host blocklist for a user-supplied endpoint |
 | `fetch` | Every outbound request; redirects are followed by hand and each hop re-checked, including SDK OAuth fetches |
 | `account` | Durable Object base and persisted SDK OAuth state: connect, refresh, revocation |
 | `facet` | Common session, catalog, action, and sharing behavior for connector-owned Durable Object facets |
-| `catalog` | One binding's tool list: fetched, cached, scoped to the grant, classified |
+| `catalog` | One binding's tool list: fetched, cached, scoped to the grant, classified; plus the cache for tools hydrated past that list |
 | `connection` | `withClient` — transport sessions, retries, credential-expiry reporting |
 | `action-store` | Staged to applied/rejected/failed, with a bound on what is retained and a claim so one approval is never sent twice |
 | `session` | The Gadget-facing capability, and the one path every tool call takes |
@@ -109,12 +110,16 @@ Fixed rather than configurable.
 
 | Limit | Value | Where | Why |
 | --- | --- | --- | --- |
-| Tools per server | 200 | `tools.ts` | A grant a person can review, and a `.d.ts` an agent can read |
+| Described or individually granted tools per server | 200 | `tools.ts` | Bounds the picker and generated `.d.ts`; a server-wide grant can discover additional tools later |
 | Catalog size | 96 KiB UTF-8 | `client.ts` | Leaves room below Durable Object's 128 KiB per-value limit for the cache wrapper and serialization overhead |
+| Filtered discovery scan | 5,000 tools / 4 MiB | `client.ts` | Bounds work spent skipping unrelated tools while searching a large endpoint |
+| Search query / results | 200 chars / 20 tools | `tool-search.ts` | Bounds agent-supplied matching work and the summaries returned to it |
+| Hydrated definitions per facet | 200 tools / 1 MiB | `catalog.ts` | Bounds definitions fetched individually beyond the described catalog |
 | Tool description | 4 KB | `client.ts` | As above, per tool, before it reaches storage |
 | Tool input schema | 20 KB | `client.ts` | Dropped rather than clipped; half a schema is not a schema |
-| `tools/list` pages | 50 | `client.ts` | A cursor that never ends would loop until the Worker is killed |
+| `tools/list` pages | 50 | `client.ts` | Stops a cursor that never ends; exhaustion truncates catalogs and fails exact/search discovery as a scan limit |
 | Response body | 1 MiB | `fetch.ts` | Every response is buffered whole before it can be parsed, and a `tools/call` result is otherwise unbounded |
+| Bounded outbound operation | 30 seconds | `fetch.ts` | OAuth and discovery callers opt into one deadline covering redirects, pagination, body streaming, and session retry |
 | Retained result | 128 KB | `action-store.ts` | Held until the Gadget collects it; oversized ones are replaced by a note |
 | Retained actions | 100 | `action-store.ts` | Records are for collecting a result, not an audit log |
 | Actions awaiting a decision | 50 | `action-store.ts` | These cannot be pruned, so uncapped they are an unbounded write |