feat(webapp,dashboard-agent): add a submit_feedback tool to the dashboard agent

The dashboard agent can now report problems with its own tools, the
dashboard, or the docs to the Trigger.dev team with a new
`submit_feedback` tool.

```ts
submit_feedback({
  message: "get_queue returned a 500 for a paused queue; expected the paused row",
  toolName: "get_queue",
});
```

The agent uses it when a tool error looks like a bug, the docs
contradict the product, or it lacks a capability it needed, and it tells
the user what it reported. It never reports the user's own failing runs,
and never includes secrets, payloads, or logs.

Reports go through a new `POST /api/v1/dashboard-agent/feedback` route,
which only accepts the agent's delegated token and attributes the report
to the chat's user, organization, project and environment. Reporting is
fire-and-forget so it never slows the agent down: the event is queued
like the rest of the dashboard's analytics, and a report that doesn't go
through (including on an instance without analytics configured) is
dropped quietly instead of surfacing as an error.

Mono-RevId: b9c67c574d621ab1ec365046f401a9b031b386d2

Author: ericallam

apps/webapp/app/components/dashboard-agent/tool-labels.ts

@@ -26,6 +26,7 @@ const TOOL_LABELS: Record<string, string> = {
   create_alert: "Creating an alert",
   delete_alert: "Deleting an alert",
   locate: "Locating the id",
+  submit_feedback: "Sending feedback",
   // Code mode.
   get_repo_info: "Reading the repo",
   list_files: "Listing files",

apps/webapp/app/routes/api.v1.dashboard-agent.feedback.ts

@@ -0,0 +1,74 @@
+import { json, type ActionFunctionArgs } from "@remix-run/server-runtime";
+import { DASHBOARD_AGENT_FEEDBACK_LIMITS } from "@internal/dashboard-agent-contracts";
+import { z } from "zod";
+import { resolveAgentAlertContext } from "~/services/dashboardAgentAlertContext.server";
+import { logger } from "~/services/logger.server";
+import { telemetry } from "~/services/telemetry.server";
+import { authenticateUatOrApiRequest } from "~/services/uatRoutePreamble.server";
+
+/**
+ * `POST` records the agent's report of a problem with its own tools or the docs. Only the
+ * agent's delegated user-actor token is accepted, and the user, organization, project and
+ * environment all come from it and the chat, never from the body.
+ */
+
+const FeedbackBodySchema = z.object({
+  chatId: z.string().min(1),
+  message: z.string().trim().min(1).max(DASHBOARD_AGENT_FEEDBACK_LIMITS.message),
+  toolName: z.string().min(1).max(DASHBOARD_AGENT_FEEDBACK_LIMITS.toolName).optional(),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json({ error: "Method not allowed" }, { status: 405 });
+  }
+
+  const authentication = await authenticateUatOrApiRequest(request);
+  const actor = authentication?.userActor;
+  if (!actor || actor.client !== "dashboard-agent") {
+    return json({ error: "Invalid or missing access token" }, { status: 401 });
+  }
+  if (!actor.environmentId) {
+    return json(
+      { error: "This chat has no environment context.", code: "invalid_target" },
+      { status: 400 }
+    );
+  }
+
+  const body = FeedbackBodySchema.safeParse(await request.json().catch(() => undefined));
+  if (!body.success) {
+    return json({ error: "Invalid request", code: "invalid_request" }, { status: 400 });
+  }
+
+  const context = await resolveAgentAlertContext({
+    userId: actor.userId,
+    environmentId: actor.environmentId,
+    chatId: body.data.chatId,
+  });
+  if (!context.ok) {
+    return json({ error: context.error, code: context.code }, { status: 404 });
+  }
+
+  const recorded = telemetry.dashboardAgent.feedback({
+    userId: actor.userId,
+    organizationId: context.environment.organizationId,
+    projectId: context.environment.project.id,
+    environmentId: context.environment.id,
+    chatId: body.data.chatId,
+    message: body.data.message,
+    toolName: body.data.toolName,
+  });
+  if (!recorded) {
+    return json(
+      { error: "Feedback isn't collected on this instance.", code: "feedback_not_configured" },
+      { status: 501 }
+    );
+  }
+
+  logger.info("Dashboard agent feedback submitted", {
+    chatId: body.data.chatId,
+    toolName: body.data.toolName,
+    messageLength: body.data.message.length,
+  });
+  return json({ recorded: true });
+}

apps/webapp/app/services/telemetry.server.ts

@@ -233,6 +233,39 @@ class Telemetry {
     },
   };
 
+  dashboardAgent = {
+    /** Returns whether the report was captured, so the agent never claims one that was dropped. */
+    feedback: ({
+      userId,
+      organizationId,
+      projectId,
+      environmentId,
+      chatId,
+      message,
+      toolName,
+    }: {
+      userId: string;
+      organizationId: string;
+      projectId: string;
+      environmentId: string;
+      chatId: string;
+      message: string;
+      toolName?: string;
+    }): boolean => {
+      if (this.#posthogClient === undefined) return false;
+      if (!IS_MANAGED_CLOUD) return false;
+      this.#capture({
+        userId,
+        event: "dashboard_agent_feedback_submitted",
+        organizationId,
+        projectId,
+        environmentId,
+        eventProperties: { chatId, message, toolName },
+      });
+      return true;
+    },
+  };
+
   #capture(event: CaptureEvent) {
     if (this.#posthogClient === undefined) return;
     let groups: Record<string, string> = {};

apps/webapp/test/dashboardAgentFeedbackRoute.test.ts

@@ -0,0 +1,111 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+  authenticate: vi.fn(),
+  resolveContext: vi.fn(),
+  feedback: vi.fn(),
+}));
+
+vi.mock("~/services/uatRoutePreamble.server", () => ({
+  authenticateUatOrApiRequest: mocks.authenticate,
+}));
+vi.mock("~/services/dashboardAgentAlertContext.server", () => ({
+  resolveAgentAlertContext: mocks.resolveContext,
+}));
+vi.mock("~/services/telemetry.server", () => ({
+  telemetry: { dashboardAgent: { feedback: mocks.feedback } },
+}));
+vi.mock("~/services/logger.server", () => ({
+  logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+}));
+
+import { action } from "~/routes/api.v1.dashboard-agent.feedback";
+
+const AGENT_ACTOR = { userId: "user_1", client: "dashboard-agent", environmentId: "env_1" };
+const ENVIRONMENT = { id: "env_1", organizationId: "org_1", project: { id: "proj_1" } };
+
+function post(body: unknown) {
+  return action({
+    request: new Request("https://app.trigger.dev/api/v1/dashboard-agent/feedback", {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: "Bearer uat" },
+      body: JSON.stringify(body),
+    }),
+    params: {},
+    context: {},
+  } as any);
+}
+
+beforeEach(() => {
+  mocks.authenticate.mockReset().mockResolvedValue({ userActor: AGENT_ACTOR });
+  mocks.resolveContext.mockReset().mockResolvedValue({ ok: true, environment: ENVIRONMENT });
+  mocks.feedback.mockReset().mockReturnValue(true);
+});
+
+describe("POST /api/v1/dashboard-agent/feedback", () => {
+  it("attributes the report from the token and the chat, never from the body", async () => {
+    const response = await post({
+      chatId: "chat_1",
+      message: "get_queue 500s on a paused queue",
+      toolName: "get_queue",
+      userId: "user_spoofed",
+      organizationId: "org_spoofed",
+    });
+
+    expect(response.status).toBe(200);
+    expect(mocks.resolveContext).toHaveBeenCalledWith({
+      userId: "user_1",
+      environmentId: "env_1",
+      chatId: "chat_1",
+    });
+    expect(mocks.feedback).toHaveBeenCalledWith({
+      userId: "user_1",
+      organizationId: "org_1",
+      projectId: "proj_1",
+      environmentId: "env_1",
+      chatId: "chat_1",
+      message: "get_queue 500s on a paused queue",
+      toolName: "get_queue",
+    });
+  });
+
+  it.each([
+    ["no token", undefined],
+    ["a token from another client", { userActor: { ...AGENT_ACTOR, client: "mcp" } }],
+  ])("refuses %s", async (_label, authentication) => {
+    mocks.authenticate.mockResolvedValue(authentication);
+
+    const response = await post({ chatId: "chat_1", message: "hi" });
+
+    expect(response.status).toBe(401);
+    expect(mocks.feedback).not.toHaveBeenCalled();
+  });
+
+  it("records nothing for a chat the user doesn't own", async () => {
+    mocks.resolveContext.mockResolvedValue({
+      ok: false,
+      code: "chat_not_found",
+      error: "Chat not found",
+    });
+
+    const response = await post({ chatId: "chat_other", message: "hi" });
+
+    expect(response.status).toBe(404);
+    expect(mocks.feedback).not.toHaveBeenCalled();
+  });
+
+  it("rejects an empty or oversized message", async () => {
+    expect((await post({ chatId: "chat_1", message: "   " })).status).toBe(400);
+    expect((await post({ chatId: "chat_1", message: "x".repeat(4001) })).status).toBe(400);
+    expect(mocks.feedback).not.toHaveBeenCalled();
+  });
+
+  it("answers 501 when the instance doesn't collect feedback", async () => {
+    mocks.feedback.mockReturnValue(false);
+
+    const response = await post({ chatId: "chat_1", message: "docs say X, product does Y" });
+
+    expect(response.status).toBe(501);
+    expect(await response.json()).toMatchObject({ code: "feedback_not_configured" });
+  });
+});

internal-packages/dashboard-agent-contracts/src/feedback.ts

@@ -0,0 +1,8 @@
+/**
+ * The bounds on a `submit_feedback` report. Shared so the tool's schema and the route that
+ * records it can't drift apart and turn a valid report into a 400.
+ */
+export const DASHBOARD_AGENT_FEEDBACK_LIMITS = {
+  message: 4000,
+  toolName: 100,
+} as const;

internal-packages/dashboard-agent-contracts/src/index.ts

@@ -5,6 +5,7 @@
 export * from "./agent-records.js";
 export * from "./blocks.js";
 export * from "./evidence.js";
+export * from "./feedback.js";
 export * from "./intent.js";
 export * from "./locate.js";
 export * from "./page-context.js";

internal-packages/dashboard-agent/src/__snapshots__/prompt-prefix.test.ts.snap

@@ -1234,6 +1234,7 @@ describe("buildDashboardAgentTools", () => {
         "run_query",
         "render_view",
         "search_docs",
+        "submit_feedback",
       ].sort()
     );
 

internal-packages/dashboard-agent/src/dashboard-agent.test.ts

@@ -130,8 +130,8 @@ describe("the head-start and agent prefixes are the same prefix", () => {
  * drift. The snapshot below is the itemised diff a reviewer reads.
  */
 const PREFIX_BUDGET = {
-  assistant: { chars: 85_300, estimatedTokens: 21_350, tools: 25, promptChars: 34_000 },
-  code: { chars: 92_500, estimatedTokens: 23_150, tools: 29, promptChars: 36_700 },
+  assistant: { chars: 86_250, estimatedTokens: 21_600, tools: 26, promptChars: 34_100 },
+  code: { chars: 93_450, estimatedTokens: 23_400, tools: 30, promptChars: 36_850 },
 } as const;
 
 // Measured with watches on: the biggest prefix a turn can hand the provider.

internal-packages/dashboard-agent/src/prompt-prefix.test.ts

@@ -0,0 +1,72 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createApiClient } from "./tool-api-client";
+import type { DashboardAgentToolContext } from "./tool-context";
+import { buildFeedbackTool } from "./tool-feedback";
+
+const ORIGIN = "https://api.example.com";
+const CTX: DashboardAgentToolContext = {
+  userActorToken: "uat",
+  apiOrigin: ORIGIN,
+  chatId: "chat_1",
+};
+
+function submit(input: { message: string; toolName?: string }, ctx = CTX) {
+  const client = createApiClient(ctx);
+  const execute = (buildFeedbackTool({ ctx, client }).submit_feedback as any).execute;
+  return execute(input, {} as any) as Promise<any>;
+}
+
+function stubFetch(response: Response | Error) {
+  const fetch = vi.fn(async (_input: any, _init?: RequestInit) => {
+    if (response instanceof Error) throw response;
+    return response;
+  });
+  vi.stubGlobal("fetch", fetch);
+  return fetch;
+}
+
+afterEach(() => vi.unstubAllGlobals());
+
+describe("submit_feedback", () => {
+  it("posts the report with the chat it came from, under the delegated token", async () => {
+    const fetch = stubFetch(new Response(JSON.stringify({ recorded: true }), { status: 200 }));
+
+    const result = await submit({
+      message: "get_queue 500s on a paused queue",
+      toolName: "get_queue",
+    });
+
+    expect(result).toMatchObject({ recorded: true });
+    const [url, init] = fetch.mock.calls[0]!;
+    expect(url).toBe(`${ORIGIN}/api/v1/dashboard-agent/feedback`);
+    expect(init?.method).toBe("POST");
+    expect((init!.headers as Record<string, string>).Authorization).toBe("Bearer uat");
+    expect(JSON.parse(String(init?.body))).toEqual({
+      chatId: "chat_1",
+      message: "get_queue 500s on a paused queue",
+      toolName: "get_queue",
+    });
+  });
+
+  it.each([
+    ["an instance that doesn't collect feedback", new Response("{}", { status: 501 })],
+    ["a server error", new Response("{}", { status: 500 })],
+    ["a transport failure", new Error("socket hang up")],
+  ])("carries on quietly on %s, without an error", async (_label, response) => {
+    stubFetch(response);
+
+    const result = await submit({ message: "something broke" });
+
+    expect(result.recorded).toBe(false);
+    expect(result.error).toBeUndefined();
+  });
+
+  it("sends nothing without a chat to attribute it to", async () => {
+    const fetch = stubFetch(new Response("{}", { status: 200 }));
+
+    const result = await submit({ message: "something broke" }, { ...CTX, chatId: undefined });
+
+    expect(result.error).toBeDefined();
+    expect(fetch).not.toHaveBeenCalled();
+  });
+});