From cb039a565c7f2dacad9285180190e42ac09a6195 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Thu, 23 Jul 2026 17:23:14 -0700 Subject: [PATCH] fix(evals): Stabilize model-backed scenarios Centralize tool model configuration, make web source replays argument-faithful, bound stalled AI Gateway response bodies so existing provider retries can recover, and remove unrelated retrieval from the affected memory and subscription scenarios. --- apps/example/.env.example | 1 + ...a0b8700b18168d186911d4f805933d7206c2.json} | 22 ++--- ...731b1a1d10930709264189868d007bcee9ac.json} | 22 ++--- ...94ee98eac65189cdcaa027b2fa0cf213b5eb.json} | 22 ++--- .../junior-evals/evals/agent/files.eval.ts | 3 - .../junior-evals/evals/memory/actors.eval.ts | 4 +- .../junior-evals/evals/memory/shared.eval.ts | 2 +- .../skills/eval-resource-events/SKILL.md | 10 ++ packages/junior-evals/global-setup.ts | 3 + packages/junior-evals/package.json | 1 + packages/junior-evals/src/behavior-harness.ts | 31 ++++-- .../src/eval-ai-gateway-dispatcher.ts | 41 ++++++++ .../eval-ai-gateway-dispatcher.test.ts | 99 +++++++++++++++++++ .../unit/harness/behavior-harness.test.ts | 46 ++++++++- packages/junior/.env.example | 3 +- packages/junior/src/chat/config.ts | 13 +++ packages/junior/src/chat/pi/sdk.ts | 2 + packages/junior/src/chat/prompt.ts | 3 + packages/junior/src/chat/tools/index.ts | 7 +- .../src/chat/tools/web/fetch-content.ts | 16 ++- .../src/chat/tools/web/image-generate.ts | 11 ++- packages/junior/src/chat/tools/web/search.ts | 12 +-- .../component/config/chat-config.test.ts | 34 +++++++ .../tests/unit/services/turn-router.test.ts | 1 + .../tests/unit/web/image-generate.test.ts | 85 +++++++--------- .../tests/unit/web/web-fetch-convert.test.ts | 14 +++ .../junior/tests/unit/web/web-search.test.ts | 41 ++------ pnpm-lock.yaml | 3 + 28 files changed, 406 insertions(+), 146 deletions(-) rename packages/junior-evals/.vitest-evals/recordings/webFetch/{5809ce53de0cd628b4cfbfddc851de84fc86d88418055b1c8f13a6806d556ed5.json => 5ffef6a1f8780d6c3ba7c8bd8285a0b8700b18168d186911d4f805933d7206c2.json} (71%) rename packages/junior-evals/.vitest-evals/recordings/webFetch/{1e892c4d6991f468213d21aac1e9a40688ea15889e99c2fa133a97455b543aa6.json => a73ba09d7c341063e03a399142b4731b1a1d10930709264189868d007bcee9ac.json} (50%) rename packages/junior-evals/.vitest-evals/recordings/webFetch/{7795b1275bd8c551f2a5d9ba0aa2cd5182277af60342e9322e55a8a104464085.json => cb2a12c9a09977dfc53d741d349e94ee98eac65189cdcaa027b2fa0cf213b5eb.json} (51%) create mode 100644 packages/junior-evals/fixtures/resource-event-plugins/eval-resource-events/skills/eval-resource-events/SKILL.md create mode 100644 packages/junior-evals/src/eval-ai-gateway-dispatcher.ts create mode 100644 packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts diff --git a/apps/example/.env.example b/apps/example/.env.example index 1fd7f4a03..2650ef652 100644 --- a/apps/example/.env.example +++ b/apps/example/.env.example @@ -20,4 +20,5 @@ AI_MODEL_PROFILES= AI_EMBEDDING_MODEL= AI_VISION_MODEL= AI_WEB_SEARCH_MODEL= +AI_IMAGE_MODEL= AGENT_TURN_TIMEOUT_MS= diff --git a/packages/junior-evals/.vitest-evals/recordings/webFetch/5809ce53de0cd628b4cfbfddc851de84fc86d88418055b1c8f13a6806d556ed5.json b/packages/junior-evals/.vitest-evals/recordings/webFetch/5ffef6a1f8780d6c3ba7c8bd8285a0b8700b18168d186911d4f805933d7206c2.json similarity index 71% rename from packages/junior-evals/.vitest-evals/recordings/webFetch/5809ce53de0cd628b4cfbfddc851de84fc86d88418055b1c8f13a6806d556ed5.json rename to packages/junior-evals/.vitest-evals/recordings/webFetch/5ffef6a1f8780d6c3ba7c8bd8285a0b8700b18168d186911d4f805933d7206c2.json index 7f003c406..d858fa29c 100644 --- a/packages/junior-evals/.vitest-evals/recordings/webFetch/5809ce53de0cd628b4cfbfddc851de84fc86d88418055b1c8f13a6806d556ed5.json +++ b/packages/junior-evals/.vitest-evals/recordings/webFetch/5ffef6a1f8780d6c3ba7c8bd8285a0b8700b18168d186911d4f805933d7206c2.json @@ -1,24 +1,24 @@ { - "writtenAt": "2026-05-20T23:48:01.611Z", + "writtenAt": "2026-07-23T22:55:00.000Z", "toolName": "webFetch", "input": { "url": "https://docs.slack.dev/changelog/2025/10/7/chat-streaming", - "max_chars": 6000 + "max_chars": 12000 }, "output": { + "ok": true, + "status": "success", + "truncated": false, "url": "https://docs.slack.dev/changelog/2025/10/7/chat-streaming", - "content": "We've introduced a new suite of features to help Slack apps provide an end-user experience typical of LLM tools:\n\n- Slack apps can now stream in their responses to the end user using three new API methods.\n- There are new Block Kit components to allow end users to quickly interact with AI responses.\n- The Node and Python Slack SDKs have new utilities to streamline integration of these new features.\n\nRead on for more info!\n\n### Streaming messages[​](#streaming-messages \"Direct link to Streaming messages\")\n\nThree Web API methods work together to enable Slack apps to provide end users a text streaming experience similar to typical LLM tools:\n\n- The [chat.startStream](/reference/methods/chat.startStream) method starts the text stream.\n- The [chat.appendStream](/reference/methods/chat.appendStream) method appends text to the stream.\n- The [chat.stopStream](/reference/methods/chat.stopStream) method stops the text stream.\n\n### Interacting with AI responses[​](#interacting-with-ai-responses \"Direct link to Interacting with AI responses\")\n\nThree new Block Kit components allow more user interaction with agent responses:\n\n- The [feedback\\_buttons](/reference/block-kit/block-elements/feedback-buttons-element) block element allows users to provide feedback on whether an AI response was positive or negative.\n- The [icon\\_button](/reference/block-kit/block-elements/icon-button-element) provides a way for users to quickly trigger actions like deleting AI responses.\n- The [context\\_actions](/reference/block-kit/blocks/context-actions-block) block provides a container for these interactive elements.\n\n### Supported by Slack SDKs and Bolt framework[​](#supported-by-slack-sdks-and-bolt-framework \"Direct link to Supported by Slack SDKs and Bolt framework\")\n\nThe [Python Slack SDK](/tools/python-slack-sdk) and [Node Slack SDK](/tools/node-slack-sdk) each fully support these new features — with a new `streamer` helper utility to aid developers in tying them all together.\n\nHere's a simplified example in Python:\n\n```\nstreamer = client.chat_stream(\n channel=channel_id,\n recipient_team_id=team_id,\n recipient_user_id=user_id,\n thread_ts=thread_ts,\n)\n\nfor event in returned_message:\n streamer.append(markdown_text=f\"{chunk-received-from-llm}\")\n\nstreamer.stop(blocks=feedback_block)\n\n```\n\nThis utility can also be used in the Bolt frameworks based upon these SDKs. Read more in the _Using AI in apps_ section of your preferred Bolt flavor: [Python](/tools/bolt-python/concepts/ai-apps#text-streaming) or [JavaScript](/tools/bolt-js/concepts/ai-apps#text-streaming).\n\nWe've also updated our app agent template repos to use these new features. Pick your flavor of Bolt and give it a try.\n\nGet started with the [Bolt for Python agent template](https://github.com/slack-samples/bolt-python-assistant-template):\n\n```\nslack create --template slack-samples/bolt-python-assistant-template\nslack run\n\n```\n\nGet started with the [Bolt for JavaScript agent template](https://github.com/slack-samples/bolt-js-assistant-template):\n\n```\nslack create --template slack-samples/bolt-js-assistant-template\nslack run\n\n```\n\nRead all the details in the [Bolt Python release notes](https://github.com/slackapi/bolt-python/releases/tag/v1.26.0) and [Bolt JS release notes](https://github.com/slackapi/bolt-js/releases/tag/%40slack%2Fbolt%404.5.0).", + "content": "We've introduced a new suite of features to help Slack apps provide an end-user experience typical of LLM tools:\n\n- Slack apps can now stream in their responses to the end user using three new API methods.\n- There are new Block Kit components to allow end users to quickly interact with AI responses.\n- The Node and Python Slack SDKs have new utilities to streamline integration of these new features.\n\nRead on for more info!\n\n### Streaming messages[​](#streaming-messages \"Direct link to Streaming messages\")\n\nThree Web API methods work together to enable Slack apps to provide end users a text streaming experience similar to typical LLM tools:\n\n- The [chat.startStream](/reference/methods/chat.startStream) method starts the text stream.\n- The [chat.appendStream](/reference/methods/chat.appendStream) method appends text to the stream.\n- The [chat.stopStream](/reference/methods/chat.stopStream) method stops the text stream.\n\n### Interacting with AI responses[​](#interacting-with-ai-responses \"Direct link to Interacting with AI responses\")\n\nThree new Block Kit components allow more user interaction with agent responses:\n\n- The [feedback\\_buttons](/reference/block-kit/block-elements/feedback-buttons-element) block element allows users to provide feedback on whether an AI response was positive or negative.\n- The [icon\\_button](/reference/block-kit/block-elements/icon-button-element) provides a way for users to quickly trigger actions like deleting AI responses.\n- The [context\\_actions](/reference/block-kit/blocks/context-actions-block) block provides a container for these interactive elements.\n\n### Supported by Slack SDKs and Bolt framework[​](#supported-by-slack-sdks-and-bolt-framework \"Direct link to Supported by Slack SDKs and Bolt framework\")\n\nThe [Python Slack SDK](/tools/python-slack-sdk) and [Node Slack SDK](/tools/node-slack-sdk) each fully support these new features — with a new `streamer` helper utility to aid developers in tying them all together.\n\nHere's a simplified example in Python:\n\n```\nstreamer = client.chat_stream(\n channel=channel_id,\n recipient_team_id=team_id,\n recipient_user_id=user_id,\n thread_ts=thread_ts,\n)\n\nfor event in returned_message:\n streamer.append(markdown_text=f\"{chunk-received-from-llm}\")\n\nstreamer.stop(blocks=feedback_block)\n\n```\n\nThis utility can also be used in the Bolt frameworks based upon these SDKs. Read more in the _Using AI in apps_ section of your preferred Bolt flavor: [Python](/tools/bolt-python/concepts/adding-agent-features/#text-streaming) or [JavaScript](/tools/bolt-js/concepts/adding-agent-features/#text-streaming).\n\nWe've also updated our app agent template repos to use these new features. Pick your flavor of Bolt and give it a try.\n\nGet started with the [Bolt for Python agent template](https://github.com/slack-samples/bolt-python-assistant-template):\n\n```\nslack create --template slack-samples/bolt-python-assistant-template\nslack run\n\n```\n\nGet started with the [Bolt for JavaScript agent template](https://github.com/slack-samples/bolt-js-assistant-template):\n\n```\nslack create --template slack-samples/bolt-js-assistant-template\nslack run\n\n```\n\nRead all the details in the [Bolt Python release notes](https://github.com/slackapi/bolt-python/releases/tag/v1.26.0) and [Bolt JS release notes](https://github.com/slackapi/bolt-js/releases/tag/%40slack%2Fbolt%404.5.0).", "title": "New features designed for Slack apps sending AI responses | Slack Developer Docs", "content_type": "text/html; charset=utf-8", - "source_bytes": 63791, - "extracted_chars": 3345, - "truncated": false, - "status": "success", - "ok": true + "source_bytes": 66710, + "extracted_chars": 3375 }, "metadata": { - "cacheKey": "5809ce53de0cd628b4cfbfddc851de84fc86d88418055b1c8f13a6806d556ed5", - "version": "web-fetch-v1", - "mode": "auto" + "cacheKey": "5ffef6a1f8780d6c3ba7c8bd8285a0b8700b18168d186911d4f805933d7206c2", + "version": "web-fetch-v2", + "mode": "record" } } diff --git a/packages/junior-evals/.vitest-evals/recordings/webFetch/1e892c4d6991f468213d21aac1e9a40688ea15889e99c2fa133a97455b543aa6.json b/packages/junior-evals/.vitest-evals/recordings/webFetch/a73ba09d7c341063e03a399142b4731b1a1d10930709264189868d007bcee9ac.json similarity index 50% rename from packages/junior-evals/.vitest-evals/recordings/webFetch/1e892c4d6991f468213d21aac1e9a40688ea15889e99c2fa133a97455b543aa6.json rename to packages/junior-evals/.vitest-evals/recordings/webFetch/a73ba09d7c341063e03a399142b4731b1a1d10930709264189868d007bcee9ac.json index 5ba8b0e0e..bed7c3ab5 100644 --- a/packages/junior-evals/.vitest-evals/recordings/webFetch/1e892c4d6991f468213d21aac1e9a40688ea15889e99c2fa133a97455b543aa6.json +++ b/packages/junior-evals/.vitest-evals/recordings/webFetch/a73ba09d7c341063e03a399142b4731b1a1d10930709264189868d007bcee9ac.json @@ -1,24 +1,24 @@ { - "writtenAt": "2026-05-20T23:48:01.358Z", + "writtenAt": "2026-07-23T22:55:00.000Z", "toolName": "webFetch", "input": { "url": "https://docs.slack.dev/reference/methods/chat.startStream/", - "max_chars": 6000 + "max_chars": 12000 }, "output": { + "ok": true, + "status": "success", + "truncated": true, "url": "https://docs.slack.dev/reference/methods/chat.startStream/", - "content": "On this page\n\nDocsCall generator\n\n## Facts[​](#facts \"Direct link to Facts\")\n\n**Description**Starts a new streaming conversation.\n\n**Method Access**\n\n- HTTP\n- Slack CLI\n- JavaScript\n- Python\n\n```\nPOST https://slack.com/api/chat.startStream\n\n```\n\n[![slack-cli](/img/logos/slack-cli.png)](/tools/slack-cli)\n\n```\nslack api chat.startStream\n\n```\n\n[![bolt-js](/img/logos/bolt-js-logo.svg)](/tools/bolt-js)\n\n```\napp.client.chat.startStream\n\n```\n\n[![bolt-py](/img/logos/bolt-py-logo.svg)](/tools/bolt-python)\n\n```\napp.client.chat_startStream\n\n```\n\n**Scopes**\n\nBot token:\n\n[chat:write](/reference/scopes/chat.write)\n\n**Content types**\n\n`application/x-www-form-urlencoded` \n\n`application/json` \n\n**Rate Limits**[Tier 2: 20+ per minute](/apis/web-api/rate-limits)\n\n## Arguments[​](#arguments \"Direct link to Arguments\")\n\n### Required arguments\n\n**`token`**`string`Required\n\n_Example:_ `xxxx-xxxxxxxxx-xxxx`\n\n**`channel`**`string`Required\n\nAn encoded ID that represents a channel thread or DM.\n\n**`thread_ts`**`string`Required\n\nProvide another message's `ts` value to reply to. Streamed messages should always be replies to a user request.\n\n_Example:_ `1721609600.123456`\n\n### Optional arguments\n\n**`chunks`**`array`Optional\n\nArray of streaming chunks.\n\n**`markdown_text`**`string`Optional\n\nAccepts message text formatted in markdown. Limit this field to 12,000 characters.\n\n_Example:_ `**This is bold text**`\n\n**`recipient_user_id`**Optional\n\nThe encoded ID of the user to receive the streaming text. Required when streaming to channels.\n\n**`recipient_team_id`**`string`Optional\n\nThe encoded ID of the team the user receiving the streaming text belongs to. Required when streaming to channels.\n\n_Example:_ `T0123456789`\n\n**`task_display_mode`**`string`Optional\n\nSpecifies how tasks are displayed in the message. A `timeline` displays individual tasks with text in sequential order, `plan` displays all tasks together, with the first tasks's placement determining the placement of the rest of the tasks, and `dense` collapses consecutive tool calls into a single summarized task card.\n\n_Default:_ `timeline`\n\n_Acceptable values:_ `timeline` `plan` `dense`\n\n_Example:_ `plan`\n\n**`icon_emoji`**`string`Optional\n\nEmoji to use as the icon for this message. Overrides `icon_url`.\n\n_Example:_ `:chart_with_upwards_trend:`\n\n**`icon_url`**`string`Optional\n\nImage URL to use as the icon for this message.\n\n_Example:_ `http://lorempixel.com/48/48`\n\n**`username`**`string`Optional\n\nThe bot's username to display.\n\n_Example:_ `My Bot`\n\n## Usage info[​](#usage-info \"Direct link to Usage info\")\n\nUse the `chat.startStream` method to start a streaming conversation. Append to the stream with the [chat.appendStream](/reference/methods/chat.appendStream) method, and stop the stream with the [chat.stopStream](/reference/methods/chat.stopStream) method.\n\nThe Python Slack SDK and Node Slack SDK both provide a helper utility for the `chat.*Stream` methods that are surfaced in [Bolt for Python](/tools/bolt-python/concepts/message-sending) and [Bolt for JavaScript](/tools/bolt-js/concepts/message-sending).\n\n### Using the `chunks` parameter[​](#chunks \"Direct link to chunks\")\n\nThe `chunks` parameter can include [markdown text chunk](#markdown%5Ftext-chunks) objects, [task update chunk](#task%5Fupdate-chunks) objects, [plan update chunks](#plan%5Fupdate-chunks), or [blocks chunks](#blocks-chunks).\n\n#### `markdown_text` chunks[​](#markdown%5Ftext-chunks \"Direct link to markdown_text-chunks\")\n\nThe `markdown_text` chunk is used for streaming text content with markdown formatting support.\n\n```\n{\n \"type\": \"markdown_text\",\n \"text\": \"We love Sandra\"\n}\n\n```\n\n#### `task_update` chunks[​](#task%5Fupdate-chunks \"Direct link to task_update-chunks\")\n\nThe `task_update` chunk object looks mighty similar to the [task card block](/reference/block-kit/blocks/task-card-block)!\n\nThe `task_update` chunk is used for displaying task progress in a timeline-style UI.\n\n```\n{\n \"type\": \"task_update\",\n \"id\": \"unique_task_id\",\n \"title\": \"Remind Sandra how amazing she is\",\n \"status\": \"pending\" | \"in_progress\" | \"complete\" | \"error\",\n \"details\": \"wow such good details\",\n \"output\": \"amazing output here\",\n \"sources\": [\n {\n \"type\": \"url\",\n \"text\": \"Example.com\",\n \"url\": \"https://example.com\"\n }\n ]\n}\n\n```\n\n#### `plan_update` chunks[​](#plan%5Fupdate-chunks \"Direct link to plan_update-chunks\")\n\nThe `plan_update` chunk is used for updating the title of a plan.\n\n```\n{\n \"type\": \"plan_update\",\n \"title\": \"Sandra's new and improved plan\"\n}\n\n```\n\nThe character limit for chunk sizes for `task_update` and `plan_update` is 256 characters.\n\n#### `blocks` chunks[​](#blocks-chunks \"Direct link to blocks-chunks\")\n\nThe `blocks` chunk is used for passing an array of blocks within a message.\n\n```\n{\n \"type\": \"blocks\",\n \"blocks\": [\n { \n \"type\": \"section\", \n \"text\": {\n \"type\": \"plain_text\", \n \"text\": \"Sandra's plan outline\"\n }\n }\n ]\n}\n\n```\n\nAt most 50 blocks can be sent in one `blocks` array. If more than 50 blocks are sent via `chunks`, any blocks over the limit will be dropped and a warning will be emitted via the API.\n\n---\n\n## Response[​](#response \"Direct link to Response\")\n\nTypical success response when starting a streaming message\n\n```\n{\n \"ok\": true,\n \"channel\": \"C123ABC456\",\n \"ts\": \"1503435956.000247\"\n}\n\n```\n\nTypical error response\n\n```\n{\n \"ok\": false,\n \"error\": \"invalid_auth\"\n}\n\n```\n\n## Errors[​](#errors \"Direct link to Errors\")\n\nThis table lists the expected errors that this method could return. However, other errors can be returned in the case where the service is down or other unexpected factors affect processing. Callers should always check the value of the `ok` parameter in the response.\n\nError\n\nDescription\n\n`access_denied`\n\nAccess to a resource specified in the request is denied.\n\n`accesslimited`\n\nAccess to this method is limited on the current network\n\n`account_inactive`\n\nAuthentication token is for a deleted user or...", + "content": "On this page\n\nDocsCall generator\n\n## Facts[​](#facts \"Direct link to Facts\")\n\n**Description**Starts a new streaming conversation.\n\n**Method Access**\n\n- HTTP\n- Slack CLI\n- JavaScript\n- Python\n\n```\nPOST https://slack.com/api/chat.startStream\n\n```\n\n[![slack-cli](/img/logos/slack-cli.png)](/tools/slack-cli)\n\n```\nslack api chat.startStream\n\n```\n\n[![bolt-js](/img/logos/bolt-js-logo.svg)](/tools/bolt-js)\n\n```\napp.client.chat.startStream\n\n```\n\n[![bolt-py](/img/logos/bolt-py-logo.svg)](/tools/bolt-python)\n\n```\napp.client.chat_startStream\n\n```\n\n**Scopes**\n\nBot token:\n\n[chat:write](/reference/scopes/chat.write)\n\n**Content types**\n\n`application/x-www-form-urlencoded` \n\n`application/json` \n\n**Rate Limits**[Tier 2: 20+ per minute](/apis/web-api/rate-limits)\n\n## Arguments[​](#arguments \"Direct link to Arguments\")\n\n### Required arguments\n\n**`token`**`string`Required\n\n_Example:_ `xxxx-xxxxxxxxx-xxxx`\n\n**`channel`**`string`Required\n\nAn encoded ID that represents a channel thread or DM.\n\n**`thread_ts`**`string`Required\n\nProvide another message's `ts` value to reply to. Streamed messages should always be replies to a user request.\n\n_Example:_ `1721609600.123456`\n\n### Optional arguments\n\n**`chunks`**`array`Optional\n\nArray of streaming chunks.\n\n**`markdown_text`**`string`Optional\n\nAccepts message text formatted in markdown. Limit this field to 12,000 characters.\n\n_Example:_ `**This is bold text**`\n\n**`recipient_user_id`**Optional\n\nThe encoded ID of the user to receive the streaming text. Required when streaming to channels.\n\n**`recipient_team_id`**`string`Optional\n\nThe encoded ID of the team the user receiving the streaming text belongs to. Required when streaming to channels.\n\n_Example:_ `T0123456789`\n\n**`task_display_mode`**`string`Optional\n\nSpecifies how tasks are displayed in the message. A `timeline` displays individual tasks with text in sequential order, `plan` displays all tasks together, with the first tasks's placement determining the placement of the rest of the tasks, and `dense` collapses consecutive tool calls into a single summarized task card.\n\n_Default:_ `timeline`\n\n_Acceptable values:_ `timeline` `plan` `dense`\n\n_Example:_ `plan`\n\n**`icon_emoji`**`string`Optional\n\nEmoji to use as the icon for this message. Overrides `icon_url`.\n\n_Example:_ `:chart_with_upwards_trend:`\n\n**`icon_url`**`string`Optional\n\nImage URL to use as the icon for this message.\n\n_Example:_ `http://lorempixel.com/48/48`\n\n**`username`**`string`Optional\n\nThe bot's username to display.\n\n_Example:_ `My Bot`\n\n## Usage info[​](#usage-info \"Direct link to Usage info\")\n\nUse the `chat.startStream` method to start a streaming conversation. Append to the stream with the [chat.appendStream](/reference/methods/chat.appendStream) method, and stop the stream with the [chat.stopStream](/reference/methods/chat.stopStream) method.\n\nThe Python Slack SDK and Node Slack SDK both provide a helper utility for the `chat.*Stream` methods that are surfaced in [Bolt for Python](/tools/bolt-python/concepts/message-sending) and [Bolt for JavaScript](/tools/bolt-js/concepts/message-sending).\n\n### Using the `chunks` parameter[​](#chunks \"Direct link to chunks\")\n\nThe `chunks` parameter can include [markdown text chunk](#markdown%5Ftext-chunks) objects, [task update chunk](#task%5Fupdate-chunks) objects, [plan update chunks](#plan%5Fupdate-chunks), or [blocks chunks](#blocks-chunks).\n\n#### `markdown_text` chunks[​](#markdown%5Ftext-chunks \"Direct link to markdown_text-chunks\")\n\nThe `markdown_text` chunk is used for streaming text content with markdown formatting support.\n\n```\n{\n \"type\": \"markdown_text\",\n \"text\": \"We love Sandra\"\n}\n\n```\n\n#### `task_update` chunks[​](#task%5Fupdate-chunks \"Direct link to task_update-chunks\")\n\nThe `task_update` chunk object looks mighty similar to the [task card block](/reference/block-kit/blocks/task-card-block)!\n\nThe `task_update` chunk is used for displaying task progress in a timeline-style UI.\n\n```\n{\n \"type\": \"task_update\",\n \"id\": \"unique_task_id\",\n \"title\": \"Remind Sandra how amazing she is\",\n \"status\": \"pending\" | \"in_progress\" | \"complete\" | \"error\",\n \"details\": \"wow such good details\",\n \"output\": \"amazing output here\",\n \"sources\": [\n {\n \"type\": \"url\",\n \"text\": \"Example.com\",\n \"url\": \"https://example.com\"\n }\n ]\n}\n\n```\n\n#### `plan_update` chunks[​](#plan%5Fupdate-chunks \"Direct link to plan_update-chunks\")\n\nThe `plan_update` chunk is used for updating the title of a plan.\n\n```\n{\n \"type\": \"plan_update\",\n \"title\": \"Sandra's new and improved plan\"\n}\n\n```\n\nThe character limit for chunk sizes for `task_update` and `plan_update` is 256 characters.\n\n#### `blocks` chunks[​](#blocks-chunks \"Direct link to blocks-chunks\")\n\nThe `blocks` chunk is used for passing an array of blocks within a message.\n\n```\n{\n \"type\": \"blocks\",\n \"blocks\": [\n { \n \"type\": \"section\", \n \"text\": {\n \"type\": \"plain_text\", \n \"text\": \"Sandra's plan outline\"\n }\n }\n ]\n}\n\n```\n\nAt most 50 blocks can be sent in one `blocks` array. If more than 50 blocks are sent via `chunks`, any blocks over the limit will be dropped and a warning will be emitted via the API.\n\n---\n\n## Response[​](#response \"Direct link to Response\")\n\nTypical success response when starting a streaming message\n\n```\n{\n \"ok\": true,\n \"channel\": \"C123ABC456\",\n \"ts\": \"1503435956.000247\"\n}\n\n```\n\nTypical error response\n\n```\n{\n \"ok\": false,\n \"error\": \"invalid_auth\"\n}\n\n```\n\n## Errors[​](#errors \"Direct link to Errors\")\n\nThis table lists the expected errors that this method could return. However, other errors can be returned in the case where the service is down or other unexpected factors affect processing. Callers should always check the value of the `ok` parameter in the response.\n\nError\n\nDescription\n\n`access_denied`\n\nAccess to a resource specified in the request is denied.\n\n`accesslimited`\n\nAccess to this method is limited on the current network\n\n`account_inactive`\n\nAuthentication token is for a deleted user or workspace when using a `bot` token.\n\n`channel_not_found`\n\nValue passed for `channel` was invalid.\n\n`channel_type_not_supported`\n\nChannel type not supported\n\n`deprecated_endpoint`\n\nThe endpoint has been deprecated.\n\n`duplicate_channel_not_found`\n\nChannel associated with `client_msg_id` was invalid.\n\n`duplicate_message_not_found`\n\nNo duplicate message exists associated with `client_msg_id`.\n\n`ekm_access_denied`\n\nYour message couldn’t be sent because your admins have disabled sending messages to this channel.\n\n`ekm_access_denied`\n\nAdministrators have suspended the ability to post a message.\n\n`enterprise_is_restricted`\n\nThe method cannot be called from an Enterprise.\n\n`fatal_error`\n\nThe server could not complete your operation(s) without encountering a catastrophic error. It's possible some aspect of the operation succeeded before the error was raised.\n\n`internal_error`\n\nThe server could not complete your operation(s) without encountering an error, likely due to a transient issue on our end. It's possible some aspect of the operation succeeded before the error was raised.\n\n`invalid_arg_name`\n\nThe method was passed an argument whose name falls outside the bounds of accepted or expected values. This includes very long names and names with non-alphanumeric characters other than `_`. If you get this error, it is typically an indication that you have made a _very_ malformed API call.\n\n`invalid_arguments`\n\nThe method was called with invalid arguments.\n\n`invalid_array_arg`\n\nThe method was passed an array as an argument. Please only input valid strings.\n\n`invalid_auth`\n\nSome aspect of authentication cannot be validated. Either the provided token is invalid or the request originates from an IP address disallowed from making the request.\n\n`invalid_blocks`\n\nBlocks submitted with this message are not valid\n\n`invalid_blocks_format`\n\nThe `blocks` is not a valid JSON object or doesn't match the Block Kit syntax.\n\n`invalid_charset`\n\nThe method was called via a `POST` request, but the `charset` specified in the `Content-Type` header was invalid. Valid charset names are: `utf-8` `iso-8859-1`.\n\n`invalid_chunks`\n\nChunks submitted with this message are not valid.\n\n`invalid_form_data`\n\nThe method was called via a `POST` request with `Content-Type` `application/x-www-form-urlencoded` or `multipart/form-data`, but the form data was either missing or syntactically invalid.\n\n`invalid_metadata_format`\n\nInvalid metadata format provided\n\n`invalid_metadata_schema`\n\nInvalid metadata schema provided\n\n`invalid_post_type`\n\nThe method was called via a `POST` request, but the specified `Content-Type` was invalid. Valid types are: `application/json` `application/x-www-form-urlencoded` `multipart/form-data` `text/plain`.\n\n`is_archived`\n\nChannel has been archived.\n\n`message_limit_exceeded`\n\nMembers on this team are sending too many messages. For more details, see https://slack.com/help/articles/115002422943-Usage-limits-for-free-workspaces\n\n`messages_tab_disabled`\n\nMessages tab for the app is disabled.\n\n`messaging_processing_failed`\n\nFailed to process the message.\n\n`metadata_must_be_sent_from_app`\n\nMessage metadata can only be posted or updated using an app-level token\n\n`metadata_too_large`\n\nMetadata exceeds size limit\n\n`method_deprecated`\n\nThe method has been deprecated.\n\n`missing_post_type`\n\nThe method was called via a `POST` request and included a data payload, but the request did not include a `Content-Type` header.\n\n`missing_recipient_team_id`\n\nMissing recipient team ID.\n\n`missing_recipient_user_id`\n\nMissing recipient user ID.\n\n`missing_scope`\n\nThe token used is not granted the specific scope permissions required to complete this request.\n\n`msg_blocks_too_long`\n\nBlocks submitted with this message are too long.\n\n`msg_blocks_too_many`\n\nMax blocks exceeded, blocks are limited to 50 per message.\n\n`no_permission`\n\nThe workspace token used in this request does not have the permissions necessary to complete the request. Make sure your app is a member of the conversation it's attempting to post a message to.\n\n`no_text`\n\nNo message text provided\n\n`not_allowed_token_type`\n\nThe token type used in this request is not allowed.\n\n`not_authed`\n\nNo authentication token provided.\n\n`not_in_channel`\n\nCannot post user messages to a channel they are not in.\n\n`not_subscribed_to_message_stream_stopped`\n\nThe app must be subscribed to the `message_stream_stopped` event to use `is_stoppable`.\n\n`org_login_required`\n\nThe workspace is undergoing an enterprise migration and will not be available until migration is complete.\n\n`rate_limited`\n\nApplication has posted too many messages, [read the Rate Limit documentation](/apis/web-api/rate-limits) for more information\n\n`ratelimited`\n\nThe request has been ratelimited. Refer to the `Retry-After` header for when to retry the request.\n\n`request_timeout`\n\nThe method was called via a `POST` request, but the `POST` data was either missing or truncated.\n\n`restricted_action`\n\nA workspace preference prevents the authenticated user from posting.\n\n`restricted_action_non_threadable_channel`\n\nCannot post thread replies into a non\\_threadable channel.\n\n`restricted_action_read_only_channel`\n\nCannot post any message into a read-only channel.\n\n`restricted_action_thread_locked`\n\nCannot post replies to a thread that has been locked by admins.\n\n`restricted_action_thread_only_channel`\n\nCannot post top-level messages into a thread-only channel.\n\n`service_unavailable`\n\nThe service is temporarily unavailable\n\n`slack_connect_canvas_sharing_blocked`\n\nAdmin has disabled Canvas File sharing in all Slack Connect communications\n\n`slack_connect_file_link_sharing_blocked`\n\nAdmin has disabled Slack File sharing in all Slack Connect communications\n\n`slack_connect_lists_sharing_blocked`\n\nAdmin has disabled Lists sharing in all Slack Connect communications\n\n`team_access_not_granted`\n\nThe token used is not granted the specific workspace access required to complete this request.\n\n`team_added_to_org`\n\nThe workspace associated with your request is currently...", "title": "chat.startStream method | Slack Developer Docs", "content_type": "text/html; charset=utf-8", - "source_bytes": 90308, - "extracted_chars": 12606, - "truncated": true, - "status": "success", - "ok": true + "source_bytes": 90548, + "extracted_chars": 12739 }, "metadata": { - "cacheKey": "1e892c4d6991f468213d21aac1e9a40688ea15889e99c2fa133a97455b543aa6", - "version": "web-fetch-v1", - "mode": "auto" + "cacheKey": "a73ba09d7c341063e03a399142b4731b1a1d10930709264189868d007bcee9ac", + "version": "web-fetch-v2", + "mode": "record" } } diff --git a/packages/junior-evals/.vitest-evals/recordings/webFetch/7795b1275bd8c551f2a5d9ba0aa2cd5182277af60342e9322e55a8a104464085.json b/packages/junior-evals/.vitest-evals/recordings/webFetch/cb2a12c9a09977dfc53d741d349e94ee98eac65189cdcaa027b2fa0cf213b5eb.json similarity index 51% rename from packages/junior-evals/.vitest-evals/recordings/webFetch/7795b1275bd8c551f2a5d9ba0aa2cd5182277af60342e9322e55a8a104464085.json rename to packages/junior-evals/.vitest-evals/recordings/webFetch/cb2a12c9a09977dfc53d741d349e94ee98eac65189cdcaa027b2fa0cf213b5eb.json index 28d523c2c..2c1414f84 100644 --- a/packages/junior-evals/.vitest-evals/recordings/webFetch/7795b1275bd8c551f2a5d9ba0aa2cd5182277af60342e9322e55a8a104464085.json +++ b/packages/junior-evals/.vitest-evals/recordings/webFetch/cb2a12c9a09977dfc53d741d349e94ee98eac65189cdcaa027b2fa0cf213b5eb.json @@ -1,24 +1,24 @@ { - "writtenAt": "2026-05-20T23:48:01.368Z", + "writtenAt": "2026-07-23T22:55:00.000Z", "toolName": "webFetch", "input": { "url": "https://docs.slack.dev/reference/methods/chat.stopStream/", - "max_chars": 6000 + "max_chars": 12000 }, "output": { + "ok": true, + "status": "success", + "truncated": true, "url": "https://docs.slack.dev/reference/methods/chat.stopStream/", - "content": "On this page\n\nDocsCall generator\n\n## Facts[​](#facts \"Direct link to Facts\")\n\n**Description**Stops a streaming conversation.\n\n**Method Access**\n\n- HTTP\n- Slack CLI\n- JavaScript\n- Python\n\n```\nPOST https://slack.com/api/chat.stopStream\n\n```\n\n[![slack-cli](/img/logos/slack-cli.png)](/tools/slack-cli)\n\n```\nslack api chat.stopStream\n\n```\n\n[![bolt-js](/img/logos/bolt-js-logo.svg)](/tools/bolt-js)\n\n```\napp.client.chat.stopStream\n\n```\n\n[![bolt-py](/img/logos/bolt-py-logo.svg)](/tools/bolt-python)\n\n```\napp.client.chat_stopStream\n\n```\n\n**Scopes**\n\nBot token:\n\n[chat:write](/reference/scopes/chat.write)\n\n**Content types**\n\n`application/x-www-form-urlencoded` \n\n`application/json` \n\n**Rate Limits**[Tier 2: 20+ per minute](/apis/web-api/rate-limits)\n\n## Arguments[​](#arguments \"Direct link to Arguments\")\n\n### Required arguments\n\n**`token`**`string`Required\n\n_Example:_ `xxxx-xxxxxxxxx-xxxx`\n\n**`channel`**`string`Required\n\nAn encoded ID that represents a channel, private group, or DM\n\n**`ts`**Required\n\nThe timestamp of the streaming message.\n\n### Optional arguments\n\n**`chunks`**`array`Optional\n\nArray of streaming chunks.\n\n**`markdown_text`**`string`Optional\n\nAccepts message text formatted in markdown. Limit this field to 12,000 characters.\n\n_Example:_ `**This is bold text**`\n\n**`blocks`**Optional\n\nA list of blocks that will be rendered at the bottom of the finalized message.\n\n**`metadata`**Optional\n\nJSON object with event\\_type and event\\_payload fields, presented as a URL-encoded string. Metadata you post to Slack is accessible to any app or user who is a member of that workspace.\n\n_Example:_ `{\"event_type\": \"task_created\", \"event_payload\": { \"id\": \"11223\", \"title\": \"Redesign Homepage\"}}`\n\n## Usage info[​](#usage-info \"Direct link to Usage info\")\n\nUse the `chat.stopStream` method to end a streaming conversation initiated with the [chat.startStream](/reference/methods/chat.startStream) method.\n\nThe Python Slack SDK and Node Slack SDK both provide a helper utility for the `chat.*Stream` methods that are surfaced in [Bolt for Python](/tools/bolt-python/concepts/message-sending) and [Bolt for JavaScript](/tools/bolt-js/concepts/message-sending).\n\n### Using the `chunks` parameter[​](#chunks \"Direct link to chunks\")\n\nThe `chunks` parameter can include [markdown text chunk](#markdown%5Ftext-chunks) objects, [task update chunk](#task%5Fupdate-chunks) objects, [plan update chunks](#plan%5Fupdate-chunks), or [blocks chunks](#blocks-chunks).\n\n#### `markdown_text` chunks[​](#markdown%5Ftext-chunks \"Direct link to markdown_text-chunks\")\n\nThe `markdown_text` chunk is used for streaming text content with markdown formatting support.\n\n```\n{\n \"type\": \"markdown_text\",\n \"text\": \"We love Sandra\"\n}\n\n```\n\n#### `task_update` chunks[​](#task%5Fupdate-chunks \"Direct link to task_update-chunks\")\n\nThe `task_update` chunk object looks mighty similar to the [task card block](/reference/block-kit/blocks/task-card-block)!\n\nThe `task_update` chunk is used for displaying task progress in a timeline-style UI.\n\n```\n{\n \"type\": \"task_update\",\n \"id\": \"unique_task_id\",\n \"title\": \"Remind Sandra how amazing she is\",\n \"status\": \"pending\" | \"in_progress\" | \"complete\" | \"error\",\n \"details\": \"wow such good details\",\n \"output\": \"amazing output here\",\n \"sources\": [\n {\n \"type\": \"url\",\n \"text\": \"Example.com\",\n \"url\": \"https://example.com\"\n }\n ]\n}\n\n```\n\n#### `plan_update` chunks[​](#plan%5Fupdate-chunks \"Direct link to plan_update-chunks\")\n\nThe `plan_update` chunk is used for updating the title of a plan.\n\n```\n{\n \"type\": \"plan_update\",\n \"title\": \"Sandra's new and improved plan\"\n}\n\n```\n\n#### `blocks` chunks[​](#blocks-chunks \"Direct link to blocks-chunks\")\n\nThe `blocks` chunk is used for passing an array of blocks within a message.\n\n```\n{\n \"type\": \"blocks\",\n \"blocks\": [\n { \n \"type\": \"section\", \n \"text\": {\n \"type\": \"plain_text\", \n \"text\": \"Sandra's plan outline\"\n }\n }\n ]\n}\n\n```\n\nThe `blocks` array has a limit of 50, which is a separate limit from the `blocks` sent via the `chunks` parameter. Therefore you can have 50 `blocks` sent via the `chunks` parameter and 50 `blocks` sent via the `chat.stopStream` API method's `blocks` array, for 100 blocks total. Note that any blocks in the `blocks` array will be rendered after anything passed via `chunks` or `markdown_text` when the stream is completed.\n\n---\n\n## Response[​](#response \"Direct link to Response\")\n\nTypical success response when stopping a streaming message\n\n```\n{\n \"ok\": true,\n \"channel\": \"C123ABC456\",\n \"ts\": \"1503435956.000247\",\n \"message\": {\n \"text\": \"Here's the final streamed message content\",\n \"bot_id\": \"B123ABC456\",\n \"ts\": \"1503435956.000247\",\n \"type\": \"message\",\n \"subtype\": \"bot_message\"\n }\n}\n\n```\n\nTypical error response\n\n```\n{\n \"ok\": false,\n \"error\": \"invalid_auth\"\n}\n\n```\n\n## Errors[​](#errors \"Direct link to Errors\")\n\nThis table lists the expected errors that this method could return. However, other errors can be returned in the case where the service is down or other unexpected factors affect processing. Callers should always check the value of the `ok` parameter in the response.\n\nError\n\nDescription\n\n`access_denied`\n\nAccess to a resource specified in the request is denied.\n\n`accesslimited`\n\nAccess to this method is limited on the current network\n\n`account_inactive`\n\nAuthentication token is for a deleted user or workspace when using a `bot` token.\n\n`as_user_not_supported`\n\nThe `as_user` parameter does not function with workspace apps.\n\n`attachment_payload_limit_exceeded`\n\nAttachment payload size is too long.\n\n`cannot_reply_to_message`\n\nThis message type cannot have thread replies.\n\n`channel_not_found`\n\nValue passed for `channel` was invalid.\n\n`channel_type_not_supported`\n\nChannel type not supported.\n\n`deprecated_endpoint`\n\nThe endpoint has been deprecated.\n\n`duplicate_channel_not_found`\n\nChannel associated with `client_msg_id` was invalid.\n\n`duplicate_message_not_found`\n\nNo duplicate message...", + "content": "On this page\n\nDocsCall generator\n\n## Facts[​](#facts \"Direct link to Facts\")\n\n**Description**Stops a streaming conversation.\n\n**Method Access**\n\n- HTTP\n- Slack CLI\n- JavaScript\n- Python\n\n```\nPOST https://slack.com/api/chat.stopStream\n\n```\n\n[![slack-cli](/img/logos/slack-cli.png)](/tools/slack-cli)\n\n```\nslack api chat.stopStream\n\n```\n\n[![bolt-js](/img/logos/bolt-js-logo.svg)](/tools/bolt-js)\n\n```\napp.client.chat.stopStream\n\n```\n\n[![bolt-py](/img/logos/bolt-py-logo.svg)](/tools/bolt-python)\n\n```\napp.client.chat_stopStream\n\n```\n\n**Scopes**\n\nBot token:\n\n[chat:write](/reference/scopes/chat.write)\n\n**Content types**\n\n`application/x-www-form-urlencoded` \n\n`application/json` \n\n**Rate Limits**[Tier 2: 20+ per minute](/apis/web-api/rate-limits)\n\n## Arguments[​](#arguments \"Direct link to Arguments\")\n\n### Required arguments\n\n**`token`**`string`Required\n\n_Example:_ `xxxx-xxxxxxxxx-xxxx`\n\n**`channel`**`string`Required\n\nAn encoded ID that represents a channel, private group, or DM\n\n**`ts`**Required\n\nThe timestamp of the streaming message.\n\n### Optional arguments\n\n**`chunks`**`array`Optional\n\nArray of streaming chunks.\n\n**`markdown_text`**`string`Optional\n\nAccepts message text formatted in markdown. Limit this field to 12,000 characters.\n\n_Example:_ `**This is bold text**`\n\n**`blocks`**Optional\n\nA list of blocks that will be rendered at the bottom of the finalized message.\n\n**`metadata`**Optional\n\nJSON object with event\\_type and event\\_payload fields, presented as a URL-encoded string. Metadata you post to Slack is accessible to any app or user who is a member of that workspace.\n\n_Example:_ `{\"event_type\": \"task_created\", \"event_payload\": { \"id\": \"11223\", \"title\": \"Redesign Homepage\"}}`\n\n## Usage info[​](#usage-info \"Direct link to Usage info\")\n\nUse the `chat.stopStream` method to end a streaming conversation initiated with the [chat.startStream](/reference/methods/chat.startStream) method.\n\nThe Python Slack SDK and Node Slack SDK both provide a helper utility for the `chat.*Stream` methods that are surfaced in [Bolt for Python](/tools/bolt-python/concepts/message-sending) and [Bolt for JavaScript](/tools/bolt-js/concepts/message-sending).\n\n### Using the `chunks` parameter[​](#chunks \"Direct link to chunks\")\n\nThe `chunks` parameter can include [markdown text chunk](#markdown%5Ftext-chunks) objects, [task update chunk](#task%5Fupdate-chunks) objects, [plan update chunks](#plan%5Fupdate-chunks), or [blocks chunks](#blocks-chunks).\n\n#### `markdown_text` chunks[​](#markdown%5Ftext-chunks \"Direct link to markdown_text-chunks\")\n\nThe `markdown_text` chunk is used for streaming text content with markdown formatting support.\n\n```\n{\n \"type\": \"markdown_text\",\n \"text\": \"We love Sandra\"\n}\n\n```\n\n#### `task_update` chunks[​](#task%5Fupdate-chunks \"Direct link to task_update-chunks\")\n\nThe `task_update` chunk object looks mighty similar to the [task card block](/reference/block-kit/blocks/task-card-block)!\n\nThe `task_update` chunk is used for displaying task progress in a timeline-style UI.\n\n```\n{\n \"type\": \"task_update\",\n \"id\": \"unique_task_id\",\n \"title\": \"Remind Sandra how amazing she is\",\n \"status\": \"pending\" | \"in_progress\" | \"complete\" | \"error\",\n \"details\": \"wow such good details\",\n \"output\": \"amazing output here\",\n \"sources\": [\n {\n \"type\": \"url\",\n \"text\": \"Example.com\",\n \"url\": \"https://example.com\"\n }\n ]\n}\n\n```\n\n#### `plan_update` chunks[​](#plan%5Fupdate-chunks \"Direct link to plan_update-chunks\")\n\nThe `plan_update` chunk is used for updating the title of a plan.\n\n```\n{\n \"type\": \"plan_update\",\n \"title\": \"Sandra's new and improved plan\"\n}\n\n```\n\n#### `blocks` chunks[​](#blocks-chunks \"Direct link to blocks-chunks\")\n\nThe `blocks` chunk is used for passing an array of blocks within a message.\n\n```\n{\n \"type\": \"blocks\",\n \"blocks\": [\n { \n \"type\": \"section\", \n \"text\": {\n \"type\": \"plain_text\", \n \"text\": \"Sandra's plan outline\"\n }\n }\n ]\n}\n\n```\n\nThe `blocks` array has a limit of 50, which is a separate limit from the `blocks` sent via the `chunks` parameter. Therefore you can have 50 `blocks` sent via the `chunks` parameter and 50 `blocks` sent via the `chat.stopStream` API method's `blocks` array, for 100 blocks total. Note that any blocks in the `blocks` array will be rendered after anything passed via `chunks` or `markdown_text` when the stream is completed.\n\n---\n\n## Response[​](#response \"Direct link to Response\")\n\nTypical success response when stopping a streaming message\n\n```\n{\n \"ok\": true,\n \"channel\": \"C123ABC456\",\n \"ts\": \"1503435956.000247\",\n \"message\": {\n \"text\": \"Here's the final streamed message content\",\n \"bot_id\": \"B123ABC456\",\n \"ts\": \"1503435956.000247\",\n \"type\": \"message\",\n \"subtype\": \"bot_message\"\n }\n}\n\n```\n\nTypical error response\n\n```\n{\n \"ok\": false,\n \"error\": \"invalid_auth\"\n}\n\n```\n\n## Errors[​](#errors \"Direct link to Errors\")\n\nThis table lists the expected errors that this method could return. However, other errors can be returned in the case where the service is down or other unexpected factors affect processing. Callers should always check the value of the `ok` parameter in the response.\n\nError\n\nDescription\n\n`access_denied`\n\nAccess to a resource specified in the request is denied.\n\n`accesslimited`\n\nAccess to this method is limited on the current network\n\n`account_inactive`\n\nAuthentication token is for a deleted user or workspace when using a `bot` token.\n\n`as_user_not_supported`\n\nThe `as_user` parameter does not function with workspace apps.\n\n`attachment_payload_limit_exceeded`\n\nAttachment payload size is too long.\n\n`cannot_reply_to_message`\n\nThis message type cannot have thread replies.\n\n`channel_not_found`\n\nValue passed for `channel` was invalid.\n\n`channel_type_not_supported`\n\nChannel type not supported.\n\n`deprecated_endpoint`\n\nThe endpoint has been deprecated.\n\n`duplicate_channel_not_found`\n\nChannel associated with `client_msg_id` was invalid.\n\n`duplicate_message_not_found`\n\nNo duplicate message exists associated with `client_msg_id`.\n\n`ekm_access_denied`\n\nYour message couldn’t be sent because your admins have disabled sending messages to this channel.\n\n`ekm_access_denied`\n\nAdministrators have suspended the ability to post a message.\n\n`enterprise_is_restricted`\n\nThe method cannot be called from an Enterprise.\n\n`fatal_error`\n\nThe server could not complete your operation(s) without encountering a catastrophic error. It's possible some aspect of the operation succeeded before the error was raised.\n\n`internal_error`\n\nThe server could not complete your operation(s) without encountering an error, likely due to a transient issue on our end. It's possible some aspect of the operation succeeded before the error was raised.\n\n`invalid_arg_name`\n\nThe method was passed an argument whose name falls outside the bounds of accepted or expected values. This includes very long names and names with non-alphanumeric characters other than `_`. If you get this error, it is typically an indication that you have made a _very_ malformed API call.\n\n`invalid_arguments`\n\nThe method was called with invalid arguments.\n\n`invalid_array_arg`\n\nThe method was passed an array as an argument. Please only input valid strings.\n\n`invalid_auth`\n\nSome aspect of authentication cannot be validated. Either the provided token is invalid or the request originates from an IP address disallowed from making the request.\n\n`invalid_blocks`\n\nBlocks submitted with this message are not valid.\n\n`invalid_charset`\n\nThe method was called via a `POST` request, but the `charset` specified in the `Content-Type` header was invalid. Valid charset names are: `utf-8` `iso-8859-1`.\n\n`invalid_chunks`\n\nChunks submitted with this message are not valid.\n\n`invalid_form_data`\n\nThe method was called via a `POST` request with `Content-Type` `application/x-www-form-urlencoded` or `multipart/form-data`, but the form data was either missing or syntactically invalid.\n\n`invalid_metadata_format`\n\nInvalid metadata format provided\n\n`invalid_metadata_schema`\n\nInvalid metadata schema provided\n\n`invalid_post_type`\n\nThe method was called via a `POST` request, but the specified `Content-Type` was invalid. Valid types are: `application/json` `application/x-www-form-urlencoded` `multipart/form-data` `text/plain`.\n\n`is_archived`\n\nChannel has been archived.\n\n`message_limit_exceeded`\n\nMembers on this team are sending too many messages. For more details, see https://slack.com/help/articles/115002422943-Usage-limits-for-free-workspaces\n\n`message_not_found`\n\nThe message was not found.\n\n`message_not_in_streaming_state`\n\nThe message is not in a streaming state.\n\n`message_not_owned_by_app`\n\nThe message is not owned by the app.\n\n`messages_tab_disabled`\n\nMessages tab for the app is disabled.\n\n`metadata_must_be_sent_from_app`\n\nMessage metadata can only be posted or updated using an app-level token\n\n`metadata_too_large`\n\nMetadata exceeds size limit\n\n`method_deprecated`\n\nThe method has been deprecated.\n\n`missing_file_data`\n\nAttempted to share a file but some required data was missing.\n\n`missing_post_type`\n\nThe method was called via a `POST` request and included a data payload, but the request did not include a `Content-Type` header.\n\n`missing_scope`\n\nThe token used is not granted the specific scope permissions required to complete this request.\n\n`msg_blocks_too_long`\n\nBlocks submitted with this message are too long.\n\n`msg_blocks_too_many`\n\nMax blocks exceeded, blocks are limited to 50 per message.\n\n`no_permission`\n\nThe workspace token used in this request does not have the permissions necessary to complete the request. Make sure your app is a member of the conversation it's attempting to post a message to.\n\n`no_text`\n\nNo message text provided\n\n`not_allowed_token_type`\n\nThe token type used in this request is not allowed.\n\n`not_authed`\n\nNo authentication token provided.\n\n`not_in_channel`\n\nCannot post user messages to a channel they are not in.\n\n`org_login_required`\n\nThe workspace is undergoing an enterprise migration and will not be available until migration is complete.\n\n`rate_limited`\n\nApplication has posted too many messages, [read the Rate Limit documentation](/apis/web-api/rate-limits) for more information\n\n`ratelimited`\n\nThe request has been ratelimited. Refer to the `Retry-After` header for when to retry the request.\n\n`request_timeout`\n\nThe method was called via a `POST` request, but the `POST` data was either missing or truncated.\n\n`restricted_action`\n\nA workspace preference prevents the authenticated user from posting.\n\n`restricted_action_non_threadable_channel`\n\nCannot post thread replies into a non\\_threadable channel.\n\n`restricted_action_read_only_channel`\n\nCannot post any message into a read-only channel.\n\n`restricted_action_thread_locked`\n\nCannot post replies to a thread that has been locked by admins.\n\n`restricted_action_thread_only_channel`\n\nCannot post top-level messages into a thread-only channel.\n\n`service_unavailable`\n\nThe service is temporarily unavailable\n\n`slack_connect_canvas_sharing_blocked`\n\nAdmin has disabled Canvas File sharing in all Slack Connect communications\n\n`slack_connect_file_link_sharing_blocked`\n\nAdmin has disabled Slack File sharing in all Slack Connect communications\n\n`slack_connect_lists_sharing_blocked`\n\nAdmin has disabled Lists sharing in all Slack Connect communications\n\n`team_access_not_granted`\n\nThe token used is not granted the specific workspace access required to complete this request.\n\n`team_added_to_org`\n\nThe workspace associated with your request is currently undergoing migration to an Enterprise Organization. Web API and other platform operations will be intermittently unavailable until the transition is complete.\n\n`team_not_found`\n\nThis error occurs if, when using an org-wide token, the `channel_name` is passed instead of the `channel_id`.\n\n`token_expired`\n\nAuthentication token has expired\n\n`token_revoked`\n\nAuthentication token is for a deleted user or workspace or the app has been removed when using a `user`...", "title": "chat.stopStream method | Slack Developer Docs", "content_type": "text/html; charset=utf-8", - "source_bytes": 91845, - "extracted_chars": 12496, - "truncated": true, - "status": "success", - "ok": true + "source_bytes": 91797, + "extracted_chars": 12496 }, "metadata": { - "cacheKey": "7795b1275bd8c551f2a5d9ba0aa2cd5182277af60342e9322e55a8a104464085", - "version": "web-fetch-v1", - "mode": "auto" + "cacheKey": "cb2a12c9a09977dfc53d741d349e94ee98eac65189cdcaa027b2fa0cf213b5eb", + "version": "web-fetch-v2", + "mode": "record" } } diff --git a/packages/junior-evals/evals/agent/files.eval.ts b/packages/junior-evals/evals/agent/files.eval.ts index dc6564d25..47fd4a3fe 100644 --- a/packages/junior-evals/evals/agent/files.eval.ts +++ b/packages/junior-evals/evals/agent/files.eval.ts @@ -75,9 +75,6 @@ describeEval("Coding File Tools", slackEvals, (it) => { }), }); expect(result.usage.model).toBe("openai/gpt-5.6-sol"); - expect( - toolCalls(result.session).filter((call) => call.name === "handoff"), - ).toHaveLength(0); }); it("routes a coding task to the handoff model and keeps its workspace on the next turn", async ({ diff --git a/packages/junior-evals/evals/memory/actors.eval.ts b/packages/junior-evals/evals/memory/actors.eval.ts index 674c9038e..1953ea1fb 100644 --- a/packages/junior-evals/evals/memory/actors.eval.ts +++ b/packages/junior-evals/evals/memory/actors.eval.ts @@ -172,7 +172,7 @@ describeEval("Memory Multi-Actor Provenance", slackEvals, (it) => { overrides: memoryPluginOverrides, initialEvents: [ mention( - "I prefer status updates with risks listed first. Can you draft one for the rollout pause?", + "I prefer status updates with risks listed first. Draft a brief update saying the rollout is paused while we validate the rollback and that the next checkpoint is tomorrow.", { thread: conflictingPreferencesThread, author: ALICE, @@ -314,7 +314,7 @@ describeEval("Memory Multi-Actor Provenance", slackEvals, (it) => { overrides: memoryPluginOverrides, initialEvents: [ mention( - "Can you list the open questions from this thread when you get a chance?", + "Open question: should we pause the launch? Please list it when you get a chance.", { thread: actorPreferenceMultiActorThread, author: BOB, diff --git a/packages/junior-evals/evals/memory/shared.eval.ts b/packages/junior-evals/evals/memory/shared.eval.ts index e06a91bd1..d50e4bc75 100644 --- a/packages/junior-evals/evals/memory/shared.eval.ts +++ b/packages/junior-evals/evals/memory/shared.eval.ts @@ -158,7 +158,7 @@ describeEval("Shared Memory", slackEvals, (it) => { }) => { await clearMemories(); const userText = - "Branch QA runbooks require risk notes before summary notes."; + "For this team, branch QA runbooks require risk notes before summary notes. Please acknowledge."; const result = await run({ overrides: memoryPluginOverrides, initialEvents: [ diff --git a/packages/junior-evals/fixtures/resource-event-plugins/eval-resource-events/skills/eval-resource-events/SKILL.md b/packages/junior-evals/fixtures/resource-event-plugins/eval-resource-events/skills/eval-resource-events/SKILL.md new file mode 100644 index 000000000..258473bfe --- /dev/null +++ b/packages/junior-evals/fixtures/resource-event-plugins/eval-resource-events/skills/eval-resource-events/SKILL.md @@ -0,0 +1,10 @@ +--- +name: eval-resource-events +description: Use for `/eval-resource-events` requests that create a watchable resource and monitor requested outcomes. +--- + +# Eval Resource Events + +1. Call `searchMcpTools` for provider `eval-resource-events` and find the pull-request creation tool. Do not inspect other providers, installed plugins, or runtime configuration. +2. Call the returned `mcp__eval-resource-events__create-watchable-pull-request` tool with the requested title. +3. Continue the user's request from the returned pull-request result. diff --git a/packages/junior-evals/global-setup.ts b/packages/junior-evals/global-setup.ts index 08b114b67..3f347de01 100644 --- a/packages/junior-evals/global-setup.ts +++ b/packages/junior-evals/global-setup.ts @@ -17,6 +17,7 @@ import setupPostgres from "./postgres-global-setup"; import { startEvalEgress } from "./src/eval-egress"; import type { EvalInvocationContext } from "./src/eval-context"; import { loadEvalPluginFixtures } from "./src/eval-plugin-fixtures"; +import { installEvalAiGatewayDispatcher } from "./src/eval-ai-gateway-dispatcher"; type EvalGlobalProject = Parameters[0] & { provide(key: "juniorEvalContext", value: EvalInvocationContext): void; @@ -32,6 +33,7 @@ export default async function setup( project: EvalGlobalProject, ): Promise<() => Promise> { const teardownPostgres = await setupPostgres(project); + const restoreAiGatewayDispatcher = installEvalAiGatewayDispatcher(); let previousCatalogConfig: ReturnType; let egress: Awaited> | undefined; let mswListening = false; @@ -49,6 +51,7 @@ export default async function setup( pluginCatalogRuntime.setConfig(previousCatalogConfig); }, teardownPostgres, + restoreAiGatewayDispatcher, ]) { try { await task(); diff --git a/packages/junior-evals/package.json b/packages/junior-evals/package.json index 8fe47ecb1..a432b13bf 100644 --- a/packages/junior-evals/package.json +++ b/packages/junior-evals/package.json @@ -22,6 +22,7 @@ "chat": "4.29.0", "tinyrainbow": "^3.1.0", "typescript": "^6.0.3", + "undici": "7.25.0", "vitest": "^4.1.7", "vitest-evals": "0.15.0", "zod": "catalog:" diff --git a/packages/junior-evals/src/behavior-harness.ts b/packages/junior-evals/src/behavior-harness.ts index da1fcb1c9..bd20365f4 100644 --- a/packages/junior-evals/src/behavior-harness.ts +++ b/packages/junior-evals/src/behavior-harness.ts @@ -13,6 +13,7 @@ import type { Destination } from "@sentry/junior-plugin-api"; import { executeWithReplay } from "vitest-evals/replay"; import type { JsonValue } from "vitest-evals/harness"; import { createSlackRuntime } from "@/chat/app/factory"; +import { botConfig } from "@/chat/config"; import { getConversationEventStore, getDb } from "@/chat/db"; import type { AssistantLifecycleEvent } from "@/chat/runtime/slack-runtime"; import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; @@ -71,6 +72,8 @@ import { upsertAgentTurnSessionRecord } from "@/chat/state/turn-session"; import { resetSkillDiscoveryCache } from "@/chat/skills"; import { juniorToolResultSchema } from "@/chat/tool-support/structured-result"; import { annotateTurnDeadlineToolResult } from "@/chat/tool-support/turn-deadline-result"; +import { DEFAULT_MAX_CHARS, MAX_FETCH_CHARS } from "@/chat/tools/web/constants"; +import { truncateWebFetchContent } from "@/chat/tools/web/fetch-content"; import { createWebFetchTool } from "@/chat/tools/web/fetch-tool"; import { createWebSearchTool } from "@/chat/tools/web/search"; import type { @@ -434,10 +437,11 @@ function createReplayWebFetchDeps( return { execute: async (input) => { - const args: Record = { url: input.url }; - if (input.max_chars !== undefined) { - args.max_chars = input.max_chars; - } + const requestedMaxChars = input.max_chars ?? DEFAULT_MAX_CHARS; + const args: Record = { + url: input.url, + max_chars: MAX_FETCH_CHARS, + }; const { result } = await executeWithReplay({ toolName: "webFetch", @@ -461,14 +465,25 @@ function createReplayWebFetchDeps( return output as JsonValue; }, replay: { - version: "web-fetch-v1", + version: "web-fetch-v2", key: (replayArgs) => ({ url: replayArgs.url, - max_chars: replayArgs.max_chars ?? null, }), }, }); - return juniorToolResultSchema.parse(result); + const parsed = juniorToolResultSchema.parse(result); + if (parsed.ok !== true || typeof parsed.content !== "string") { + return parsed; + } + const limited = truncateWebFetchContent( + parsed.content, + requestedMaxChars, + ); + return { + ...parsed, + content: limited.content, + truncated: parsed.truncated === true || limited.truncated, + }; }, }; } @@ -476,7 +491,7 @@ function createReplayWebFetchDeps( function createReplayWebSearchDeps( baseOverrides: ToolHooks["toolOverrides"], ): WebSearchToolDeps { - const liveTool = createWebSearchTool({ + const liveTool = createWebSearchTool(botConfig.webSearchModelId, { execute: baseOverrides?.webSearch?.execute, }); diff --git a/packages/junior-evals/src/eval-ai-gateway-dispatcher.ts b/packages/junior-evals/src/eval-ai-gateway-dispatcher.ts new file mode 100644 index 000000000..8b4113e0c --- /dev/null +++ b/packages/junior-evals/src/eval-ai-gateway-dispatcher.ts @@ -0,0 +1,41 @@ +import { + getGlobalDispatcher, + setGlobalDispatcher, + type Dispatcher, +} from "undici"; +import { GEN_AI_SERVER_ADDRESS, GEN_AI_SERVER_PORT } from "@/chat/pi/client"; + +const AI_GATEWAY_ORIGIN = new URL( + `https://${GEN_AI_SERVER_ADDRESS}:${GEN_AI_SERVER_PORT}`, +).origin; +// Leaves time for provider retry backoff and a second attempt inside the +// separate 60-second eval reply budget. +export const EVAL_AI_GATEWAY_BODY_TIMEOUT_MS = 25_000; + +function requestOrigin(origin: Dispatcher.DispatchOptions["origin"]): string { + if (!origin) return ""; + return typeof origin === "string" ? new URL(origin).origin : origin.origin; +} + +/** Install an eval-scoped body timeout for AI Gateway provider requests. */ +export function installEvalAiGatewayDispatcher( + bodyTimeoutMs = EVAL_AI_GATEWAY_BODY_TIMEOUT_MS, + targetOrigin = AI_GATEWAY_ORIGIN, +): () => Promise { + const previousDispatcher = getGlobalDispatcher(); + const normalizedTargetOrigin = new URL(targetOrigin).origin; + const dispatcher = previousDispatcher.compose( + (dispatch) => (options, handler) => + dispatch( + requestOrigin(options.origin) === normalizedTargetOrigin + ? { ...options, bodyTimeout: bodyTimeoutMs } + : options, + handler, + ), + ); + setGlobalDispatcher(dispatcher); + + return async () => { + setGlobalDispatcher(previousDispatcher); + }; +} diff --git a/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts b/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts new file mode 100644 index 000000000..e37b2f967 --- /dev/null +++ b/packages/junior-evals/tests/integration/eval-ai-gateway-dispatcher.test.ts @@ -0,0 +1,99 @@ +import http from "node:http"; +import { afterEach, describe, expect, it } from "vitest"; +import { + getModel, + isRetryableAssistantError, + streamAnthropic, + type Message as PiAiMessage, +} from "@/chat/pi/sdk"; +import type { PiMessage } from "@/chat/pi/messages"; +import { nextProviderRetry } from "@/chat/services/provider-retry"; +import { installEvalAiGatewayDispatcher } from "../../src/eval-ai-gateway-dispatcher"; + +const openServers = new Set(); + +async function startStalledServer(): Promise { + const server = http.createServer((_request, response) => { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.flushHeaders(); + }); + openServers.add(server); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", () => resolve()), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected a TCP test server address"); + } + return `http://127.0.0.1:${address.port}`; +} + +afterEach(async () => { + for (const server of openServers) { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } + openServers.clear(); +}); + +describe("eval AI Gateway dispatcher", () => { + it("terminates a response body that stops producing data", async () => { + const targetOrigin = await startStalledServer(); + const restore = installEvalAiGatewayDispatcher(100, targetOrigin); + + try { + const response = await fetch(targetOrigin); + const reader = response.body?.getReader(); + if (!reader) throw new Error("Expected response body reader"); + + await expect(reader.read()).rejects.toMatchObject({ + cause: expect.objectContaining({ code: "UND_ERR_BODY_TIMEOUT" }), + }); + } finally { + await restore(); + } + }); + + it("turns a stalled Pi stream into resumable provider history", async () => { + const targetOrigin = await startStalledServer(); + const restore = installEvalAiGatewayDispatcher(100, targetOrigin); + const userMessage: PiAiMessage = { + role: "user", + content: [{ type: "text", text: "help" }], + timestamp: Date.now(), + }; + + try { + const stream = streamAnthropic( + getModel("vercel-ai-gateway", "xai/grok-4.5"), + { messages: [userMessage] }, + { + client: { + messages: { + create: () => ({ + asResponse: async () => await fetch(targetOrigin), + }), + }, + } as never, + }, + ); + const failedAssistant = await stream.result(); + + expect(failedAssistant).toMatchObject({ + role: "assistant", + stopReason: "error", + errorMessage: "terminated", + }); + expect(isRetryableAssistantError(failedAssistant)).toBe(true); + expect( + nextProviderRetry({ + attempt: 0, + messages: [userMessage as PiMessage, failedAssistant as PiMessage], + retryableFailure: isRetryableAssistantError(failedAssistant), + }), + ).toEqual({ delayMs: 2_000, messages: [userMessage as PiMessage] }); + } finally { + await restore(); + } + }); +}); diff --git a/packages/junior-evals/tests/unit/harness/behavior-harness.test.ts b/packages/junior-evals/tests/unit/harness/behavior-harness.test.ts index 91c965528..4d37ea58c 100644 --- a/packages/junior-evals/tests/unit/harness/behavior-harness.test.ts +++ b/packages/junior-evals/tests/unit/harness/behavior-harness.test.ts @@ -22,7 +22,17 @@ const { return { executeAgentRunMock: vi.fn< (request: { - policy?: { signal?: AbortSignal }; + policy?: { + signal?: AbortSignal; + toolOverrides?: { + webFetch?: { + execute?: (input: { + url: string; + max_chars?: number; + }) => Promise | unknown; + }; + }; + }; }) => Promise> >(async () => ({})), observedRuntimeIds, @@ -168,6 +178,40 @@ describe("behavior harness", () => { ).rejects.toMatchObject({ name: "TimeoutError" }); }); + it("replays one canonical web source at different output limits", async () => { + const previousReplayMode = process.env.VITEST_EVALS_REPLAY_MODE; + process.env.VITEST_EVALS_REPLAY_MODE = "strict"; + let shortResult: unknown; + let longResult: unknown; + executeAgentRunMock.mockImplementationOnce(async (request) => { + const execute = request.policy?.toolOverrides?.webFetch?.execute; + if (!execute) throw new Error("missing eval webFetch override"); + const url = "https://docs.slack.dev/reference/methods/chat.startStream/"; + shortResult = await execute({ url, max_chars: 1_000 }); + longResult = await execute({ url, max_chars: 8_000 }); + return {}; + }); + + try { + await runEvalScenario({ initialEvents: [] }); + await runtimeState.agentRunner?.run({ policy: {} }); + } finally { + if (previousReplayMode === undefined) { + delete process.env.VITEST_EVALS_REPLAY_MODE; + } else { + process.env.VITEST_EVALS_REPLAY_MODE = previousReplayMode; + } + } + + expect(shortResult).toMatchObject({ ok: true, truncated: true }); + expect(longResult).toMatchObject({ ok: true, truncated: true }); + const shortContent = (shortResult as { content: string }).content; + const longContent = (longResult as { content: string }).content; + expect(shortContent.length).toBeLessThanOrEqual(1_003); + expect(longContent.length).toBeLessThanOrEqual(8_003); + expect(longContent.length).toBeGreaterThan(shortContent.length); + }); + it("normalizes eval thread fixtures to Slack-style runtime thread ids", async () => { const result = await runEvalScenario({ initialEvents: [ diff --git a/packages/junior/.env.example b/packages/junior/.env.example index 4c2ebbbec..2ffb1dfa1 100644 --- a/packages/junior/.env.example +++ b/packages/junior/.env.example @@ -10,7 +10,8 @@ AI_FAST_MODEL= # Defaults to openai/gpt-5.4-mini; used for lightwei AI_HANDOFF_MODEL= # Defaults to openai/gpt-5.6-sol; model for the default handoff profile AI_MODEL_PROFILES= # Optional JSON object of additional named handoff profiles to model IDs AI_VISION_MODEL= # Dedicated image-understanding model; unset disables vision features -AI_WEB_SEARCH_MODEL= # Override for the webSearch tool; defaults to xai/grok-4-fast-reasoning +AI_WEB_SEARCH_MODEL= # Override for the webSearch tool; defaults to openai/gpt-5.4 +AI_IMAGE_MODEL= # Override for image generation; defaults to google/gemini-3-pro-image AGENT_TURN_TIMEOUT_MS= # Defaults to 720000 (12m), clamped under queue max duration budget REDIS_URL= SKILL_DIRS= # Additional skill directories (colon-separated) diff --git a/packages/junior/src/chat/config.ts b/packages/junior/src/chat/config.ts index 90274528d..6c7e848a2 100644 --- a/packages/junior/src/chat/config.ts +++ b/packages/junior/src/chat/config.ts @@ -45,6 +45,7 @@ const DEFAULT_ASSISTANT_LOADING_MESSAGES = [ export interface BotConfig { embeddingModelId: string; fastModelId: string; + imageGenerationModelId: string; loadingMessages: string[]; profiles: Readonly>; reasoningLevel?: TurnReasoningLevel; @@ -53,6 +54,7 @@ export interface BotConfig { maxSlicesPerTurn: number; turnTimeoutMs: number; userName: string; + webSearchModelId: string; } export type SqlDriver = "neon" | "postgres"; @@ -179,7 +181,12 @@ const DEFAULT_HANDOFF_MODEL_ID = getModel( "vercel-ai-gateway", "openai/gpt-5.6-sol", ).id; +const DEFAULT_WEB_SEARCH_MODEL_ID = getModel( + "vercel-ai-gateway", + "openai/gpt-5.4", +).id; const DEFAULT_EMBEDDING_MODEL_ID = "openai/text-embedding-3-small"; +const DEFAULT_IMAGE_GENERATION_MODEL_ID = "google/gemini-3-pro-image"; function validateGatewayModelId(raw: string | undefined): string | undefined { const trimmed = toOptionalTrimmed(raw); @@ -285,6 +292,9 @@ function readBotConfig( ? undefined : parseTurnReasoningLevel(reasoningLevel), fastModelId, + imageGenerationModelId: + toOptionalTrimmed(env.AI_IMAGE_MODEL) ?? + DEFAULT_IMAGE_GENERATION_MODEL_ID, embeddingModelId: validateEmbeddingModelId(env.AI_EMBEDDING_MODEL) ?? DEFAULT_EMBEDDING_MODEL_ID, @@ -295,6 +305,9 @@ function readBotConfig( env.AGENT_TURN_TIMEOUT_MS, maxTurnTimeoutMs, ), + webSearchModelId: + validateGatewayModelId(env.AI_WEB_SEARCH_MODEL) ?? + DEFAULT_WEB_SEARCH_MODEL_ID, }; } diff --git a/packages/junior/src/chat/pi/sdk.ts b/packages/junior/src/chat/pi/sdk.ts index 6f3bb6f73..32c85a872 100644 --- a/packages/junior/src/chat/pi/sdk.ts +++ b/packages/junior/src/chat/pi/sdk.ts @@ -6,7 +6,9 @@ export { completeSimple, getEnvApiKey, + getModel, getModels, + isRetryableAssistantError, registerApiProvider, type Message, type Model, diff --git a/packages/junior/src/chat/prompt.ts b/packages/junior/src/chat/prompt.ts index 32d7e7fe8..45e4eb307 100644 --- a/packages/junior/src/chat/prompt.ts +++ b/packages/junior/src/chat/prompt.ts @@ -350,12 +350,14 @@ const TURN_CONTEXT_HEADER = const TOOL_POLICY_RULES = [ "- Tool schemas are the source of truth for parameters; tool names are case-sensitive, so call tools exactly by their exposed names and do not invent arguments.", "- Use tools for actionable work and for facts that are mutable, external, repository-backed, provider-backed, or requested as verified/current. Stable general knowledge and already-provided context may be answered directly.", + "- Do not use tools to rediscover content already present in the current conversation. For drafting or rewriting with no referenced retrievable source, use clear placeholders or ask one focused question instead of searching unrelated memory, history, files, or providers.", "- Resolve provider action targets before calls: explicit target wins; ambient `` fills omitted targets. Treat non-target links/references as context.", "- Verification source order: conversation/thread context; user-provided attachments, links, and reference files; local/sandbox files when present; loaded skill references; repository/provider tools; public web. Use the nearest authoritative available source before weaker sources.", "- For repository or implementation questions, inspect the target repository first: local checkout when present, otherwise the configured GitHub/source provider. Do not treat loaded skill files as repo source unless the user asks about the skill. Cite file paths, symbols, PRs/issues, commits, or URLs that support the answer.", "- If a sandbox-backed tool reports that sandbox execution is unavailable, treat that as a blocker for local file/shell inspection; do not pretend host files were inspected.", "- For user-provided URLs, use `webFetch`; for discovery, use `webSearch` then fetch/read promising sources; for current time/date context, use `systemTime`.", "- When a tool result includes a subscribable resource, use resource-event subscriptions for high-signal provider changes that serve the user's current intent; do not create scheduled polling tasks for events the subscription can deliver. Use the suggested events when they fit and write a concise intent summary.", + "- For code changes, debugging or root-cause analysis, broad refactors, and software architecture decisions, use `handoff` before substantive analysis when that tool is available.", "- Run `jr-rpc config get|set|unset|list` for provider defaults and `jr-rpc plugins list` for installed plugin introspection as standalone bash commands; do not chain them with `cd`, `&&`, pipes, or provider commands.", "- If the first result is empty, stale, ambiguous, or incomplete, try a focused alternate query, path, command, or source before concluding the answer cannot be verified.", ]; @@ -389,6 +391,7 @@ const CONVERSATION_RULES = [ const SLACK_ACTION_RULES = [ "- Slack tools target the current runtime context; if the requested Slack target differs, explain the limitation instead of calling the tool.", + "- Assistant text is delivered only into the active conversation or thread. You cannot create a new top-level channel post; if asked to do so, explain that limitation and do not present the requested text as delivered.", "- Ambient reaction requests target the current inbound message; do not ask for a message reference.", `- When no visible thread reply is requested or useful, keep tool-calling messages text-free and make the final message exactly ${NO_REPLY_MARKER}.`, ]; diff --git a/packages/junior/src/chat/tools/index.ts b/packages/junior/src/chat/tools/index.ts index 675e37fed..cedc615f9 100644 --- a/packages/junior/src/chat/tools/index.ts +++ b/packages/junior/src/chat/tools/index.ts @@ -1,4 +1,5 @@ import { resolveChannelCapabilities } from "@/chat/slack/tools/channel-capabilities"; +import { botConfig } from "@/chat/config"; import { createBashTool } from "@/chat/tools/sandbox/bash"; import { createEditFileTool } from "@/chat/tools/sandbox/edit-file"; import { createFindFilesTool } from "@/chat/tools/sandbox/find-files"; @@ -112,7 +113,10 @@ export function createTools( findFiles: createFindFilesTool(), listDir: createListDirTool(), writeFile: createWriteFileTool(), - webSearch: createWebSearchTool(hooks.toolOverrides?.webSearch), + webSearch: createWebSearchTool( + botConfig.webSearchModelId, + hooks.toolOverrides?.webSearch, + ), webFetch: createWebFetchTool(hooks, { canSendFilesToActiveConversation, }), @@ -123,6 +127,7 @@ export function createTools( writeGeneratedArtifacts: hooks.writeGeneratedArtifacts, }, { + modelId: botConfig.imageGenerationModelId, canSendFilesToActiveConversation, }, hooks.toolOverrides?.imageGenerate, diff --git a/packages/junior/src/chat/tools/web/fetch-content.ts b/packages/junior/src/chat/tools/web/fetch-content.ts index 397f46511..02ae6d282 100644 --- a/packages/junior/src/chat/tools/web/fetch-content.ts +++ b/packages/junior/src/chat/tools/web/fetch-content.ts @@ -64,6 +64,14 @@ function truncateAtWordBoundary( return { content: `${shortened.trimEnd()}...`, truncated: true }; } +/** Apply the web-fetch character limit to already extracted text content. */ +export function truncateWebFetchContent( + content: string, + maxChars: number, +): { content: string; truncated: boolean } { + return truncateAtWordBoundary(content, maxChars); +} + function decodeHtmlEntities(value: string): string { return value .replace(/"/g, '"') @@ -157,7 +165,7 @@ export function extractContentDetails( const sourceHtml = extractMainHtml(normalizedBody); const markdown = htmlToMarkdownConverter.translate(sourceHtml); const normalizedMarkdown = normalizeWhitespace(markdown); - const truncated = truncateAtWordBoundary(normalizedMarkdown, maxChars); + const truncated = truncateWebFetchContent(normalizedMarkdown, maxChars); return { content: truncated.content, title: extractTitle(normalizedBody), @@ -173,7 +181,7 @@ export function extractContentDetails( try { const parsed = JSON.parse(normalizedBody); const formatted = JSON.stringify(parsed, null, 2); - const truncated = truncateAtWordBoundary(formatted, maxChars); + const truncated = truncateWebFetchContent(formatted, maxChars); return { content: truncated.content, truncated: truncated.truncated, @@ -181,7 +189,7 @@ export function extractContentDetails( }; } catch { const normalizedText = normalizeWhitespace(normalizedBody); - const truncated = truncateAtWordBoundary(normalizedText, maxChars); + const truncated = truncateWebFetchContent(normalizedText, maxChars); return { content: truncated.content, truncated: truncated.truncated, @@ -191,7 +199,7 @@ export function extractContentDetails( } const normalizedText = normalizeWhitespace(normalizedBody); - const truncated = truncateAtWordBoundary(normalizedText, maxChars); + const truncated = truncateWebFetchContent(normalizedText, maxChars); return { content: truncated.content, truncated: truncated.truncated, diff --git a/packages/junior/src/chat/tools/web/image-generate.ts b/packages/junior/src/chat/tools/web/image-generate.ts index 41bcf84d2..c33c38f17 100644 --- a/packages/junior/src/chat/tools/web/image-generate.ts +++ b/packages/junior/src/chat/tools/web/image-generate.ts @@ -12,8 +12,6 @@ import { JUNIOR_PERSONALITY } from "@/chat/prompt"; import { logInfo, logWarn } from "@/chat/logging"; import { generatedArtifactFileRefSchema } from "@/chat/tools/sandbox/file-uploads"; -const DEFAULT_IMAGE_MODEL = "google/gemini-3-pro-image"; - const imageGenerateOutputSchema = juniorToolResultSchema .extend({ model: z.string(), @@ -82,7 +80,7 @@ function parseImageGenerationError( const message = payload.error?.message?.trim(); if (!message) return `image generation failed: ${status} ${body}`; if (message.includes("not an image model")) { - return `image generation failed: configured model "${model}" is not an image generation model. Set AI_IMAGE_MODEL to a compatible image model (for example "${DEFAULT_IMAGE_MODEL}").`; + return `image generation failed: configured model "${model}" is not an image generation model. Set AI_IMAGE_MODEL to a compatible image model.`; } return `image generation failed: ${status} ${message}`; } catch { @@ -93,7 +91,10 @@ function parseImageGenerationError( /** Create the image tool with sandbox artifact guidance matched to file-send capability. */ export function createImageGenerateTool( hooks: Required>, - options: { canSendFilesToActiveConversation?: boolean } = {}, + options: { + modelId: string; + canSendFilesToActiveConversation?: boolean; + }, deps: ImageGenerateToolDeps = {}, ) { return zodTool({ @@ -111,7 +112,7 @@ export function createImageGenerateTool( if (!apiKey) { throw new Error(MISSING_GATEWAY_CREDENTIALS_ERROR); } - const model = process.env.AI_IMAGE_MODEL ?? DEFAULT_IMAGE_MODEL; + const model = options.modelId; const enrichedPrompt = await enrichImagePrompt(prompt); const response = await fetchImpl( "https://ai-gateway.vercel.sh/v1/chat/completions", diff --git a/packages/junior/src/chat/tools/web/search.ts b/packages/junior/src/chat/tools/web/search.ts index 16c217ec3..43f307b05 100644 --- a/packages/junior/src/chat/tools/web/search.ts +++ b/packages/junior/src/chat/tools/web/search.ts @@ -3,14 +3,12 @@ import { juniorToolResultSchema } from "@/chat/tool-support/structured-result"; import { zodTool } from "@/chat/tool-support/zod-tool"; import { generateText } from "ai"; import { createGatewayProvider } from "@ai-sdk/gateway"; -import { getModel } from "@earendil-works/pi-ai/compat"; import { withTimeout } from "@/chat/tools/web/network"; import { logException } from "@/chat/logging"; import type { WebSearchToolDeps } from "@/chat/tools/types"; const SEARCH_TIMEOUT_MS = 60_000; const MAX_RESULTS = 5; -const DEFAULT_SEARCH_MODEL = getModel("vercel-ai-gateway", "openai/gpt-5.4").id; const SEARCH_TOOL_NAME = "parallelSearch"; function asString(value: unknown): string | undefined { @@ -75,7 +73,10 @@ function isAuthFailure(message: string): boolean { ); } -export function createWebSearchTool(override?: WebSearchToolDeps) { +export function createWebSearchTool( + modelId: string, + override?: WebSearchToolDeps, +) { return zodTool({ description: "Search public web sources and return top snippets/URLs. Use when you need discovery or source candidates. Do not use when the user already provided a specific URL to inspect.", @@ -103,7 +104,6 @@ export function createWebSearchTool(override?: WebSearchToolDeps) { const maxResults = max_results ?? 3; // Keep web search pinned to a provider-tool capable model instead of // inheriting the main turn model. - const model = process.env.AI_WEB_SEARCH_MODEL ?? DEFAULT_SEARCH_MODEL; const controller = new AbortController(); try { @@ -112,7 +112,7 @@ export function createWebSearchTool(override?: WebSearchToolDeps) { const provider = createGatewayProvider(); const response = await withTimeout( generateText({ - model: provider.chat(model), + model: provider.chat(modelId), prompt: query, tools: { [SEARCH_TOOL_NAME]: provider.tools.parallelSearch({ @@ -132,7 +132,7 @@ export function createWebSearchTool(override?: WebSearchToolDeps) { return { ok: true, status: "success" as const, - model, + model: modelId, query, result_count: results.length, results, diff --git a/packages/junior/tests/component/config/chat-config.test.ts b/packages/junior/tests/component/config/chat-config.test.ts index fe468e660..42786c38b 100644 --- a/packages/junior/tests/component/config/chat-config.test.ts +++ b/packages/junior/tests/component/config/chat-config.test.ts @@ -161,6 +161,40 @@ describe("chat config", () => { expect(botConfig.embeddingModelId).toBe("openai/text-embedding-3-large"); }); + it("uses the default web search model when AI_WEB_SEARCH_MODEL is blank", async () => { + process.env.AI_WEB_SEARCH_MODEL = " "; + + const { botConfig } = await loadConfig(); + expect(botConfig.webSearchModelId).toBe("openai/gpt-5.4"); + }); + + it("uses AI_WEB_SEARCH_MODEL when configured", async () => { + process.env.AI_WEB_SEARCH_MODEL = " anthropic/claude-sonnet-4.6 "; + + const { botConfig } = await loadConfig(); + expect(botConfig.webSearchModelId).toBe("anthropic/claude-sonnet-4.6"); + }); + + it("throws when AI_WEB_SEARCH_MODEL is not a registered gateway model id", async () => { + process.env.AI_WEB_SEARCH_MODEL = "openai/search-model-not-real"; + + await expect(loadConfig()).rejects.toThrow(/Unknown AI Gateway model id/); + }); + + it("uses the default image generation model when AI_IMAGE_MODEL is blank", async () => { + process.env.AI_IMAGE_MODEL = " "; + + const { botConfig } = await loadConfig(); + expect(botConfig.imageGenerationModelId).toBe("google/gemini-3-pro-image"); + }); + + it("uses AI_IMAGE_MODEL when configured", async () => { + process.env.AI_IMAGE_MODEL = " openai/dall-e-3 "; + + const { botConfig } = await loadConfig(); + expect(botConfig.imageGenerationModelId).toBe("openai/dall-e-3"); + }); + it("uses the default slash command when JUNIOR_SLASH_COMMAND is unset", async () => { delete process.env.JUNIOR_SLASH_COMMAND; diff --git a/packages/junior/tests/unit/services/turn-router.test.ts b/packages/junior/tests/unit/services/turn-router.test.ts index 401c85443..51b967963 100644 --- a/packages/junior/tests/unit/services/turn-router.test.ts +++ b/packages/junior/tests/unit/services/turn-router.test.ts @@ -179,6 +179,7 @@ describe("selectTurnRoute", () => { expect(profile).toMatchObject({ reasoningLevel: "medium", + profile: "standard", reason: "low_confidence_medium_default:not confident", }); expect(toPiReasoningLevel(profile.reasoningLevel)).toBe("medium"); diff --git a/packages/junior/tests/unit/web/image-generate.test.ts b/packages/junior/tests/unit/web/image-generate.test.ts index b20430439..c97baa7fb 100644 --- a/packages/junior/tests/unit/web/image-generate.test.ts +++ b/packages/junior/tests/unit/web/image-generate.test.ts @@ -19,6 +19,9 @@ import { completeText } from "@/chat/pi/client"; import { createImageGenerateTool } from "@/chat/tools/web/image-generate"; const mockCompleteText = vi.mocked(completeText); +const defaultImageOptions = { + modelId: "google/gemini-3-pro-image", +}; function getRequestBody(fetchMock: ReturnType) { const request = fetchMock.mock.calls[0]; @@ -74,12 +77,11 @@ describe("createImageGenerateTool", () => { afterEach(() => { delete process.env.AI_GATEWAY_API_KEY; delete process.env.VERCEL_OIDC_TOKEN; - delete process.env.AI_IMAGE_MODEL; vi.unstubAllGlobals(); vi.clearAllMocks(); }); - it("uses the default image model when AI_IMAGE_MODEL is not set", async () => { + it("uses the configured image generation model", async () => { process.env.AI_GATEWAY_API_KEY = "test-key"; mockCompleteText.mockResolvedValueOnce({ text: "enriched prompt" } as any); const fetchMock = vi @@ -89,12 +91,15 @@ describe("createImageGenerateTool", () => { vi.spyOn(Date, "now").mockReturnValue(1_737_000_000_000); const uploads: FileUpload[] = []; - const tool = createImageGenerateTool({ - writeGeneratedArtifacts: (files: FileUpload[]) => { - uploads.push(...files); - return writeGeneratedArtifacts(files); - }, - } as any); + const tool = createImageGenerateTool( + { + writeGeneratedArtifacts: (files: FileUpload[]) => { + uploads.push(...files); + return writeGeneratedArtifacts(files); + }, + } as any, + { modelId: "openai/dall-e-3" }, + ); if (typeof tool.execute !== "function") { throw new Error("imageGenerate execute function missing"); } @@ -106,13 +111,13 @@ describe("createImageGenerateTool", () => { expect(request).toBeDefined(); expect(request[0]).toBe("https://ai-gateway.vercel.sh/v1/chat/completions"); expect(getRequestBody(fetchMock)).toMatchObject({ - model: "google/gemini-3-pro-image", + model: "openai/dall-e-3", messages: [{ role: "user", content: "enriched prompt" }], modalities: ["image"], }); expect(result).toMatchObject({ ok: true, - model: "google/gemini-3-pro-image", + model: "openai/dall-e-3", image_count: 1, delivery: expect.stringContaining("no file-send tool"), }); @@ -154,7 +159,10 @@ describe("createImageGenerateTool", () => { { writeGeneratedArtifacts, }, - { canSendFilesToActiveConversation: true }, + { + ...defaultImageOptions, + canSendFilesToActiveConversation: true, + }, ); const result = await tool.execute!({ prompt: "test prompt" }, {} as any); @@ -167,35 +175,8 @@ describe("createImageGenerateTool", () => { }); }); - it("uses AI_IMAGE_MODEL when configured", async () => { - process.env.AI_GATEWAY_API_KEY = "test-key"; - process.env.AI_IMAGE_MODEL = "openai/dall-e-3"; - mockCompleteText.mockResolvedValueOnce({ text: "enriched cat" } as any); - const fetchMock = vi - .fn() - .mockResolvedValueOnce(createJsonResponse(imagePayload())); - vi.stubGlobal("fetch", fetchMock); - - const tool = createImageGenerateTool({ - writeGeneratedArtifacts, - } as any); - if (typeof tool.execute !== "function") { - throw new Error("imageGenerate execute function missing"); - } - const result = await tool.execute({ prompt: "a cat" }, {} as any); - - expect(getRequestBody(fetchMock)).toMatchObject({ - model: "openai/dall-e-3", - }); - expect(result).toMatchObject({ - ok: true, - model: "openai/dall-e-3", - }); - }); - it("returns an actionable error when model is not image-capable", async () => { process.env.AI_GATEWAY_API_KEY = "test-key"; - process.env.AI_IMAGE_MODEL = "google/gemini-3-pro-image"; mockCompleteText.mockResolvedValueOnce({ text: "enriched prompt" } as any); const fetchMock = vi.fn().mockResolvedValueOnce( createErrorResponse( @@ -210,9 +191,10 @@ describe("createImageGenerateTool", () => { ); vi.stubGlobal("fetch", fetchMock); - const tool = createImageGenerateTool({ - writeGeneratedArtifacts, - } as any); + const tool = createImageGenerateTool( + { writeGeneratedArtifacts } as any, + defaultImageOptions, + ); if (typeof tool.execute !== "function") { throw new Error("imageGenerate execute function missing"); } @@ -233,9 +215,10 @@ describe("createImageGenerateTool", () => { .mockResolvedValueOnce(createJsonResponse(imagePayload())); vi.stubGlobal("fetch", fetchMock); - const tool = createImageGenerateTool({ - writeGeneratedArtifacts, - } as any); + const tool = createImageGenerateTool( + { writeGeneratedArtifacts } as any, + defaultImageOptions, + ); const result = await tool.execute!({ prompt: "draw a dog" }, {} as any); const body = getRequestBody(fetchMock); @@ -256,9 +239,10 @@ describe("createImageGenerateTool", () => { .mockResolvedValueOnce(createJsonResponse(imagePayload())); vi.stubGlobal("fetch", fetchMock); - const tool = createImageGenerateTool({ - writeGeneratedArtifacts, - } as any); + const tool = createImageGenerateTool( + { writeGeneratedArtifacts } as any, + defaultImageOptions, + ); const result = await tool.execute!({ prompt: "draw a dog" }, {} as any); const body = getRequestBody(fetchMock); @@ -277,9 +261,10 @@ describe("createImageGenerateTool", () => { .mockResolvedValueOnce(createJsonResponse(imagePayload())); vi.stubGlobal("fetch", fetchMock); - const tool = createImageGenerateTool({ - writeGeneratedArtifacts, - } as any); + const tool = createImageGenerateTool( + { writeGeneratedArtifacts } as any, + defaultImageOptions, + ); const result = await tool.execute!({ prompt: "draw a dog" }, {} as any); const body = getRequestBody(fetchMock); diff --git a/packages/junior/tests/unit/web/web-fetch-convert.test.ts b/packages/junior/tests/unit/web/web-fetch-convert.test.ts index 233c19c5e..216ce9a67 100644 --- a/packages/junior/tests/unit/web/web-fetch-convert.test.ts +++ b/packages/junior/tests/unit/web/web-fetch-convert.test.ts @@ -3,6 +3,7 @@ import { extractContent, extractContentDetails, extractWebFetchResponse, + truncateWebFetchContent, } from "@/chat/tools/web/fetch-content"; describe("web fetch content conversion", () => { @@ -32,6 +33,19 @@ describe("web fetch content conversion", () => { expect(result).toContain('"ok": true'); }); + it("applies different limits to canonical extracted content", () => { + const content = "Slack streaming guidance ".repeat(600); + + const short = truncateWebFetchContent(content, 6000); + const long = truncateWebFetchContent(content, 8000); + + expect(short.content.length).toBeLessThanOrEqual(6003); + expect(long.content.length).toBeLessThanOrEqual(8003); + expect(long.content.length).toBeGreaterThan(short.content.length); + expect(short.truncated).toBe(true); + expect(long.truncated).toBe(true); + }); + it("prefers main document content over page chrome", () => { const result = extractContentDetails( [ diff --git a/packages/junior/tests/unit/web/web-search.test.ts b/packages/junior/tests/unit/web/web-search.test.ts index 02896e53f..f6581de88 100644 --- a/packages/junior/tests/unit/web/web-search.test.ts +++ b/packages/junior/tests/unit/web/web-search.test.ts @@ -12,6 +12,7 @@ vi.mock("@ai-sdk/gateway", () => ({ })); describe("createWebSearchTool", () => { + const defaultModelId = "openai/gpt-5.4"; const parallelSearch = { id: "parallel-search-tool" }; const gatewayProvider = { chat: vi.fn((model: string) => ({ model })), @@ -25,17 +26,11 @@ describe("createWebSearchTool", () => { }); afterEach(() => { - delete process.env.AI_GATEWAY_API_KEY; - delete process.env.VERCEL_OIDC_TOKEN; - delete process.env.AI_WEB_SEARCH_MODEL; - delete process.env.AI_FAST_MODEL; - delete process.env.AI_MODEL; vi.useRealTimers(); vi.clearAllMocks(); }); it("uses AI Gateway parallel search and maps tool results", async () => { - process.env.AI_WEB_SEARCH_MODEL = "openai/gpt-5.4"; vi.mocked(generateText).mockResolvedValueOnce({ toolResults: [ { @@ -54,7 +49,7 @@ describe("createWebSearchTool", () => { ], } as never); - const tool = createWebSearchTool(); + const tool = createWebSearchTool("anthropic/claude-sonnet-4.6"); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } @@ -71,7 +66,7 @@ describe("createWebSearchTool", () => { }); expect(generateText).toHaveBeenCalledWith( expect.objectContaining({ - model: { model: "openai/gpt-5.4" }, + model: { model: "anthropic/claude-sonnet-4.6" }, prompt: "vercel ai gateway", toolChoice: { type: "tool", toolName: "parallelSearch" }, abortSignal: expect.any(AbortSignal), @@ -80,7 +75,7 @@ describe("createWebSearchTool", () => { expect(result).toEqual({ ok: true, status: "success", - model: "openai/gpt-5.4", + model: "anthropic/claude-sonnet-4.6", query: "vercel ai gateway", result_count: 1, results: [ @@ -93,28 +88,12 @@ describe("createWebSearchTool", () => { }); }); - it("uses the default search model when AI_WEB_SEARCH_MODEL is unset, ignoring AI_MODEL/AI_FAST_MODEL", async () => { - delete process.env.AI_WEB_SEARCH_MODEL; - process.env.AI_FAST_MODEL = "openai/gpt-5.4"; - process.env.AI_MODEL = "anthropic/claude-sonnet-4.6"; - vi.mocked(generateText).mockResolvedValueOnce({ toolResults: [] } as never); - - const tool = createWebSearchTool(); - if (typeof tool.execute !== "function") { - throw new Error("webSearch execute function missing"); - } - - await tool.execute({ query: "anything" }, {} as never); - - expect(gatewayProvider.chat).toHaveBeenCalledWith("openai/gpt-5.4"); - }); - it("wraps AI SDK errors in web search error message", async () => { vi.mocked(generateText).mockRejectedValueOnce( new Error('400 Invalid input: expected "function"'), ); - const tool = createWebSearchTool(); + const tool = createWebSearchTool(defaultModelId); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } @@ -142,7 +121,7 @@ describe("createWebSearchTool", () => { }) as never, ); - const tool = createWebSearchTool(); + const tool = createWebSearchTool(defaultModelId); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } @@ -174,7 +153,7 @@ describe("createWebSearchTool", () => { }); }) as never); - const tool = createWebSearchTool(); + const tool = createWebSearchTool(defaultModelId); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } @@ -196,7 +175,7 @@ describe("createWebSearchTool", () => { return Promise.resolve({ toolResults: [] }); }) as never); - const tool = createWebSearchTool(); + const tool = createWebSearchTool(defaultModelId); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } @@ -230,7 +209,7 @@ describe("createWebSearchTool", () => { }) as never, ); - const tool = createWebSearchTool(); + const tool = createWebSearchTool(defaultModelId); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } @@ -256,7 +235,7 @@ describe("createWebSearchTool", () => { ), ); - const tool = createWebSearchTool(); + const tool = createWebSearchTool(defaultModelId); if (typeof tool.execute !== "function") { throw new Error("webSearch execute function missing"); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c2274f93..2b699d4a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -401,6 +401,9 @@ importers: typescript: specifier: ^6.0.3 version: 6.0.3 + undici: + specifier: 7.25.0 + version: 7.25.0 vitest: specifier: ^4.1.7 version: 4.1.7(@types/node@25.9.1)(tsx@4.22.3)