From e9b2d68ebfe8d9b5683486fb0900ef162b3e6aec Mon Sep 17 00:00:00 2001 From: Aditya Parikh Date: Wed, 14 Oct 2026 10:31:30 +0100 Subject: [PATCH] test(eval): add the harness and Gradle task for LLM tool-selection evals First part of #256. `./gradlew eval` runs tests tagged `llm-eval`; every other Test task excludes the tag, so `./gradlew build` never calls a model. The eval context is Spring Boot auto-configuration, set up by properties: - spring-ai-starter-model-openai is the agent, on Spring AI's defaults with OPENAI_API_KEY; nothing is configured. - spring-ai-starter-mcp-client starts the built jar over STDIO against a Testcontainers Solr (stdio connection in application-integration-test.properties). - The ChatClient bean gets the server's instructions, its tools and ToolCallAuditAdvisor, which observes the tool-calling loop from inside (ordered after ToolCallingAdvisor, as the Spring AI reference describes under "Observing the tool-calling loop"). When the model gives its answer, that iteration's request holds every tool call and its ToolResponseMessage; the advisor records them as spring-ai-typesafe JevJudgeInput.ToolCall (name, arguments, result). An ApplicationRunner loads the DevNexus 2026 sessions through the server's own tools. ToolCallAuditAdvisorTest runs a real ChatClient tool loop over a scripted model in ./gradlew build. The first scenario follows in the next change. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Aditya Parikh --- build.gradle.kts | 39 +++++- gradle/libs.versions.toml | 12 ++ .../mcp/server/eval/EvalConfiguration.java | 95 +++++++++++++ .../mcp/server/eval/ToolCallAuditAdvisor.java | 105 ++++++++++++++ .../server/eval/ToolCallAuditAdvisorTest.java | 131 ++++++++++++++++++ .../solr/mcp/server/eval/package-info.java | 25 ++++ .../application-integration-test.properties | 30 ++++ 7 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/apache/solr/mcp/server/eval/EvalConfiguration.java create mode 100644 src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisor.java create mode 100644 src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisorTest.java create mode 100644 src/test/java/org/apache/solr/mcp/server/eval/package-info.java create mode 100644 src/test/resources/application-integration-test.properties diff --git a/build.gradle.kts b/build.gradle.kts index a9ab9a75..2b0171e2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -197,6 +197,7 @@ dependencies { errorprone(libs.nullaway) testImplementation(libs.bundles.test) + testImplementation(libs.bundles.eval) testRuntimeOnly(libs.junit.platform.launcher) } @@ -252,6 +253,10 @@ tasks.withType { if (name != "dockerIntegrationTest") { excludeTags("docker-integration") } + // LLM evals call a hosted model, so only ./gradlew eval runs them + if (name != "eval") { + excludeTags("llm-eval") + } } // McpClientStdioIntegrationTest spawns `java -jar build/libs/` as a // subprocess. Without an explicit dependency, `:test` runs before `:bootJar` @@ -263,7 +268,7 @@ tasks.withType { // Forward the Testcontainers image pins (or a per-run -D override) to test JVMs. systemProperty("solr.test.image", solrTestImage) systemProperty("lgtm.test.image", lgtmTestImage) - if (name != "dockerIntegrationTest") { + if (name != "dockerIntegrationTest" && name != "eval") { finalizedBy(tasks.jacocoTestReport) } } @@ -417,6 +422,38 @@ changelog { groups = listOf("Added", "Changed", "Fixed", "Security") } +// LLM Tool-Selection Evals +// ======================== +// Opt-in: checks whether a model, given only the server's tool descriptions, picks the +// right tool and arguments (https://github.com/apache/solr-mcp/issues/256). Needs Docker +// and OPENAI_API_KEY; ./gradlew build never runs it. The agent uses Spring AI's OpenAI +// defaults, so nothing is configured here. +// +// ./gradlew eval +tasks.register("eval") { + description = "Runs the LLM tool-selection evals (needs OPENAI_API_KEY)" + group = "verification" + + useJUnitPlatform { + includeTags("llm-eval") + } + + testClassesDirs = sourceSets["test"].output.classesDirs + classpath = sourceSets["test"].runtimeClasspath + + // Model output varies between runs, so never reuse a previous result + outputs.upToDateWhen { false } + + // The jar the MCP client starts (application-integration-test.properties) + val bootJarFile = tasks.bootJar.flatMap { it.archiveFile } + doFirst { systemProperty("solr.mcp.jar", bootJarFile.get().asFile.absolutePath) } + + reports { + html.outputLocation.set(layout.buildDirectory.dir("reports/eval")) + junitXml.outputLocation.set(layout.buildDirectory.dir("test-results/eval")) + } +} + // Docker Integration Test Task // ============================= // Runs integration tests against the appropriate Docker image: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0c7771e9..d81a8448 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,8 @@ changelog-plugin = "2.5.0" spring-ai = "2.0.1" solr = "10.0.0" mcp-server-security = "0.1.14" +# spring-ai-typesafe: the judge and its input types, for the opt-in evals (test classpath only) +typesafe = "0.4.0" # MCP Java SDK. Spring AI 2.0.1 pulls 2.0.0 transitively; 2.0.1 fixes error # handling when a handler is not registered on a stateless server (our HTTP # profile), empty-completion handling for unmatched refs, and adds bounded @@ -74,6 +76,10 @@ spring-boot-testcontainers = { module = "org.springframework.boot:spring-boot-te spring-ai-starter-mcp-server-webmvc = { module = "org.springframework.ai:spring-ai-starter-mcp-server-webmvc" } spring-ai-spring-boot-docker-compose = { module = "org.springframework.ai:spring-ai-spring-boot-docker-compose" } spring-ai-spring-boot-testcontainers = { module = "org.springframework.ai:spring-ai-spring-boot-testcontainers" } +# Opt-in tool-selection evals (./gradlew eval); test classpath only +spring-ai-starter-model-openai = { module = "org.springframework.ai:spring-ai-starter-model-openai" } +spring-ai-starter-mcp-client = { module = "org.springframework.ai:spring-ai-starter-mcp-client" } +typesafe-spring-ai = { module = "org.springaicommunity:typesafe-spring-ai", version.ref = "typesafe" } # MCP Java SDK (pinned ahead of what Spring AI 2.0.1 resolves; see [versions]) mcp-bom = { module = "io.modelcontextprotocol.sdk:mcp-bom", version.ref = "mcp-sdk" } @@ -146,6 +152,12 @@ test = [ "jetty-util" ] +eval = [ + "spring-ai-starter-model-openai", + "spring-ai-starter-mcp-client", + "typesafe-spring-ai" +] + errorprone = [ "errorprone-core", "nullaway" diff --git a/src/test/java/org/apache/solr/mcp/server/eval/EvalConfiguration.java b/src/test/java/org/apache/solr/mcp/server/eval/EvalConfiguration.java new file mode 100644 index 00000000..ecd615d5 --- /dev/null +++ b/src/test/java/org/apache/solr/mcp/server/eval/EvalConfiguration.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mcp.server.eval; + +import io.modelcontextprotocol.client.McpSyncClient; +import io.modelcontextprotocol.spec.McpSchema.CallToolRequest; +import io.modelcontextprotocol.spec.McpSchema.CallToolResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.solr.mcp.server.TestDocuments; +import org.apache.solr.mcp.server.TestcontainersConfiguration; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.mcp.SyncMcpToolCallbackProvider; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; + +/** + * The evals' Spring context. Auto-configuration supplies the model, with Spring AI's OpenAI + * defaults, and an MCP client that starts solr-mcp over STDIO, as configured in {@code + * application-integration-test.properties}. Solr runs in Testcontainers. + */ +@TestConfiguration(proxyBeanMethods = false) +@EnableAutoConfiguration +@Import(TestcontainersConfiguration.class) +class EvalConfiguration { + + static final String COLLECTION = "devnexus"; + + static final Path SESSIONS = Path.of("mydata/devnexus-2026.json"); + + /** + * The agent: the server's instructions and tools, and nothing else. {@link ToolCallAuditAdvisor} + * records the tool calls it makes. + */ + @Bean + ChatClient chatClient( + ChatClient.Builder chatClientBuilder, + List mcpClients, + SyncMcpToolCallbackProvider mcpTools) { + String instructions = + Objects.requireNonNull( + mcpClients.getFirst().getServerInstructions(), "solr-mcp sent no instructions"); + return chatClientBuilder + .defaultSystem(instructions) + .defaultTools(mcpTools) + .defaultAdvisors(new ToolCallAuditAdvisor()) + .build(); + } + + /** Loads the DevNexus 2026 sessions through the server's own tools. */ + @Bean + ApplicationRunner indexSessions(List mcpClients) { + return args -> { + McpSyncClient solrMcp = mcpClients.getFirst(); + callTool(solrMcp, "create-collection", Map.of("name", COLLECTION)); + callTool( + solrMcp, + "index-json-documents", + Map.of( + "collection", + COLLECTION, + "documents", + TestDocuments.json(Files.readString(SESSIONS)))); + }; + } + + /** Calls a tool outside any model run, failing if the tool reports an error. */ + static CallToolResult callTool(McpSyncClient client, String tool, Map arguments) { + CallToolResult result = client.callTool(new CallToolRequest(tool, arguments)); + if (Boolean.TRUE.equals(result.isError())) { + throw new IllegalStateException(tool + " failed: " + result.content()); + } + return result; + } +} diff --git a/src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisor.java b/src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisor.java new file mode 100644 index 00000000..b25ad443 --- /dev/null +++ b/src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisor.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mcp.server.eval; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.springaicommunity.typesafe.judge.JevJudgeInput.ToolCall; +import org.springframework.ai.chat.client.ChatClientRequest; +import org.springframework.ai.chat.client.ChatClientResponse; +import org.springframework.ai.chat.client.advisor.ToolCallingAdvisor; +import org.springframework.ai.chat.client.advisor.api.CallAdvisor; +import org.springframework.ai.chat.client.advisor.api.CallAdvisorChain; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.ToolResponseMessage.ToolResponse; + +/** + * Records every tool call the model makes during a request, with its arguments and result. Read + * them back with {@link #toolCalls(ChatClientResponse)}. + * + *

It observes the tool-calling loop from inside, ordered after {@link ToolCallingAdvisor}, so it + * runs on every iteration. Each iteration's request carries the turn so far, so when the model + * answers without asking for another tool, that request holds every tool call and the {@link + * ToolResponseMessage} that answered it. + */ +final class ToolCallAuditAdvisor implements CallAdvisor { + + static final int ORDER = ToolCallingAdvisor.DEFAULT_ORDER + 100; + + private static final String TOOL_CALLS = ToolCallAuditAdvisor.class.getName() + ".toolCalls"; + + @Override + public String getName() { + return ToolCallAuditAdvisor.class.getSimpleName(); + } + + @Override + public int getOrder() { + return ORDER; + } + + @Override + public ChatClientResponse adviseCall(ChatClientRequest request, CallAdvisorChain chain) { + ChatClientResponse response = chain.nextCall(request); + if (response.chatResponse() != null && response.chatResponse().hasToolCalls()) { + return response; + } + return response + .mutate() + .context(TOOL_CALLS, toolCalls(request.prompt().getInstructions())) + .build(); + } + + /** The tool calls this advisor recorded for a response, in order. */ + @SuppressWarnings("unchecked") + static List toolCalls(ChatClientResponse response) { + return (List) + Objects.requireNonNull( + response.context().get(TOOL_CALLS), "ToolCallAuditAdvisor is not on this ChatClient"); + } + + /** + * Pairs each tool call the model asked for with the result that answered it. In the history, a + * call is a tool call on an {@link AssistantMessage}, and its result is the {@link ToolResponse} + * with the same id in a later {@link ToolResponseMessage}. + */ + private static List toolCalls(List history) { + Map resultById = new HashMap<>(); + for (Message message : history) { + if (message instanceof ToolResponseMessage toolResponses) { + for (ToolResponse response : toolResponses.getResponses()) { + resultById.put(response.id(), response.responseData()); + } + } + } + List toolCalls = new ArrayList<>(); + for (Message message : history) { + if (message instanceof AssistantMessage assistant) { + for (AssistantMessage.ToolCall call : assistant.getToolCalls()) { + String result = resultById.getOrDefault(call.id(), ""); + toolCalls.add(new ToolCall(call.name(), call.arguments(), result)); + } + } + } + return toolCalls; + } +} diff --git a/src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisorTest.java b/src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisorTest.java new file mode 100644 index 00000000..6ae01143 --- /dev/null +++ b/src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisorTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mcp.server.eval; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.springaicommunity.typesafe.judge.JevJudgeInput.ToolCall; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.model.tool.ToolCallingChatOptions; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; + +/** + * Runs a real {@link ChatClient} tool loop over a scripted model, so the advisor sees tool calls + * the way it does in an eval. + */ +class ToolCallAuditAdvisorTest { + + private static final String DEVNEXUS = + """ + {"collection": "devnexus"} + """; + + @Test + void recordsEveryToolCallInOrder() { + var model = + new ScriptedChatModel( + toolCall("list-collections", "{}"), + toolCall("search", DEVNEXUS), + answer("There are three workshops.")); + ChatClient chatClient = + ChatClient.builder(model) + .defaultTools( + tool("list-collections", "[\"devnexus\"]"), tool("search", "{\"numFound\": 3}")) + .defaultAdvisors(new ToolCallAuditAdvisor()) + .build(); + + var response = chatClient.prompt("List the workshops.").call().chatClientResponse(); + + assertThat(ToolCallAuditAdvisor.toolCalls(response)) + .containsExactly( + new ToolCall("list-collections", "{}", "[\"devnexus\"]"), + new ToolCall("search", DEVNEXUS, "{\"numFound\": 3}")); + } + + @Test + void recordsNoToolCallsWhenTheModelAnswersDirectly() { + var model = new ScriptedChatModel(answer("I can't tell without searching.")); + ChatClient chatClient = + ChatClient.builder(model) + .defaultTools(tool("search", "{}")) + .defaultAdvisors(new ToolCallAuditAdvisor()) + .build(); + + var response = chatClient.prompt("List the workshops.").call().chatClientResponse(); + + assertThat(ToolCallAuditAdvisor.toolCalls(response)).isEmpty(); + } + + private static ChatResponse toolCall(String tool, String arguments) { + var call = new AssistantMessage.ToolCall(tool + "-id", "function", tool, arguments); + AssistantMessage message = + AssistantMessage.builder().content("").toolCalls(List.of(call)).build(); + return new ChatResponse(List.of(new Generation(message))); + } + + private static ChatResponse answer(String text) { + return new ChatResponse(List.of(new Generation(new AssistantMessage(text)))); + } + + private static ToolCallback tool(String name, String result) { + ToolDefinition definition = + DefaultToolDefinition.builder().name(name).description(name).inputSchema("{}").build(); + return new ToolCallback() { + @Override + public ToolDefinition getToolDefinition() { + return definition; + } + + @Override + public String call(String toolInput) { + return result; + } + }; + } + + /** Returns its responses in order, one per model call. */ + private static final class ScriptedChatModel implements ChatModel { + + private final Deque responses; + + ScriptedChatModel(ChatResponse... responses) { + this.responses = new ArrayDeque<>(List.of(responses)); + } + + @Override + public ChatResponse call(Prompt prompt) { + return responses.removeFirst(); + } + + @Override + public ChatOptions getOptions() { + return ToolCallingChatOptions.builder().build(); + } + } +} diff --git a/src/test/java/org/apache/solr/mcp/server/eval/package-info.java b/src/test/java/org/apache/solr/mcp/server/eval/package-info.java new file mode 100644 index 00000000..462a8401 --- /dev/null +++ b/src/test/java/org/apache/solr/mcp/server/eval/package-info.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Opt-in LLM evals: does a model, given only the server's tool descriptions, pick the right tool + * and arguments? Run with {@code ./gradlew eval}; see #256. + */ +@NullMarked +package org.apache.solr.mcp.server.eval; + +import org.jspecify.annotations.NullMarked; diff --git a/src/test/resources/application-integration-test.properties b/src/test/resources/application-integration-test.properties new file mode 100644 index 00000000..550fb42b --- /dev/null +++ b/src/test/resources/application-integration-test.properties @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Profile for the opt-in LLM evals (./gradlew eval). The agent uses Spring AI's +# OpenAI defaults and reads its key from OPENAI_API_KEY, so it is not configured here. + +# The eval context runs Spring AI's auto-configuration for its ChatClient and MCP +# client; without this it would also start an MCP STDIO server on the test JVM's +# stdin and stdout. +spring.ai.mcp.server.enabled=false + +# solr-mcp under test: the built jar (passed by the eval task), on the test JVM's +# own Java, against the Testcontainers Solr (solr.url, set by TestcontainersConfiguration). +spring.ai.mcp.client.stdio.connections.solr-mcp.command=${java.home}/bin/java +spring.ai.mcp.client.stdio.connections.solr-mcp.args=-jar,${solr.mcp.jar} +spring.ai.mcp.client.stdio.connections.solr-mcp.env[SOLR_URL]=${solr.url} +spring.ai.mcp.client.stdio.connections.solr-mcp.env[SPRING_DOCKER_COMPOSE_ENABLED]=false