test(eval): add the harness and Gradle task for LLM tool-selection evals

build.gradle.kts

@@ -197,6 +197,7 @@ dependencies {
     errorprone(libs.nullaway)
 
     testImplementation(libs.bundles.test)
+    testImplementation(libs.bundles.eval)
     testRuntimeOnly(libs.junit.platform.launcher)
 }
 
@@ -252,6 +253,10 @@ tasks.withType<Test> {
         if (name != "dockerIntegrationTest") {
             excludeTags("docker-integration")
         }
+        // LLM evals call a hosted model, so only ./gradlew eval runs them
+        if (name != "eval") {
+            excludeTags("llm-eval")
+        }
     }
     // McpClientStdioIntegrationTest spawns `java -jar build/libs/<bootJar>` as a
     // subprocess. Without an explicit dependency, `:test` runs before `:bootJar`
@@ -263,7 +268,7 @@ tasks.withType<Test> {
     // Forward the Testcontainers image pins (or a per-run -D override) to test JVMs.
     systemProperty("solr.test.image", solrTestImage)
     systemProperty("lgtm.test.image", lgtmTestImage)
-    if (name != "dockerIntegrationTest") {
+    if (name != "dockerIntegrationTest" && name != "eval") {
         finalizedBy(tasks.jacocoTestReport)
     }
 }
@@ -417,6 +422,38 @@ changelog {
     groups = listOf("Added", "Changed", "Fixed", "Security")
 }
 
+// LLM Tool-Selection Evals
+// ========================
+// Opt-in: checks whether a model, given only the server's tool descriptions, picks the
+// right tool and arguments (https://github.com/apache/solr-mcp/issues/256). Needs Docker
+// and OPENAI_API_KEY; ./gradlew build never runs it. The agent uses Spring AI's OpenAI
+// defaults, so nothing is configured here.
+//
+//   ./gradlew eval
+tasks.register<Test>("eval") {
+    description = "Runs the LLM tool-selection evals (needs OPENAI_API_KEY)"
+    group = "verification"
+
+    useJUnitPlatform {
+        includeTags("llm-eval")
+    }
+
+    testClassesDirs = sourceSets["test"].output.classesDirs
+    classpath = sourceSets["test"].runtimeClasspath
+
+    // Model output varies between runs, so never reuse a previous result
+    outputs.upToDateWhen { false }
+
+    // The jar the MCP client starts (application-integration-test.properties)
+    val bootJarFile = tasks.bootJar.flatMap { it.archiveFile }
+    doFirst { systemProperty("solr.mcp.jar", bootJarFile.get().asFile.absolutePath) }
+
+    reports {
+        html.outputLocation.set(layout.buildDirectory.dir("reports/eval"))
+        junitXml.outputLocation.set(layout.buildDirectory.dir("test-results/eval"))
+    }
+}
+
 // Docker Integration Test Task
 // =============================
 // Runs integration tests against the appropriate Docker image:

gradle/libs.versions.toml

@@ -29,6 +29,8 @@ changelog-plugin = "2.5.0"
 spring-ai = "2.0.1"
 solr = "10.0.0"
 mcp-server-security = "0.1.14"
+# spring-ai-typesafe: the judge and its input types, for the opt-in evals (test classpath only)
+typesafe = "0.4.0"
 # MCP Java SDK. Spring AI 2.0.1 pulls 2.0.0 transitively; 2.0.1 fixes error
 # handling when a handler is not registered on a stateless server (our HTTP
 # profile), empty-completion handling for unmatched refs, and adds bounded
@@ -74,6 +76,10 @@ spring-boot-testcontainers = { module = "org.springframework.boot:spring-boot-te
 spring-ai-starter-mcp-server-webmvc = { module = "org.springframework.ai:spring-ai-starter-mcp-server-webmvc" }
 spring-ai-spring-boot-docker-compose = { module = "org.springframework.ai:spring-ai-spring-boot-docker-compose" }
 spring-ai-spring-boot-testcontainers = { module = "org.springframework.ai:spring-ai-spring-boot-testcontainers" }
+# Opt-in tool-selection evals (./gradlew eval); test classpath only
+spring-ai-starter-model-openai = { module = "org.springframework.ai:spring-ai-starter-model-openai" }
+spring-ai-starter-mcp-client = { module = "org.springframework.ai:spring-ai-starter-mcp-client" }
+typesafe-spring-ai = { module = "org.springaicommunity:typesafe-spring-ai", version.ref = "typesafe" }
 
 # MCP Java SDK (pinned ahead of what Spring AI 2.0.1 resolves; see [versions])
 mcp-bom = { module = "io.modelcontextprotocol.sdk:mcp-bom", version.ref = "mcp-sdk" }
@@ -146,6 +152,12 @@ test = [
     "jetty-util"
 ]
 
+eval = [
+    "spring-ai-starter-model-openai",
+    "spring-ai-starter-mcp-client",
+    "typesafe-spring-ai"
+]
+
 errorprone = [
     "errorprone-core",
     "nullaway"

src/test/java/org/apache/solr/mcp/server/eval/EvalConfiguration.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.mcp.server.eval;
+
+import io.modelcontextprotocol.client.McpSyncClient;
+import io.modelcontextprotocol.spec.McpSchema.CallToolRequest;
+import io.modelcontextprotocol.spec.McpSchema.CallToolResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.solr.mcp.server.TestDocuments;
+import org.apache.solr.mcp.server.TestcontainersConfiguration;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.mcp.SyncMcpToolCallbackProvider;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Import;
+
+/**
+ * The evals' Spring context. Auto-configuration supplies the model, with Spring AI's OpenAI
+ * defaults, and an MCP client that starts solr-mcp over STDIO, as configured in {@code
+ * application-integration-test.properties}. Solr runs in Testcontainers.
+ */
+@TestConfiguration(proxyBeanMethods = false)
+@EnableAutoConfiguration
+@Import(TestcontainersConfiguration.class)
+class EvalConfiguration {
+
+  static final String COLLECTION = "devnexus";
+
+  static final Path SESSIONS = Path.of("mydata/devnexus-2026.json");
+
+  /**
+   * The agent: the server's instructions and tools, and nothing else. {@link ToolCallAuditAdvisor}
+   * records the tool calls it makes.
+   */
+  @Bean
+  ChatClient chatClient(
+      ChatClient.Builder chatClientBuilder,
+      List<McpSyncClient> mcpClients,
+      SyncMcpToolCallbackProvider mcpTools) {
+    String instructions =
+        Objects.requireNonNull(
+            mcpClients.getFirst().getServerInstructions(), "solr-mcp sent no instructions");
+    return chatClientBuilder
+        .defaultSystem(instructions)
+        .defaultTools(mcpTools)
+        .defaultAdvisors(new ToolCallAuditAdvisor())
+        .build();
+  }
+
+  /** Loads the DevNexus 2026 sessions through the server's own tools. */
+  @Bean
+  ApplicationRunner indexSessions(List<McpSyncClient> mcpClients) {
+    return args -> {
+      McpSyncClient solrMcp = mcpClients.getFirst();
+      callTool(solrMcp, "create-collection", Map.of("name", COLLECTION));
+      callTool(
+          solrMcp,
+          "index-json-documents",
+          Map.of(
+              "collection",
+              COLLECTION,
+              "documents",
+              TestDocuments.json(Files.readString(SESSIONS))));
+    };
+  }
+
+  /** Calls a tool outside any model run, failing if the tool reports an error. */
+  static CallToolResult callTool(McpSyncClient client, String tool, Map<String, Object> arguments) {
+    CallToolResult result = client.callTool(new CallToolRequest(tool, arguments));
+    if (Boolean.TRUE.equals(result.isError())) {
+      throw new IllegalStateException(tool + " failed: " + result.content());
+    }
+    return result;
+  }
+}

src/test/java/org/apache/solr/mcp/server/eval/ToolCallAuditAdvisor.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.mcp.server.eval;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import org.springaicommunity.typesafe.judge.JevJudgeInput.ToolCall;
+import org.springframework.ai.chat.client.ChatClientRequest;
+import org.springframework.ai.chat.client.ChatClientResponse;
+import org.springframework.ai.chat.client.advisor.ToolCallingAdvisor;
+import org.springframework.ai.chat.client.advisor.api.CallAdvisor;
+import org.springframework.ai.chat.client.advisor.api.CallAdvisorChain;
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.messages.Message;
+import org.springframework.ai.chat.messages.ToolResponseMessage;
+import org.springframework.ai.chat.messages.ToolResponseMessage.ToolResponse;
+
+/**
+ * Records every tool call the model makes during a request, with its arguments and result. Read
+ * them back with {@link #toolCalls(ChatClientResponse)}.
+ *
+ * <p>It observes the tool-calling loop from inside, ordered after {@link ToolCallingAdvisor}, so it
+ * runs on every iteration. Each iteration's request carries the turn so far, so when the model
+ * answers without asking for another tool, that request holds every tool call and the {@link
+ * ToolResponseMessage} that answered it.
+ */
+final class ToolCallAuditAdvisor implements CallAdvisor {
+
+  static final int ORDER = ToolCallingAdvisor.DEFAULT_ORDER + 100;
+
+  private static final String TOOL_CALLS = ToolCallAuditAdvisor.class.getName() + ".toolCalls";
+
+  @Override
+  public String getName() {
+    return ToolCallAuditAdvisor.class.getSimpleName();
+  }
+
+  @Override
+  public int getOrder() {
+    return ORDER;
+  }
+
+  @Override
+  public ChatClientResponse adviseCall(ChatClientRequest request, CallAdvisorChain chain) {
+    ChatClientResponse response = chain.nextCall(request);
+    if (response.chatResponse() != null && response.chatResponse().hasToolCalls()) {
+      return response;
+    }
+    return response
+        .mutate()
+        .context(TOOL_CALLS, toolCalls(request.prompt().getInstructions()))
+        .build();
+  }
+
+  /** The tool calls this advisor recorded for a response, in order. */
+  @SuppressWarnings("unchecked")
+  static List<ToolCall> toolCalls(ChatClientResponse response) {
+    return (List<ToolCall>)
+        Objects.requireNonNull(
+            response.context().get(TOOL_CALLS), "ToolCallAuditAdvisor is not on this ChatClient");
+  }
+
+  /**
+   * Pairs each tool call the model asked for with the result that answered it. In the history, a
+   * call is a tool call on an {@link AssistantMessage}, and its result is the {@link ToolResponse}
+   * with the same id in a later {@link ToolResponseMessage}.
+   */
+  private static List<ToolCall> toolCalls(List<Message> history) {
+    Map<String, String> resultById = new HashMap<>();
+    for (Message message : history) {
+      if (message instanceof ToolResponseMessage toolResponses) {
+        for (ToolResponse response : toolResponses.getResponses()) {
+          resultById.put(response.id(), response.responseData());
+        }
+      }
+    }
+    List<ToolCall> toolCalls = new ArrayList<>();
+    for (Message message : history) {
+      if (message instanceof AssistantMessage assistant) {
+        for (AssistantMessage.ToolCall call : assistant.getToolCalls()) {
+          String result = resultById.getOrDefault(call.id(), "");
+          toolCalls.add(new ToolCall(call.name(), call.arguments(), result));
+        }
+      }
+    }
+    return toolCalls;
+  }
+}