feat(evaluations): return tool errors to the model and mark eval spans for viewers

packages/client/src/__tests__/evaluations-spans.test.ts

@@ -294,9 +294,9 @@ describe('a plain function that calls a tool, with one judge, capture on', () =>
     }
   });
 
-  it('names no gen_ai operation and no evaluation on the row root', () => {
+  it('marks the row root as an invoke_workflow envelope that names no evaluation', () => {
     const root = only('launchdarkly.dataset_row');
-    expect(root.attributes['gen_ai.operation.name']).toBeUndefined();
+    expect(root.attributes['gen_ai.operation.name']).toBe('invoke_workflow');
     expect(root.name).not.toMatch(/evaluat/);
   });
 
@@ -384,9 +384,12 @@ describe('a plain function that calls a tool, with one judge, capture on', () =>
     expect(events[0].attributes).toEqual({
       'gen_ai.evaluation.name': 'accuracy',
       'gen_ai.evaluation.score.value': 0.9,
+      'launchdarkly.evaluation.score.min': 0,
+      'launchdarkly.evaluation.score.max': 1,
       'gen_ai.evaluation.explanation': 'Names the right capital.',
     });
     expect(evaluate.attributes['gen_ai.evaluation.score.value']).toBe(0.9);
+    expect(evaluate.attributes['launchdarkly.evaluation.score.max']).toBe(1);
     for (const key of Object.keys(evaluate.attributes)) {
       expect(key).not.toMatch(/score\.label|verdict|passed|success_?direction/i);
     }
@@ -449,6 +452,8 @@ describe('content capture follows the one gate', () => {
     expect(resultEvent(only('evaluate accuracy'))?.attributes).toEqual({
       'gen_ai.evaluation.name': 'accuracy',
       'gen_ai.evaluation.score.value': 0.9,
+      'launchdarkly.evaluation.score.min': 0,
+      'launchdarkly.evaluation.score.max': 1,
       'gen_ai.evaluation.explanation': 'Names the right capital.',
     });
     // A non-evaluation span written with the same resolved default.
@@ -462,10 +467,12 @@ describe('content capture follows the one gate', () => {
     await runWith({ handlers: [plain('OpenAI', callsTool), judgeHandler()] });
     expect(spans()).toHaveLength(5);
     for (const span of spans()) expect(contentOn(span), span.name).toEqual([]);
-    // Name and score stay.
+    // Name, score and range stay.
     expect(resultEvent(only('evaluate accuracy'))?.attributes).toEqual({
       'gen_ai.evaluation.name': 'accuracy',
       'gen_ai.evaluation.score.value': 0.9,
+      'launchdarkly.evaluation.score.min': 0,
+      'launchdarkly.evaluation.score.max': 1,
     });
     // The model, usage and tool name are not content.
     expect(only('chat gpt-4o').attributes['gen_ai.request.model']).toBe('gpt-4o');
@@ -580,6 +587,8 @@ describe('criterion spans', () => {
     expect(resultEvent(evaluate)?.attributes).toEqual({
       'gen_ai.evaluation.name': 'exact',
       'gen_ai.evaluation.score.value': 1,
+      'launchdarkly.evaluation.score.min': 0,
+      'launchdarkly.evaluation.score.max': 1,
     });
   });
 });
@@ -593,19 +602,18 @@ describe('a failed tool call', () => {
       },
       {},
     );
+    let received: unknown;
     const generation = plain(
       'OpenAI',
       async (_config, _input, tools) => {
-        try {
-          await (tools?.lookup_capital as (args: unknown) => Promise<string>)({ country: 'Atlantis' });
-        } catch {
-          // The handler recovers and answers anyway.
-        }
+        // The model gets the error as the tool result and answers anyway.
+        received = (tools?.lookup_capital as (args: unknown) => unknown)({ country: 'Atlantis' });
         return { output: 'I do not know.' };
       },
       true,
     );
     await runWith({ handlers: [generation], tools: [failing], criteria: [] });
+    expect(received).toBe('Error: Error: no such country');
     const tool = only('execute_tool lookup_capital');
     expect(tool.status.code).toBe(2);
     expect(tool.attributes['error.type']).toBe('tool_error');

packages/client/src/__tests__/evaluations-tool-errors.test.ts

@@ -0,0 +1,64 @@
+/** A throwing tool in an evaluation answers the model with an error result. */
+
+import { describe, expect, it } from 'vitest';
+import { returnToolErrors, toolErrorResult } from '../evaluations-tools.js';
+import { TrajectoryRecorder } from '../trajectory.js';
+import { NativeTool } from '../types.js';
+
+type Fn = (...args: unknown[]) => unknown;
+
+class RangeErrorish extends Error {}
+
+const noStation = (args: { city: string }) => {
+  throw new TypeError(`No weather station for '${args.city}'`);
+};
+
+describe('returnToolErrors', () => {
+  it('returns a sync throw as the result, stays sync, and the recorder still records the error', () => {
+    const recorder = new TrajectoryRecorder();
+    const tools = returnToolErrors(recorder.wrap({ get_weather: noStation }));
+
+    const result = (tools.get_weather as Fn)({ city: 'Reykjavik' });
+
+    expect(result).toBe("Error: TypeError: No weather station for 'Reykjavik'");
+    expect(recorder.calls).toHaveLength(1);
+    expect(recorder.calls[0]).toMatchObject({ error: "No weather station for 'Reykjavik'" });
+    expect(recorder.calls[0].result).toBeUndefined();
+  });
+
+  it('returns an async rejection as the result', async () => {
+    const recorder = new TrajectoryRecorder();
+    const tools = returnToolErrors(
+      recorder.wrap({
+        get_weather: async (args: { city: string }) => {
+          throw new RangeErrorish(`No weather station for '${args.city}'`);
+        },
+      }),
+    );
+
+    const result = (tools.get_weather as Fn)({ city: 'Reykjavik' });
+
+    expect(result).toBeInstanceOf(Promise);
+    expect(await result).toBe("Error: RangeErrorish: No weather station for 'Reykjavik'");
+    expect(recorder.calls[0]).toMatchObject({ error: "No weather station for 'Reykjavik'", done: true });
+  });
+
+  it('leaves a working tool and a native tool unchanged', async () => {
+    const native = new NativeTool(Symbol('search'), 'WebSearch');
+    const tools = returnToolErrors({
+      get_weather: (args: { city: string }) => ({ city: args.city, temp_c: 14 }),
+      later: async () => 'done',
+      search: native,
+    });
+
+    expect((tools.get_weather as Fn)({ city: 'Tokyo' })).toEqual({ city: 'Tokyo', temp_c: 14 });
+    expect(await (tools.later as Fn)()).toBe('done');
+    expect(tools.search).toBe(native);
+  });
+});
+
+describe('toolErrorResult', () => {
+  it('names a non-Error throw by its type', () => {
+    expect(toolErrorResult('quota exceeded')).toBe('Error: string: quota exceeded');
+  });
+});

packages/client/src/conversation.ts

@@ -161,15 +161,26 @@ export function countSdkSpans<T>(fn: (counter: SpanCounter) => T): T {
 /**
  * Adds `gen_ai.evaluation.result` to `span`, with the name, the score and, when given, the
  * explanation. With `mirrorAttributes`, the same values are also written as span attributes.
+ * `scoreRange` adds `launchdarkly.evaluation.score.min` and `.max`, so a reader can show the
+ * score as a fraction of its range rather than a bare number.
  *
  * Not part of the public API — shared by every path that records a judge's score.
  */
-export const recordEvaluationOnSpan = (span: Span, evaluation: JudgeEvaluation, mirrorAttributes: boolean): void => {
+export const recordEvaluationOnSpan = (
+  span: Span,
+  evaluation: JudgeEvaluation,
+  mirrorAttributes: boolean,
+  scoreRange?: readonly [min: number, max: number],
+): void => {
   if (!span.isRecording()) return;
   const attrs: Attributes = {
     'gen_ai.evaluation.name': evaluation.name,
     'gen_ai.evaluation.score.value': evaluation.score,
   };
+  if (scoreRange) {
+    attrs['launchdarkly.evaluation.score.min'] = scoreRange[0];
+    attrs['launchdarkly.evaluation.score.max'] = scoreRange[1];
+  }
   if (evaluation.explanation) attrs['gen_ai.evaluation.explanation'] = evaluation.explanation;
   span.addEvent('gen_ai.evaluation.result', attrs);
   // A classifier records one event per label. Span attributes hold a single name and score,

packages/client/src/evaluations-runner.ts

@@ -29,7 +29,7 @@ import {
   withActiveSpan,
   withEvaluationRow,
 } from './evaluations-spans.js';
-import { createBodyTools, type EvalTool, handlerConfigTools } from './evaluations-tools.js';
+import { createBodyTools, type EvalTool, handlerConfigTools, returnToolErrors } from './evaluations-tools.js';
 import { noteAiInvocation } from './sdk-options.js';
 import { TrajectoryRecorder } from './trajectory.js';
 import type {
@@ -562,7 +562,7 @@ export function generateRows(
     // One recorder per row: rows share the tool map, so a shared recorder
     // would splice their calls together.
     const trajectory = new TrajectoryRecorder();
-    const rowTools = trajectory.wrap(tools);
+    const rowTools = returnToolErrors(trajectory.wrap(tools));
     const span = evaluationRunId === undefined ? undefined : startRowSpan(evaluationRunId, row.rowIndex);
     const startedAt = new Date().toISOString();
     const started = performance.now();

packages/client/src/evaluations-spans.ts

@@ -131,9 +131,15 @@ export const SPAN_FLUSH_BOUND = 500;
 /** Ends a span started by this module. */
 export const endSpan = (span: Span, endTime = spanTimestamp()): void => span.end(endTime);
 
-/** Opens the row's root. The caller ends it with {@link endSpan} after the row's criteria. */
+/**
+ * Opens the row's root. The caller ends it with {@link endSpan} after the row's criteria.
+ *
+ * `invoke_workflow` marks it as the run's envelope, so a viewer reads a failed row as a
+ * failed run even when every span under it succeeded.
+ */
 export function startRowSpan(runId: string, rowIndex: number): Span {
   const span = startSpan(ROW_SPAN_NAME, undefined);
+  span.setAttribute('gen_ai.operation.name', 'invoke_workflow');
   span.setAttributes(rowAttributes(runId, rowIndex));
   return span;
 }
@@ -380,7 +386,8 @@ export type CriterionOutcome = {
 export function finishCriterionSpan(span: Span, name: string, outcome: CriterionOutcome, capture: boolean): void {
   if (outcome.status === 'COMPLETE' && typeof outcome.score === 'number') {
     const explanation = capture && outcome.reason ? outcome.reason : undefined;
-    recordEvaluationOnSpan(span, { name, score: outcome.score, explanation }, true);
+    // Every criterion scores from 0 to 1, so the event carries that range.
+    recordEvaluationOnSpan(span, { name, score: outcome.score, explanation }, true, [0, 1]);
     span.setStatus({ code: SpanStatusCode.OK });
     return;
   }

packages/client/src/evaluations-tools.ts

@@ -84,7 +84,11 @@ export class EvalTool {
    * Define an inline tool.
    *
    * @param key - The tool key. Must be lowercase.
-   * @param implementation - The function the handler calls.
+   * @param implementation - The function the handler calls, sync or async. It receives one
+   *   argument: an object of the arguments the model sent, keyed as in `schema`. Write
+   *   `(args) => getWeather(args.city)`, not `(city) => getWeather(city)`. What it returns
+   *   goes back to the model as the tool result. If it throws, the model receives
+   *   `"Error: <type>: <message>"` as the result and the row continues.
    * @param schema - JSON Schema for the tool's parameters. Required; `{}` is valid.
    * @param description - Description passed to the model. Defaults to `''`.
    */
@@ -213,6 +217,50 @@ export function toolHandlerMap(tools: readonly EvalTool[]): Record<string, ToolH
   return Object.fromEntries(tools.map((tool) => [tool.key, tool.implementation as ToolHandlerFn | NativeTool]));
 }
 
+/** The tool result a model receives when its tool call threw. */
+export function toolErrorResult(err: unknown): string {
+  const type = err instanceof Error ? err.constructor.name || err.name : typeof err;
+  return `Error: ${type}: ${errorMessage(err)}`;
+}
+
+/**
+ * Returns `handlers` with each throwing call answered by an error result.
+ *
+ * An evaluation measures how an agent copes with a failing tool, so the error
+ * goes back to the model as the tool's result instead of failing the row. Wrap
+ * outside the trajectory recorder: it still records the call as an error, for
+ * the judge and the `execute_tool` span.
+ */
+export function returnToolErrors(
+  handlers: Record<string, ToolHandlerFn | NativeTool>,
+): Record<string, ToolHandlerFn | NativeTool> {
+  const wrapped: Record<string, ToolHandlerFn | NativeTool> = {};
+  for (const [key, implementation] of Object.entries(handlers)) {
+    wrapped[key] =
+      implementation instanceof NativeTool || typeof implementation !== 'function'
+        ? implementation
+        : answerErrors(implementation as (...args: unknown[]) => unknown);
+  }
+  return wrapped;
+}
+
+function answerErrors(implementation: (...args: unknown[]) => unknown): ToolHandlerFn {
+  // A sync tool stays sync, as in the trajectory recorder: a handler may call
+  // it directly and use the value.
+  return function answered(this: unknown, ...args: unknown[]): unknown {
+    let result: unknown;
+    try {
+      result = implementation.apply(this, args);
+    } catch (err) {
+      return toolErrorResult(err);
+    }
+    if (typeof (result as PromiseLike<unknown> | undefined)?.then === 'function') {
+      return Promise.resolve(result).catch(toolErrorResult);
+    }
+    return result;
+  } as ToolHandlerFn;
+}
+
 /** The `tools` entry of a handler config. One shape for both sources. */
 export function handlerConfigTools(
   tools: readonly EvalTool[],