diff --git a/packages/client/src/__tests__/evaluations-spans.test.ts b/packages/client/src/__tests__/evaluations-spans.test.ts index dba4865a..dd5476a7 100644 --- a/packages/client/src/__tests__/evaluations-spans.test.ts +++ b/packages/client/src/__tests__/evaluations-spans.test.ts @@ -294,9 +294,9 @@ describe('a plain function that calls a tool, with one judge, capture on', () => } }); - it('names no gen_ai operation and no evaluation on the row root', () => { + it('marks the row root as an invoke_workflow envelope that names no evaluation', () => { const root = only('launchdarkly.dataset_row'); - expect(root.attributes['gen_ai.operation.name']).toBeUndefined(); + expect(root.attributes['gen_ai.operation.name']).toBe('invoke_workflow'); expect(root.name).not.toMatch(/evaluat/); }); @@ -384,9 +384,12 @@ describe('a plain function that calls a tool, with one judge, capture on', () => expect(events[0].attributes).toEqual({ 'gen_ai.evaluation.name': 'accuracy', 'gen_ai.evaluation.score.value': 0.9, + 'launchdarkly.evaluation.score.min': 0, + 'launchdarkly.evaluation.score.max': 1, 'gen_ai.evaluation.explanation': 'Names the right capital.', }); expect(evaluate.attributes['gen_ai.evaluation.score.value']).toBe(0.9); + expect(evaluate.attributes['launchdarkly.evaluation.score.max']).toBe(1); for (const key of Object.keys(evaluate.attributes)) { expect(key).not.toMatch(/score\.label|verdict|passed|success_?direction/i); } @@ -449,6 +452,8 @@ describe('content capture follows the one gate', () => { expect(resultEvent(only('evaluate accuracy'))?.attributes).toEqual({ 'gen_ai.evaluation.name': 'accuracy', 'gen_ai.evaluation.score.value': 0.9, + 'launchdarkly.evaluation.score.min': 0, + 'launchdarkly.evaluation.score.max': 1, 'gen_ai.evaluation.explanation': 'Names the right capital.', }); // A non-evaluation span written with the same resolved default. @@ -462,10 +467,12 @@ describe('content capture follows the one gate', () => { await runWith({ handlers: [plain('OpenAI', callsTool), judgeHandler()] }); expect(spans()).toHaveLength(5); for (const span of spans()) expect(contentOn(span), span.name).toEqual([]); - // Name and score stay. + // Name, score and range stay. expect(resultEvent(only('evaluate accuracy'))?.attributes).toEqual({ 'gen_ai.evaluation.name': 'accuracy', 'gen_ai.evaluation.score.value': 0.9, + 'launchdarkly.evaluation.score.min': 0, + 'launchdarkly.evaluation.score.max': 1, }); // The model, usage and tool name are not content. expect(only('chat gpt-4o').attributes['gen_ai.request.model']).toBe('gpt-4o'); @@ -580,6 +587,8 @@ describe('criterion spans', () => { expect(resultEvent(evaluate)?.attributes).toEqual({ 'gen_ai.evaluation.name': 'exact', 'gen_ai.evaluation.score.value': 1, + 'launchdarkly.evaluation.score.min': 0, + 'launchdarkly.evaluation.score.max': 1, }); }); }); @@ -593,19 +602,18 @@ describe('a failed tool call', () => { }, {}, ); + let received: unknown; const generation = plain( 'OpenAI', async (_config, _input, tools) => { - try { - await (tools?.lookup_capital as (args: unknown) => Promise)({ country: 'Atlantis' }); - } catch { - // The handler recovers and answers anyway. - } + // The model gets the error as the tool result and answers anyway. + received = (tools?.lookup_capital as (args: unknown) => unknown)({ country: 'Atlantis' }); return { output: 'I do not know.' }; }, true, ); await runWith({ handlers: [generation], tools: [failing], criteria: [] }); + expect(received).toBe('Error: Error: no such country'); const tool = only('execute_tool lookup_capital'); expect(tool.status.code).toBe(2); expect(tool.attributes['error.type']).toBe('tool_error'); diff --git a/packages/client/src/__tests__/evaluations-tool-errors.test.ts b/packages/client/src/__tests__/evaluations-tool-errors.test.ts new file mode 100644 index 00000000..3aabeaba --- /dev/null +++ b/packages/client/src/__tests__/evaluations-tool-errors.test.ts @@ -0,0 +1,64 @@ +/** A throwing tool in an evaluation answers the model with an error result. */ + +import { describe, expect, it } from 'vitest'; +import { returnToolErrors, toolErrorResult } from '../evaluations-tools.js'; +import { TrajectoryRecorder } from '../trajectory.js'; +import { NativeTool } from '../types.js'; + +type Fn = (...args: unknown[]) => unknown; + +class RangeErrorish extends Error {} + +const noStation = (args: { city: string }) => { + throw new TypeError(`No weather station for '${args.city}'`); +}; + +describe('returnToolErrors', () => { + it('returns a sync throw as the result, stays sync, and the recorder still records the error', () => { + const recorder = new TrajectoryRecorder(); + const tools = returnToolErrors(recorder.wrap({ get_weather: noStation })); + + const result = (tools.get_weather as Fn)({ city: 'Reykjavik' }); + + expect(result).toBe("Error: TypeError: No weather station for 'Reykjavik'"); + expect(recorder.calls).toHaveLength(1); + expect(recorder.calls[0]).toMatchObject({ error: "No weather station for 'Reykjavik'" }); + expect(recorder.calls[0].result).toBeUndefined(); + }); + + it('returns an async rejection as the result', async () => { + const recorder = new TrajectoryRecorder(); + const tools = returnToolErrors( + recorder.wrap({ + get_weather: async (args: { city: string }) => { + throw new RangeErrorish(`No weather station for '${args.city}'`); + }, + }), + ); + + const result = (tools.get_weather as Fn)({ city: 'Reykjavik' }); + + expect(result).toBeInstanceOf(Promise); + expect(await result).toBe("Error: RangeErrorish: No weather station for 'Reykjavik'"); + expect(recorder.calls[0]).toMatchObject({ error: "No weather station for 'Reykjavik'", done: true }); + }); + + it('leaves a working tool and a native tool unchanged', async () => { + const native = new NativeTool(Symbol('search'), 'WebSearch'); + const tools = returnToolErrors({ + get_weather: (args: { city: string }) => ({ city: args.city, temp_c: 14 }), + later: async () => 'done', + search: native, + }); + + expect((tools.get_weather as Fn)({ city: 'Tokyo' })).toEqual({ city: 'Tokyo', temp_c: 14 }); + expect(await (tools.later as Fn)()).toBe('done'); + expect(tools.search).toBe(native); + }); +}); + +describe('toolErrorResult', () => { + it('names a non-Error throw by its type', () => { + expect(toolErrorResult('quota exceeded')).toBe('Error: string: quota exceeded'); + }); +}); diff --git a/packages/client/src/conversation.ts b/packages/client/src/conversation.ts index 724ff651..48135a6d 100644 --- a/packages/client/src/conversation.ts +++ b/packages/client/src/conversation.ts @@ -161,15 +161,26 @@ export function countSdkSpans(fn: (counter: SpanCounter) => T): T { /** * Adds `gen_ai.evaluation.result` to `span`, with the name, the score and, when given, the * explanation. With `mirrorAttributes`, the same values are also written as span attributes. + * `scoreRange` adds `launchdarkly.evaluation.score.min` and `.max`, so a reader can show the + * score as a fraction of its range rather than a bare number. * * Not part of the public API — shared by every path that records a judge's score. */ -export const recordEvaluationOnSpan = (span: Span, evaluation: JudgeEvaluation, mirrorAttributes: boolean): void => { +export const recordEvaluationOnSpan = ( + span: Span, + evaluation: JudgeEvaluation, + mirrorAttributes: boolean, + scoreRange?: readonly [min: number, max: number], +): void => { if (!span.isRecording()) return; const attrs: Attributes = { 'gen_ai.evaluation.name': evaluation.name, 'gen_ai.evaluation.score.value': evaluation.score, }; + if (scoreRange) { + attrs['launchdarkly.evaluation.score.min'] = scoreRange[0]; + attrs['launchdarkly.evaluation.score.max'] = scoreRange[1]; + } if (evaluation.explanation) attrs['gen_ai.evaluation.explanation'] = evaluation.explanation; span.addEvent('gen_ai.evaluation.result', attrs); // A classifier records one event per label. Span attributes hold a single name and score, diff --git a/packages/client/src/evaluations-runner.ts b/packages/client/src/evaluations-runner.ts index d080a489..baca5dbf 100644 --- a/packages/client/src/evaluations-runner.ts +++ b/packages/client/src/evaluations-runner.ts @@ -29,7 +29,7 @@ import { withActiveSpan, withEvaluationRow, } from './evaluations-spans.js'; -import { createBodyTools, type EvalTool, handlerConfigTools } from './evaluations-tools.js'; +import { createBodyTools, type EvalTool, handlerConfigTools, returnToolErrors } from './evaluations-tools.js'; import { noteAiInvocation } from './sdk-options.js'; import { TrajectoryRecorder } from './trajectory.js'; import type { @@ -562,7 +562,7 @@ export function generateRows( // One recorder per row: rows share the tool map, so a shared recorder // would splice their calls together. const trajectory = new TrajectoryRecorder(); - const rowTools = trajectory.wrap(tools); + const rowTools = returnToolErrors(trajectory.wrap(tools)); const span = evaluationRunId === undefined ? undefined : startRowSpan(evaluationRunId, row.rowIndex); const startedAt = new Date().toISOString(); const started = performance.now(); diff --git a/packages/client/src/evaluations-spans.ts b/packages/client/src/evaluations-spans.ts index 36d51a29..68326b23 100644 --- a/packages/client/src/evaluations-spans.ts +++ b/packages/client/src/evaluations-spans.ts @@ -131,9 +131,15 @@ export const SPAN_FLUSH_BOUND = 500; /** Ends a span started by this module. */ export const endSpan = (span: Span, endTime = spanTimestamp()): void => span.end(endTime); -/** Opens the row's root. The caller ends it with {@link endSpan} after the row's criteria. */ +/** + * Opens the row's root. The caller ends it with {@link endSpan} after the row's criteria. + * + * `invoke_workflow` marks it as the run's envelope, so a viewer reads a failed row as a + * failed run even when every span under it succeeded. + */ export function startRowSpan(runId: string, rowIndex: number): Span { const span = startSpan(ROW_SPAN_NAME, undefined); + span.setAttribute('gen_ai.operation.name', 'invoke_workflow'); span.setAttributes(rowAttributes(runId, rowIndex)); return span; } @@ -380,7 +386,8 @@ export type CriterionOutcome = { export function finishCriterionSpan(span: Span, name: string, outcome: CriterionOutcome, capture: boolean): void { if (outcome.status === 'COMPLETE' && typeof outcome.score === 'number') { const explanation = capture && outcome.reason ? outcome.reason : undefined; - recordEvaluationOnSpan(span, { name, score: outcome.score, explanation }, true); + // Every criterion scores from 0 to 1, so the event carries that range. + recordEvaluationOnSpan(span, { name, score: outcome.score, explanation }, true, [0, 1]); span.setStatus({ code: SpanStatusCode.OK }); return; } diff --git a/packages/client/src/evaluations-tools.ts b/packages/client/src/evaluations-tools.ts index 239dd703..695ac76e 100644 --- a/packages/client/src/evaluations-tools.ts +++ b/packages/client/src/evaluations-tools.ts @@ -84,7 +84,11 @@ export class EvalTool { * Define an inline tool. * * @param key - The tool key. Must be lowercase. - * @param implementation - The function the handler calls. + * @param implementation - The function the handler calls, sync or async. It receives one + * argument: an object of the arguments the model sent, keyed as in `schema`. Write + * `(args) => getWeather(args.city)`, not `(city) => getWeather(city)`. What it returns + * goes back to the model as the tool result. If it throws, the model receives + * `"Error: : "` as the result and the row continues. * @param schema - JSON Schema for the tool's parameters. Required; `{}` is valid. * @param description - Description passed to the model. Defaults to `''`. */ @@ -213,6 +217,50 @@ export function toolHandlerMap(tools: readonly EvalTool[]): Record [tool.key, tool.implementation as ToolHandlerFn | NativeTool])); } +/** The tool result a model receives when its tool call threw. */ +export function toolErrorResult(err: unknown): string { + const type = err instanceof Error ? err.constructor.name || err.name : typeof err; + return `Error: ${type}: ${errorMessage(err)}`; +} + +/** + * Returns `handlers` with each throwing call answered by an error result. + * + * An evaluation measures how an agent copes with a failing tool, so the error + * goes back to the model as the tool's result instead of failing the row. Wrap + * outside the trajectory recorder: it still records the call as an error, for + * the judge and the `execute_tool` span. + */ +export function returnToolErrors( + handlers: Record, +): Record { + const wrapped: Record = {}; + for (const [key, implementation] of Object.entries(handlers)) { + wrapped[key] = + implementation instanceof NativeTool || typeof implementation !== 'function' + ? implementation + : answerErrors(implementation as (...args: unknown[]) => unknown); + } + return wrapped; +} + +function answerErrors(implementation: (...args: unknown[]) => unknown): ToolHandlerFn { + // A sync tool stays sync, as in the trajectory recorder: a handler may call + // it directly and use the value. + return function answered(this: unknown, ...args: unknown[]): unknown { + let result: unknown; + try { + result = implementation.apply(this, args); + } catch (err) { + return toolErrorResult(err); + } + if (typeof (result as PromiseLike | undefined)?.then === 'function') { + return Promise.resolve(result).catch(toolErrorResult); + } + return result; + } as ToolHandlerFn; +} + /** The `tools` entry of a handler config. One shape for both sources. */ export function handlerConfigTools( tools: readonly EvalTool[],