Test failed Mastra classifier evaluations

Author: andreiborza

dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs

@@ -0,0 +1,48 @@
+import * as Sentry from '@sentry/node';
+import { Mastra } from '@mastra/core';
+import { Classifier } from '@mastra/core/classifier';
+import { Observability } from '@mastra/observability';
+import { SentryMastraExporter } from '@sentry/node';
+
+// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir.
+const evaluationModel = {
+  specificationVersion: 'v4',
+  provider: 'typesafe-ai.evaluation',
+  modelId: 'jev-latest',
+  supportedQuestionTypes: ['choice', 'score', 'boolean'],
+  async doEvaluate() {
+    throw new Error('evaluation failed');
+  },
+};
+
+async function run() {
+  const classifier = new Classifier({
+    id: 'request-classifier',
+    model: evaluationModel,
+    questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } },
+  });
+
+  const mastra = new Mastra({
+    classifiers: { classifier },
+    logger: false,
+    observability: new Observability({
+      configs: {
+        default: {
+          serviceName: 'mastra-test',
+          exporters: [new SentryMastraExporter()],
+        },
+      },
+    }),
+  });
+
+  await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => {
+    await mastra
+      .getClassifier('classifier')
+      .evaluate({ state: { message: 'I was charged twice.' }, maxRetries: 0 })
+      .catch(() => undefined);
+  });
+
+  await mastra.observability.shutdown();
+}
+
+run();

dev-packages/node-integration-tests/suites/tracing/mastra/test.ts

@@ -274,6 +274,32 @@ conditionalTest({ min: 22 })('Mastra integration', () => {
     MASTRA_CLASSIFIER_DEPENDENCIES,
   );
 
+  createEsmAndCjsTests(
+    __dirname,
+    'scenario-classifier-error.mjs',
+    'instrument-with-pii.mjs',
+    (createRunner, test) => {
+      test('ends a failed classifier evaluation span with an error status', async () => {
+        await createRunner()
+          .ignore('event')
+          .expect({ transaction: { transaction: 'mastra-test' } })
+          .expect({
+            span: container => {
+              expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']);
+
+              const evaluateSpan = container.items[0]!;
+              expect(evaluateSpan.status).toBe('error');
+              expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.');
+              expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
+            },
+          })
+          .start()
+          .completed();
+      });
+    },
+    MASTRA_CLASSIFIER_DEPENDENCIES,
+  );
+
   createEsmAndCjsTests(
     __dirname,
     'scenario-auto.mjs',