diff --git a/src/strands_evals/evaluators/__init__.py b/src/strands_evals/evaluators/__init__.py index 870509ae..5c1f7aac 100644 --- a/src/strands_evals/evaluators/__init__.py +++ b/src/strands_evals/evaluators/__init__.py @@ -1,7 +1,7 @@ from .coherence_evaluator import CoherenceEvaluator from .conciseness_evaluator import ConcisenessEvaluator from .correctness_evaluator import CorrectnessEvaluator -from .deterministic import Contains, Equals, SkillInvoked, StartsWith, StateEquals, ToolCalled +from .deterministic import Contains, Equals, FuzzyEquals, SkillInvoked, StartsWith, StateEquals, ToolCalled from .evaluator import Evaluator from .faithfulness_evaluator import FaithfulnessEvaluator from .goal_success_rate_evaluator import GoalSuccessRateEvaluator @@ -52,6 +52,7 @@ "InstructionFollowingEvaluator", "Contains", "Equals", + "FuzzyEquals", "StartsWith", "StateEquals", "ToolCalled", diff --git a/src/strands_evals/evaluators/deterministic/__init__.py b/src/strands_evals/evaluators/deterministic/__init__.py index 8a44d549..cd7c90fe 100644 --- a/src/strands_evals/evaluators/deterministic/__init__.py +++ b/src/strands_evals/evaluators/deterministic/__init__.py @@ -1,4 +1,5 @@ from .environment_state import StateEquals +from .fuzzy import FuzzyEquals from .output import Contains, Equals, StartsWith from .skill_invoked import SkillInvoked from .trajectory import ToolCalled @@ -7,6 +8,7 @@ "SkillInvoked", "Contains", "Equals", + "FuzzyEquals", "StartsWith", "StateEquals", "ToolCalled", diff --git a/src/strands_evals/evaluators/deterministic/fuzzy.py b/src/strands_evals/evaluators/deterministic/fuzzy.py new file mode 100644 index 00000000..d11507a4 --- /dev/null +++ b/src/strands_evals/evaluators/deterministic/fuzzy.py @@ -0,0 +1,227 @@ +from typing_extensions import Any + +from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT +from ..evaluator import Evaluator + +# Sentinel so an explicitly-passed ``value=None`` is distinguishable from "compare against +# ``expected_output``". ``Equals`` uses a plain ``None`` default and therefore cannot check an +# expected value that is itself ``None``; ``FuzzyEquals`` keeps that door open. +_UNSET = object() + + +class FuzzyEquals(Evaluator[InputT, OutputT]): + """Deterministic structural comparison with configurable tolerance. + + Exact matching (``Equals``) is too brittle for structured outputs such as SQL result sets, + JSON objects, and tabular data, where row order, key order, numeric precision, or whitespace + should not decide a pass. ``FuzzyEquals`` compares ``actual_output`` against an expected value + recursively across lists and dicts, applying the knobs below at every level. + + Attributes: + value: The expected value. When omitted, ``expected_output`` from the case is used. Pass + ``value=None`` explicitly to assert against a literal ``None``. + ignore_order: Treat lists as multisets so element order does not matter. + ignore_key_order: When ``False``, dict comparison additionally requires the same key + insertion order; when ``True`` (default) only the key/value pairs must match, matching + ordinary Python dict-equality semantics. + numeric_tolerance: Relative tolerance for numeric values, e.g. ``0.01`` accepts values + within +/-1%. ``0.0`` (default) requires exact numeric equality. + case_sensitive: When ``False``, strings are compared case-insensitively. + ignore_whitespace: When ``True``, leading/trailing whitespace is stripped and internal + runs of whitespace collapse to a single space before string comparison. + subset_mode: When ``True``, pass if the expected value is *contained* in the actual value + (``actual`` superset of ``expected``): expected dict keys must be present with matching + values (extra actual keys are ignored) and every expected list element must match a + distinct actual element. + type_coercion: When ``True``, numeric strings compare equal to numbers (``"42" == 42``) + and boolean strings compare equal to booleans (``"true" == True``). + """ + + def __init__( + self, + value: Any = _UNSET, + *, + ignore_order: bool = False, + ignore_key_order: bool = True, + numeric_tolerance: float = 0.0, + case_sensitive: bool = True, + ignore_whitespace: bool = False, + subset_mode: bool = False, + type_coercion: bool = False, + name: str | None = None, + ): + super().__init__(name=name) + self.value = value + self.ignore_order = ignore_order + self.ignore_key_order = ignore_key_order + self.numeric_tolerance = numeric_tolerance + self.case_sensitive = case_sensitive + self.ignore_whitespace = ignore_whitespace + self.subset_mode = subset_mode + self.type_coercion = type_coercion + + def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: + expected = self.value if self.value is not _UNSET else evaluation_case.expected_output + match, reason = self._compare(expected, evaluation_case.actual_output, path="$") + return [ + EvaluationOutput( + score=1.0 if match else 0.0, + test_pass=match, + reason="actual_output matches expected value" if match else reason, + ) + ] + + async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]: + return self.evaluate(evaluation_case) + + def to_dict(self) -> dict: + # ``value`` may be the private ``_UNSET`` sentinel; strip it so serialization stays clean. + _dict = super().to_dict() + if _dict.get("value") is _UNSET: + _dict.pop("value", None) + return _dict + + # --- comparison core --- + + def _compare(self, expected: Any, actual: Any, path: str) -> tuple[bool, str]: + """Recursively compare ``expected`` and ``actual``, returning (match, reason_on_fail).""" + if isinstance(expected, dict): + if not isinstance(actual, dict): + return False, f"at {path}: expected an object, got {type(actual).__name__}" + return self._compare_dicts(expected, actual, path) + + # Strings are sequences but must be compared as scalars, never element-wise. + if isinstance(expected, (list, tuple)) and not isinstance(expected, (str, bytes)): + if not isinstance(actual, (list, tuple)) or isinstance(actual, (str, bytes)): + return False, f"at {path}: expected a list, got {type(actual).__name__}" + return self._compare_lists(list(expected), list(actual), path) + + return self._compare_scalars(expected, actual, path) + + def _compare_dicts(self, expected: dict, actual: dict, path: str) -> tuple[bool, str]: + if self.subset_mode: + missing = [key for key in expected if key not in actual] + if missing: + return False, f"at {path}: missing expected key(s) {sorted(map(str, missing))}" + else: + if set(expected.keys()) != set(actual.keys()): + return False, ( + f"at {path}: key set differs " + f"(expected {sorted(map(str, expected))}, got {sorted(map(str, actual))})" + ) + if not self.ignore_key_order and list(expected.keys()) != list(actual.keys()): + return False, f"at {path}: key order differs" + + for key, expected_value in expected.items(): + match, reason = self._compare(expected_value, actual[key], f"{path}.{key}") + if not match: + return False, reason + return True, "" + + def _compare_lists(self, expected: list, actual: list, path: str) -> tuple[bool, str]: + if self.ignore_order: + return self._compare_lists_unordered(expected, actual, path) + + if self.subset_mode: + # Ordered containment: expected must appear as an in-order subsequence of actual. + actual_index = 0 + for expected_index, expected_value in enumerate(expected): + found = False + while actual_index < len(actual): + match, _ = self._compare(expected_value, actual[actual_index], f"{path}[{expected_index}]") + actual_index += 1 + if match: + found = True + break + if not found: + return False, f"at {path}: expected element at index {expected_index} not found in order" + return True, "" + + if len(expected) != len(actual): + return False, f"at {path}: length differs (expected {len(expected)}, got {len(actual)})" + for index, (expected_value, actual_value) in enumerate(zip(expected, actual, strict=False)): + match, reason = self._compare(expected_value, actual_value, f"{path}[{index}]") + if not match: + return False, reason + return True, "" + + def _compare_lists_unordered(self, expected: list, actual: list, path: str) -> tuple[bool, str]: + remaining = list(range(len(actual))) + for expected_index, expected_value in enumerate(expected): + matched_at = None + for pos, actual_index in enumerate(remaining): + match, _ = self._compare(expected_value, actual[actual_index], f"{path}[{expected_index}]") + if match: + matched_at = pos + break + if matched_at is None: + return False, f"at {path}: no match found for expected element at index {expected_index}" + remaining.pop(matched_at) + + if not self.subset_mode and remaining: + return False, f"at {path}: {len(remaining)} unexpected element(s) in actual" + return True, "" + + def _compare_scalars(self, expected: Any, actual: Any, path: str) -> tuple[bool, str]: + left, right = expected, actual + if self.type_coercion: + left = _coerce_scalar(left) + right = _coerce_scalar(right) + + # bool is an int subclass; compare booleans exactly and never numerically. A boolean only + # matches another boolean, so True != 1 even though Python's == says otherwise. + if isinstance(left, bool) or isinstance(right, bool): + match = isinstance(left, bool) and isinstance(right, bool) and left == right + return match, "" if match else f"at {path}: expected {expected!r}, got {actual!r}" + + if _is_number(left) and _is_number(right): + match = _numbers_close(float(left), float(right), self.numeric_tolerance) + return match, "" if match else f"at {path}: expected {expected!r}, got {actual!r}" + + if isinstance(left, str) and isinstance(right, str): + left_norm = self._normalize_string(left) + right_norm = self._normalize_string(right) + match = left_norm == right_norm + return match, "" if match else f"at {path}: expected {expected!r}, got {actual!r}" + + match = left == right + return match, "" if match else f"at {path}: expected {expected!r}, got {actual!r}" + + def _normalize_string(self, value: str) -> str: + result = value + if self.ignore_whitespace: + result = " ".join(result.split()) + if not self.case_sensitive: + result = result.lower() + return result + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _numbers_close(left: float, right: float, tolerance: float) -> bool: + if left == right: + return True + if tolerance <= 0.0: + return False + scale = max(abs(left), abs(right)) + return abs(left - right) <= tolerance * scale + + +def _coerce_scalar(value: Any) -> Any: + """Coerce numeric/boolean strings into their scalar types for cross-type comparison.""" + if not isinstance(value, str): + return value + stripped = value.strip() + lowered = stripped.lower() + if lowered in ("true", "false"): + return lowered == "true" + try: + return int(stripped) + except ValueError: + pass + try: + return float(stripped) + except ValueError: + return value diff --git a/tests/strands_evals/evaluators/deterministic/test_fuzzy.py b/tests/strands_evals/evaluators/deterministic/test_fuzzy.py new file mode 100644 index 00000000..2bc3a2e4 --- /dev/null +++ b/tests/strands_evals/evaluators/deterministic/test_fuzzy.py @@ -0,0 +1,298 @@ +import pytest + +from strands_evals.evaluators.deterministic.fuzzy import FuzzyEquals +from strands_evals.types import EvaluationData + + +class TestFuzzyEqualsExact: + def test_matches_expected_output(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output="Paris", expected_output="Paris") + results = evaluator.evaluate(data) + assert len(results) == 1 + assert results[0].score == 1.0 + assert results[0].test_pass is True + + def test_fails_when_actual_differs(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output="London", expected_output="Paris") + results = evaluator.evaluate(data) + assert results[0].score == 0.0 + assert results[0].test_pass is False + + def test_explicit_value_takes_precedence_over_expected_output(self): + evaluator = FuzzyEquals(value="Paris") + data = EvaluationData(input="q", actual_output="Paris", expected_output="London") + results = evaluator.evaluate(data) + assert results[0].test_pass is True + + def test_explicit_none_value_asserts_against_none(self): + evaluator = FuzzyEquals(value=None) + data = EvaluationData(input="q", actual_output=None, expected_output="ignored") + results = evaluator.evaluate(data) + assert results[0].test_pass is True + + def test_explicit_none_value_fails_non_none_actual(self): + evaluator = FuzzyEquals(value=None) + data = EvaluationData(input="q", actual_output="something") + results = evaluator.evaluate(data) + assert results[0].test_pass is False + + +class TestIgnoreOrder: + def test_list_order_ignored(self): + evaluator = FuzzyEquals(ignore_order=True) + data = EvaluationData(input="q", actual_output=[3, 1, 2], expected_output=[1, 2, 3]) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_list_order_matters_by_default(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output=[3, 1, 2], expected_output=[1, 2, 3]) + assert evaluator.evaluate(data)[0].test_pass is False + + def test_unordered_reports_extra_elements(self): + evaluator = FuzzyEquals(ignore_order=True) + data = EvaluationData(input="q", actual_output=[1, 2, 3], expected_output=[1, 2]) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "unexpected" in result.reason + + def test_unordered_nested_dicts(self): + evaluator = FuzzyEquals(ignore_order=True) + expected = [{"id": 1}, {"id": 2}] + actual = [{"id": 2}, {"id": 1}] + data = EvaluationData(input="q", actual_output=actual, expected_output=expected) + assert evaluator.evaluate(data)[0].test_pass is True + + +class TestIgnoreKeyOrder: + def test_key_order_ignored_by_default(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output={"b": 2, "a": 1}, expected_output={"a": 1, "b": 2}) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_key_order_enforced_when_disabled(self): + evaluator = FuzzyEquals(ignore_key_order=False) + data = EvaluationData(input="q", actual_output={"b": 2, "a": 1}, expected_output={"a": 1, "b": 2}) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "key order" in result.reason + + def test_key_order_match_when_disabled_and_ordered(self): + evaluator = FuzzyEquals(ignore_key_order=False) + data = EvaluationData(input="q", actual_output={"a": 1, "b": 2}, expected_output={"a": 1, "b": 2}) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_missing_key_reported(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output={"a": 1}, expected_output={"a": 1, "b": 2}) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "key set differs" in result.reason + + +class TestNumericTolerance: + def test_within_tolerance_passes(self): + evaluator = FuzzyEquals(numeric_tolerance=0.01) + data = EvaluationData(input="q", actual_output=3.14, expected_output=3.14159) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_outside_tolerance_fails(self): + evaluator = FuzzyEquals(numeric_tolerance=0.01) + data = EvaluationData(input="q", actual_output=3.0, expected_output=3.14159) + assert evaluator.evaluate(data)[0].test_pass is False + + def test_exact_required_by_default(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output=3.14, expected_output=3.14159) + assert evaluator.evaluate(data)[0].test_pass is False + + def test_tolerance_applies_nested(self): + evaluator = FuzzyEquals(numeric_tolerance=0.05) + data = EvaluationData( + input="q", + actual_output={"total": 100.0, "avg": 33.0}, + expected_output={"total": 102.0, "avg": 33.5}, + ) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_zero_values_exact_within_tolerance(self): + evaluator = FuzzyEquals(numeric_tolerance=0.1) + data = EvaluationData(input="q", actual_output=0.0, expected_output=0.0) + assert evaluator.evaluate(data)[0].test_pass is True + + +class TestCaseSensitivity: + def test_case_insensitive_passes(self): + evaluator = FuzzyEquals(case_sensitive=False) + data = EvaluationData(input="q", actual_output="PARIS", expected_output="paris") + assert evaluator.evaluate(data)[0].test_pass is True + + def test_case_sensitive_by_default(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output="PARIS", expected_output="paris") + assert evaluator.evaluate(data)[0].test_pass is False + + +class TestIgnoreWhitespace: + def test_leading_trailing_whitespace_ignored(self): + evaluator = FuzzyEquals(ignore_whitespace=True) + data = EvaluationData(input="q", actual_output=" hello ", expected_output="hello") + assert evaluator.evaluate(data)[0].test_pass is True + + def test_internal_whitespace_collapsed(self): + evaluator = FuzzyEquals(ignore_whitespace=True) + data = EvaluationData(input="q", actual_output="a\t b\n c", expected_output="a b c") + assert evaluator.evaluate(data)[0].test_pass is True + + def test_whitespace_matters_by_default(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output=" hello ", expected_output="hello") + assert evaluator.evaluate(data)[0].test_pass is False + + +class TestSubsetMode: + def test_dict_subset_passes(self): + evaluator = FuzzyEquals(subset_mode=True) + data = EvaluationData( + input="q", + actual_output={"a": 1, "b": 2, "c": 3}, + expected_output={"a": 1, "b": 2}, + ) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_dict_subset_fails_on_missing_key(self): + evaluator = FuzzyEquals(subset_mode=True) + data = EvaluationData(input="q", actual_output={"a": 1}, expected_output={"a": 1, "z": 9}) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "missing expected key" in result.reason + + def test_dict_subset_fails_on_wrong_value(self): + evaluator = FuzzyEquals(subset_mode=True) + data = EvaluationData(input="q", actual_output={"a": 2}, expected_output={"a": 1}) + assert evaluator.evaluate(data)[0].test_pass is False + + def test_ordered_list_subsequence_passes(self): + evaluator = FuzzyEquals(subset_mode=True) + data = EvaluationData(input="q", actual_output=[1, 2, 3, 4], expected_output=[2, 4]) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_ordered_list_subsequence_fails_when_out_of_order(self): + evaluator = FuzzyEquals(subset_mode=True) + data = EvaluationData(input="q", actual_output=[1, 2, 3, 4], expected_output=[4, 2]) + assert evaluator.evaluate(data)[0].test_pass is False + + def test_unordered_subset_list_passes(self): + evaluator = FuzzyEquals(subset_mode=True, ignore_order=True) + data = EvaluationData(input="q", actual_output=[3, 1, 2], expected_output=[2, 3]) + assert evaluator.evaluate(data)[0].test_pass is True + + +class TestTypeCoercion: + def test_numeric_string_matches_number(self): + evaluator = FuzzyEquals(type_coercion=True) + data = EvaluationData(input="q", actual_output="42", expected_output=42) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_float_string_matches_float(self): + evaluator = FuzzyEquals(type_coercion=True) + data = EvaluationData(input="q", actual_output="3.14", expected_output=3.14) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_bool_string_matches_bool(self): + evaluator = FuzzyEquals(type_coercion=True) + data = EvaluationData(input="q", actual_output="true", expected_output=True) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_no_coercion_by_default(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output="42", expected_output=42) + assert evaluator.evaluate(data)[0].test_pass is False + + def test_coercion_with_tolerance(self): + evaluator = FuzzyEquals(type_coercion=True, numeric_tolerance=0.01) + data = EvaluationData(input="q", actual_output="3.14", expected_output=3.14159) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_bool_not_matched_numerically(self): + # bool is an int subclass; True must not equal 1 under coercion. + evaluator = FuzzyEquals(type_coercion=True) + data = EvaluationData(input="q", actual_output=True, expected_output=1) + assert evaluator.evaluate(data)[0].test_pass is False + + +class TestTypeMismatch: + def test_dict_vs_list_fails(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output=[1, 2], expected_output={"a": 1}) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "expected an object" in result.reason + + def test_list_vs_scalar_fails(self): + evaluator = FuzzyEquals() + data = EvaluationData(input="q", actual_output="not-a-list", expected_output=[1, 2]) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "expected a list" in result.reason + + +class TestStructuredScenarios: + def test_sql_result_rows_unordered(self): + evaluator = FuzzyEquals(ignore_order=True, ignore_key_order=True) + expected = [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}] + actual = [{"name": "Bob", "id": 2}, {"name": "Alice", "id": 1}] + data = EvaluationData(input="q", actual_output=actual, expected_output=expected) + assert evaluator.evaluate(data)[0].test_pass is True + + def test_nested_json_all_knobs(self): + evaluator = FuzzyEquals( + ignore_order=True, + numeric_tolerance=0.01, + case_sensitive=False, + ignore_whitespace=True, + ) + expected = {"users": [{"name": "Alice", "score": 90.0}], "total": 100.0} + actual = {"total": 100.5, "users": [{"score": 90.5, "name": " ALICE "}]} + data = EvaluationData(input="q", actual_output=actual, expected_output=expected) + assert evaluator.evaluate(data)[0].test_pass is True + + +class TestReasonAndSerialization: + def test_reason_on_match(self): + evaluator = FuzzyEquals(value="Paris") + data = EvaluationData(input="q", actual_output="Paris") + assert "matches" in evaluator.evaluate(data)[0].reason + + def test_reason_includes_path(self): + evaluator = FuzzyEquals() + data = EvaluationData( + input="q", + actual_output={"a": {"b": 1}}, + expected_output={"a": {"b": 2}}, + ) + result = evaluator.evaluate(data)[0] + assert result.test_pass is False + assert "$.a.b" in result.reason + + def test_to_dict_records_non_default_knobs(self): + evaluator = FuzzyEquals(value=[1, 2], ignore_order=True, numeric_tolerance=0.01) + d = evaluator.to_dict() + assert d["evaluator_type"] == "FuzzyEquals" + assert d["value"] == [1, 2] + assert d["ignore_order"] is True + assert d["numeric_tolerance"] == 0.01 + + def test_to_dict_omits_unset_value(self): + evaluator = FuzzyEquals() + d = evaluator.to_dict() + assert "value" not in d + assert d["evaluator_type"] == "FuzzyEquals" + + @pytest.mark.asyncio + async def test_evaluate_async_delegates(self): + evaluator = FuzzyEquals(value="Paris") + data = EvaluationData(input="q", actual_output="Paris") + results = await evaluator.evaluate_async(data) + assert results[0].test_pass is True