diff --git a/tools/skill-evals/src/skill_evals/runner.py b/tools/skill-evals/src/skill_evals/runner.py index f0218a8d9..698bf4cf6 100644 --- a/tools/skill-evals/src/skill_evals/runner.py +++ b/tools/skill-evals/src/skill_evals/runner.py @@ -714,9 +714,18 @@ def evaluate_deterministic_assertion(spec: dict, actual: object) -> tuple[bool | ``--passphrase`` token does not appear). Negation is applied only to a concrete True/False result; a spec/usage error (None) is passed through unchanged so the typo still fails loudly. + + ``"negate"`` must be an exact boolean; any other type (e.g. the string + ``"false"``, a number, or a list) raises ``TypeError`` immediately so the + author sees a clear error rather than silently getting the wrong behavior. """ + negate = spec.get("negate", False) + if not isinstance(negate, bool): + raise TypeError( + f"'negate' must be a boolean (true or false), got {type(negate).__name__!r}: {negate!r}" + ) holds, note = _evaluate_deterministic_assertion_raw(spec, actual) - if spec.get("negate") and holds is not None: + if negate and holds is not None: holds = not holds return holds, note diff --git a/tools/skill-evals/tests/test_runner.py b/tools/skill-evals/tests/test_runner.py index 0f4725e05..d8646016f 100644 --- a/tools/skill-evals/tests/test_runner.py +++ b/tools/skill-evals/tests/test_runner.py @@ -1655,6 +1655,41 @@ def test_assert_negate_passes_spec_error_through(): assert "pattern" in note +def test_assert_negate_true_inverts_result(): + # Explicit boolean True: the predicate result must be inverted. + spec = {"field": "body", "type": "contains", "substring": "present", "negate": True} + holds, _note = evaluate_deterministic_assertion(spec, {"body": "present"}) + assert holds is False + + holds, _note = evaluate_deterministic_assertion(spec, {"body": "absent"}) + assert holds is True + + +def test_assert_negate_false_does_not_invert_result(): + # Explicit boolean False: the predicate result must be returned unchanged. + spec = {"field": "body", "type": "contains", "substring": "present", "negate": False} + holds, _note = evaluate_deterministic_assertion(spec, {"body": "present"}) + assert holds is True + + holds, _note = evaluate_deterministic_assertion(spec, {"body": "absent"}) + assert holds is False + + +def test_assert_negate_non_boolean_raises_type_error(): + # Non-boolean values for 'negate' must raise TypeError immediately. + # The string "false" is a common mistake: it is truthy in Python, so + # without this guard it would silently invert the assertion. + for bad_value in ("false", "true", 0, 1, 0.1, [], {}): + spec = { + "field": "body", + "type": "contains", + "substring": "x", + "negate": bad_value, + } + with pytest.raises(TypeError, match="'negate' must be a boolean"): + evaluate_deterministic_assertion(spec, {"body": "x"}) + + # --------------------------------------------------------------------------- # Structural assertions: batch_judge_assertions # ---------------------------------------------------------------------------