From 40a0dc713d2cb3be18b201db310b120e283e4768 Mon Sep 17 00:00:00 2001
From: suraj kumar <157868021+suraj-k-umar@users.noreply.github.com>
Date: Mon, 27 Jul 2026 15:47:39 +0530
Subject: [PATCH] fix(tools/skill-evals): require exact boolean for negate in
 assertions

Why: spec.get("negate") used Python truthiness, so a string like
"false" (non-empty, therefore truthy) would silently invert a
security assertion that the author intended to leave un-negated.
This is a silent correctness bug with no error surfaced.

The fix reads the negate field into a local variable and raises
TypeError immediately if the value is not an exact bool.  Existing
callers that already pass true/false are unaffected.

Three new tests cover the three required cases:
- negate: true  -> result is inverted
- negate: false -> result is unchanged
- non-boolean   -> TypeError is raised with a clear message
---
 tools/skill-evals/src/skill_evals/runner.py | 11 ++++++-
 tools/skill-evals/tests/test_runner.py      | 35 +++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/tools/skill-evals/src/skill_evals/runner.py b/tools/skill-evals/src/skill_evals/runner.py
index f0218a8d9..698bf4cf6 100644
--- a/tools/skill-evals/src/skill_evals/runner.py
+++ b/tools/skill-evals/src/skill_evals/runner.py
@@ -714,9 +714,18 @@ def evaluate_deterministic_assertion(spec: dict, actual: object) -> tuple[bool |
     ``--passphrase`` token does not appear). Negation is applied only to a
     concrete True/False result; a spec/usage error (None) is passed through
     unchanged so the typo still fails loudly.
+
+    ``"negate"`` must be an exact boolean; any other type (e.g. the string
+    ``"false"``, a number, or a list) raises ``TypeError`` immediately so the
+    author sees a clear error rather than silently getting the wrong behavior.
     """
+    negate = spec.get("negate", False)
+    if not isinstance(negate, bool):
+        raise TypeError(
+            f"'negate' must be a boolean (true or false), got {type(negate).__name__!r}: {negate!r}"
+        )
     holds, note = _evaluate_deterministic_assertion_raw(spec, actual)
-    if spec.get("negate") and holds is not None:
+    if negate and holds is not None:
         holds = not holds
     return holds, note
 
diff --git a/tools/skill-evals/tests/test_runner.py b/tools/skill-evals/tests/test_runner.py
index 0f4725e05..d8646016f 100644
--- a/tools/skill-evals/tests/test_runner.py
+++ b/tools/skill-evals/tests/test_runner.py
@@ -1655,6 +1655,41 @@ def test_assert_negate_passes_spec_error_through():
     assert "pattern" in note
 
 
+def test_assert_negate_true_inverts_result():
+    # Explicit boolean True: the predicate result must be inverted.
+    spec = {"field": "body", "type": "contains", "substring": "present", "negate": True}
+    holds, _note = evaluate_deterministic_assertion(spec, {"body": "present"})
+    assert holds is False
+
+    holds, _note = evaluate_deterministic_assertion(spec, {"body": "absent"})
+    assert holds is True
+
+
+def test_assert_negate_false_does_not_invert_result():
+    # Explicit boolean False: the predicate result must be returned unchanged.
+    spec = {"field": "body", "type": "contains", "substring": "present", "negate": False}
+    holds, _note = evaluate_deterministic_assertion(spec, {"body": "present"})
+    assert holds is True
+
+    holds, _note = evaluate_deterministic_assertion(spec, {"body": "absent"})
+    assert holds is False
+
+
+def test_assert_negate_non_boolean_raises_type_error():
+    # Non-boolean values for 'negate' must raise TypeError immediately.
+    # The string "false" is a common mistake: it is truthy in Python, so
+    # without this guard it would silently invert the assertion.
+    for bad_value in ("false", "true", 0, 1, 0.1, [], {}):
+        spec = {
+            "field": "body",
+            "type": "contains",
+            "substring": "x",
+            "negate": bad_value,
+        }
+        with pytest.raises(TypeError, match="'negate' must be a boolean"):
+            evaluate_deterministic_assertion(spec, {"body": "x"})
+
+
 # ---------------------------------------------------------------------------
 # Structural assertions: batch_judge_assertions
 # ---------------------------------------------------------------------------