fix(skill-evals): extend default prose_fields with grading-schema.json

tools/skill-evals/README.md

@@ -242,18 +242,17 @@ writes stdout works). Pass `--exact` to disable grading entirely.
 
 The default prose-field set is `rationale`, `reason`, `reasons`,
 `drop_reason`, `blockers`, `notes`, `summary`, `explanation`,
-`details`, `description`. Override it per fixtures dir by placing a
+`details`, `description`. Extend it per fixtures dir by placing a
 `grading-schema.json` next to `step-config.json`:
 
 ```json
 {
-  "prose_fields": ["rationale", "drop_reason"]
+  "prose_fields": ["custom_prose_field"]
 }
 ```
 
-An empty list (`"prose_fields": []`) makes every field decision-graded
-even with the grader on, equivalent to passing `--exact` for that
-fixtures dir. The grader is called fresh on every run; nothing is
+The fields listed in `grading-schema.json` are added to the default
+prose-field set. The grader is called fresh on every run; nothing is
 cached.
 
 **Self-eval caveat.** When the model invoked by `--cli` is the same

tools/skill-evals/src/skill_evals/runner.py

@@ -397,9 +397,7 @@ def load_grading_schema(fixtures_dir: Path) -> set[str]:
     """Return the set of prose field names for cases in this fixtures dir.
 
     Reads ``fixtures_dir/grading-schema.json`` when present. The file may
-    set ``prose_fields`` to a string list that *replaces* the default set
-    (use ``["rationale", "reason", ...]`` to be explicit, or ``[]`` to
-    grade everything by exact match).
+    set ``prose_fields`` to a string list extending the default set.
 
     Falls back to :data:`DEFAULT_PROSE_FIELDS` when no schema file exists.
     """
@@ -412,7 +410,7 @@ def load_grading_schema(fixtures_dir: Path) -> set[str]:
         return set(DEFAULT_PROSE_FIELDS)
     if not isinstance(fields, list) or not all(isinstance(f, str) for f in fields):
         raise ValueError(f"{path} must contain a string-list 'prose_fields' field")
-    return set(fields)
+    return set(DEFAULT_PROSE_FIELDS) | set(fields)
 
 
 def _render_field_value(value: object) -> str:

tools/skill-evals/tests/test_runner.py

@@ -1029,18 +1029,18 @@ def test_load_grading_schema_defaults_when_no_file(tmp_path: Path):
     assert load_grading_schema(fixtures_dir) == set(DEFAULT_PROSE_FIELDS)
 
 
-def test_load_grading_schema_override_replaces_default(tmp_path: Path):
+def test_load_grading_schema_override_extends_default(tmp_path: Path):
     fixtures_dir = tmp_path / "fixtures"
     fixtures_dir.mkdir()
     (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": ["why"]}))
-    assert load_grading_schema(fixtures_dir) == {"why"}
+    assert load_grading_schema(fixtures_dir) == set(DEFAULT_PROSE_FIELDS) | {"why"}
 
 
-def test_load_grading_schema_empty_list_disables_grader(tmp_path: Path):
+def test_load_grading_schema_empty_list_keeps_default(tmp_path: Path):
     fixtures_dir = tmp_path / "fixtures"
     fixtures_dir.mkdir()
     (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": []}))
-    assert load_grading_schema(fixtures_dir) == set()
+    assert load_grading_schema(fixtures_dir) == set(DEFAULT_PROSE_FIELDS)
 
 
 def test_load_grading_schema_rejects_non_string_entries(tmp_path: Path):
@@ -1412,24 +1412,23 @@ def test_cli_grader_mode_fails_when_grader_rejects_prose(tmp_path: Path, capsys:
 
 
 def test_cli_grader_mode_respects_grading_schema_override(tmp_path: Path, capsys: pytest.CaptureFixture[str]):
-    """grading-schema.json with prose_fields=[] forces exact compare on `reason`."""
-    expected = {"verdict": "BUG", "reason": "crashes on null input"}
-    actual = {"verdict": "BUG", "reason": "null pointer on first call"}
+    """grading-schema.json extends prose_fields to include `custom_prose`."""
+    expected = {"verdict": "BUG", "custom_prose": "crashes on null input"}
+    actual = {"verdict": "BUG", "custom_prose": "null pointer on first call"}
     fixtures_dir, _ = _make_cli_case(tmp_path, expected=expected)
-    (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": []}))
+    (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": ["custom_prose"]}))
     rc, stdout, _ = _run_main(
         capsys,
         [
             "--cli",
             f"echo '{json.dumps(actual)}'",
             "--grader-cli",
-            _GRADER_YES,  # would say YES, but reason should be graded exact now
+            _GRADER_YES,  # grader will be called for custom_prose and say YES
             str(fixtures_dir),
         ],
     )
-    assert rc == 1
-    assert "FAIL" in stdout
-    assert "reason" in stdout
+    assert rc == 0
+    assert "PASS" in stdout
 
 
 def test_grader_cli_requires_cli_flag(tmp_path: Path, capsys: pytest.CaptureFixture[str]):