diff --git a/tools/skill-evals/README.md b/tools/skill-evals/README.md index 2fc9f826a..565fb7d39 100644 --- a/tools/skill-evals/README.md +++ b/tools/skill-evals/README.md @@ -242,18 +242,17 @@ writes stdout works). Pass `--exact` to disable grading entirely. The default prose-field set is `rationale`, `reason`, `reasons`, `drop_reason`, `blockers`, `notes`, `summary`, `explanation`, -`details`, `description`. Override it per fixtures dir by placing a +`details`, `description`. Extend it per fixtures dir by placing a `grading-schema.json` next to `step-config.json`: ```json { - "prose_fields": ["rationale", "drop_reason"] + "prose_fields": ["custom_prose_field"] } ``` -An empty list (`"prose_fields": []`) makes every field decision-graded -even with the grader on, equivalent to passing `--exact` for that -fixtures dir. The grader is called fresh on every run; nothing is +The fields listed in `grading-schema.json` are added to the default +prose-field set. The grader is called fresh on every run; nothing is cached. **Self-eval caveat.** When the model invoked by `--cli` is the same diff --git a/tools/skill-evals/evals/release-audit-report/step-2-assemble-record/fixtures/grading-schema.json b/tools/skill-evals/evals/release-audit-report/step-2-assemble-record/fixtures/grading-schema.json deleted file mode 100644 index 44a848187..000000000 --- a/tools/skill-evals/evals/release-audit-report/step-2-assemble-record/fixtures/grading-schema.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "prose_fields": ["record_markdown"] -} diff --git a/tools/skill-evals/src/skill_evals/runner.py b/tools/skill-evals/src/skill_evals/runner.py index f0218a8d9..ff838f8c9 100644 --- a/tools/skill-evals/src/skill_evals/runner.py +++ b/tools/skill-evals/src/skill_evals/runner.py @@ -397,9 +397,7 @@ def load_grading_schema(fixtures_dir: Path) -> set[str]: """Return the set of prose field names for cases in this fixtures dir. Reads ``fixtures_dir/grading-schema.json`` when present. The file may - set ``prose_fields`` to a string list that *replaces* the default set - (use ``["rationale", "reason", ...]`` to be explicit, or ``[]`` to - grade everything by exact match). + set ``prose_fields`` to a string list extending the default set. Falls back to :data:`DEFAULT_PROSE_FIELDS` when no schema file exists. """ @@ -412,7 +410,7 @@ def load_grading_schema(fixtures_dir: Path) -> set[str]: return set(DEFAULT_PROSE_FIELDS) if not isinstance(fields, list) or not all(isinstance(f, str) for f in fields): raise ValueError(f"{path} must contain a string-list 'prose_fields' field") - return set(fields) + return set(DEFAULT_PROSE_FIELDS) | set(fields) def _render_field_value(value: object) -> str: diff --git a/tools/skill-evals/tests/test_runner.py b/tools/skill-evals/tests/test_runner.py index 0f4725e05..2b119a605 100644 --- a/tools/skill-evals/tests/test_runner.py +++ b/tools/skill-evals/tests/test_runner.py @@ -1029,18 +1029,18 @@ def test_load_grading_schema_defaults_when_no_file(tmp_path: Path): assert load_grading_schema(fixtures_dir) == set(DEFAULT_PROSE_FIELDS) -def test_load_grading_schema_override_replaces_default(tmp_path: Path): +def test_load_grading_schema_override_extends_default(tmp_path: Path): fixtures_dir = tmp_path / "fixtures" fixtures_dir.mkdir() (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": ["why"]})) - assert load_grading_schema(fixtures_dir) == {"why"} + assert load_grading_schema(fixtures_dir) == set(DEFAULT_PROSE_FIELDS) | {"why"} -def test_load_grading_schema_empty_list_disables_grader(tmp_path: Path): +def test_load_grading_schema_empty_list_keeps_default(tmp_path: Path): fixtures_dir = tmp_path / "fixtures" fixtures_dir.mkdir() (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": []})) - assert load_grading_schema(fixtures_dir) == set() + assert load_grading_schema(fixtures_dir) == set(DEFAULT_PROSE_FIELDS) def test_load_grading_schema_rejects_non_string_entries(tmp_path: Path): @@ -1412,24 +1412,23 @@ def test_cli_grader_mode_fails_when_grader_rejects_prose(tmp_path: Path, capsys: def test_cli_grader_mode_respects_grading_schema_override(tmp_path: Path, capsys: pytest.CaptureFixture[str]): - """grading-schema.json with prose_fields=[] forces exact compare on `reason`.""" - expected = {"verdict": "BUG", "reason": "crashes on null input"} - actual = {"verdict": "BUG", "reason": "null pointer on first call"} + """grading-schema.json extends prose_fields to include `custom_prose`.""" + expected = {"verdict": "BUG", "custom_prose": "crashes on null input"} + actual = {"verdict": "BUG", "custom_prose": "null pointer on first call"} fixtures_dir, _ = _make_cli_case(tmp_path, expected=expected) - (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": []})) + (fixtures_dir / "grading-schema.json").write_text(json.dumps({"prose_fields": ["custom_prose"]})) rc, stdout, _ = _run_main( capsys, [ "--cli", f"echo '{json.dumps(actual)}'", "--grader-cli", - _GRADER_YES, # would say YES, but reason should be graded exact now + _GRADER_YES, # grader will be called for custom_prose and say YES str(fixtures_dir), ], ) - assert rc == 1 - assert "FAIL" in stdout - assert "reason" in stdout + assert rc == 0 + assert "PASS" in stdout def test_grader_cli_requires_cli_flag(tmp_path: Path, capsys: pytest.CaptureFixture[str]):