Fix agentic SWE-bench eval dispatch: dedicated agentic_evals bucket + sweep job / 修复 agentic SWE-bench 评估调度：新增 agentic_evals 独立分桶与扫描任务

.github/workflows/run-sweep.yml

@@ -628,6 +628,58 @@ jobs:
             run-eval: true
             eval-only: true
 
+    # Agentic (SWE-bench) eval rows carry the agentic input shape, so they are
+    # dispatched with sweep-agentic's inputs rather than sweep-evals' fixed-seq-len
+    # inputs (isl/osl/max-model-len, which agentic rows don't have).
+    sweep-agentic-evals:
+        needs: [setup, canary-select, canary-sweep]
+        if: >-
+            ${{
+              !cancelled() &&
+              needs.setup.result == 'success' &&
+              needs.setup.outputs.reuse-enabled != 'true' &&
+              (needs.canary-sweep.result == 'success' || needs.canary-sweep.result == 'skipped') &&
+              toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != '[]' &&
+              toJson(fromJson(needs.setup.outputs.search-space-config).agentic_evals) != 'null'
+            }}
+        uses: ./.github/workflows/benchmark-tmpl.yml
+        name: agentic eval /
+        strategy:
+            fail-fast: ${{ contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') }}
+            matrix:
+                config: ${{ fromJson(needs.setup.outputs.search-space-config).agentic_evals }}
+        secrets: inherit
+        with:
+            exp-name: ${{ matrix.config.exp-name }}
+            runner: ${{ matrix.config.runner }}
+            image: ${{ matrix.config.image }}
+            model: ${{ matrix.config.model }}
+            model-prefix: ${{ matrix.config.model-prefix }}
+            framework: ${{ matrix.config.framework }}
+            precision: ${{ matrix.config.precision }}
+            router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
+            kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
+            tp: ${{ matrix.config.tp }}
+            pp: ${{ matrix.config.pp }}
+            dcp-size: ${{ matrix.config.dcp-size }}
+            pcp-size: ${{ matrix.config.pcp-size }}
+            ep: ${{ matrix.config.ep }}
+            dp-attn: ${{ matrix.config.dp-attn }}
+            conc: ${{ matrix.config.conc }}
+            kv-offloading: ${{ matrix.config.kv-offloading }}
+            kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }}
+            kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }}
+            total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }}
+            duration: ${{ matrix.config.duration }}
+            isl: '0'
+            osl: '0'
+            max-model-len: '0'
+            spec-decoding: ${{ matrix.config.spec-decoding }}
+            disagg: ${{ 'false' }}
+            run-eval: true
+            eval-only: true
+            scenario-type: agentic-coding
+
     sweep-multi-node-evals:
         needs: [setup, canary-select, canary-sweep]
         if: >-
@@ -714,8 +766,8 @@ jobs:
             result-prefix: "bmk"
 
     collect-evals:
-        needs: [sweep-evals, sweep-multi-node-evals, setup]
-        if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }}
+        needs: [sweep-evals, sweep-agentic-evals, sweep-multi-node-evals, setup]
+        if: ${{ always() && needs.setup.result != 'skipped' && (needs.sweep-evals.result != 'skipped' || needs.sweep-agentic-evals.result != 'skipped' || needs.sweep-multi-node-evals.result != 'skipped') }}
         uses: ./.github/workflows/collect-evals.yml
         secrets: inherit
 

utils/matrix_logic/test_validation.py

@@ -18,6 +18,7 @@
     SingleNodeMasterConfigEntry,
     MultiNodeMasterConfigEntry,
     ChangelogEntry,
+    ChangelogMatrixEntry,
     validate_matrix_entry,
     validate_master_config,
     validate_runner_config,
@@ -1400,6 +1401,49 @@ def test_scenario_type_must_be_nonempty_and_supported(self, scenario_type):
             })
 
 
+# =============================================================================
+# Test ChangelogMatrixEntry
+# =============================================================================
+
+AGENTIC_EVAL_ROW = {
+    "image": "vllm/vllm-openai:nightly", "model": "deepseek-ai/DeepSeek-V4-Pro",
+    "model-prefix": "dsv4", "precision": "fp4", "framework": "vllm",
+    "runner": "cluster:b300-nv", "tp": 8, "pp": 1, "dcp-size": 1,
+    "pcp-size": 1, "ep": 8, "dp-attn": True, "spec-decoding": "mtp",
+    "conc": 224, "kv-offloading": "none", "total-cpu-dram-gb": 0,
+    "duration": 3600, "exp-name": "dsv4_tp8_conc224_kvnone_spec-mtp",
+    "scenario-type": "agentic-coding", "run-eval": True, "eval-only": True,
+}
+
+CHANGELOG_METADATA = {
+    "base_ref": "base", "head_ref": "head",
+    "entries": [{
+        "config-keys": ["test-config"], "description": ["Test entry"],
+        "pr-link": "https://github.com/SemiAnalysisAI/InferenceX/pull/1",
+    }],
+}
+
+
+class TestChangelogMatrixEntry:
+    """Tests for the final search-space contract consumed by run-sweep.yml."""
+
+    def test_agentic_eval_rows_live_in_agentic_evals_only(self):
+        """`evals` is dispatched with fixed-seq-len inputs (isl/osl/
+        max-model-len), so agentic rows must only validate in agentic_evals."""
+        entry = ChangelogMatrixEntry.model_validate({
+            "agentic_evals": [AGENTIC_EVAL_ROW],
+            "changelog_metadata": CHANGELOG_METADATA,
+        })
+        assert entry.agentic_evals[0].run_eval is True
+        assert entry.agentic_evals[0].eval_only is True
+
+        with pytest.raises(ValueError):
+            ChangelogMatrixEntry.model_validate({
+                "evals": [AGENTIC_EVAL_ROW],
+                "changelog_metadata": CHANGELOG_METADATA,
+            })
+
+
 # =============================================================================
 # Test load_config_files
 # =============================================================================

utils/matrix_logic/validation.py

@@ -291,6 +291,10 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
     duration: int = Field(alias=Fields.DURATION.value)
     exp_name: str = Field(alias=Fields.EXP_NAME.value)
     scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value)
+    # Agentic eval rows (SWE-bench) carry run-eval/eval-only; benchmark rows
+    # omit them, and exclude_none keeps them out of dumped benchmark output.
+    run_eval: Optional[bool] = Field(default=None, alias=Fields.RUN_EVAL.value)
+    eval_only: Optional[bool] = Field(default=None, alias=Fields.EVAL_ONLY.value)
 
     @model_validator(mode='after')
     def validate_kv_offload_fields(self):
@@ -890,6 +894,13 @@ class ChangelogMatrixEntry(BaseModel):
     multi_node: dict[str, list[Union[MultiNodeMatrixEntry, MultiNodeAgenticMatrixEntry]]
                      ] = Field(default_factory=dict)
     evals: list[SingleNodeMatrixEntry] = Field(default_factory=list)
+    # Agentic (SWE-bench) eval rows live in their own bucket rather than a
+    # union inside `evals`: each bucket maps 1:1 to a run-sweep.yml job with a
+    # static input block, so an agentic row can never reach the fixed-seq-len
+    # eval dispatch (which reads isl/osl/max-model-len and would launch the
+    # wrong benchmark script).
+    agentic_evals: list[SingleNodeAgenticMatrixEntry] = Field(
+        default_factory=list)
     multinode_evals: list[MultiNodeMatrixEntry] = Field(default_factory=list)
     changelog_metadata: ChangelogMetadata
 

utils/process_changelog.py

@@ -137,6 +137,7 @@ def main():
         "single_node": defaultdict(list),
         "multi_node": defaultdict(list),
         "evals": [],
+        "agentic_evals": [],
         "multinode_evals": [],
         "changelog_metadata": {
             "base_ref": args.base_ref,
@@ -271,7 +272,18 @@ def main():
             seq_len_str = seq_len_to_str(result["isl"], result["osl"])
             final_results["single_node"][seq_len_str].append(result)
 
-    final_results["evals"] = [e for e in all_eval_results if e.get("prefill") is None]
+    # Agentic eval rows go to their own bucket so run-sweep.yml can dispatch
+    # them with agentic inputs (scenario-type, kv-offloading, ...) instead of
+    # the fixed-seq-len inputs (isl/osl/max-model-len) they don't have.
+    single_node_evals = [e for e in all_eval_results if e.get("prefill") is None]
+    final_results["evals"] = [
+        e for e in single_node_evals
+        if e.get("scenario-type") != "agentic-coding"
+    ]
+    final_results["agentic_evals"] = [
+        e for e in single_node_evals
+        if e.get("scenario-type") == "agentic-coding"
+    ]
     final_results["multinode_evals"] = [e for e in all_eval_results if e.get("prefill") is not None]
 
     # Validate final results structure

utils/recover_failed_ingest.py

@@ -492,6 +492,7 @@ def build_config(
         )
         + len(config.get("multi_node", {}).get("agentic", []) or []),
         "eval_jobs": len(config.get("evals", []) or [])
+        + len(config.get("agentic_evals", []) or [])
         + len(config.get("multinode_evals", []) or []),
     }
 

utils/test_process_changelog.py

@@ -583,3 +583,59 @@ def fake_run(command, **kwargs):
     assert "--all-evals" not in commands[2]
     assert _scenario_values(commands[2]) == ["fixed-seq-len"]
     json.loads(capsys.readouterr().out)
+
+
+def test_eval_rows_split_into_fixed_and_agentic_buckets(
+    monkeypatch,
+    capsys,
+):
+    """Realistic eval rows must pass final validation and land in the bucket
+    matching their dispatch job: fixed-seq-len rows in `evals`, agentic
+    (SWE-bench) rows in `agentic_evals`."""
+    added_yaml = """
+- config-keys:
+    - test-config
+  description:
+    - Mixed fixed-seq-len and agentic eval selection
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1
+"""
+    common = {
+        "image": "vllm/vllm-openai:v0.11.0",
+        "model": "deepseek-ai/DeepSeek-V4-Pro", "model-prefix": "dsv4",
+        "precision": "fp4", "framework": "vllm", "spec-decoding": "mtp",
+        "runner": "cluster:b300-nv", "tp": 8, "pp": 1, "dcp-size": 1,
+        "pcp-size": 1, "ep": 8, "dp-attn": True, "conc": 224,
+        "run-eval": True, "eval-only": True,
+    }
+    fixed_eval_row = {
+        **common, "isl": 8192, "osl": 1024, "max-model-len": 10240,
+        "disagg": False, "exp-name": "fixed_eval",
+    }
+    agentic_eval_row = {
+        **common, "kv-offloading": "none", "total-cpu-dram-gb": 0,
+        "duration": 3600, "scenario-type": "agentic-coding",
+        "exp-name": "agentic_eval",
+    }
+
+    monkeypatch.setattr(
+        process_changelog, "get_added_lines", lambda *_: added_yaml)
+    monkeypatch.setattr(
+        process_changelog, "load_config_files", lambda _: {"test-config": {}})
+
+    def fake_run(command, **kwargs):
+        is_evals = "--evals-only" in command
+        rows = [fixed_eval_row, agentic_eval_row] if is_evals else []
+        return SimpleNamespace(stdout=json.dumps(rows))
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+    monkeypatch.setattr(sys, "argv", [
+        "process_changelog.py", "--base-ref", "base", "--head-ref", "head",
+        "--changelog-file", "perf-changelog.yaml",
+    ])
+
+    process_changelog.main()
+
+    output = json.loads(capsys.readouterr().out)
+    assert [r["exp-name"] for r in output["evals"]] == ["fixed_eval"]
+    assert [r["exp-name"] for r in output["agentic_evals"]] == ["agentic_eval"]
+    assert output["multinode_evals"] == []