Add Kernel-backed Harbor control arm

CHANGELOG.md

@@ -10,11 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 ### Added
 - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia).
 - Added `clawbench-analyze` entrypoint for aggregate batch error analysis.
+- Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification.
 
 ### Changed
 - Updated the harbor adaptor to support the full V2 lenient & strict and reports numeric results.
 
 ### Fixed
+- Fail task setup when PurelyMail returns an API error instead of emitting credentials for an account that was not created.
 - Fixed an issue where malformed per-run metadata could prevent `batch-summary.json` from being written and, when configured, uploaded.
 - Fixed the issue that an invalid judge model would lose the `run-meta.json` file.
 

docs/harbor.md

@@ -108,6 +108,25 @@ uvx --from harbor==0.15.0 harbor run \
   --jobs-dir ./harbor-jobs/hermes-deepseek-flash
 ```
 
+## Kernel browser runtime (control arm)
+
+By default each Harbor trial runs Chromium inside its own container. Pass `--browser-runtime kernel` to the adapter to run the same tasks against one Kernel cloud browser per task instead:
+
+```bash
+uv run clawbench-harbor-adapt \
+  --output-dir ./harbor-datasets/clawbench-v2-kernel \
+  --browser-runtime kernel \
+  --browser-runtime-options '{"stealth": true}' \
+  --task-ids v2-1134-chapter-finder-redcross \
+  --overwrite
+```
+
+During task setup, the environment creates exactly one Kernel browser and replay, starts the ClawBench runtime server against it, and exposes only the local credential-free CDP bridge (`http://127.0.0.1:7878`) to the agent — the Kernel API key is never visible to the benchmark agent. Session identity and cleanup metadata land in `/my-info/kernel_browser.json`. During verification the provider replay is finalized, `recording.mp4` is downloaded into `/data`, and the browser is deleted (idempotently, including failure paths via a setup trap).
+
+Generated tasks register a pinned Playwright MCP package (`@playwright/mcp@0.0.79`) pointed at the CDP bridge, so Harbor's stock Claude Code and Codex agents drive the Kernel browser with native Playwright MCP tool calls — structurally identical to ClawBench's native Claude/Codex harnesses.
+
+Export `KERNEL_API_KEY` (and optionally `KERNEL_BASE_URL` for non-production gateways) before `harbor run`; no extra flags are needed.
+
 ## Making it fast
 
 A full V2 sweep is 129 containerized browser sessions, each capped by the task's `time_limit`. Serial, that is a very long night. What actually moves the needle, in order:

src/clawbench/eval/harbor_adapter.py

@@ -11,11 +11,24 @@
 from pathlib import Path
 from typing import Any
 
+from clawbench.runner.run_support.browser_runtime.providers import (
+    BrowserRuntimeError,
+    _parse_options,
+)
 from clawbench.runner.run_support.task import build_instruction, validate_task_data
 from clawbench.utils.paths import RUNTIME_ROOT, asset_path
 
 DEFAULT_CASES_DIR = asset_path("test-cases", "v2")
 STEP_NAME = "run"
+HARBOR_BROWSER_RUNTIMES = ("local", "kernel")
+LOCAL_CDP_URL = "http://127.0.0.1:9223"
+# Remote browser runtimes expose the runtime server's credential-free CDP
+# bridge at the same local endpoint used by container-local Chromium.
+REMOTE_BRIDGE_CDP_URL = LOCAL_CDP_URL
+# Pinned Playwright MCP package Harbor's stock Claude Code and Codex agents
+# use to drive the ClawBench browser through the CDP bridge.
+PLAYWRIGHT_MCP_PACKAGE = "@playwright/mcp"
+PLAYWRIGHT_MCP_VERSION = "0.0.79"
 
 
 def sanitize_task_name(raw: str) -> str:
@@ -96,6 +109,16 @@ def copy_environment(env_dir: Path) -> None:
     copytree_filtered(RUNTIME_ROOT / "shared", env_dir / "shared")
     copytree_filtered(RUNTIME_ROOT / "harbor", env_dir / "harbor")
     (env_dir / "harbor" / "Dockerfile").unlink(missing_ok=True)
+    # The Kernel lifecycle scripts reuse the same provider implementation as
+    # the native runner.
+    shutil.copy2(
+        Path(__file__).resolve().parents[1]
+        / "runner"
+        / "run_support"
+        / "browser_runtime"
+        / "providers.py",
+        env_dir / "harbor" / "browser_runtime_providers.py",
+    )
     shutil.copy2(
         Path(__file__).resolve().parents[1]
         / "runner"
@@ -108,6 +131,36 @@ def copy_environment(env_dir: Path) -> None:
             chmod_executable(script)
 
 
+def playwright_mcp_server(cdp_url: str) -> dict[str, Any]:
+    return {
+        "name": "playwright",
+        "transport": "stdio",
+        "command": "npx",
+        "args": [
+            "-y",
+            f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}",
+            "--cdp-endpoint",
+            cdp_url,
+        ],
+    }
+
+
+def mcp_servers_toml(servers: list[dict[str, Any]]) -> str:
+    if not servers:
+        return ""
+    blocks = []
+    for server in servers:
+        args = ", ".join(json.dumps(arg) for arg in server["args"])
+        blocks.append(
+            "[[environment.mcp_servers]]\n"
+            f"name = {json.dumps(server['name'])}\n"
+            f"transport = {json.dumps(server['transport'])}\n"
+            f"command = {json.dumps(server['command'])}\n"
+            f"args = [{args}]\n"
+        )
+    return "\n" + "\n".join(blocks)
+
+
 def harbor_instruction(task: dict[str, Any]) -> str:
     instruction = build_instruction(task)
     return (
@@ -130,12 +183,37 @@ def task_toml(
     dataset_name: str,
     timeout_sec: int,
     task_dir_name: str,
+    browser_runtime: str = "local",
+    browser_runtime_options: str | None = None,
 ) -> str:
     escaped_description = json.dumps(description)
     escaped_dataset = json.dumps(dataset_name)
     escaped_source = json.dumps(task_dir_name)
     escaped_package = json.dumps(package_name)
-    return f"""schema_version = "1.3"
+    cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL
+    kernel_env = ""
+    mcp_servers = ""
+    if browser_runtime == "kernel":
+        runtime_options_line = (
+            f"\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = {json.dumps(browser_runtime_options)}"
+            if browser_runtime_options
+            else '\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = "${CLAWBENCH_BROWSER_RUNTIME_OPTIONS:-}"'
+        )
+        kernel_env = (
+            '\nCLAWBENCH_HARBOR_BROWSER_RUNTIME = "kernel"'
+            '\nKERNEL_API_KEY = "${KERNEL_API_KEY}"'
+            '\nKERNEL_BASE_URL = "${KERNEL_BASE_URL:-}"'
+            + runtime_options_line
+            + '\nCLAWBENCH_RECORDING_MODE = "provider-download"'
+        )
+        mcp_servers = mcp_servers_toml([playwright_mcp_server(REMOTE_BRIDGE_CDP_URL)])
+    healthcheck_command = (
+        "curl -sf http://127.0.0.1:7878/api/status | grep -q '"
+        + '\\"eval_interceptor_ready\\":true'
+        + f"' && curl -sf {cdp_url}/json/version >/dev/null"
+    )
+    return (
+        f"""schema_version = "1.3"
 source = "clawbench-v2"
 artifacts = ["/data"]
 
@@ -147,20 +225,24 @@ def task_toml(
 [metadata]
 dataset = {escaped_dataset}
 source_task = {escaped_source}
+browser_runtime = "{browser_runtime}"
 
 [environment]
 build_timeout_sec = 1200.0
 network_mode = "public"
-workdir = "/app"
+# The container root doubles as the step workdir so every exec runs with a
+# cwd that exists even on overlay drivers that cannot resolve image-created
+# directories during `docker exec`.
+workdir = "/"
 
 [environment.env]
 PURELY_MAIL_API_KEY = "${{PURELY_MAIL_API_KEY}}"
 PURELY_MAIL_DOMAIN = "${{PURELY_MAIL_DOMAIN}}"
-CLAWBENCH_CDP_URL = "http://127.0.0.1:9223"
-BROWSER_CDP_URL = "http://127.0.0.1:9223"
-CDP_URL = "http://127.0.0.1:9223"
-CHROME_CDP_URL = "http://127.0.0.1:9223"
-PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223"
+CLAWBENCH_CDP_URL = "{cdp_url}"
+BROWSER_CDP_URL = "{cdp_url}"
+CDP_URL = "{cdp_url}"
+CHROME_CDP_URL = "{cdp_url}"
+PLAYWRIGHT_CDP_URL = "{cdp_url}"{kernel_env}
 CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html"
 CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878"
 CLAWBENCH_JUDGE_BASE_URL = "${{CLAWBENCH_JUDGE_BASE_URL:-}}"
@@ -178,37 +260,59 @@ def task_toml(
 timeout_sec = 300.0
 
 [steps.healthcheck]
-command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\\\"eval_interceptor_ready\\\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null"
+command = "{healthcheck_command}"
 interval_sec = 2.0
 timeout_sec = 5.0
 start_period_sec = 2.0
 start_interval_sec = 1.0
 retries = 30
 """
+        + mcp_servers
+    )
 
 
-def setup_script() -> str:
-    return """#!/bin/bash
+def setup_script(browser_runtime: str = "local") -> str:
+    kernel_setup = ""
+    readiness = (
+        "  if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n"
+        "    && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n"
+    )
+    if browser_runtime == "kernel":
+        readiness = (
+            "  if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n"
+            "    && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n"
+        )
+        kernel_setup = (
+            "# Create the Kernel browser and replay before the runtime server"
+            " starts so it can bridge the provider CDP endpoint.\n"
+            "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py start\n"
+            "export CLAWBENCH_BROWSER_CDP_URL_FILE=/tmp/clawbench-run/kernel-cdp-url\n"
+            "\n"
+            "cleanup_browser() {\n"
+            "  /app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py cleanup || true\n"
+            "}\n"
+            "trap cleanup_browser EXIT\n"
+            "\n"
+        )
+    return f"""#!/bin/bash
 set -euo pipefail
 
-mkdir -p /data /logs/verifier /app/extra_info
-cp /app/eval-schema.json /eval-schema.json
+mkdir -p /data /logs/verifier /extra_info
 
 /app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py \
-  --task-json /app/task.json \
-  --extra-info-dir /app/extra_info \
-  --output-dir /app/my-info
+  --task-json /task.json \
+  --extra-info-dir /extra_info \
+  --output-dir /my-info
 
 # Harbor installs its stock agent before step setup. Wrap that executable so
 # ClawBench's existing /data/.stop-requested signal ends the agent cleanly.
 /app/src/harbor/wrap-harbor-agent.sh
 
-/app/src/harbor/start-runtime.sh
+{kernel_setup}/app/src/harbor/start-runtime.sh
 
 for _ in $(seq 1 60); do
-  if curl -sf http://127.0.0.1:7878/api/status >/dev/null \
-    && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then
-    rm -f /app/setup.sh
+{readiness}    rm -f /app/setup.sh
+    trap - EXIT
     exit 0
   fi
   sleep 1
@@ -219,15 +323,21 @@ def setup_script() -> str:
 """
 
 
-def test_script() -> str:
-    return """#!/bin/bash
+def test_script(browser_runtime: str = "local") -> str:
+    kernel_finalize = ""
+    if browser_runtime == "kernel":
+        kernel_finalize = (
+            "# Stop the provider replay, download the recording, delete the browser.\n"
+            "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py finalize\n"
+        )
+    return f"""#!/bin/bash
 set -euo pipefail
 
 curl -sf -X POST http://127.0.0.1:7878/api/stop || true
 curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true
 sleep 2
 rm -f /data/.stop-requested
-rm -rf /logs/verifier/data
+{kernel_finalize}rm -rf /logs/verifier/data
 cp -a /data /logs/verifier/data
 
 /app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py
@@ -269,6 +379,8 @@ def write_harbor_task(
     output_name: str,
     org: str,
     dataset_name: str,
+    browser_runtime: str = "local",
+    browser_runtime_options: str | None = None,
 ) -> Path:
     dest = output_root / output_name
     if dest.exists():
@@ -296,15 +408,17 @@ def write_harbor_task(
             dataset_name=dataset_name,
             timeout_sec=timeout_sec,
             task_dir_name=task_dir.name,
+            browser_runtime=browser_runtime,
+            browser_runtime_options=browser_runtime_options,
         )
     )
     (step_dir / "instruction.md").write_text(harbor_instruction(task))
     (workdir / "eval-schema.json").write_text(json.dumps(task["eval_schema"], indent=2))
     (workdir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False))
     copy_extra_info(task, task_dir, workdir / "extra_info")
-    write_text_executable(workdir / "setup.sh", setup_script())
+    write_text_executable(workdir / "setup.sh", setup_script(browser_runtime))
     (tests_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False))
-    write_text_executable(tests_dir / "test.sh", test_script())
+    write_text_executable(tests_dir / "test.sh", test_script(browser_runtime))
     write_text_executable(solution_dir / "solve.sh", solve_script())
     copy_environment(env_dir)
     return dest
@@ -343,13 +457,32 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Overwrite an existing output directory",
     )
+    parser.add_argument(
+        "--browser-runtime",
+        choices=HARBOR_BROWSER_RUNTIMES,
+        default="local",
+        help="Browser runtime for the generated tasks; kernel creates one "
+        "Kernel browser per task during Harbor setup",
+    )
+    parser.add_argument(
+        "--browser-runtime-options",
+        default=None,
+        help="JSON object with runtime options, e.g. '{\"stealth\":true}'",
+    )
     return parser
 
 
 def main(argv: list[str] | None = None) -> int:
     parser = build_parser()
     args = parser.parse_args(argv)
 
+    options: dict[str, Any] = {}
+    if args.browser_runtime_options:
+        try:
+            options = _parse_options(args.browser_runtime_options)
+        except BrowserRuntimeError as exc:
+            parser.error(str(exc))
+
     cases_dir = (args.cases_dir or DEFAULT_CASES_DIR).resolve()
     default_cases = args.cases_dir is None
     if not cases_dir.exists():
@@ -388,10 +521,16 @@ def main(argv: list[str] | None = None) -> int:
                 output_name=out_name,
                 org=args.org,
                 dataset_name=args.dataset_name,
+                browser_runtime=args.browser_runtime,
+                browser_runtime_options=(json.dumps(options) if options else None),
             )
         )
 
     print(f"Wrote {len(written)} Harbor task(s) to {output_dir}")
+    if args.browser_runtime != "local":
+        print(f"Browser runtime: {args.browser_runtime}")
+        if options:
+            print(f"Runtime options: {json.dumps(options)}")
     return 0
 
 

src/clawbench/runner/run_support/browser_runtime/providers.py

@@ -558,6 +558,16 @@ def _delete(self, session_id: str) -> str:
             raise
         return "deleted"
 
+    def session_exists(self, session_id: str) -> bool:
+        """Return True if the provider still reports this browser session."""
+        try:
+            self._request("GET", f"/browsers/{session_id}")
+        except _KernelApiError as e:
+            if e.status == 404:
+                return False
+            raise
+        return True
+
     def _stop_replay(self, session_id: str, replay_id: str) -> None:
         try:
             self._request(
@@ -575,13 +585,9 @@ def start(self, task: dict[str, Any], time_limit_s: int) -> BrowserSession:
         timeout_seconds = min(259200, max(10, time_limit_s + 120))
         payload = {
             **self.options,
+            "stealth": self.options.get("stealth", True),
             "headless": False,
             "timeout_seconds": timeout_seconds,
-            "viewport": {
-                "width": 1920,
-                "height": 1080,
-                "refresh_rate": 25,
-            },
         }
         try:
             result = self._request_json("POST", "/browsers", payload)