diff --git a/CHANGELOG.md b/CHANGELOG.md index ef1b4197..04776b5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Changed - The `claw-eval` port now uses the same `test-cases///task.json` layout as the native corpora, instead of flat `.json` files. Case discovery in `clawbench-batch` and the TUI is a plain `*/task.json` search again, and the `validate-task` workflow covers the suite without special-casing. +- Changed the default Harbor version to `0.22.0`. ### Fixed - Fixed a judge-provider outage (or an unparseable judge reply) being recorded as an agent failure. `run.py` now exits 3 instead of 1 when the judge never renders a verdict, `batch.py` gives it its own `judge_inconclusive` bucket in `batch-summary.json` instead of folding it into `failed`, and `clawbench-rescore` now retries a cached `match: null` verdict even without `--force`. diff --git a/docs/harbor.md b/docs/harbor.md index fba41381..7814111b 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -4,21 +4,30 @@ **Use Harbor when** you already run other benchmarks through it, you want Harbor's agent registry (`-a openclaw`, `-a hermes`, …) instead of ClawBench's own harnesses, or you need Harbor's retry/attempt semantics. **Use `clawbench-batch` instead** when you just want to score a model on ClawBench — it is the shorter path and it is what the leaderboard uses. -- [Prerequisites](#prerequisites) -- [1. Convert V2 into a Harbor dataset](#1-convert-v2-into-a-harbor-dataset) -- [2. Wire up the judge](#2-wire-up-the-judge) -- [3. Run it](#3-run-it) -- [Making it fast](#making-it-fast) -- [What the generated environment contains](#what-the-generated-environment-contains) -- [Troubleshooting](#troubleshooting) +- [Running ClawBench through Harbor](#running-clawbench-through-harbor) + - [Prerequisites](#prerequisites) + - [Harbor versions](#harbor-versions) + - [1. Convert V2 into a Harbor dataset](#1-convert-v2-into-a-harbor-dataset) + - [2. Wire up the judge](#2-wire-up-the-judge) + - [3. Run it](#3-run-it) + - [OpenClaw through an OpenAI-compatible endpoint](#openclaw-through-an-openai-compatible-endpoint) + - [Hermes through OpenRouter](#hermes-through-openrouter) + - [Making it fast](#making-it-fast) + - [What the generated environment contains](#what-the-generated-environment-contains) + - [Troubleshooting](#troubleshooting) ## Prerequisites +- **Harbor 0.22.0.** Every command below pins `harbor==0.22.0`, which is the version the generated dataset is verified against — see [Harbor versions](#harbor-versions). - **Docker.** Harbor runs use Harbor's Docker provider, so Docker must be available even if you normally use Podman for native ClawBench runs. - **ClawBench installed** (`uv tool install clawbench-eval`, or a source checkout with `uv run` prefixes). - **Judge credentials.** Scoring requires both an intercepted request *and* a judge verdict; without judge credentials every intercepted task scores `0`. - **PurelyMail credentials** from `.env`, passed through with `--env-file .env`. +## Harbor versions + +The commands here pin **`harbor==0.22.0`**, the current release at the time of writing. The previous pin, `0.15.0`, was six releases stale: anyone following these docs installed an old Harbor, and anyone who already had a current Harbor found the pin fighting their install. + ## 1. Convert V2 into a Harbor dataset ```bash @@ -59,7 +68,7 @@ Use `deepseek-v4-pro` if you want numbers comparable to the published leaderboar ## 3. Run it ```bash -uvx --from harbor==0.15.0 harbor run \ +uvx --from harbor==0.22.0 harbor run \ -p ./harbor-datasets/clawbench-v2 \ -a "" \ -m "" \ @@ -70,7 +79,7 @@ uvx --from harbor==0.15.0 harbor run \ --ve CLAWBENCH_JUDGE_API_TYPE="${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" ``` -Drop `uvx --from harbor==0.15.0` if Harbor is already installed. +Drop `uvx --from harbor==0.22.0` if Harbor is already installed. ### OpenClaw through an OpenAI-compatible endpoint @@ -78,7 +87,7 @@ Drop `uvx --from harbor==0.15.0` if Harbor is already installed. export OPENAI_BASE_URL="https://openrouter.ai/api/v1" export OPENAI_API_KEY="$OPENROUTER_API_KEY" -uvx --from harbor==0.15.0 harbor run \ +uvx --from harbor==0.22.0 harbor run \ -p ./harbor-datasets/clawbench-v2 \ -a openclaw \ -m openai/deepseek/deepseek-v4-flash \ @@ -96,7 +105,7 @@ uvx --from harbor==0.15.0 harbor run \ ```bash export OPENROUTER_API_KEY="your-openrouter-key" -uvx --from harbor==0.15.0 harbor run \ +uvx --from harbor==0.22.0 harbor run \ -p ./harbor-datasets/clawbench-v2 \ -a hermes \ -m deepseek/deepseek-v4-flash \ @@ -115,7 +124,7 @@ A full V2 sweep is 129 containerized browser sessions, each capped by the task's **1. Raise concurrency.** `-n / --n-concurrent` is the single biggest lever: ```bash -uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2 \ +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2 \ -a hermes -m deepseek/deepseek-v4-flash -n 8 --env-file .env --ve ... ``` @@ -124,7 +133,7 @@ Each trial is a full Chromium container, so budget roughly **1 CPU core and ~2 G **2. Build the image once.** The first trial builds the ClawBench environment image; parallel cold starts all build at once. Warm the cache with the smoke dataset before the real sweep: ```bash -uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \ +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \ -a hermes -m deepseek/deepseek-v4-flash --env-file .env --ve ... ``` diff --git a/src/README.md b/src/README.md index 8fa8f728..e2b1c8e2 100644 --- a/src/README.md +++ b/src/README.md @@ -26,10 +26,10 @@ uv run clawbench-harbor-adapt --output-dir ./harbor-datasets/clawbench-v2 --over # Example Harbor runs after adaptation: # OpenClaw via OpenRouter/OpenAI-compatible API: -uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2 -a openclaw -m openai/deepseek/deepseek-v4-flash --ak thinking=off --env-file .env +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2 -a openclaw -m openai/deepseek/deepseek-v4-flash --ak thinking=off --env-file .env # Hermes via OpenRouter: -uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2 -a hermes -m deepseek/deepseek-v4-flash --env-file .env +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2 -a hermes -m deepseek/deepseek-v4-flash --env-file .env ``` The console scripts are defined in the root `pyproject.toml`: diff --git a/tests/test_harbor_version_compatibility.py b/tests/test_harbor_version_compatibility.py new file mode 100644 index 00000000..770e6ecd --- /dev/null +++ b/tests/test_harbor_version_compatibility.py @@ -0,0 +1,73 @@ +"""Does the adapter's output still load in the Harbor version the docs pin? + +Skipped unless Harbor is installed -- it is not a ClawBench dependency. The +point of the test is to make a version bump checkable instead of assumed: + + uv run --with harbor== pytest tests/test_harbor_version_compatibility.py + +The docs pinned harbor==0.15.0 for six releases past upstream (#294). Nothing +in the repo could tell you whether that mattered, because nothing ever loaded +a generated task with Harbor's own loader. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from clawbench.eval import harbor_adapter + +harbor_task = pytest.importorskip( + "harbor.models.task.task", + reason="Harbor is not installed; run with `uv run --with harbor==`", +) + +# The version docs/harbor.md pins and claims to have verified. A mismatch is +# not a failure -- the point is to make the tested version visible in output. +DOCUMENTED_HARBOR_VERSION = "0.22.0" + + +@pytest.fixture(scope="module") +def generated_dataset(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Convert the real V2 corpus, not a synthetic task. + + A hand-built fixture would only prove the fixture loads. What has to hold + is that the corpus we tell people to convert produces a dataset Harbor + accepts. + """ + out = tmp_path_factory.mktemp("harbor-dataset") / "clawbench-v2" + rc = harbor_adapter.main(["--output-dir", str(out), "--overwrite"]) + assert rc == 0 + return out + + +def test_installed_harbor_version_is_reported(record_property) -> None: + from importlib.metadata import version + + installed = version("harbor") + record_property("harbor_version", installed) + if installed != DOCUMENTED_HARBOR_VERSION: + pytest.skip( + f"harbor {installed} installed, docs pin {DOCUMENTED_HARBOR_VERSION}; " + "the load test below still runs" + ) + + +def test_every_generated_task_loads_with_harbors_own_loader( + generated_dataset: Path, +) -> None: + task_dirs = sorted(d for d in generated_dataset.iterdir() if d.is_dir()) + assert task_dirs + + failures: list[tuple[str, str]] = [] + for task_dir in task_dirs: + if not harbor_task.Task.is_valid_dir(task_dir): + failures.append((task_dir.name, "Harbor does not recognise the directory")) + continue + try: + harbor_task.Task(task_dir) + except Exception as exc: # noqa: BLE001 - report whatever Harbor raises + failures.append((task_dir.name, f"{type(exc).__name__}: {exc}")) + + assert not failures, "\n".join(f"{name}: {why}" for name, why in failures)