feat(evals): add deterministic VQA dataset generation

dimos/cli/dimos.py

@@ -63,6 +63,7 @@
 from dimos.cli.commands.tuis import agentspy, humancli, lcmspy, spy, top
 from dimos.cli.hardware_cli import app as hardware_app
 from dimos.cli.shell import shell
+from dimos.cli.vqa import app as vqa_app
 from dimos.robot.unitree.go2.cli.go2tool import app as go2tool_app
 
 main = typer.Typer(
@@ -138,6 +139,7 @@ def cli_main() -> None:
 
 from dimos.evals.cli import app as evals_app
 
+evals_app.add_typer(vqa_app, name="vqa")
 main.add_typer(evals_app, name="evals")
 
 main.command()(cameracalibrate)

dimos/cli/test_vqa.py

@@ -0,0 +1,158 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from click import unstyle
+import pytest
+from typer.testing import CliRunner
+
+from dimos.cli.dimos import main as app
+from dimos.evals import runner as runner_module
+from dimos.evals.types import EvalResult
+from dimos.evals.vqa import generate as generate_module, suite as suite_module
+from dimos.evals.vqa.generate import GenerationRequest, GenerationResult, PublicCase
+
+
+def test_vqa_cli_exposes_generate_and_run() -> None:
+    result = CliRunner().invoke(app, ["evals", "vqa", "--help"])
+    output = unstyle(result.stdout)
+
+    assert result.exit_code == 0
+    assert "generate" in output
+    assert "run" in output
+
+
+def test_vqa_generate_cli_declares_single_image_input() -> None:
+    result = CliRunner().invoke(app, ["evals", "vqa", "generate", "--help"])
+    output = unstyle(result.stdout)
+
+    assert result.exit_code == 0
+    assert "DATASET" in output
+    assert "--image-index" in output
+    assert "--start" in output
+    assert "--stop" in output
+    assert "--stride" in output
+    assert "--output" in output
+
+
+def test_vqa_run_cli_declares_standalone_dataset_input() -> None:
+    result = CliRunner().invoke(app, ["evals", "vqa", "run", "--help"])
+    output = unstyle(result.stdout)
+
+    assert result.exit_code == 0
+    assert "DATASET" in output
+    assert "--model" in output
+
+
+def test_vqa_generate_cli_runs_generation(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    seen: list[GenerationRequest] = []
+
+    def fake_generate(request: GenerationRequest) -> GenerationResult:
+        seen.append(request)
+        return GenerationResult(
+            output=request.output_directory(),
+            cases=(
+                PublicCase(
+                    id="q",
+                    image="assets/frame.jpg",
+                    question="Is there a chair?",
+                    choices=("yes", "no"),
+                ),
+            ),
+        )
+
+    monkeypatch.setattr(generate_module, "generate_dataset", fake_generate)
+
+    result = CliRunner().invoke(
+        app,
+        [
+            "evals",
+            "vqa",
+            "generate",
+            "recording.db",
+            "--image-index",
+            "3",
+            "--output",
+            str(tmp_path),
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert seen == [GenerationRequest(dataset="recording.db", image_index=3, output=tmp_path)]
+    assert "Generated 1 VQA case" in result.stdout
+
+
+def test_vqa_generate_cli_accepts_frame_range(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    seen: list[GenerationRequest] = []
+
+    def fake_generate(request: GenerationRequest) -> GenerationResult:
+        seen.append(request)
+        return GenerationResult(output=request.output_directory(), cases=())
+
+    monkeypatch.setattr(generate_module, "generate_dataset", fake_generate)
+
+    result = CliRunner().invoke(
+        app,
+        [
+            "evals",
+            "vqa",
+            "generate",
+            "recording.db",
+            "--start",
+            "2",
+            "--stop",
+            "9",
+            "--stride",
+            "3",
+            "--output",
+            str(tmp_path),
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert seen == [
+        GenerationRequest(
+            dataset="recording.db",
+            start=2,
+            stop=9,
+            stride=3,
+            output=tmp_path,
+        )
+    ]
+
+
+def test_vqa_run_cli_runs_shared_evaluator(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    class FakeRunner:
+        def __init__(self, **kwargs: object) -> None:
+            assert kwargs == {"model": "test-model"}
+            self.run_dir = tmp_path / "results"
+
+        def run(self, cases: object) -> list[EvalResult]:
+            assert cases == ("case",)
+            return [EvalResult(case_id="q", outputs="yes", score=1.0, passed=True)]
+
+    monkeypatch.setattr(suite_module, "load_suite", lambda dataset: ("case",))
+    monkeypatch.setattr(runner_module, "EvalRunner", FakeRunner)
+
+    result = CliRunner().invoke(
+        app,
+        ["evals", "vqa", "run", str(tmp_path), "--model", "test-model"],
+    )
+
+    assert result.exit_code == 0
+    assert "PASS" in result.stdout
+    assert "mean 1.00" in result.stdout

dimos/cli/vqa.py

@@ -0,0 +1,74 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CLI commands for generating and evaluating standalone VQA datasets."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+app = typer.Typer(help="Generate and evaluate standalone visual question-answering datasets.")
+
+
+@app.command("generate")
+def generate(
+    dataset: str = typer.Argument(help="Memory dataset name or .db/.mcap path"),
+    image_index: int | None = typer.Option(None, min=0, help="Process one color_image index"),
+    start: int | None = typer.Option(None, min=0, help="First color_image index in range mode"),
+    stop: int | None = typer.Option(None, min=1, help="Exclusive color_image stop index"),
+    stride: int | None = typer.Option(None, min=1, help="Frame stride in range mode"),
+    output: Path | None = typer.Option(None, help="Override the generated dataset directory"),
+) -> None:
+    """Generate questions for one image or an indexed image range."""
+    # Keep generation's optional model stack out of global CLI startup.
+    from dimos.evals.vqa.generate import GenerationRequest, generate_dataset
+
+    request = GenerationRequest(
+        dataset=dataset,
+        output=output,
+        image_index=image_index,
+        start=start,
+        stop=stop,
+        stride=stride,
+    )
+    result = generate_dataset(request)
+    typer.echo(f"Generated {len(result.cases)} VQA case(s) in {result.output}")
+
+
+@app.command("run")
+def run(
+    dataset: Path = typer.Argument(help="Generated standalone VQA dataset"),
+    model: str = typer.Option("", help="Override chat model"),
+) -> None:
+    """Evaluate a generated standalone VQA dataset."""
+    # Keep evaluation implementation imports out of global CLI startup.
+    from dimos.evals.runner import EvalRunner, summarize
+    from dimos.evals.vqa.suite import load_suite
+
+    overrides: dict[str, object] = {}
+    if model:
+        overrides["model"] = model
+    runner = EvalRunner(**overrides)
+    results = runner.run(load_suite(dataset))
+    for result in results:
+        status = "ERROR" if result.error else ("PASS" if result.passed else "fail")
+        detail = result.error or f"answer={result.outputs[:60]!r}"
+        typer.echo(f"{status:5} {result.case_id:30} {detail}")
+    summary = summarize(results)
+    typer.echo(
+        f"\n{summary.n} cases | mean {summary.mean_score:.2f} | "
+        f"pass {summary.pass_rate:.0%} | errors {summary.errors} | {runner.run_dir}"
+    )

dimos/evals/runner.py

@@ -32,7 +32,7 @@
 
 from dimos.constants import STATE_DIR
 from dimos.core.resource import CompositeResource
-from dimos.evals.types import EvalCase, EvalResult, InteractiveEval, Suite
+from dimos.evals.types import EvalCase, EvalResult, InteractiveEval, ResponseT, Suite
 from dimos.protocol.service.spec import BaseConfig, Configurable
 from dimos.utils.logging_config import setup_logger
 
@@ -254,6 +254,31 @@ def ask(self, context: Sequence[dict[str, Any]], question: str) -> str:
         response = self.model.invoke([SystemMessage(EVAL_SYSTEM_PROMPT), message])
         return str(response.text)
 
+    def ask_structured(
+        self,
+        context: Sequence[dict[str, Any]],
+        question: str,
+        schema: type[ResponseT],
+    ) -> ResponseT:
+        """Ask for a provider-native response validated against a Pydantic schema."""
+        from langchain_core.messages import HumanMessage, SystemMessage
+
+        blocks = list(context) if context else [BLIND_BLOCK]
+        message = HumanMessage(content=[*blocks, {"type": "text", "text": question}])
+        try:
+            structured_model = self.model.with_structured_output(schema)
+        except NotImplementedError as exc:
+            raise RuntimeError(
+                f"model {self.config.model!r} does not support structured output"
+            ) from exc
+        response = structured_model.invoke([SystemMessage(EVAL_SYSTEM_PROMPT), message])
+        if not isinstance(response, schema):
+            raise TypeError(
+                f"model {self.config.model!r} returned {type(response).__name__}, "
+                f"expected {schema.__name__}"
+            )
+        return response
+
     @property
     def model(self) -> BaseChatModel:
         if self.config.chat_model is not None:

dimos/evals/test_evals.py

@@ -25,6 +25,7 @@
 from pathlib import Path
 from typing import Any
 
+from pydantic import BaseModel
 import pytest
 
 from dimos.evals.scorers import (
@@ -95,6 +96,11 @@ def ask(self, context: Sequence[dict[str, Any]], question: str) -> str:
         self.calls.append("ask")
         return self.answer
 
+    def ask_structured(
+        self, context: Sequence[dict[str, Any]], question: str, schema: type[Any]
+    ) -> Any:
+        raise NotImplementedError
+
     def call_skill(self, name: str, args: Mapping[str, object]) -> str:
         self.calls.append(f"skill:{name}")
         return self.answer
@@ -121,6 +127,10 @@ def sample(
         return self.series
 
 
+class _AnswerResponse(BaseModel):
+    answer: str
+
+
 # -- scorers ------------------------------------------------------------------------
 
 
@@ -150,6 +160,45 @@ def test_parsers() -> None:
     assert choice(" Chairs. ") == "chairs"
 
 
+def test_runner_uses_model_native_structured_output(tmp_path: Path) -> None:
+    from dimos.evals.runner import EvalRunner
+
+    seen: list[tuple[type[BaseModel], object]] = []
+
+    class StructuredModel:
+        def with_structured_output(self, schema: type[BaseModel]) -> StructuredModel:
+            seen.append((schema, None))
+            return self
+
+        def invoke(self, messages: object) -> BaseModel:
+            seen[-1] = (seen[-1][0], messages)
+            return _AnswerResponse(answer="yes")
+
+    runner = EvalRunner(chat_model=StructuredModel(), out_dir=tmp_path)
+
+    response = runner.ask_structured([], "Is there a chair?", _AnswerResponse)
+
+    assert response == _AnswerResponse(answer="yes")
+    assert seen[0][0] is _AnswerResponse
+    assert seen[0][1] is not None
+
+
+def test_runner_rejects_wrong_structured_response_type(tmp_path: Path) -> None:
+    from dimos.evals.runner import EvalRunner
+
+    class WrongTypeModel:
+        def with_structured_output(self, schema: type[BaseModel]) -> WrongTypeModel:
+            return self
+
+        def invoke(self, messages: object) -> dict[str, str]:
+            return {"answer": "yes"}
+
+    runner = EvalRunner(chat_model=WrongTypeModel(), out_dir=tmp_path)
+
+    with pytest.raises(TypeError, match="expected _AnswerResponse"):
+        runner.ask_structured([], "Is there a chair?", _AnswerResponse)
+
+
 # -- case dispatch ---------------------------------------------------------------------
 
 

dimos/evals/types.py

@@ -36,6 +36,8 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar
 
+from pydantic import BaseModel
+
 from dimos.evals.scorers import exact, final
 
 if TYPE_CHECKING:
@@ -44,6 +46,7 @@
     from dimos.memory.stream import Stream
 
 T = TypeVar("T")
+ResponseT = TypeVar("ResponseT", bound=BaseModel)
 
 Select = Callable[["Store"], "Stream[Any, Any]"]
 """Context selector — hands the model real mem2 streams, whole or windowed::
@@ -79,6 +82,12 @@ def open_dataset(self, name: str) -> Store: ...
     def live_store(self) -> Store: ...
     def encode(self, stream: Stream[Any, Any]) -> list[dict[str, Any]]: ...
     def ask(self, context: Sequence[dict[str, Any]], question: str) -> str: ...
+    def ask_structured(
+        self,
+        context: Sequence[dict[str, Any]],
+        question: str,
+        schema: type[ResponseT],
+    ) -> ResponseT: ...
     def call_skill(self, name: str, args: Mapping[str, object]) -> str: ...
     def agent_loop(self, case: EvalCase) -> str: ...
     def mcp_ready(self) -> bool: ...