Emit v1.0 schema bundles from benchmark scripts

docs/index.rst

@@ -134,6 +134,7 @@ Table of Contents
    Tiled Rendering</source/overview/core-concepts/sensors/camera>
    source/features/ray
    source/features/reproducibility
+   source/features/benchmarking
 
 
 .. toctree::

docs/source/features/benchmarking.rst

@@ -0,0 +1,110 @@
+Benchmarking
+============
+
+Isaac Lab ships three standalone benchmark scripts that emit a common
+``v1.0`` JSON schema for training-performance and startup-performance data.
+The schema is defined in :mod:`isaaclab.benchmark.schema`, and the scripts
+are fully usable standalone — any tool that can read JSON can consume the
+output.
+
+.. contents::
+   :local:
+   :depth: 2
+
+
+Scripts
+-------
+
+``benchmark_startup.py``
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Profiles five IsaacLab startup phases with ``cProfile``: ``app_launch``,
+``python_imports``, ``task_config``, ``env_creation``, and ``first_step``. For
+each phase it records wall-clock time and the top N self-time functions.
+
+.. code-block:: bash
+
+   ./isaaclab.sh -p scripts/benchmarks/benchmark_startup.py \
+       --task Isaac-Ant-Direct-v0 --num_envs 4096 --headless \
+       --schema_v1_output /tmp/startup.json
+
+``benchmark_rsl_rl.py``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Trains a task with the RSL-RL PPO agent and records runtime / resource /
+learning metrics, including exponentially-smoothed reward and episode-length
+curves.
+
+.. code-block:: bash
+
+   ./isaaclab.sh -p scripts/benchmarks/benchmark_rsl_rl.py \
+       --task Isaac-Ant-Direct-v0 --num_envs 4096 \
+       --max_iterations 500 --headless \
+       --schema_v1_output /tmp/training.json
+
+``benchmark_skrl.py``
+~~~~~~~~~~~~~~~~~~~~~
+
+The SKRL-framework counterpart to ``benchmark_rsl_rl.py``. Emits the same
+schema with ``framework: "skrl"``.
+
+.. code-block:: bash
+
+   ./isaaclab.sh -p scripts/benchmarks/benchmark_skrl.py \
+       --task Isaac-Ant-Direct-v0 --num_envs 4096 \
+       --max_iterations 500 --headless \
+       --schema_v1_output /tmp/training_skrl.json
+
+
+v1.0 schema summary
+-------------------
+
+Each script writes a single self-contained JSON file. The shape is defined by
+dataclasses in :mod:`isaaclab.benchmark.schema` — refer to the module for
+per-field units and descriptions.
+
+:class:`~isaaclab.benchmark.schema.TrainingBundle` (training scripts)
+top-level keys:
+
+* ``run`` — run identity (``run_id``, ``framework``, ``backend``, ``task``,
+  ``seed``, ``num_envs``, ``max_iterations``, timestamps, ``status``).
+* ``versions`` — software versions at run time (Isaac Lab, Isaac Sim, Kit,
+  Newton, Warp, Torch, RSL-RL / SKRL, git metadata).
+* ``hardware`` — host snapshot (hostname, GPU devices, CPU, RAM).
+* ``runtime`` — aggregated timings (``iterations_completed``,
+  ``iteration_time_s``, ``env_steps_per_s``, ``iterations_per_s``,
+  ``startup_phase_times_s``).
+* ``resources`` — aggregated GPU/CPU/RAM utilisation (mean/std/peak).
+* ``learning`` — final-value and EMA-smoothed reward / episode-length curves,
+  with full per-iteration series unless ``--no_series`` is passed.
+
+:class:`~isaaclab.benchmark.schema.StartupBundle` (``benchmark_startup.py``)
+replaces ``runtime`` / ``resources`` / ``learning`` with:
+
+* ``phases`` — mapping from phase name to ``{total_time_s, top_functions}``.
+* ``config`` — CLI configuration (``top_n``, ``whitelist``).
+
+
+Common CLI flags
+----------------
+
+``--schema_v1_output <path>``
+    Write the v1.0 JSON bundle to this path. If omitted, the script falls
+    back to the legacy per-backend output format.
+
+``--backend {physx, newton}``
+    Physics backend tag recorded in the bundle. Defaults to ``physx`` if
+    omitted.
+
+``--run_id <string>``
+    Explicit run-identity string. If omitted, a synthetic run_id of the
+    form ``<framework>_<backend>_<task>_<YYYYMMDD-HHMMSS>_seed<seed>`` is
+    generated.
+
+``--ema_alpha <float>`` (training scripts)
+    EMA smoothing factor for reward / episode-length curves (default
+    ``0.05``, roughly a 20-sample window).
+
+``--no_series`` (training scripts)
+    Omit per-iteration series from the bundle, leaving only the
+    ``final_raw`` + ``final_ema`` scalars.

scripts/benchmarks/_action_sampling.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Random-action sampler shared across the benchmark scripts.
+
+Single-agent (``DirectRLEnv`` / ``ManagerBasedRLEnv``) envs expose
+``single_action_space``; multi-agent (``DirectMARLEnv``) envs expose
+``action_spaces`` — a dict keyed by agent id. ``env.step`` accepts the
+matching shape: a stacked tensor for single-agent, a dict of stacked
+tensors for multi-agent. The benchmark startup phase needs random
+actions for the first env step and previously assumed single-agent;
+this helper picks the right shape.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import torch
+
+__all__ = ["sample_random_actions"]
+
+
+def sample_random_actions(env: Any) -> torch.Tensor | dict[str, torch.Tensor]:
+    """Sample one random action per env from the env's action space(s).
+
+    Discriminates single-agent from multi-agent by duck typing on
+    ``action_spaces`` (plural, dict-valued). DirectRLEnv and
+    ManagerBasedRLEnv expose ``single_action_space``; DirectMARLEnv
+    exposes ``action_spaces``. Both shapes ultimately get fed straight
+    to ``env.step``.
+
+    Args:
+        env: The benchmark target — typically a ``gym.Env`` returned by
+            ``gym.make``. The unwrapped env must expose ``num_envs``
+            and ``device`` plus either ``single_action_space`` or
+            ``action_spaces``.
+
+    Returns:
+        A ``torch.Tensor`` of shape ``(num_envs, action_dim)`` for
+        single-agent envs, or a dict ``{agent: tensor}`` for
+        multi-agent envs. dtype is ``torch.float32`` on the env's
+        device.
+    """
+    unwrapped = env.unwrapped
+    if hasattr(unwrapped, "action_spaces"):
+        return {
+            agent: torch.as_tensor(
+                np.stack([space.sample() for _ in range(unwrapped.num_envs)]),
+                dtype=torch.float32,
+                device=unwrapped.device,
+            )
+            for agent, space in unwrapped.action_spaces.items()
+        }
+    np_actions = np.stack([unwrapped.single_action_space.sample() for _ in range(unwrapped.num_envs)])
+    return torch.as_tensor(np_actions, dtype=torch.float32, device=unwrapped.device)

scripts/benchmarks/_schema_helpers.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Shared helpers for the v1.0 benchmark bundle writers.
+
+Used by ``benchmark_startup.py``, ``benchmark_rsl_rl.py``, and
+``benchmark_skrl.py`` to build schema-v1 ``Versions`` and ``Hardware``
+dataclasses from the benchmark's manual recorders, and to synthesise a
+fallback run_id when the caller did not provide one.
+"""
+
+from __future__ import annotations
+
+import socket
+from datetime import datetime, timezone
+
+from isaaclab.test.benchmark import BaseIsaacLabBenchmark
+from isaaclab.benchmark.schema import GpuDeviceInfo, Hardware, Versions
+
+
+def capture_versions(bm: BaseIsaacLabBenchmark) -> Versions:
+    """Build a :class:`Versions` from the benchmark's ``VersionInfoRecorder``.
+
+    Must be called before :meth:`BaseIsaacLabBenchmark._finalize_impl`, which
+    clears ``_manual_recorders``.
+    """
+    meta = {m.name: m.data for m in bm._manual_recorders["VersionInfo"].get_data().metadata}
+    dev = meta.get("dev", {}) or {}
+    return Versions(
+        isaaclab=meta.get("isaaclab_version", "unknown"),
+        isaacsim=meta.get("isaacsim_version"),
+        kit=meta.get("kit_version"),
+        newton=meta.get("newton_version"),
+        warp=meta.get("warp_version"),
+        mjwarp=meta.get("mujoco_warp_version"),
+        torch=meta.get("torch_version", "unknown"),
+        rsl_rl=meta.get("rsl_rl_version"),
+        skrl=meta.get("skrl_version"),
+        git_commit=dev.get("commit_hash"),
+        git_branch=dev.get("branch"),
+        git_dirty=bool(dev.get("dirty", False)),
+    )
+
+
+def capture_hardware(bm: BaseIsaacLabBenchmark) -> Hardware:
+    """Build a :class:`Hardware` from GPU/CPU/Memory recorders.
+
+    Must be called before :meth:`BaseIsaacLabBenchmark._finalize_impl`, which
+    clears ``_manual_recorders``.
+    """
+    gpu_meta = {m.name: m.data for m in bm._manual_recorders["GPUInfo"].get_data().metadata}
+    cpu_meta = {m.name: m.data for m in bm._manual_recorders["CPUInfo"].get_data().metadata}
+    mem_meta = {m.name: m.data for m in bm._manual_recorders["MemoryInfo"].get_data().metadata}
+    devices_raw = gpu_meta.get("gpu_devices", {}) or {}
+    devices = [
+        GpuDeviceInfo(
+            name=str(d.get("name", "unknown")),
+            mem_gb=float(d.get("total_memory_gb", 0.0) or 0.0),
+            compute_cap=str(d.get("compute_capability", "unknown")),
+        )
+        for d in devices_raw.values()
+    ]
+    return Hardware(
+        hostname=socket.gethostname(),
+        gpu_devices=devices,
+        cpu_name=str(cpu_meta.get("cpu_name", "unknown")),
+        cpu_count=int(cpu_meta.get("physical_cores", 0) or 0),
+        ram_gb=float(mem_meta.get("total_ram_gb", 0.0) or 0.0),
+    )
+
+
+def synth_run_id(framework: str, backend: str, task: str, seed: int) -> str:
+    """Fallback run_id when the caller did not supply ``--run_id``.
+
+    Format: ``<framework>_<backend>_<task>_<YYYYMMDD-HHMMSS>_seed<seed>``,
+    with underscores in ``framework`` replaced by hyphens (so ``rsl_rl``
+    becomes ``rsl-rl``).
+    """
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    fw = framework.replace("_", "-")
+    return f"{fw}_{backend}_{task}_{stamp}_seed{seed}"