Add SGLang disaggregated P/D inference and Primus Megatron-LM scaleout benchmarks + mad-slurm-multinode skill

.claude/skills/mad-slurm-multinode/assets/mad.env/mad.env.amd-ainic.template

@@ -0,0 +1,75 @@
+# mad.env — AMD-AINIC / Pollara archetype (AMD AINIC rdma* HCAs, gfx950)
+# Copy to <WORKDIR>/rundir/mad.env, resolve every <FILL_...>, then `source mad.env`.
+# Do NOT commit a filled copy into this skill — it carries cluster-private paths.
+#
+# AINIC specifics vs CX7: gfx950, eno0, GID 1, ionic drivers, RCCL_AINIC_ROCE.
+# Confirm against the node (see references/cluster-types.md +
+# scripts/detect_cluster_env.sh).
+
+# --- secrets (file-based, never inline a literal hf_... token) ---
+# This file is meant to be `source`d, so we warn (not exit) on a missing/empty
+# token: an exit here would kill the caller's shell instead of just this file.
+if [[ ! -s ~/.huggingface/token ]]; then
+  echo "WARNING: ~/.huggingface/token is missing or empty; MAD_SECRETS_HFTOKEN" >&2
+  echo "         will be empty and HF downloads will fail later. Run" >&2
+  echo "         'huggingface-cli login' or place a token at that path." >&2
+fi
+export MAD_SECRETS_HFTOKEN=$(cat ~/.huggingface/token 2>/dev/null)
+
+# --- system ---
+export MAD_SYSTEM_GPU_ARCHITECTURE=gfx950   # AINIC/MI355-class default; confirm on node
+export MAD_VERBOSE_CONFIG=true
+
+# --- MAD source (point at the MAD checkout) ---
+export MAD_SETUP_MODEL_DIR=false
+export MODEL_DIR=<FILL_PATH_TO_MAD>/   # e.g. <WORKDIR>/MAD/
+
+# --- shared dir convenience (set once, reuse below) ---
+SHARE_DIR=<FILL_SHARED_ROOT>   # shared FS visible to every node, e.g. /shared/<user>
+
+# --- data / cache roots ---
+export MAD_DATAHOME=$SHARE_DIR/data/models
+export HF_HOME=$SHARE_DIR/data/cache/huggingface
+export TORCH_HOME=$SHARE_DIR/data/cache/torch
+export XDG_CACHE_HOME=$SHARE_DIR/data/cache/xdg
+export PIP_CACHE_DIR=$SHARE_DIR/data/cache/pip
+export TRITON_CACHE_DIR=$SHARE_DIR/data/cache/triton
+
+# --- image tar cache: MUST be on shared FS visible to every compute node ---
+export MAD_DOCKER_BUILDS=$SHARE_DIR/mad_docker_builds
+
+# --- HF helper aliases ---
+export TRANSFORMERS_CACHE=$HF_HOME
+export HUGGINGFACE_HUB_CACHE=$HF_HOME/hub
+
+# --- MAD metadata ---
+export MAD_DEPLOYMENT_TYPE=slurm
+export BUILD_NUMBER=${BUILD_NUMBER:-0}
+
+# --- AINIC RoCE / NCCL transport (Pollara, vendor 0x1dd8, exposed as rdma0..7) ---
+# NO mlx5 devices. NCCL_IB_HCA (the rdma* list) is set PER-MANIFEST.
+export NCCL_IB_DISABLE=0
+export NCCL_SOCKET_IFNAME=eno0    # management iface; confirm with `ip link`
+export GLOO_SOCKET_IFNAME=eno0
+export NCCL_IB_GID_INDEX=1        # RoCEv2 GID on AINIC; confirm with show_gids
+
+# --- AINIC transport selection (also required in BOTH manifest env blocks) ---
+# Without these RCCL silently falls back to verbs/sockets and certifies the
+# wrong codepath.
+export RCCL_AINIC_ROCE=1
+export RDMAV_DRIVERS=ionic
+export IBV_DRIVERS=ionic
+
+# --- AINIC driver mounts (manifest-level, documented here for reference) ---
+# The ionic verbs driver lives on the host and must be bind-mounted into every
+# container. Add to built_models.additional_docker_run_options:
+#   -v /usr/lib/x86_64-linux-gnu/libionic.so:/usr/lib/x86_64-linux-gnu/libionic.so:ro
+#   -v /usr/lib/x86_64-linux-gnu/libionic.so.1:/usr/lib/x86_64-linux-gnu/libionic.so.1:ro
+#   -v /etc/libibverbs.d:/etc/libibverbs.d:ro
+# And to context.docker_mounts:
+#   "/etc/libibverbs.d": "/etc/libibverbs.d"
+#   "/usr/lib/x86_64-linux-gnu/libibverbs": "/usr/lib/x86_64-linux-gnu/libibverbs"
+# See references/cluster-types.md "AINIC driver mounts" for the full rationale.
+
+# Workload-specific data paths are NOT set here — they live in the workload
+# manifest, which keeps this file cluster/cache-only and reusable across workloads.

.claude/skills/mad-slurm-multinode/assets/mad.env/mad.env.cx7-roce.template

@@ -0,0 +1,55 @@
+# mad.env — CX7 / Mellanox-RoCE archetype (Mellanox mlx5 HCAs, gfx942)
+# Copy to <WORKDIR>/rundir/mad.env, resolve every <FILL_...>, then `source mad.env`.
+# Do NOT commit a filled copy into this skill — it carries cluster-private paths.
+#
+# Verify the cluster-specific values against the actual node (see
+# references/cluster-types.md and scripts/detect_cluster_env.sh):
+#   MAD_SYSTEM_GPU_ARCHITECTURE, NCCL_SOCKET_IFNAME, NCCL_IB_GID_INDEX
+
+# --- secrets (file-based, never inline a literal hf_... token) ---
+# This file is meant to be `source`d, so we warn (not exit) on a missing/empty
+# token: an exit here would kill the caller's shell instead of just this file.
+if [[ ! -s ~/.huggingface/token ]]; then
+  echo "WARNING: ~/.huggingface/token is missing or empty; MAD_SECRETS_HFTOKEN" >&2
+  echo "         will be empty and HF downloads will fail later. Run" >&2
+  echo "         'huggingface-cli login' or place a token at that path." >&2
+fi
+export MAD_SECRETS_HFTOKEN=$(cat ~/.huggingface/token 2>/dev/null)
+
+# --- system ---
+export MAD_SYSTEM_GPU_ARCHITECTURE=gfx942   # CX7/MI300-class default; confirm on node
+export MAD_VERBOSE_CONFIG=true
+
+# --- MAD source (point at the MAD checkout) ---
+export MAD_SETUP_MODEL_DIR=false
+export MODEL_DIR=<FILL_PATH_TO_MAD>/   # e.g. <WORKDIR>/MAD/
+
+# --- data / cache roots: keep large artifacts off /home, on shared FS ---
+export MAD_DATAHOME=<FILL_DATA_ROOT>/models
+export HF_HOME=<FILL_DATA_ROOT>/cache/huggingface
+export TORCH_HOME=<FILL_DATA_ROOT>/cache/torch
+export XDG_CACHE_HOME=<FILL_DATA_ROOT>/cache/xdg
+export PIP_CACHE_DIR=<FILL_DATA_ROOT>/cache/pip
+export TRITON_CACHE_DIR=<FILL_DATA_ROOT>/cache/triton
+
+# --- image tar cache: MUST be on shared FS visible to every compute node ---
+export MAD_DOCKER_BUILDS=<FILL_SHARED_DOCKER_BUILDS_DIR>
+
+# --- HF helper aliases ---
+export TRANSFORMERS_CACHE=$HF_HOME
+export HUGGINGFACE_HUB_CACHE=$HF_HOME/hub
+
+# --- MAD metadata ---
+export MAD_DEPLOYMENT_TYPE=slurm
+export BUILD_NUMBER=${BUILD_NUMBER:-0}
+
+# --- RoCE / NCCL transport (CX7: Mellanox mlx5 HCAs, RoCEv2 over rdma*) ---
+# Bootstrap/control plane is TCP over the management iface; data plane is RDMA
+# via NCCL_IB_HCA which is set PER-MANIFEST (device list differs per node).
+export NCCL_IB_DISABLE=0
+export NCCL_SOCKET_IFNAME=eth0    # management iface; confirm with `ip link`
+export GLOO_SOCKET_IFNAME=eth0
+export NCCL_IB_GID_INDEX=3        # RoCEv2 GID; confirm with show_gids
+
+# Workload-specific data paths are NOT set here — they live in the workload
+# manifest, which keeps this file cluster/cache-only and reusable across workloads.

.claude/skills/mad-slurm-multinode/assets/mad.env/mad.env.thor2-bnxt.template

@@ -0,0 +1,57 @@
+# mad.env — Broadcom Thor2 / RoCE archetype (bnxt_re HCAs, gfx950 / MI355X-class)
+# Copy to <WORKDIR>/rundir/mad.env, resolve every <FILL_...>, then `source mad.env`.
+# Do NOT commit a filled copy into this skill — it carries cluster-private paths.
+#
+# Verify the cluster-specific values against the actual node (see
+# references/cluster-types.md and scripts/detect_cluster_env.sh):
+#   MAD_SYSTEM_GPU_ARCHITECTURE, NCCL_SOCKET_IFNAME, NCCL_IB_GID_INDEX
+
+# --- secrets (file-based, never inline a literal hf_... token) ---
+# This file is meant to be `source`d, so we warn (not exit) on a missing/empty
+# token: an exit here would kill the caller's shell instead of just this file.
+if [[ ! -s ~/.huggingface/token ]]; then
+  echo "WARNING: ~/.huggingface/token is missing or empty; MAD_SECRETS_HFTOKEN" >&2
+  echo "         will be empty and HF downloads will fail later. Run" >&2
+  echo "         'huggingface-cli login' or place a token at that path." >&2
+fi
+export MAD_SECRETS_HFTOKEN=$(cat ~/.huggingface/token 2>/dev/null)
+
+# --- system ---
+export MAD_SYSTEM_GPU_ARCHITECTURE=gfx950   # MI355X-class default; confirm on node (gfx942 = MI300X)
+export MAD_VERBOSE_CONFIG=true
+
+# --- MAD source (point at the MAD checkout) ---
+export MAD_SETUP_MODEL_DIR=false
+export MODEL_DIR=<FILL_PATH_TO_MAD>/   # e.g. <WORKDIR>/MAD/
+
+# --- data / cache roots: keep large artifacts off /home, on shared FS ---
+export MAD_DATAHOME=<FILL_DATA_ROOT>/models
+export HF_HOME=<FILL_DATA_ROOT>/cache/huggingface
+export TORCH_HOME=<FILL_DATA_ROOT>/cache/torch
+export XDG_CACHE_HOME=<FILL_DATA_ROOT>/cache/xdg
+export PIP_CACHE_DIR=<FILL_DATA_ROOT>/cache/pip
+export TRITON_CACHE_DIR=<FILL_DATA_ROOT>/cache/triton
+
+# --- image tar cache: MUST be on shared FS visible to every compute node ---
+export MAD_DOCKER_BUILDS=<FILL_SHARED_DOCKER_BUILDS_DIR>
+
+# --- HF helper aliases ---
+export TRANSFORMERS_CACHE=$HF_HOME
+export HUGGINGFACE_HUB_CACHE=$HF_HOME/hub
+
+# --- MAD metadata ---
+export MAD_DEPLOYMENT_TYPE=slurm
+export BUILD_NUMBER=${BUILD_NUMBER:-0}
+
+# --- RoCE / NCCL transport (Broadcom Thor2 bnxt_re HCAs, RoCEv2) ---
+# Bootstrap/control plane is TCP over the management iface (fenic0 here); data
+# plane is RDMA via NCCL_IB_HCA which is set PER-MANIFEST (bnxt_re device list).
+export NCCL_IB_DISABLE=0
+export NCCL_SOCKET_IFNAME=fenic0  # management iface; confirm with `ip -br link`
+export GLOO_SOCKET_IFNAME=fenic0
+export NCCL_IB_GID_INDEX=3        # RoCEv2 GID; confirm with show_gids
+export RDMAV_DRIVERS=bnxt_re      # Broadcom Thor2 ibverbs provider
+export IBV_DRIVERS=bnxt_re
+
+# Workload-specific data paths are NOT set here — they live in the workload
+# manifest, which keeps this file cluster/cache-only and reusable across workloads.

.claude/skills/mad-slurm-multinode/assets/manifests/primus_llama-3.1-70b.template.json

@@ -0,0 +1,111 @@
+{
+  "built_images": {
+    "rocm-primus-llama31-70b": {
+      "model": "primus_pyt_megatron_lm_train_llama-3.1-70b_scaleout",
+      "docker_image": "rocm/primus:v26.4-<FILL_RCCL_TAG e.g. rccl-7.2.1 or rccl-develop-<sha7>>",
+      "dockerfile": "docker/primus_megatron_train_rccl_overlay.ubuntu.amd.Dockerfile",
+      "base_docker": "rocm/primus:v26.4",
+      "build_duration": 0,
+      "local_image": true,
+      "registry_image": null,
+      "registry": null,
+      "gpu_vendor": "AMD"
+    }
+  },
+  "built_models": {
+    "rocm-primus-llama31-70b": {
+      "name": "primus_pyt_megatron_lm_train_llama-3.1-70b_scaleout",
+      "url": "",
+      "dockerfile": "docker/primus_megatron_train_rccl_overlay.ubuntu.amd.Dockerfile",
+      "scripts": "scripts/primus/megatron-lm/run.sh",
+      "n_gpus": "-1",
+      "owner": "mad.support@amd.com",
+      "training_precision": "",
+      "multiple_results": "perf_primus-megatron-Llama-3.1-70B.csv",
+      "tags": ["pyt", "pretrain", "llama-3.1-70b", "training"],
+      "timeout": -1,
+      "args": "--model_repo primus_pyt_megatron_lm_train_llama-3.1-70b",
+      "additional_docker_run_options": "--privileged --group-add render --shm-size 64G --device=/dev/infiniband --cap-add IPC_LOCK --ulimit memlock=-1 -v /sys:/sys:ro -v /run/udev:/run/udev:ro"
+    }
+  },
+  "context": {
+    "docker_env_vars": {
+      "NCCL_DEBUG": "INFO",
+      "NCCL_DEBUG_SUBSYS": "INIT,NET",
+      "NCCL_IB_DISABLE": "0",
+      "NCCL_NET": "IB",
+      "NCCL_IB_HCA": "<FILL_NCCL_IB_HCA  data-plane HCA list, see references/cluster-types.md>",
+      "NCCL_IB_GID_INDEX": "<FILL_GID  RoCEv2 GID index, see references/cluster-types.md>",
+      "NCCL_SOCKET_IFNAME": "<FILL_IFNAME  management iface, see references/cluster-types.md>",
+      "GLOO_SOCKET_IFNAME": "<FILL_IFNAME  management iface, see references/cluster-types.md>",
+      "RDMAV_DRIVERS": "<FILL_DRIVER  see references/cluster-types.md>",
+      "IBV_DRIVERS": "<FILL_DRIVER  see references/cluster-types.md>",
+      "RCCL_AINIC_ROCE": "<FILL_RCCL_AINIC_ROCE  per archetype, see references/cluster-types.md; delete this key if not needed>",
+      "NCCL_NET_PLUGIN": "none",
+      "IBV_SHOW_WARNINGS": "1"
+    },
+    "docker_mounts": {
+      "/dev/infiniband": "/dev/infiniband"
+    },
+    "docker_build_arg": {},
+    "gpu_vendor": "AMD",
+    "guest_os": "UBUNTU",
+    "docker_gpus": "0,1,2,3,4,5,6,7"
+  },
+  "credentials_required": [],
+  "summary": {
+    "successful_builds": [],
+    "failed_builds": [],
+    "total_build_time": 0,
+    "successful_pushes": [],
+    "failed_pushes": []
+  },
+  "deployment_config": {
+    "target": "slurm",
+    "slurm": {
+      "partition": "<FILL_PARTITION>",
+      "account": "<FILL_ACCOUNT or remove if cluster has none>",
+      "qos": "<FILL_QOS>",
+      "nodes": 2,
+      "gpus_per_node": 8,
+      "time": "12:00:00",
+      "output_dir": "./slurm_output",
+      "exclusive": true,
+      "network_interface": "<FILL_IFNAME  management iface, see references/cluster-types.md>"
+    },
+    "distributed": {
+      "launcher": "torchrun",
+      "backend": "nccl",
+      "port": 29500,
+      "nnodes": 2,
+      "nproc_per_node": 8
+    },
+    "env_vars": {
+      "HF_HOME": "${HF_HOME}",
+      "TORCH_HOME": "${TORCH_HOME}",
+      "XDG_CACHE_HOME": "${XDG_CACHE_HOME}",
+      "PIP_CACHE_DIR": "${PIP_CACHE_DIR}",
+      "MAD_DATAHOME": "${MAD_DATAHOME}",
+      "NCCL_DEBUG": "INFO",
+      "NCCL_DEBUG_SUBSYS": "INIT,NET",
+      "NCCL_IB_DISABLE": "0",
+      "NCCL_NET": "IB",
+      "NCCL_SOCKET_IFNAME": "<FILL_IFNAME  management iface, see references/cluster-types.md>",
+      "GLOO_SOCKET_IFNAME": "<FILL_IFNAME  management iface, see references/cluster-types.md>",
+      "NCCL_IB_GID_INDEX": "<FILL_GID  RoCEv2 GID index, see references/cluster-types.md>",
+      "NCCL_IB_HCA": "<FILL_NCCL_IB_HCA  same value as context.docker_env_vars>",
+      "RDMAV_DRIVERS": "<FILL_DRIVER  see references/cluster-types.md>",
+      "IBV_DRIVERS": "<FILL_DRIVER  see references/cluster-types.md>",
+      "RCCL_AINIC_ROCE": "<FILL_RCCL_AINIC_ROCE  per archetype, see references/cluster-types.md; delete this key if not needed>",
+      "NCCL_NET_PLUGIN": "none",
+      "NCCL_TIMEOUT": "900",
+      "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1",
+      "TORCH_NCCL_HIGH_PRIORITY": "1",
+      "OMP_NUM_THREADS": "8",
+      "MIOPEN_FIND_MODE": "1",
+      "MIOPEN_USER_DB_PATH": "${XDG_CACHE_HOME}/miopen"
+    },
+    "debug": false,
+    "docker_gpus": "0,1,2,3,4,5,6,7"
+  }
+}