From 9c192c2c391c800db2628e5da050eb37b4b19f35 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Mon, 13 May 2024 15:36:36 -0700 Subject: [PATCH 01/10] [wip] delete build scripts + configs included in aznhc --- .../cluster-init/files/build_bandwidthtest.sh | 8 ---- .../cluster-init/files/build_perftest_gdr.sh | 18 ------- .../cluster-init/files/nc48v4-graph.xml | 48 ------------------- .../cluster-init/files/nc48v4-topo.xml | 19 -------- .../scripts/002_build_bandwidthtest.sh | 4 -- .../scripts/003_build_perftest_gdr.sh | 4 -- .../cluster-init/scripts/004_topo_graph.sh | 8 ---- 7 files changed, 109 deletions(-) delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_bandwidthtest.sh delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_perftest_gdr.sh delete mode 100644 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-graph.xml delete mode 100644 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-topo.xml delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/002_build_bandwidthtest.sh delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/003_build_perftest_gdr.sh delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/004_topo_graph.sh diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_bandwidthtest.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_bandwidthtest.sh deleted file mode 100755 index 454d736a8..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_bandwidthtest.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -source $CYCLECLOUD_SPEC_PATH/files/common_functions.sh - -if ! is_slurm_controller; then - cd /usr/local/cuda/samples/1_Utilities/bandwidthTest - make -fi diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_perftest_gdr.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_perftest_gdr.sh deleted file mode 100755 index 154bee997..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/build_perftest_gdr.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -VERSION=4.5-0.12 -VERSION_HASH=ge93c538 -INSTALL_DIR=/opt - -source $CYCLECLOUD_SPEC_PATH/files/common_functions.sh - -if ! is_slurm_controller; then - apt-get install -y libpci-dev - cd ${INSTALL_DIR} - wget https://github.com/linux-rdma/perftest/releases/download/v${VERSION}/perftest-${VERSION}.${VERSION_HASH}.tar.gz - tar xvf perftest-${VERSION}.${VERSION_HASH}.tar.gz - cd perftest-4.5 - ./configure CUDA_H_PATH=/usr/local/cuda/include/cuda.h - make - rm ${INSTALL_DIR}/perftest-${VERSION}.${VERSION_HASH}.tar.gz -fi diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-graph.xml b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-graph.xml deleted file mode 100644 index 154188cf1..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-graph.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-topo.xml b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-topo.xml deleted file mode 100644 index 117827d6c..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48v4-topo.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/002_build_bandwidthtest.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/002_build_bandwidthtest.sh deleted file mode 100755 index bdf21f6ad..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/002_build_bandwidthtest.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -chmod +x $CYCLECLOUD_SPEC_PATH/files/build_bandwidthtest.sh -$CYCLECLOUD_SPEC_PATH/files/build_bandwidthtest.sh diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/003_build_perftest_gdr.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/003_build_perftest_gdr.sh deleted file mode 100755 index 0950c3386..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/003_build_perftest_gdr.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -chmod +x $CYCLECLOUD_SPEC_PATH/files/build_perftest_gdr.sh -$CYCLECLOUD_SPEC_PATH/files/build_perftest_gdr.sh diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/004_topo_graph.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/004_topo_graph.sh deleted file mode 100755 index 5cd91d0e7..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/004_topo_graph.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if ! [ -d /opt/microsoft/ncv4 ]; then - mkdir /opt/microsoft/ncv4 -fi - -cp $CYCLECLOUD_SPEC_PATH/files/nc48v4-topo.xml /opt/microsoft/ncv4 -cp $CYCLECLOUD_SPEC_PATH/files/nc48v4-graph.xml /opt/microsoft/ncv4 From 17d8d6b2a33e853dc80c95a1e2692fdee36600a3 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Mon, 13 May 2024 15:40:12 -0700 Subject: [PATCH 02/10] [wip] remove nhc tests included in aznhc --- .../files/azure_cpu_drop_cache_mem.nhc | 58 --------- .../files/azure_cuda_bandwidth.nhc | 112 ------------------ .../files/azure_gpu_app_clocks.nhc | 46 ------- .../files/azure_gpu_clock_throttling.nhc | 43 ------- .../cluster-init/files/azure_gpu_ecc.nhc | 80 ------------- .../files/azure_gpu_persistence.nhc | 43 ------- .../cluster-init/files/azure_gpu_xid.nhc | 26 ---- .../files/azure_ib_link_flapping.nhc | 49 -------- .../files/azure_ib_write_bw_gdr.nhc | 80 ------------- .../files/azure_nccl_allreduce.nhc | 108 ----------------- .../azure_nccl_allreduce_ib_loopback.nhc | 74 ------------ .../cluster-init/files/csc_nvidia_smi.nhc | 67 ----------- 12 files changed, 786 deletions(-) delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cpu_drop_cache_mem.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cuda_bandwidth.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_app_clocks.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_clock_throttling.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_ecc.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_persistence.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_xid.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_link_flapping.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_write_bw_gdr.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce_ib_loopback.nhc delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/csc_nvidia_smi.nhc diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cpu_drop_cache_mem.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cpu_drop_cache_mem.nhc deleted file mode 100755 index 5a2bb46ce..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cpu_drop_cache_mem.nhc +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Check size of cached CPU memory, if large than argument then drop the memory caches. - - -function collect_cached_cpu_memory_data() { - - meminfo_out=$(cat /proc/meminfo) - meminfo_out_rc=$? - if [[ $meminfo_out_rc != 0 ]]; then - log "$meminfo_out" - die 1 "$FUNCNAME: Does /proc/meminfo exist, returned error code $meminfo_out_rc" - fi - IFS=$'\n' - meminfo_lines=( $meminfo_out ) - IFS=$' \t\n' -} - -function find_cpu_cached_mem() { - - for ((i=0; i<${#meminfo_lines[*]}; i++)) - do - if [[ "${meminfo_lines[$i]//Cached}" != "${meminfo_lines[$i]}" ]] - then - IFS=$' \t\n' - line=( ${meminfo_lines[$i]} ) - echo "${line[1]}" - break - fi - done -} - - -function check_cpu_drop_mem_cache() { - - collect_cached_cpu_memory_data - - CPU_CACHED_MEM_KB=$1 - dbg "CPU_CACHED_MEM_KB=$CPU_CACHED_MEM_KB KB" - if [[ -n $CPU_CACHED_MEM_KB ]]; then - current_cpu_cached_mem_kb=$(find_cpu_cached_mem) - dbg "current_cpu_cached_mem_kb=$current_cpu_cached_mem_kb KB" - if [[ $current_cpu_cached_mem_kb -gt $CPU_CACHED_MEM_KB ]]; then - dbg "$FUNCNAME: We will attempt to drop the cpu memory cache" - cpu_drop_mem_cache_out=$(sync;echo 3 > /proc/sys/vm/drop_caches) - cpu_drop_mem_cache_out_rc=$? - if [[ $cpu_drop_mem_cache_out_rc != 0 ]]; then - log "$cpu_drop_mem_cache_out" - die 1 "$FUNCNAME: Could not drop the cpu memory cache, returned error code $cpu_drop_mem_cache_out_rc" - else - log "Dropped the CPU memory cache successfully" - fi - fi - else - dbg "$FUNCNAME: Will not check the CPU cached memory, (CPU_CACHED_MEM_KB argument is not set)" - fi - return 0 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cuda_bandwidth.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cuda_bandwidth.nhc deleted file mode 100755 index 458e64f6d..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_cuda_bandwidth.nhc +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# Expected bandwidth > 22GB/s -BANDWIDTHTEST_EXE_PATH=/usr/local/cuda/samples/1_Utilities/bandwidthTest/bandwidthTest -BANDWIDTHTEST=`basename $BANDWIDTHTEST_EXE_PATH` - - -function find_vm_gpu_numa() { - vm_size=`/opt/cycle/jetpack/bin/jetpack config azure.metadata.compute.vmSize | tr '[:upper:]' '[:lower:]'` - dbg "vm_size=$vm_size" - case $vm_size in - standard_nd96isr_h100_v5) - GPU_NUMA=( 0 0 0 0 1 1 1 1 ) - NUM_GPU_DEVICES=8 - ;; - standard_nd96amsr_a100_v4) -#NUMA mapping for NDv4(A100) - GPU_NUMA=( 1 1 0 0 3 3 2 2 ) - NUM_GPU_DEVICES=8 - ;; - standard_nd96asr_v4) - GPU_NUMA=( 1 1 0 0 3 3 2 2 ) - NUM_GPU_DEVICES=8 - ;; - standard_nc96ads_a100_v4) - GPU_NUMA=( 0 1 2 3 ) - NUM_GPU_DEVICES=4 - ;; - standard_nc48ads_a100_v4) - GPU_NUMA=( 0 1 ) - NUM_GPU_DEVICES=2 - ;; - *) - NUM_GPU_DEVICES="NotFound" - esac -} - - -function check_cuda_bw() -{ -EXP_CUDA_BW=$1 -REPEATS="${2:-1}" - -if [ -f /opt/cycle/jetpack/bin/jetpack ]; then - find_vm_gpu_numa - if [ $NUM_GPU_DEVICES == "NotFound" ]; then - log "Warning : $vm_size is not supported, CUDA Bandwidth test skipped" - return 0 - fi -else - log "Warning: NHC is not running in Azure Cyclecloud, skip CUDA Bandwidth test." - return 0 -fi - -for iter in $(seq 1 $REPEATS) -do - FAIL=0 - for test in "dtoh" "htod" - do - for ((device=0; device < $NUM_GPU_DEVICES; device++)) - do - IFS=$'\n' - CUDA_BW=$(numactl -N ${GPU_NUMA[$device]} -m ${GPU_NUMA[$device]} $BANDWIDTHTEST_EXE_PATH --device=$device --$test) - CUDA_BW_RC=$? - if [[ $CUDA_BW_RC != 0 ]] - then - log "$CUDA_BW" - die 1 "$FUNCNAME: $BANDWIDTHTEST retuned error code $CUDA_BW_RC " - return 1 - fi - CUDA_BW_LINES=( $CUDA_BW ) - for ((i=0; i<${#CUDA_BW_LINES[*]}; i++)) - do - if [[ "${CUDA_BW_LINES[$i]//32000000}" != "${CUDA_BW_LINES[$i]}" ]] - then - IFS=$' \t\n' - LINE=( ${CUDA_BW_LINES[$i]} ) - cuda_bandwidth=${LINE[1]} - dbg "gpu id=$device: numa domain=${GPU_NUMA[$device]}, Measured CUDA ${test^^} BW $cuda_bandwidth GB/s" - break - fi - done - - if [[ $cuda_bandwidth < $EXP_CUDA_BW ]] - then - FAIL=1 - log "Iteration ${iter} of ${REPEATS} failed: gpu id=$device: numa domain=${GPU_NUMA[$device]}, Measured CUDA ${test^^} BW $cuda_bandwidth GB/s" - break - fi - done - - if [[ $FAIL == 1 ]] - then - break - fi - - done - - if [[ $FAIL == 0 ]] - then - break - elif [[ $FAIL == 1 && $iter == $REPEATS ]] - then - die 1 "$FUNCNAME: $BANDWIDTHTEST, gpu=$device, CUDA BW $test (expected > $EXP_CUDA_BW GB/s, but measured $cuda_bandwidth GB/s" - return 1 - fi -done - -IFS=$' \t\n' -return 0 -} - diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_app_clocks.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_app_clocks.nhc deleted file mode 100755 index c305058f6..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_app_clocks.nhc +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Check if application GPU clock frequencies are set to their maximum values, if not will attempt to set them. - -GPU_QUERY_CLOCKS="clocks.max.memory,clocks.applications.memory,clocks.max.graphics,clocks.applications.graphics" - - -function collect_clocks_data() { - - gpu_freq_out=$(nvidia-smi --query-gpu=$GPU_QUERY_CLOCKS --format=csv,noheader,nounits) - gpu_freq_out_rc=$? - if [[ $gpu_freq_out_rc != 0 ]]; then - log "$gpu_freq_out" - die 1 "$FUNCNAME: nvidia-smi (get clock freqs) returned error code $gpu_freq_out_rc" - fi - IFS=$'\n' - gpu_freq_out_lines=( $gpu_freq_out ) - IFS=$' \t\n' -} - - -function check_app_gpu_clocks() { - - collect_clocks_data - - for ((i=0; i<${#gpu_freq_out_lines[*]}; i++)) - do - IFS=$', ' - gpu_freq_out_line=( ${gpu_freq_out_lines[$i]} ) - IFS=$' \t\n' - if [[ ${gpu_freq_out_line[0]} -gt ${gpu_freq_out_line[1]} || ${gpu_freq_out_line[2]} -gt ${gpu_freq_out_line[3]} ]]; then - log "Warning: GPU Id $i: GPU memory freq (max,current)= (${gpu_freq_out_line[0]},${gpu_freq_out_line[1]}) MHz, GPU graphics freq (max,current) = (${gpu_freq_out_line[2]},${gpu_freq_out_line[3]}) MHz" - log "Attempting to set application GPU clock frequencies to maximum frequencies" - set_gpu_freq_out=$(nvidia-smi -i $i -ac ${gpu_freq_out_line[0]},${gpu_freq_out_line[2]}) - set_gpu_freq_out_rc=$? - if [[ $set_gpu_freq_out_rc != 0 ]]; then - log "$set_gpu_freq_out" - die 1 "$FUNCNAME: nvidia-smi (set gpu max clock freqs) returned error code $set_gpu_freq_out_rc" - fi - log "On GPU Id $i: $set_gpu_freq_out" - else - dbg "GPU Id $i: max application GPU clocks are already set, GPU memory is ${gpu_freq_out_line[0]} MHz and GPU graphics is ${gpu_freq_out_line[2]} MHz" - fi -done -return 0 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_clock_throttling.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_clock_throttling.nhc deleted file mode 100755 index 4a8bfee2b..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_clock_throttling.nhc +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -GPU_THROTTLE_QUERY="clocks_throttle_reasons.active" - -GPU_CLOCKS_THROTTLE_REASON_HW_SLOWDOWN=0x0000000000000008 -GPU_CLOCKS_THROTTLE_REASON_HW_THERMAL_SLOWDOWN=0x000000000000004 -GPU_CLOCKS_THROTTLE_REASON_APPLICATIONS_CLOCK_SETTINGS=0x0000000000000002 -GPU_CLOCKS_THROTTLE_READON_DISPLAY_SETTINGS=0x0000000000000100 -GPU_CLOCKS_THROTTLE_REASON_GPU_IDLE=0x0000000000000001 -GPU_CLOCKS_THROTTLE_REASON_POWER_BRAKE_SLOWDOWN=0x0000000000000080 -GPU_CLOCKS_THROTTLE_REASON_NONE=0x0000000000000000 -GPU_CLOCKS_THROTTLE_REASON_SW_POWER_CAP=0x0000000000000004 -GPU_CLOCKS_THROTTLE_REASON_SW_THERMAL_SLOWDOWN=0x0000000000000020 -GPU_CLOCKS_THROTTLE_REASON_SYNC_BOOST=0x0000000000000010 - - -function collect_gpu_clock_throttle_data() { - gpu_clock_throttle_query_out=$(nvidia-smi --query-gpu=$GPU_THROTTLE_QUERY --format=csv,noheader,nounits) - gpu_clock_throttle_query_rc=$? - if [[ $gpu_clock_throttle_query_rc != 0 ]]; then - log "$gpu_clock_throttle_query_out" - die 1 "$FUNCNAME: nvidia-smi (get gpu clock throttle data) returned error code $gpu_clock_throttle_query_rc" - fi - dbg "gpu_clock_throttle_query_out=$gpu_clock_throttle_query_out" - IFS=$'\n' - gpu_clock_throttle_out_lines=( $gpu_clock_throttle_query_out ) - IFS=$' \t\n' -} - -function check_gpu_clock_throttling() { - collect_gpu_clock_throttle_data - for ((i=0; i<${#gpu_clock_throttle_out_lines[*]}; i++)) - do - IFS=$', ' - gpu_clock_throttle_out_line=( ${gpu_clock_throttle_out_lines[$i]} ) - IFS=$' \t\n' - if [[ ${gpu_clock_throttle_out_line[0]} != $GPU_CLOCKS_THROTTLE_REASON_GPU_IDLE && ${gpu_clock_throttle_out_line[0]} != $GPU_CLOCKS_THROTTLE_REASON_NONE && ${gpu_clock_throttle_out_line[0]} != $GPU_CLOCKS_THROTTLE_REASON_SW_POWER_CAP ]]; then - log "Warning: GPU $i throttled, reason=${gpu_clock_throttle_out_line[0]}" -# Just log GPU throttling (but do not DRAIN node) -# die 1 "$FUNCNAME: GPU $i clock throttled, reason=${gpu_clock_throttle_out_line[0]}" - fi - done -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_ecc.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_ecc.nhc deleted file mode 100755 index 486e0b2f6..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_ecc.nhc +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Check for GPU ECC errors - -GPU_REMAPPED_ROWS_QUERY="remapped_rows.pending,remapped_rows.failure,remapped_rows.uncorrectable" -GPU_QUERY="ecc.errors.uncorrected.volatile.sram,ecc.errors.uncorrected.aggregate.sram,ecc.errors.uncorrected.volatile.dram,ecc.errors.uncorrected.aggregate.dram,ecc.errors.corrected.volatile.sram,ecc.errors.corrected.aggregate.sram,ecc.errors.corrected.volatile.dram,ecc.errors.corrected.aggregate.dram" - - -function collect_ecc_data() { - - gpu_query_out=$(nvidia-smi --query-gpu=$GPU_QUERY --format=csv,noheader) - gpu_query_out_rc=$? - if [[ $gpu_query_out_rc != 0 ]] - then - log "$gpu_query_out" - die 1 "$FUNCNAME: nvidia-smi (get gpu uncorrected counts) returned error code $gpu_query_out_rc" - fi - gpu_remapped_rows_out=$(nvidia-smi --query-remapped-rows=$GPU_REMAPPED_ROWS_QUERY --format=csv,noheader) - gpu_remapped_rows_out_rc=$? - if [[ $gpu_remapped_rows_out_rc != 0 ]] - then - log "$gpu_remaped_rows_out" - die 1 "$FUNCNAME: nvidia-smi (get gpu remapped rows) returned error code $gpu_freq_out_rc" - fi - IFS=$'\n' - gpu_query_out_lines=( $gpu_query_out ) - gpu_remapped_rows_query_out_lines=( $gpu_remapped_rows_out ) - IFS=$' \t\n' -} - - -function check_gpu_ecc() { - - collect_ecc_data - - ecc_error_threshold=$1 - ecc_sram_threshold=$2 - - if [[ ${#gpu_query_out_lines[*]} != ${#gpu_remapped_rows_query_out_lines[*]} ]]; then - die 1 "$FUNCNAME: nvidia-smi (Number GPU's not correct), (${#gpu_query_out_lines[*]},${#gpu_remapped_rows_query_out_lines[*]})" - fi - for ((i=0; i<${#gpu_remapped_rows_query_out_lines[*]}; i++)) - do - IFS=$', ' - gpu_remapped_rows_query_out_line=( ${gpu_remapped_rows_query_out_lines[$i]} ) - gpu_query_out_line=( ${gpu_query_out_lines[$i]} ) - IFS=$' \t\n' - dbg "GPU id $i: row remap pending, (${gpu_remapped_rows_query_out_line[0]})" - if [[ ${gpu_remapped_rows_query_out_line[0]} > 0 ]] - then - die 1 "$FUNCNAME: GPU id $i: Row remap pending" - fi - dbg "GPU id $i: row remap error, (${gpu_remapped_rows_query_out_line[1]})" - if [[ ${gpu_remapped_rows_query_out_line[1]} > 0 ]] - then - die 1 "$FUNCNAME: GPU id $i: Row remap error" - fi - dbg "GPU id $i: row remap uncorrectable error count, (${gpu_remapped_rows_query_out_line[3]})" - if [[ ${gpu_remapped_rows_query_out_line[3]} > 512 ]] - then - die 1 "$FUNCNAME: GPU id $i: Row remap uncorrectable error count is too high" - fi - dbg "GPU id $i: No GPU row remap pending, row remap errors or row remap high count errors" - if [[ ${gpu_query_out_line[4]} -gt $ecc_sram_threshold || ${gpu_query_out_line[5]} -gt $ecc_sram_threshold ]]; then - die 1 "$FUNCNAME: GPU id $i: High SRAM correctable ECC error count detected, (${gpu_query_out_line[4]},${gpu_query_out_line[5]})" - elif [[ ${gpu_query_out_line[0]} -gt 0 || ${gpu_query_out_line[1]} -gt 0 ]]; then - die 1 "$FUNCNAME: GPU id $i: SRAM Uncorrectable ECC error count detected, (${gpu_query_out_line[0]},${gpu_query_out_line[1]})" - else - dbg "GPU id $i: Normal SRAM Uncorrectable/correctable ECC error count, (${gpu_query_out_line[0]},${gpu_query_out_line[1]},${gpu_query_out_line[4]},${gpu_query_out_line[5]})" - fi - if [[ -n $ecc_error_threshold ]]; then - if [[ ${gpu_query_out_line[2]} -gt $ecc_error_threshold || ${gpu_query_out_line[3]} -gt $ecc_error_threshold || ${gpu_query_out_line[6]} -gt $ecc_error_threshold || ${gpu_query_out_line[7]} -gt $ecc_error_threshold ]]; then - die 1 "$FUNCNAME: GPU id $i: High DRAM Uncorrectable/correctable ECC error count detected, (${gpu_query_out_line[2]},${gpu_query_out_line[3]},${gpu_query_out_line[6]},${gpu_query_out_line[7]})" - else - dbg "GPU id $i: Normal DRAM Uncorrectable/correctable ECC error count, (${gpu_query_out_line[2]},${gpu_query_out_line[3]},${gpu_query_out_line[6]},${gpu_query_out_line[7]})" - fi - fi - done -return 0 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_persistence.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_persistence.nhc deleted file mode 100755 index 9781da452..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_persistence.nhc +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Check GPU persistence mode, if not enabled, attempt to enable. - -PERSISTENCE_GPU_QUERY="persistence_mode" - - -function collect_persistence_data() { - - gpu_query_out=$(nvidia-smi --query-gpu=$PERSISTENCE_GPU_QUERY --format=csv,noheader) - gpu_query_out_rc=$? - if [[ $gpu_query_out_rc != 0 ]]; then - log "$gpu_query_out" - die 1 "$FUNCNAME: nvidia-smi (persistence mode) returned error code $gpu_query_out_rc" - fi - IFS=$'\n' - gpu_query_out_lines=( $gpu_query_out ) - IFS=$' \t\n' -} - - -function check_gpu_persistence() { - - collect_persistence_data - - for ((i=0; i<${#gpu_query_out_lines[*]}; i++)) - do - if [[ ${gpu_query_out_lines[$i]} == Disabled ]]; then - dbg "$FUNCNAME: GPU id $i: Persistence mode is disabled, will attempt to enable" - gpu_persistence_out=$(nvidia-smi -i $i -pm 1) - gpu_persistence_out_rc=$? - if [[ $gpu_query_out_rc != 0 ]]; then - log "$gpu_persistence_out" - die 1 "$FUNCNAME: nvidia-smi (enable persistence mode) returned error code $gpu_persistence_out_rc" - else - dbg "$gpu_persistence_out" - fi - else - dbg "$FUNCNAME: GPU id $i: Persistence mode is already enabled" - fi -done -return 0 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_xid.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_xid.nhc deleted file mode 100755 index fcc66675c..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_gpu_xid.nhc +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Check for the following GPU Xid errors in dmesg -XID_EC="48 56 57 58 62 63 64 65 68 69 73 74 79 80 81 92 119 120" -GPU_XID_TEST="GPU Xid errors detected" - - -function check_gpu_xid() -{ -/bin/dmesg | grep -q "Xid" -RC=$? -if [ $RC == 0 ]; then - for XID in $XID_EC; do - xid_found_line=$(/bin/dmesg | grep "Xid.*: $XID," | head -n 1) - if [ "$xid_found_line" != "" ]; then - log "$GPU_XID_TEST: $xid_found_line" - die 1 "$FUNCNAME: $GPU_XID_TEST: $xid_found_line" - else - dbg "No GPU Xid $XID error found in dmesg" - fi - done -else - dbg "No GPU Xid error found in dmesg" - return 0 -fi -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_link_flapping.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_link_flapping.nhc deleted file mode 100755 index f1373746c..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_link_flapping.nhc +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -#expect to not have any IB link flaps within a given time interval (in hours) -IB_FLAPPING_LINK_TEST="IB link flapping detected" - - -function check_ib_link_flapping() -{ -TIME_INTERVAL_HOURS=$1 -lost_carrier_file=/tmp/last_lost_carrier_date -lost_carrier_line=$(grep -i "ib.*lost carrier" /var/log/syslog | tail -n 1) - -if [ "$lost_carrier_line" != "" ]; then - dbg "IB link flapping entry in syslog, $lost_carrier_line" - lost_carrier_array=( $lost_carrier_line ) - last_date_str="${lost_carrier_array[0]} ${lost_carrier_array[1]} ${lost_carrier_array[2]}" - last_date_sec=$(date --date "$last_date_str" +%s) - dbg "last_date_sec = $last_date_sec" - - if [ -f $lost_carrier_file ]; then - log "File $lost_carrier_file exists" - previous_stored_date=$(cat $lost_carrier_file) - dbg "File $lost_carrier_file contains, $previous_stored_date" - - if [ "$last_date_str" != "$previous_stored_date" ]; then - previous_stored_date_sec=$(date --date "$previous_stored_date" +%s) - dbg "previous_stored_date_sec=$previous_stored_date_sec" - ((diff_secs=$last_date_sec-$previous_stored_date_sec)) - dbg "IB link flap time interval= $diff_sec sec" - ((diff_hours=$diff/(60*60))) - dbg "IB link flap time interval= $diff_hours hours" - - if [ $diff_hours -lt $TIME_INTERVAL_HOURS ]; then - log "$IB_FLAPPING_LINK_TEST, multiple IB link flapping events within $TIME_INTERVAL_HOURS hours($previous_stored_date, $last_date_str)" - die 1 "$FUNCNAME: $IB_FLAPPING_LINK_TEST, multiple IB link flapping events within $TIME_INTERVAL_HOURS hours" - else - rm $lost_carrier_file - log "Time interval > $TIME_INTERVAL_HOURS, Remove $lost_carrier_file" - fi - fi - else - log "$lost_carrier_file does not exist, so will create it with $last_date_str" - echo $last_date_str > $lost_carrier_file - fi -else - dbg "No IB link flapping entry in syslog" - return 0 -fi -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_write_bw_gdr.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_write_bw_gdr.nhc deleted file mode 100755 index 664f96598..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_ib_write_bw_gdr.nhc +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Expected bandwidth > 180 Gbps - -IB_WRITE_BW_EXE_PATH=/opt/perftest-4.5/ib_write_bw -IB_WRITE_BW=`basename $IB_WRITE_BW_EXE_PATH` -IB_WRITE_BW_DURATION=10 -IB_WRITE_BW_ARGS="-s $(( 1 * 1024 * 1024 )) -D ${IB_WRITE_BW_DURATION} -x 0 -F --report_gbits" -SLEEP_TIME=5 - -HOSTNAME=`hostname` - -function select_numa_mapping() -{ - vm_size=`/opt/cycle/jetpack/bin/jetpack config azure.metadata.compute.vmSize | tr '[:upper:]' '[:lower:]'` - dbg "vm_size=$vm_size" - case $vm_size in - standard_nd96isr_h100_v5) - GPU_NUMA=( 0 0 0 0 1 1 1 1 ) - ;; - standard_nd96amsr_a100_v4) - GPU_NUMA=( 1 1 0 0 3 3 2 2 ) - ;; - standard_nd96asr_v4) - GPU_NUMA=( 1 1 0 0 3 3 2 2 ) - ;; - esac -} - -function check_ib_bw_gdr() -{ -EXP_IB_BW=$1 - -select_numa_mapping - -for device in {0..3}; do - IB_WRITE_BW_OUT1=$(numactl -N ${GPU_NUMA[$device]} -m ${GPU_NUMA[$device]} $IB_WRITE_BW_EXE_PATH $IB_WRITE_BW_ARGS --use_cuda=${device} -d mlx5_ib${device} > /dev/null &) - IB_WRITE_BW_OUT1_RC=$? - if [[ $IB_WRITE_BW_OUT1_RC != 0 ]]; then - log "$IB_WRITE_BW_OUT1" - PORT=$(echo $IB_WRITE_BW_OUT1 | grep -oP '(?<=port\s)\w+') - NETSTAT_OUT=$(netstat -lnp | grep $PORT) - log "Running: netstat -lnp | grep $PORT:" - log "$NETSTAT_OUT" - die 1 "$FUNCNAME: $IB_WRITE_BW returned error code $IB_WRITE_BW_OUT1_RC" - return 1 - fi - sleep $SLEEP_TIME - device_peer=$(( device+4 )) - IB_WRITE_BW_OUT2=$(numactl -N ${GPU_NUMA[$device_peer]} -m ${GPU_NUMA[$device_peer]} $IB_WRITE_BW_EXE_PATH $IB_WRITE_BW_ARGS --use_cuda=${device_peer} -d mlx5_ib${device_peer} $HOSTNAME) - IB_WRITE_BW_OUT2_RC=$? - if [[ $IB_WRITE_BW_OUT2_RC != 0 ]]; then - log "$IB_WRITE_BW_OUT2" - PORT=$(echo $IB_WRITE_BW_OUT2 | grep -oP '(?<=port\s)\w+') - NETSTAT_OUT=$(netstat -lnp | grep $PORT) - log "Running: netstat -lnp | grep $PORT:" - log "$NETSTAT_OUT" - die 1 "$FUNCNAME: $IB_WRITE_BW returned error code $IB_WRITE_BW_OUT2_RC" - return 1 - fi - IFS=$'\n' - IB_WRITE_BW_OUT2_LINES=( $IB_WRITE_BW_OUT2 ) - IFS=$' \t\n' - for ((i=0; i<${#IB_WRITE_BW_OUT2_LINES[*]}; i++)) - do - if [[ "${IB_WRITE_BW_OUT2_LINES[$i]//1048576}" != "${IB_WRITE_BW_OUT2_LINES[$i]}" ]]; then - LINE=( ${IB_WRITE_BW_OUT2_LINES[$i]} ) - ib_bandwidth=${LINE[3]} - dbg "IB devices=mlx5_ib${device}, mlx5_ib${device_peer}: numa domains=${GPU_NUMA[$device]},${GPU_NUMA[$device_peer]}, Measured IB BW $ib_bandwidth Gbps" - break - fi - done - if [[ $ib_bandwidth < $EXP_IB_BW ]]; then - log "$IB_WRITE_BW_OUT2" - die 1 "$FUNCNAME: $IB_WRITE_BW, IB=mlx5_ib${device}, mlx5_ib${device_peer}, IB BW (expected > $EXP_IB_BW Gbps, but measured $ib_bandwidth Gbps" - return 1 - fi -done -return 0 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce.nhc deleted file mode 100755 index 30963337a..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce.nhc +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Check for NVlink ssues by running NCCL allreduce. -# Expected performance is > 228 GB/s - -NCCL_ARGS_2="-b 1G -f 2 -g 1 -e 2G -c 1" - - -function find_vm() { - vm_size=`/opt/cycle/jetpack/bin/jetpack config azure.metadata.compute.vmSize | tr '[:upper:]' '[:lower:]'` - dbg "vm_size=$vm_size" - case $vm_size in - standard_nd96isr_h100_v5) - MPI_MODULE="mpi/hpcx" - MPI_ARGS_2="-np 8 --map-by ppr:8:node -bind-to none -mca coll_hcoll_enable 0 --allow-run-as-root" - ENVIRON_VARS_2="-x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH -x NCCL_IB_PCI_RELAXED_ORDERING=1 -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_TOPO_FILE=/opt/microsoft/ndv5-topo.xml -x NCCL_MIN_NCHANNELS=32" - NCCL_ARGS_2="-b 16G -f 2 -g 1 -e 16G -c 1" - ;; - standard_nd96amsr_a100_v4) - MPI_MODULE="mpi/hpcx" - MPI_ARGS_2="-np 8 --map-by ppr:8:node -bind-to numa -mca coll_hcoll_enable 0 --allow-run-as-root" - ENVIRON_VARS_2="-x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH -x NCCL_IB_PCI_RELAXED_ORDERING=1 -x UCX_IB_PCI_RELAXED_ORDERING=on -x UCX_TLS=tcp -x UCX_NET_DEVICES=eth0 -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_TOPO_FILE=/opt/microsoft/ndv4-topo.xml" - ;; - standard_nd96asr_v4) - MPI_MODULE="mpi/hpcx" - MPI_ARGS_2="-np 8 --map-by ppr:8:node -bind-to numa -mca coll_hcoll_enable 0 --allow-run-as-root" - ENVIRON_VARS_2="-x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH -x NCCL_IB_PCI_RELAXED_ORDERING=1 -x UCX_IB_PCI_RELAXED_ORDERING=on -x UCX_TLS=tcp -x UCX_NET_DEVICES=eth0 -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_TOPO_FILE=/opt/microsoft/ndv4-topo.xml" - ;; - standard_nc96ads_a100_v4) - MPI_MODULE="mpi/openmpi" - MPI_ARGS_2="-np 4 --map-by ppr:4:node -bind-to numa --allow-run-as-root" - ENVIRON_VARS_2="-x NCCL_IB_DISABLE=1 -x NCCL_TOPO_FILE=/opt/microsoft/ncv4/topo.xml -x NCCL_GRAPH_FILE=/opt/microsoft/ncv4/graph.xml" - ;; - standard_nc48ads_a100_v4) - MPI_MODULE="mpi/openmpi" - MPI_ARGS_2="-np 2 --map-by ppr:2:node -bind-to numa --allow-run-as-root" - ENVIRON_VARS_2="-x NCCL_IB_DISABLE=1 -x NCCL_TOPO_FILE=/opt/microsoft/ncv4/nc48v4-topo.xml -x NCCL_GRAPH_FILE=/opt/microsoft/ncv4/nc48v4-graph.xml" - ;; - *) - MPI_MODULE="NotFound" - esac -} - - -function collect_nccl_allreduce_data() { - - nccl_allreduce_out=$(source /etc/profile.d/modules.sh && module load $MPI_MODULE && mpirun $MPI_ARGS_2 $ENVIRON_VARS_2 /opt/nccl-tests/build/all_reduce_perf $NCCL_ARGS_2) - nccl_allreduce_out_rc=$? - if [[ $nccl_allreduce_out_rc != 0 ]]; then - log "nccl_allreduce_freq_out" - die 1 "$FUNCNAME: nccl_allreduce returned error code $nccl_allreduce_out_rc" - fi - IFS=$'\n' - nccl_allreduce_out_lines=( $nccl_allreduce_out ) - IFS=$' \t\n' -} - - -function check_nccl_allreduce() { - - EXP_NCCL_ALLREDUCE_BW=$1 - REPEATS="${2:-1}" - - if [ -f /opt/cycle/jetpack/bin/jetpack ]; then - find_vm - if [ $MPI_MODULE == "NotFound" ]; then - log "Warning : $vm_size is not supported, NCCL test skipped" - return 0 - fi - else - log "Warning: NHC is not running in Azure Cyclecloud, skip NCCL allreduce test." - return 0 - fi - - for iter in $(seq 1 $REPEATS) - do - collect_nccl_allreduce_data - - for ((i=0; i<${#nccl_allreduce_out_lines[*]}; i++)) - do - if [[ "${nccl_allreduce_out_lines[$i]//FAILED}" != "${nccl_allreduce_out_lines[$i]}" ]] - then - log "$nccl_allreduce_out" - die 1 "$FUNCNAME: NCCL allreduce, Out of bounds values failed" - return 1 - fi - if [[ "${nccl_allreduce_out_lines[$i]//bandwidth}" != "${nccl_allreduce_out_lines[$i]}" ]] - then - IFS=$' \t\n' - nccl_allreduce_out_line=( ${nccl_allreduce_out_lines[$i]} ) - avg_bus_bw=${nccl_allreduce_out_line[5]} - dbg "Measured Avg NCCL allreduce bus BW $avg_bus_bw GB/s (expected >=$EXP_NCCL_ALLREDUCE_BW GB/s)" - break - fi - done - - if (( $(echo "$avg_bus_bw < $EXP_NCCL_ALLREDUCE_BW" | bc -l) )) - then - dbg "$nccl_allreduce_out" - log "Iteration ${iter} of ${REPEATS} failed: NCCL allreduce bandwidth $avg_bus_bw GB/s < $EXP_NCCL_ALLREDUCE_BW GB/s" - else - return 0 - fi - done - - die 1 "$FUNCNAME: NCCL allreduce, BUS BW (expected >=$EXP_NCCL_ALLREDUCE_BW GB/s, but measured $avg_bus_bw GB/s)" - return 1 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce_ib_loopback.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce_ib_loopback.nhc deleted file mode 100755 index 2b4c6de0d..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_nccl_allreduce_ib_loopback.nhc +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -# Check for IB issues by running NCCL allreduce disabling NCCL shared memory. - -function select_sku() { - vm_size=`/opt/cycle/jetpack/bin/jetpack config azure.metadata.compute.vmSize | tr '[:upper:]' '[:lower:]'` - dbg "vm_size=$vm_size" - case $vm_size in - standard_nd96isr_h100_v5) - MPI_ARGS="-np 8 --map-by ppr:8:node -bind-to none -mca coll_hcoll_enable 0 --allow-run-as-root" - ENVIRON_VARS="-x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH -x NCCL_IB_PCI_RELAXED_ORDERING=1 -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_TOPO_FILE=/opt/microsoft/ndv5-topo.xml -x NCCL_MIN_NCHANNELS=32" - ;; - standard_nd96amsr_a100_v4) - MPI_ARGS_2="-np 8 --map-by ppr:8:node -bind-to numa -mca coll_hcoll_enable 0 --allow-run-as-root" - ENVIRON_VARS_2="-x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH -x NCCL_IB_PCI_RELAXED_ORDERING=1 -x UCX_IB_PCI_RELAXED_ORDERING=on -x UCX_TLS=tcp -x UCX_NET_DEVICES=eth0 -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_TOPO_FILE=/opt/microsoft/ndv4-topo.xml" - ;; - standard_nd96asr_v4) - MPI_ARGS_2="-np 8 --map-by ppr:8:node -bind-to numa -mca coll_hcoll_enable 0 --allow-run-as-root" - ENVIRON_VARS_2="-x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH -x NCCL_IB_PCI_RELAXED_ORDERING=1 -x UCX_IB_PCI_RELAXED_ORDERING=on -x UCX_TLS=tcp -x UCX_NET_DEVICES=eth0 -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_TOPO_FILE=/opt/microsoft/ndv4-topo.xml" - ;; - esac -} - -ENVIRON_VARS="${ENVIRON_VARS} -x NCCL_SHM_DISABLE=1 -x NCCL_P2P_DISABLE=1" -NCCL_ARGS="-b 500M -f 2 -g 1 -e 1G -c 1" - -function collect_nccl_allreduce_ib_loopback_data() { - - select_sku - - nccl_allreduce_ib_loopback_out=$(source /etc/profile.d/modules.sh && module load mpi/hpcx && mpirun $MPI_ARGS $ENVIRON_VARS /opt/nccl-tests/build/all_reduce_perf $NCCL_ARGS) - nccl_allreduce_ib_loopback_out_rc=$? - if [[ $nccl_allreduce_ib_loopback_out_rc != 0 ]]; then - log "nccl_allreduce_ib_loopback_freq_out" - die 1 "$FUNCNAME: nccl_allreduce (IB loopback) returned error code $nccl_allreduce_ib_loopback_out_rc" - fi - IFS=$'\n' - nccl_allreduce_ib_loopback_out_lines=( $nccl_allreduce_ib_loopback_out ) - IFS=$' \t\n' -} - -function check_nccl_allreduce_ib_loopback() { - - EXP_NCCL_ALLREDUCE_IB_LOOPBACK_BW=$1 - REPEATS="${2:-1}" - - for iter in $(seq 1 $REPEATS) - do - collect_nccl_allreduce_ib_loopback_data - - for ((i=0; i<${#nccl_allreduce_ib_loopback_out_lines[*]}; i++)) - do - if [[ "${nccl_allreduce_ib_loopback_out_lines[$i]//bandwidth}" != "${nccl_allreduce_ib_loopback_out_lines[$i]}" ]] - then - IFS=$' \t\n' - nccl_allreduce_ib_loopback_out_line=( ${nccl_allreduce_ib_loopback_out_lines[$i]} ) - avg_bus_bw=${nccl_allreduce_ib_loopback_out_line[5]} - dbg "Measured Avg NCCL allreduce ib loopback bus BW $avg_bus_bw GB/s (expected >=$EXP_NCCL_ALLREDUCE_IB_LOOPBACK_BW GB/s)" - break - fi - done - - if (( $(echo "$avg_bus_bw < $EXP_NCCL_ALLREDUCE_IB_LOOPBACK_BW" | bc -l) )) - then - dbg "$nccl_allreduce_ib_loopback_out" - log "Iteration ${iter} of ${REPEATS} failed: NCCL allreduce IB loopback bandwidth $avg_bus_bw GB/s < $EXP_NCCL_ALLREDUCE_IB_LOOPBACK_BW GB/s" - else - return 0 - fi - done - - die 1 "$FUNCNAME: NCCL allreduce, BUS BW (expected >=$EXP_NCCL_ALLREDUCE_IB_LOOPBACK_BW GB/s, but measured $avg_bus_bw GB/s)" - return 1 -} diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/csc_nvidia_smi.nhc b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/csc_nvidia_smi.nhc deleted file mode 100755 index df1f79c1f..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/csc_nvidia_smi.nhc +++ /dev/null @@ -1,67 +0,0 @@ -# NHC - nVidia GPU Checks -# -# Johan Guldmyr -# 17 Dec 2015 -# - -NVIDIA_SMI_HEALTHMON="${NVIDIA_SMI_HEALTHMON:-nvidia-smi}" -NVIDIA_SMI_HEALTHMON_ARGS="${NVIDIA_SMI_HEALTHMON_ARGS}" - -NVSMI_HEALTHMON_LINES=( ) -NVSMI_HEALTHMON_OUTPUT="" -NVSMI_HEALTHMON_RC="" - -export NVSMI_HEALTHMON_LINES NVSMI_HEALTHMON_OUTPUT NVSMI_HEALTHMON_RC - -function nhc_nvsmi_gather_data() { - local IFS - - NVSMI_HEALTHMON_OUTPUT=$($NVIDIA_SMI_HEALTHMON $NVIDIA_SMI_HEALTHMON_ARGS 2>/dev/null) - NVSMI_HEALTHMON_RC=$? - IFS=$'\n' - NVSMI_HEALTHMON_LINES=( $NVSMI_HEALTHMON_OUTPUT ) -} - -# Run the nvidia-smi utility and verify that all GPUs -# are functioning properly. -function check_nvsmi_healthmon() { - if [[ -z "$NVSMI_HEALTHMON_RC" ]]; then - nhc_nvsmi_gather_data - fi - - if [[ $NVSMI_HEALTHMON_RC -eq 0 ]]; then - dbg "$FUNCNAME: $NVIDIA_SMI_HEALTHMON completed successfully" - return 0 - elif [[ $NVSMI_HEALTHMON_RC -eq 4 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: Permission denied" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 8 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: Power cables not attached" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 2 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: Invalid argument or flag" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 9 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: NVIDIA driver not loaded" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 10 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: Interrupt issue with a GPU" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 12 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: NVML shared library could not be found" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 14 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: InfoROM is corrupted" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -eq 15 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: The GPU has fallen off the bus or has otherwise become inaccessible" - return 1 - elif [[ $NVSMI_HEALTHMON_RC -gt 127 ]]; then - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: Caught fatal signal $((NVSMI_HEALTHMON_RC&0x7f))" - return 1 - else - log "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: \"$NVSMI_HEALTHMON_OUTPUT\"" - die 1 "$FUNCNAME: $NVIDIA_SMI_HEALTHMON: Returned failure code $NVSMI_HEALTHMON_RC" - return 1 - fi -} From d8d066d46a81219f77415af916f45fd2fb572b14 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Mon, 13 May 2024 16:44:00 -0700 Subject: [PATCH 03/10] [wip] update SKU confs --- .../cluster-init/files/hb120-16rs_v3.conf | 75 +------------- .../cluster-init/files/hb120-32rs_v3.conf | 75 +------------- .../cluster-init/files/hb120-64rs_v3.conf | 75 +------------- .../cluster-init/files/hb120-96rs_v3.conf | 75 +------------- .../cluster-init/files/hb120rs_v3.conf | 75 +------------- .../cluster-init/files/nc48ads_v4.conf | 81 ++------------- .../cluster-init/files/nc96ads_v4.conf | 83 ++-------------- .../cluster-init/files/nd96amsr_v4.conf | 98 +++---------------- .../cluster-init/files/nd96asr_v4.conf | 96 +++--------------- .../cluster-init/files/nd96isr_v5.conf | 33 +------ 10 files changed, 54 insertions(+), 712 deletions(-) diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-16rs_v3.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-16rs_v3.conf index ba62c3e8c..40050bb47 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-16rs_v3.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-16rs_v3.conf @@ -5,76 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Fri Jun 17 15:26:28 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Firmware Version: Not Specified" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda2" -f "/" - * || check_fs_mount_rw -t "selinuxfs" -s "selinuxfs" -f "/sys/fs/selinux" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda1" -f "/boot" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt/resource" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_free /boot 40MB @@ -116,17 +56,4 @@ ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -r rpcbind - * || check_ps_service -r -d qmgr postfix - * || check_ps_service -S -u root atd - * || check_ps_service -S -u root crond - * || check_ps_service -S -u root sshd - * || check_ps_service -r -d rpc.statd nfslock - * || check_ps_service -S -d rsyslogd -u root rsyslog - -####################################################################### -#### -#### Additional memory checks -#### - * || check_cpu_drop_mem_cache 1000000 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-32rs_v3.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-32rs_v3.conf index e64592dd1..c6b32c8bf 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-32rs_v3.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-32rs_v3.conf @@ -5,76 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Fri Jun 17 15:26:28 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Firmware Version: Not Specified" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda2" -f "/" - * || check_fs_mount_rw -t "selinuxfs" -s "selinuxfs" -f "/sys/fs/selinux" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda1" -f "/boot" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt/resource" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_free /boot 40MB @@ -116,17 +56,4 @@ ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -r rpcbind - * || check_ps_service -r -d qmgr postfix - * || check_ps_service -S -u root atd - * || check_ps_service -S -u root crond - * || check_ps_service -S -u root sshd - * || check_ps_service -r -d rpc.statd nfslock - * || check_ps_service -S -d rsyslogd -u root rsyslog - -####################################################################### -#### -#### Additional memory checks -#### - * || check_cpu_drop_mem_cache 1000000 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-64rs_v3.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-64rs_v3.conf index e516e6910..4a9ff4c6c 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-64rs_v3.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-64rs_v3.conf @@ -5,76 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Fri Jun 17 15:26:28 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Firmware Version: Not Specified" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda2" -f "/" - * || check_fs_mount_rw -t "selinuxfs" -s "selinuxfs" -f "/sys/fs/selinux" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda1" -f "/boot" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt/resource" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_free /boot 40MB @@ -116,17 +56,4 @@ ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -r rpcbind - * || check_ps_service -r -d qmgr postfix - * || check_ps_service -S -u root atd - * || check_ps_service -S -u root crond - * || check_ps_service -S -u root sshd - * || check_ps_service -r -d rpc.statd nfslock - * || check_ps_service -S -d rsyslogd -u root rsyslog - -####################################################################### -#### -#### Additional memory checks -#### - * || check_cpu_drop_mem_cache 1000000 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-96rs_v3.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-96rs_v3.conf index 3eb5f001b..fb39d09ad 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-96rs_v3.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120-96rs_v3.conf @@ -5,76 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Fri Jun 17 15:26:28 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Firmware Version: Not Specified" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda2" -f "/" - * || check_fs_mount_rw -t "selinuxfs" -s "selinuxfs" -f "/sys/fs/selinux" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda1" -f "/boot" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt/resource" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_free /boot 40MB @@ -116,17 +56,4 @@ ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -r rpcbind - * || check_ps_service -r -d qmgr postfix - * || check_ps_service -S -u root atd - * || check_ps_service -S -u root crond - * || check_ps_service -S -u root sshd - * || check_ps_service -r -d rpc.statd nfslock - * || check_ps_service -S -d rsyslogd -u root rsyslog - -####################################################################### -#### -#### Additional memory checks -#### - * || check_cpu_drop_mem_cache 1000000 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120rs_v3.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120rs_v3.conf index b7a0372e0..6359aeac0 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120rs_v3.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/hb120rs_v3.conf @@ -5,76 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Fri Jun 17 15:26:28 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V73X 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3525 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 1850 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Firmware Version: Not Specified" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Memory Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Firmware Version: Not Specified" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda2" -f "/" - * || check_fs_mount_rw -t "selinuxfs" -s "selinuxfs" -f "/sys/fs/selinux" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "xfs" -s "/dev/sda1" -f "/boot" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt/resource" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_free /boot 40MB @@ -116,17 +56,4 @@ ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -r rpcbind - * || check_ps_service -r -d qmgr postfix - * || check_ps_service -S -u root atd - * || check_ps_service -S -u root crond - * || check_ps_service -S -u root sshd - * || check_ps_service -r -d rpc.statd nfslock - * || check_ps_service -S -d rsyslogd -u root rsyslog - -####################################################################### -#### -#### Additional memory checks -#### - * || check_cpu_drop_mem_cache 1000000 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48ads_v4.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48ads_v4.conf index ff5823119..10e8eba83 100644 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48ads_v4.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc48ads_v4.conf @@ -5,71 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Wed May 18 18:11:44 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Clock Speed: Unknown" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_rw -t "ext4" -s "/dev/root" -f "/" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run/lock" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "fusectl" -s "fusectl" -f "/sys/fs/fuse/connections" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt" - * || check_fs_mount_rw -t "tracefs" -s "tracefs" -f "/sys/kernel/debug/tracing" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs*" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_used /boot/efi 90% @@ -85,7 +30,6 @@ * || check_fs_iused /sched 100% * || check_fs_iused /shared 98% * || check_fs_iused /mnt/resource_nvme 98% - * || check_raid /mnt/resource_nvme 2 ####################################################################### ### @@ -96,8 +40,8 @@ * || check_hw_swap 0kB 0kB 3% * || check_hw_eth lo * || check_hw_eth eth0 - * || check_hw_eth docker0 * || check_hw_eth eth1 + * || check_hw_eth docker0 ####################################################################### @@ -105,27 +49,16 @@ ### Process checks ### * || check_ps_loadavg 96 - * || check_ps_service -S -u root -m 'sshd:' sshd - * || check_ps_service -r -d rpc.statd nfslock - ####################################################################### #### -#### Additional memory checks +#### GPU checks #### - * || check_cpu_drop_mem_cache 1000000 - - -####################################################################### -##### -##### GPU checks -##### + * || check_gpu_count 4 + * || check_gpu_xid * || check_nvsmi_healthmon * || check_gpu_persistence - * || check_nv_healthmon * || check_app_gpu_clocks - * || check_cuda_bw 24.0 10 - * || check_gpu_ecc 20000000 10000 * || check_gpu_clock_throttling - * || check_gpu_xid - * || check_nccl_allreduce 56.0 10 + # Path and env var AZ_NHC_ROOT defined in container + * || check_nccl_allreduce 56.0 10 $AZ_NHC_ROOT/topofiles/ncv5-topo.xml 16G diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc96ads_v4.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc96ads_v4.conf index 31b043afa..f35b36709 100644 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc96ads_v4.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nc96ads_v4.conf @@ -5,71 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Wed May 18 18:11:44 UTC 2022 +# Fri Jan 14 21:17:09 UTC 2022 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Clock Speed: Unknown" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_rw -t "ext4" -s "/dev/root" -f "/" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run/lock" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "fusectl" -s "fusectl" -f "/sys/fs/fuse/connections" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt" - * || check_fs_mount_rw -t "tracefs" -s "tracefs" -f "/sys/kernel/debug/tracing" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs*" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_used /boot/efi 90% @@ -85,7 +30,6 @@ * || check_fs_iused /sched 100% * || check_fs_iused /shared 98% * || check_fs_iused /mnt/resource_nvme 98% - * || check_raid /mnt/resource_nvme 4 ####################################################################### ### @@ -96,36 +40,25 @@ * || check_hw_swap 0kB 0kB 3% * || check_hw_eth lo * || check_hw_eth eth0 - * || check_hw_eth docker0 * || check_hw_eth eth1 - + * || check_hw_eth docker0 ####################################################################### ### ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -S -u root -m 'sshd:' sshd - * || check_ps_service -r -d rpc.statd nfslock - ####################################################################### #### -#### Additional memory checks +#### GPU checks #### - * || check_cpu_drop_mem_cache 1000000 - - -####################################################################### -##### -##### GPU checks -##### + * || check_gpu_count 4 + * || check_gpu_xid * || check_nvsmi_healthmon * || check_gpu_persistence - * || check_nv_healthmon * || check_app_gpu_clocks - * || check_cuda_bw 24.0 10 - * || check_gpu_ecc 20000000 10000 * || check_gpu_clock_throttling - * || check_gpu_xid - * || check_nccl_allreduce 25.0 10 +# Path and env var AZ_NHC_ROOT defined in container +* || check_nccl_allreduce 228.0 1 $AZ_NHC_ROOT/topofiles/ndv4-topo.xml 16G + * || check_nccl_allreduce 25.0 10 $AZ_NHC_ROOT/topofiles/ndv4-topo.xml 500M diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96amsr_v4.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96amsr_v4.conf index 002415e05..fdf87edee 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96amsr_v4.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96amsr_v4.conf @@ -5,72 +5,16 @@ # Comments begin with '#' # # This file was automatically generated by nhc-genconf -# Wed May 18 18:11:44 UTC 2022 +# Sat May 6 00:05:37 UTC 2023 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Clock Speed: Unknown" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_rw -t "ext4" -s "/dev/root" -f "/" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run/lock" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "fusectl" -s "fusectl" -f "/sys/fs/fuse/connections" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt" -# * || check_fs_mount_rw -t "fuse.lxcfs" -s "lxcfs" -f "/var/lib/lxcfs" - * || check_fs_mount_rw -t "tracefs" -s "tracefs" -f "/sys/kernel/debug/tracing" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs*" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_used /boot/efi 90% @@ -86,7 +30,6 @@ * || check_fs_iused /sched 100% * || check_fs_iused /shared 98% * || check_fs_iused /mnt/resource_nvme 98% - * || check_raid /mnt/resource_nvme 8 ####################################################################### ### @@ -104,49 +47,37 @@ * || check_hw_ib 200 mlx5_ib5:1 * || check_hw_ib 200 mlx5_ib6:1 * || check_hw_ib 200 mlx5_ib7:1 - * || check_hw_eth ib1 * || check_hw_eth lo - * || check_hw_eth ib4 - * || check_hw_eth ib2 - * || check_hw_eth ib7 * || check_hw_eth eth0 - * || check_hw_eth ib5 - * || check_hw_eth ib3 - * || check_hw_eth ib6 - * || check_hw_eth docker0 * || check_hw_eth eth1 + * || check_hw_eth docker0 * || check_hw_eth ib0 - + * || check_hw_eth ib1 + * || check_hw_eth ib2 + * || check_hw_eth ib3 + * || check_hw_eth ib4 + * || check_hw_eth ib5 + * || check_hw_eth ib6 + * || check_hw_eth ib7 ####################################################################### ### ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -S -u root -m 'sshd:' sshd - * || check_ps_service -r -d rpc.statd nfslock - ####################################################################### #### -#### Additional memory checks +#### GPU checks #### - * || check_cpu_drop_mem_cache 1000000 - - -####################################################################### -##### -##### GPU checks -##### + * || check_gpu_count 8 * || check_gpu_xid * || check_nvsmi_healthmon * || check_gpu_persistence - * || check_nv_healthmon * || check_app_gpu_clocks - * || check_cuda_bw 24.0 10 - * || check_gpu_ecc 20000000 10000 * || check_gpu_clock_throttling - * || check_nccl_allreduce 228.0 10 +# Path and env var AZ_NHC_ROOT defined in container + * || check_nccl_allreduce 228.0 10 $AZ_NHC_ROOT/topofiles/ndv5-topo.xml 16G ######################################################################## @@ -154,5 +85,6 @@ ##### Additional IB checks ##### * || check_ib_bw_gdr 185.0 -* || check_nccl_allreduce_ib_loopback 18.0 10 +# Path and env var AZ_NHC_ROOT defined in container +* || check_nccl_allreduce_ib_loopback 18.0 10 $AZ_NHC_ROOT/topofiles/ndv5-topo.xml 500M * || check_ib_link_flapping 6 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96asr_v4.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96asr_v4.conf index 66f468c4c..47bd6a515 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96asr_v4.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96asr_v4.conf @@ -8,69 +8,13 @@ # Fri Jan 14 21:17:09 UTC 2022 # -####################################################################### -### -### NHC Configuration Variables -### -# * || export MARK_OFFLINE=1 NHC_CHECK_ALL=0 - - -####################################################################### -### -### DMI Checks -### -# * || check_dmi_data_match -h 0x0000 -t 0 "BIOS Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0001 -t 1 "System Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0002 -t 3 "Chassis Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0003 -t 2 "Base Board Information: Version: Hyper-V UEFI Release v4.1" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0004 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Version: AMD EPYC 7V12 64-Core Processor " -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Max Speed: 3300 MHz" -# * || check_dmi_data_match -h 0x0005 -t 4 "Processor Information: Current Speed: 2450 MHz" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0008 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000B -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x000E -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0011 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0014 -t 17 "Memory Device: Configured Clock Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Speed: Unknown" -# * || check_dmi_data_match -h 0x0017 -t 17 "Memory Device: Configured Clock Speed: Unknown" - - ####################################################################### ### ### Filesystem checks ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_rw -t "ext4" -s "/dev/root" -f "/" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run/lock" - * || check_fs_mount_ro -t "tmpfs" -s "tmpfs" -f "/sys/fs/cgroup" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "fusectl" -s "fusectl" -f "/sys/fs/fuse/connections" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt" -# * || check_fs_mount_rw -t "fuse.lxcfs" -s "lxcfs" -f "/var/lib/lxcfs" - * || check_fs_mount_rw -t "tracefs" -s "tracefs" -f "/sys/kernel/debug/tracing" - * || check_fs_mount_rw -t "nfs4" -s "*:/*" -f "/sched" - * || check_fs_mount_rw -t "nfs*" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_used /boot/efi 90% @@ -84,7 +28,6 @@ * || check_fs_iused /sched 100% * || check_fs_iused /shared 98% * || check_fs_iused /mnt/resource_nvme 98% - * || check_raid /mnt/resource_nvme 8 ####################################################################### ### @@ -102,54 +45,43 @@ * || check_hw_ib 200 mlx5_ib5:1 * || check_hw_ib 200 mlx5_ib6:1 * || check_hw_ib 200 mlx5_ib7:1 - * || check_hw_eth ib7 - * || check_hw_eth ib3 - * || check_hw_eth docker0 * || check_hw_eth lo - * || check_hw_eth ib6 - * || check_hw_eth ib5 - * || check_hw_eth ib4 - * || check_hw_eth ib0 - * || check_hw_eth ib2 * || check_hw_eth eth0 + * || check_hw_eth eth1 + * || check_hw_eth docker0 + * || check_hw_eth ib0 * || check_hw_eth ib1 - + * || check_hw_eth ib2 + * || check_hw_eth ib3 + * || check_hw_eth ib4 + * || check_hw_eth ib5 + * || check_hw_eth ib6 + * || check_hw_eth ib7 ####################################################################### ### ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -S -u root -m 'sshd:' sshd - * || check_ps_service -r -d rpc.statd nfslock - - -####################################################################### -#### -#### Additional memory checks -#### - * || check_cpu_drop_mem_cache 1000000 - ####################################################################### #### #### GPU checks #### + * || check_gpu_count 8 * || check_gpu_xid * || check_nvsmi_healthmon * || check_gpu_persistence - * || check_nv_healthmon * || check_app_gpu_clocks - * || check_cuda_bw 24.0 10 - * || check_gpu_ecc 20000000 10000 * || check_gpu_clock_throttling - * || check_nccl_allreduce 228.0 10 - +# Path and env var AZ_NHC_ROOT defined in container +* || check_nccl_allreduce 228.0 1 $AZ_NHC_ROOT/topofiles/ndv4-topo.xml 16G ####################################################################### #### #### Additional IB checks #### * || check_ib_bw_gdr 185.0 - * || check_nccl_allreduce_ib_loopback 18.0 10 +# Path and env var AZ_NHC_ROOT defined in container + * || check_nccl_allreduce_ib_loopback 18.0 10 $AZ_NHC_ROOT/topofiles/ndv4-topo.xml 500M * || check_ib_link_flapping 6 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96isr_v5.conf b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96isr_v5.conf index 5d9740aee..b01c6ff77 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96isr_v5.conf +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96isr_v5.conf @@ -14,26 +14,7 @@ ### * || check_fs_mount_rw -t "sysfs" -s "sysfs" -f "/sys" * || check_fs_mount_rw -t "proc" -s "proc" -f "/proc" - * || check_fs_mount_rw -t "devtmpfs" -s "devtmpfs" -f "/dev" - * || check_fs_mount_rw -t "devpts" -s "devpts" -f "/dev/pts" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run" - * || check_fs_mount_rw -t "ext4" -s "/dev/root" -f "/" - * || check_fs_mount_rw -t "securityfs" -s "securityfs" -f "/sys/kernel/security" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/dev/shm" - * || check_fs_mount_rw -t "tmpfs" -s "tmpfs" -f "/run/lock" - * || check_fs_mount_rw -t "pstore" -s "pstore" -f "/sys/fs/pstore" - * || check_fs_mount_rw -t "efivarfs" -s "efivarfs" -f "/sys/firmware/efi/efivars" - * || check_fs_mount_rw -t "hugetlbfs" -s "hugetlbfs" -f "/dev/hugepages" - * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" - * || check_fs_mount_rw -t "debugfs" -s "debugfs" -f "/sys/kernel/debug" - * || check_fs_mount_rw -t "vfat" -s "/dev/sda15" -f "/boot/efi" - * || check_fs_mount_rw -t "fusectl" -s "fusectl" -f "/sys/fs/fuse/connections" - * || check_fs_mount_rw -t "configfs" -s "configfs" -f "/sys/kernel/config" - * || check_fs_mount_rw -t "ext4" -s "/dev/sdb1" -f "/mnt" - * || check_fs_mount_rw -t "tracefs" -s "tracefs" -f "/sys/kernel/debug/tracing" - * || check_fs_mount_rw -t "nfs4" -s "*:/sched" -f "/sched" - * || check_fs_mount_rw -t "nfs*" -s "*:/*" -f "/shared" - * || check_fs_mount_rw -t "xfs" -s "/dev/*" -f "/mnt/resource_nvme" + * || check_fs_mount_rw -t "mqueue" -s "mqueue" -f "/dev/mqueue" * || check_fs_used /dev 90% * || check_fs_used / 90% * || check_fs_used /boot/efi 90% @@ -49,7 +30,6 @@ * || check_fs_iused /sched 100% * || check_fs_iused /shared 98% * || check_fs_iused /mnt/resource_nvme 98% - * || check_raid /mnt/resource_nvme 8 ####################################################################### ### @@ -85,8 +65,6 @@ ### Process checks ### * || check_ps_loadavg 192 - * || check_ps_service -S -u root -m 'sshd:' sshd - * || check_ps_service -r -d rpc.statd nfslock ####################################################################### #### @@ -96,17 +74,16 @@ * || check_gpu_xid * || check_nvsmi_healthmon * || check_gpu_persistence - * || check_nv_healthmon * || check_app_gpu_clocks - * || check_cuda_bw 52 10 - * || check_gpu_ecc 20000000 10000 * || check_gpu_clock_throttling - * || check_nccl_allreduce 460.0 10 +# Path and env var AZ_NHC_ROOT defined in container + * || check_nccl_allreduce 460.0 1 $AZ_NHC_ROOT/topofiles/ndv5-topo.xml 16G ####################################################################### #### #### Additional IB checks #### * || check_ib_bw_gdr 375 - * || check_nccl_allreduce_ib_loopback 40.0 10 +# Path and env var AZ_NHC_ROOT defined in container + * || check_nccl_allreduce_ib_loopback 40 10 $AZ_NHC_ROOT/topofiles/ndv5-topo.xml 500M * || check_ib_link_flapping 6 From ac872ba120400eb6ddc8d747fda4ce0bb3d5a6df Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 09:19:09 -0700 Subject: [PATCH 04/10] [wip] update configure_nhc.sh: use run_nhc.sh for health checks and remove duplicated tests --- .../specs/default/cluster-init/files/configure_nhc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/configure_nhc.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/configure_nhc.sh index b7262217d..470d0302d 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/configure_nhc.sh +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/configure_nhc.sh @@ -10,7 +10,7 @@ NHC_TIMEOUT=300 NHC_VERBOSE=1 NHC_DETACHED_MODE=1 NHC_DEBUG=0 -NHC_EXE=/usr/sbin/nhc +NHC_EXE=/sched/scripts/run_aznhc.sh NHC_NVIDIA_HEALTHMON=dcgmi NHC_NVIDIA_HEALTHMON_ARGS="diag -r 1" SLURM_CONF=/etc/slurm/slurm.conf @@ -21,7 +21,7 @@ NHC_EPILOG=0 AUTOSCALING=0 PROLOG_NOHOLD_REQUEUE=0 PROLOG_RUN_NHC=0 -NHC_EXTRA_TEST_FILES="csc_nvidia_smi.nhc azure_cuda_bandwidth.nhc azure_gpu_app_clocks.nhc azure_gpu_ecc.nhc azure_gpu_persistence.nhc azure_ib_write_bw_gdr.nhc azure_nccl_allreduce_ib_loopback.nhc azure_ib_link_flapping.nhc azure_gpu_clock_throttling.nhc azure_cpu_drop_cache_mem.nhc azure_gpu_xid.nhc azure_nccl_allreduce.nhc azure_raid_health.nhc" +NHC_EXTRA_TEST_FILES="azure_raid_health.nhc" source $CYCLECLOUD_SPEC_PATH/files/common_functions.sh From feacbec31b37a658f2d12a9ecc1cef2148f3ca05 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 09:32:06 -0700 Subject: [PATCH 05/10] [wip] Change NHC bin to use AzNHC in AzHPC image --- .../specs/default/cluster-init/files/run_nhc.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/run_nhc.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/run_nhc.sh index ccdc58c07..adca2e555 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/run_nhc.sh +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/run_nhc.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Comma separated mounts to pass to run-health-checks.sh +MOUNTS='/mnt/resource_nvme' +LOG_FILE='/var/log/nhc.log' +NHC_CMD="sudo /shared/home/cycleadmin/azurehpc-health-checks/run-health-checks.sh -d $MOUNTS -c /etc/nhc/nhc.conf -o $LOG_FILE" + function set_detached_mode() { TARGET_MODE=$1 sudo sed -i "s/DETACHED_MODE.*/DETACHED_MODE=${TARGET_MODE}/g" /etc/default/nhc @@ -29,7 +34,7 @@ NHC_RC=0 if [ $exclusive_node_rc -eq 0 ]; then set_detached_mode 0 echo "[$prolog_eplilog] execute nhc" >> /var/log/nhc.log - sudo /usr/sbin/nhc + NHC_CMD NHC_RC=$? set_detached_mode 1 fi From 59867fb76b89aa12672d5eb7aaf31668e86420f8 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 10:45:24 -0700 Subject: [PATCH 06/10] [wip] Update check for aznhc container and removed detached mode checks --- .../cluster-init/files/wait_for_nhc.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/wait_for_nhc.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/wait_for_nhc.sh index 512ddf8fe..08a22a832 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/wait_for_nhc.sh +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/wait_for_nhc.sh @@ -4,13 +4,19 @@ PROLOG_RUN_NHC=$2 while [ ! -f /usr/sbin/nhc ];do sleep 2 -echo "[Prolog] waiting for /usr/sbin/nhc" >> /var/log/nhc.log +echo "[Prolog] waiting for aznhc" >> /var/log/nhc.log done -pid=`ps -ef | grep -v grep | grep /usr/sbin/nhc | tr -s ' ' | cut -d ' ' -f2 | head -n 1` +# Is AzNHC running? +# - Returns 1 if running, returns 0 if not running. +function is_aznhc_running() { + sudo docker ps --format '{{.Names}}' | grep -v -q "^aznhc$" +} -if [ -n "$pid" ]; then - while ps -p $pid > /dev/null 2>&1 +is_aznhc_running +aznhc_rc=$? +if [ aznhc_rc -eq 1 ]; then + while is_aznhc_running > /dev/null 2>&1 do sleep 10 TIMESTAMP=$(/bin/date '+%Y%m%d %H:%M:%S') @@ -19,11 +25,6 @@ if [ -n "$pid" ]; then TIMESTAMP=$(/bin/date '+%Y%m%d %H:%M:%S') echo "${TIMESTAMP} [prolog] NHC processes finished and job can start" >> /var/log/nhc.log - if [ -f /var/run/nhc/nhc.status ]; then - exit 1 - else - exit 0 - fi elif [[ $PROLOG_RUN_NHC == 1 ]]; then /sched/scripts/run_nhc.sh fi From 864ef31058511c2324f57df2c07e3573114c0014 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 10:50:23 -0700 Subject: [PATCH 07/10] [wip] replace killing process with stopping container --- .../specs/default/cluster-init/files/kill_nhc.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/kill_nhc.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/kill_nhc.sh index 8bd621d90..2f50ff00e 100755 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/kill_nhc.sh +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/kill_nhc.sh @@ -1,7 +1,8 @@ #!/bin/bash -# Kill NHC processes -pkill -9 -f /usr/sbin/nhc +AZNHC_CONT_NAME='aznhc' +# Stop AzNHC container +docker stop $AZNHC_CONT_NAME TIMESTAMP=$(/bin/date '+%Y%m%d %H:%M:%S') echo "${TIMESTAMP} [prolog] NHC processes killed at job start" >> /var/log/nhc.log From 12c026fbb89ce877d91d34d92cc90b8383cf00b7 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 11:56:16 -0700 Subject: [PATCH 08/10] [wip] remove unnecessary install nhc scripts --- .../default/cluster-init/files/install_nhc.sh | 49 ------------------- .../cluster-init/scripts/000_install_nhc.sh | 4 -- 2 files changed, 53 deletions(-) delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/install_nhc.sh delete mode 100755 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/000_install_nhc.sh diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/install_nhc.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/install_nhc.sh deleted file mode 100755 index bd98bab46..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/install_nhc.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Tagged Version 1.4.2 failed the tests (lbnl_file.nhc) - -TMPDIR=/tmp -#TAR_FILE=$CYCLECLOUD_SPEC_PATH/files/lbnl-nhc-01-16-2022.tar.gz -TAR_FILE=$CYCLECLOUD_SPEC_PATH/files/ - - -source $CYCLECLOUD_SPEC_PATH/files/common_functions.sh - - -function get_source() { - - if [[ -f $TAR_FILE ]] - then - tar xvf $TAR_FILE - else - git clone https://github.com/mej/nhc.git - fi -} - - -function install_from_source() { - - if ! [[ -f /usr/sbin/nhc ]] && ! [[ -d /etc/nhc ]] - then - cd $TMPDIR - get_source - cd nhc - ./autogen.sh - if [ -f /etc/centos-release ]; then - ./configure --prefix=/usr --sysconfdir=/etc --libexecdir=/usr/libexec - else - ./configure --prefix=/usr --sysconfdir=/etc --libexecdir=/usr/lib - fi - make test - make install - - rm -rf $TMPDIR/nhc - else - echo "Warning: Did not install NHC (looks like it already has been installed)" - fi - -} - -if ! is_slurm_controller; then - install_from_source -fi diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/000_install_nhc.sh b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/000_install_nhc.sh deleted file mode 100755 index b5b610e7c..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/000_install_nhc.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -chmod +x $CYCLECLOUD_SPEC_PATH/files/install_nhc.sh -$CYCLECLOUD_SPEC_PATH/files/install_nhc.sh From 5b1c45c9fc98baf6379c4d2c78ffd0ed4b2c4c39 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 11:58:42 -0700 Subject: [PATCH 09/10] [wip] remove script to create nhc tarball --- experimental/cc_slurm_nhc/create_nhc_src_tar.sh | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100755 experimental/cc_slurm_nhc/create_nhc_src_tar.sh diff --git a/experimental/cc_slurm_nhc/create_nhc_src_tar.sh b/experimental/cc_slurm_nhc/create_nhc_src_tar.sh deleted file mode 100755 index cdc2b4f4f..000000000 --- a/experimental/cc_slurm_nhc/create_nhc_src_tar.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -TODAY=$(date +"%m-%d-%Y") -TAR_FILE=lbnl-nhc-${TODAY}.tar -CWD=`pwd` - -cd /tmp -git clone https://github.com/mej/nhc.git -tar -cvf ${CWD}/$TAR_FILE nhc -gzip ${CWD}/$TAR_FILE -rm -rf /tmp/nhc - From 9439824ebcdf3be435c597c8d852fffe39ad411a Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 14 May 2024 13:39:59 -0700 Subject: [PATCH 10/10] [wip] remove unneessary readme files. Major version increment in project.ini --- experimental/cc_slurm_nhc/cc_slurm_nhc/project.ini | 2 +- .../specs/default/cluster-init/files/README.txt | 5 ----- .../specs/default/cluster-init/scripts/README.txt | 10 ---------- 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/README.txt delete mode 100644 experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/README.txt diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/project.ini b/experimental/cc_slurm_nhc/cc_slurm_nhc/project.ini index f2706d2c3..1aa796a97 100644 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/project.ini +++ b/experimental/cc_slurm_nhc/cc_slurm_nhc/project.ini @@ -1,5 +1,5 @@ [project] name = cc_slurm_nhc type = application -version = 1.0.0 +version = 2.0.0 diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/README.txt b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/README.txt deleted file mode 100644 index e110007a4..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/README.txt +++ /dev/null @@ -1,5 +0,0 @@ - -Files in this directory are automatically synced to any node using this spec. Content here -can be anything from software packages to config files. Scripts can be used to install -software packages or move files into the appropriate location on the node. - \ No newline at end of file diff --git a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/README.txt b/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/README.txt deleted file mode 100644 index 66c952f98..000000000 --- a/experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/scripts/README.txt +++ /dev/null @@ -1,10 +0,0 @@ - -Files in this directory are executed on the host in alphabetical order. -It is recommended that files are named start with digits to ensure they -are executed in the correct order, example: - - 000_run_me_first.sh - - 001_run_me_second.sh - -Allowable file extensions on Linux: .sh -Allowable file extensions on Windows: .bat, .cmd, .exe - \ No newline at end of file