From ff382ffe46480d9a4abc078b652be384f5819e32 Mon Sep 17 00:00:00 2001 From: Adrien Bernede <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 4 Jun 2026 11:31:42 +0200 Subject: [PATCH 1/3] Add collapsible GitLab CI sections to build script Assisted-By: Copilot --- scripts/gitlab/build_and_test.sh | 332 +++++++++++++++++++++++++------ 1 file changed, 267 insertions(+), 65 deletions(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 0500f0fa..ca4d2860 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -13,9 +13,25 @@ fi # SPDX-License-Identifier: BSD-3-Clause ############################################################################### +# Navigation: +# - VARIABLES +# - HELPER FUNCTIONS +# - SETUP +# - BUILD DEPENDENCIES +# - HOST CONFIG / CMAKE CACHE FILES +# - BUILD PROJECT +# - TEST PROJECT +# - CLEANUP + +############################################################################### +# VARIABLES +############################################################################### + set -o errexit set -o nounset +exec 2>&1 + option=${1:-""} hostname="$(hostname)" truehostname=${hostname//[0-9]/} @@ -37,12 +53,157 @@ ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/radiuss/care"} export ci_registry_user=${CI_REGISTRY_USER:-"${USER}"} export ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"} -timed_message () +############################################################################### +# HELPER FUNCTIONS +############################################################################### + +# Helper function to print errors in red +print_error () +{ + local error_msg="${1}" + echo -e "\e[31m[Error]: ${error_msg}\e[0m" +} + +# Helper function to print warnings in gray +print_warning () +{ + local warning_msg="${1}" + echo -e "\e[1;30m[Warning]: ${warning_msg}\e[0m" +} + +# Helper function to print information +print_info () +{ + local info_msg="${1}" + echo -e "[Information]: ${info_msg}" +} + +# Portable UTC timestamp formatter for epoch seconds. +format_utc_timestamp () +{ + local timestamp="${1}" + # BSD/macOS date supports epoch conversion via: date -r + if date -u -r "${timestamp}" "+%Y-%m-%d %H:%M:%S UTC" >/dev/null 2>&1 + then + date -u -r "${timestamp}" "+%Y-%m-%d %H:%M:%S UTC" + else + # GNU date supports epoch conversion via: date -d "@" + date -u -d "@${timestamp}" "+%Y-%m-%d %H:%M:%S UTC" + fi +} + +# Portable elapsed time formatter (HH:MM:SS). +format_elapsed_hms () { - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~ $(date --rfc-3339=seconds) ~ ${1}" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + local elapsed="${1}" + printf '%02d:%02d:%02d' $((elapsed / 3600)) $(((elapsed % 3600) / 60)) $((elapsed % 60)) } + +# Track script start time for elapsed time calculations +script_start_time=$(date +%s) + +# Storage for section start times (supports nesting) +declare -A section_start_times + +# Section stack for tracking nested sections +section_id_stack=() +section_counter=0 +section_indent="" + +# GitLab CI collapsible section helpers with nesting support +section_start () +{ + local section_name="${1}" + local section_title="${2}" + local section_state="${3:-""}" + + local collapsed="false" + if [[ "${section_state}" == "collapsed" ]] + then + collapsed="true" + fi + + # Generate unique section ID + section_counter=$((section_counter + 1)) + local section_id="${section_name}_${section_counter}" + + local timestamp=$(date +%s) + local current_time=$(format_utc_timestamp "${timestamp}") + local total_elapsed=$((timestamp - script_start_time)) + local total_elapsed_formatted=$(format_elapsed_hms "${total_elapsed}") + + # Store section start time for later calculation + section_start_times[${section_id}]=${timestamp} + + # Push section ID onto stack + section_id_stack+=("${section_id}") + + echo -e "\e[1;30m${section_indent}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\e[0m" + echo -e "\e[1;30m${section_indent}~ TIME | TOTAL | SECTION \e[0m" + echo -e "\e[1;30m${section_indent}~ ${current_time} | ${total_elapsed_formatted} | ${section_title}\e[0m" + echo -e "\e[0Ksection_start:${timestamp}:${section_id}[collapsed=${collapsed}]\r\e[0K${section_indent}~ ${section_title}" + + # Increase indentation for nested sections + section_indent="${section_indent} " +} + +section_end () +{ + # Pop section ID from stack + if [[ ${#section_id_stack[@]} -eq 0 ]]; then + print_warning "section_end called with empty stack" + return 1 + fi + + # Decrease indentation before displaying + section_indent="${section_indent% }" + + local stack_index=$((${#section_id_stack[@]} - 1)) + local section_id="${section_id_stack[$stack_index]}" + unset section_id_stack[$stack_index] + + local timestamp=$(date +%s) + local current_time=$(format_utc_timestamp "${timestamp}") + local total_elapsed=$((timestamp - script_start_time)) + local total_elapsed_formatted=$(format_elapsed_hms "${total_elapsed}") + + # Calculate section elapsed time + local section_start=${section_start_times[${section_id}]:-${timestamp}} + local section_elapsed=$((timestamp - section_start)) + local section_elapsed_formatted=$(format_elapsed_hms "${section_elapsed}") + + echo -e "\e[0Ksection_end:${timestamp}:${section_id}\r\e[0K\e[0m" + echo -e "\e[1;30m${section_indent}~ ${current_time} | ${total_elapsed_formatted} | ${section_elapsed_formatted}\e[0m" + echo -e "\e[1;30m${section_indent}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\e[0m" + + # Clean up stored time + unset section_start_times[${section_id}] +} + +# For convenience, a helper function to run a command within a section and handle errors +run_section () +{ + local id="$1" + local title="$2" + local collapsed="$3" + local err_msg="$4" + local status=0 + shift 4 + + section_start "$id" "$title" "$collapsed" + if "$@"; then + section_end + else + status=$? + section_end + print_error "$err_msg" + exit $status + fi +} + +############################################################################### +# SETUP +############################################################################### camp_version=${UPDATE_CAMP:-""} raja_version=${UPDATE_RAJA:-""} umpire_version=${UPDATE_UMPIRE:-""} @@ -50,12 +211,10 @@ chai_version=${UPDATE_CHAI:-""} if [[ ${debug_mode} == true ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Debug mode:" - echo "~~~~~ - Spack debug mode." - echo "~~~~~ - Deactivated shared memory." - echo "~~~~~ - Do not push to buildcache." - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + print_info "Debug mode:" + print_info "- Spack debug mode." + print_info "- Deactivated shared memory." + print_info "- Do not push to buildcache." use_dev_shm=false spack_debug=true push_to_registry=false @@ -63,7 +222,7 @@ fi if [[ -n ${module_list} ]] then - timed_message "Modules to load: ${module_list}" + print_info "Loading modules: ${module_list}" module load ${module_list} fi @@ -86,28 +245,30 @@ else prefix="${project_dir}/../spack-and-build-root" fi -echo "Creating directory ${prefix}" -echo "project_dir: ${project_dir}" +print_info "Creating directory ${prefix}" +print_info "project_dir: ${project_dir}" mkdir -p ${prefix} spack_cmd="${prefix}/spack/bin/spack" spack_env_path="${prefix}/spack_env" -uberenv_cmd="./scripts/uberenv/uberenv.py" +uberenv_cmd="${project_dir}/scripts/uberenv/uberenv.py" if [[ ${spack_debug} == true ]] then spack_cmd="${spack_cmd} --debug --stacktrace" uberenv_cmd="${uberenv_cmd} --spack-debug" fi -# Dependencies +############################################################################### +# BUILD DEPENDENCIES +############################################################################### if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then - timed_message "Building dependencies" + section_start "dependencies" "Building Dependencies" if [[ -z ${spec} ]] then - echo "[Error]: SPEC is undefined, aborting..." + section_end ; print_error "SPEC is undefined, aborting..." exit 1 fi @@ -146,45 +307,51 @@ then mkdir -p ${spack_user_cache} # generate cmake cache file with uberenv and radiuss spack package - timed_message "Spack setup and environment" - ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt} + run_section "spack_setup" "Spack setup and environment" "collapsed" \ + "Spack environment setup failed (Uberenv)" \ + ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt} if [[ -n ${ci_registry_token} ]] then - timed_message "GitLab registry as Spack Buildcache" - ${spack_cmd} -D ${spack_env_path} mirror add --unsigned --oci-username-variable ci_registry_user --oci-password-variable ci_registry_token gitlab_ci oci://${ci_registry_image} + run_section "registry_setup" "GitLab registry as Spack Buildcache" "collapsed" \ + "Adding gitlab registry to spack environment failed" \ + ${spack_cmd} -D ${spack_env_path} mirror add --unsigned --oci-username-variable ci_registry_user --oci-password-variable ci_registry_token gitlab_ci oci://${ci_registry_image} fi - timed_message "Spack build of dependencies" - ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} + run_section "spack_build" "Spack build of dependencies" "collapsed" \ + "Spack build of dependencies failed (Uberenv)" \ + ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} if [[ -n ${ci_registry_token} && ${push_to_registry} == true ]] then - timed_message "Push dependencies to buildcache" - ${spack_cmd} -D ${spack_env_path} buildcache push --only dependencies gitlab_ci + run_section "buildcache_push" "Push dependencies to buildcache" "collapsed" \ + "Pushing dependencies to gitlab registry failed" \ + ${spack_cmd} -D ${spack_env_path} buildcache push --only dependencies gitlab_ci fi - timed_message "Dependencies built" + section_end fi -# Find cmake cache file (hostconfig) +############################################################################### +# HOST CONFIG / CMAKE CACHE FILE +############################################################################### if [[ -z ${hostconfig} ]] then # If no host config file was provided, we assume it was generated. # This means we are looking of a unique one in project dir. - hostconfigs=( $( ls "${project_dir}/"*.cmake ) ) + shopt -s nullglob; hostconfigs=( "${project_dir}"/*.cmake ); shopt -u nullglob if [[ ${#hostconfigs[@]} == 1 ]] then hostconfig_path=${hostconfigs[0]} elif [[ ${#hostconfigs[@]} == 0 ]] then - echo "[Error]: No result for: ${project_dir}/*.cmake" - echo "[Error]: Spack generated host-config not found." + print_error "No result for: ${project_dir}/*.cmake" + print_error "Spack generated host-config not found." exit 1 else - echo "[Error]: More than one result for: ${project_dir}/*.cmake" - echo "[Error]: ${hostconfigs[@]}" - echo "[Error]: Please specify one with HOST_CONFIG variable" + print_error "More than one result for: ${project_dir}/*.cmake" + print_error "${hostconfigs[@]}" + print_error "Please specify one with HOST_CONFIG variable" exit 1 fi else @@ -193,8 +360,11 @@ else fi hostconfig=$(basename ${hostconfig_path}) -echo "[Information]: Found hostconfig ${hostconfig_path}" +print_info "Found hostconfig ${hostconfig_path}" +############################################################################### +# BUILD PROJECT +############################################################################### # Build Directory # When using /dev/shm, we use prefix for both spack builds and source build, unless BUILD_ROOT was defined build_root=${BUILD_ROOT:-"${prefix}"} @@ -202,20 +372,17 @@ build_root=${BUILD_ROOT:-"${prefix}"} build_dir="${build_root}/build_${hostconfig//.cmake/}" install_dir="${build_root}/install_${hostconfig//.cmake/}" -cmake_exe=`grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs` +cmake_exe=$(grep 'CMake executable' ${hostconfig_path} | cut -d ':' -f 2 | xargs 2>/dev/null) -# Build if [[ "${option}" != "--deps-only" && "${option}" != "--test-only" ]] then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "~~~~~ Prefix: ${prefix}" - echo "~~~~~ Host-config: ${hostconfig_path}" - echo "~~~~~ Build Dir: ${build_dir}" - echo "~~~~~ Project Dir: ${project_dir}" - echo "~~~~~ Install Dir: ${install_dir}" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "" - timed_message "Cleaning working directory" + print_info "Prefix ${prefix}" + print_info "Host-config ${hostconfig_path}" + print_info "Build Dir ${build_dir}" + print_info "Project Dir ${project_dir}" + print_info "Install Dir ${install_dir}" + + section_start "clean" "Cleaning working directory" "collapsed" # Map CPU core allocations declare -A core_counts=(["lassen"]=40 ["poodle"]=28 ["dane"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48 ["matrix"]=48) @@ -226,8 +393,8 @@ then # use max cores. rm -rf ${build_dir} 2>/dev/null mkdir -p ${build_dir} && cd ${build_dir} + section_end - timed_message "Building CARE" # We set the MPI tests command to allow overlapping. # Shared allocation: Allows build_and_test.sh to run within a sub-allocation (see CI config). # Use /dev/shm: Prevent MPI tests from running on a node where the build dir doesn't exist. @@ -237,63 +404,98 @@ then cmake_options="-DBLT_MPI_COMMAND_APPEND:STRING=--overlap" fi - $cmake_exe \ + section_start "cmake_config" "CMake Configuration" "collapsed" + if $cmake_exe \ -C ${hostconfig_path} \ ${cmake_options} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ ${project_dir} - if ! $cmake_exe --build . -j ${core_counts[$truehostname]} then - echo "[Error]: Compilation failed, building with verbose output..." - timed_message "Re-building with --verbose" - $cmake_exe --build . --verbose -j 1 + section_end + else + status=$? + section_end ; print_error "CMake configuration failed" + exit ${status} + fi + + section_start "build" "Building CARE" "collapsed" + if $cmake_exe --build . -j ${core_counts[$truehostname]} + then + section_end else - timed_message "Installing" - $cmake_exe --install . + status=$? + section_end ; print_error "Compilation failed, building with verbose output..." + + section_start "build_verbose" "Verbose Rebuild" + $cmake_exe --build . --verbose -j 1 + section_end + + exit ${status} fi - timed_message "CARE built and installed" + run_section "install" "Installing CARE" "collapsed" \ + "Installation failed" \ + $cmake_exe --install . fi -# Test +############################################################################### +# TEST PROJECT +############################################################################### if [[ "${option}" != "--build-only" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then if [[ ! -d ${build_dir} ]] then - echo "[Error]: Build directory not found : ${build_dir}" && exit 1 + print_error "Build directory not found : ${build_dir}" + exit 1 fi cd ${build_dir} - timed_message "Testing CARE" + section_start "tests" "Running Tests" "collapsed" ctest --output-on-failure --no-compress-output -T test -VV 2>&1 | tee tests_output.txt + ctest_status=${PIPESTATUS[0]} no_test_str="No tests were found!!!" if [[ "$(tail -n 1 tests_output.txt)" == "${no_test_str}" ]] then - echo "[Error]: No tests were found" && exit 1 + section_end ; print_error "No tests were found (ctest status: ${ctest_status})" + exit 1 fi - timed_message "Preparing tests xml reports for export" tree Testing xsltproc -o junit.xml ${project_dir}/scripts/radiuss-spack-configs/utilities/ctest-to-junit.xsl Testing/*/Test.xml mv junit.xml ${project_dir}/junit.xml if grep -q "Errors while running CTest" ./tests_output.txt then - echo "[Error]: Failure(s) while running CTest" && exit 1 + section_end ; print_error "Failure(s) while running CTest (ctest status: ${ctest_status})" + exit 1 fi if [[ ! -d ${install_dir} ]] then - echo "[Error]: Install directory not found : ${install_dir}" && exit 1 + section_end ; print_error "Install directory not found : ${install_dir}" + exit 1 fi - timed_message "CARE tests completed" + section_end fi -timed_message "Cleaning up" -make clean +############################################################################### +# CLEANUP +############################################################################### + +section_start "cleanup" "Cleaning up" "collapsed" +if make clean +then + section_end +else + status=$? + section_end ; print_error "Cleanup failed" + exit ${status} +fi -timed_message "Build and test completed" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo "~ Build and test completed" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From 472baaa9352b102b7216c0540b93a3c9da9d6bdb Mon Sep 17 00:00:00 2001 From: Adrien Bernede <51493078+adrienbernede@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:35:57 +0200 Subject: [PATCH 2/3] Print debug outputs when CMake configuration phase failed + minor changes --- scripts/gitlab/build_and_test.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index ca4d2860..31f9c9c9 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -7,7 +7,7 @@ then fi ############################################################################### -# Copyright (c) 2020-25, Lawrence Livermore National Security, LLC and CARE +# Copyright (c) 2020-26, Lawrence Livermore National Security, LLC and CARE # project contributors. See the CARE LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause @@ -385,7 +385,7 @@ then section_start "clean" "Cleaning working directory" "collapsed" # Map CPU core allocations - declare -A core_counts=(["lassen"]=40 ["poodle"]=28 ["dane"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48 ["matrix"]=48) + declare -A core_counts=(["dane"]=28 ["matrix"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48) # If building, then delete everything first # NOTE: 'cmake --build . -j core_counts' attempts to reduce individual build resources. @@ -414,7 +414,14 @@ then section_end else status=$? - section_end ; print_error "CMake configuration failed" + section_end ; print_error "CMake configuration failed, dumping output..." + + $cmake_exe \ + -C ${hostconfig_path} \ + ${cmake_options} \ + -DCMAKE_INSTALL_PREFIX=${install_dir} \ + ${project_dir} --debug-output --trace-expand + exit ${status} fi From 93cdf3e4710b8ab9ffe9b8d523367ff7329cc219 Mon Sep 17 00:00:00 2001 From: Alan Dayton <6393677+adayton1@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:36:54 -0700 Subject: [PATCH 3/3] Remove rzansel core counts --- scripts/gitlab/build_and_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 31f9c9c9..575bcd4b 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -385,7 +385,7 @@ then section_start "clean" "Cleaning working directory" "collapsed" # Map CPU core allocations - declare -A core_counts=(["dane"]=28 ["matrix"]=28 ["corona"]=32 ["rzansel"]=48 ["tioga"]=32 ["tuolumne"]=48) + declare -A core_counts=(["dane"]=28 ["matrix"]=28 ["corona"]=32 ["tioga"]=32 ["tuolumne"]=48) # If building, then delete everything first # NOTE: 'cmake --build . -j core_counts' attempts to reduce individual build resources.