diff --git a/gdb/testsuite/Makefile.in b/gdb/testsuite/Makefile.in index 74e69e5aa2b..78058898bba 100644 --- a/gdb/testsuite/Makefile.in +++ b/gdb/testsuite/Makefile.in @@ -258,7 +258,7 @@ check-single-racy: check-parallel: -rm -f *core* -rm -rf cache outputs temp - -rm -f gpu-parallel.lock + -rm -f gpu-parallel*.lock $(MAKE) -k do-check-parallel; \ result=$$?; \ if test -d outputs; then \ diff --git a/gdb/testsuite/README b/gdb/testsuite/README index 479e81f90f5..7c82ef5694a 100644 --- a/gdb/testsuite/README +++ b/gdb/testsuite/README @@ -354,6 +354,167 @@ by switching to a different user on the same machine. These users will have random files copied into their $HOME directories, so it is a good idea to setup new users just for this purpose. +ROCm GPU Parallel Testing +************************* + +The gdb.rocm/ test suite runs GPU-bound tests under a file-based lock +managed by with_rocm_gpu_lock (see lib/rocm.exp). The lock layer keeps +the suite safe to run with GDB_PARALLEL on multi-GPU nodes by gating +how many tests may share a GPU at once and which physical GPU each +test sees. + +Per-test concurrency marker +--------------------------- + +Each .exp file selects one of three modes by setting +::rocm_gpu_concurrency near its load_lib rocm.exp, before any call to +with_rocm_gpu_lock: + + set rocm_gpu_concurrency parallel + set rocm_gpu_concurrency serial ;# the default + set rocm_gpu_concurrency machine + +The three modes are: + + parallel + Take one slot on any one GPU. Multiple parallel-marked tests + may run simultaneously, spread across GPUs first and stacked on + the same GPU only once every GPU has at least one tenant. Use + this for tests that launch a small kernel (typically one + workgroup with a handful of workitems) and only inspect + per-inferior state. This is the only mode that allows more + than one test to share a single GPU. + + serial (default) + Take every slot of one GPU. No other test runs on that GPU + until this one releases it, but tests on other GPUs are + unaffected. Use this for tests that touch device-wide state + (such as device-wide attach/interrupt or precise-memory + toggles) or that spawn multiple concurrent GPU inferiors + within a single test. This is the historical "one rocm test + at a time" behaviour, except now scoped to a single GPU + instead of the whole machine. + + machine + Take every slot of every active GPU. No other rocm test runs + anywhere on the node until this one releases. Use this only + for tests whose inferiors spawn sibling HIP processes whose + runtime initialisation does not tolerate concurrent ROCm + activity on neighbouring GPUs. Note: the inferior under + machine mode still sees a single device via + ROCR_VISIBLE_DEVICES (the same first-pool pin as serial mode); + the exclusivity is about keeping other rocm tests off the node, + not about handing the body all GPUs. + +There is a failsafe: if any local device lacks multi-process debug +support (per hip_devices_support_debug_multi_process), parallel mode +is downgraded to serial regardless of the per-test marker. This +covers older gfx generations that cannot have two debugged processes +on the GPU at once. + +ROCGDB_ROCM_PARALLEL_SLOTS + +Number of concurrent parallel-marked tests allowed on a single GPU. +Must be a positive integer. Default 1. + + ROCGDB_ROCM_PARALLEL_SLOTS=1 (default) + Each GPU hosts at most one rocm test at any moment. A + parallel-marked test takes the GPU's single slot for its + duration; a serial-marked test takes the GPU's single slot + (which is the same thing, since there is only one). On an + N-GPU node this yields up to N concurrent rocm tests, each on + its own GPU. Matches the historical "one rocm test per GPU" + behaviour. + + ROCGDB_ROCM_PARALLEL_SLOTS=K (K > 1) + Up to K parallel-marked tests may share a GPU. Serial-marked + tests still take all K slots of one GPU, so they fully + exclude parallel-marked siblings from that GPU. On an N-GPU + node this raises the ceiling to N*K concurrent rocm tests. + Raise K only on hardware/stacks known to tolerate higher + kfd/dbgapi concurrency; can cause spurious "no ROCm-capable + device" failures if pushed past what the stack supports. + +ROCGDB_ROCM_GPU_MAX_PARALLEL + +Cap on the number of active GPU pools (i.e. distinct GPUs used by the +test run). Must be a positive integer. Default is the number of +eligible GPUs detected on the system (i.e. uncapped). + +The active pool count is set to: + + min(eligible_gpus, ROCGDB_ROCM_GPU_MAX_PARALLEL) + +where eligible_gpus is the device set surviving any ROCR_VISIBLE_DEVICES +or ROCM_TEST_ARCH filtering applied before the lock layer starts. When +the cap reduces the active set, the internal pool->physical device map +is truncated to the first N eligible devices (preserving physical +device indices, see ROCR_VISIBLE_DEVICES below); each test then sees +the corresponding physical device through the per-slot +ROCR_VISIBLE_DEVICES that with_rocm_gpu_lock sets around its body. + + unset (default) + Use every eligible GPU on the node. On an N-GPU node with no + ROCR_VISIBLE_DEVICES / ROCM_TEST_ARCH filtering this gives N + pools. + + ROCGDB_ROCM_GPU_MAX_PARALLEL=1 + Disables multi-GPU spread; the whole rocm suite shares one + GPU. Equivalent to GDB_PARALLEL being unset for the purposes + of GPU selection. + + ROCGDB_ROCM_GPU_MAX_PARALLEL=N + Cap the active pool count at N. The effective ceiling is + min(N, eligible_gpus). Lower this if kfd/dbgapi contention + from many simultaneous debug sessions is producing spurious + mid-run failures even in tests that only touch a small slice + of a single GPU. + +ROCR_VISIBLE_DEVICES + +Pre-set this in the environment before invoking the testsuite to +restrict the rocm lock layer (and every GDB inferior it spawns) to a +chosen subset of the node's physical GPUs. Format is the standard +HSA comma-separated list of physical device indices, for example: + + ROCR_VISIBLE_DEVICES=4,5,6 make check TESTS=gdb.rocm/*.exp + +Effects: + + - The lock layer enumerates only the listed devices, so + eligible_gpus (see ROCGDB_ROCM_GPU_MAX_PARALLEL above) is at most + the length of the list. + + - Each pool's per-test GPU pin uses the physical index from the + list, not a renumbered 0..N-1 logical index. In the example + above, pool 0 pins ROCR_VISIBLE_DEVICES=4, pool 1 pins =5, etc., + so the inferiors actually run on physical devices 4, 5, 6. This + is the property you want when sharing a node with other workloads + pinned to the lower-numbered GPUs. + + - ROCM_TEST_ARCH filtering, if also set, is applied within the + pre-masked set and preserves the surviving physical indices. + + - If the pre-set list's length does not match the number of devices + the HIP enumerator reports (typo, stale value, device the runtime + rejected), the lock layer logs a warning and falls back to an + identity 0..N-1 mapping for the surviving devices, so the run + still proceeds but on whichever devices HIP exposed. + +If ROCR_VISIBLE_DEVICES is unset the lock layer uses every physical +GPU the HIP enumerator reports, subject to the cap. + +Combined ceiling + +The maximum number of concurrent rocm tests on a node is: + + min(eligible_gpus, ROCGDB_ROCM_GPU_MAX_PARALLEL) + * ROCGDB_ROCM_PARALLEL_SLOTS + +assuming every concurrent test is parallel-marked. Serial- and +machine-marked tests effectively reduce that ceiling for the duration +of their lock hold. + Testing All Simple Boards ************************* diff --git a/gdb/testsuite/gdb.rocm/addr-bp-gpu-no-deb-info.exp b/gdb/testsuite/gdb.rocm/addr-bp-gpu-no-deb-info.exp index a77c628f81d..29ef1a302b7 100644 --- a/gdb/testsuite/gdb.rocm/addr-bp-gpu-no-deb-info.exp +++ b/gdb/testsuite/gdb.rocm/addr-bp-gpu-no-deb-info.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + standard_testfile .cpp require allow_hip_tests @@ -26,17 +30,18 @@ if {[build_executable "failed to prepare" $testfile $srcfile {hip}]} { return } -clean_restart $::testfile +# with_rocm_gpu_lock pins ROCR_VISIBLE_DEVICES to a single GPU per pool, +# so the inferior sees exactly one device and the breakpoint resolves to +# a single address. GDB has to start inside the lock so it inherits the +# pinned ROCR_VISIBLE_DEVICES from the environment. -# We may have multiple GPUs, resulting in many possible locations. This is -# needed to ensure we get a single address to break on, so make device 0 the -# only visible one. -gdb_test_no_output "set environment ROCR_VISIBLE_DEVICES=0" +with_rocm_gpu_lock { + clean_restart $::testfile -# Make the HIP runtime load all the GPU code objects during initialization. -gdb_test_no_output "set environment HIP_ENABLE_DEFERRED_LOADING=0" + # Make the HIP runtime load all the GPU code objects during + # initialization. + gdb_test_no_output "set environment HIP_ENABLE_DEFERRED_LOADING=0" -with_rocm_gpu_lock { if { ![runto_main] } { return } diff --git a/gdb/testsuite/gdb.rocm/alu-exceptions.exp b/gdb/testsuite/gdb.rocm/alu-exceptions.exp index a1b9b9f8802..42dfe4089f0 100644 --- a/gdb/testsuite/gdb.rocm/alu-exceptions.exp +++ b/gdb/testsuite/gdb.rocm/alu-exceptions.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/aspace-user-input.exp b/gdb/testsuite/gdb.rocm/aspace-user-input.exp index 099def6754f..ffb0cdaf58d 100644 --- a/gdb/testsuite/gdb.rocm/aspace-user-input.exp +++ b/gdb/testsuite/gdb.rocm/aspace-user-input.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile bit-extract.cpp diff --git a/gdb/testsuite/gdb.rocm/aspace-watchpoint.exp b/gdb/testsuite/gdb.rocm/aspace-watchpoint.exp index f39ac239f87..d3b6037fe8d 100644 --- a/gdb/testsuite/gdb.rocm/aspace-watchpoint.exp +++ b/gdb/testsuite/gdb.rocm/aspace-watchpoint.exp @@ -24,6 +24,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/bit-extract.exp b/gdb/testsuite/gdb.rocm/bit-extract.exp index de750b9309a..8b14f9d6110 100644 --- a/gdb/testsuite/gdb.rocm/bit-extract.exp +++ b/gdb/testsuite/gdb.rocm/bit-extract.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/branch-fault.exp b/gdb/testsuite/gdb.rocm/branch-fault.exp index 2f8f790303d..ca9d13b4810 100644 --- a/gdb/testsuite/gdb.rocm/branch-fault.exp +++ b/gdb/testsuite/gdb.rocm/branch-fault.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/break-kernel-no-debug-info.exp b/gdb/testsuite/gdb.rocm/break-kernel-no-debug-info.exp index 81de1c5adb2..ea83fbf072d 100644 --- a/gdb/testsuite/gdb.rocm/break-kernel-no-debug-info.exp +++ b/gdb/testsuite/gdb.rocm/break-kernel-no-debug-info.exp @@ -23,6 +23,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + standard_testfile .cpp require allow_hip_tests diff --git a/gdb/testsuite/gdb.rocm/breakpoint-after-exit.exp b/gdb/testsuite/gdb.rocm/breakpoint-after-exit.exp index 2c0887b1c73..f28450dbd98 100644 --- a/gdb/testsuite/gdb.rocm/breakpoint-after-exit.exp +++ b/gdb/testsuite/gdb.rocm/breakpoint-after-exit.exp @@ -24,6 +24,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/code-object-load-while-breakpoint-hit.exp b/gdb/testsuite/gdb.rocm/code-object-load-while-breakpoint-hit.exp index c1fdbe69c1b..a5c2b783d53 100644 --- a/gdb/testsuite/gdb.rocm/code-object-load-while-breakpoint-hit.exp +++ b/gdb/testsuite/gdb.rocm/code-object-load-while-breakpoint-hit.exp @@ -33,6 +33,10 @@ # value). load_lib rocm.exp + +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel standard_testfile .cpp require allow_hip_tests diff --git a/gdb/testsuite/gdb.rocm/continue-over-kernel-exit.exp b/gdb/testsuite/gdb.rocm/continue-over-kernel-exit.exp index 01625b09389..0b307a25538 100644 --- a/gdb/testsuite/gdb.rocm/continue-over-kernel-exit.exp +++ b/gdb/testsuite/gdb.rocm/continue-over-kernel-exit.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + standard_testfile .cpp require allow_hip_tests diff --git a/gdb/testsuite/gdb.rocm/convenience_variables.exp b/gdb/testsuite/gdb.rocm/convenience_variables.exp index 60815f695da..fde14e0ac71 100644 --- a/gdb/testsuite/gdb.rocm/convenience_variables.exp +++ b/gdb/testsuite/gdb.rocm/convenience_variables.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/core-no-read-special-files.exp b/gdb/testsuite/gdb.rocm/core-no-read-special-files.exp index 86bd819b650..32abecccb72 100644 --- a/gdb/testsuite/gdb.rocm/core-no-read-special-files.exp +++ b/gdb/testsuite/gdb.rocm/core-no-read-special-files.exp @@ -21,6 +21,11 @@ load_lib rocm.exp +# Each instance writes its corefile under its own outputs/ directory +# and only inspects per-inferior state, so allow it to run concurrently +# with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + # The /dev/kfd special file is Linux-specific. require {istarget *-linux*} diff --git a/gdb/testsuite/gdb.rocm/corefile.exp b/gdb/testsuite/gdb.rocm/corefile.exp index a019bc08d78..c1da3f859af 100644 --- a/gdb/testsuite/gdb.rocm/corefile.exp +++ b/gdb/testsuite/gdb.rocm/corefile.exp @@ -21,6 +21,11 @@ load_lib rocm.exp +# Each instance writes its corefile under its own outputs/ directory +# and only inspects per-inferior state, so allow it to run concurrently +# with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_rocm_core_tests require allow_hip_tests diff --git a/gdb/testsuite/gdb.rocm/debugtrap.exp b/gdb/testsuite/gdb.rocm/debugtrap.exp index 02939624f8f..18196707fcc 100644 --- a/gdb/testsuite/gdb.rocm/debugtrap.exp +++ b/gdb/testsuite/gdb.rocm/debugtrap.exp @@ -22,6 +22,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/deep-stack.exp b/gdb/testsuite/gdb.rocm/deep-stack.exp index 93284384d61..526f29167ee 100644 --- a/gdb/testsuite/gdb.rocm/deep-stack.exp +++ b/gdb/testsuite/gdb.rocm/deep-stack.exp @@ -20,6 +20,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/deref-scoped-pointer.exp b/gdb/testsuite/gdb.rocm/deref-scoped-pointer.exp index 80ccdd698d6..d46dc648142 100644 --- a/gdb/testsuite/gdb.rocm/deref-scoped-pointer.exp +++ b/gdb/testsuite/gdb.rocm/deref-scoped-pointer.exp @@ -26,6 +26,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/detach-while-breakpoints-inserted.exp b/gdb/testsuite/gdb.rocm/detach-while-breakpoints-inserted.exp index 6ba842a3437..52d7a6045c4 100644 --- a/gdb/testsuite/gdb.rocm/detach-while-breakpoints-inserted.exp +++ b/gdb/testsuite/gdb.rocm/detach-while-breakpoints-inserted.exp @@ -27,6 +27,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/device-barrier.exp b/gdb/testsuite/gdb.rocm/device-barrier.exp index f75fbbe7127..506bc6532e0 100644 --- a/gdb/testsuite/gdb.rocm/device-barrier.exp +++ b/gdb/testsuite/gdb.rocm/device-barrier.exp @@ -19,6 +19,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/devicecode-breakpoint.exp b/gdb/testsuite/gdb.rocm/devicecode-breakpoint.exp index 86742d9cf21..32a3ffebd6f 100644 --- a/gdb/testsuite/gdb.rocm/devicecode-breakpoint.exp +++ b/gdb/testsuite/gdb.rocm/devicecode-breakpoint.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/disassemble.exp b/gdb/testsuite/gdb.rocm/disassemble.exp index 1b65ac4f1b3..87ed3556004 100644 --- a/gdb/testsuite/gdb.rocm/disassemble.exp +++ b/gdb/testsuite/gdb.rocm/disassemble.exp @@ -26,6 +26,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/displaced-stepping.exp b/gdb/testsuite/gdb.rocm/displaced-stepping.exp index d3a4b15acbe..198272875da 100644 --- a/gdb/testsuite/gdb.rocm/displaced-stepping.exp +++ b/gdb/testsuite/gdb.rocm/displaced-stepping.exp @@ -22,6 +22,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/exec-bit-extract.exp b/gdb/testsuite/gdb.rocm/exec-bit-extract.exp index 1251f8572a1..9d79d6fa548 100644 --- a/gdb/testsuite/gdb.rocm/exec-bit-extract.exp +++ b/gdb/testsuite/gdb.rocm/exec-bit-extract.exp @@ -22,6 +22,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests if { ![istarget "*-linux*"] } then { diff --git a/gdb/testsuite/gdb.rocm/finish.exp b/gdb/testsuite/gdb.rocm/finish.exp index 1443f8b9efa..d9ed3c521ab 100644 --- a/gdb/testsuite/gdb.rocm/finish.exp +++ b/gdb/testsuite/gdb.rocm/finish.exp @@ -22,6 +22,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/gcore-after-attach.exp b/gdb/testsuite/gdb.rocm/gcore-after-attach.exp index 09a3cd21289..9c50e5d4bdc 100644 --- a/gdb/testsuite/gdb.rocm/gcore-after-attach.exp +++ b/gdb/testsuite/gdb.rocm/gcore-after-attach.exp @@ -25,6 +25,11 @@ load_lib rocm.exp +# Each instance writes its corefile under its own outputs/ directory +# and only inspects per-inferior state, so allow it to run concurrently +# with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_rocm_core_tests require allow_hip_tests require can_spawn_for_attach diff --git a/gdb/testsuite/gdb.rocm/generic-address.exp b/gdb/testsuite/gdb.rocm/generic-address.exp index 4fdfb800784..5450c18de0f 100644 --- a/gdb/testsuite/gdb.rocm/generic-address.exp +++ b/gdb/testsuite/gdb.rocm/generic-address.exp @@ -25,6 +25,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/hip-builtin-completions.exp b/gdb/testsuite/gdb.rocm/hip-builtin-completions.exp index d5cf01bbb62..3575c098a23 100644 --- a/gdb/testsuite/gdb.rocm/hip-builtin-completions.exp +++ b/gdb/testsuite/gdb.rocm/hip-builtin-completions.exp @@ -26,6 +26,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile hip-builtin-variables.cpp diff --git a/gdb/testsuite/gdb.rocm/hip-builtin-variables.exp b/gdb/testsuite/gdb.rocm/hip-builtin-variables.exp index 67fc6de28cf..e0e86c13d2f 100644 --- a/gdb/testsuite/gdb.rocm/hip-builtin-variables.exp +++ b/gdb/testsuite/gdb.rocm/hip-builtin-variables.exp @@ -20,6 +20,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/hip-lang-detect.exp b/gdb/testsuite/gdb.rocm/hip-lang-detect.exp index 1642d00ccda..ea6072c44a6 100644 --- a/gdb/testsuite/gdb.rocm/hip-lang-detect.exp +++ b/gdb/testsuite/gdb.rocm/hip-lang-detect.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile hip-builtin-variables.cpp diff --git a/gdb/testsuite/gdb.rocm/illegal-insn-sigill.exp b/gdb/testsuite/gdb.rocm/illegal-insn-sigill.exp index cd4d04a9f51..4258429f47d 100644 --- a/gdb/testsuite/gdb.rocm/illegal-insn-sigill.exp +++ b/gdb/testsuite/gdb.rocm/illegal-insn-sigill.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/info-dispatches.exp b/gdb/testsuite/gdb.rocm/info-dispatches.exp index e9f0f2cc5e0..75bf9015902 100644 --- a/gdb/testsuite/gdb.rocm/info-dispatches.exp +++ b/gdb/testsuite/gdb.rocm/info-dispatches.exp @@ -22,6 +22,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/info-sharedlibrary.exp b/gdb/testsuite/gdb.rocm/info-sharedlibrary.exp index aaa9569f367..8d6b24c1628 100644 --- a/gdb/testsuite/gdb.rocm/info-sharedlibrary.exp +++ b/gdb/testsuite/gdb.rocm/info-sharedlibrary.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/instruction-stepping-commands.exp b/gdb/testsuite/gdb.rocm/instruction-stepping-commands.exp index 3ebf0c58dec..577e0c45dc7 100644 --- a/gdb/testsuite/gdb.rocm/instruction-stepping-commands.exp +++ b/gdb/testsuite/gdb.rocm/instruction-stepping-commands.exp @@ -20,6 +20,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/interrupt-twice.exp b/gdb/testsuite/gdb.rocm/interrupt-twice.exp index cfefdff9ff0..8fc9602c56d 100644 --- a/gdb/testsuite/gdb.rocm/interrupt-twice.exp +++ b/gdb/testsuite/gdb.rocm/interrupt-twice.exp @@ -19,6 +19,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/lane-execution.exp b/gdb/testsuite/gdb.rocm/lane-execution.exp index 82a81286945..aede7678710 100644 --- a/gdb/testsuite/gdb.rocm/lane-execution.exp +++ b/gdb/testsuite/gdb.rocm/lane-execution.exp @@ -34,6 +34,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/lane-info.exp b/gdb/testsuite/gdb.rocm/lane-info.exp index 2b5b8163d0e..b9d369a6a29 100644 --- a/gdb/testsuite/gdb.rocm/lane-info.exp +++ b/gdb/testsuite/gdb.rocm/lane-info.exp @@ -28,6 +28,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/lane-pc-vega20.exp b/gdb/testsuite/gdb.rocm/lane-pc-vega20.exp index 00cba079e30..acd34ef7ff0 100644 --- a/gdb/testsuite/gdb.rocm/lane-pc-vega20.exp +++ b/gdb/testsuite/gdb.rocm/lane-pc-vega20.exp @@ -20,6 +20,10 @@ load_lib dwarf.exp load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests # Return the memory address of SYMBOL. diff --git a/gdb/testsuite/gdb.rocm/line-breakpoint-in-kernel.exp b/gdb/testsuite/gdb.rocm/line-breakpoint-in-kernel.exp index 14c10c44158..27338afaefa 100644 --- a/gdb/testsuite/gdb.rocm/line-breakpoint-in-kernel.exp +++ b/gdb/testsuite/gdb.rocm/line-breakpoint-in-kernel.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/mi-aspace.exp b/gdb/testsuite/gdb.rocm/mi-aspace.exp index fe7465c7d0e..89f552ecacc 100644 --- a/gdb/testsuite/gdb.rocm/mi-aspace.exp +++ b/gdb/testsuite/gdb.rocm/mi-aspace.exp @@ -17,6 +17,10 @@ # This testcase exercises MI support for address spaces. # load_lib rocm.exp + +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel load_lib mi-support.exp set MIFLAGS "-i=mi" diff --git a/gdb/testsuite/gdb.rocm/mi-attach.exp b/gdb/testsuite/gdb.rocm/mi-attach.exp index 22aaae5e6e3..d6e66fa3058 100644 --- a/gdb/testsuite/gdb.rocm/mi-attach.exp +++ b/gdb/testsuite/gdb.rocm/mi-attach.exp @@ -17,6 +17,10 @@ load_lib rocm.exp load_lib mi-support.exp set MIFLAGS "-i=mi" +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require can_spawn_for_attach allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/mi-lanes.exp b/gdb/testsuite/gdb.rocm/mi-lanes.exp index 13f22c6075b..8e2603505db 100644 --- a/gdb/testsuite/gdb.rocm/mi-lanes.exp +++ b/gdb/testsuite/gdb.rocm/mi-lanes.exp @@ -17,6 +17,10 @@ # This testcase exercises MI support for lane debugging features. # load_lib rocm.exp + +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel load_lib mi-support.exp set MIFLAGS "-i=mi" diff --git a/gdb/testsuite/gdb.rocm/names.exp b/gdb/testsuite/gdb.rocm/names.exp index bb860f28bd1..1b9695146b3 100644 --- a/gdb/testsuite/gdb.rocm/names.exp +++ b/gdb/testsuite/gdb.rocm/names.exp @@ -17,6 +17,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/nonstop-displaced.exp b/gdb/testsuite/gdb.rocm/nonstop-displaced.exp index b2af0d495d1..695bf4ef7dc 100644 --- a/gdb/testsuite/gdb.rocm/nonstop-displaced.exp +++ b/gdb/testsuite/gdb.rocm/nonstop-displaced.exp @@ -20,6 +20,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/nonstop-mode.exp b/gdb/testsuite/gdb.rocm/nonstop-mode.exp index 77a8dcae5ed..bff7a25e1d6 100644 --- a/gdb/testsuite/gdb.rocm/nonstop-mode.exp +++ b/gdb/testsuite/gdb.rocm/nonstop-mode.exp @@ -20,6 +20,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/ocp_mx.exp b/gdb/testsuite/gdb.rocm/ocp_mx.exp index b016d2ab62d..ad0a01e9dbb 100644 --- a/gdb/testsuite/gdb.rocm/ocp_mx.exp +++ b/gdb/testsuite/gdb.rocm/ocp_mx.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests allow_python_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/program-execution.exp b/gdb/testsuite/gdb.rocm/program-execution.exp index dc11edd52b7..2b122d13b99 100644 --- a/gdb/testsuite/gdb.rocm/program-execution.exp +++ b/gdb/testsuite/gdb.rocm/program-execution.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/register-watchpoint.exp b/gdb/testsuite/gdb.rocm/register-watchpoint.exp index 1c91f88b94d..61ff0d4e0b7 100644 --- a/gdb/testsuite/gdb.rocm/register-watchpoint.exp +++ b/gdb/testsuite/gdb.rocm/register-watchpoint.exp @@ -20,6 +20,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile bit-extract.cpp diff --git a/gdb/testsuite/gdb.rocm/resume-exception.exp b/gdb/testsuite/gdb.rocm/resume-exception.exp index f5b4d2c1e0b..e93500e3961 100644 --- a/gdb/testsuite/gdb.rocm/resume-exception.exp +++ b/gdb/testsuite/gdb.rocm/resume-exception.exp @@ -23,6 +23,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/runtime-core-abort-lightweight-file.exp b/gdb/testsuite/gdb.rocm/runtime-core-abort-lightweight-file.exp new file mode 100644 index 00000000000..510258f5aa9 --- /dev/null +++ b/gdb/testsuite/gdb.rocm/runtime-core-abort-lightweight-file.exp @@ -0,0 +1,24 @@ +# Copyright (C) 2026 Free Software Foundation, Inc. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This file is part of GDB. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Exercise runtime-core with: abort / lightweight core / file output. + +set fault abort +set core_type lightweight +set output_type file +source $srcdir/$subdir/runtime-core.exp.tcl diff --git a/gdb/testsuite/gdb.rocm/runtime-core-assert-lightweight-file.exp b/gdb/testsuite/gdb.rocm/runtime-core-assert-lightweight-file.exp new file mode 100644 index 00000000000..233ecdf70d2 --- /dev/null +++ b/gdb/testsuite/gdb.rocm/runtime-core-assert-lightweight-file.exp @@ -0,0 +1,24 @@ +# Copyright (C) 2026 Free Software Foundation, Inc. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This file is part of GDB. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Exercise runtime-core with: assert / lightweight core / file output. + +set fault assert +set core_type lightweight +set output_type file +source $srcdir/$subdir/runtime-core.exp.tcl diff --git a/gdb/testsuite/gdb.rocm/runtime-core-pagefault-full-file.exp b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-full-file.exp new file mode 100644 index 00000000000..82f6465e224 --- /dev/null +++ b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-full-file.exp @@ -0,0 +1,24 @@ +# Copyright (C) 2026 Free Software Foundation, Inc. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This file is part of GDB. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Exercise runtime-core with: pagefault / full core / file output. + +set fault pagefault +set core_type full +set output_type file +source $srcdir/$subdir/runtime-core.exp.tcl diff --git a/gdb/testsuite/gdb.rocm/runtime-core-pagefault-full-pipe.exp b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-full-pipe.exp new file mode 100644 index 00000000000..55e4cb65a16 --- /dev/null +++ b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-full-pipe.exp @@ -0,0 +1,24 @@ +# Copyright (C) 2026 Free Software Foundation, Inc. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This file is part of GDB. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Exercise runtime-core with: pagefault / full core / pipe output. + +set fault pagefault +set core_type full +set output_type pipe +source $srcdir/$subdir/runtime-core.exp.tcl diff --git a/gdb/testsuite/gdb.rocm/runtime-core-pagefault-lightweight-file.exp b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-lightweight-file.exp new file mode 100644 index 00000000000..388c862aafd --- /dev/null +++ b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-lightweight-file.exp @@ -0,0 +1,24 @@ +# Copyright (C) 2026 Free Software Foundation, Inc. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This file is part of GDB. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Exercise runtime-core with: pagefault / lightweight core / file output. + +set fault pagefault +set core_type lightweight +set output_type file +source $srcdir/$subdir/runtime-core.exp.tcl diff --git a/gdb/testsuite/gdb.rocm/runtime-core-pagefault-lightweight-pipe.exp b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-lightweight-pipe.exp new file mode 100644 index 00000000000..81f72ba19b1 --- /dev/null +++ b/gdb/testsuite/gdb.rocm/runtime-core-pagefault-lightweight-pipe.exp @@ -0,0 +1,24 @@ +# Copyright (C) 2026 Free Software Foundation, Inc. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This file is part of GDB. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Exercise runtime-core with: pagefault / lightweight core / pipe output. + +set fault pagefault +set core_type lightweight +set output_type pipe +source $srcdir/$subdir/runtime-core.exp.tcl diff --git a/gdb/testsuite/gdb.rocm/runtime-core.exp b/gdb/testsuite/gdb.rocm/runtime-core.exp.tcl similarity index 87% rename from gdb/testsuite/gdb.rocm/runtime-core.exp rename to gdb/testsuite/gdb.rocm/runtime-core.exp.tcl index 4c8d6506744..2e16b4a6219 100644 --- a/gdb/testsuite/gdb.rocm/runtime-core.exp +++ b/gdb/testsuite/gdb.rocm/runtime-core.exp.tcl @@ -20,13 +20,16 @@ # on the GPU (pagefault, abort, or assert (false)) which triggers the # runtime to generate a core dump. The test will find the host and GPU # core dumps, create a unified core using coremerge and open it in GDB. +# +# This file is sourced by the runtime-core-*.exp shards, each of which +# sets $fault, $core_type and $output_type before sourcing. load_lib rocm.exp require allow_rocm_core_tests require allow_hip_tests -standard_testfile .cpp +standard_testfile runtime-core.cpp if { [build_executable "failed to prepare" ${testfile} ${srcfile} \ {debug hip}] } { @@ -37,12 +40,12 @@ if { [build_executable "failed to prepare" ${testfile} ${srcfile} \ # Run the test exercising generation of a core dump on fault FAULT # using a file or a pipe. # -# FAULT is one of "page", "abort", or "assert", and selects which +# FAULT is one of "pagefault", "abort", or "assert", and selects which # fault the application should generate. # # CORE_TYPE is one of "full" or "lightweight". # -# OUTPUT_TYPE is one of "file" of "pipe". +# OUTPUT_TYPE is one of "file" or "pipe". # proc do_test { fault core_type output_type } { set use_pipe [expr {$output_type eq "pipe"}] @@ -137,17 +140,5 @@ proc do_test { fault core_type output_type } { } with_rocm_gpu_lock { - # Check the runtime can configure the core dump generation based - # on env variables. - foreach_with_prefix output_type { file pipe } { - foreach_with_prefix core_type {full lightweight} { - do_test pagefault $core_type $output_type - } - } - - # Check that various GPU events can cause a core dump. - # "pagefault" was covered by previous cases already. - foreach_with_prefix fault {abort assert} { - do_test $fault lightweight file - } + do_test $fault $core_type $output_type } diff --git a/gdb/testsuite/gdb.rocm/scheduler-locking.exp b/gdb/testsuite/gdb.rocm/scheduler-locking.exp index d5ed9a5b1ee..1b441905b29 100644 --- a/gdb/testsuite/gdb.rocm/scheduler-locking.exp +++ b/gdb/testsuite/gdb.rocm/scheduler-locking.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/shared-memory.exp b/gdb/testsuite/gdb.rocm/shared-memory.exp index 06a0b136da8..69b48e898ba 100644 --- a/gdb/testsuite/gdb.rocm/shared-memory.exp +++ b/gdb/testsuite/gdb.rocm/shared-memory.exp @@ -19,6 +19,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + # Python support is required to use $_streq. require allow_hip_tests allow_python_tests diff --git a/gdb/testsuite/gdb.rocm/show-info.exp b/gdb/testsuite/gdb.rocm/show-info.exp index 7bc29e706d6..7453ae0ecec 100644 --- a/gdb/testsuite/gdb.rocm/show-info.exp +++ b/gdb/testsuite/gdb.rocm/show-info.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/simple-outside-debugger.exp b/gdb/testsuite/gdb.rocm/simple-outside-debugger.exp index 214e238b22a..8e33d9f1154 100644 --- a/gdb/testsuite/gdb.rocm/simple-outside-debugger.exp +++ b/gdb/testsuite/gdb.rocm/simple-outside-debugger.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile simple.cpp diff --git a/gdb/testsuite/gdb.rocm/simple.exp b/gdb/testsuite/gdb.rocm/simple.exp index 320839f457c..248db9fd72a 100644 --- a/gdb/testsuite/gdb.rocm/simple.exp +++ b/gdb/testsuite/gdb.rocm/simple.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/snapshot-objfile-on-load.exp b/gdb/testsuite/gdb.rocm/snapshot-objfile-on-load.exp index 383f3455999..dbe11dae46d 100644 --- a/gdb/testsuite/gdb.rocm/snapshot-objfile-on-load.exp +++ b/gdb/testsuite/gdb.rocm/snapshot-objfile-on-load.exp @@ -42,6 +42,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/static-global.exp b/gdb/testsuite/gdb.rocm/static-global.exp index d678a7c5f19..735018bdb3b 100644 --- a/gdb/testsuite/gdb.rocm/static-global.exp +++ b/gdb/testsuite/gdb.rocm/static-global.exp @@ -38,6 +38,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/step-over-kernel-exit.exp b/gdb/testsuite/gdb.rocm/step-over-kernel-exit.exp index 1ccdc44a25f..1d5da57dab5 100644 --- a/gdb/testsuite/gdb.rocm/step-over-kernel-exit.exp +++ b/gdb/testsuite/gdb.rocm/step-over-kernel-exit.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + standard_testfile .cpp require allow_hip_tests diff --git a/gdb/testsuite/gdb.rocm/unaligned-memory-access.exp b/gdb/testsuite/gdb.rocm/unaligned-memory-access.exp index 3b1c9b90403..b9f97525f18 100644 --- a/gdb/testsuite/gdb.rocm/unaligned-memory-access.exp +++ b/gdb/testsuite/gdb.rocm/unaligned-memory-access.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/until-tests.exp b/gdb/testsuite/gdb.rocm/until-tests.exp index 09286fb19c4..c5e0b3dde8d 100644 --- a/gdb/testsuite/gdb.rocm/until-tests.exp +++ b/gdb/testsuite/gdb.rocm/until-tests.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/update-thread-list.exp b/gdb/testsuite/gdb.rocm/update-thread-list.exp index 7656c722184..6c27d1c6921 100644 --- a/gdb/testsuite/gdb.rocm/update-thread-list.exp +++ b/gdb/testsuite/gdb.rocm/update-thread-list.exp @@ -21,6 +21,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/watch-gpu-global-from-host.exp b/gdb/testsuite/gdb.rocm/watch-gpu-global-from-host.exp index c6364d7de7b..84299f6b9e7 100644 --- a/gdb/testsuite/gdb.rocm/watch-gpu-global-from-host.exp +++ b/gdb/testsuite/gdb.rocm/watch-gpu-global-from-host.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/watchpoint-at-end-of-shader.exp b/gdb/testsuite/gdb.rocm/watchpoint-at-end-of-shader.exp index 24bd5bc7042..3f84b8d0ae1 100644 --- a/gdb/testsuite/gdb.rocm/watchpoint-at-end-of-shader.exp +++ b/gdb/testsuite/gdb.rocm/watchpoint-at-end-of-shader.exp @@ -18,6 +18,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.rocm/watchpoint-basic.exp b/gdb/testsuite/gdb.rocm/watchpoint-basic.exp index b041c742797..18cea187347 100644 --- a/gdb/testsuite/gdb.rocm/watchpoint-basic.exp +++ b/gdb/testsuite/gdb.rocm/watchpoint-basic.exp @@ -17,6 +17,10 @@ load_lib rocm.exp +# This test only exercises a small slice of the GPU; allow it to +# run concurrently with other gdb.rocm tests. +set rocm_gpu_concurrency parallel + require allow_hip_tests standard_testfile .cpp diff --git a/gdb/testsuite/gdb.testsuite/slot-pool-lock-helper.tcl b/gdb/testsuite/gdb.testsuite/slot-pool-lock-helper.tcl new file mode 100644 index 00000000000..0145d2bcbf6 --- /dev/null +++ b/gdb/testsuite/gdb.testsuite/slot-pool-lock-helper.tcl @@ -0,0 +1,108 @@ +# Copyright 2026 Free Software Foundation, Inc. +# Copyright 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Child driver for slot-pool-lock.exp. Each invocation acquires one +# slot-pool lock, holds it for HOLD_MS milliseconds, releases, and +# exits. Acquire / hold / release events are appended to LOG_FILE +# with microsecond timestamps so the parent can reason about +# concurrency. When done, the child touches DONE_FILE. +# +# Usage: +# tclsh slot-pool-lock-helper.tcl \ +# LOCK_DIR KIND HOLD_MS LOG_FILE DONE_FILE ID +# +# KIND is one of: +# shared-- +# exclusive-- +# machine-- + +# Stubs for the bits of gdb-utils.exp that touch DejaGnu state. +proc verbose {args} {} +set ::subdir "gdb.testsuite" +set ::gdb_test_file_name "slot-pool-lock-helper" + +set here [file dirname [file normalize [info script]]] +source [file join $here .. lib gdb-utils.exp] + +if {[llength $argv] != 6} { + puts stderr "usage: $argv0 LOCK_DIR KIND HOLD_MS LOG_FILE DONE_FILE ID" + exit 2 +} +lassign $argv lock_dir kind hold_ms log_file done_file id + +if {![regexp {^(shared|exclusive|machine)-(\d+)-(\d+)$} $kind -> mode npools per_pool]} { + puts stderr "bad KIND: $kind" + exit 2 +} +if {$npools <= 0 || $per_pool <= 0} { + puts stderr "bad KIND: npools and per_pool must be > 0 (got $npools, $per_pool)" + exit 2 +} + +proc ev {log id tag} { + set ch [open $log a] + puts $ch "[clock microseconds] $id $tag" + close $ch +} + +ev $log_file $id want + +# Wrap acquire/release in catch so a regression in the lock layer +# surfaces as a clean stderr line + non-zero exit instead of a silent +# hang or an unhelpful Tcl stack trace. +if {[catch { + switch -- $mode { + shared { + set tok [lock_file_acquire_shared_multi $lock_dir slot \ + $npools $per_pool barrier] + } + exclusive { + set tok [lock_file_acquire_exclusive_multi $lock_dir slot \ + $npools $per_pool barrier] + } + machine { + set tok [lock_file_acquire_machine_multi $lock_dir slot \ + $npools $per_pool barrier] + } + } +} msg]} { + puts stderr "acquire failed: $msg" + exit 1 +} + +# Token shapes: +# {shared } +# {exclusive 0} +# {machine } +if {$mode eq "machine"} { + set pool "all" + set slot "all" +} else { + set pool [lindex $tok end-1] + set slot [lindex $tok end] +} +ev $log_file $id "hold pool=$pool slot=$slot" + +after $hold_ms + +ev $log_file $id release +if {[catch {lock_file_release_shared $tok} msg]} { + puts stderr "release failed: $msg" + exit 1 +} + +set ch [open $done_file w] +close $ch diff --git a/gdb/testsuite/gdb.testsuite/slot-pool-lock.exp b/gdb/testsuite/gdb.testsuite/slot-pool-lock.exp new file mode 100644 index 00000000000..13e9a96a26f --- /dev/null +++ b/gdb/testsuite/gdb.testsuite/slot-pool-lock.exp @@ -0,0 +1,347 @@ +# Copyright 2026 Free Software Foundation, Inc. +# Copyright 2026 Advanced Micro Devices, Inc. All rights reserved. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Self-test for the slot-pool shared/exclusive lock layer in +# lib/gdb-utils.exp -- the foundation under with_rocm_gpu_lock. +# Children are spawned as separate tclsh processes so the test +# exercises the cross-process file locks for real. + +load_lib gdb-utils.exp + +if {[auto_execok tclsh] eq ""} { + untested "tclsh not available" + return +} + +set helper [file normalize \ + [file join $srcdir $subdir slot-pool-lock-helper.tcl]] +set tmpdir [standard_output_file lockdir] +set child_pids {} + +# Spawn a helper. Returns the done-marker path; wait_all polls for it. +# Records the PID so reset_tmpdir can clean up stragglers from a +# previous (possibly timed-out) scenario. +proc spawn_child {id kind hold_ms} { + global helper tmpdir log_file child_pids + set done [file join $tmpdir "done-$id"] + set err [file join $tmpdir "err-$id"] + verbose -log " spawn child id=$id kind=$kind hold=${hold_ms}ms" + set pid [exec tclsh $helper $tmpdir $kind $hold_ms $log_file $done $id \ + &$err &] + lappend child_pids $pid + return $done +} + +proc wait_all {done_files {timeout_ms 60000}} { + global child_pids + set deadline [expr {[clock milliseconds] + $timeout_ms}] + foreach d $done_files { + while {![file exists $d]} { + if {[clock milliseconds] > $deadline} { + verbose -log " wait_all: timeout after ${timeout_ms}ms waiting for [file tail $d]" + return 0 + } + after 50 + } + } + # All children done -- forget their PIDs so the next reset_tmpdir + # doesn't waste kills on already-reaped processes. + set child_pids {} + return 1 +} + +# Busy-wait until any file matching GLOB_PATTERN exists. Used to +# synchronise scenarios on observable lock-file state (e.g. an +# exclusive waiter creating its barrier file) instead of guessing +# fork+exec latency. Polls at 25 ms, so files that appear and +# disappear faster than that may be missed -- intended for long-lived +# lock files, not transient markers. +proc wait_for_file {glob_pattern {timeout_ms 5000}} { + set deadline [expr {[clock milliseconds] + $timeout_ms}] + while {[clock milliseconds] < $deadline} { + if {[llength [glob -nocomplain $glob_pattern]] > 0} { + return 1 + } + after 25 + } + return 0 +} + +# Read the merged event log. No reader/writer locking: children +# append one short line per event via {open a; puts; close}, where Tcl +# opens the file with O_APPEND and emits the buffered line in a single +# write(2) at close. On Linux, an O_APPEND write atomically advances +# the file offset, so concurrent appenders cannot interleave bytes +# within a line. Porting this self-test to a platform without that +# guarantee would need an explicit lock around the writes. +proc read_events {log_file} { + set events {} + set ch [open $log_file r] + while {[gets $ch line] >= 0} { + if {$line eq ""} continue + lappend events $line + } + close $ch + # Sort by timestamp so interleaved appends are easy to read. + return [lsort -integer -index 0 $events] +} + +# Return the microsecond timestamp of event {ID TAG} (TAG is the first +# whitespace-separated token after the id), or -1 if not found. +proc ts_of {events id tag} { + foreach e $events { + set ts [lindex $e 0] + set eid [lindex $e 1] + set etag [lindex $e 2] + if {$eid == $id && $etag eq $tag} { return $ts } + } + return -1 +} + +# Dump the merged event timeline to gdb.log with millisecond +# timestamps (3 fractional digits) rebased to the first event. Also +# surface any non-empty child stderr. +proc dump_events {events} { + global tmpdir + if {[llength $events] == 0} { + verbose -log " (no events recorded)" + } else { + set t0 [lindex [lindex $events 0] 0] + verbose -log " timeline (t=ms relative to first event):" + foreach e $events { + set rel [expr {([lindex $e 0] - $t0) / 1000.0}] + set rest [lrange $e 1 end] + verbose -log [format " %10.3f %s" $rel $rest] + } + } + foreach errf [lsort [glob -nocomplain [file join $tmpdir err-*]]] { + if {[file size $errf] > 0} { + verbose -log " child stderr ([file tail $errf]):" + set ch [open $errf r] + while {[gets $ch line] >= 0} { + verbose -log " | $line" + } + close $ch + } + } +} + +# Wipe the lock dir between scenarios and kill any helper still alive +# from a previous (likely timed-out) scenario, so its lingering writes +# can't contaminate the next log. +proc reset_tmpdir {tag} { + global tmpdir log_file child_pids + verbose -log "scenario: $tag" + foreach pid $child_pids { + catch {exec kill -9 $pid} + } + set child_pids {} + file delete -force $tmpdir + file mkdir $tmpdir + set log_file [file join $tmpdir "log-$tag"] + close [open $log_file w] +} + +# === Capacity: N shared can hold concurrently within a pool. === +with_test_prefix "shared-capacity" { + reset_tmpdir capacity + set dones {} + lappend dones [spawn_child 1 shared-1-2 500] + lappend dones [spawn_child 2 shared-1-2 500] + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set h1 [ts_of $events 1 hold] + set h2 [ts_of $events 2 hold] + set r1 [ts_of $events 1 release] + set r2 [ts_of $events 2 release] + gdb_assert {$h1 != -1 && $h2 != -1} "both acquired" + # Both holds before either release => concurrent occupancy. + gdb_assert {$h1 < $r2 && $h2 < $r1} "concurrent holds within capacity" +} + +# === Mutex: exclusive waits for shared to drain. === +with_test_prefix "shared-exclusive-mutex" { + reset_tmpdir mutex + set dones {} + lappend dones [spawn_child 1 shared-1-1 500] + after 100 + lappend dones [spawn_child 2 exclusive-1-1 100] + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set r1 [ts_of $events 1 release] + set h2 [ts_of $events 2 hold] + gdb_assert {$r1 != -1 && $h2 != -1} "both ran" + gdb_assert {$h2 > $r1} "exclusive waited for shared" +} + +# === Multi-pool spread: 4 shared on 4 pools land on 4 distinct pools. === +# Row-major fill takes slot 0 of every pool before slot 1 of any pool, +# so each of the first NPOOLS shared acquirers must get its own pool. +with_test_prefix "multi-pool-spread" { + reset_tmpdir spread + set dones {} + for {set i 1} {$i <= 4} {incr i} { + lappend dones [spawn_child $i shared-4-2 500] + } + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set pools {} + foreach e $events { + if {[regexp {hold pool=(\d+)} $e -> p]} { lappend pools $p } + } + set distinct [llength [lsort -unique $pools]] + verbose -log " pool assignment: $pools ([llength $pools] holds, $distinct distinct)" + gdb_assert {$distinct == 4} "4 distinct pools used" +} + +# === Writer priority: a queued exclusive blocks new shared acquirers. === +# Two shared fill a one-pool x two-slot pool; an exclusive arrives while +# they hold; a late shared arrives after the exclusive's barrier is +# observable in the lockdir, but before the pool drains. Writer +# priority requires the late shared to wait until the exclusive +# completes, even though slots free up momentarily when the first two +# release. We poll for the barrier file (instead of guessing fork +# latency) so the test stays deterministic under load. +with_test_prefix "writer-priority" { + reset_tmpdir priority + set dones {} + lappend dones [spawn_child 1 shared-1-2 1500] + lappend dones [spawn_child 2 shared-1-2 1500] + after 100 + lappend dones [spawn_child 3 exclusive-1-2 200] + gdb_assert {[wait_for_file [file join $tmpdir barrier-0.lock]]} \ + "exclusive barrier observable" + lappend dones [spawn_child 4 shared-1-2 100] + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set rex [ts_of $events 3 release] + set h4 [ts_of $events 4 hold] + gdb_assert {$rex != -1 && $h4 != -1} "all four ran" + gdb_assert {$h4 > $rex} "late shared waited for exclusive" +} + +# === Exclusive queued behind exclusive on the same pool. === +# With one pool, two exclusives serialise: the second's hold must +# follow the first's release. +with_test_prefix "exclusive-behind-exclusive" { + reset_tmpdir excl_queue + set dones {} + lappend dones [spawn_child 1 exclusive-1-1 300] + gdb_assert {[wait_for_file [file join $tmpdir barrier-0.lock]]} \ + "first exclusive's barrier observable" + lappend dones [spawn_child 2 exclusive-1-1 100] + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set r1 [ts_of $events 1 release] + set h2 [ts_of $events 2 hold] + gdb_assert {$r1 != -1 && $h2 != -1} "both ran" + gdb_assert {$h2 > $r1} "second exclusive waited for first" +} + +# === Machine lock: blocks every pool. === +# A 2-pool x 2-slot setup with one shared holding a slot in some +# pool; a machine acquirer must wait for that shared to release +# (it needs every pool), and once it holds, no other shared can +# acquire anywhere. +with_test_prefix "machine-lock" { + reset_tmpdir machine + set dones {} + lappend dones [spawn_child 1 shared-2-2 400] + after 100 + lappend dones [spawn_child 2 machine-2-2 200] + # Wait until both of the machine acquirer's barriers are + # visible, then try to slip another shared in -- it must see + # the machine's barriers and yield, regardless of which pool + # it tries first. + gdb_assert {[wait_for_file [file join $tmpdir barrier-1.lock]]} \ + "machine's pool-1 barrier observable" + lappend dones [spawn_child 3 shared-2-2 50] + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set r1 [ts_of $events 1 release] + set hM [ts_of $events 2 hold] + set rM [ts_of $events 2 release] + set h3 [ts_of $events 3 hold] + gdb_assert {$r1 != -1 && $hM != -1 && $h3 != -1} "all three ran" + gdb_assert {$hM > $r1} "machine waited for shared" + gdb_assert {$h3 > $rM} "later shared waited for machine" +} + +# === Randomised pool start offset is actually random. === +# Each isolated single-acquirer run picks its starting pool with +# rand(). Across many runs, the chosen pool must not always be the +# same -- otherwise concurrent acquirers would concentrate on pool 0 +# under load. We require at least 3 distinct starting pools out of +# 16 trials on a 4-pool setup. A working RNG should essentially never +# trip this; a failure here most likely means rand() is stuck on a +# constant (e.g. an unseeded interpreter) rather than a real +# statistical fluke. +with_test_prefix "randomised-start-offset" { + reset_tmpdir randstart + set dones {} + for {set i 1} {$i <= 16} {incr i} { + lappend dones [spawn_child $i shared-4-1 20] + # Tiny stagger so each child sees an empty pool and the + # rand() outcome alone decides the starting pool. + after 30 + } + gdb_assert {[wait_all $dones]} "children completed" + set events [read_events $log_file] + dump_events $events + set pools {} + foreach e $events { + if {[regexp {hold pool=(\d+)} $e -> p]} { lappend pools $p } + } + set distinct [llength [lsort -unique $pools]] + verbose -log " start pools: $pools ($distinct distinct)" + gdb_assert {$distinct >= 3} "rand spreads start" +} + +# === Error in body releases the lock and rethrows. === +# In-process: drive the wrapper with a thrown body, confirm the +# error propagates. Then re-acquire from a child with a tight +# wait_all timeout, so a regression that leaks the slot fails the +# scenario instead of hanging until DejaGnu's top-level timeout. +with_test_prefix "error-releases-lock" { + reset_tmpdir errprop + save_vars { ::GDB_PARALLEL ::GDB_LOCK_DIR } { + set ::GDB_PARALLEL 1 + set ::GDB_LOCK_DIR $tmpdir + verbose -log " invoking with_shared_lock_multi with body that throws" + set caught 0 + set msg "" + if {[catch { + with_shared_lock_multi slot 1 1 barrier { error "boom" } + } msg]} { + set caught 1 + } + verbose -log " caught=$caught msg='$msg'" + gdb_assert {$caught} "error from body propagated" + gdb_assert {$msg eq "boom"} "message preserved" + } + verbose -log " re-acquiring same slot via child (3s deadline)" + set dones [list [spawn_child 99 shared-1-1 50]] + gdb_assert {[wait_all $dones 3000]} \ + "re-acquire completed within deadline (slot was released)" + set events [read_events $log_file] + dump_events $events +} diff --git a/gdb/testsuite/lib/gdb-utils.exp b/gdb/testsuite/lib/gdb-utils.exp index cabeb7aa1ed..d33eec80a65 100644 --- a/gdb/testsuite/lib/gdb-utils.exp +++ b/gdb/testsuite/lib/gdb-utils.exp @@ -197,6 +197,159 @@ proc lock_file_acquire {lockfile} { } } +# Shared/exclusive file-lock layer +# ================================ +# +# The procs below build a reader/writer lock out of the existing +# lock_file_acquire primitive. Two API tiers are provided: +# +# Single-pool: N slot files + 1 barrier file. Shared callers grab +# one slot; exclusive callers grab all N. +# Multi-pool: NPOOLS independent pools, each PER_POOL slots wide, +# with one barrier per pool. Models "N independent +# resources with bounded concurrency each" (used by the +# rocm GPU lock to spread tests across GPUs). +# +# Layering: +# +# open EXCL --+--> lock_file_acquire (blocking) +# +--> lock_file_try_acquire (non-blocking) +# | +# v +# single-pool primitives: +# lock_file_acquire_shared \ +# lock_file_acquire_exclusive >--> lock_file_release_shared +# ^ / +# | +# with_shared_lock --+ +# with_exclusive_lock --+ (GDB_PARALLEL gate + uplevel body) +# +# multi-pool primitives: +# lock_file_acquire_shared_multi \ +# lock_file_acquire_exclusive_multi >--> lock_file_release_shared +# lock_file_acquire_machine_multi / +# ^ +# | +# with_shared_lock_multi --+ (GDB_PARALLEL gate + uplevel body; +# with_exclusive_lock_multi --+ also exposes _lock_pool / +# with_machine_lock_multi --+ _lock_slot to the body) +# +# Writer-priority barrier: an exclusive (writer) acquirer creates the +# barrier file before going after slots. Shared (reader) acquirers +# check the barrier each pass and yield while it exists, so a steady +# stream of readers cannot starve a queued writer. In the multi-pool +# variant, writer-priority is scoped per pool: the writer first picks +# a pool (by try-acquiring its barrier), and only once that barrier +# is held are readers on *that* pool blocked -- readers can still +# enter other pools. A machine acquirer raises every pool's barrier +# up-front, so it blocks readers everywhere. +# +# Mode summary (multi-pool, NPOOLS pools x PER_POOL slots each): +# +# slots held other callers can run +# shared_multi 1 slot, 1 pool shared (any pool, incl. same +# pool up to PER_POOL); +# exclusive on other pools only +# exclusive_multi all slots, 1 pool any mode on other pools +# machine_multi all slots, all nothing +# pools +# +# When GDB_PARALLEL is unset (single-threaded test run), all with_* +# wrappers skip locking entirely. + +# Try once to acquire LOCKFILE without blocking. Returns the same +# token shape as lock_file_acquire on success, or the empty string on +# failure. + +proc lock_file_try_acquire {lockfile} { + if {[catch {open $lockfile {WRONLY CREAT EXCL}} rc]} { + return "" + } + set msg "locked by $::subdir/${::gdb_test_file_name}.exp" + puts $rc $msg + flush $rc + return [list $rc $lockfile] +} + +# Acquire one of N slot files in SLOT_DIR named SLOT_PREFIX-0 .. +# SLOT_PREFIX-(NSLOTS-1), implementing a shared (reader) lock with at +# most NSLOTS concurrent holders. BARRIER_FILE, if present, indicates +# an exclusive (writer) waiter is queued; shared acquirers yield to it +# to avoid writer starvation. Returns a token for +# lock_file_release_shared. + +proc lock_file_acquire_shared {slot_dir slot_prefix nslots barrier_file} { + verbose -log "acquiring shared lock ($slot_prefix x$nslots):\ + $::subdir/${::gdb_test_file_name}.exp" + while {true} { + # Yield to any pending exclusive waiter. + if {[file exists $barrier_file]} { + after 10 + continue + } + for {set i 0} {$i < $nslots} {incr i} { + set slot [file join $slot_dir "$slot_prefix-$i.lock"] + set tok [lock_file_try_acquire $slot] + if {$tok ne ""} { + verbose -log "shared lock: acquired slot $i" + return [list shared $tok] + } + } + after 10 + } +} + +# Acquire an exclusive (writer) lock over the NSLOTS-slot set in +# SLOT_DIR named SLOT_PREFIX-0 .. SLOT_PREFIX-(NSLOTS-1) -- the same +# set used by lock_file_acquire_shared. First creates BARRIER_FILE to +# prevent new shared acquirers from grabbing slots, then waits to hold +# all NSLOTS slots simultaneously. Multiple exclusive waiters queue +# on BARRIER_FILE. Returns a token for lock_file_release_shared. + +proc lock_file_acquire_exclusive {slot_dir slot_prefix nslots barrier_file} { + verbose -log "acquiring exclusive lock ($slot_prefix x$nslots):\ + $::subdir/${::gdb_test_file_name}.exp" + # Queue on the barrier file: only one exclusive waiter is active + # at a time. + set barrier_tok [lock_file_acquire $barrier_file] + set slot_toks {} + for {set i 0} {$i < $nslots} {incr i} { + set slot [file join $slot_dir "$slot_prefix-$i.lock"] + lappend slot_toks [lock_file_acquire $slot] + } + verbose -log "exclusive lock: acquired all $nslots slots" + return [list exclusive $barrier_tok $slot_toks] +} + +# Release a token returned by lock_file_acquire_shared, +# lock_file_acquire_exclusive, or lock_file_acquire_machine_multi. +# The token's first element tags which kind it is. + +proc lock_file_release_shared {info} { + set kind [lindex $info 0] + verbose -log "releasing $kind lock: $::subdir/${::gdb_test_file_name}.exp" + if {$kind eq "shared"} { + lock_file_release [lindex $info 1] + } elseif {$kind eq "exclusive"} { + foreach tok [lindex $info 2] { + lock_file_release $tok + } + # Release the barrier last so queued shared acquirers don't + # race against slot release. + lock_file_release [lindex $info 1] + } elseif {$kind eq "machine"} { + # Release every slot, then every barrier, mirroring exclusive. + foreach tok [lindex $info 2] { + lock_file_release $tok + } + foreach tok [lindex $info 1] { + lock_file_release $tok + } + } else { + error "invalid shared/exclusive lock token: $info" + } +} + # Release a lock file. proc lock_file_release {info} { @@ -249,6 +402,254 @@ proc with_lock { lock_file body } { } } +# Run BODY holding a shared (reader) slot in an NSLOTS-wide pool +# named SLOT_PREFIX, with BARRIER_NAME as the writer-priority barrier +# file. All filenames are relative to [lock_dir]. When GDB_PARALLEL +# is unset, BODY runs unlocked, matching with_lock. + +proc with_shared_lock { slot_prefix nslots barrier_name body } { + if {[info exists ::GDB_PARALLEL]} { + set dir [lock_dir] + set barrier_file [file join $dir $barrier_name] + set tok [lock_file_acquire_shared $dir $slot_prefix $nslots \ + $barrier_file] + } + + set code [catch {uplevel 1 $body} result] + + if {[info exists ::GDB_PARALLEL]} { + lock_file_release_shared $tok + } + + if {$code == 1} { + global errorInfo errorCode + return -code $code -errorinfo $errorInfo -errorcode $errorCode $result + } else { + return -code $code $result + } +} + +# Run BODY holding all slots in the NSLOTS-wide pool named +# SLOT_PREFIX (exclusive/writer access), with BARRIER_NAME as the +# writer-priority barrier file. All filenames are relative to +# [lock_dir]. When GDB_PARALLEL is unset, BODY runs unlocked, +# matching with_lock. + +proc with_exclusive_lock { slot_prefix nslots barrier_name body } { + if {[info exists ::GDB_PARALLEL]} { + set dir [lock_dir] + set barrier_file [file join $dir $barrier_name] + set tok [lock_file_acquire_exclusive $dir $slot_prefix $nslots \ + $barrier_file] + } + + set code [catch {uplevel 1 $body} result] + + if {[info exists ::GDB_PARALLEL]} { + lock_file_release_shared $tok + } + + if {$code == 1} { + global errorInfo errorCode + return -code $code -errorinfo $errorInfo -errorcode $errorCode $result + } else { + return -code $code $result + } +} + +# Multi-pool variants of the slot-pool locks above. These manage NPOOLS +# independent pools, each PER_POOL slots wide, with a per-pool barrier +# file. A shared (reader) acquirer takes one slot in any pool whose +# barrier is clear; an exclusive (writer) acquirer takes every slot in a +# single pool while leaving the other pools available. This models +# "N independent resources, each with bounded concurrency" -- e.g. one +# pool per GPU. +# +# Slot file name: SLOT_PREFIX--.lock +# Barrier file name: BARRIER_PREFIX-.lock + +# Acquire one slot in one pool. Fills slot 0 of every pool before +# touching slot 1 of any pool (so on a multi-GPU box each GPU gets at +# most one tenant before any GPU gets two). Within a slot row, pools +# are scanned starting at a random offset so concurrent acquirers +# don't all land on pool 0. Returns {shared }. + +proc lock_file_acquire_shared_multi {slot_dir slot_prefix npools per_pool \ + barrier_prefix} { + verbose -log "acquiring shared lock ($slot_prefix x$npools x$per_pool):\ + $::subdir/${::gdb_test_file_name}.exp" + while {true} { + # Outer loop walks slot rows (0 .. per_pool-1) and the inner + # loop sweeps every pool within a row -- this is what gives + # the row-major "fill slot 0 of every pool before slot 1 of + # any pool" property the multi-pool-spread self-test relies + # on. Drawing a fresh random start per row (rather than per + # inner iteration) keeps that property while still spreading + # concurrent acquirers off pool 0. + for {set s 0} {$s < $per_pool} {incr s} { + set start [expr {int(rand() * $npools)}] + for {set i 0} {$i < $npools} {incr i} { + set pool [expr {($start + $i) % $npools}] + set barrier [file join $slot_dir \ + "$barrier_prefix-$pool.lock"] + if {[file exists $barrier]} { + continue + } + set slot [file join $slot_dir \ + "$slot_prefix-$pool-$s.lock"] + set tok [lock_file_try_acquire $slot] + if {$tok ne ""} { + verbose -log "shared lock: pool $pool slot $s" + return [list shared $tok $pool $s] + } + } + } + after 10 + } +} + +# Acquire all slots in one pool (writer access to that pool only). +# Picks a pool by scanning from a random offset. Returns +# {exclusive 0}. + +proc lock_file_acquire_exclusive_multi {slot_dir slot_prefix npools per_pool \ + barrier_prefix} { + verbose -log "acquiring exclusive lock ($slot_prefix x$npools x$per_pool):\ + $::subdir/${::gdb_test_file_name}.exp" + # First, claim a pool's barrier so we have a definite target. Try + # pools in random order; if all barriers are held by other + # exclusive waiters, sleep and retry. + set barrier_tok "" + set pool -1 + while {$barrier_tok eq ""} { + set start [expr {int(rand() * $npools)}] + for {set i 0} {$i < $npools} {incr i} { + set p [expr {($start + $i) % $npools}] + set barrier [file join $slot_dir "$barrier_prefix-$p.lock"] + set tok [lock_file_try_acquire $barrier] + if {$tok ne ""} { + set barrier_tok $tok + set pool $p + break + } + } + if {$barrier_tok eq ""} { after 10 } + } + # Now block until we hold every slot in that pool. + set slot_toks {} + for {set s 0} {$s < $per_pool} {incr s} { + lappend slot_toks [lock_file_acquire \ + [file join $slot_dir "$slot_prefix-$pool-$s.lock"]] + } + verbose -log "exclusive lock: acquired pool $pool" + return [list exclusive $barrier_tok $slot_toks $pool 0] +} + +# Wrappers matching with_shared_lock / with_exclusive_lock. Before +# evaluating BODY, set caller-visible variables _lock_pool and +# _lock_slot to the acquired indices, so BODY can map them to a +# resource (e.g. a GPU device). + +proc with_shared_lock_multi {slot_prefix npools per_pool barrier_prefix \ + body} { + set pool 0 + set slot 0 + if {[info exists ::GDB_PARALLEL]} { + set dir [lock_dir] + set tok [lock_file_acquire_shared_multi $dir $slot_prefix \ + $npools $per_pool $barrier_prefix] + set pool [lindex $tok 2] + set slot [lindex $tok 3] + } + uplevel 1 [list set _lock_pool $pool] + uplevel 1 [list set _lock_slot $slot] + set code [catch {uplevel 1 $body} result] + if {[info exists ::GDB_PARALLEL]} { + lock_file_release_shared $tok + } + if {$code == 1} { + global errorInfo errorCode + return -code $code -errorinfo $errorInfo -errorcode $errorCode $result + } else { + return -code $code $result + } +} + +proc with_exclusive_lock_multi {slot_prefix npools per_pool barrier_prefix \ + body} { + set pool 0 + if {[info exists ::GDB_PARALLEL]} { + set dir [lock_dir] + set tok [lock_file_acquire_exclusive_multi $dir $slot_prefix \ + $npools $per_pool $barrier_prefix] + set pool [lindex $tok 3] + } + uplevel 1 [list set _lock_pool $pool] + uplevel 1 [list set _lock_slot 0] + set code [catch {uplevel 1 $body} result] + if {[info exists ::GDB_PARALLEL]} { + lock_file_release_shared $tok + } + if {$code == 1} { + global errorInfo errorCode + return -code $code -errorinfo $errorInfo -errorcode $errorCode $result + } else { + return -code $code $result + } +} + +# Acquire writer access to every pool at once (machine-wide +# exclusive). Barriers are taken in ascending pool order to avoid +# deadlock between two machine acquirers; slots are then taken +# everywhere. Returns {machine }. + +proc lock_file_acquire_machine_multi {slot_dir slot_prefix npools per_pool \ + barrier_prefix} { + verbose -log "acquiring machine lock ($slot_prefix x$npools x$per_pool):\ + $::subdir/${::gdb_test_file_name}.exp" + set barrier_toks {} + for {set p 0} {$p < $npools} {incr p} { + lappend barrier_toks [lock_file_acquire \ + [file join $slot_dir "$barrier_prefix-$p.lock"]] + } + set slot_toks {} + for {set p 0} {$p < $npools} {incr p} { + for {set s 0} {$s < $per_pool} {incr s} { + lappend slot_toks [lock_file_acquire \ + [file join $slot_dir "$slot_prefix-$p-$s.lock"]] + } + } + verbose -log "machine lock: acquired all $npools pools" + return [list machine $barrier_toks $slot_toks] +} + +# Run BODY holding every slot in every pool. No other shared, +# exclusive, or machine acquirer can run anywhere during BODY. Sets +# _lock_pool / _lock_slot to 0 in the caller's frame. + +proc with_machine_lock_multi {slot_prefix npools per_pool barrier_prefix \ + body} { + set pool 0 + set slot 0 + if {[info exists ::GDB_PARALLEL]} { + set dir [lock_dir] + set tok [lock_file_acquire_machine_multi $dir $slot_prefix \ + $npools $per_pool $barrier_prefix] + } + uplevel 1 [list set _lock_pool $pool] + uplevel 1 [list set _lock_slot $slot] + set code [catch {uplevel 1 $body} result] + if {[info exists ::GDB_PARALLEL]} { + lock_file_release_shared $tok + } + if {$code == 1} { + global errorInfo errorCode + return -code $code -errorinfo $errorInfo -errorcode $errorCode $result + } else { + return -code $code $result + } +} + # Alias for subst -nobackslashes -nocommands. proc subst_vars { str } { diff --git a/gdb/testsuite/lib/rocm.exp b/gdb/testsuite/lib/rocm.exp index 215035afa5e..8a8a73b321f 100644 --- a/gdb/testsuite/lib/rocm.exp +++ b/gdb/testsuite/lib/rocm.exp @@ -235,19 +235,272 @@ proc rocm_assemble {source dest {options ""}} { return [lindex $res 1] } -# The lock file used to ensure that only one GDB has access to the GPU -# at a time. -set gpu_lock_filename gpu-parallel.lock +# Names used for the rocm GPU lock pools. One pool per visible GPU, +# each with $gpu_slots_per_gpu slots. Shared (parallel) tests take +# one slot in any pool; exclusive (serial) tests take every slot of a +# single pool (one whole GPU), leaving the other GPUs free. A +# per-pool barrier file gives writer priority on that pool. +set gpu_slot_prefix gpu-parallel-slot +set gpu_barrier_prefix gpu-parallel-barrier + +# Per-GPU concurrency cap. Configurable via ROCGDB_ROCM_PARALLEL_SLOTS. +# Default 1 keeps the historical "one rocm test per GPU" behaviour; raise +# it to let multiple parallel-marked tests share a GPU on hardware/stacks +# known to tolerate higher kfd/dbgapi concurrency. +if {[info exists ::env(ROCGDB_ROCM_PARALLEL_SLOTS)] + && [string is integer -strict $::env(ROCGDB_ROCM_PARALLEL_SLOTS)] + && $::env(ROCGDB_ROCM_PARALLEL_SLOTS) > 0} { + set gpu_slots_per_gpu $::env(ROCGDB_ROCM_PARALLEL_SLOTS) +} else { + set gpu_slots_per_gpu 1 +} + +# Decide how many GPUs to spread tests across, and build the +# rocm_pool_to_physical map so the per-body lock can translate a pool +# index back to a physical device index (the actual ROCR_VISIBLE_DEVICES +# masking happens inside with_rocm_gpu_lock, not here). +# +# - GDB_PARALLEL unset: pin to one GPU (device 0). Reproduces the +# pre-parallel single-tenant behaviour. +# - ROCM_TEST_ARCH set: keep only GPUs whose gfx name matches. +# - Heterogeneous fleet, no ROCM_TEST_ARCH: failsafe to one GPU, +# since a fat binary may not cover every arch present and per-arch +# dispatch is not yet implemented. +# - Homogeneous fleet: use every GPU. +# +# The eligible-GPU count may then be capped by +# ROCGDB_ROCM_GPU_MAX_PARALLEL. Default is uncapped (use every +# eligible GPU); lower it if kfd/dbgapi contention from many concurrent +# debug sessions on the same node starts producing spurious "no +# ROCm-capable device" failures mid-run. +# +# The block is wrapped in an apply lambda so its helper variables stay +# local and don't pollute the global namespace; gpu_pool_count and +# rocm_pool_to_physical escape via ::. +apply {{} { + set all_devices [find_amdgpu_devices] + set unique_archs [lsort -unique $all_devices] + + # Build rocm_pool_to_physical: a list whose i-th element is the + # physical device index that lock pool i pins to. Pool indices + # are an internal, dense 0..gpu_pool_count-1 numbering used by + # the lock layer; physical indices are what HSA / KFD see and + # what the user wrote in ROCR_VISIBLE_DEVICES. Keeping the two + # spaces separate lets a user pre-mask the testsuite to a subset + # of the machine's GPUs (e.g. ROCR_VISIBLE_DEVICES=4,5,6 to stay + # off devices 0-3) without the lock layer silently rewriting the + # mask to 0,1,2. + # + # find_amdgpu_devices runs a HIP enumerator child that *inherits* + # our ROCR_VISIBLE_DEVICES, so $all_devices is already the + # post-filter view: one entry per device the user's mask exposed, + # numbered 0..N-1 from HIP's perspective. We therefore start + # with an identity map (pool i -> physical i) sized to the + # surviving devices, which is correct when RVD is unset. + set ::rocm_pool_to_physical {} + for {set i 0} {$i < [llength $all_devices]} {incr i} { + lappend ::rocm_pool_to_physical $i + } + + # If the user pre-set ROCR_VISIBLE_DEVICES, replace the identity + # map with the user's physical indices in order: the k-th + # surviving HIP device corresponds to the k-th entry of the user + # mask. We only trust the mask if its length matches the + # enumeration; a mismatch (typo, stale value, device the runtime + # rejected) would misroute every per-test pin, so we log and + # fall back to identity instead. + if {[info exists ::env(ROCR_VISIBLE_DEVICES)] + && $::env(ROCR_VISIBLE_DEVICES) ne ""} { + set user_rvd [split $::env(ROCR_VISIBLE_DEVICES) ","] + if {[llength $user_rvd] == [llength $all_devices]} { + set ::rocm_pool_to_physical $user_rvd + } else { + verbose -log "rocm: ROCR_VISIBLE_DEVICES=$::env(ROCR_VISIBLE_DEVICES)\ + does not match enumerated device count\ + ([llength $all_devices]); using identity mapping" + } + } + + # Patch the degenerate no-GPU case up front so every branch below + # can assume rocm_pool_to_physical has at least one entry; without + # this, a branch that falls back to "first device" via + # [lrange ... 0 0] would silently produce an empty pool map and + # later set ROCR_VISIBLE_DEVICES to the empty string. + if {[llength $::rocm_pool_to_physical] == 0} { + set ::rocm_pool_to_physical {0} + } + + # Now narrow rocm_pool_to_physical to the devices we actually + # intend to spread tests across, keeping pool indices contiguous + # from 0. Each branch below sets gpu_pool_count and shrinks the + # map in lockstep so element i is always pool i's physical + # device. + if {![info exists ::GDB_PARALLEL]} { + # Non-parallel run: collapse to a single pool on the first + # available physical device (preserves the user's choice if + # RVD was pre-set, otherwise physical 0). + set ::gpu_pool_count 1 + set ::rocm_pool_to_physical [lrange $::rocm_pool_to_physical 0 0] + } elseif {[info exists ::env(ROCM_TEST_ARCH)]} { + # Keep only devices matching the requested arch. We index + # into rocm_pool_to_physical (not raw 0..N-1) so the surviving + # entries are physical indices, not HIP-renumbered ones. + set keep {} + for {set i 0} {$i < [llength $all_devices]} {incr i} { + if {[lindex $all_devices $i] eq $::env(ROCM_TEST_ARCH)} { + lappend keep [lindex $::rocm_pool_to_physical $i] + } + } + if {[llength $keep] == 0} { + verbose -log "ROCM_TEST_ARCH=$::env(ROCM_TEST_ARCH) matched no GPU;\ + falling back to first eligible device\ + ([lindex $::rocm_pool_to_physical 0])" + set ::gpu_pool_count 1 + set ::rocm_pool_to_physical [lrange $::rocm_pool_to_physical 0 0] + } else { + set ::gpu_pool_count [llength $keep] + set ::rocm_pool_to_physical $keep + } + } elseif {[llength $unique_archs] > 1} { + # Heterogeneous fleet, no arch picked: same single-pool + # fallback as the GDB_PARALLEL-unset branch. + verbose -log "rocm: heterogeneous arches ($unique_archs);\ + pinning to first eligible device\ + ([lindex $::rocm_pool_to_physical 0])\ + (set ROCM_TEST_ARCH to override)" + set ::gpu_pool_count 1 + set ::rocm_pool_to_physical [lrange $::rocm_pool_to_physical 0 0] + } else { + # Homogeneous fleet: use every device. rocm_pool_to_physical + # is already the right length and content from the identity / + # user mask step above. + set ::gpu_pool_count [llength $::rocm_pool_to_physical] + } + + # Cap the active pool count. Default is the full eligible set + # (every GPU surviving the ROCR_VISIBLE_DEVICES / ROCM_TEST_ARCH + # / arch homogeneity filtering above); users can lower it via + # the environment if kfd/dbgapi contention from many concurrent + # debug sessions starts producing spurious mid-run failures on + # their stack. + if {[info exists ::env(ROCGDB_ROCM_GPU_MAX_PARALLEL)] + && [string is integer -strict $::env(ROCGDB_ROCM_GPU_MAX_PARALLEL)] + && $::env(ROCGDB_ROCM_GPU_MAX_PARALLEL) > 0} { + set max_pools $::env(ROCGDB_ROCM_GPU_MAX_PARALLEL) + } else { + set max_pools $::gpu_pool_count + } + if {$::gpu_pool_count > $max_pools} { + verbose -log "rocm: capping pool count from $::gpu_pool_count to\ + $max_pools (ROCGDB_ROCM_GPU_MAX_PARALLEL)" + set ::gpu_pool_count $max_pools + set ::rocm_pool_to_physical \ + [lrange $::rocm_pool_to_physical 0 [expr {$::gpu_pool_count - 1}]] + } +}} -# Run body under the GPU lock. Also calls gdb_exit before releasing -# the GPU lock. +# Run BODY under the rocm GPU lock. Three concurrency modes: +# +# parallel - takes one slot on any GPU. Pick this for tests that +# only exercise a small slice of the GPU and have no +# device-wide state. +# serial - (default) takes every slot of one GPU. No other test +# runs on the assigned GPU during BODY, but other GPUs +# remain free. +# machine - takes every slot of every GPU. No other rocm test +# runs anywhere on the machine. Pick this for tests +# that spawn additional HIP processes (hogs, helpers) +# or are otherwise sensitive to system-wide ROCm / +# dbgapi contention. +# +# Select non-default modes near the top of the .exp file, before +# calling with_rocm_gpu_lock: +# +# set rocm_gpu_concurrency parallel +# set rocm_gpu_concurrency machine +# +# BODY runs with ROCR_VISIBLE_DEVICES set to the assigned GPU index, +# so the inferior launched by GDB sees only that device. Also calls +# gdb_exit before releasing the lock. proc with_rocm_gpu_lock { body } { - with_lock $::gpu_lock_filename {uplevel 1 $body} + set mode "serial" + if {[info exists ::rocm_gpu_concurrency]} { + set mode $::rocm_gpu_concurrency + } + + # Failsafe: on devices that do not support debugging multiple + # processes concurrently (older gfx generations), force serial + # execution regardless of the per-test marker. Otherwise two + # rocm tests sharing the GPU would race. + if {$mode eq "parallel" + && ![hip_devices_support_debug_multi_process]} { + verbose -log "with_rocm_gpu_lock: forcing serial mode\ + (device lacks multi-process debug support)" + set mode "serial" + } + + # With only one pool there is no difference between serial and + # machine modes; collapse to serial so the lock helper is + # uniform. + if {$mode eq "machine" && $::gpu_pool_count <= 1} { + set mode "serial" + } - # In case BODY returned early due to some testcase failing, and - # left GDB running, debugging the GPU. - gdb_exit + # Wrapper that pins the inferior to the GPU corresponding to the + # acquired pool, then runs the user body. _lock_pool and + # _lock_slot are set by the with_*_lock_multi wrappers. + # + # Only ROCR_VISIBLE_DEVICES is set: it filters at the HSA layer + # and renumbers the surviving devices from 0, so HIP already sees + # the right view. Setting HIP_VISIBLE_DEVICES to the same + # absolute index would re-filter the renumbered set and select a + # device that no longer exists for any pool other than 0. + set inner { + # Translate the dense pool index into the physical device index + # the user (or the startup auto-detect) selected. Without this + # indirection a pre-set ROCR_VISIBLE_DEVICES=4,5,6 would still + # pin pool 0 to physical 0. + set _gpu [lindex $::rocm_pool_to_physical $_lock_pool] + set _had [info exists ::env(ROCR_VISIBLE_DEVICES)] + if {$_had} { set _saved_rvd $::env(ROCR_VISIBLE_DEVICES) } + set ::env(ROCR_VISIBLE_DEVICES) $_gpu + verbose -log "rocm: pool=$_lock_pool slot=$_lock_slot ->\ + ROCR_VISIBLE_DEVICES=$_gpu" + set _rc [catch {uplevel 1 $body} _res] + global errorInfo errorCode + set _einfo $errorInfo + set _ecode $errorCode + if {$_had} { + set ::env(ROCR_VISIBLE_DEVICES) $_saved_rvd + } else { + unset -nocomplain ::env(ROCR_VISIBLE_DEVICES) + } + # Tear GDB down while we still hold the slot, so the pinned GPU + # is fully released before the next test acquires this slot. In + # case BODY returned early due to some testcase failing and left + # GDB running, debugging the GPU. + gdb_exit + if {$_rc == 1} { + return -code $_rc -errorinfo $_einfo -errorcode $_ecode $_res + } elseif {$_rc} { + return -code $_rc $_res + } + } + + if {$mode eq "parallel"} { + with_shared_lock_multi $::gpu_slot_prefix \ + $::gpu_pool_count $::gpu_slots_per_gpu \ + $::gpu_barrier_prefix $inner + } elseif {$mode eq "machine"} { + with_machine_lock_multi $::gpu_slot_prefix \ + $::gpu_pool_count $::gpu_slots_per_gpu \ + $::gpu_barrier_prefix $inner + } else { + with_exclusive_lock_multi $::gpu_slot_prefix \ + $::gpu_pool_count $::gpu_slots_per_gpu \ + $::gpu_barrier_prefix $inner + } } # Build an OpenCL kernel library from SOURCES. @@ -407,7 +660,7 @@ proc get_current_gpu_arch {} { # Return true if all the devices support debugging multiple processes # using the GPU. -proc hip_devices_support_debug_multi_process {} { +gdb_caching_proc hip_devices_support_debug_multi_process {} { set unsupported_targets { gfx900 gfx906 gfx908 gfx1010 gfx1011 gfx1012 gfx1030 gfx1031 gfx1032 diff --git a/gdb/testsuite/tclint-plugin.py b/gdb/testsuite/tclint-plugin.py index 5a09397c3a7..e6779d3b409 100644 --- a/gdb/testsuite/tclint-plugin.py +++ b/gdb/testsuite/tclint-plugin.py @@ -159,6 +159,14 @@ def _with_lock(args, parser): return _script_last_arg("with_lock", 2, args, parser) +def _with_shared_lock(args, parser): + return _script_last_arg("with_shared_lock", 4, args, parser) + + +def _with_exclusive_lock(args, parser): + return _script_last_arg("with_exclusive_lock", 4, args, parser) + + def _with_rocm_gpu_lock(args, parser): return _script_last_arg("with_rocm_gpu_lock", 1, args, parser) @@ -206,6 +214,8 @@ def _ns_term_with_tuiterm(args, parser): "with_override": _with_override, "with_ansi_styling_terminal": _with_ansi_styling_terminal, "with_lock": _with_lock, + "with_shared_lock": _with_shared_lock, + "with_exclusive_lock": _with_exclusive_lock, "with_rocm_gpu_lock": _with_rocm_gpu_lock, "Term::_log_cur": _ns_term_log_cur, "Term::with_term": _ns_term_with_term,