From fcca720a9dba0f27002d73b0192e1793e125a1ba Mon Sep 17 00:00:00 2001 From: Olivier Cots Date: Mon, 27 Jul 2026 15:46:55 +0200 Subject: [PATCH 1/4] ci on windows --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d98c0043..246413f8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,7 +21,8 @@ jobs: (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run ci')) uses: control-toolbox/CTActions/.github/workflows/ci.yml@main with: - runs_on: '["ubuntu-latest", "macos-latest"]' + # runs_on: '["ubuntu-latest", "macos-latest"]' + runs_on: '["windows-latest"]' use_ct_registry: true secrets: SSH_KEY: ${{ secrets.SSH_KEY }} \ No newline at end of file From 76706ba07c8bf0463762e9d5b454cc22a56ecbbb Mon Sep 17 00:00:00 2001 From: Olivier Cots Date: Mon, 27 Jul 2026 16:27:20 +0200 Subject: [PATCH 2/4] fix(TestRunner): normalize test file paths to forward slashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit relpath returns backslash-separated paths on Windows, which silently broke every glob-based test selection (including the default "suite/*/test_*") since patterns are always written with forward slashes โ€” resulting in 0 tests being collected and CI reporting a false green. Add a CI job that dry-runs test selection with and without arguments on windows-latest/ubuntu-latest to catch this class of regression going forward. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/CI.yml | 57 ++++++++++++++++++- ext/TestRunner/test_selection.jl | 3 +- .../extensions/test_testrunner_selection.jl | 18 +++++- 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 246413f8..ec44b34c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,8 +21,59 @@ jobs: (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run ci')) uses: control-toolbox/CTActions/.github/workflows/ci.yml@main with: - # runs_on: '["ubuntu-latest", "macos-latest"]' - runs_on: '["windows-latest"]' + runs_on: '["ubuntu-latest", "macos-latest", "windows-latest"]' use_ct_registry: true secrets: - SSH_KEY: ${{ secrets.SSH_KEY }} \ No newline at end of file + SSH_KEY: ${{ secrets.SSH_KEY }} + + # Smoke check for the TestRunner extension's test-selection logic (glob + # matching over `_collect_test_files_recursive`). This exists independently + # of the `call` job above because it needs to pass custom `test_args` to + # `Pkg.test`, which the reusable CTActions ci.yml workflow doesn't expose. + # It caught a regression where Windows returned backslash-separated relative + # paths, silently breaking forward-slash glob patterns and making every + # selection (including the default "run everything") resolve to zero tests. + test-selection-smoke: + if: > + github.event_name != 'pull_request' || + (github.event.action == 'labeled' && github.event.label.name == 'run ci') || + (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run ci')) + strategy: + fail-fast: false + matrix: + runs_on: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.runs_on }} + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@latest + with: + version: '1.12' + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + + - name: Dry run - no selection (all tests) + shell: bash + run: | + julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["--dryrun"])' | tee dryrun_all.log + count=$(sed -n '/^Dry run:/,$p' dryrun_all.log | tail -n +2 | grep -c .) + echo "Discovered $count test(s) with no selection" + if [ "$count" -lt 10 ]; then + echo "::error::Expected at least 10 tests with no selection, got $count" + exit 1 + fi + + - name: Dry run - directory selection (suite/data) + shell: bash + run: | + julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["suite/data", "--dryrun"])' | tee dryrun_data.log + selected=$(sed -n '/^Dry run:/,$p' dryrun_data.log | tail -n +2 | grep .) + count=$(echo "$selected" | grep -c .) + echo "Discovered $count test(s) under suite/data" + if [ "$count" -lt 1 ]; then + echo "::error::Expected at least 1 test under suite/data, got $count" + exit 1 + fi + if echo "$selected" | grep -qv '^suite/data/'; then + echo "::error::Selection 'suite/data' returned tests outside suite/data" + exit 1 + fi \ No newline at end of file diff --git a/ext/TestRunner/test_selection.jl b/ext/TestRunner/test_selection.jl index ec152b4e..9f5dcf5a 100644 --- a/ext/TestRunner/test_selection.jl +++ b/ext/TestRunner/test_selection.jl @@ -30,7 +30,8 @@ function _collect_test_files_recursive(test_dir::AbstractString) for f in fs if endswith(f, ".jl") && f != "runtests.jl" full = joinpath(root, f) - push!(files, relpath(full, test_dir)) + rel = relpath(full, test_dir) + push!(files, replace(rel, '\\' => '/')) end end end diff --git a/test/suite/extensions/test_testrunner_selection.jl b/test/suite/extensions/test_testrunner_selection.jl index 9ef7f1fb..a8ca98ec 100644 --- a/test/suite/extensions/test_testrunner_selection.jl +++ b/test/suite/extensions/test_testrunner_selection.jl @@ -197,11 +197,27 @@ function test_testrunner_selection() touch(joinpath(temp_dir, "a", "b", "test_x.jl")) rel = find_symbol_file(:x, n -> "test_" * String(n); test_dir=temp_dir) - Test.@test rel == joinpath("a", "test_x.jl") + Test.@test rel == "a/test_x.jl" end end end + Test.@testset verbose = VERBOSE showtiming = SHOWTIMING "_collect_test_files_recursive uses forward slashes" begin + collect_files = TestRunner._collect_test_files_recursive + + # Regression: on Windows, `relpath` returns backslash-separated paths, + # which broke glob matching against forward-slash patterns like + # "suite/*/test_*" (available_tests filters always end up empty). + mktempdir() do temp_dir + mkpath(joinpath(temp_dir, "suite", "core")) + touch(joinpath(temp_dir, "suite", "core", "test_core.jl")) + + files = collect_files(temp_dir) + Test.@test files == ["suite/core/test_core.jl"] + Test.@test all(f -> !occursin('\\', f), files) + end + end + Test.@testset verbose = VERBOSE showtiming = SHOWTIMING "order preservation" begin parse_args = TestRunner._parse_test_args select_tests = TestRunner._select_tests From 55b62c72a9c8c51c4f7ca1a4d9d9199fb5869ff3 Mon Sep 17 00:00:00 2001 From: Olivier Cots Date: Mon, 27 Jul 2026 16:43:09 +0200 Subject: [PATCH 3/4] release(0.28.7-beta): bump version and changelog for Windows TestRunner fix Also make the new test-selection-smoke CI job robust to a Windows-only Pkg.test quirk: after a cold precompile, julia can exit non-zero after already printing "tests passed" with no error output, which pipefail was turning into a false failure of the smoke job. Judge success from the dry-run listing content instead of julia's raw exit code. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/CI.yml | 22 ++++++++++++++++++-- BREAKING.md | 13 ++++++++++++ CHANGELOG.md | 43 ++++++++++++++++++++++++++++++++++++++++ Project.toml | 2 +- 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ec44b34c..9e3c4816 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -51,10 +51,23 @@ jobs: - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 + # Julia's own process exit code is not trustworthy here: on Windows, + # after a cold/fresh precompile, `Pkg.test` can print its "tests + # passed" success message and still return a nonzero process exit + # code from unrelated teardown (no exception, no stderr output). + # Piping through `tee` would let bash's pipefail abort the step on + # that spurious code before the dry-run output is even inspected, so + # redirect to a file instead and judge success from the *content* of + # the dry-run listing, not from julia's raw exit status. - name: Dry run - no selection (all tests) shell: bash run: | - julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["--dryrun"])' | tee dryrun_all.log + set +e + julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["--dryrun"])' > dryrun_all.log 2>&1 + julia_exit=$? + set -e + cat dryrun_all.log + echo "julia exit code: $julia_exit" count=$(sed -n '/^Dry run:/,$p' dryrun_all.log | tail -n +2 | grep -c .) echo "Discovered $count test(s) with no selection" if [ "$count" -lt 10 ]; then @@ -65,7 +78,12 @@ jobs: - name: Dry run - directory selection (suite/data) shell: bash run: | - julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["suite/data", "--dryrun"])' | tee dryrun_data.log + set +e + julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["suite/data", "--dryrun"])' > dryrun_data.log 2>&1 + julia_exit=$? + set -e + cat dryrun_data.log + echo "julia exit code: $julia_exit" selected=$(sed -n '/^Dry run:/,$p' dryrun_data.log | tail -n +2 | grep .) count=$(echo "$selected" | grep -c .) echo "Discovered $count test(s) under suite/data" diff --git a/BREAKING.md b/BREAKING.md index bb17026b..cd0c1c27 100644 --- a/BREAKING.md +++ b/BREAKING.md @@ -3,6 +3,19 @@ This document outlines all breaking changes introduced in CTBase v0.18.0-beta compared to v0.17.4. Use this guide to migrate your code and understand the impact of these changes. +## Non-breaking note (0.28.7-beta) + +- **`TestRunner`: fixed a Windows-only bug where test selection silently + discovered 0 tests.** `_collect_test_files_recursive` now normalizes + relative test-file paths to forward slashes on every platform; previously + `relpath` returned backslash-separated paths on Windows, which never + matched the forward-slash glob patterns used throughout the extension + (including the default `"suite/*/test_*"` in `test/runtests.jl`), so + `Pkg.test` exited 0 having run nothing. **No breaking change**: purely an + internal path-normalization fix; the public `run_tests` API, its + arguments, and behavior on non-Windows platforms are unchanged. No + migration required. + ## Non-breaking note (0.28.6-beta) - **`Strategies`: option values whose type is a parametric struct are now diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ae0a58e..530f24d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,49 @@ All notable changes to CTBase will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.28.7-beta] - 2026-07-27 + +### ๐Ÿ› Bug Fixes + +#### **TestRunner** โ€” test selection silently discovered 0 tests on Windows + +- **`_collect_test_files_recursive`** (`ext/TestRunner/test_selection.jl`) now + normalizes relative test file paths to forward slashes (`/`) regardless of + platform. `relpath` returns OS-native separators, so on Windows it produced + backslash-separated paths (e.g. `suite\core\test_core.jl`). Every + glob-based selection in the extension โ€” including the default + `available_tests=("suite/*/test_*",)` used by `test/runtests.jl` โ€” is + written with forward slashes, so on Windows the glob never matched any + candidate file, and `Pkg.test` silently ran and reported **0 tests** with + exit code 0 instead of failing loudly. +- Fixes a false-green Windows CI run + (control-toolbox/CTBase.jl#512) where "CTBase tests" reported "None" and + passed. + +### ๐Ÿงช Testing + +- New unit test in `test_testrunner_selection.jl`: + `_collect_test_files_recursive` returns forward-slash paths for nested + directories, verified to contain no `\` characters. +- Fixed an existing unit test (`symbol resolution: shallowest match`) that + compared against `joinpath("a", "test_x.jl")` (OS-native separator) โ€” now + compares against the literal `"a/test_x.jl"`, matching the normalized + behavior. +- New CI-level regression guard: `.github/workflows/CI.yml` gained a + `test-selection-smoke` job (matrix: `ubuntu-latest`, `windows-latest`) + that dry-runs `Pkg.test` both with no selection and with a directory + selection (`suite/data`), asserting a non-zero, correctly-scoped test + count on both platforms. +- Re-enabled `macos-latest` and `ubuntu-latest` in the main CI matrix + alongside `windows-latest` (previously commented out). +- Full suite green: **4909/4909**. + +### โœ… Compatibility + +- **No breaking changes**: internal path-handling fix in the `TestRunner` + extension; no public API, type, or signature changes. See + [BREAKING.md](BREAKING.md). + ## [0.28.6-beta] - 2026-07-26 ### ๐Ÿ› Bug Fixes diff --git a/Project.toml b/Project.toml index c2b0e652..786d5d96 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "CTBase" uuid = "54762871-cc72-4466-b8e8-f6c8b58076cd" -version = "0.28.6-beta" +version = "0.28.7-beta" authors = ["Olivier Cots ", "Jean-Baptiste Caillau "] [deps] From 5004285666c4cea44774bf9ecddc0d9ef0a4b1e7 Mon Sep 17 00:00:00 2001 From: Olivier Cots Date: Mon, 27 Jul 2026 16:53:00 +0200 Subject: [PATCH 4/4] fix(ci): match only printed test paths in test-selection-smoke sed -n '/^Dry run:/,$p' captured everything to end of file, including Pkg's own trailing "Testing CTBase tests passed" status line printed after run_tests returns. That extra line inflated the "no selection" count by one (harmless there) but broke the "suite/data" prefix check, since it doesn't start with "suite/data/". Match "^suite/" directly instead, which only ever matches real test paths. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/CI.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9e3c4816..a0724ba6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -68,7 +68,10 @@ jobs: set -e cat dryrun_all.log echo "julia exit code: $julia_exit" - count=$(sed -n '/^Dry run:/,$p' dryrun_all.log | tail -n +2 | grep -c .) + # Match only printed test paths (they all live under "suite/"), not the + # sed-range-to-EOF approach, which also swept up Pkg's own trailing + # "Testing CTBase tests passed" status line printed after run_tests returns. + count=$(grep -c '^suite/' dryrun_all.log) echo "Discovered $count test(s) with no selection" if [ "$count" -lt 10 ]; then echo "::error::Expected at least 10 tests with no selection, got $count" @@ -84,7 +87,7 @@ jobs: set -e cat dryrun_data.log echo "julia exit code: $julia_exit" - selected=$(sed -n '/^Dry run:/,$p' dryrun_data.log | tail -n +2 | grep .) + selected=$(grep '^suite/' dryrun_data.log || true) count=$(echo "$selected" | grep -c .) echo "Discovered $count test(s) under suite/data" if [ "$count" -lt 1 ]; then