Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 75 additions & 2 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,80 @@ jobs:
(github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run ci'))
uses: control-toolbox/CTActions/.github/workflows/ci.yml@main
with:
runs_on: '["ubuntu-latest", "macos-latest"]'
runs_on: '["ubuntu-latest", "macos-latest", "windows-latest"]'
use_ct_registry: true
secrets:
SSH_KEY: ${{ secrets.SSH_KEY }}
SSH_KEY: ${{ secrets.SSH_KEY }}

# Smoke check for the TestRunner extension's test-selection logic (glob
# matching over `_collect_test_files_recursive`). This exists independently
# of the `call` job above because it needs to pass custom `test_args` to
# `Pkg.test`, which the reusable CTActions ci.yml workflow doesn't expose.
# It caught a regression where Windows returned backslash-separated relative
# paths, silently breaking forward-slash glob patterns and making every
# selection (including the default "run everything") resolve to zero tests.
test-selection-smoke:
if: >
github.event_name != 'pull_request' ||
(github.event.action == 'labeled' && github.event.label.name == 'run ci') ||
(github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'run ci'))
strategy:
fail-fast: false
matrix:
runs_on: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.runs_on }}
steps:
- uses: actions/checkout@v6
- uses: julia-actions/setup-julia@latest
with:
version: '1.12'
- uses: julia-actions/cache@v2
- uses: julia-actions/julia-buildpkg@v1

# Julia's own process exit code is not trustworthy here: on Windows,
# after a cold/fresh precompile, `Pkg.test` can print its "tests
# passed" success message and still return a nonzero process exit
# code from unrelated teardown (no exception, no stderr output).
# Piping through `tee` would let bash's pipefail abort the step on
# that spurious code before the dry-run output is even inspected, so
# redirect to a file instead and judge success from the *content* of
# the dry-run listing, not from julia's raw exit status.
- name: Dry run - no selection (all tests)
shell: bash
run: |
set +e
julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["--dryrun"])' > dryrun_all.log 2>&1
julia_exit=$?
set -e
cat dryrun_all.log
echo "julia exit code: $julia_exit"
# Match only printed test paths (they all live under "suite/"), not the
# sed-range-to-EOF approach, which also swept up Pkg's own trailing
# "Testing CTBase tests passed" status line printed after run_tests returns.
count=$(grep -c '^suite/' dryrun_all.log)
echo "Discovered $count test(s) with no selection"
if [ "$count" -lt 10 ]; then
echo "::error::Expected at least 10 tests with no selection, got $count"
exit 1
fi

- name: Dry run - directory selection (suite/data)
shell: bash
run: |
set +e
julia --project=. -e 'using Pkg; Pkg.test("CTBase"; test_args=["suite/data", "--dryrun"])' > dryrun_data.log 2>&1
julia_exit=$?
set -e
cat dryrun_data.log
echo "julia exit code: $julia_exit"
selected=$(grep '^suite/' dryrun_data.log || true)
count=$(echo "$selected" | grep -c .)
echo "Discovered $count test(s) under suite/data"
if [ "$count" -lt 1 ]; then
echo "::error::Expected at least 1 test under suite/data, got $count"
exit 1
fi
if echo "$selected" | grep -qv '^suite/data/'; then
echo "::error::Selection 'suite/data' returned tests outside suite/data"
exit 1
fi
13 changes: 13 additions & 0 deletions BREAKING.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,19 @@

This document outlines all breaking changes introduced in CTBase v0.18.0-beta compared to v0.17.4. Use this guide to migrate your code and understand the impact of these changes.

## Non-breaking note (0.28.7-beta)

- **`TestRunner`: fixed a Windows-only bug where test selection silently
discovered 0 tests.** `_collect_test_files_recursive` now normalizes
relative test-file paths to forward slashes on every platform; previously
`relpath` returned backslash-separated paths on Windows, which never
matched the forward-slash glob patterns used throughout the extension
(including the default `"suite/*/test_*"` in `test/runtests.jl`), so
`Pkg.test` exited 0 having run nothing. **No breaking change**: purely an
internal path-normalization fix; the public `run_tests` API, its
arguments, and behavior on non-Windows platforms are unchanged. No
migration required.

## Non-breaking note (0.28.6-beta)

- **`Strategies`: option values whose type is a parametric struct are now
Expand Down
43 changes: 43 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,49 @@ All notable changes to CTBase will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [0.28.7-beta] - 2026-07-27

### 🐛 Bug Fixes

#### **TestRunner** — test selection silently discovered 0 tests on Windows

- **`_collect_test_files_recursive`** (`ext/TestRunner/test_selection.jl`) now
normalizes relative test file paths to forward slashes (`/`) regardless of
platform. `relpath` returns OS-native separators, so on Windows it produced
backslash-separated paths (e.g. `suite\core\test_core.jl`). Every
glob-based selection in the extension — including the default
`available_tests=("suite/*/test_*",)` used by `test/runtests.jl` — is
written with forward slashes, so on Windows the glob never matched any
candidate file, and `Pkg.test` silently ran and reported **0 tests** with
exit code 0 instead of failing loudly.
- Fixes a false-green Windows CI run
(control-toolbox/CTBase.jl#512) where "CTBase tests" reported "None" and
passed.

### 🧪 Testing

- New unit test in `test_testrunner_selection.jl`:
`_collect_test_files_recursive` returns forward-slash paths for nested
directories, verified to contain no `\` characters.
- Fixed an existing unit test (`symbol resolution: shallowest match`) that
compared against `joinpath("a", "test_x.jl")` (OS-native separator) — now
compares against the literal `"a/test_x.jl"`, matching the normalized
behavior.
- New CI-level regression guard: `.github/workflows/CI.yml` gained a
`test-selection-smoke` job (matrix: `ubuntu-latest`, `windows-latest`)
that dry-runs `Pkg.test` both with no selection and with a directory
selection (`suite/data`), asserting a non-zero, correctly-scoped test
count on both platforms.
- Re-enabled `macos-latest` and `ubuntu-latest` in the main CI matrix
alongside `windows-latest` (previously commented out).
- Full suite green: **4909/4909**.

### ✅ Compatibility

- **No breaking changes**: internal path-handling fix in the `TestRunner`
extension; no public API, type, or signature changes. See
[BREAKING.md](BREAKING.md).

## [0.28.6-beta] - 2026-07-26

### 🐛 Bug Fixes
Expand Down
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "CTBase"
uuid = "54762871-cc72-4466-b8e8-f6c8b58076cd"
version = "0.28.6-beta"
version = "0.28.7-beta"
authors = ["Olivier Cots <olivier.cots@irit.fr>", "Jean-Baptiste Caillau <caillau@univ-cotedazur.fr>"]

[deps]
Expand Down
3 changes: 2 additions & 1 deletion ext/TestRunner/test_selection.jl
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ function _collect_test_files_recursive(test_dir::AbstractString)
for f in fs
if endswith(f, ".jl") && f != "runtests.jl"
full = joinpath(root, f)
push!(files, relpath(full, test_dir))
rel = relpath(full, test_dir)
push!(files, replace(rel, '\\' => '/'))
end
end
end
Expand Down
18 changes: 17 additions & 1 deletion test/suite/extensions/test_testrunner_selection.jl
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,27 @@ function test_testrunner_selection()
touch(joinpath(temp_dir, "a", "b", "test_x.jl"))

rel = find_symbol_file(:x, n -> "test_" * String(n); test_dir=temp_dir)
Test.@test rel == joinpath("a", "test_x.jl")
Test.@test rel == "a/test_x.jl"
end
end
end

Test.@testset verbose = VERBOSE showtiming = SHOWTIMING "_collect_test_files_recursive uses forward slashes" begin
collect_files = TestRunner._collect_test_files_recursive

# Regression: on Windows, `relpath` returns backslash-separated paths,
# which broke glob matching against forward-slash patterns like
# "suite/*/test_*" (available_tests filters always end up empty).
mktempdir() do temp_dir
mkpath(joinpath(temp_dir, "suite", "core"))
touch(joinpath(temp_dir, "suite", "core", "test_core.jl"))

files = collect_files(temp_dir)
Test.@test files == ["suite/core/test_core.jl"]
Test.@test all(f -> !occursin('\\', f), files)
end
end

Test.@testset verbose = VERBOSE showtiming = SHOWTIMING "order preservation" begin
parse_args = TestRunner._parse_test_args
select_tests = TestRunner._select_tests
Expand Down
Loading