diff --git a/.github/actions/setup-prebuild/action.yml b/.github/actions/setup-prebuild/action.yml index 9cfc837b680..b5213795e7e 100644 --- a/.github/actions/setup-prebuild/action.yml +++ b/.github/actions/setup-prebuild/action.yml @@ -39,7 +39,7 @@ runs: # Fallback path: full setup for forks - name: Full Rust setup - if: github.repository != 'vortex-data/vortex' + if: github.repository == 'massive-com/vortex' uses: ./.github/actions/setup-rust with: repo-token: ${{ inputs.repo-token }} @@ -47,3 +47,10 @@ runs: components: ${{ inputs.components }} targets: ${{ inputs.targets }} enable-sccache: ${{ inputs.enable-sccache }} + + # uv is preinstalled on the prebuilt images; install it for forks. + - name: Install uv + if: github.repository == 'massive-com/vortex' + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 4f42d38167a..867efd18e1e 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -80,6 +80,11 @@ runs: shell: bash run: sccache --start-server + - name: Install cargo tools (cargo-hack, cargo-nextest, cargo-minimal-versions) + uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2 + with: + tool: cargo-hack,cargo-nextest,cargo-minimal-versions + - name: Install Protoc (for lance-encoding build step) if: runner.os != 'Windows' uses: ./.github/actions/setup-protoc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54a3960ae28..523ccd14905 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,16 +108,23 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + # doxygen is preinstalled on the prebuilt images; install it for forks. + - name: Install Doxygen + if: github.repository == 'massive-com/vortex' + uses: ssciwr/doxygen-install@329d88f5a303066a5bd006db7516b1925b86350e # v2 - name: Pytest - Vortex run: | uv run --all-packages pytest --benchmark-disable -n auto test/ working-directory: vortex-python/ + # Benchmark tuning requires the self-hosted prebuilt runner; skip on forks. - name: Setup benchmark environment + if: github.repository == 'vortex-data/vortex' run: sudo bash scripts/setup-benchmark.sh - name: Pytest Benchmarks - Vortex + if: github.repository == 'vortex-data/vortex' run: | bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/ working-directory: vortex-python/ @@ -193,7 +200,8 @@ jobs: build-rust: name: "Rust build (${{matrix.config.name}})" - timeout-minutes: 30 + # Forks build without sccache, so give the uncached build more headroom. + timeout-minutes: ${{ github.repository == 'vortex-data/vortex' && 30 || 60 }} runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) @@ -533,6 +541,9 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + # flatc is preinstalled on the prebuilt images; install it for forks. + - uses: ./.github/actions/setup-flatc + if: github.repository == 'massive-com/vortex' - name: Install nightly for cbindgen macro expansion run: rustup toolchain install $NIGHTLY_TOOLCHAIN - name: "regenerate all .fbs/.proto Rust code" diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 2cdcbaa9dbf..89797a21136 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -24,6 +24,8 @@ env: jobs: bench-codspeed: + # CodSpeed requires an upstream account/token; skip on forks. + if: github.repository == 'vortex-data/vortex' strategy: matrix: include: diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index 49f539a19e4..ce00716cf17 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -24,6 +24,9 @@ env: jobs: python-wheel-build: name: "Python (wheel build)" + # Publish validation is upstream-only (git deps are unpublishable, and some + # jobs require the upstream self-hosted runners); skip on forks. + if: github.repository == 'vortex-data/vortex' runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -64,6 +67,7 @@ jobs: working-directory: vortex-python/ check-java-publish-build: + if: github.repository == 'vortex-data/vortex' timeout-minutes: 30 strategy: fail-fast: false @@ -96,12 +100,14 @@ jobs: compat-check: name: "Compat check" + if: github.repository == 'vortex-data/vortex' uses: ./.github/workflows/compat-validation.yml with: mode: last rust-publish-dry-run: name: "Rust publish dry-run" + if: github.repository == 'vortex-data/vortex' timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index bf45fc7be13..2e43266784f 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -43,6 +43,10 @@ jobs: rust-coverage: name: "Rust tests (coverage) (${{ matrix.suite }})" + # Coverage instruments and runs the whole test suite, which OOMs the + # GitHub-hosted runners forks use (upstream runs it on larger self-hosted + # runners); it also uploads to Codecov via OIDC that forks lack. Skip on forks. + if: github.repository == 'vortex-data/vortex' needs: duckdb-ready timeout-minutes: 30 permissions: @@ -130,7 +134,8 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 30 + # Forks build instrumented without sccache; give the uncached run more headroom. + timeout-minutes: ${{ github.repository == 'vortex-data/vortex' && 30 || 90 }} env: ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" LSAN_OPTIONS: "report_objects=1" @@ -154,6 +159,27 @@ jobs: run: | rustup toolchain install $NIGHTLY_TOOLCHAIN rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview + # The prebuilt image ships /usr/bin/llvm-symbolizer; GitHub-hosted runners + # do not. Without it TSan/ASan cannot symbolize frames, so the + # tsan_suppressions.txt entries (benign crossbeam/oneshot fence races) + # never match and fail the job. Symlink an available llvm-symbolizer. + - name: Ensure llvm-symbolizer (forks) + if: github.repository == 'massive-com/vortex' + shell: bash + run: | + if [ ! -x /usr/bin/llvm-symbolizer ]; then + sym="$(command -v llvm-symbolizer \ + || ls -1 /usr/bin/llvm-symbolizer-* 2>/dev/null | sort -V | tail -1 \ + || ls -1 /usr/lib/llvm-*/bin/llvm-symbolizer 2>/dev/null | sort -V | tail -1)" + if [ -z "$sym" ]; then + sudo apt-get update + sudo apt-get install -y llvm + sym="$(ls -1 /usr/bin/llvm-symbolizer-* 2>/dev/null | sort -V | tail -1 \ + || ls -1 /usr/lib/llvm-*/bin/llvm-symbolizer 2>/dev/null | sort -V | tail -1)" + fi + sudo ln -sf "$sym" /usr/bin/llvm-symbolizer + fi + /usr/bin/llvm-symbolizer --version - name: Build tests with sanitizer run: | RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ diff --git a/Cargo.lock b/Cargo.lock index 0a84c673f61..1dbfe5d9eb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1443,7 +1443,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width 0.2.2", + "unicode-width 0.1.14", ] [[package]] @@ -1545,7 +1545,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1999,9 +1999,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" +checksum = "201f2332bd98022973bbf50c87f2d2f8151200f43e640da370901b20f1bea9d3" dependencies = [ "cc", "cxx-build", @@ -2014,9 +2014,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" +checksum = "02da7762e3807681f61c4561f34e1ff791417334294b225dbe22236459c9deef" dependencies = [ "cc", "codespan-reporting", @@ -2029,9 +2029,9 @@ dependencies = [ [[package]] name = "cxxbridge-cmd" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" +checksum = "21c216b8cd7c26c2798bbecdb13de2f8dc65239b9ed892756deacff10fc0fa64" dependencies = [ "clap", "codespan-reporting", @@ -2043,15 +2043,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" +checksum = "61c618f09812e388d1d065fd43fdc4340fade506ab2e0903397bf238650fce45" [[package]] name = "cxxbridge-macro" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +checksum = "415b113deba00cc605302b9b0b4974ab3d57c41135d81737cd47b89dbaa17c38" dependencies = [ "indexmap 2.14.0", "proc-macro2", @@ -3577,7 +3577,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3763,7 +3763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4944,7 +4944,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6317,7 +6317,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8094,7 +8094,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8152,7 +8152,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9018,7 +9018,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9037,7 +9037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11068,7 +11068,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3293c5b1a54..ca3694b4c38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,7 +137,9 @@ cudarc = { version = "0.19.0", features = [ ] } custom-labels = "0.4.4" dashmap = "6.1.0" -datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949", default-features = false, features = ["sql"] } +datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949", default-features = false, features = [ + "sql", +] } datafusion-catalog = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949" } datafusion-common = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949" } datafusion-common-runtime = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949" } diff --git a/deny.toml b/deny.toml index b9f9c79a796..3a38fd19fec 100644 --- a/deny.toml +++ b/deny.toml @@ -16,7 +16,10 @@ ignore = [ # Paste is no longer maintained because its essentially "finished". "RUSTSEC-2024-0436", # proc-macro-error-2 is unmaintained, only used by the `test_with` test dependency - "RUSTSEC-2026-0173" + "RUSTSEC-2026-0173", + # quick-xml is a transitive dependency that we cannot upgrade ourselves + "RUSTSEC-2026-0195", + "RUSTSEC-2026-0194" ] [licenses] diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index 95b3354310d..ffcab4ea884 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -265,7 +265,7 @@ impl ArrayRef { .optimize() } - /// Wraps the array in a [`ReversedArray`](crate::arrays::ReversedArray) such that it is + /// Wraps the array in a [`ReversedArray`] such that it is /// logically yielded in reverse element order. The optimizer is run immediately, which may /// eliminate the wrapper (for example, for `Reversed(Reversed(x))` or `Reversed(Dict(...))`). pub fn reverse(&self) -> VortexResult { diff --git a/vortex-array/src/arrays/patched/mod.rs b/vortex-array/src/arrays/patched/mod.rs index 0d4a4dbdb5a..a88628e5587 100644 --- a/vortex-array/src/arrays/patched/mod.rs +++ b/vortex-array/src/arrays/patched/mod.rs @@ -43,7 +43,7 @@ //! `indices` and `values` are aligned and accessed together. //! //! ```text -//! +//! //! chunk 0 chunk 0 chunk 0 chunk 0 chunk 0 chunk 0 //! lane 0 lane 1 lane 2 lane 3 lane 4 lane 5 //! ┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┐ diff --git a/vortex-array/src/arrays/reversed/mod.rs b/vortex-array/src/arrays/reversed/mod.rs index b85c06705c7..36c565bc322 100644 --- a/vortex-array/src/arrays/reversed/mod.rs +++ b/vortex-array/src/arrays/reversed/mod.rs @@ -15,7 +15,7 @@ //! dictionary-encoded, so the per-chunk reversal cost is O(n_codes) rather than O(n_rows). //! //! For encodings that have no reduce rule the `ReversedArray` wrapper survives to -//! decode time, where [`execute.rs`](self::execute) reverses the canonical form +//! decode time, where `execute.rs` reverses the canonical form //! directly: //! //! * `Primitive`: iterates the typed buffer backwards — O(n), fully sequential. diff --git a/vortex-array/src/arrays/reversed/vtable.rs b/vortex-array/src/arrays/reversed/vtable.rs index f2627e45fba..f6e98af3444 100644 --- a/vortex-array/src/arrays/reversed/vtable.rs +++ b/vortex-array/src/arrays/reversed/vtable.rs @@ -55,13 +55,9 @@ impl VTable for Reversed { len: usize, slots: &[Option], ) -> VortexResult<()> { - vortex_ensure!( - slots[CHILD_SLOT].is_some(), - "ReversedArray child slot must be present" - ); - let child = slots[CHILD_SLOT] - .as_ref() - .expect("validated child slot presence"); + let Some(child) = slots[CHILD_SLOT].as_ref() else { + vortex_bail!("ReversedArray child slot must be present"); + }; vortex_ensure!( child.dtype() == dtype, "ReversedArray dtype {} does not match child dtype {}", diff --git a/vortex-ffi/src/sink.rs b/vortex-ffi/src/sink.rs index 639d22196e3..20b9fb0167f 100644 --- a/vortex-ffi/src/sink.rs +++ b/vortex-ffi/src/sink.rs @@ -242,6 +242,12 @@ mod tests { let vx_array_ptr = vx_array::new(Arc::new(array.into_array())); vx_array_sink_push(sink, vx_array_ptr, &raw mut error); vx_array_free(vx_array_ptr); + // Free any push error before reusing `error` for close, otherwise + // close overwrites the pointer and the push error leaks. + if !error.is_null() { + vx_error_free(error); + error = std::ptr::null_mut(); + } // Close should fail due to invalid path vx_array_sink_close(sink, &raw mut error); diff --git a/vortex-layout/src/scan/tasks.rs b/vortex-layout/src/scan/tasks.rs index fe6e02b6dfa..05c868067e1 100644 --- a/vortex-layout/src/scan/tasks.rs +++ b/vortex-layout/src/scan/tasks.rs @@ -255,7 +255,7 @@ mod tests { let remaining = Arc::new(AtomicU64::new(LIMIT)); let handles: Vec<_> = (0..THREADS) .map(|_| { - let remaining = remaining.clone(); + let remaining = Arc::clone(&remaining); thread::spawn(move || { let mut total: usize = 0; while total < PER_THREAD_REQUEST { diff --git a/vortex-layout/src/segments/cache.rs b/vortex-layout/src/segments/cache.rs index a26701959a7..64808180047 100644 --- a/vortex-layout/src/segments/cache.rs +++ b/vortex-layout/src/segments/cache.rs @@ -209,7 +209,7 @@ impl SegmentCacheSourceAdapter { impl SegmentSource for SegmentCacheSourceAdapter { fn request(&self, id: SegmentId) -> SegmentFuture { - let cache = self.cache.clone(); + let cache = Arc::clone(&self.cache); let delegate = self.source.request(id); async move {