Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .github/actions/setup-prebuild/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,18 @@ runs:

# Fallback path: full setup for forks
- name: Full Rust setup
if: github.repository != 'vortex-data/vortex'
if: github.repository == 'massive-com/vortex'
uses: ./.github/actions/setup-rust
with:
repo-token: ${{ inputs.repo-token }}
toolchain: ${{ inputs.toolchain }}
components: ${{ inputs.components }}
targets: ${{ inputs.targets }}
enable-sccache: ${{ inputs.enable-sccache }}

# uv is preinstalled on the prebuilt images; install it for forks.
- name: Install uv
if: github.repository == 'massive-com/vortex'
uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false
5 changes: 5 additions & 0 deletions .github/actions/setup-rust/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ runs:
shell: bash
run: sccache --start-server

- name: Install cargo tools (cargo-hack, cargo-nextest, cargo-minimal-versions)
uses: taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2
with:
tool: cargo-hack,cargo-nextest,cargo-minimal-versions

- name: Install Protoc (for lance-encoding build step)
if: runner.os != 'Windows'
uses: ./.github/actions/setup-protoc
13 changes: 12 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,16 +108,23 @@ jobs:
sccache: s3
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: ./.github/actions/setup-prebuild
# doxygen is preinstalled on the prebuilt images; install it for forks.
- name: Install Doxygen
if: github.repository == 'massive-com/vortex'
uses: ssciwr/doxygen-install@329d88f5a303066a5bd006db7516b1925b86350e # v2

- name: Pytest - Vortex
run: |
uv run --all-packages pytest --benchmark-disable -n auto test/
working-directory: vortex-python/

# Benchmark tuning requires the self-hosted prebuilt runner; skip on forks.
- name: Setup benchmark environment
if: github.repository == 'vortex-data/vortex'
run: sudo bash scripts/setup-benchmark.sh

- name: Pytest Benchmarks - Vortex
if: github.repository == 'vortex-data/vortex'
run: |
bash ../scripts/bench-taskset.sh uv run --all-packages pytest --benchmark-only benchmark/
working-directory: vortex-python/
Expand Down Expand Up @@ -193,7 +200,8 @@ jobs:

build-rust:
name: "Rust build (${{matrix.config.name}})"
timeout-minutes: 30
# Forks build without sccache, so give the uncached build more headroom.
timeout-minutes: ${{ github.repository == 'vortex-data/vortex' && 30 || 60 }}
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name)
Expand Down Expand Up @@ -533,6 +541,9 @@ jobs:
sccache: s3
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- uses: ./.github/actions/setup-prebuild
# flatc is preinstalled on the prebuilt images; install it for forks.
- uses: ./.github/actions/setup-flatc
if: github.repository == 'massive-com/vortex'
- name: Install nightly for cbindgen macro expansion
run: rustup toolchain install $NIGHTLY_TOOLCHAIN
- name: "regenerate all .fbs/.proto Rust code"
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/codspeed.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ env:

jobs:
bench-codspeed:
# CodSpeed requires an upstream account/token; skip on forks.
if: github.repository == 'vortex-data/vortex'
strategy:
matrix:
include:
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/publish-dry-runs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ env:
jobs:
python-wheel-build:
name: "Python (wheel build)"
# Publish validation is upstream-only (git deps are unpublishable, and some
# jobs require the upstream self-hosted runners); skip on forks.
if: github.repository == 'vortex-data/vortex'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
Expand Down Expand Up @@ -64,6 +67,7 @@ jobs:
working-directory: vortex-python/

check-java-publish-build:
if: github.repository == 'vortex-data/vortex'
timeout-minutes: 30
strategy:
fail-fast: false
Expand Down Expand Up @@ -96,12 +100,14 @@ jobs:

compat-check:
name: "Compat check"
if: github.repository == 'vortex-data/vortex'
uses: ./.github/workflows/compat-validation.yml
with:
mode: last

rust-publish-dry-run:
name: "Rust publish dry-run"
if: github.repository == 'vortex-data/vortex'
timeout-minutes: 30
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
Expand Down
28 changes: 27 additions & 1 deletion .github/workflows/rust-instrumented.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ jobs:

rust-coverage:
name: "Rust tests (coverage) (${{ matrix.suite }})"
# Coverage instruments and runs the whole test suite, which OOMs the
# GitHub-hosted runners forks use (upstream runs it on larger self-hosted
# runners); it also uploads to Codecov via OIDC that forks lack. Skip on forks.
if: github.repository == 'vortex-data/vortex'
needs: duckdb-ready
timeout-minutes: 30
permissions:
Expand Down Expand Up @@ -130,7 +134,8 @@ jobs:
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 30
# Forks build instrumented without sccache; give the uncached run more headroom.
timeout-minutes: ${{ github.repository == 'vortex-data/vortex' && 30 || 90 }}
env:
ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1"
LSAN_OPTIONS: "report_objects=1"
Expand All @@ -154,6 +159,27 @@ jobs:
run: |
rustup toolchain install $NIGHTLY_TOOLCHAIN
rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview
# The prebuilt image ships /usr/bin/llvm-symbolizer; GitHub-hosted runners
# do not. Without it TSan/ASan cannot symbolize frames, so the
# tsan_suppressions.txt entries (benign crossbeam/oneshot fence races)
# never match and fail the job. Symlink an available llvm-symbolizer.
- name: Ensure llvm-symbolizer (forks)
if: github.repository == 'massive-com/vortex'
shell: bash
run: |
if [ ! -x /usr/bin/llvm-symbolizer ]; then
sym="$(command -v llvm-symbolizer \
|| ls -1 /usr/bin/llvm-symbolizer-* 2>/dev/null | sort -V | tail -1 \
|| ls -1 /usr/lib/llvm-*/bin/llvm-symbolizer 2>/dev/null | sort -V | tail -1)"
if [ -z "$sym" ]; then
sudo apt-get update
sudo apt-get install -y llvm
sym="$(ls -1 /usr/bin/llvm-symbolizer-* 2>/dev/null | sort -V | tail -1 \
|| ls -1 /usr/lib/llvm-*/bin/llvm-symbolizer 2>/dev/null | sort -V | tail -1)"
fi
sudo ln -sf "$sym" /usr/bin/llvm-symbolizer
fi
/usr/bin/llvm-symbolizer --version
- name: Build tests with sanitizer
run: |
RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \
Expand Down
46 changes: 23 additions & 23 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,9 @@ cudarc = { version = "0.19.0", features = [
] }
custom-labels = "0.4.4"
dashmap = "6.1.0"
datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949", default-features = false, features = ["sql"] }
datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949", default-features = false, features = [
"sql",
] }
datafusion-catalog = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949" }
datafusion-common = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949" }
datafusion-common-runtime = { git = "https://github.com/massive-com/arrow-datafusion", rev = "6363fbeb2a593e78fb6d73ad07ddba13c0406949" }
Expand Down
5 changes: 4 additions & 1 deletion deny.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ ignore = [
# Paste is no longer maintained because its essentially "finished".
"RUSTSEC-2024-0436",
# proc-macro-error-2 is unmaintained, only used by the `test_with` test dependency
"RUSTSEC-2026-0173"
"RUSTSEC-2026-0173",
# quick-xml is a transitive dependency that we cannot upgrade ourselves
"RUSTSEC-2026-0195",
"RUSTSEC-2026-0194"
]

[licenses]
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/array/erased.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ impl ArrayRef {
.optimize()
}

/// Wraps the array in a [`ReversedArray`](crate::arrays::ReversedArray) such that it is
/// Wraps the array in a [`ReversedArray`] such that it is
/// logically yielded in reverse element order. The optimizer is run immediately, which may
/// eliminate the wrapper (for example, for `Reversed(Reversed(x))` or `Reversed(Dict(...))`).
pub fn reverse(&self) -> VortexResult<ArrayRef> {
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/patched/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
//! `indices` and `values` are aligned and accessed together.
//!
//! ```text
//!
//!
//! chunk 0 chunk 0 chunk 0 chunk 0 chunk 0 chunk 0
//! lane 0 lane 1 lane 2 lane 3 lane 4 lane 5
//! ┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┐
Expand Down
2 changes: 1 addition & 1 deletion vortex-array/src/arrays/reversed/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
//! dictionary-encoded, so the per-chunk reversal cost is O(n_codes) rather than O(n_rows).
//!
//! For encodings that have no reduce rule the `ReversedArray` wrapper survives to
//! decode time, where [`execute.rs`](self::execute) reverses the canonical form
//! decode time, where `execute.rs` reverses the canonical form
//! directly:
//!
//! * `Primitive`: iterates the typed buffer backwards — O(n), fully sequential.
Expand Down
Loading
Loading