diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 347c725ac49..a255eea06fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,11 +72,12 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 10 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -104,11 +105,12 @@ jobs: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -248,23 +250,25 @@ jobs: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - components: cargo - enable-sccache: "true" + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + components: cargo + enable-sccache: "true" + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false - name: Pin rustup proxy to repository toolchain run: | TOOLCHAIN="$(grep '^channel' rust-toolchain.toml | cut -d '"' -f 2)" echo "RUSTUP_TOOLCHAIN=$TOOLCHAIN" >> "$GITHUB_ENV" - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - name: Pytest - PyVortex CUDA bridge run: | @@ -281,11 +285,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -307,9 +312,6 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) || 'ubuntu-latest' }} - env: - # disable lints for build, they will be caught in Rust lint job. - RUSTFLAGS: "-A warnings" strategy: fail-fast: false matrix: @@ -331,11 +333,12 @@ jobs: rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'" args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -347,6 +350,9 @@ jobs: run: | git status --porcelain test -z "$(git status --porcelain)" + - name: sccache stats + if: always() && github.repository == 'vortex-data/vortex' + run: sccache --show-stats check-min-deps: name: "Check build with minimal dependencies" @@ -356,11 +362,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -376,11 +383,12 @@ jobs: env: RUSTFLAGS: "-A warnings" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -396,24 +404,23 @@ jobs: && format('runs-on={0}/runner=amd64-ci-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - name: Install nightly for fmt run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt + # Run one Cargo command at a time so it can use the whole runner and reuse + # artifacts from the shared target directory in subsequent checks. - name: Rust Lint - Format id: fmt continue-on-error: true run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check - - name: Rustc check - id: check - continue-on-error: true - run: RUSTFLAGS="-D warnings" cargo check --profile ci --locked --all-features --all-targets - name: Rustc check (release) id: check-release continue-on-error: true @@ -433,7 +440,6 @@ jobs: script: | const failed = Object.entries({ fmt: '${{ steps.fmt.outcome }}', - check: '${{ steps.check.outcome }}', 'check-release': '${{ steps.check-release.outcome }}', 'clippy-all': '${{ steps.clippy-all.outcome }}', 'clippy-default': '${{ steps.clippy-default.outcome }}', @@ -453,20 +459,9 @@ jobs: } core.setFailed(`Lint failed: ${failed.join(', ')}`); - cpp-lint: - name: "C/C++ (lint)" - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: C/C++ Lint - clang-format - run: | - git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \ - | grep -E '\.(cpp|hpp|cu|cuh|h)$' \ - | grep -v 'arrow/reference/arrow_c_device\.h$' \ - | grep -v 'kernels/src/bit_unpack_.*\.cu$' \ - | grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \ - | xargs clang-format --dry-run --Werror --style=file + - name: sccache stats + if: always() && github.repository == 'vortex-data/vortex' + run: sccache --show-stats rust-lint-no-default: name: "Rust (lint, no default)" @@ -476,11 +471,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -489,6 +485,21 @@ jobs: run: | cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings + cpp-lint: + name: "C/C++ (lint)" + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: C/C++ Lint - clang-format + run: | + git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \ + | grep -E '\.(cpp|hpp|cu|cuh|h)$' \ + | grep -v 'arrow/reference/arrow_c_device\.h$' \ + | grep -v 'kernels/src/bit_unpack_.*\.cu$' \ + | grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \ + | xargs clang-format --dry-run --Werror --style=file + rust-test-windows: name: "Rust tests (windows-x64)" needs: duckdb-ready @@ -498,11 +509,12 @@ jobs: && format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id) || 'windows-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Setup (Windows) run: | echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH @@ -540,10 +552,11 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64 steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -570,11 +583,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -602,17 +616,19 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - - run: ./gradlew javadoc - working-directory: ./java - - run: ./gradlew check + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6 + - name: Build documentation and run Java checks + run: ./gradlew --parallel javadoc check working-directory: ./java license-check-and-audit-check: @@ -663,11 +679,12 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -709,11 +726,12 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -735,27 +753,29 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - - name: Build DuckDB tests - shell: bash - run: | - cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run - - name: Start MinIO - shell: bash - run: | - curl -sSL https://dl.min.io/server/minio/release/linux-amd64/minio -o "${RUNNER_TEMP}/minio" - chmod +x "${RUNNER_TEMP}/minio" - mkdir -p "${RUNNER_TEMP}/minio-data/vortex-test" - MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin \ - "${RUNNER_TEMP}/minio" server "${RUNNER_TEMP}/minio-data" --address 127.0.0.1:9000 & - until curl -sf http://127.0.0.1:9000/minio/health/live; do sleep 1; done + - parallel: + - name: Build DuckDB tests + shell: bash + run: | + cargo nextest run --cargo-profile ci --locked -p vortex-duckdb --no-run + - name: Start MinIO + shell: bash + run: | + curl -sSL https://dl.min.io/server/minio/release/linux-amd64/minio -o "${RUNNER_TEMP}/minio" + chmod +x "${RUNNER_TEMP}/minio" + mkdir -p "${RUNNER_TEMP}/minio-data/vortex-test" + MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin \ + "${RUNNER_TEMP}/minio" server "${RUNNER_TEMP}/minio-data" --address 127.0.0.1:9000 & + until curl -sf http://127.0.0.1:9000/minio/health/live; do sleep 1; done - name: Run tests shell: bash env: @@ -776,19 +796,21 @@ jobs: && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=wasm-integration', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - targets: "wasm32-wasip1" - - name: Setup Wasmer - uses: wasmerio/setup-wasmer@6c5d5cfde33555ebd3e0172ba547497890c4b1c0 # v3 - with: - version: "v7.4.0" + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + targets: "wasm32-wasip1" + - name: Setup Wasmer + uses: wasmerio/setup-wasmer@6c5d5cfde33555ebd3e0172ba547497890c4b1c0 # v3 + with: + version: "v7.4.0" - name: Verify Wasmer installation run: wasmer --version - run: cargo build --profile ci --target wasm32-wasip1 @@ -797,32 +819,31 @@ jobs: working-directory: ./wasm-test generated-files: - name: "Check generated source files are up to date" + name: "Generated source files and C API" runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=generated-files', github.run_id) || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - # check-editions compares against the merge base, so it needs real history. - fetch-depth: 0 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # check-editions compares against the merge base, so it needs real history. + fetch-depth: 0 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - name: Install nightly for cbindgen macro expansion run: rustup toolchain install $NIGHTLY_TOOLCHAIN - - name: "regenerate all .fbs/.proto Rust code" + - name: "regenerate Rust sources and edition records" run: | cargo run --profile ci -p xtask -- generate-fbs cargo run --profile ci -p xtask -- generate-proto - - name: "regenerate the edition records" - run: | cargo run --profile ci -p xtask -- generate-editions - name: "check frozen edition records never change" # Independent of the regeneration above: a stale record must not mask a frozen one @@ -831,9 +852,16 @@ jobs: run: | BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}" cargo run --profile ci -p xtask -- check-editions --base "$BASE" - - name: "regenerate FFI header file" + - name: "regenerate FFI header and test the C API" + env: + # Nightly regenerates cinclude/vortex.h with cbindgen during the build. + RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }} run: | - cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi + cmake -S vortex-ffi -B "${RUNNER_TEMP}/ffi-c-build" \ + -DVORTEX_CARGO_PROFILE=ci \ + -DVORTEX_BUILD_TESTS=ON + cmake --build "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" + ctest --test-dir "${RUNNER_TEMP}/ffi-c-build" --parallel "$(nproc)" --output-on-failure - name: "Make sure no files changed after regenerating" run: | git status --porcelain @@ -849,30 +877,3 @@ jobs: working-directory: vortex-flatbuffers/ run: | find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{} - - ffi-c-test: - name: "C API test build" - timeout-minutes: 10 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) - || 'ubuntu-latest' }} - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: "true" - - name: Build and run C API tests - env: - # Nightly regenerates cinclude/vortex.h with cbindgen during the build. - RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }} - run: | - cmake -S vortex-ffi -B vortex-ffi/build \ - -DVORTEX_CARGO_PROFILE=ci \ - -DVORTEX_BUILD_TESTS=ON - cmake --build vortex-ffi/build --parallel "$(nproc)" - ctest --test-dir vortex-ffi/build --output-on-failure diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index d8012193614..3e8746950a6 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -29,11 +29,12 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index c57598092fb..0db70b5e26f 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -56,10 +56,11 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-build steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -91,23 +92,25 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-tests steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "true" - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "true" + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false - name: Install nextest uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2 with: @@ -161,15 +164,16 @@ jobs: - sanitizer: initcheck runner_flags: "--tool initcheck --error-exitcode 1" steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -190,15 +194,16 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-test-cudf steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index bb10fc9fc82..45a1210a6ff 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -74,21 +74,23 @@ jobs: runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=check-java-publish-build-arm64 runs-on: ${{ matrix.runner }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: "corretto" - java-version: "17" - - uses: ./.github/actions/setup-prebuild - with: - enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + - parallel: + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 + with: + distribution: "corretto" + java-version: "17" + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install cargo-zigbuild uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: @@ -109,11 +111,12 @@ jobs: && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-publish-dry-run', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index 1128526d0a1..54fc5f60072 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -61,11 +61,12 @@ jobs: LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -126,11 +127,12 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -182,11 +184,12 @@ jobs: # unset -Zsanitizer flag and we should allow that. RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -245,11 +248,12 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -295,11 +299,12 @@ jobs: RUSTFLAGS: "-A warnings" RUST_BACKTRACE: full steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - parallel: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true"