rust.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Rust

on:
  # always trigger
  push:
  pull_request:

jobs:

  # build the library, a compilation step used by multiple steps below
  linux-build-lib:
    name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2022-03-08]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          # these represent dependencies downloaded by cargo
          # and thus do not depend on the OS, arch nor rust version.
          path: /github/home/.cargo
          key: cargo-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          # these represent compiled steps of both dependencies and arrow
          # and thus are specific for a particular OS, arch and rust version.
          path: /github/home/target
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}-
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt

      - name: Build Workspace
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cargo build

  # test the crate
  linux-test:
    name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2022-03-08]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
        ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
        PARQUET_TEST_DATA: /__w/arrow-rs/arrow-rs/parquet-testing/data
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run tests
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"

          # run tests on all workspace members with default feature list
          cargo test

          # Switch to arrow crate
          cd arrow
          # re-run tests on arrow crate with additional features
          cargo test --features=prettyprint
          # run test on arrow crate with minimal set of features
          cargo test --no-default-features
          cargo run --example builders
          cargo run --example dynamic_types
          cargo run --example read_csv
          cargo run --example read_csv_infer_schema
          cargo check --no-default-features

          # Switch to parquet crate
          cd ../parquet
          # re-run tests on parquet crate with async feature enabled
          cargo test --features=async
          cargo check --no-default-features

          # Switch to arrow-flight
          cd ../arrow-flight
          cargo test --features=flight-sql-experimental
          cargo check --no-default-features

  # test the --features "simd" of the arrow crate. This requires nightly.
  linux-test-simd:
    name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
        ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          key: cargo-nightly-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          key: ${{ runner.os }}-${{ matrix.arch }}-target-nightly-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run tests
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cd arrow
          cargo test --features "simd"
      - name: Check new project build with simd features
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cd arrow/test/dependency/simd
          cargo check

  windows-and-macos:
    name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, macos-latest]
        rust: [nightly-2022-03-08]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      # TODO: this won't cache anything, which is expensive. Setup this action
      # with a OS-dependent path.
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run tests
        shell: bash
        run: |
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          # do not produce debug symbols to keep memory usage down
          export RUSTFLAGS="-C debuginfo=0"
          cargo test

  clippy:
    name: Clippy
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2022-03-08]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt clippy
      - name: Run clippy
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cargo clippy --features test_common --features prettyprint  --features=async --all-targets --workspace -- -D warnings

  check_benches:
    name: Check Benchmarks (but don't run them)
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2022-03-08]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt clippy
      - name: Check benchmarks
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cargo check --benches --workspace

  lint:
    name: Lint (cargo fmt)
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v2
      - name: Setup toolchain
        run: |
          rustup toolchain install nightly-2022-03-08
          rustup default nightly-2022-03-08
          rustup component add rustfmt
      - name: Run
        run: cargo fmt --all -- --check

  coverage:
    name: Coverage
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [stable]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt clippy
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /home/runner/.cargo
          # this key is not equal because the user is different than on a container (runner vs github)
          key: cargo-coverage-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /home/runner/target
          # this key is not equal because coverage uses different compilation flags.
          key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache3-${{ matrix.rust }}-
      - name: Run coverage
        run: |
          export CARGO_HOME="/home/runner/.cargo"
          export CARGO_TARGET_DIR="/home/runner/target"

          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data

          rustup toolchain install stable
          rustup default stable
          cargo install --version 0.18.2 cargo-tarpaulin
          cargo tarpaulin --all --out Xml
      - name: Report coverage
        continue-on-error: true
        run: bash <(curl -s https://codecov.io/bash)

  # test the arrow crate builds against wasm32 in stable rust
  wasm32-build:
    name: Build wasm32 on AMD64 Rust ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
        ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
        PARQUET_TEST_DATA: /__w/arrow/arrow/parquet-testing/data
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          key: cargo-wasm32-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup override set ${{ matrix.rust }}
          rustup component add rustfmt
          rustup target add wasm32-unknown-unknown
          rustup target add wasm32-wasi
      - name: Build arrow crate
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cd arrow
          cargo build --no-default-features --features=csv,ipc,simd --target wasm32-unknown-unknown
          cargo build --no-default-features --features=csv,ipc,simd --target wasm32-wasi

  # test doc links still work
  docs:
    name: Docs are clean on AMD64 Rust ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Install python dev
        run: |
          apt update
          apt install -y libpython3.9-dev
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          key: cargo-nightly-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          key: ${{ runner.os }}-${{ matrix.arch }}-target-nightly-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run cargo doc
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          export RUSTDOCFLAGS="-Dwarnings"
          cargo doc --document-private-items --no-deps --workspace --all-features


  # test builds with various feature flag combinations outside the main workspace
  default-build:
    name: Feature Flag Builds ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2022-03-08]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable debug symbol generation to speed up CI build and keep memory down
        RUSTFLAGS: "-C debuginfo=0"
    steps:
      - uses: actions/checkout@v2
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache3-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache3-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup override set ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Arrow Build with default features
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cd arrow/test/dependency/default-features
          cargo check
      - name: Arrow Build with default-features=false
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cd arrow/test/dependency/no-default-features
          cargo check
      - name: Parquet Derive build with default-features
        run: |
          export CARGO_HOME="/github/home/.cargo"
          export CARGO_TARGET_DIR="/github/home/target"
          cd parquet_derive/test/dependency/default-features
          cargo check