rust.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Rust

on:
  # always trigger
  push:
  pull_request:

jobs:
  # build the library, a compilation step used by multiple steps below
  linux-build-lib:
    name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2024-01-29]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          # these represent dependencies downloaded by cargo
          # and thus do not depend on the OS, arch nor rust version.
          path: /github/home/.cargo
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          # these represent compiled steps of both dependencies and arrow
          # and thus are specific for a particular OS, arch and rust version.
          path: /github/home/target
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}-
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Build Workspace
        run: |
          cargo build
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"

  # test the crate
  linux-test:
    name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
    needs: [linux-build-lib]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        arch: [amd64]
        rust: [nightly-2024-01-29]
    container:
      image: ${{ matrix.arch }}/rust
      env:
        # Disable full debug symbol generation to speed up CI build and keep memory down
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      - name: Cache Cargo
        uses: actions/cache@v2
        with:
          path: /github/home/.cargo
          # this key equals the ones on `linux-build-lib` for re-use
          key: cargo-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v2
        with:
          path: /github/home/target
          # this key equals the ones on `linux-build-lib` for re-use
          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run tests
        run: |
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          # run tests on all workspace members with default feature list
          cargo test
          # test datafusion examples
          #cd datafusion-examples
          #cargo test --no-default-features
          #cargo run --example csv_sql
          #cargo run --example parquet_sql
        env:
          CARGO_HOME: "/github/home/.cargo"
          CARGO_TARGET_DIR: "/github/home/target"
      # Ballista is currently not part of the main workspace so requires a separate test step
#      - name: Run Ballista tests
#        run: |
#          export ARROW_TEST_DATA=$(pwd)/testing/data
#          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
#          cd ballista/rust
#          # snmalloc requires cmake so build without default features
#          cargo test --no-default-features --features sled
#        env:
#          CARGO_HOME: "/github/home/.cargo"
#          CARGO_TARGET_DIR: "/github/home/target"

#  integration-test:
#    name: "Integration Test"
#    needs: [linux-build-lib]
#    runs-on: ubuntu-latest
#    services:
#      postgres:
#        image: postgres:13
#        env:
#          POSTGRES_PASSWORD: postgres
#          POSTGRES_DB: db_test
#        ports:
#          - 5432/tcp
#        options: >-
#          --health-cmd pg_isready
#          --health-interval 10s
#          --health-timeout 5s
#          --health-retries 5
#    steps:
#      - uses: actions/checkout@v2
#        with:
#          submodules: true
#      - uses: actions/setup-python@v2
#        with:
#          python-version: "3.8"
#      - name: Install Python dependencies
#        run: |
#          python -m pip install --upgrade pip setuptools wheel
#          python -m pip install -r integration-tests/requirements.txt
#      - name: Allow access of psql
#        run: |
#          # make sure psql can access the server
#          echo "$POSTGRES_HOST:$POSTGRES_PORT:$POSTGRES_DB:$POSTGRES_USER:$POSTGRES_PASSWORD" | tee ~/.pgpass
#          chmod 0600 ~/.pgpass
#          psql -d "$POSTGRES_DB" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -c 'CREATE TABLE IF NOT EXISTS test (
#            c1 character varying NOT NULL,
#            c2 integer NOT NULL,
#            c3 smallint NOT NULL,
#            c4 smallint NOT NULL,
#            c5 integer NOT NULL,
#            c6 bigint NOT NULL,
#            c7 smallint NOT NULL,
#            c8 integer NOT NULL,
#            c9 bigint NOT NULL,
#            c10 character varying NOT NULL,
#            c11 double precision NOT NULL,
#            c12 double precision NOT NULL,
#            c13 character varying NOT NULL
#          );'
#          psql -d "$POSTGRES_DB" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -c "\copy test FROM '$(pwd)/testing/data/csv/aggregate_test_100.csv' WITH (FORMAT csv, HEADER true);"
#        env:
#          POSTGRES_HOST: localhost
#          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
#          POSTGRES_DB: db_test
#          POSTGRES_USER: postgres
#          POSTGRES_PASSWORD: postgres
#      - name: Build datafusion-cli
#        run: cargo build --bin datafusion-cli
#      - name: Test Psql Parity
#        run: python -m pytest -v integration-tests/test_psql_parity.py
#        env:
#          POSTGRES_HOST: localhost
#          POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
#          POSTGRES_DB: db_test
#          POSTGRES_USER: postgres
#          POSTGRES_PASSWORD: postgres

  windows-and-macos:
    name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-latest]
        rust: [nightly-2024-01-29]
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
      # TODO: this won't cache anything, which is expensive. Setup this action
      # with a OS-dependent path.
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt
      - name: Run tests
        shell: bash
        run: |
          export ARROW_TEST_DATA=$(pwd)/testing/data
          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
          cargo test
        env:
          # do not produce debug symbols to keep memory usage down
          RUSTFLAGS: "-C debuginfo=0"

  lint:
    name: Lint
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v2
      - name: Setup toolchain
        run: |
          rustup toolchain install nightly-2024-01-29
          rustup default nightly-2024-01-29
          rustup component add rustfmt
      - name: Run
        run: cargo fmt --all -- --check

#  clippy:
#    name: Clippy
#    needs: [linux-build-lib]
#    runs-on: ubuntu-latest
#    strategy:
#      matrix:
#        arch: [amd64]
#        rust: [nightly-2021-07-04]
#    container:
#      image: ${{ matrix.arch }}/rust
#      env:
#        # Disable full debug symbol generation to speed up CI build and keep memory down
#        # "1" means line tables only, which is useful for panic tracebacks.
#        RUSTFLAGS: "-C debuginfo=1"
#    steps:
#      - uses: actions/checkout@v2
#        with:
#          submodules: true
#      - name: Cache Cargo
#        uses: actions/cache@v2
#        with:
#          path: /github/home/.cargo
#          # this key equals the ones on `linux-build-lib` for re-use
#          key: cargo-cache-
#      - name: Cache Rust dependencies
#        uses: actions/cache@v2
#        with:
#          path: /github/home/target
#          # this key equals the ones on `linux-build-lib` for re-use
#          key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
#      - name: Setup Rust toolchain
#        run: |
#          rustup toolchain install ${{ matrix.rust }}
#          rustup default ${{ matrix.rust }}
#          rustup component add rustfmt clippy
#      - name: Run clippy
#        run: |
#          cargo clippy --all-targets --workspace -- -D warnings
#        env:
#          CARGO_HOME: "/github/home/.cargo"
#          CARGO_TARGET_DIR: "/github/home/target"
#
#  miri-checks:
#    name: MIRI
#    runs-on: ubuntu-latest
#    strategy:
#      matrix:
#        arch: [amd64]
#        rust: [nightly-2021-07-04]
#    steps:
#      - uses: actions/checkout@v2
#        with:
#          submodules: true
#      - uses: actions/cache@v2
#        with:
#          path: |
#            ~/.cargo/registry
#            ~/.cargo/git
#            target
#          key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }}
#      - name: Setup Rust toolchain
#        run: |
#          rustup toolchain install ${{ matrix.rust }}
#          rustup default ${{ matrix.rust }}
#          rustup component add rustfmt clippy miri
#      - name: Run Miri Checks
#        env:
#          RUST_BACKTRACE: full
#          RUST_LOG: "trace"
#          MIRIFLAGS: "-Zmiri-disable-isolation"
#        run: |
#          cargo miri setup
#          cargo clean
#          # Ignore MIRI errors until we can get a clean run
#          cargo miri test || true

# Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it

  # coverage:
  #   name: Coverage
  #   runs-on: ubuntu-latest
  #   strategy:
  #     matrix:
  #       arch: [amd64]
  #       rust: [nightly-2021-07-04]
  #   steps:
  #     - uses: actions/checkout@v2
  #       with:
  #         submodules: true
  #     - name: Cache Cargo
  #       uses: actions/cache@v2
  #       with:
  #         path: /home/runner/.cargo
  #         # this key is not equal because the user is different than on a container (runner vs github)
  #         key: cargo-coverage-cache-
  #     - name: Cache Rust dependencies
  #       uses: actions/cache@v2
  #       with:
  #         path: /home/runner/target
  #         # this key is not equal because coverage uses different compilation flags.
  #         key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}-
  #     - name: Run coverage
  #       run: |
  #         export ARROW_TEST_DATA=$(pwd)/testing/data
  #         export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data

  #         # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
  #         # see https://github.com/xd009642/tarpaulin/issues/618
  #         cargo install --version 0.16.0 cargo-tarpaulin
  #         cargo tarpaulin --out Xml
  #       env:
  #         CARGO_HOME: "/home/runner/.cargo"
  #         CARGO_TARGET_DIR: "/home/runner/target"
  #     - name: Report coverage
  #       continue-on-error: true
  #       run: bash <(curl -s https://codecov.io/bash)