Skip to content
Rust

feat: Add MetadataCacheFactory #1167

Sign in to view logs

feat: Add MetadataCacheFactory

feat: Add MetadataCacheFactory #1167

Workflow file for this run

.github/workflows/rust.yml at 32c4a03
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Rust
on:
# always trigger
push:
pull_request:
jobs:
# build the library, a compilation step used by multiple steps below
linux-build-lib:
name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
strategy:
matrix:
arch: [amd64]
rust: [nightly-2024-01-29]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- uses: actions/checkout@v2
- name: Cache Cargo
uses: actions/cache@v2
with:
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
path: /github/home/.cargo
key: cargo-cache-
- name: Cache Rust dependencies
uses: actions/cache@v2
with:
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
path: /github/home/target
key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}-
- name: Setup Rust toolchain
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
rustup component add rustfmt
- name: Build Workspace
run: |
cargo build
env:
CARGO_HOME: "/github/home/.cargo"
CARGO_TARGET_DIR: "/github/home/target"
# test the crate
linux-test:
name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
needs: [linux-build-lib]
runs-on: ubuntu-latest
strategy:
matrix:
arch: [amd64]
rust: [nightly-2024-01-29]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# this key equals the ones on `linux-build-lib` for re-use
key: cargo-cache-
- name: Cache Rust dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# this key equals the ones on `linux-build-lib` for re-use
key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
- name: Setup Rust toolchain
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
rustup component add rustfmt
- name: Run tests
run: |
export ARROW_TEST_DATA=$(pwd)/testing/data
export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# run tests on all workspace members with default feature list
cargo test
# test datafusion examples
#cd datafusion-examples
#cargo test --no-default-features
#cargo run --example csv_sql
#cargo run --example parquet_sql
env:
CARGO_HOME: "/github/home/.cargo"
CARGO_TARGET_DIR: "/github/home/target"
# Ballista is currently not part of the main workspace so requires a separate test step
# - name: Run Ballista tests
# run: |
# export ARROW_TEST_DATA=$(pwd)/testing/data
# export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# cd ballista/rust
# # snmalloc requires cmake so build without default features
# cargo test --no-default-features --features sled
# env:
# CARGO_HOME: "/github/home/.cargo"
# CARGO_TARGET_DIR: "/github/home/target"
# integration-test:
# name: "Integration Test"
# needs: [linux-build-lib]
# runs-on: ubuntu-latest
# services:
# postgres:
# image: postgres:13
# env:
# POSTGRES_PASSWORD: postgres
# POSTGRES_DB: db_test
# ports:
# - 5432/tcp
# options: >-
# --health-cmd pg_isready
# --health-interval 10s
# --health-timeout 5s
# --health-retries 5
# steps:
# - uses: actions/checkout@v2
# with:
# submodules: true
# - uses: actions/setup-python@v2
# with:
# python-version: "3.8"
# - name: Install Python dependencies
# run: |
# python -m pip install --upgrade pip setuptools wheel
# python -m pip install -r integration-tests/requirements.txt
# - name: Allow access of psql
# run: |
# # make sure psql can access the server
# echo "$POSTGRES_HOST:$POSTGRES_PORT:$POSTGRES_DB:$POSTGRES_USER:$POSTGRES_PASSWORD" | tee ~/.pgpass
# chmod 0600 ~/.pgpass
# psql -d "$POSTGRES_DB" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -c 'CREATE TABLE IF NOT EXISTS test (
# c1 character varying NOT NULL,
# c2 integer NOT NULL,
# c3 smallint NOT NULL,
# c4 smallint NOT NULL,
# c5 integer NOT NULL,
# c6 bigint NOT NULL,
# c7 smallint NOT NULL,
# c8 integer NOT NULL,
# c9 bigint NOT NULL,
# c10 character varying NOT NULL,
# c11 double precision NOT NULL,
# c12 double precision NOT NULL,
# c13 character varying NOT NULL
# );'
# psql -d "$POSTGRES_DB" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -c "\copy test FROM '$(pwd)/testing/data/csv/aggregate_test_100.csv' WITH (FORMAT csv, HEADER true);"
# env:
# POSTGRES_HOST: localhost
# POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
# POSTGRES_DB: db_test
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# - name: Build datafusion-cli
# run: cargo build --bin datafusion-cli
# - name: Test Psql Parity
# run: python -m pytest -v integration-tests/test_psql_parity.py
# env:
# POSTGRES_HOST: localhost
# POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
# POSTGRES_DB: db_test
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
windows-and-macos:
name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-latest]
rust: [nightly-2024-01-29]
steps:
- uses: actions/checkout@v2
with:
submodules: true
# TODO: this won't cache anything, which is expensive. Setup this action
# with a OS-dependent path.
- name: Setup Rust toolchain
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
rustup component add rustfmt
- name: Run tests
shell: bash
run: |
export ARROW_TEST_DATA=$(pwd)/testing/data
export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
cargo test
env:
# do not produce debug symbols to keep memory usage down
RUSTFLAGS: "-C debuginfo=0"
lint:
name: Lint
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- name: Setup toolchain
run: |
rustup toolchain install nightly-2024-01-29
rustup default nightly-2024-01-29
rustup component add rustfmt
- name: Run
run: cargo fmt --all -- --check
# clippy:
# name: Clippy
# needs: [linux-build-lib]
# runs-on: ubuntu-latest
# strategy:
# matrix:
# arch: [amd64]
# rust: [nightly-2021-07-04]
# container:
# image: ${{ matrix.arch }}/rust
# env:
# # Disable full debug symbol generation to speed up CI build and keep memory down
# # "1" means line tables only, which is useful for panic tracebacks.
# RUSTFLAGS: "-C debuginfo=1"
# steps:
# - uses: actions/checkout@v2
# with:
# submodules: true
# - name: Cache Cargo
# uses: actions/cache@v2
# with:
# path: /github/home/.cargo
# # this key equals the ones on `linux-build-lib` for re-use
# key: cargo-cache-
# - name: Cache Rust dependencies
# uses: actions/cache@v2
# with:
# path: /github/home/target
# # this key equals the ones on `linux-build-lib` for re-use
# key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
# - name: Setup Rust toolchain
# run: |
# rustup toolchain install ${{ matrix.rust }}
# rustup default ${{ matrix.rust }}
# rustup component add rustfmt clippy
# - name: Run clippy
# run: |
# cargo clippy --all-targets --workspace -- -D warnings
# env:
# CARGO_HOME: "/github/home/.cargo"
# CARGO_TARGET_DIR: "/github/home/target"
#
# miri-checks:
# name: MIRI
# runs-on: ubuntu-latest
# strategy:
# matrix:
# arch: [amd64]
# rust: [nightly-2021-07-04]
# steps:
# - uses: actions/checkout@v2
# with:
# submodules: true
# - uses: actions/cache@v2
# with:
# path: |
# ~/.cargo/registry
# ~/.cargo/git
# target
# key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }}
# - name: Setup Rust toolchain
# run: |
# rustup toolchain install ${{ matrix.rust }}
# rustup default ${{ matrix.rust }}
# rustup component add rustfmt clippy miri
# - name: Run Miri Checks
# env:
# RUST_BACKTRACE: full
# RUST_LOG: "trace"
# MIRIFLAGS: "-Zmiri-disable-isolation"
# run: |
# cargo miri setup
# cargo clean
# # Ignore MIRI errors until we can get a clean run
# cargo miri test || true
# Coverage job was failing. https://github.com/apache/arrow-datafusion/issues/590 tracks re-instating it
# coverage:
# name: Coverage
# runs-on: ubuntu-latest
# strategy:
# matrix:
# arch: [amd64]
# rust: [nightly-2021-07-04]
# steps:
# - uses: actions/checkout@v2
# with:
# submodules: true
# - name: Cache Cargo
# uses: actions/cache@v2
# with:
# path: /home/runner/.cargo
# # this key is not equal because the user is different than on a container (runner vs github)
# key: cargo-coverage-cache-
# - name: Cache Rust dependencies
# uses: actions/cache@v2
# with:
# path: /home/runner/target
# # this key is not equal because coverage uses different compilation flags.
# key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}-
# - name: Run coverage
# run: |
# export ARROW_TEST_DATA=$(pwd)/testing/data
# export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
# # see https://github.com/xd009642/tarpaulin/issues/618
# cargo install --version 0.16.0 cargo-tarpaulin
# cargo tarpaulin --out Xml
# env:
# CARGO_HOME: "/home/runner/.cargo"
# CARGO_TARGET_DIR: "/home/runner/target"
# - name: Report coverage
# continue-on-error: true
# run: bash <(curl -s https://codecov.io/bash)