From bbf0662f6de9c652fea319a4f84bc5b512d0400f Mon Sep 17 00:00:00 2001 From: Christian Zentgraf Date: Wed, 18 Dec 2024 14:48:51 -0500 Subject: [PATCH] [native] Remove old CircleCI jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes the remaining CircleCI jobs from the repo so they won’t get run on new PRs. --- .circleci/config.yml | 30 --- .circleci/continue_config.yml | 375 ---------------------------------- 2 files changed, 405 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/continue_config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 1d29ee58b851..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -version: 2.1 - -setup: true - -orbs: - path-filtering: circleci/path-filtering@0.1.5 - -workflows: - always-run: - jobs: - - path-filtering/filter: - name: check-updated-files - # 3-column, whitespace-delimited mapping. One mapping per line - # - mapping: | - presto-native-execution/.* run_native_specific_jobs true - (?!presto-docs).*(?> - - linux-presto-e2e-tests: - run_linux_tests: << pipeline.parameters.run_linux_tests >> - requires: - - linux-build-and-unit-test - - linux-spark-e2e-tests: - run_linux_tests: << pipeline.parameters.run_linux_tests >> - requires: - - linux-build-and-unit-test - - linux-presto-native-sidecar-tests: - run_linux_tests: << pipeline.parameters.run_linux_tests >> - requires: - - linux-build-and-unit-test - - conditionals: - when: << pipeline.parameters.run_native_specific_jobs >> - jobs: - - format-check - - header-check - - linux-build-all - -commands: - maven_install: - description: "Maven Install" - parameters: - maven_install_opts: - type: string - maven_fast_install: - type: string - steps: - - run: - name: "Maven Install" - command: | - export MAVEN_OPTS=<< parameters.maven_install_opts>> - # Sometimes mvn build fails because of network failures. Add logic to rerun the build to avoid non PR-related CI failures. - for i in $(seq 1 3); do ./mvnw clean install << parameters.maven_fast_install>> -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) - -executors: - build: - docker: - # These images are managed by the Presto Release team. - - image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090 - resource_class: 2xlarge - environment: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError -XX:CompressedClassSpaceSize=3g" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError -XX:CompressedClassSpaceSize=3g" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" - check: - docker: - - image: public.ecr.aws/oss-presto/velox-dev:check - macos-intel: - macos: - xcode: "14.3.0" - resource_class: macos.x86.medium.gen2 - -jobs: - linux-build-and-unit-test: - executor: build - parameters: - run_linux_tests: - type: boolean - default: false - steps: - - run: echo "Run Linux tests is << parameters.run_linux_tests >>" - - when: - condition: << parameters.run_linux_tests >> - steps: - - checkout - - run: - name: "Update velox" - command: | - cd presto-native-execution - make velox-submodule - - run: - name: "Calculate merge-base date for CCache" - command: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date - - restore_cache: - name: "Restore CCache cache" - keys: - - native-exe-linux-ccache-{{ arch }}-{{ checksum "merge-base-date" }} - - run: - name: Build - command: | - mkdir -p .ccache - export CCACHE_DIR=$(realpath .ccache) - ccache -sz -M 8Gi - source /opt/rh/gcc-toolset-12/enable - cd presto-native-execution - cmake \ - -B _build/debug \ - -GNinja \ - -DTREAT_WARNINGS_AS_ERRORS=1 \ - -DENABLE_ALL_WARNINGS=1 \ - -DCMAKE_BUILD_TYPE=Debug \ - -DPRESTO_ENABLE_PARQUET=ON \ - -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ - -DPRESTO_ENABLE_JWT=ON \ - -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ - -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DMAX_LINK_JOBS=2 - ninja -C _build/debug -j 8 - ccache -s - - save_cache: - name: "Save CCache cache" - key: native-exe-linux-ccache-{{ arch }}-{{ checksum "merge-base-date" }} - paths: - - .ccache/ - - run: - name: 'Run Unit Tests' - command: | - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64" - cd presto-native-execution/_build/debug - ctest -j 10 -VV --output-on-failure --no-tests=error - - persist_to_workspace: - root: presto-native-execution - paths: - - _build/debug/presto_cpp/main/presto_server - - _build/debug/velox/velox/functions/remote/server/velox_functions_remote_server_main - - linux-presto-e2e-tests: - executor: build - parameters: - run_linux_tests: - type: boolean - default: false - parallelism: 5 - steps: - - run: echo "Run Linux tests is << parameters.run_linux_tests >>" - - when: - condition: << parameters.run_linux_tests >> - steps: - - checkout - - attach_workspace: - at: presto-native-execution - - maven_install: - maven_install_opts: ${MAVEN_INSTALL_OPTS} - maven_fast_install: ${MAVEN_FAST_INSTALL} - - run: - name: 'Run presto-native e2e tests' - command: | - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64" - export PRESTO_SERVER_PATH="${HOME}/project/presto-native-execution/_build/debug/presto_cpp/main/presto_server" - export TEMP_PATH="/tmp" - TESTFILES=$(circleci tests glob "presto-native-execution/src/test/**/TestPrestoNative*.java" | circleci tests split --split-by=timings) - # Convert file paths to comma separated class names - export TESTCLASSES= - for test_file in $TESTFILES - do - tmp=${test_file##*/} - test_class=${tmp%%\.*} - export TESTCLASSES="${TESTCLASSES},$test_class" - done - export TESTCLASSES=${TESTCLASSES#,} - - # TODO: neeed to enable remote function tests with - # "-Ppresto-native-execution-remote-functions" once - # > https://github.com/facebookincubator/velox/discussions/6163 - # is fixed. - if [ ! -z $TESTCLASSES ]; then - mvn test \ - ${MAVEN_TEST} \ - -pl 'presto-native-execution' \ - -Dtest="${TESTCLASSES}" \ - -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ - -DDATA_DIR=${TEMP_PATH} \ - -Duser.timezone=America/Bahia_Banderas \ - -T1C - fi - - store_artifacts: - path: '/tmp/PrestoNativeQueryRunnerUtils' - - linux-presto-native-sidecar-tests: - executor: build - parameters: - run_linux_tests: - type: boolean - default: false - parallelism: 5 - steps: - - run: echo "Run Linux tests is << parameters.run_linux_tests >>" - - when: - condition: << parameters.run_linux_tests >> - steps: - - checkout - - attach_workspace: - at: presto-native-execution - - maven_install: - maven_install_opts: ${MAVEN_INSTALL_OPTS} - maven_fast_install: ${MAVEN_FAST_INSTALL} - - run: - name: 'Run Presto native sidecar tests' - command: | - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64" - export PRESTO_SERVER_PATH="${HOME}/project/presto-native-execution/_build/debug/presto_cpp/main/presto_server" - export TEMP_PATH="/tmp" - TESTFILES=$(circleci tests glob "presto-native-sidecar-plugin/src/test/**/Test*.java" | circleci tests split --split-by=timings) - # Convert file paths to comma separated class names - export TESTCLASSES= - for test_file in $TESTFILES - do - tmp=${test_file##*/} - test_class=${tmp%%\.*} - export TESTCLASSES="${TESTCLASSES},$test_class" - done - export TESTCLASSES=${TESTCLASSES#,} - if [ ! -z $TESTCLASSES ]; then - mvn test \ - ${MAVEN_TEST} \ - -pl 'presto-native-sidecar-plugin' \ - -Dtest="${TESTCLASSES}" \ - -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ - -DDATA_DIR=${TEMP_PATH} \ - -Duser.timezone=America/Bahia_Banderas \ - -T1C - fi - - linux-spark-e2e-tests: - executor: build - parameters: - run_linux_tests: - type: boolean - default: false - parallelism: 5 - steps: - - run: echo "Run Linux tests is << parameters.run_linux_tests >>" - - when: - condition: << parameters.run_linux_tests >> - steps: - - checkout - - attach_workspace: - at: presto-native-execution - - maven_install: - maven_install_opts: ${MAVEN_INSTALL_OPTS} - maven_fast_install: ${MAVEN_FAST_INSTALL} - - run: - name: 'Run spark e2e tests' - command: | - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64" - export PRESTO_SERVER_PATH="${HOME}/project/presto-native-execution/_build/debug/presto_cpp/main/presto_server" - export TEMP_PATH="/tmp" - TESTFILES=$(circleci tests glob "presto-native-execution/src/test/**/TestPrestoSpark*.java" | circleci tests split --split-by=timings) - # Convert file paths to comma separated class names - export TESTCLASSES= - for test_file in $TESTFILES - do - tmp=${test_file##*/} - test_class=${tmp%%\.*} - export TESTCLASSES="${TESTCLASSES},$test_class" - done - export TESTCLASSES=${TESTCLASSES#,} - if [ ! -z $TESTCLASSES ]; then - mvn test \ - ${MAVEN_TEST} \ - -pl 'presto-native-execution' \ - -Dtest="${TESTCLASSES}" \ - -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ - -DDATA_DIR=${TEMP_PATH} \ - -Duser.timezone=America/Bahia_Banderas \ - -T1C - fi - - linux-build-all: - executor: build - steps: - - checkout - - run: - name: "Update velox submodule" - command: | - cd presto-native-execution - make velox-submodule - - run: - name: "Install all adapter dependencies" - command: | - mkdir -p ${HOME}/adapter-deps/install - source /opt/rh/gcc-toolset-12/enable - set -xu - cd presto-native-execution - DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh jwt - - run: - name: "Build All" - command: | - source /opt/rh/gcc-toolset-12/enable - cd presto-native-execution - cmake \ - -B _build/release \ - -GNinja \ - -DAWSSDK_ROOT_DIR=${HOME}/adapter-deps/install \ - -DTREAT_WARNINGS_AS_ERRORS=1 \ - -DENABLE_ALL_WARNINGS=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -DPRESTO_ENABLE_PARQUET=ON \ - -DPRESTO_ENABLE_S3=ON \ - -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ - -DPRESTO_ENABLE_JWT=ON \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DMAX_LINK_JOBS=2 - ninja -C _build/release -j 8 - - format-check: - executor: check - steps: - - checkout - - run: - name: Check formatting - command: | - cd presto-native-execution - make format-check - - header-check: - executor: check - steps: - - checkout - - run: - name: "Check license headers" - command: | - cd presto-native-execution - make header-check - - native-specific-macos: - executor: macos-intel - steps: - - checkout - - run: - name: "Update submodules" - command: | - cd presto-native-execution - make submodules - - run: - name: "Setup MacOS" - command: | - set -xu - mkdir ~/deps ~/deps-src - git clone --depth 1 https://github.com/Homebrew/brew ~/deps - PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh - # Calculate the prefix path before we delete brew's repos and taps. - echo "$(pwd)/deps;$(brew --prefix openssl@1.1);$(brew --prefix icu4c)" > ~/deps/PREFIX_PATH - rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%. - rm -rf ~/deps-src - - run: - name: "Build presto_cpp on MacOS" - command: | - export PATH=~/deps/bin:${PATH} - cd presto-native-execution - cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=$(cat ~/deps/PREFIX_PATH) -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - ninja -C _build/debug