From 305ca199377af155484fdb4f3c915b500d03bbc7 Mon Sep 17 00:00:00 2001 From: Piotr Bulawa Date: Fri, 20 Sep 2024 13:21:55 +0200 Subject: [PATCH] Initial CI/CD scripts --- .github/workflows/build-test.yml | 110 ++++++++++++++++++ .github/workflows/cla_bot.yml | 29 +++++ .github/workflows/parameters_aws.json.gpg | Bin 0 -> 268 bytes .github/workflows/parameters_azure.json.gpg | 3 + .github/workflows/parameters_gcp.json.gpg | Bin 0 -> 284 bytes .github/workflows/snyk-issue.yml | 33 ++++++ .github/workflows/snyk-pr.yml | 37 ++++++ ci/_init.sh | 18 +++ ci/build.sh | 10 ++ ci/container/build_component.sh | 12 ++ ci/container/change_snowflake_test_pwd.py | 28 +++++ ci/container/create_schema.py | 20 ++++ ci/container/download_artifact.sh | 30 +++++ ci/container/drop_schema.py | 20 ++++ ci/container/sf_test_utils.py | 33 ++++++ ci/container/test_component.sh | 81 +++++++++++++ ci/container/upload_artifact.sh | 42 +++++++ ci/log_analyze_setup.sh | 59 ++++++++++ ci/scripts/login_docker.sh | 10 ++ ci/scripts/login_internal_docker.sh | 18 +++ ci/scripts/set_git_info.sh | 34 ++++++ ci/test.sh | 62 ++++++++++ ci/test_mac.sh | 21 ++++ ci/test_windows.bat | 120 ++++++++++++++++++++ 24 files changed, 830 insertions(+) create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/cla_bot.yml create mode 100644 .github/workflows/parameters_aws.json.gpg create mode 100644 .github/workflows/parameters_azure.json.gpg create mode 100644 .github/workflows/parameters_gcp.json.gpg create mode 100644 .github/workflows/snyk-issue.yml create mode 100644 .github/workflows/snyk-pr.yml create mode 100755 ci/_init.sh create mode 100755 ci/build.sh create mode 100755 ci/container/build_component.sh create mode 100644 ci/container/change_snowflake_test_pwd.py create mode 100644 ci/container/create_schema.py create mode 100755 ci/container/download_artifact.sh create mode 100644 ci/container/drop_schema.py create mode 100644 ci/container/sf_test_utils.py create mode 100755 ci/container/test_component.sh create mode 100755 ci/container/upload_artifact.sh create mode 100755 ci/log_analyze_setup.sh create mode 100644 ci/scripts/login_docker.sh create mode 100644 ci/scripts/login_internal_docker.sh create mode 100644 ci/scripts/set_git_info.sh create mode 100755 ci/test.sh create mode 100755 ci/test_mac.sh create mode 100644 ci/test_windows.bat diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..b0bc03f --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,110 @@ +name: Build and Test + +on: + push: + branches: + - master + tags: + - v* + pull_request: + branches: + - master + - prep-** + workflow_dispatch: + inputs: + logLevel: + default: warning + description: "Log level" + required: true + tags: + description: "Test scenario tags" + +concurrency: + # older builds for the same pull request numer or branch should be cancelled + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build + shell: bash + env: + WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }} + run: ./ci/build.sh + + test-windows: + needs: build + name: ${{ matrix.runConfig.cloud }} Windows java ${{ matrix.runConfig.javaVersion }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + runConfig: [ {cloud: 'AWS', javaVersion: '8'}, {cloud: 'GCP', javaVersion: '11'}, {cloud: 'AZURE', javaVersion: '17'}, {cloud: 'AWS', javaVersion: '21'}] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.runConfig.javaVersion }} + distribution: 'temurin' + cache: maven + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + architecture: 'x64' + - name: Tests + shell: cmd + env: + PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} + CLOUD_PROVIDER: ${{ matrix.runConfig.cloud }} + run: ci\\test_windows.bat + + test-mac: + needs: build + name: ${{ matrix.runConfig.cloud }} Mac java ${{ matrix.runConfig.javaVersion }} + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + runConfig: [ {cloud: 'AWS', javaVersion: '8'}, {cloud: 'GCP', javaVersion: '11'}, {cloud: 'AZURE', javaVersion: '17'}, {cloud: 'AWS', javaVersion: '21'}] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.runConfig.javaVersion }} + distribution: 'temurin' + cache: maven + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + - name: Install Homebrew Bash + shell: bash + run: brew install bash + - name: Tests + shell: bash + env: + PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} + CLOUD_PROVIDER: ${{ matrix.runConfig.cloud }} + run: /usr/local/bin/bash ./ci/test_mac.sh + + test-linux: + needs: build + name: ${{ matrix.cloud }} Linux java on ${{ matrix.image }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: [ 'jdbc-centos7-openjdk8', 'jdbc-centos7-openjdk11', 'jdbc-centos7-openjdk17', 'jdbc-centos7-openjdk21' ] + cloud: [ 'AWS', 'AZURE', 'GCP' ] + steps: + - uses: actions/checkout@v4 + - name: Tests + shell: bash + env: + PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} + CLOUD_PROVIDER: ${{ matrix.cloud }} + TARGET_DOCKER_TEST_IMAGE: ${{ matrix.image }} + run: ./ci/test.sh diff --git a/.github/workflows/cla_bot.yml b/.github/workflows/cla_bot.yml new file mode 100644 index 0000000..d825a11 --- /dev/null +++ b/.github/workflows/cla_bot.yml @@ -0,0 +1,29 @@ +name: "CLA Assistant" +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened,closed,synchronize] + +jobs: + CLAAssistant: + runs-on: ubuntu-latest + permissions: + actions: write + contents: write + pull-requests: write + statuses: write + steps: + - name: "CLA Assistant" + if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' + uses: contributor-assistant/github-action/@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }} + with: + path-to-signatures: 'signatures/version1.json' + path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md' + branch: 'main' + allowlist: 'dependabot[bot],github-actions,Jenkins User,_jenkins,sfc-gh-snyk-sca-sa,snyk-bot' + remote-organization-name: 'snowflakedb' + remote-repository-name: 'cla-db' diff --git a/.github/workflows/parameters_aws.json.gpg b/.github/workflows/parameters_aws.json.gpg new file mode 100644 index 0000000000000000000000000000000000000000..0be590eba1c8dd368bd0959d3e3e77a637fe4233 GIT binary patch literal 268 zcmV+n0rUQh4Fm}T2q^0DymcI+lKRrXIstepOkhp{9rrg&v%t{W3y<`5O>T091+39+ zW zlbTgK(wl01jA*U8BRNUWecPx*OFsK0xwbz=+lqz^_99V`a{6nRLEBZBNS0^ zvaQ#9d;S6miFLfcDn}zMyBsKTG|cw)WKv}^NZW77J1W?E3jCZrkpTC5Lm;8bD#Ugo SSFN0^^8y5ZCD5tHsocYGrhYX5 literal 0 HcmV?d00001 diff --git a/.github/workflows/parameters_azure.json.gpg b/.github/workflows/parameters_azure.json.gpg new file mode 100644 index 0000000..993be75 --- /dev/null +++ b/.github/workflows/parameters_azure.json.gpg @@ -0,0 +1,3 @@ +  ɂTH^?[Lm rl1{tقYEm}:!e Y/I~,kA-)gx?DڒKk/hq)3яNB\I$t<ʟ2|S#[ RcMY;9n41(۷xA3 + +ȆҰ_K3RbeaO1.T?6Ox"Hqc6L[_M]I^**\W99F` \ No newline at end of file diff --git a/.github/workflows/parameters_gcp.json.gpg b/.github/workflows/parameters_gcp.json.gpg new file mode 100644 index 0000000000000000000000000000000000000000..4b4c83896c2cb12b3049c99df847a1a664e7f3b7 GIT binary patch literal 284 zcmV+%0ptFR4Fm}T2(j-+{28BOJ^IqXN&##duN|d9rI9C;*v*!0jct-yNgC)a#|Qly zkD3pKr)4Tj<-})S_Qfotj&%3ij-qmqz_p`%(5>Yq;#IW57L9%J{qOuhn($M%IIY43 z5x-+~w-N!F5Zv@SB`*k{Cf!9yM64~#o@J)j#0fsALHb#Q#P-msrIsiArS!@M(9_|! z-D*v$@s9NXpX}aUS0h=s&AB~Zd@^f(kaR}H1?^OjU0JcW(exJweES9b3ld3TXOXie zt_G-(7&-{mr~=p9u4TQR`0p@ZlG5`$Yo6+#r!(VYhl@K}%S2%h4-mD&{xT>d1rB&b iZp0c=)<(tEM#C(=!%aAYFOOXr=x8MS`)y=`hnD{-1&`nW literal 0 HcmV?d00001 diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml new file mode 100644 index 0000000..7b58bb1 --- /dev/null +++ b/.github/workflows/snyk-issue.yml @@ -0,0 +1,33 @@ +name: Snyk Issue + +on: + schedule: + - cron: '* */12 * * *' + +permissions: + contents: read + issues: write + pull-requests: write + +concurrency: snyk-issue + +jobs: + snyk: + runs-on: ubuntu-latest + steps: + - name: checkout action + uses: actions/checkout@v3 + with: + repository: snowflakedb/whitesource-actions + token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} + path: whitesource-actions + - name: set-env + run: echo "REPO=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV + - name: Jira Creation + uses: ./whitesource-actions/snyk-issue + with: + snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }} + snyk_token: ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN_PUBLIC_REPO }} + jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml new file mode 100644 index 0000000..0c101e3 --- /dev/null +++ b/.github/workflows/snyk-pr.yml @@ -0,0 +1,37 @@ +name: snyk-pr +on: + pull_request: + branches: + - master + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + snyk: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} + steps: + - name: checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: checkout action + uses: actions/checkout@v4 + with: + repository: snowflakedb/whitesource-actions + token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} + path: whitesource-actions + + - name: PR + uses: ./whitesource-actions/snyk-pr + env: + PR_TITLE: ${{ github.event.pull_request.title }} + with: + jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + amend: false # true if you want the commit to be amended with the JIRA number diff --git a/ci/_init.sh b/ci/_init.sh new file mode 100755 index 0000000..a95e70f --- /dev/null +++ b/ci/_init.sh @@ -0,0 +1,18 @@ +#!/usr/local/bin/env bash +set -e + +export PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]') +export INTERNAL_REPO=nexus.int.snowflakecomputing.com:8086 +if [[ -z "$GITHUB_ACTIONS" ]]; then + # Use the internal Docker Registry + export DOCKER_REGISTRY_NAME=$INTERNAL_REPO/docker + export WORKSPACE=${WORKSPACE:-/tmp} +else + # Use Docker Hub + export DOCKER_REGISTRY_NAME=snowflakedb + export WORKSPACE=$GITHUB_WORKSPACE +fi +mkdir -p $WORKSPACE + +export DRIVER_NAME=hibernate-orm + diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 0000000..4d5173a --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +# +# Build Hibernate +# +set -o pipefail +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export WORKSPACE=${WORKSPACE:=/tmp} +"$THIS_DIR/container/build_component.sh" diff --git a/ci/container/build_component.sh b/ci/container/build_component.sh new file mode 100755 index 0000000..bca0af4 --- /dev/null +++ b/ci/container/build_component.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e +# +# Build Hibernate +# +set -o pipefail +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +HIBERNATE_ROOT=$(cd "${THIS_DIR}/../../" && pwd) + +cd "$HIBERNATE_ROOT" +rm -f lib/*.jar +mvn clean install --batch-mode --show-version -DskipTests +"$THIS_DIR"/upload_artifact.sh diff --git a/ci/container/change_snowflake_test_pwd.py b/ci/container/change_snowflake_test_pwd.py new file mode 100644 index 0000000..62932b4 --- /dev/null +++ b/ci/container/change_snowflake_test_pwd.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# +# Set a complex password for test user snowman +# + +import os +import sys +import snowflake.connector + +params = { + 'account': '', + 'user': os.getenv("SNOWFLAKE_TEST_USER"), + 'password': os.getenv("SNOWFLAKE_TEST_PASSWORD"), + 'database': os.getenv("SNOWFLAKE_TEST_DATABASE"), + 'role': os.getenv("SNOWFLAKE_TEST_ROLE"), + 'host': os.getenv("SNOWFLAKE_TEST_HOST"), + 'port': os.getenv("SNOWFLAKE_TEST_PORT"), + 'protocol': os.getenv("SNOWFLAKE_TEST_PROTOCOL"), +} + +for account in ["testaccount", "s3testaccount", "azureaccount", "gcpaccount"]: + params['account'] = account + conn = snowflake.connector.connect(**params) + conn.cursor().execute("use role accountadmin") + cmd = "alter user set password = '{}'".format(os.getenv("SNOWFLAKE_TEST_PASSWORD_NEW")) + print(cmd) + conn.cursor().execute(cmd) + conn.close() diff --git a/ci/container/create_schema.py b/ci/container/create_schema.py new file mode 100644 index 0000000..adacd33 --- /dev/null +++ b/ci/container/create_schema.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# +# Create test schema +# +import os +import sys +import snowflake.connector + +import sf_test_utils + +test_schema = sf_test_utils.get_test_schema() +if not test_schema: + sys.exit(2) + +params = sf_test_utils.init_connection_params() + +con = snowflake.connector.connect(**params) +con.cursor().execute("create or replace schema {0}".format(test_schema)) + +sys.exit(0) diff --git a/ci/container/download_artifact.sh b/ci/container/download_artifact.sh new file mode 100755 index 0000000..3d33c67 --- /dev/null +++ b/ci/container/download_artifact.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e +# +# Download Artifact +# +set -o pipefail +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +HIBERNATE_ROOT=$(cd "${THIS_DIR}/../../" && pwd) + +if [[ -z "$GITHUB_ACTIONS" ]] ;then + export GIT_BRANCH=${GIT_BRANCH:-origin/$(git rev-parse --abbrev-ref HEAD)} + + BRANCH=$(basename ${GIT_BRANCH}) + + # Place to hold downloaded library + export LIB_DIR=$WORKSPACE/lib + + mkdir -p $LIB_DIR + pushd $LIB_DIR >& /dev/null + base_stage=s3://sfc-eng-jenkins/repository/hibernate/${BRANCH} + export GIT_COMMIT=${GIT_COMMIT:-$(aws s3 cp $base_stage/latest_commit -)} + source_stage=$base_stage/${GIT_COMMIT} + echo "[INFO] downloading ${source_stage}/" + aws s3 cp --only-show-errors $source_stage/ . --recursive + popd >& /dev/null + mkdir -p /mnt/host/lib + cp -p $LIB_DIR/*.jar /mnt/host/lib +else + export GIT_BRANCH=origin/$(basename ${GITHUB_REF}) + export GIT_COMMIT=${GITHUB_SHA} +fi diff --git a/ci/container/drop_schema.py b/ci/container/drop_schema.py new file mode 100644 index 0000000..9900226 --- /dev/null +++ b/ci/container/drop_schema.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# +# Create test schema +# +import os +import sys +import snowflake.connector + +import sf_test_utils + +test_schema = sf_test_utils.get_test_schema() +if not test_schema: + sys.exit(0) + +params = sf_test_utils.init_connection_params() + +con = snowflake.connector.connect(**params) +con.cursor().execute("drop schema if exists {0}".format(test_schema)) + +sys.exit(0) diff --git a/ci/container/sf_test_utils.py b/ci/container/sf_test_utils.py new file mode 100644 index 0000000..30d7061 --- /dev/null +++ b/ci/container/sf_test_utils.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# +# Snowflake test utils +# +import os +import sys + +def get_test_schema(): + return os.getenv("TARGET_SCHEMA_NAME", "LOCAL_reg_1") + + +def init_connection_params(): + params = { + 'account': os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + 'user': os.getenv("SNOWFLAKE_TEST_USER"), + 'password': os.getenv("SNOWFLAKE_TEST_PASSWORD"), + 'database': os.getenv("SNOWFLAKE_TEST_DATABASE"), + 'role': os.getenv("SNOWFLAKE_TEST_ROLE"), + } + host = os.getenv("SNOWFLAKE_TEST_HOST") + if host: + params['host'] = host + port = os.getenv("SNOWFLAKE_TEST_PORT") + if port: + params['port'] = port + protocol = os.getenv("SNOWFLAKE_TEST_PROTOCOL") + if protocol: + params['protocol'] = protocol + warehouse = os.getenv("SNOWFLAKE_TEST_WAREHOUSE") + if warehouse: + params['warehouse'] = warehouse + + return params diff --git a/ci/container/test_component.sh b/ci/container/test_component.sh new file mode 100755 index 0000000..9aaeb4a --- /dev/null +++ b/ci/container/test_component.sh @@ -0,0 +1,81 @@ +#!/bin/bash -e +# +# Test Hibernate for Linux/MAC +# +set -o pipefail +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export WORKSPACE=${WORKSPACE:-/mnt/workspace} +export SOURCE_ROOT=${SOURCE_ROOT:-/mnt/host} +MVNW_EXE=$SOURCE_ROOT/mvnw + +echo "[INFO] Download Hibernate Integration test cases and libraries" +source "$THIS_DIR/download_artifact.sh" + +if [[ -f "$WORKSPACE/parameters.json" ]]; then + echo "[INFO] Found parameter file in $WORKSPACE" + PARAMETER_FILE=$WORKSPACE/parameters.json +else + echo "[INFO] Use the default test parameters.json" + PARAMETER_FILE=$SOURCE_ROOT/src/test/resources/parameters.json +fi +# shellcheck disable=SC2046 +eval $(jq -r '.testconnection | to_entries | map("export \(.key)=\(.value|tostring)")|.[]' "$PARAMETER_FILE") + +if [[ -n "$GITHUB_SHA" ]]; then + # Github Action + export TARGET_SCHEMA_NAME=${RUNNER_TRACKING_ID//-/_}_${GITHUB_SHA} + + function finish() { + pushd "$SOURCE_ROOT"/ci/container >& /dev/null + echo "[INFO] Drop schema $TARGET_SCHEMA_NAME" + python3 drop_schema.py + popd >& /dev/null + } + trap finish EXIT + + pushd "$SOURCE_ROOT/ci/container" >& /dev/null + echo "[INFO] Create schema $TARGET_SCHEMA_NAME" + if python3 create_schema.py; then + export SNOWFLAKE_TEST_SCHEMA=$TARGET_SCHEMA_NAME + else + echo "[WARN] SNOWFLAKE_TEST_SCHEMA: $SNOWFLAKE_TEST_SCHEMA" + fi + popd >& /dev/null +fi + +# we change password, create SSM_KNOWN_FILE +source "$THIS_DIR/../log_analyze_setup.sh" +if [[ "${ENABLE_CLIENT_LOG_ANALYZE}" == "true" ]]; then + echo "[INFO] Log Analyze is enabled." + + setup_log_env + + if [[ "$SNOWFLAKE_TEST_HOST" == *"snowflake.reg"*".local"* && "$SNOWFLAKE_TEST_ACCOUNT" == "s3testaccount" && "$SNOWFLAKE_TEST_USER" == "snowman" && "$SNOWFLAKE_TEST_PASSWORD" == "test" ]]; then + echo "[INFO] Run test with local instance. Will set a more complex password" + + python3 "$THIS_DIR/change_snowflake_test_pwd.py" + export SNOWFLAKE_TEST_PASSWORD=$SNOWFLAKE_TEST_PASSWORD_NEW + + echo "$SNOWFLAKE_TEST_PASSWORD" >> "$CLIENT_KNOWN_SSM_FILE_PATH" + else + echo "[INFO] Not running test with local instance. Won't set a new password" + fi +fi + +env | grep SNOWFLAKE_ | grep -v PASS | sort + + +# Avoid connection timeouts +export MAVEN_OPTS="$MAVEN_OPTS -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120" + +cd "$SOURCE_ROOT" + +# Avoid connection timeout on plugin dependency fetch or fail-fast when dependency cannot be fetched +$MVNW_EXE --batch-mode --show-version dependency:go-offline + +echo "[INFO] Run Hibernate tests" +$MVNW_EXE -Djava.io.tmpdir="$WORKSPACE" \ + -Djacoco.skip.instrument=false \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + verify \ + --batch-mode --show-version diff --git a/ci/container/upload_artifact.sh b/ci/container/upload_artifact.sh new file mode 100755 index 0000000..b2ce5f4 --- /dev/null +++ b/ci/container/upload_artifact.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e +# +# Upload jar files to S3 +# +set -o pipefail +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +HIBERNATE_ROOT=$(cd "${THIS_DIR}/../../" && pwd) + +if [[ -z "$GITHUB_ACTIONS" ]] ;then + export GIT_BRANCH=${GIT_BRANCH:-origin/$(git rev-parse --abbrev-ref HEAD)} + export GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse HEAD)} + export WORKSPACE=${WORKSPACE:-/tmp} + echo "[INFO] Git Branch is $GIT_BRANCH" + if [[ "$GIT_BRANCH" == PR-* ]]; then + BRANCH=$GIT_BRANCH + else + BRANCH=$(basename "${GIT_BRANCH}") + fi + + target_stage=s3://sfc-eng-jenkins/repository/hibernate/$BRANCH/${GIT_COMMIT} + echo "[INFO] Uploading jar to $target_stage/" + aws s3 cp --only-show-errors "$HIBERNATE_ROOT"/lib/ "$target_stage/" --recursive --exclude "*" --include "*.jar" + aws s3 cp --only-show-errors "$HIBERNATE_ROOT/FIPS/lib $target_stage/" --recursive --exclude "*" --include "*.jar" + + COMMIT_FILE=$(mktemp) + cat > "$COMMIT_FILE" <parameters.bat +jq -r ".testconnection | to_entries | map(\"set \(.key)=\(.value)\") | .[]" parameters.json >> parameters.bat +call parameters.bat +if %ERRORLEVEL% NEQ 0 ( + echo === failed to set the test parameters + exit /b 1 +) +set SNOWFLAKE_TEST_SCHEMA=%RUNNER_TRACKING_ID:-=_%_%GITHUB_SHA% +set TARGET_SCHEMA_NAME=%SNOWFLAKE_TEST_SCHEMA% + +echo [INFO] Account: %SNOWFLAKE_TEST_ACCOUNT% +echo [INFO] User : %SNOWFLAKE_TEST_USER% +echo [INFO] Database: %SNOWFLAKE_TEST_DATABASE% +echo [INFO] Schema: %SNOWFLAKE_TEST_SCHEMA% +echo [INFO] Warehouse: %SNOWFLAKE_TEST_WAREHOUSE% +echo [INFO] Role: %SNOWFLAKE_TEST_ROLE% +echo [INFO] PROVIDER: %CLOUD_PROVIDER% + +echo [INFO] Creating schema %SNOWFLAKE_TEST_SCHEMA% +pushd %GITHUB_WORKSPACE%\ci\container +python create_schema.py +popd + +REM setup log + +set CLIENT_LOG_DIR_PATH=%GITHUB_WORKSPACE%\jenkins_rt_logs +echo "[INFO] CLIENT_LOG_DIR_PATH=%CLIENT_LOG_DIR_PATH%" + +set CLIENT_LOG_FILE_PATH=%CLIENT_LOG_DIR_PATH%\ssnowflake_ssm_rt.log +echo "[INFO] CLIENT_LOG_FILE_PATH=%CLIENT_LOG_FILE_PATH%" + +set CLIENT_KNOWN_SSM_FILE_PATH=%CLIENT_LOG_DIR_PATH%\rt_jenkins_log_known_ssm.txt +echo "[INFO] CLIENT_KNOWN_SSM_FILE_PATH=%CLIENT_KNOWN_SSM_FILE_PATH%" + +REM To close log analyze, just set ENABLE_CLIENT_LOG_ANALYZE to not "true", e.g. "false". +set ENABLE_CLIENT_LOG_ANALYZE=true + +REM The new complex password we use for jenkins test +set SNOWFLAKE_TEST_PASSWORD_NEW="ThisIsRandomPassword123!" + +set LOG_PROPERTY_FILE=%GITHUB_WORKSPACE%\src\test\resources\logging.properties + +echo "[INFO] LOG_PROPERTY_FILE=%LOG_PROPERTY_FILE%" + +set CLIENT_DRIVER_NAME=Hibernate-ORM + +powershell -Command "(Get-Content %LOG_PROPERTY_FILE%) | Foreach-Object { $_ -replace '^java.util.logging.FileHandler.pattern.*', 'java.util.logging.FileHandler.pattern = %CLIENT_LOG_FILE_PATH%' } | Set-Content %LOG_PROPERTY_FILE%" + +echo "[INFO] Create log directory" + +IF NOT EXIST %CLIENT_LOG_DIR_PATH% MD %CLIENT_LOG_DIR_PATH% 2>nul + +echo "[INFO] Delete ssm file" +IF EXIST "%CLIENT_KNOWN_SSM_FILE_PATH%" DEL /F /Q "%CLIENT_KNOWN_SSM_FILE_PATH%" + +echo "[INFO] Create ssm file" +echo.>"%CLIENT_KNOWN_SSM_FILE_PATH%" + +echo "[INFO] Finish log setup" +REM end setup log + +for /F "tokens=1,* delims==" %%i in ('set ^| findstr /I /R "^SNOWFLAKE_[^=]*$" ^| findstr /I /V /R "^SNOWFLAKE_PASS_[^=]*$" ^| sort') do ( + echo %%i=%%j +) + +echo [INFO] Testing + +set MVNW_EXE=%GITHUB_WORKSPACE%\mvnw.cmd + +REM Avoid connection timeouts +set MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120" +echo "MAVEN OPTIONS %MAVEN_OPTS%" + +REM Avoid connection timeout on plugin dependency fetch or fail-fast when dependency cannot be fetched +cmd /c %MVNW_EXE% --batch-mode --show-version dependency:go-offline + +echo "[INFO] Run Hibernate tests" +cmd /c %MVNW_EXE% -B -Djava.io.tmpdir=%GITHUB_WORKSPACE% ^ + -Djacoco.skip.instrument=false ^ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn ^ + verify ^ + --batch-mode --show-version > log.txt & type log.txt +echo "[INFO] Check for test execution status" +find /i /c "BUILD FAILURE" log.txt > NUL +set isfound=!errorlevel! +if !isfound! equ 0 ( + echo [ERROR] Failed run %%a test + exit /b 1 +) else ( + echo [INFO] Success run %%a test +) + +echo [INFO] Dropping schema %SNOWFLAKE_TEST_SCHEMA% +pushd %GITHUB_WORKSPACE%\ci\container +python drop_schema.py +popd