cron-job-its.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "Cron Job ITs"
on:
  schedule: # Runs by default on master branch
    - cron: '0 3 * * *' # Runs every day at 3:00 AM UTC
  pull_request:
    paths:
      - 'owasp-dependency-check-suppressions.xml'
    branches:
      - master
      - '[0-9]+.[0-9]+.[0-9]+' # release branches
      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches

jobs:
  build:
    if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
    name: build (jdk8)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout branch
        uses: actions/checkout@v3

      - name: Setup java
        run: export JAVA_HOME=$JAVA_HOME_8_X64

      - name: Cache Maven m2 repository
        id: maven
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: maven-${{ runner.os }}-8-${{ github.sha }}

      - name: Maven build
        id: maven_build
        run: |
          ./it.sh ci

  integration-index-tests-middleManager:
    strategy:
      fail-fast: false
      matrix:
        testing_group: [batch-index, input-format, input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow, kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format, ldap-security, realtime-index, append-ingestion, compaction]
    uses: ./.github/workflows/reusable-standard-its.yml
    needs: build
    with:
      build_jdk: 8
      runtime_jdk: 11
      testing_groups: -Dgroups=${{ matrix.testing_group }}
      use_indexer: middleManager
      group: ${{ matrix.testing_group }}

  integration-index-tests-indexer:
    strategy:
      fail-fast: false
      matrix:
        testing_group: [ input-source, perfect-rollup-parallel-batch-index, kafka-index, kafka-transactional-index, kafka-index-slow, kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction ]
    uses: ./.github/workflows/reusable-standard-its.yml
    needs: build
    with:
      build_jdk: 8
      runtime_jdk: 11
      testing_groups: -Dgroups=${{ matrix.testing_group }}
      use_indexer: indexer
      group: ${{ matrix.testing_group }}

  integration-query-tests-middleManager:
    strategy:
      fail-fast: false
      matrix:
        testing_group: [ query, query-retry, query-error, security, high-availability ]
    uses: ./.github/workflows/reusable-standard-its.yml
    needs: build
    with:
      build_jdk: 8
      runtime_jdk: 11
      testing_groups: -Dgroups=${{ matrix.testing_group }}
      use_indexer: middleManager
      override_config_path: ./environment-configs/test-groups/prepopulated-data
      group: ${{ matrix.testing_group }}

  integration-other-tests:
    strategy:
      fail-fast: false
      matrix:
        indexer: [ middleManager, indexer ]
    uses: ./.github/workflows/reusable-standard-its.yml
    needs: build
    with:
      build_jdk: 8
      runtime_jdk: 11
      testing_groups: -DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,query-error,realtime-index,security,ldap-security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability,upgrade,shuffle-deep-store,custom-coordinator-duties
      use_indexer: ${{ matrix.indexer }}
      group: other

  security_vulnerabilities:
    if: github.repository == 'apache/druid'
    name: security vulnerabilities
    strategy:
      fail-fast: false
      matrix:
        HADOOP_PROFILE: [ '', '-Phadoop2' ]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout branch
        uses: actions/checkout@v3

      - name: setup java
        uses: actions/setup-java@v3
        with:
          java-version: '8'
          distribution: 'zulu'
          cache: maven

      - name: maven build # needed to rebuild incase of maven snapshot resolution fails
        run: mvn clean install dependency:go-offline -P dist -P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -Dcyclonedx.skip=true -Dweb.console.skip=true

      - name: security vulnerabilities check
        env:
          HADOOP_PROFILE: ${{ matrix.HADOOP_PROFILE }}
        run: |
          mvn dependency-check:purge dependency-check:check ${HADOOP_PROFILE} || { echo "
          The OWASP dependency check has found security vulnerabilities. Please use a newer version
          of the dependency that does not have vulnerabilities. To see a report run
          `mvn dependency-check:check`
          If the analysis has false positives,
          they can be suppressed by adding entries to owasp-dependency-check-suppressions.xml (for more
          information, see https://jeremylong.github.io/DependencyCheck/general/suppression.html).
          " && false; }