ci.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: CI Build
on:  # yamllint disable-line rule:truthy
  schedule:
    - cron: '28 0 * * *'
  push:
    branches: ['master', 'v1-10-test', 'v1-10-stable']
  pull_request:
    branches: ['master', 'v1-10-test', 'v1-10-stable']

env:

  MOUNT_LOCAL_SOURCES: "false"
  MOUNT_FILES: "true"
  FORCE_ANSWER_TO_QUESTIONS: "yes"
  FORCE_PULL_IMAGES: "true"
  CHECK_IMAGE_FOR_REBUILD: "true"
  SKIP_CHECK_REMOTE_IMAGE: "true"
  DB_RESET: "true"
  VERBOSE: "true"
  UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }}
  DOCKER_CACHE: "pulled"
  USE_GITHUB_REGISTRY: "true"
  GITHUB_REPOSITORY: ${{ github.repository }}
  GITHUB_USERNAME: ${{ github.actor }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}"
  GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
  INSTALL_PROVIDERS_FROM_SOURCES: "true"

  # You can switch between building the image in "Build Images" workflow or building them in CI workflow
  # Separately for each job.
  #
  # a) Using images build in the separate "Build Image" workflow:
  #
  #  GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
  #
  # b) Building images in CI workflow - separately for each job:
  #
  #  GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
  #
  # You can also switch back to building images locally and disabling the "Build Images" workflow
  # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false"

  GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}

jobs:

  build-info:
    name: "Build info"
    runs-on: ubuntu-latest
    env:
      GITHUB_CONTEXT: ${{ toJson(github) }}
    outputs:
      waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
      pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
      pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
      defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
      kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
      defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
      kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
      defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
      postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
      defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
      mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
      defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
      helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
      defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
      kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
      defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
      testTypes: ${{ steps.selective-checks.outputs.test-types }}
      postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
      mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
      sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
      kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude }}
      run-tests: ${{ steps.selective-checks.outputs.run-tests }}
      run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
      basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
      image-build: ${{ steps.selective-checks.outputs.image-build }}
      docs-build: ${{ steps.selective-checks.outputs.docs-build }}
      needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
      needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
      pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
      pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
    steps:
      - name: "Get information about the PR"
        uses: potiuk/get-workflow-origin@2ef0b065db6b688a2231f8a7f464df1aac254328  # v1_2
        id: source-run-info
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: >
          Event: ${{ github.event_name }}
          Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
          Branch: ${{ github.head_ref }}
          Run id: ${{ github.run_id }}
          Sha: ${{ github.sha }}
          Ref: ${{ github.ref }}
        run: printenv
      - name: Set wait for image
        id: wait-for-image
        run: |
          if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then
              echo "::set-output name=wait-for-image::true"
          else
              echo "::set-output name=wait-for-image::false"
          fi
      - name: Fetch incoming commit ${{ github.sha }} with its parent
        uses: actions/checkout@v2
        with:
          ref: ${{ github.sha }}
          fetch-depth: 2
        if: github.event_name  == 'pull_request'
      - name: Selective checks
        id: selective-checks
        env:
          EVENT_NAME: ${{ github.event_name }}
          INCOMING_COMMIT_SHA: ${{ github.sha }}
          PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
        run: |
          if [[ ${EVENT_NAME} == "pull_request" ]]; then
            # Run selective checks
            ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
          else
            # Run all checks
            ./scripts/ci/selective_ci_checks.sh
          fi

  test-openapi-client-generation:
    name: "Test OpenAPI client generation"
    runs-on: ubuntu-latest
    needs: [build-info]
    if: >
      needs.build-info.outputs.needs-api-tests == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Generate client codegen diff"
        run: ./scripts/ci/openapi/client_codegen_diff.sh

  ci-images:
    timeout-minutes: 120
    name: "Wait for CI images"
    runs-on: ubuntu-latest
    needs: [build-info]
    if: needs.build-info.outputs.image-build == 'true'
    env:
      BACKEND: sqlite
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
        if: needs.build-info.outputs.waitForImage == 'true'
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
        if: needs.build-info.outputs.waitForImage == 'true'
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
        if: needs.build-info.outputs.waitForImage == 'true'
      - name: >
          Wait for CI images
          ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        env:
          CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
            ${{needs.build-info.outputs.pythonVersionsListAsString}}
        # We wait for the images to be available either from the build-ci-image step or from
        # "build-images-workflow-run.yml' run as pull_request_target (it has the write
        # permissions in case pull_request from fork is run.
        # We are utilising single job to wait for all images because this job merely waits
        # For the images to be available. The test jobs wait for it to complete!
        run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh
        if: needs.build-info.outputs.waitForImage == 'true'

  static-checks:
    timeout-minutes: 30
    name: "Static checks"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    env:
      SKIP: "pylint"
      MOUNT_LOCAL_SOURCES: "true"
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
    if: needs.build-info.outputs.basic-checks-only == 'false'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
      - name: Cache pre-commit env
        uses: actions/cache@v2
        env:
          cache-name: cache-pre-commit-master-no-pylint-v1
        with:
          path: ~/.cache/pre-commit
          key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Static checks: except pylint"
        run: ./scripts/ci/static_checks/run_static_checks.sh
        env:
          VERBOSE: false

  # Those checks are run if no image needs to be built for checks. This is for simple changes that
  # Do not touch any of the python code or any of the important files that might require building
  # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
  static-checks-basic-checks-only:
    timeout-minutes: 30
    name: "Static checks: basic checks only"
    runs-on: ubuntu-latest
    needs: [build-info]
    env:
      SKIP: "build,mypy,flake8,pylint,bats-in-container-tests"
      MOUNT_LOCAL_SOURCES: "true"
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
    if: needs.build-info.outputs.basic-checks-only == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
      - name: Cache pre-commit env
        uses: actions/cache@v2
        env:
          cache-name: cache-pre-commit-master-basic-checks-v1
        with:
          path: ~/.cache/pre-commit
          key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
      - name: >
          Fetch incoming commit ${{ github.sha }} with its parent
        uses: actions/checkout@v2
        with:
          ref: ${{ github.sha }}
          fetch-depth: 2
      - name: "Static checks: basic checks only"
        run: |
          ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}"
        env:
          VERBOSE: false


  static-checks-pylint:
    timeout-minutes: 30
    name: "Pylint"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    if: needs.build-info.outputs.basic-checks-only == 'false'
    env:
      # We want to make sure we have latest sources as only in_container scripts are added
      # to the image but we want to static-check all of them
      MOUNT_LOCAL_SOURCES: "true"
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
      - name: "Cache pre-commit env"
        uses: actions/cache@v2
        env:
          cache-name: cache-pre-commit-master-pylint-v1
        with:
          path: ~/.cache/pre-commit
          key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Static checks: pylint"
        run: ./scripts/ci/static_checks/run_static_checks.sh pylint
        env:
          VERBOSE: false

  docs:
    timeout-minutes: 30
    name: "Build docs"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    if: needs.build-info.outputs.docs-build == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Build docs"
        run: ./scripts/ci/docs/ci_docs.sh --docs-only
      - name: "Upload documentation"
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: airflow-documentation
          path: "./files/documentation"

  docs-spell-check:
    timeout-minutes: 30
    name: "Spell check docs"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    env:
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
    if: needs.build-info.outputs.docs-build == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Spell check docs"
        run: ./scripts/ci/docs/ci_docs.sh --spellcheck-only

  prepare-backport-provider-packages:
    timeout-minutes: 30
    name: "Backport packages"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    env:
      INSTALL_AIRFLOW_VERSION: "1.10.12"
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
      BACKPORT_PACKAGES: "true"
      VERSION_SUFFIX_FOR_SVN: "rc1"
    if: needs.build-info.outputs.image-build == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Prepare & test provider packages"
        run: ./scripts/ci/provider_packages/ci_prepare_and_test_provider_packages.sh
      - name: "Upload provider package artifacts"
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: airflow-backport-packages
          path: "./files/airflow-packages-*"
      - name: "Upload readme artifacts"
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: airflow-backport-readmes
          path: "./files/airflow-readme-*"

  prepare-provider-packages:
    timeout-minutes: 30
    name: "Provider packages"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    env:
      INSTALL_AIRFLOW_VERSION: "2.0.0-dev"   # Note that this causes local installation
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
      VERSION_SUFFIX_FOR_PYPI: "a0"
      VERSION_SUFFIX_FOR_SVN: "a0"
    if: needs.build-info.outputs.image-build == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Prepare & test provider packages"
        run: ./scripts/ci/provider_packages/ci_prepare_and_test_provider_packages.sh
      - name: "Upload provider package artifacts"
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: airflow-provider-packages
          path: "./files/airflow-packages-*"
      - name: "Upload readme artifacts"
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: airflow-provider-readmes
          path: "./files/airflow-readme-*"

  tests-helm:
    timeout-minutes: 5
    name: "Python unit tests for helm chart"
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    env:
      MOUNT_LOCAL_SOURCES: "true"
      RUN_TESTS: true
      TEST_TYPES: "Helm"
      BACKEND: "sqlite"
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
    if: >
      needs.build-info.outputs.needs-helm-tests == 'true' &&
      (github.repository == 'apache/airflow' || github.event_name != 'schedule')
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
      - name: "Upload airflow logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: airflow-logs-helm
          path: "./files/airflow_logs*"
      - name: "Upload container logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: container-logs-helm
          path: "./files/container_logs*"
      - name: "Upload artifact for coverage"
        uses: actions/upload-artifact@v2
        with:
          name: >
            coverage-helm
          path: "./files/coverage.xml"

  tests-postgres:
    timeout-minutes: 80
    name: >
      Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
      ${{needs.build-info.outputs.testTypes}}
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
        postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
        exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
      fail-fast: false
    env:
      BACKEND: postgres
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
      POSTGRES_VERSION: ${{ matrix.postgres-version }}
      RUN_TESTS: true
      TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
      TEST_TYPE: ""
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
      - name: "Upload airflow logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}}
          path: "./files/airflow_logs*"
      - name: "Upload container logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
          path: "./files/container_logs*"
      - name: "Upload artifact for coverage"
        uses: actions/upload-artifact@v2
        with:
          name: >
            coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
          path: "./files/coverage.xml"

  tests-mysql:
    timeout-minutes: 80
    name: >
      MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
        mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
        exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
      fail-fast: false
    env:
      BACKEND: mysql
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
      MYSQL_VERSION: ${{ matrix.mysql-version }}
      RUN_TESTS: true
      TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
      TEST_TYPE: ""
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
      - name: "Upload airflow logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}}
          path: "./files/airflow_logs*"
      - name: "Upload container logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
          path: "./files/container_logs*"
      - name: "Upload artifact for coverage"
        uses: actions/upload-artifact@v2
        with:
          name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
          path: "./files/coverage.xml"

  tests-sqlite:
    timeout-minutes: 60
    name: >
      Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
    runs-on: ubuntu-latest
    needs: [build-info, ci-images]
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
        exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }}
      fail-fast: false
    env:
      BACKEND: sqlite
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
      RUN_TESTS: true
      TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
      TEST_TYPE: ""
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
      - name: "Upload airflow logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: airflow-logs-${{matrix.python-version}}
          path: './files/airflow_logs*'
      - name: "Upload container logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: container-logs-sqlite-${{matrix.python-version}}
          path: "./files/container_logs*"
      - name: "Upload artifact for coverage"
        uses: actions/upload-artifact@v2
        with:
          name: coverage-sqlite-${{matrix.python-version}}
          path: ./files/coverage.xml

  tests-quarantined:
    timeout-minutes: 60
    name: "Quarantined tests"
    runs-on: ubuntu-latest
    continue-on-error: true
    needs: [build-info, ci-images]
    strategy:
      matrix:
        include:
          - backend: mysql
          - backend: postgres
          - backend: sqlite
    env:
      BACKEND: ${{ matrix.backend }}
      PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
      MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
      POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
      RUN_TESTS: true
      TEST_TYPES: "Quarantined"
      TEST_TYPE: ""
      NUM_RUNS: 10
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    if: needs.build-info.outputs.run-tests == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Set issue id for master"
        if: github.ref == 'refs/heads/master'
        run: |
          echo "ISSUE_ID=10118" >> $GITHUB_ENV
      - name: "Set issue id for v1-10-stable"
        if: github.ref == 'refs/heads/v1-10-stable'
        run: |
          echo "ISSUE_ID=10127" >> $GITHUB_ENV
      - name: "Set issue id for v1-10-test"
        if: github.ref == 'refs/heads/v1-10-test'
        run: |
          echo "ISSUE_ID=10128" >> $GITHUB_ENV
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Tests: Quarantined"
        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
      - name: "Upload Quarantine test results"
        uses: actions/upload-artifact@v2
        if: always()
        with:
          name: quarantined_tests
          path: "files/test_result.xml"
      - name: "Upload airflow logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: airflow-logs-quarantined-${{ matrix.backend }}
          path: "./files/airflow_logs*"
      - name: "Upload container logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: container-logs-quarantined-${{ matrix.backend }}
          path: "./files/container_logs*"
      - name: "Upload artifact for coverage"
        uses: actions/upload-artifact@v2
        with:
          name: coverage-quarantined-${{ matrix.backend }}
          path: "./files/coverage.xml"

  upload-coverage:
    timeout-minutes: 5
    name: "Upload coverage"
    runs-on: ubuntu-latest
    continue-on-error: true
    needs:
      - tests-kubernetes
      - tests-postgres
      - tests-sqlite
      - tests-mysql
      - tests-quarantined
    steps:
      - name: "Download all artifacts from the current build"
        uses: actions/download-artifact@v2
        with:
          path: ./coverage-files
      - name: "Removes unnecessary artifacts"
        run: ls ./coverage-files | grep -v coverage | xargs rm -rf
      - name: "Upload all coverage reports to codecov"
        uses: codecov/codecov-action@v1
        with:
          directory: "./coverage-files"

  prod-images:
    timeout-minutes: 120
    name: "Wait for PROD images"
    runs-on: ubuntu-latest
    needs: [build-info]
    env:
      BACKEND: sqlite
      PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
    if: needs.build-info.outputs.image-build == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
        if: needs.build-info.outputs.waitForImage == 'true'
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
        if: needs.build-info.outputs.waitForImage == 'true'
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
        if: needs.build-info.outputs.waitForImage == 'true'
      - name: >
          Wait for PROD images
          ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        env:
          CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
            ${{needs.build-info.outputs.pythonVersionsListAsString}}
        run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh
        if: needs.build-info.outputs.waitForImage == 'true'

  tests-kubernetes:
    timeout-minutes: 50
    name: K8s ${{matrix.python-version}} ${{matrix.kubernetes-version}} ${{matrix.kubernetes-mode}}
    runs-on: ubuntu-latest
    needs: [build-info, prod-images]
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
        kubernetes-version: ${{ fromJson(needs.build-info.outputs.kubernetesVersions) }}
        kubernetes-mode: ${{ fromJson(needs.build-info.outputs.kubernetesModes) }}
        exclude: ${{ fromJson(needs.build-info.outputs.kubernetesExclude) }}
      fail-fast: false
    env:
      BACKEND: postgres
      RUN_TESTS: "true"
      RUNTIME: "kubernetes"
      ENABLE_KIND_CLUSTER: "true"
      PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
      KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}"
      KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
      KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
      HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
    if: needs.build-info.outputs.run-kubernetes-tests == 'true'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Setup Kind Cluster ${{ env.KIND_VERSION }}"
        uses: engineerd/setup-kind@v0.4.0
        with:
          version: "${{ env.KIND_VERSION }}"
          name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}}
          config: "scripts/ci/kubernetes/kind-cluster-conf.yaml"
      - name: "Prepare PROD Image"
        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
      - name: "Deploy airflow to cluster"
        id: deploy-app
        run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
        env:
          # We have the right image pulled already by the previous step
          SKIP_BUILDING_PROD_IMAGE: "true"
          # due to some instabilities, in CI we try to increase port numbers when trying to establish
          # port forwarding
          INCREASE_PORT_NUMBER_FOR_KUBERNETES: "true"
      - name: "Cache virtualenv for kubernetes testing"
        uses: actions/cache@v2
        env:
          cache-name: cache-kubernetes-tests-virtualenv-v6
        with:
          path: .build/.kubernetes_venv
          key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}"
      - name: "Cache bin folder with tools for kubernetes testing"
        uses: actions/cache@v2
        env:
          cache-name: cache-kubernetes-tests-bin-v6
        with:
          path: .build/bin
          key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}\
-${{ needs.build-info.outputs.defaultKindVersion }}\
-${{ needs.build-info.outputs.defaultHelmVersion }}\
-$${{ matrix.kubernetes-version }}"
      - name: "Kubernetes Tests"
        run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
      - name: "Upload KinD logs"
        uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: >
            kind-logs-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}
          path: /tmp/kind_logs_*
      - name: "Upload artifact for coverage"
        uses: actions/upload-artifact@v2
        with:
          name: >
            coverage-k8s-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}
          path: "./files/coverage.xml"

  push-prod-images-to-github-registry:
    timeout-minutes: 10
    name: "Push PROD images"
    runs-on: ubuntu-latest
    needs:
      - build-info
      - static-checks
      - static-checks-pylint
      - tests-sqlite
      - tests-postgres
      - tests-mysql
      - tests-kubernetes
      - prepare-backport-provider-packages
      - prepare-provider-packages
      - prod-images
      - docs
      - docs-spell-check
    if: >
      (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test') &&
      github.event_name != 'schedule'
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
      GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
      - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_push_production_images.sh

  push-ci-images-to-github-registry:
    timeout-minutes: 10
    name: "Push CI images"
    runs-on: ubuntu-latest
    needs:
      - build-info
      - static-checks
      - static-checks-pylint
      - tests-sqlite
      - tests-postgres
      - tests-mysql
      - tests-kubernetes
      - prepare-backport-provider-packages
      - ci-images
      - docs
      - docs-spell-check
    if: >
      (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
      github.event_name != 'schedule'
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
    env:
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
      GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
        run: ./scripts/ci/images/ci_push_ci_images.sh

  constraints:
    timeout-minutes: 10
    name: "Constraints"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
      fail-fast: false
    needs:
      - build-info
      - ci-images
    env:
      PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Setup python"
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ github.sha }}"
        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
      - name: "Generate constraints"
        run: ./scripts/ci/constraints/ci_generate_constraints.sh
      - name: "Upload constraint artifacts"
        uses: actions/upload-artifact@v2
        with:
          name: 'constraints-${{matrix.python-version}}'
          path: './files/constraints-${{matrix.python-version}}/constraints-${{matrix.python-version}}.txt'

  constraints-push:
    timeout-minutes: 10
    name: "Constraints push"
    runs-on: ubuntu-latest
    needs:
      - build-info
      - constraints
      - static-checks
      - static-checks-pylint
      - tests-sqlite
      - tests-mysql
      - tests-postgres
      - tests-kubernetes
    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Set constraints branch name"
        id: constraints-branch
        run: ./scripts/ci/constraints/ci_branch_constraints.sh
      - name: Checkout ${{ steps.constraints-branch.outputs.branch }}
        uses: actions/checkout@v2
        with:
          path: "repo"
          ref: ${{ steps.constraints-branch.outputs.branch }}
      - name: "Get all artifacts (constraints)"
        uses: actions/download-artifact@v2
        with:
          path: 'artifacts'
      - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
        run: ./scripts/ci/constraints/ci_commit_constraints.sh
      - name: "Push changes"
        uses: ad-m/github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6  # v0.6.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ steps.constraints-branch.outputs.branch }}
          directory: "repo"

  tag-repo-nightly:
    timeout-minutes: 10
    name: "Tag repo nightly"
    runs-on: ubuntu-latest
    needs:
      - docs
      - docs-spell-check
      - static-checks
      - static-checks-pylint
      - tests-sqlite
      - tests-postgres
      - tests-mysql
      - tests-kubernetes
      - constraints-push
      - prepare-provider-packages
    if: github.event_name == 'schedule' &&  github.repository == 'apache/airflow'
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@v2
      - name: "Free space"
        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
      - name: "Tag commit"
        run: |
          BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///')
          echo "Tagging ${BRANCH_NAME}"
          git tag -f nightly-${BRANCH_NAME} HEAD
      - name: "Push tags"
        uses: ad-m/github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6  # v0.6.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          tags: true
          force: true
          branch: master