Skip to content

[pre-commit.ci] pre-commit autoupdate #104

[pre-commit.ci] pre-commit autoupdate

[pre-commit.ci] pre-commit autoupdate #104

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: CI Build
on: # yamllint disable-line rule:truthy
schedule:
- cron: '28 0 * * *'
push:
branches: ['master', 'v1-10-test', 'v1-10-stable']
pull_request:
branches: ['master', 'v1-10-test', 'v1-10-stable']
env:
MOUNT_LOCAL_SOURCES: "false"
MOUNT_FILES: "true"
FORCE_ANSWER_TO_QUESTIONS: "yes"
FORCE_PULL_IMAGES: "true"
CHECK_IMAGE_FOR_REBUILD: "true"
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
VERBOSE: "true"
UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }}
DOCKER_CACHE: "pulled"
USE_GITHUB_REGISTRY: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}"
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
INSTALL_PROVIDERS_FROM_SOURCES: "true"
# You can switch between building the image in "Build Images" workflow or building them in CI workflow
# Separately for each job.
#
# a) Using images build in the separate "Build Image" workflow:
#
# GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true"
#
# b) Building images in CI workflow - separately for each job:
#
# GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
#
# You can also switch back to building images locally and disabling the "Build Images" workflow
# by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false"
GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
jobs:
build-info:
name: "Build info"
runs-on: ubuntu-latest
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
testTypes: ${{ steps.selective-checks.outputs.test-types }}
postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
image-build: ${{ steps.selective-checks.outputs.image-build }}
docs-build: ${{ steps.selective-checks.outputs.docs-build }}
needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
steps:
- name: "Get information about the PR"
uses: potiuk/get-workflow-origin@2ef0b065db6b688a2231f8a7f464df1aac254328 # v1_2
id: source-run-info
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: >
Event: ${{ github.event_name }}
Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
Branch: ${{ github.head_ref }}
Run id: ${{ github.run_id }}
Sha: ${{ github.sha }}
Ref: ${{ github.ref }}
run: printenv
- name: Set wait for image
id: wait-for-image
run: |
if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then
echo "::set-output name=wait-for-image::true"
else
echo "::set-output name=wait-for-image::false"
fi
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v2
with:
ref: ${{ github.sha }}
fetch-depth: 2
if: github.event_name == 'pull_request'
- name: Selective checks
id: selective-checks
env:
EVENT_NAME: ${{ github.event_name }}
INCOMING_COMMIT_SHA: ${{ github.sha }}
PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
run: |
if [[ ${EVENT_NAME} == "pull_request" ]]; then
# Run selective checks
./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
else
# Run all checks
./scripts/ci/selective_ci_checks.sh
fi
test-openapi-client-generation:
name: "Test OpenAPI client generation"
runs-on: ubuntu-latest
needs: [build-info]
if: >
needs.build-info.outputs.needs-api-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Generate client codegen diff"
run: ./scripts/ci/openapi/client_codegen_diff.sh
ci-images:
timeout-minutes: 120
name: "Wait for CI images"
runs-on: ubuntu-latest
needs: [build-info]
if: needs.build-info.outputs.image-build == 'true'
env:
BACKEND: sqlite
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
if: needs.build-info.outputs.waitForImage == 'true'
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.waitForImage == 'true'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
if: needs.build-info.outputs.waitForImage == 'true'
- name: >
Wait for CI images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
# We wait for the images to be available either from the build-ci-image step or from
# "build-images-workflow-run.yml' run as pull_request_target (it has the write
# permissions in case pull_request from fork is run.
# We are utilising single job to wait for all images because this job merely waits
# For the images to be available. The test jobs wait for it to complete!
run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh
if: needs.build-info.outputs.waitForImage == 'true'
static-checks:
timeout-minutes: 30
name: "Static checks"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
env:
SKIP: "pylint"
MOUNT_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- name: Cache pre-commit env
uses: actions/cache@v2
env:
cache-name: cache-pre-commit-master-no-pylint-v1
with:
path: ~/.cache/pre-commit
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Static checks: except pylint"
run: ./scripts/ci/static_checks/run_static_checks.sh
env:
VERBOSE: false
# Those checks are run if no image needs to be built for checks. This is for simple changes that
# Do not touch any of the python code or any of the important files that might require building
# The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
runs-on: ubuntu-latest
needs: [build-info]
env:
SKIP: "build,mypy,flake8,pylint,bats-in-container-tests"
MOUNT_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- name: Cache pre-commit env
uses: actions/cache@v2
env:
cache-name: cache-pre-commit-master-basic-checks-v1
with:
path: ~/.cache/pre-commit
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
- name: >
Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v2
with:
ref: ${{ github.sha }}
fetch-depth: 2
- name: "Static checks: basic checks only"
run: |
./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}"
env:
VERBOSE: false
static-checks-pylint:
timeout-minutes: 30
name: "Pylint"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
if: needs.build-info.outputs.basic-checks-only == 'false'
env:
# We want to make sure we have latest sources as only in_container scripts are added
# to the image but we want to static-check all of them
MOUNT_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- name: "Cache pre-commit env"
uses: actions/cache@v2
env:
cache-name: cache-pre-commit-master-pylint-v1
with:
path: ~/.cache/pre-commit
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Static checks: pylint"
run: ./scripts/ci/static_checks/run_static_checks.sh pylint
env:
VERBOSE: false
docs:
timeout-minutes: 30
name: "Build docs"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
if: needs.build-info.outputs.docs-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Build docs"
run: ./scripts/ci/docs/ci_docs.sh --docs-only
- name: "Upload documentation"
uses: actions/upload-artifact@v2
if: always()
with:
name: airflow-documentation
path: "./files/documentation"
docs-spell-check:
timeout-minutes: 30
name: "Spell check docs"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
env:
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.docs-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Spell check docs"
run: ./scripts/ci/docs/ci_docs.sh --spellcheck-only
prepare-backport-provider-packages:
timeout-minutes: 30
name: "Backport packages"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
env:
INSTALL_AIRFLOW_VERSION: "1.10.12"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
BACKPORT_PACKAGES: "true"
VERSION_SUFFIX_FOR_SVN: "rc1"
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare & test provider packages"
run: ./scripts/ci/provider_packages/ci_prepare_and_test_provider_packages.sh
- name: "Upload provider package artifacts"
uses: actions/upload-artifact@v2
if: always()
with:
name: airflow-backport-packages
path: "./files/airflow-packages-*"
- name: "Upload readme artifacts"
uses: actions/upload-artifact@v2
if: always()
with:
name: airflow-backport-readmes
path: "./files/airflow-readme-*"
prepare-provider-packages:
timeout-minutes: 30
name: "Provider packages"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
env:
INSTALL_AIRFLOW_VERSION: "2.0.0-dev" # Note that this causes local installation
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
VERSION_SUFFIX_FOR_PYPI: "a0"
VERSION_SUFFIX_FOR_SVN: "a0"
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare & test provider packages"
run: ./scripts/ci/provider_packages/ci_prepare_and_test_provider_packages.sh
- name: "Upload provider package artifacts"
uses: actions/upload-artifact@v2
if: always()
with:
name: airflow-provider-packages
path: "./files/airflow-packages-*"
- name: "Upload readme artifacts"
uses: actions/upload-artifact@v2
if: always()
with:
name: airflow-provider-readmes
path: "./files/airflow-readme-*"
tests-helm:
timeout-minutes: 5
name: "Python unit tests for helm chart"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
env:
MOUNT_LOCAL_SOURCES: "true"
RUN_TESTS: true
TEST_TYPES: "Helm"
BACKEND: "sqlite"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: airflow-logs-helm
path: "./files/airflow_logs*"
- name: "Upload container logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: container-logs-helm
path: "./files/container_logs*"
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: >
coverage-helm
path: "./files/coverage.xml"
tests-postgres:
timeout-minutes: 80
name: >
Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
${{needs.build-info.outputs.testTypes}}
runs-on: ubuntu-latest
needs: [build-info, ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
fail-fast: false
env:
BACKEND: postgres
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
POSTGRES_VERSION: ${{ matrix.postgres-version }}
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}}
path: "./files/airflow_logs*"
- name: "Upload container logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
path: "./files/container_logs*"
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: >
coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
path: "./files/coverage.xml"
tests-mysql:
timeout-minutes: 80
name: >
MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
runs-on: ubuntu-latest
needs: [build-info, ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
fail-fast: false
env:
BACKEND: mysql
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
MYSQL_VERSION: ${{ matrix.mysql-version }}
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}}
path: "./files/airflow_logs*"
- name: "Upload container logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
path: "./files/container_logs*"
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
path: "./files/coverage.xml"
tests-sqlite:
timeout-minutes: 60
name: >
Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
runs-on: ubuntu-latest
needs: [build-info, ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }}
fail-fast: false
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: airflow-logs-${{matrix.python-version}}
path: './files/airflow_logs*'
- name: "Upload container logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: container-logs-sqlite-${{matrix.python-version}}
path: "./files/container_logs*"
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: coverage-sqlite-${{matrix.python-version}}
path: ./files/coverage.xml
tests-quarantined:
timeout-minutes: 60
name: "Quarantined tests"
runs-on: ubuntu-latest
continue-on-error: true
needs: [build-info, ci-images]
strategy:
matrix:
include:
- backend: mysql
- backend: postgres
- backend: sqlite
env:
BACKEND: ${{ matrix.backend }}
PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
RUN_TESTS: true
TEST_TYPES: "Quarantined"
TEST_TYPE: ""
NUM_RUNS: 10
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Set issue id for master"
if: github.ref == 'refs/heads/master'
run: |
echo "ISSUE_ID=10118" >> $GITHUB_ENV
- name: "Set issue id for v1-10-stable"
if: github.ref == 'refs/heads/v1-10-stable'
run: |
echo "ISSUE_ID=10127" >> $GITHUB_ENV
- name: "Set issue id for v1-10-test"
if: github.ref == 'refs/heads/v1-10-test'
run: |
echo "ISSUE_ID=10128" >> $GITHUB_ENV
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: Quarantined"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- name: "Upload Quarantine test results"
uses: actions/upload-artifact@v2
if: always()
with:
name: quarantined_tests
path: "files/test_result.xml"
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: airflow-logs-quarantined-${{ matrix.backend }}
path: "./files/airflow_logs*"
- name: "Upload container logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: container-logs-quarantined-${{ matrix.backend }}
path: "./files/container_logs*"
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: coverage-quarantined-${{ matrix.backend }}
path: "./files/coverage.xml"
upload-coverage:
timeout-minutes: 5
name: "Upload coverage"
runs-on: ubuntu-latest
continue-on-error: true
needs:
- tests-kubernetes
- tests-postgres
- tests-sqlite
- tests-mysql
- tests-quarantined
steps:
- name: "Download all artifacts from the current build"
uses: actions/download-artifact@v2
with:
path: ./coverage-files
- name: "Removes unnecessary artifacts"
run: ls ./coverage-files | grep -v coverage | xargs rm -rf
- name: "Upload all coverage reports to codecov"
uses: codecov/codecov-action@v1
with:
directory: "./coverage-files"
prod-images:
timeout-minutes: 120
name: "Wait for PROD images"
runs-on: ubuntu-latest
needs: [build-info]
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
if: needs.build-info.outputs.waitForImage == 'true'
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
if: needs.build-info.outputs.waitForImage == 'true'
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
if: needs.build-info.outputs.waitForImage == 'true'
- name: >
Wait for PROD images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh
if: needs.build-info.outputs.waitForImage == 'true'
tests-kubernetes:
timeout-minutes: 50
name: K8s ${{matrix.python-version}} ${{matrix.kubernetes-version}} ${{matrix.kubernetes-mode}}
runs-on: ubuntu-latest
needs: [build-info, prod-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
kubernetes-version: ${{ fromJson(needs.build-info.outputs.kubernetesVersions) }}
kubernetes-mode: ${{ fromJson(needs.build-info.outputs.kubernetesModes) }}
exclude: ${{ fromJson(needs.build-info.outputs.kubernetesExclude) }}
fail-fast: false
env:
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
ENABLE_KIND_CLUSTER: "true"
PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}"
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
if: needs.build-info.outputs.run-kubernetes-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Setup Kind Cluster ${{ env.KIND_VERSION }}"
uses: engineerd/[email protected]
with:
version: "${{ env.KIND_VERSION }}"
name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}}
config: "scripts/ci/kubernetes/kind-cluster-conf.yaml"
- name: "Prepare PROD Image"
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- name: "Deploy airflow to cluster"
id: deploy-app
run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
env:
# We have the right image pulled already by the previous step
SKIP_BUILDING_PROD_IMAGE: "true"
# due to some instabilities, in CI we try to increase port numbers when trying to establish
# port forwarding
INCREASE_PORT_NUMBER_FOR_KUBERNETES: "true"
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v2
env:
cache-name: cache-kubernetes-tests-virtualenv-v6
with:
path: .build/.kubernetes_venv
key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}"
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v2
env:
cache-name: cache-kubernetes-tests-bin-v6
with:
path: .build/bin
key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}\
-${{ needs.build-info.outputs.defaultKindVersion }}\
-${{ needs.build-info.outputs.defaultHelmVersion }}\
-$${{ matrix.kubernetes-version }}"
- name: "Kubernetes Tests"
run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
- name: "Upload KinD logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: >
kind-logs-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}
path: /tmp/kind_logs_*
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: >
coverage-k8s-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}
path: "./files/coverage.xml"
push-prod-images-to-github-registry:
timeout-minutes: 10
name: "Push PROD images"
runs-on: ubuntu-latest
needs:
- build-info
- static-checks
- static-checks-pylint
- tests-sqlite
- tests-postgres
- tests-mysql
- tests-kubernetes
- prepare-backport-provider-packages
- prepare-provider-packages
- prod-images
- docs
- docs-spell-check
if: >
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test') &&
github.event_name != 'schedule'
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_push_production_images.sh
push-ci-images-to-github-registry:
timeout-minutes: 10
name: "Push CI images"
runs-on: ubuntu-latest
needs:
- build-info
- static-checks
- static-checks-pylint
- tests-sqlite
- tests-postgres
- tests-mysql
- tests-kubernetes
- prepare-backport-provider-packages
- ci-images
- docs
- docs-spell-check
if: >
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
github.event_name != 'schedule'
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_push_ci_images.sh
constraints:
timeout-minutes: 10
name: "Constraints"
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
fail-fast: false
needs:
- build-info
- ci-images
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ github.sha }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Generate constraints"
run: ./scripts/ci/constraints/ci_generate_constraints.sh
- name: "Upload constraint artifacts"
uses: actions/upload-artifact@v2
with:
name: 'constraints-${{matrix.python-version}}'
path: './files/constraints-${{matrix.python-version}}/constraints-${{matrix.python-version}}.txt'
constraints-push:
timeout-minutes: 10
name: "Constraints push"
runs-on: ubuntu-latest
needs:
- build-info
- constraints
- static-checks
- static-checks-pylint
- tests-sqlite
- tests-mysql
- tests-postgres
- tests-kubernetes
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
uses: actions/checkout@v2
with:
path: "repo"
ref: ${{ steps.constraints-branch.outputs.branch }}
- name: "Get all artifacts (constraints)"
uses: actions/download-artifact@v2
with:
path: 'artifacts'
- name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
- name: "Push changes"
uses: ad-m/github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6 # v0.6.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.constraints-branch.outputs.branch }}
directory: "repo"
tag-repo-nightly:
timeout-minutes: 10
name: "Tag repo nightly"
runs-on: ubuntu-latest
needs:
- docs
- docs-spell-check
- static-checks
- static-checks-pylint
- tests-sqlite
- tests-postgres
- tests-mysql
- tests-kubernetes
- constraints-push
- prepare-provider-packages
if: github.event_name == 'schedule' && github.repository == 'apache/airflow'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Tag commit"
run: |
BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///')
echo "Tagging ${BRANCH_NAME}"
git tag -f nightly-${BRANCH_NAME} HEAD
- name: "Push tags"
uses: ad-m/github-push-action@40bf560936a8022e68a3c00e7d2abefaf01305a6 # v0.6.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
tags: true
force: true
branch: master