From 197b7f46328043e96e09385218d57f88566d7ae7 Mon Sep 17 00:00:00 2001 From: Martin Yushko <139386268+mayushko26@users.noreply.github.com> Date: Mon, 26 Feb 2024 12:17:12 -0800 Subject: [PATCH] support for version 2.8.1 (#362) support for version 2.8.1 (#362) --- VERSION | 2 +- docker/Dockerfile | 10 +- docker/config/airflow.cfg | 4 +- docker/config/constraints.txt | 854 +++++++++--------- .../mwaa-base-providers-requirements.txt | 16 +- docker/docker-compose-local.yml | 4 +- docker/docker-compose-resetdb.yml | 4 +- docker/docker-compose-sequential.yml | 2 +- docker/script/verification.sh | 4 +- mwaa-local-env | 2 +- plugins/README.md | 2 +- requirements/requirements.txt | 6 +- 12 files changed, 476 insertions(+), 434 deletions(-) diff --git a/VERSION b/VERSION index 37c2961c2..dbe590065 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.7.2 +2.8.1 diff --git a/docker/Dockerfile b/docker/Dockerfile index a0d91798a..240efd2f6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,9 +8,9 @@ LABEL maintainer="amazon" # Airflow ## Version specific ARGs -ARG AIRFLOW_VERSION=2.7.2 -ARG WATCHTOWER_VERSION=2.0.1 -ARG PROVIDER_AMAZON_VERSION=8.7.1 +ARG AIRFLOW_VERSION=2.8.1 +ARG WATCHTOWER_VERSION=3.0.1 +ARG PROVIDER_AMAZON_VERSION=8.16.0 ## General ARGs ARG AIRFLOW_USER_HOME=/usr/local/airflow @@ -19,8 +19,8 @@ ARG PYTHON_DEPS="" ARG SYSTEM_DEPS="" ARG INDEX_URL="" ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} -ENV PATH="$PATH:/usr/local/airflow/.local/bin:/root/.local/bin:/usr/local/airflow/.local/lib/python3.10/site-packages" -ENV PYTHON_VERSION=3.11.6 +ENV PATH="/usr/local/airflow/.local/bin:/root/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/airflow/.local/lib/python3.11/site-packages:/usr/local/airflow/.local/bin:/usr/local/airflow/.local/lib/python3.11/site-packages:/usr/local/airflow/.local/bin:/usr/local/airflow/.local/lib/python3.11/site-packages" +ENV PYTHON_VERSION=3.11.7 COPY script/bootstrap.sh /bootstrap.sh COPY script/systemlibs.sh /systemlibs.sh diff --git a/docker/config/airflow.cfg b/docker/config/airflow.cfg index 9b3ceff7d..e5a0db248 100644 --- a/docker/config/airflow.cfg +++ b/docker/config/airflow.cfg @@ -276,7 +276,7 @@ simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s task_log_prefix_template = # Formatting for how airflow generates file names/paths for each task run. -log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log +log_filename_template = dag_id={{ti.dag_id}}/run_id={{ti.run_id}}/task_id={{ti.task_id}}/{%% if ti.map_index >= 0 %%}map_index={{ti.map_index}}/{%% endif %%}attempt={{try_number}}.log # Formatting for how airflow generates file names for log log_processor_filename_template = {{ filename }}.log @@ -628,7 +628,7 @@ kubernetes_queue = kubernetes # This section only applies if you are using the CeleryExecutor in # ``[core]`` section above # The app name that will be used by celery -celery_app_name = airflow.executors.celery_executor +celery_app_name = airflow.providers.celery.executors.celery_executor # The concurrency that will be used when starting workers with the # ``airflow celery worker`` command. This defines the number of task instances that diff --git a/docker/config/constraints.txt b/docker/config/constraints.txt index c94c01a99..47a5fba08 100644 --- a/docker/config/constraints.txt +++ b/docker/config/constraints.txt @@ -1,7 +1,7 @@ -# https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.11.txt +# https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.11.txt # -# This constraints file was automatically generated on 2023-10-08T09:54:23Z -# via "eager-upgrade" mechanism of PIP. For the "v2-7-test" branch of Airflow. +# This constraints file was automatically generated on 2024-01-16T07:06:10.475440 +# via "eager-upgrade" mechanism of PIP. For the "v2-8-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. # @@ -22,223 +22,227 @@ # 1. Reproducible installation of airflow with selected providers (note constraints are used): # # pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ -# --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.8.txt" +# --constraint \ +# "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.11.txt" # # 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not # used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. # -# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]==2.9.0" +# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" # -Authlib==1.2.1 -Babel==2.13.0 -ConfigUpdater==3.1.1 +Authlib==1.3.0 +Babel==2.14.0 +ConfigUpdater==3.2 Deprecated==1.2.14 -Flask-AppBuilder==4.3.6 +Flask-AppBuilder==4.3.10 Flask-Babel==2.0.0 Flask-Bcrypt==1.0.1 -Flask-Caching==2.0.2 -Flask-JWT-Extended==4.5.3 +Flask-Caching==2.1.0 +Flask-JWT-Extended==4.6.0 Flask-Limiter==3.5.0 -Flask-Login==0.6.2 +Flask-Login==0.6.3 Flask-SQLAlchemy==2.5.1 Flask-Session==0.5.0 Flask-WTF==1.2.1 Flask==2.2.5 -GitPython==3.1.37 -JPype1==1.4.1 +GitPython==3.1.41 +JPype1==1.5.0 JayDeBeApi==1.2.3 -Jinja2==3.1.2 -Mako==1.2.4 -Markdown==3.5 +Jinja2==3.1.3 +Js2Py==0.74 +Mako==1.3.0 +Markdown==3.5.2 MarkupSafe==2.1.3 PyGithub==2.1.1 PyHive==0.7.0 PyJWT==2.8.0 PyNaCl==1.5.0 PyYAML==6.0.1 -Pygments==2.16.1 -SQLAlchemy-JSONField==1.0.1.post0 +Pygments==2.17.2 +SQLAlchemy-JSONField==1.0.2 SQLAlchemy-Utils==0.41.1 -SQLAlchemy==1.4.49 +SQLAlchemy==1.4.51 SecretStorage==3.3.3 -Shapely==1.8.5.post1 Sphinx==5.3.0 -WTForms==3.0.1 +WTForms==3.1.2 Werkzeug==2.2.3 adal==1.2.7 -aiobotocore==2.6.0 +adlfs==2023.12.0 +aiobotocore==2.9.0 aiofiles==23.2.1 -aiohttp==3.8.6 +aiohttp==3.9.1 aioitertools==0.11.0 -aioresponses==0.7.4 +aioresponses==0.7.6 aiosignal==1.3.1 -alabaster==0.7.13 -alembic==1.12.0 -alibabacloud-adb20211201==1.0.0 -alibabacloud-credentials==0.3.2 -alibabacloud-endpoint-util==0.0.3 -alibabacloud-gateway-spi==0.0.1 -alibabacloud-openapi-util==0.2.1 -alibabacloud-tea-openapi==0.3.7 -alibabacloud-tea-util==0.3.11 -alibabacloud-tea-xml==0.0.2 -alibabacloud-tea==0.3.3 +alabaster==0.7.16 +alembic==1.13.1 +alibabacloud-adb20211201==1.2.6 +alibabacloud-tea==0.3.5 +alibabacloud_credentials==0.3.2 +alibabacloud_endpoint_util==0.0.3 +alibabacloud_gateway_spi==0.0.1 +alibabacloud_openapi_util==0.2.2 +alibabacloud_tea_openapi==0.3.8 +alibabacloud_tea_util==0.3.11 +alibabacloud_tea_xml==0.0.2 aliyun-python-sdk-core==2.14.0 aliyun-python-sdk-kms==2.16.2 -amqp==5.1.1 -analytics-python==1.4.post1 +amqp==5.2.0 +analytics-python==1.2.9 annotated-types==0.6.0 -ansiwrap==0.8.4 anyascii==0.3.2 -anyio==4.0.0 -apache-airflow-providers-airbyte==3.3.2 -apache-airflow-providers-alibaba==2.5.3 -apache-airflow-providers-amazon==8.7.1 -apache-airflow-providers-apache-beam==5.2.3 -apache-airflow-providers-apache-cassandra==3.2.1 -apache-airflow-providers-apache-drill==2.4.4 -apache-airflow-providers-apache-druid==3.5.0 -apache-airflow-providers-apache-flink==1.1.3 -apache-airflow-providers-apache-hdfs==4.1.1 -apache-airflow-providers-apache-impala==1.1.3 -apache-airflow-providers-apache-kafka==1.1.2 -apache-airflow-providers-apache-kylin==3.2.1 -apache-airflow-providers-apache-livy==3.5.4 -apache-airflow-providers-apache-pig==4.1.2 -apache-airflow-providers-apache-pinot==4.1.4 -apache-airflow-providers-apache-spark==4.1.5 -apache-airflow-providers-apache-sqoop==4.0.0 -apache-airflow-providers-apprise==1.0.2 -apache-airflow-providers-arangodb==2.2.2 -apache-airflow-providers-asana==2.2.2 -apache-airflow-providers-atlassian-jira==2.1.1 -apache-airflow-providers-celery==3.3.4 -apache-airflow-providers-cloudant==3.2.1 -apache-airflow-providers-cncf-kubernetes==7.6.0 -apache-airflow-providers-common-sql==1.7.2 -apache-airflow-providers-daskexecutor==1.0.1 -apache-airflow-providers-databricks==4.5.0 -apache-airflow-providers-datadog==3.3.2 -apache-airflow-providers-dbt-cloud==3.3.0 -apache-airflow-providers-dingding==3.2.1 -apache-airflow-providers-discord==3.3.1 -apache-airflow-providers-docker==3.7.5 -apache-airflow-providers-elasticsearch==5.0.2 -apache-airflow-providers-exasol==4.2.5 -apache-airflow-providers-facebook==3.2.2 -apache-airflow-providers-ftp==3.5.2 -apache-airflow-providers-github==2.3.2 -apache-airflow-providers-google==10.9.0 -apache-airflow-providers-grpc==3.2.2 -apache-airflow-providers-hashicorp==3.4.3 -apache-airflow-providers-http==4.5.2 -apache-airflow-providers-imap==3.3.2 -apache-airflow-providers-influxdb==2.2.3 -apache-airflow-providers-jdbc==4.0.2 -apache-airflow-providers-jenkins==3.3.2 -apache-airflow-providers-microsoft-azure==7.0.0 -apache-airflow-providers-microsoft-mssql==3.4.2 -apache-airflow-providers-microsoft-psrp==2.3.2 -apache-airflow-providers-microsoft-winrm==3.2.2 -apache-airflow-providers-mongo==3.2.2 -apache-airflow-providers-mysql==5.3.1 -apache-airflow-providers-neo4j==3.3.3 -apache-airflow-providers-odbc==4.0.0 -apache-airflow-providers-openfaas==3.2.1 -apache-airflow-providers-openlineage==1.1.0 -apache-airflow-providers-opsgenie==5.1.1 -apache-airflow-providers-oracle==3.7.4 -apache-airflow-providers-pagerduty==3.3.1 -apache-airflow-providers-papermill==3.2.1 -apache-airflow-providers-plexus==3.2.2 -apache-airflow-providers-postgres==5.6.1 -apache-airflow-providers-presto==5.1.4 -apache-airflow-providers-redis==3.3.2 -apache-airflow-providers-salesforce==5.4.3 -apache-airflow-providers-samba==4.2.2 -apache-airflow-providers-segment==3.2.1 -apache-airflow-providers-sendgrid==3.2.2 -apache-airflow-providers-sftp==4.6.1 -apache-airflow-providers-singularity==3.2.2 -apache-airflow-providers-slack==8.1.0 -apache-airflow-providers-smtp==1.3.2 -apache-airflow-providers-snowflake==5.0.1 -apache-airflow-providers-sqlite==3.4.3 -apache-airflow-providers-ssh==3.7.3 -apache-airflow-providers-tableau==4.2.2 -apache-airflow-providers-tabular==1.2.1 -apache-airflow-providers-telegram==4.1.1 -apache-airflow-providers-trino==5.3.1 -apache-airflow-providers-vertica==3.5.2 -apache-airflow-providers-zendesk==4.3.2 -apache-beam==2.50.0 -apispec==6.3.0 -apprise==1.5.0 -argcomplete==3.1.2 -arrow==1.3.0 -asana==3.2.1 +anyio==4.2.0 +apache-airflow-providers-airbyte==3.5.1 +apache-airflow-providers-alibaba==2.7.1 +apache-airflow-providers-amazon==8.16.0 +apache-airflow-providers-apache-beam==5.5.0 +apache-airflow-providers-apache-cassandra==3.4.1 +apache-airflow-providers-apache-drill==2.6.0 +apache-airflow-providers-apache-druid==3.7.0 +apache-airflow-providers-apache-flink==1.3.0 +apache-airflow-providers-apache-hdfs==4.3.2 +apache-airflow-providers-apache-hive==6.4.1 +apache-airflow-providers-apache-impala==1.3.0 +apache-airflow-providers-apache-kafka==1.3.1 +apache-airflow-providers-apache-kylin==3.5.0 +apache-airflow-providers-apache-livy==3.7.1 +apache-airflow-providers-apache-pig==4.3.0 +apache-airflow-providers-apache-pinot==4.3.0 +apache-airflow-providers-apache-spark==4.7.0 +apache-airflow-providers-apprise==1.2.1 +apache-airflow-providers-arangodb==2.4.1 +apache-airflow-providers-asana==2.4.1 +apache-airflow-providers-atlassian-jira==2.5.0 +apache-airflow-providers-celery==3.5.1 +apache-airflow-providers-cloudant==3.4.1 +apache-airflow-providers-cncf-kubernetes==7.13.0 +apache-airflow-providers-cohere==1.1.1 +apache-airflow-providers-common-io==1.2.0 +apache-airflow-providers-common-sql==1.10.0 +apache-airflow-providers-databricks==6.0.0 +apache-airflow-providers-datadog==3.5.1 +apache-airflow-providers-dbt-cloud==3.5.1 +apache-airflow-providers-dingding==3.4.0 +apache-airflow-providers-discord==3.5.0 +apache-airflow-providers-docker==3.9.1 +apache-airflow-providers-elasticsearch==5.3.1 +apache-airflow-providers-exasol==4.4.1 +apache-airflow-providers-facebook==3.4.0 +apache-airflow-providers-ftp==3.7.0 +apache-airflow-providers-github==2.5.1 +apache-airflow-providers-google==10.13.1 +apache-airflow-providers-grpc==3.4.1 +apache-airflow-providers-hashicorp==3.6.1 +apache-airflow-providers-http==4.8.0 +apache-airflow-providers-imap==3.5.0 +apache-airflow-providers-influxdb==2.4.0 +apache-airflow-providers-jdbc==4.2.1 +apache-airflow-providers-jenkins==3.5.1 +apache-airflow-providers-microsoft-azure==8.5.1 +apache-airflow-providers-microsoft-mssql==3.6.0 +apache-airflow-providers-microsoft-psrp==2.5.0 +apache-airflow-providers-microsoft-winrm==3.4.0 +apache-airflow-providers-mongo==3.5.0 +apache-airflow-providers-mysql==5.5.1 +apache-airflow-providers-neo4j==3.5.0 +apache-airflow-providers-odbc==4.4.0 +apache-airflow-providers-openai==1.1.0 +apache-airflow-providers-openfaas==3.4.0 +apache-airflow-providers-openlineage==1.4.0 +apache-airflow-providers-opensearch==1.1.1 +apache-airflow-providers-opsgenie==5.5.0 +apache-airflow-providers-oracle==3.9.1 +apache-airflow-providers-pagerduty==3.6.0 +apache-airflow-providers-papermill==3.6.0 +apache-airflow-providers-pgvector==1.1.0 +apache-airflow-providers-pinecone==1.1.1 +apache-airflow-providers-postgres==5.10.0 +apache-airflow-providers-presto==5.4.0 +apache-airflow-providers-redis==3.6.0 +apache-airflow-providers-salesforce==5.6.1 +apache-airflow-providers-samba==4.5.0 +apache-airflow-providers-segment==3.4.0 +apache-airflow-providers-sendgrid==3.4.0 +apache-airflow-providers-sftp==4.8.1 +apache-airflow-providers-singularity==3.4.0 +apache-airflow-providers-slack==8.5.1 +apache-airflow-providers-smtp==1.6.0 +apache-airflow-providers-snowflake==5.2.1 +apache-airflow-providers-sqlite==3.7.0 +apache-airflow-providers-ssh==3.10.0 +apache-airflow-providers-tableau==4.4.0 +apache-airflow-providers-tabular==1.4.1 +apache-airflow-providers-telegram==4.3.0 +apache-airflow-providers-trino==5.6.0 +apache-airflow-providers-vertica==3.7.0 +apache-airflow-providers-weaviate==1.3.0 +apache-airflow-providers-yandex==3.7.1 +apache-airflow-providers-zendesk==4.6.0 +apache-beam==2.53.0 +apispec==6.4.0 +apprise==1.7.1 +argcomplete==3.2.1 +asana==3.2.2 asgiref==3.7.2 asn1crypto==1.5.1 astroid==2.15.8 -asttokens==2.4.0 -async-timeout==4.0.3 +asttokens==2.4.1 atlasclient==1.0.0 -atlassian-python-api==3.41.2 -attrs==23.1.0 -aws-xray-sdk==2.12.0 -azure-batch==14.0.0 +atlassian-python-api==3.41.5 +attrs==23.2.0 +aws-sam-translator==1.83.0 +aws-xray-sdk==2.12.1 +azure-batch==14.1.0 azure-common==1.1.28 -azure-core==1.29.4 +azure-core==1.29.6 azure-cosmos==4.5.1 azure-datalake-store==0.0.53 -azure-identity==1.14.0 +azure-identity==1.15.0 azure-keyvault-secrets==4.7.0 -azure-kusto-data==4.2.0 -azure-mgmt-containerinstance==8.0.0 -azure-mgmt-containerregistry==10.2.0 +azure-kusto-data==4.3.1 +azure-mgmt-containerinstance==10.1.0 +azure-mgmt-containerregistry==10.3.0 azure-mgmt-core==1.4.0 -azure-mgmt-cosmosdb==9.3.0 -azure-mgmt-datafactory==1.1.0 +azure-mgmt-cosmosdb==9.4.0 +azure-mgmt-datafactory==4.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==23.0.1 azure-mgmt-storage==21.1.0 azure-nspkg==3.0.2 -azure-servicebus==7.11.2 -azure-storage-blob==12.18.2 -azure-storage-common==2.1.0 -azure-storage-file-datalake==12.13.1 -azure-storage-file-share==12.14.1 -azure-storage-file==2.1.0 +azure-servicebus==7.11.4 +azure-storage-blob==12.19.0 +azure-storage-file-datalake==12.14.0 +azure-storage-file-share==12.15.0 +azure-synapse-artifacts==0.18.0 azure-synapse-spark==0.7.0 -backcall==0.2.0 -backoff==1.10.0 -bcrypt==4.0.1 +backoff==2.2.1 +bcrypt==4.1.2 beautifulsoup4==4.12.2 -billiard==4.1.0 -bitarray==2.8.2 -black==23.9.1 -blinker==1.6.3 -boto3==1.28.17 -botocore==1.31.17 +billiard==4.2.0 +bitarray==2.9.2 +black==24.1a1 +blinker==1.7.0 +boto3==1.33.13 +botocore==1.33.13 cachelib==0.9.0 -cachetools==5.3.1 -cassandra-driver==3.28.0 -cattrs==23.1.2 -# The original file specifies 5.3.2, but due to the problem described here: +cachetools==5.3.2 +cassandra-driver==3.29.0 +cattrs==23.2.3 +# The original file specifies 5.3.6, but due to the problem described here: # https://github.com/celery/kombu/pull/1807, we are manually installing # celery v5.3.1. celery==5.3.1 -certifi==2023.7.22 +certifi==2023.11.17 cffi==1.16.0 cfgv==3.4.0 +cfn-lint==0.83.8 cgroupspy==0.2.2 chardet==5.2.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.2 checksumdir==1.2.0 -ciso8601==2.3.0 -click-default-group==1.2.4 +ciso8601==2.3.1 click-didyoumean==0.3.0 click-plugins==1.1.1 click-repl==0.3.0 @@ -246,151 +250,169 @@ click==8.1.7 clickclick==20.10.2 cloudant==2.15.0 cloudpickle==2.2.1 +cohere==4.44 colorama==0.4.6 colorlog==4.8.0 -confluent-kafka==2.2.0 +comm==0.2.1 +confluent-kafka==2.3.0 connexion==2.14.2 -coverage==7.3.2 +coverage==7.4.0 crcmod==1.7 cron-descriptor==1.4.0 -croniter==1.4.1 -cryptography==41.0.4 +croniter==2.0.1 +cryptography==41.0.7 curlify==2.2.1 -dask==2023.9.3 databricks-sql-connector==2.9.3 datadog==0.47.0 -db-dtypes==1.1.1 +db-dtypes==1.2.0 +debugpy==1.8.0 decorator==5.1.1 defusedxml==0.7.1 -deprecation==2.1.0 +deltalake==0.15.1 dill==0.3.1.1 -distlib==0.3.7 -distributed==2023.9.3 +distlib==0.3.8 +distro==1.9.0 dnspython==2.4.2 -docker==6.1.3 +docker==7.0.0 docopt==0.6.2 docutils==0.20.1 -elastic-transport==8.4.1 -elasticsearch==8.10.0 +duckdb==0.9.2 +ecdsa==0.18.0 +editables==0.5 +elastic-transport==8.11.0 +elasticsearch==8.11.1 email-validator==1.3.1 entrypoints==0.4 -eralchemy2==1.3.7 +eralchemy2==1.3.8 et-xmlfile==1.1.0 -eventlet==0.33.3 +eventlet==0.34.3 execnet==2.0.2 -executing==2.0.0 -facebook-business==18.0.2 -fastavro==1.8.4 +executing==2.0.1 +facebook-business==18.0.5 +fastavro==1.9.3 fasteners==0.19 -fastjsonschema==2.18.1 -filelock==3.12.4 +fastjsonschema==2.19.1 +filelock==3.13.1 flower==2.0.1 -frozenlist==1.4.0 -fsspec==2023.9.2 +frozenlist==1.4.1 +fsspec==2023.12.2 future==0.18.3 gcloud-aio-auth==4.2.3 gcloud-aio-bigquery==7.0.0 gcloud-aio-storage==9.0.0 -gcsfs==2023.9.2 +gcsfs==2023.12.2.post1 geomet==0.2.1.post1 gevent==23.9.1 -gitdb==4.0.10 -google-ads==22.0.0 -google-api-core==2.12.0 -google-api-python-client==2.102.0 -google-auth-httplib2==0.1.1 -google-auth-oauthlib==1.1.0 -google-auth==2.23.2 -google-cloud-aiplatform==1.34.0 -google-cloud-appengine-logging==1.3.2 +gitdb==4.0.11 +google-ads==22.1.0 +google-analytics-admin==0.22.2 +google-api-core==2.15.0 +google-api-python-client==2.113.0 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-auth==2.26.2 +google-cloud-aiplatform==1.39.0 +google-cloud-appengine-logging==1.4.0 google-cloud-audit-log==0.2.5 -google-cloud-automl==2.11.2 -google-cloud-batch==0.17.1 -google-cloud-bigquery-datatransfer==3.12.1 -google-cloud-bigquery-storage==2.22.0 -google-cloud-bigquery==3.12.0 -google-cloud-bigtable==2.21.0 -google-cloud-build==3.20.0 -google-cloud-compute==1.14.1 -google-cloud-container==2.32.0 -google-cloud-core==2.3.3 -google-cloud-datacatalog==3.16.0 -google-cloud-dataflow-client==0.8.4 -google-cloud-dataform==0.5.3 -google-cloud-dataplex==1.6.3 -google-cloud-dataproc-metastore==1.13.0 -google-cloud-dataproc==5.6.0 -google-cloud-dlp==3.12.3 -google-cloud-kms==2.19.1 -google-cloud-language==2.11.1 -google-cloud-logging==3.8.0 -google-cloud-memcache==1.7.3 -google-cloud-monitoring==2.15.1 -google-cloud-orchestration-airflow==1.9.2 -google-cloud-os-login==2.10.0 -google-cloud-pubsub==2.18.4 -google-cloud-redis==2.13.2 -google-cloud-resource-manager==1.10.4 -google-cloud-run==0.9.1 -google-cloud-secret-manager==2.16.4 -google-cloud-spanner==3.40.1 -google-cloud-speech==2.21.0 -google-cloud-storage-transfer==1.9.2 -google-cloud-storage==2.11.0 -google-cloud-tasks==2.14.2 -google-cloud-texttospeech==2.14.2 -google-cloud-translate==3.12.0 -google-cloud-videointelligence==2.11.4 -google-cloud-vision==3.4.4 -google-cloud-workflows==1.12.1 +google-cloud-automl==2.12.0 +google-cloud-batch==0.17.7 +google-cloud-bigquery-datatransfer==3.13.0 +google-cloud-bigquery-storage==2.24.0 +google-cloud-bigquery==3.16.0 +google-cloud-bigtable==2.22.0 +google-cloud-build==3.22.0 +google-cloud-compute==1.15.0 +google-cloud-container==2.37.0 +google-cloud-core==2.4.1 +google-cloud-datacatalog==3.17.2 +google-cloud-dataflow-client==0.8.6 +google-cloud-dataform==0.5.5 +google-cloud-dataplex==1.11.0 +google-cloud-dataproc-metastore==1.14.0 +google-cloud-dataproc==5.8.0 +google-cloud-dlp==3.14.0 +google-cloud-kms==2.20.0 +google-cloud-language==2.12.0 +google-cloud-logging==3.9.0 +google-cloud-memcache==1.8.0 +google-cloud-monitoring==2.18.0 +google-cloud-orchestration-airflow==1.10.0 +google-cloud-os-login==2.13.0 +google-cloud-pubsub==2.19.0 +google-cloud-redis==2.14.0 +google-cloud-resource-manager==1.11.0 +google-cloud-run==0.10.1 +google-cloud-secret-manager==2.17.0 +google-cloud-spanner==3.41.0 +google-cloud-speech==2.23.0 +google-cloud-storage-transfer==1.10.0 +google-cloud-storage==2.14.0 +google-cloud-tasks==2.15.0 +google-cloud-texttospeech==2.15.1 +google-cloud-translate==3.14.0 +google-cloud-videointelligence==2.12.0 +google-cloud-vision==3.5.0 +google-cloud-workflows==1.13.0 google-crc32c==1.5.0 google-re2==1.1 -google-resumable-media==2.6.0 -googleapis-common-protos==1.60.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +graphql-core==3.2.3 graphviz==0.20.1 -greenlet==3.0.0 -grpc-google-iam-v1==0.12.6 +greenlet==3.0.3 +grpc-google-iam-v1==0.13.0 grpcio-gcp==0.2.2 -grpcio-status==1.59.0 -grpcio==1.59.0 +grpcio-status==1.60.0 +grpcio==1.60.0 gssapi==1.8.3 gunicorn==21.2.0 h11==0.14.0 -hdfs==2.7.2 +hatch==1.9.1 +hatchling==1.21.0 +hdfs==2.7.3 hmsclient==0.1.1 httpcore==0.16.3 httplib2==0.22.0 httpx==0.23.3 -humanize==4.8.0 -hvac==1.2.1 -identify==2.5.30 -idna==3.4 +humanize==4.9.0 +hvac==2.1.0 +hyperlink==21.0.0 +icdiff==2.0.7 +identify==2.5.33 +idna==3.6 ijson==3.2.3 imagesize==1.4.1 -importlib-metadata==6.8.0 -importlib-resources==6.1.0 -impyla==0.18.0 +importlib-metadata==6.11.0 +importlib-resources==6.1.1 +impyla==0.19.0 incremental==22.10.0 inflection==0.5.1 -influxdb-client==1.38.0 +influxdb-client==1.39.0 iniconfig==2.0.0 ipdb==0.13.13 -ipython==8.16.1 +ipykernel==6.28.0 +ipython==8.20.0 isodate==0.6.1 itsdangerous==2.1.2 jaraco.classes==3.3.0 jedi==0.19.1 jeepney==0.8.0 jmespath==0.10.0 +jschema-to-python==1.2.3 json-merge-patch==0.2 -jsonpath-ng==1.6.0 -jsonschema-spec==0.2.4 -jsonschema-specifications==2023.7.1 -jsonschema==4.19.1 -jupyter_client==8.3.1 -jupyter_core==5.3.2 -keyring==24.2.0 -# The original file specifies 5.3.2, but due to the problem described here: +jsondiff==2.0.0 +jsonpatch==1.33 +jsonpath-ng==1.6.1 +jsonpickle==3.0.2 +jsonpointer==2.4 +jsonschema-path==0.3.2 +jsonschema-specifications==2023.12.1 +jsonschema==4.20.0 +junit-xml==1.9 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +keyring==24.3.0 +# The original file specifies 5.3.4, but due to the problem described here: # https://github.com/celery/kombu/pull/1807, we are manually installing # kombu v5.3.1. kombu==5.3.1 @@ -398,161 +420,169 @@ krb5==0.5.1 kubernetes-asyncio==24.2.3 kubernetes==23.6.0 kylinpy==2.8.4 -lazy-object-proxy==1.9.0 +lazy-object-proxy==1.10.0 ldap3==2.9.1 -limits==3.6.0 +limits==3.7.0 linkify-it-py==2.0.2 -locket==1.0.0 lockfile==0.12.2 -looker-sdk==23.16.0 -lxml==4.9.3 -lz4==4.3.2 +loguru==0.7.2 +looker-sdk==23.20.1 +lxml==5.1.0 +lz4==4.3.3 markdown-it-py==3.0.0 marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.26.1 -marshmallow==3.20.1 +marshmallow==3.20.2 matplotlib-inline==0.1.6 mdit-py-plugins==0.4.0 mdurl==0.1.2 +mmhash3==3.0.1 mongomock==4.1.2 -monotonic==1.6 -more-itertools==10.1.0 -moto==4.2.5 -msal-extensions==1.0.0 -msal==1.24.1 -msgpack==1.0.7 +more-itertools==10.2.0 +moto==4.2.13 +mpmath==1.3.0 +msal-extensions==1.1.0 +msal==1.26.0 msrest==0.7.1 msrestazure==0.6.4 -multi-key-dict==2.0.3 +multi_key_dict==2.0.3 multidict==6.0.4 -mypy-boto3-appflow==1.28.42 -mypy-boto3-rds==1.28.61 -mypy-boto3-redshift-data==1.28.36 -mypy-boto3-s3==1.28.55 +mypy-boto3-appflow==1.34.0 +mypy-boto3-rds==1.34.6 +mypy-boto3-redshift-data==1.34.0 +mypy-boto3-s3==1.34.14 mypy-extensions==1.0.0 mypy==1.2.0 -mysql-connector-python==8.1.0 -mysqlclient==2.2.0 -nbclient==0.8.0 +mysql-connector-python==8.0.29 +mysqlclient==2.2.1 +nbclient==0.9.0 nbformat==5.9.2 -neo4j==5.13.0 -nh3==0.2.14 +neo4j==5.16.0 +nest-asyncio==1.5.9 +networkx==3.2.1 +nh3==0.2.15 nodeenv==1.8.0 numpy==1.24.4 oauthlib==3.2.2 objsize==0.6.1 +openai==1.7.2 openapi-schema-validator==0.6.2 -openapi-spec-validator==0.6.0 -openlineage-integration-common==1.3.1 -openlineage-python==1.3.1 -openlineage_sql==1.3.1 +openapi-spec-validator==0.7.1 +openlineage-integration-common==1.7.0 +openlineage-python==1.7.0 +openlineage_sql==1.7.0 openpyxl==3.1.2 -opentelemetry-api==1.20.0 -opentelemetry-exporter-otlp-proto-common==1.20.0 -opentelemetry-exporter-otlp-proto-grpc==1.20.0 -opentelemetry-exporter-otlp-proto-http==1.20.0 -opentelemetry-exporter-otlp==1.20.0 -opentelemetry-exporter-prometheus==1.12.0rc1 -opentelemetry-proto==1.20.0 -opentelemetry-sdk==1.20.0 -opentelemetry-semantic-conventions==0.41b0 +opensearch-py==2.4.2 +opentelemetry-api==1.22.0 +opentelemetry-exporter-otlp-proto-common==1.22.0 +opentelemetry-exporter-otlp-proto-grpc==1.22.0 +opentelemetry-exporter-otlp-proto-http==1.22.0 +opentelemetry-exporter-otlp==1.22.0 +opentelemetry-exporter-prometheus==0.43b0 +opentelemetry-proto==1.22.0 +opentelemetry-sdk==1.22.0 +opentelemetry-semantic-conventions==0.43b0 opsgenie-sdk==2.1.5 -oracledb==1.4.1 +oracledb==2.0.1 ordered-set==4.1.0 -orjson==3.9.7 -oscrypto==1.3.0 -oss2==2.18.2 +orjson==3.9.10 +oss2==2.18.4 packaging==23.2 -pandas-gbq==0.19.2 -pandas==2.1.1 -papermill==2.4.0 -paramiko==3.3.1 +pandas-gbq==0.20.0 +pandas-stubs==2.0.2.230605 +pandas==2.1.4 +papermill==2.5.0 +paramiko==3.4.0 parso==0.8.3 -partd==1.4.1 pathable==0.4.3 -pathspec==0.11.2 -pbr==5.11.1 -pdpyras==5.1.2 -pendulum==2.1.2 -pexpect==4.8.0 -pickleshare==0.7.5 -pinotdb==5.1.0 -pipdeptree==2.13.0 -pipx==1.2.0 +pathspec==0.12.1 +pbr==6.0.0 +pdpyras==5.2.0 +pendulum==3.0.0 +pexpect==4.9.0 +pgvector==0.2.4 +pinecone-client==2.2.4 +pinotdb==5.1.2 +pipdeptree==2.13.2 +pipx==1.4.2 pkginfo==1.9.6 platformdirs==3.11.0 pluggy==1.3.0 ply==3.11 -plyvel==1.5.0 +plyvel==1.5.1 portalocker==2.8.2 -pre-commit==3.4.0 +pprintpp==0.4.0 +pre-commit==3.6.0 presto-python-client==0.8.4 prison==0.2.1 -prometheus-client==0.17.1 -prompt-toolkit==3.0.39 -proto-plus==1.22.3 -protobuf==4.21.12 -psutil==5.9.5 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +proto-plus==1.23.0 +protobuf==4.25.2 +psutil==5.9.7 psycopg2-binary==2.9.9 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 +py-partiql-parser==0.5.0 py4j==0.10.9.7 -pyOpenSSL==23.2.0 -pyarrow==11.0.0 +pyOpenSSL==23.3.0 +pyarrow-hotfix==0.6 +pyarrow==14.0.2 pyasn1-modules==0.3.0 -pyasn1==0.5.0 -pycountry==22.3.5 +pyasn1==0.5.1 +pycountry==23.12.11 pycparser==2.21 -pycryptodome==3.19.0 -pycryptodomex==3.19.0 -pydantic==2.4.2 -pydantic_core==2.10.1 +pycryptodome==3.20.0 +pydantic==2.5.3 +pydantic_core==2.14.6 pydata-google-auth==1.8.2 pydot==1.4.2 -pydruid==0.6.5 +pydruid==0.6.6 pyenchant==3.2.2 pyexasol==0.25.2 -pygraphviz==1.11 -pyhcl==0.4.5 +pygraphviz==1.12 +pyiceberg==0.5.1 +pyjsparser==2.7.1 pykerberos==1.2.4 -pymongo==4.5.0 -pymssql==2.2.8 -pyodbc==4.0.39 +pymongo==4.6.1 +pymssql==2.2.11 +pyodbc==5.0.1 pyparsing==3.1.1 pypsrp==0.8.1 pyspark==3.5.0 pyspnego==0.10.2 -pytest-asyncio==0.21.1 -pytest-capture-warnings==0.0.4 +pytest-asyncio==0.23.3 pytest-cov==4.1.0 pytest-httpx==0.21.3 +pytest-icdiff==0.9 pytest-instafail==0.5.0 -pytest-mock==3.11.1 -pytest-rerunfailures==12.0 +pytest-mock==3.12.0 +pytest-rerunfailures==13.0 pytest-timeouts==1.2.1 -pytest-xdist==3.3.1 -pytest==7.4.2 -python-arango==7.7.0 +pytest-xdist==3.5.0 +pytest==7.4.4 +python-arango==7.9.0 python-daemon==3.0.1 python-dateutil==2.8.2 python-dotenv==1.0.0 python-http-client==3.3.7 -python-jenkins==1.7.0 -python-ldap==3.4.3 +python-jenkins==1.8.2 +python-jose==3.3.0 +python-ldap==3.4.4 python-nvd3==0.15.0 python-slugify==8.0.1 python-telegram-bot==20.2 +python3-saml==1.16.0 pytz==2023.3.post1 -pytzdata==2020.1 pywinrm==0.4.3 -pyzmq==25.1.1 +pyzmq==25.1.2 reactivex==4.0.4 readme-renderer==42.0 redis==4.6.0 -redshift-connector==2.0.914 -referencing==0.30.2 -regex==2023.10.3 +redshift-connector==2.0.918 +referencing==0.32.1 +regex==2023.12.25 requests-file==1.5.1 requests-kerberos==0.14.0 requests-mock==1.11.0 @@ -560,32 +590,37 @@ requests-ntlm==1.2.0 requests-oauthlib==1.3.1 requests-toolbelt==1.0.0 requests==2.31.0 -responses==0.23.3 +responses==0.24.1 +restructuredtext_lint==1.4.0 rfc3339-validator==0.1.4 rfc3986==1.5.0 -rich-argparse==1.3.0 -rich-click==1.6.1 -rich==13.6.0 -rpds-py==0.10.4 +rich-argparse==1.4.0 +rich-click==1.7.3 +rich==13.7.0 +rpds-py==0.17.1 rsa==4.9 -ruff==0.0.292 -s3transfer==0.6.2 +ruff==0.1.11 +s3fs==2023.12.2 +s3transfer==0.8.2 +sarif-om==1.0.4 scramp==1.4.4 scrapbook==0.5.0 -semver==3.0.1 -sendgrid==6.10.0 +semver==3.0.2 +sendgrid==6.11.0 sentinels==1.0.0 -sentry-sdk==1.31.0 +sentry-sdk==1.39.2 setproctitle==1.3.3 +shapely==2.0.2 +shellingham==1.5.4 simple-salesforce==1.12.5 six==1.16.0 -slack-sdk==3.23.0 -smbprotocol==1.11.0 +slack_sdk==3.26.2 +smbprotocol==1.12.0 smmap==5.0.1 sniffio==1.3.0 snowballstemmer==2.2.0 -snowflake-connector-python==3.2.1 -snowflake-sqlalchemy==1.5.0 +snowflake-connector-python==3.6.0 +snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.5 sphinx-airflow-theme==0.0.12 @@ -593,87 +628,94 @@ sphinx-argparse==0.4.0 sphinx-autoapi==2.1.1 sphinx-copybutton==0.5.2 sphinx-jinja==2.0.2 -sphinx-rtd-theme==1.3.0 -sphinxcontrib-applehelp==1.0.7 -sphinxcontrib-devhelp==1.0.5 -sphinxcontrib-htmlhelp==2.0.4 +sphinx-rtd-theme==2.0.0 +sphinx_design==0.5.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 sphinxcontrib-httpdomain==1.8.1 sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-spelling==8.0.0 -spython==0.3.0 -sqlalchemy-bigquery==1.8.0 -sqlalchemy-drill==1.1.2 +spython==0.3.13 +sqlalchemy-bigquery==1.9.0 sqlalchemy-redshift==0.8.14 sqlalchemy-spanner==1.6.2 +sqlalchemy_drill==1.1.4 sqlparse==0.4.4 +sshpubkeys==3.3.1 sshtunnel==0.4.0 stack-data==0.6.3 starkbank-ecdsa==2.2.0 statsd==4.0.1 -tableauserverclient==0.25 +strictyaml==1.7.3 +sympy==1.12 +tableauserverclient==0.29 tabulate==0.9.0 -tblib==2.0.0 tenacity==8.2.3 -termcolor==2.3.0 +termcolor==2.4.0 text-unidecode==1.3 -textwrap3==0.9.2 thrift-sasl==0.4.3 thrift==0.16.0 time-machine==2.13.0 -tomlkit==0.12.1 -toolz==0.12.0 -tornado==6.3.3 -towncrier==23.6.0 +tomli_w==1.0.0 +tomlkit==0.12.3 +tornado==6.4 +towncrier==23.11.0 tqdm==4.66.1 -traitlets==5.11.2 +traitlets==5.14.1 trino==0.327.0 +trove-classifiers==2024.1.8 twine==4.0.2 -types-Deprecated==1.2.9.3 -types-Markdown==3.5.0.0 +types-Deprecated==1.2.9.20240106 +types-Markdown==3.5.0.20240106 types-PyMySQL==1.1.0.1 types-PyYAML==6.0.12.12 +types-aiofiles==23.2.0.20240106 types-certifi==2021.10.8.3 -types-croniter==1.4.0.1 -types-docutils==0.20.0.3 -types-paramiko==3.3.0.0 -types-protobuf==4.24.0.2 -types-pyOpenSSL==23.2.0.2 -types-python-dateutil==2.8.19.14 +types-croniter==2.0.0.20240106 +types-docutils==0.20.0.20240106 +types-paramiko==3.4.0.20240106 +types-protobuf==4.24.0.20240106 +types-pyOpenSSL==23.3.0.20240106 +types-python-dateutil==2.8.19.20240106 types-python-slugify==8.0.0.3 types-pytz==2023.3.1.1 -types-redis==4.6.0.7 -types-requests==2.31.0.6 -types-setuptools==68.2.0.0 -types-tabulate==0.9.0.3 +types-redis==4.6.0.20240106 +types-requests==2.31.0.20240106 +types-setuptools==69.0.0.20240115 +types-tabulate==0.9.0.20240106 types-termcolor==1.1.6.2 types-toml==0.10.8.7 -types-urllib3==1.26.25.14 -typing_extensions==4.8.0 -tzdata==2023.3 -tzlocal==5.1 +typing_extensions==4.9.0 +tzdata==2023.4 +tzlocal==5.2 uc-micro-py==1.0.2 unicodecsv==0.14.1 +universal-pathlib==0.1.4 uritemplate==4.1.1 -urllib3==1.26.17 +urllib3==2.0.7 userpath==1.9.1 -vertica-python==1.3.5 -vine==5.0.0 -virtualenv==20.24.5 -watchtower==2.0.1 -wcwidth==0.2.8 -websocket-client==1.6.4 -wrapt==1.15.0 +validators==0.22.0 +vertica-python==1.3.8 +vine==5.1.0 +virtualenv==20.25.0 +watchtower==3.0.1 +wcwidth==0.2.13 +weaviate-client==3.26.1 +websocket-client==1.7.0 +wrapt==1.16.0 +xmlsec==1.3.13 xmltodict==0.13.0 -yamllint==1.32.0 -yarl==1.9.2 +yamllint==1.33.0 +yandexcloud==0.253.0 +yarl==1.9.4 zeep==4.2.1 -zenpy==2.0.40 -zict==3.0.0 +zenpy==2.0.45 zipp==3.17.0 zope.event==5.0 zope.interface==6.1 -zstandard==0.21.0 +zstandard==0.22.0 \ No newline at end of file diff --git a/docker/config/mwaa-base-providers-requirements.txt b/docker/config/mwaa-base-providers-requirements.txt index 7fee1d371..5900474e2 100644 --- a/docker/config/mwaa-base-providers-requirements.txt +++ b/docker/config/mwaa-base-providers-requirements.txt @@ -1,8 +1,8 @@ -apache-airflow-providers-amazon==8.7.1 -apache-airflow-providers-celery==3.3.4 -apache-airflow-providers-common-sql==1.7.2 -apache-airflow-providers-ftp==3.5.2 -apache-airflow-providers-http==4.5.2 -apache-airflow-providers-imap==3.3.2 -apache-airflow-providers-postgres==5.6.1 -apache-airflow-providers-sqlite==3.4.3 \ No newline at end of file +apache-airflow-providers-amazon==8.16.0 +apache-airflow-providers-celery==3.5.1 +apache-airflow-providers-common-sql==1.10.0 +apache-airflow-providers-ftp==3.7.0 +apache-airflow-providers-http==4.8.0 +apache-airflow-providers-imap==3.5.0 +apache-airflow-providers-postgres==5.10.0 +apache-airflow-providers-sqlite==3.7.0 \ No newline at end of file diff --git a/docker/docker-compose-local.yml b/docker/docker-compose-local.yml index 575d42db2..510824857 100644 --- a/docker/docker-compose-local.yml +++ b/docker/docker-compose-local.yml @@ -1,7 +1,7 @@ version: '3.7' services: postgres: - image: postgres:11-alpine + image: postgres:13-alpine environment: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflow @@ -14,7 +14,7 @@ services: - "${PWD}/db-data:/var/lib/postgresql/data" local-runner: - image: amazon/mwaa-local:2_7 + image: amazon/mwaa-local:2_8 restart: always depends_on: - postgres diff --git a/docker/docker-compose-resetdb.yml b/docker/docker-compose-resetdb.yml index 17392b20a..448d3cd5f 100644 --- a/docker/docker-compose-resetdb.yml +++ b/docker/docker-compose-resetdb.yml @@ -1,7 +1,7 @@ version: '3.7' services: postgres: - image: postgres:11-alpine + image: postgres:13-alpine environment: - POSTGRES_USER=airflow - POSTGRES_PASSWORD=airflow @@ -14,7 +14,7 @@ services: - "${PWD}/db-data:/var/lib/postgresql/data" resetdb: - image: amazon/mwaa-local:2_7 + image: amazon/mwaa-local:2_8 depends_on: - postgres environment: diff --git a/docker/docker-compose-sequential.yml b/docker/docker-compose-sequential.yml index 000b8c715..54f9f8753 100644 --- a/docker/docker-compose-sequential.yml +++ b/docker/docker-compose-sequential.yml @@ -1,7 +1,7 @@ version: '3.7' services: webserver: - image: amazon/mwaa-local:2_7 + image: amazon/mwaa-local:2_8 restart: always environment: - LOAD_EX=n diff --git a/docker/script/verification.sh b/docker/script/verification.sh index 7b4a76dc6..ea3ddd21d 100644 --- a/docker/script/verification.sh +++ b/docker/script/verification.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -EXPECTED_AIRFLOW_VERSION="2.7.2" -EXPECTED_WATCHTOWER_VERSION="2.0.1" +EXPECTED_AIRFLOW_VERSION="2.8.1" +EXPECTED_WATCHTOWER_VERSION="3.0.1" validate_version_with_pip() { if [[ $(pip3 show "$1" | grep 'Version' | grep -o '[0-9].*') != "$2" ]] diff --git a/mwaa-local-env b/mwaa-local-env index 3cd330b62..7eafde13e 100755 --- a/mwaa-local-env +++ b/mwaa-local-env @@ -1,6 +1,6 @@ #!/bin/bash -AIRFLOW_VERSION=2_7 +AIRFLOW_VERSION=2_8 DOCKER_COMPOSE_PROJECT_NAME=aws-mwaa-local-runner-$AIRFLOW_VERSION display_help() { diff --git a/plugins/README.md b/plugins/README.md index 04c491d98..202d290d5 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,3 +1,3 @@ -Please see https://airflow.apache.org/docs/apache-airflow/2.7.2/authoring-and-scheduling/plugins.html for information about creating Airflow plugin +Please see https://airflow.apache.org/docs/apache-airflow/2.8.1/authoring-and-scheduling/plugins.html for information about creating Airflow plugin Note that, per the above documentation, importing operators, sensors, hooks added in plugins via airflow.{operators,sensors,hooks}. is no longer supported, and these extensions should just be imported as regular python modules. diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 185cda4e4..fe69e131a 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,4 @@ ---constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.11.txt" +--constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.11.txt" -apache-airflow-providers-snowflake==5.0.1 -apache-airflow-providers-mysql==5.3.1 +apache-airflow-providers-snowflake==5.2.1 +apache-airflow-providers-mysql==5.5.1