From fd4c145ad2275ee8785f4a2572ac6c6aa0a79057 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:32:26 +0000 Subject: [PATCH 1/2] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.1.9 → v1.5.5](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.1.9...v1.5.5) - [github.com/thlorenz/doctoc.git: v1.4.0 → v2.2.0](https://github.com/thlorenz/doctoc.git/compare/v1.4.0...v2.2.0) - [github.com/psf/black: 20.8b1 → 24.10.0](https://github.com/psf/black/compare/20.8b1...24.10.0) - [github.com/pre-commit/pre-commit-hooks: v3.3.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v3.3.0...v5.0.0) - [github.com/pre-commit/pygrep-hooks: v1.6.0 → v1.10.0](https://github.com/pre-commit/pygrep-hooks/compare/v1.6.0...v1.10.0) - [github.com/adrienverge/yamllint: v1.25.0 → v1.35.1](https://github.com/adrienverge/yamllint/compare/v1.25.0...v1.35.1) - https://github.com/timothycrosley/isort → https://github.com/PyCQA/isort - [github.com/PyCQA/isort: 5.6.4 → 5.13.2](https://github.com/PyCQA/isort/compare/5.6.4...5.13.2) - [github.com/pycqa/pydocstyle: 5.1.1 → 6.3.0](https://github.com/pycqa/pydocstyle/compare/5.1.1...6.3.0) --- .pre-commit-config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c2047da67e107..475e789c31299a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,14 +15,14 @@ # specific language governing permissions and limitations # under the License. --- -default_stages: [commit, push] +default_stages: [pre-commit, pre-push] default_language_version: # force all unspecified python hooks to run python3 python: python3 minimum_pre_commit_version: "1.20.0" repos: - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.9 + rev: v1.5.5 hooks: - id: forbid-tabs exclude: ^docs/Makefile$|^clients/gen/go.sh @@ -138,7 +138,7 @@ repos: - --fuzzy-match-generates-todo files: \.mermaid$ - repo: https://github.com/thlorenz/doctoc.git - rev: v1.4.0 + rev: v2.2.0 hooks: - id: doctoc name: Add TOC for md files @@ -156,14 +156,14 @@ repos: hooks: - id: check-hooks-apply - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 24.10.0 hooks: - id: black files: api_connexion/.*\.py|.*providers.*\.py|^chart/tests/.*\.py exclude: .*kubernetes_pod\.py|.*google/common/hooks/base_google\.py$ args: [--config=./pyproject.toml] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.3.0 + rev: v5.0.0 hooks: - id: check-merge-conflict - id: debug-statements @@ -178,12 +178,12 @@ repos: args: - --remove - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.6.0 + rev: v1.10.0 hooks: - id: rst-backticks - id: python-no-log-warn - repo: https://github.com/adrienverge/yamllint - rev: v1.25.0 + rev: v1.35.1 hooks: - id: yamllint name: Check yaml files with yamllint @@ -191,8 +191,8 @@ repos: types: [yaml] exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml - - repo: https://github.com/timothycrosley/isort - rev: 5.6.4 + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 hooks: - id: isort name: Run isort to sort imports @@ -200,7 +200,7 @@ repos: # To keep consistent with the global isort skip config defined in setup.cfg exclude: ^build/.*$|^.tox/.*$|^venv/.*$|.*api_connexion/.*\.py|.*providers.*\.py - repo: https://github.com/pycqa/pydocstyle - rev: 5.1.1 + rev: 6.3.0 hooks: - id: pydocstyle name: Run pydocstyle From d9b25d5349098ba35c21becf63e0acabfe67f448 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:34:34 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../api_connexion/endpoints/pool_endpoint.py | 8 +++--- .../example_dags/example_s3_to_redshift.py | 12 ++++++--- .../dingding/example_dags/example_dingding.py | 25 ++++++++++++------- .../elasticsearch/log/es_task_handler.py | 2 +- .../example_bigquery_operations.py | 16 +++++++++--- .../providers/google/cloud/hooks/bigquery.py | 2 +- .../providers/google/cloud/hooks/mlengine.py | 4 +-- airflow/providers/mongo/hooks/mongo.py | 8 +++--- .../example_dags/example_snowflake.py | 16 +++++++----- .../apache/hive/transfers/test_s3_to_hive.py | 4 +-- tests/providers/jenkins/hooks/test_jenkins.py | 2 +- tests/providers/slack/hooks/test_slack.py | 12 ++++----- tests/providers/yandex/hooks/test_yandex.py | 3 +-- 13 files changed, 70 insertions(+), 44 deletions(-) diff --git a/airflow/api_connexion/endpoints/pool_endpoint.py b/airflow/api_connexion/endpoints/pool_endpoint.py index 6f8b4d5150f785..ff02cf341be20a 100644 --- a/airflow/api_connexion/endpoints/pool_endpoint.py +++ b/airflow/api_connexion/endpoints/pool_endpoint.py @@ -88,9 +88,11 @@ def patch_pool(pool_name, session, update_mask=None): _patch_body = {} try: update_mask = [ - pool_schema.declared_fields[field].attribute - if pool_schema.declared_fields[field].attribute - else field + ( + pool_schema.declared_fields[field].attribute + if pool_schema.declared_fields[field].attribute + else field + ) for field in update_mask ] except KeyError as err: diff --git a/airflow/providers/amazon/aws/example_dags/example_s3_to_redshift.py b/airflow/providers/amazon/aws/example_dags/example_s3_to_redshift.py index 76c79e521df17a..1a7c9118a96d7f 100644 --- a/airflow/providers/amazon/aws/example_dags/example_s3_to_redshift.py +++ b/airflow/providers/amazon/aws/example_dags/example_s3_to_redshift.py @@ -75,7 +75,11 @@ def _remove_sample_data_from_s3(): teardown__task_remove_sample_data_from_s3 = PythonOperator( python_callable=_remove_sample_data_from_s3, task_id='teardown__remove_sample_data_from_s3' ) - [setup__task_add_sample_data_to_s3, setup__task_create_table] >> task_transfer_s3_to_redshift >> [ - teardown__task_drop_table, - teardown__task_remove_sample_data_from_s3, - ] + ( + [setup__task_add_sample_data_to_s3, setup__task_create_table] + >> task_transfer_s3_to_redshift + >> [ + teardown__task_drop_table, + teardown__task_remove_sample_data_from_s3, + ] + ) diff --git a/airflow/providers/dingding/example_dags/example_dingding.py b/airflow/providers/dingding/example_dags/example_dingding.py index 907fc5ed808b90..13cd5e58f02380 100644 --- a/airflow/providers/dingding/example_dags/example_dingding.py +++ b/airflow/providers/dingding/example_dags/example_dingding.py @@ -205,12 +205,19 @@ def failure_callback(context): message="", ) - [ - text_msg_remind_none, - text_msg_remind_specific, - text_msg_remind_include_invalid, - text_msg_remind_all, - ] >> link_msg >> markdown_msg >> [ - single_action_card_msg, - multi_action_card_msg, - ] >> feed_card_msg >> msg_failure_callback + ( + [ + text_msg_remind_none, + text_msg_remind_specific, + text_msg_remind_include_invalid, + text_msg_remind_all, + ] + >> link_msg + >> markdown_msg + >> [ + single_action_card_msg, + multi_action_card_msg, + ] + >> feed_card_msg + >> msg_failure_callback + ) diff --git a/airflow/providers/elasticsearch/log/es_task_handler.py b/airflow/providers/elasticsearch/log/es_task_handler.py index 35c99ea4f74534..ad2614aeba5a49 100644 --- a/airflow/providers/elasticsearch/log/es_task_handler.py +++ b/airflow/providers/elasticsearch/log/es_task_handler.py @@ -253,7 +253,7 @@ def set_context(self, ti: TaskInstance) -> None: 'execution_date': self._clean_execution_date(ti.execution_date), 'try_number': str(ti.try_number), 'log_id': self._render_log_id(ti, ti.try_number), - 'offset': int(time() * (10 ** 9)), + 'offset': int(time() * (10**9)), }, ) diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py b/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py index 07f900ad07c652..9d543b5bdd0fd8 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py @@ -168,10 +168,18 @@ create_dataset >> patch_dataset >> update_dataset >> get_dataset >> get_dataset_result >> delete_dataset - update_dataset >> create_table >> create_view >> [ - get_dataset_tables, - delete_view, - ] >> update_table >> delete_table >> delete_dataset + ( + update_dataset + >> create_table + >> create_view + >> [ + get_dataset_tables, + delete_view, + ] + >> update_table + >> delete_table + >> delete_dataset + ) update_dataset >> create_external_table >> delete_dataset with models.DAG( diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index f130df7def718e..041ef360589aa1 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -2755,7 +2755,7 @@ def _bind_parameters(operation: str, parameters: dict) -> str: """Helper method that binds parameters to a SQL query""" # inspired by MySQL Python Connector (conversion.py) string_parameters = {} # type Dict[str, str] - for (name, value) in parameters.items(): + for name, value in parameters.items(): if value is None: string_parameters[name] = 'NULL' elif isinstance(value, str): diff --git a/airflow/providers/google/cloud/hooks/mlengine.py b/airflow/providers/google/cloud/hooks/mlengine.py index a843b3bf03fb0c..288250b62f9e6c 100644 --- a/airflow/providers/google/cloud/hooks/mlengine.py +++ b/airflow/providers/google/cloud/hooks/mlengine.py @@ -62,13 +62,13 @@ def _poll_with_exponential_delay(request, execute_num_retries, max_n, is_done_fu log.info('Operation is done: %s', response) return response - time.sleep((2 ** i) + (random.randint(0, 1000) / 1000)) + time.sleep((2**i) + (random.randint(0, 1000) / 1000)) except HttpError as e: if e.resp.status != 429: log.info('Something went wrong. Not retrying: %s', format(e)) raise else: - time.sleep((2 ** i) + (random.randint(0, 1000) / 1000)) + time.sleep((2**i) + (random.randint(0, 1000) / 1000)) raise ValueError('Connection could not be established after {} retries.'.format(max_n)) diff --git a/airflow/providers/mongo/hooks/mongo.py b/airflow/providers/mongo/hooks/mongo.py index 5d71438e8a98d7..e19b2412b62e04 100644 --- a/airflow/providers/mongo/hooks/mongo.py +++ b/airflow/providers/mongo/hooks/mongo.py @@ -55,9 +55,11 @@ def __init__(self, conn_id: str = 'mongo_default', *args, **kwargs) -> None: self.uri = '{scheme}://{creds}{host}{port}/{database}'.format( scheme=scheme, - creds='{}:{}@'.format(self.connection.login, self.connection.password) - if self.connection.login - else '', + creds=( + '{}:{}@'.format(self.connection.login, self.connection.password) + if self.connection.login + else '' + ), host=self.connection.host, port='' if self.connection.port is None else ':{}'.format(self.connection.port), database=self.connection.schema, diff --git a/airflow/providers/snowflake/example_dags/example_snowflake.py b/airflow/providers/snowflake/example_dags/example_snowflake.py index 9b174f160dcd9b..936933280bee75 100644 --- a/airflow/providers/snowflake/example_dags/example_snowflake.py +++ b/airflow/providers/snowflake/example_dags/example_snowflake.py @@ -123,9 +123,13 @@ # [END howto_operator_snowflake_to_slack] -snowflake_op_sql_str >> [ - snowflake_op_with_params, - snowflake_op_sql_list, - snowflake_op_template_file, - copy_into_table, -] >> slack_report +( + snowflake_op_sql_str + >> [ + snowflake_op_with_params, + snowflake_op_sql_list, + snowflake_op_template_file, + copy_into_table, + ] + >> slack_report +) diff --git a/tests/providers/apache/hive/transfers/test_s3_to_hive.py b/tests/providers/apache/hive/transfers/test_s3_to_hive.py index 4c0ed475176c01..00103bda5ec069 100644 --- a/tests/providers/apache/hive/transfers/test_s3_to_hive.py +++ b/tests/providers/apache/hive/transfers/test_s3_to_hive.py @@ -212,7 +212,7 @@ def test_execute(self, mock_hiveclihook): conn.create_bucket(Bucket='bucket') # Testing txt, zip, bz2 files with and without header row - for (ext, has_header) in product(['.txt', '.gz', '.bz2', '.GZ'], [True, False]): + for ext, has_header in product(['.txt', '.gz', '.bz2', '.GZ'], [True, False]): self.kwargs['headers'] = has_header self.kwargs['check_headers'] = has_header logging.info("Testing %s format %s header", ext, 'with' if has_header else 'without') @@ -248,7 +248,7 @@ def test_execute_with_select_expression(self, mock_hiveclihook): # Only testing S3ToHiveTransfer calls S3Hook.select_key with # the right parameters and its execute method succeeds here, # since Moto doesn't support select_object_content as of 1.3.2. - for (ext, has_header) in product(['.txt', '.gz', '.GZ'], [True, False]): + for ext, has_header in product(['.txt', '.gz', '.GZ'], [True, False]): input_compressed = ext.lower() != '.txt' key = self.s3_key + ext diff --git a/tests/providers/jenkins/hooks/test_jenkins.py b/tests/providers/jenkins/hooks/test_jenkins.py index 69ce7833b88def..b42b23e96e257a 100644 --- a/tests/providers/jenkins/hooks/test_jenkins.py +++ b/tests/providers/jenkins/hooks/test_jenkins.py @@ -25,7 +25,7 @@ class TestJenkinsHook(unittest.TestCase): @mock.patch('airflow.hooks.base_hook.BaseHook.get_connection') def test_client_created_default_http(self, get_connection_mock): - """tests `init` method to validate http client creation when all parameters are passed """ + """tests `init` method to validate http client creation when all parameters are passed""" default_connection_id = 'jenkins_default' connection_host = 'http://test.com' diff --git a/tests/providers/slack/hooks/test_slack.py b/tests/providers/slack/hooks/test_slack.py index 4ca0060f8fbaf1..3d7cb8ad586f0e 100644 --- a/tests/providers/slack/hooks/test_slack.py +++ b/tests/providers/slack/hooks/test_slack.py @@ -27,7 +27,7 @@ class TestSlackHook(unittest.TestCase): def test_get_token_with_token_only(self): - """tests `__get_token` method when only token is provided """ + """tests `__get_token` method when only token is provided""" # Given test_token = 'test_token' test_conn_id = None @@ -43,7 +43,7 @@ def test_get_token_with_token_only(self): @mock.patch('airflow.providers.slack.hooks.slack.WebClient') @mock.patch('airflow.providers.slack.hooks.slack.SlackHook.get_connection') def test_get_token_with_valid_slack_conn_id_only(self, get_connection_mock, mock_slack_client): - """tests `__get_token` method when only connection is provided """ + """tests `__get_token` method when only connection is provided""" # Given test_token = None test_conn_id = 'x' @@ -63,7 +63,7 @@ def test_get_token_with_valid_slack_conn_id_only(self, get_connection_mock, mock @mock.patch('airflow.providers.slack.hooks.slack.SlackHook.get_connection') def test_get_token_with_no_password_slack_conn_id_only(self, get_connection_mock): - """tests `__get_token` method when only connection is provided """ + """tests `__get_token` method when only connection is provided""" # Mock conn = mock.Mock() @@ -75,7 +75,7 @@ def test_get_token_with_no_password_slack_conn_id_only(self, get_connection_mock @mock.patch('airflow.providers.slack.hooks.slack.SlackHook.get_connection') def test_get_token_with_empty_password_slack_conn_id_only(self, get_connection_mock): - """tests `__get_token` method when only connection is provided """ + """tests `__get_token` method when only connection is provided""" # Mock get_connection_mock.return_value = mock.Mock(password=None) @@ -84,7 +84,7 @@ def test_get_token_with_empty_password_slack_conn_id_only(self, get_connection_m self.assertRaises(AirflowException, SlackHook, token=None, slack_conn_id='x') def test_get_token_with_token_and_slack_conn_id(self): - """tests `__get_token` method when both arguments are provided """ + """tests `__get_token` method when both arguments are provided""" # Given test_token = 'test_token' test_conn_id = 'x' @@ -98,7 +98,7 @@ def test_get_token_with_token_and_slack_conn_id(self): self.assertEqual(output, expected) def test_get_token_with_out_token_nor_slack_conn_id(self): - """tests `__get_token` method when no arguments are provided """ + """tests `__get_token` method when no arguments are provided""" self.assertRaises(AirflowException, SlackHook, token=None, slack_conn_id=None) diff --git a/tests/providers/yandex/hooks/test_yandex.py b/tests/providers/yandex/hooks/test_yandex.py index 55b0f3b3e9de1b..34a7c619247d84 100644 --- a/tests/providers/yandex/hooks/test_yandex.py +++ b/tests/providers/yandex/hooks/test_yandex.py @@ -27,7 +27,7 @@ class TestYandexHook(unittest.TestCase): @mock.patch('airflow.hooks.base_hook.BaseHook.get_connection') @mock.patch('airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook._get_credentials') def test_client_created_without_exceptions(self, get_credentials_mock, get_connection_mock): - """tests `init` method to validate client creation when all parameters are passed """ + """tests `init` method to validate client creation when all parameters are passed""" # Inputs to constructor default_folder_id = 'test_id' @@ -46,7 +46,6 @@ def test_client_created_without_exceptions(self, get_credentials_mock, get_conne @mock.patch('airflow.hooks.base_hook.BaseHook.get_connection') def test_get_credentials_raise_exception(self, get_connection_mock): - """tests 'get_credentials' method raising exception if none of the required fields are passed.""" # Inputs to constructor