From 1d6e2ce41035bd096473fd62a1282556de7e33cf Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Tue, 20 Feb 2024 15:15:34 -0500 Subject: [PATCH] [dagster-fivetran] tests tweak to avoid duplicate asset keys --- .../test_load_from_instance.py | 18 ++++++++++-------- .../dagster_fivetran_tests/utils.py | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/test_load_from_instance.py b/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/test_load_from_instance.py index 82bfeb938f1f0..2c7adc431a7c7 100644 --- a/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/test_load_from_instance.py +++ b/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/test_load_from_instance.py @@ -103,7 +103,7 @@ def load_input(self, context: InputContext) -> Any: rsps.add( rsps.GET, f"{ft_resource.api_connector_url}{DEFAULT_CONNECTOR_ID_2}/schemas", - json=get_complex_sample_connector_schema_config(), + json=get_complex_sample_connector_schema_config("_xyz1", "_abc"), match=[matchers.header_matcher(expected_auth_header)], ) @@ -130,7 +130,6 @@ def load_input(self, context: InputContext) -> Any: ft_cacheable_assets.compute_cacheable_data() ) ft_assets = with_resources(ft_assets, {"test_io_manager": test_io_manager}) - if filter_connector: assert len(ft_assets) == 0 return @@ -203,19 +202,22 @@ def downstream_asset(xyz): ) with responses.RequestsMock() as rsps: - api_prefixes = [ - f"{ft_resource.api_connector_url}{DEFAULT_CONNECTOR_ID}", - ] + api_prefixes = [(f"{ft_resource.api_connector_url}{DEFAULT_CONNECTOR_ID}", tuple())] if multiple_connectors: - api_prefixes.append(f"{ft_resource.api_connector_url}{DEFAULT_CONNECTOR_ID_2}") - for api_prefix in api_prefixes: + api_prefixes.append( + (f"{ft_resource.api_connector_url}{DEFAULT_CONNECTOR_ID_2}", ("_xyz1", "_abc")) + ) + # for api_prefix in api_prefixes: + for api_prefix, schema_args in api_prefixes: rsps.add(rsps.PATCH, api_prefix, json=get_sample_update_response()) rsps.add(rsps.POST, f"{api_prefix}/force", json=get_sample_sync_response()) + # connector schema rsps.add( rsps.GET, f"{api_prefix}/schemas", - json=get_complex_sample_connector_schema_config(), + # json=get_complex_sample_connector_schema_config(), + json=get_complex_sample_connector_schema_config(*schema_args), ) # initial state rsps.add(rsps.GET, api_prefix, json=get_sample_connector_response()) diff --git a/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/utils.py b/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/utils.py index 3b4b4979ca73e..2fda05f18cb53 100644 --- a/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/utils.py +++ b/python_modules/libraries/dagster-fivetran/dagster_fivetran_tests/utils.py @@ -63,14 +63,20 @@ def get_sample_connector_schema_config(tables): } -def get_complex_sample_connector_schema_config(): +# Set schema names if you want to reuse this on multiple connectors for the same fivetran service +# without getting duplicate asset keys. +def get_complex_sample_connector_schema_config( + schema_name_1: str = "xyz1", + schema_name_2: str = "abc", + schema_name_3: str = "qwerty", +): return { "code": "Success", "data": { "enable_new_by_default": False, "schemas": { "schema_1": { - "name_in_destination": "xyz1", + "name_in_destination": schema_name_1, "enabled": True, "tables": { "table_1": { @@ -116,7 +122,7 @@ def get_complex_sample_connector_schema_config(): }, }, "schema_2": { - "name_in_destination": "abc", + "name_in_destination": schema_name_2, "enabled": True, "tables": { "table_1": { @@ -139,7 +145,7 @@ def get_complex_sample_connector_schema_config(): }, }, "schema_3": { - "name_in_destination": "qwerty", + "name_in_destination": schema_name_3, "enabled": False, "tables": { "table_1": { @@ -226,8 +232,8 @@ def get_sample_connectors_response_multiple(): }, { "id": "FAKE", - "service": "some_other_service", - "schema": "some_other_service.some_name", + "service": "some_fake_service", + "schema": "some_fake_service.some_name", "status": { "setup_state": "broken", },