Merge pull request #3818 from SFDO-Tooling/feature/select_records

@W-16485311: Core Logic for Selecting Records from Target Org
SFDO-Tooling · Nov 22, 2024 · e261563 · e261563
2 parents abe92e7 + 8a984fd
commit e261563
Show file tree

Hide file tree

Showing 25 changed files with 5,443 additions and 253 deletions.
diff --git a/cumulusci/core/tests/test_datasets_e2e.py b/cumulusci/core/tests/test_datasets_e2e.py
@@ -304,6 +304,7 @@ def write_yaml(filename: str, json: Any):
                                 "after": "Insert Account",
                             }
                         },
+                        "select_options": {},
                     },
                     "Insert Event": {
                         "sf_object": "Event",
@@ -316,16 +317,19 @@ def write_yaml(filename: str, json: Any):
                                 "after": "Insert Lead",
                             }
                         },
+                        "select_options": {},
                     },
                     "Insert Account": {
                         "sf_object": "Account",
                         "table": "Account",
                         "fields": ["Name"],
+                        "select_options": {},
                     },
                     "Insert Lead": {
                         "sf_object": "Lead",
                         "table": "Lead",
                         "fields": ["Company", "LastName"],
+                        "select_options": {},
                     },
                 }
                 assert tuple(actual.items()) == tuple(expected.items()), actual.items()

diff --git a/cumulusci/tasks/bulkdata/extract_dataset_utils/extract_yml.py b/cumulusci/tasks/bulkdata/extract_dataset_utils/extract_yml.py
@@ -5,7 +5,7 @@
 from pydantic import Field, validator
 
 from cumulusci.core.enums import StrEnum
-from cumulusci.tasks.bulkdata.step import DataApi
+from cumulusci.tasks.bulkdata.utils import DataApi
 from cumulusci.utils.yaml.model_parser import CCIDictModel, HashableBaseModel
 
 object_decl = re.compile(r"objects\((\w+)\)", re.IGNORECASE)

diff --git a/...sks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py b/...sks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py
@@ -41,6 +41,7 @@ def test_simple_generate_mapping_from_declarations(self, org_config):
                     "sf_object": "Account",
                     "table": "Account",
                     "fields": ["Name", "Description"],
+                    "select_options": {},
                 }
             }
 
@@ -74,11 +75,13 @@ def test_generate_mapping_from_both_kinds_of_declarations(self, org_config):
                         "sf_object": "Contact",
                         "table": "Contact",
                         "fields": ["FirstName", "LastName"],
+                        "select_options": {},
                     },
                     "Insert Account": {
                         "sf_object": "Account",
                         "table": "Account",
                         "fields": ["Name", "Description"],
+                        "select_options": {},
                     },
                 }.items()
             )
@@ -111,6 +114,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config):
                     "sf_object": "Account",
                     "table": "Account",
                     "fields": ["Name", "Description"],
+                    "select_options": {},
                 },
                 "Insert Contact": {
                     "sf_object": "Contact",
@@ -119,6 +123,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config):
                     "lookups": {
                         "AccountId": {"table": ["Account"], "key_field": "AccountId"}
                     },
+                    "select_options": {},
                 },
             }
 
@@ -157,6 +162,7 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
                     "sf_object": "Account",
                     "table": "Account",
                     "fields": ["Name", "Description"],
+                    "select_options": {},
                 },
                 "Insert Contact": {
                     "sf_object": "Contact",
@@ -165,11 +171,13 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
                     "lookups": {
                         "AccountId": {"table": ["Account"], "key_field": "AccountId"}
                     },
+                    "select_options": {},
                 },
                 "Insert Lead": {
                     "sf_object": "Lead",
                     "table": "Lead",
                     "fields": ["LastName", "Company"],
+                    "select_options": {},
                 },
                 "Insert Event": {
                     "sf_object": "Event",
@@ -178,6 +186,7 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
                     "lookups": {
                         "WhoId": {"table": ["Contact", "Lead"], "key_field": "WhoId"}
                     },
+                    "select_options": {},
                 },
             }
 
@@ -221,6 +230,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups(
                     },
                     "sf_object": "Account",
                     "table": "Account",
+                    "select_options": {},
                 },
                 "Insert Contact": {
                     "sf_object": "Contact",
@@ -229,6 +239,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups(
                     "lookups": {
                         "AccountId": {"table": ["Account"], "key_field": "AccountId"}
                     },
+                    "select_options": {},
                 },
             }, mf
 
@@ -252,11 +263,13 @@ def test_generate_load_mapping__with_load_declarations(self, org_config):
                 "sf_object": "Account",
                 "api": DataApi.REST,
                 "table": "Account",
+                "select_options": {},
             },
             "Insert Contact": {
                 "sf_object": "Contact",
                 "api": DataApi.BULK,
                 "table": "Contact",
+                "select_options": {},
             },
         }, mf
 
@@ -288,24 +301,28 @@ def test_generate_load_mapping__with_upserts(self, org_config):
             "Insert Account": {
                 "sf_object": "Account",
                 "table": "Account",
+                "select_options": {},
             },
             "Upsert Account Name": {
                 "sf_object": "Account",
                 "table": "Account",
                 "action": DataOperationType.UPSERT,
                 "update_key": ("Name",),
                 "fields": ["Name"],
+                "select_options": {},
             },
             "Etl_Upsert Account AccountNumber_Name": {
                 "sf_object": "Account",
                 "table": "Account",
                 "action": DataOperationType.ETL_UPSERT,
                 "update_key": ("AccountNumber", "Name"),
                 "fields": ["AccountNumber", "Name"],
+                "select_options": {},
             },
             "Insert Contact": {
                 "sf_object": "Contact",
                 "table": "Contact",
+                "select_options": {},
             },
         }, mf