Skip to content

Commit

Permalink
Add tests for parent level similarity and priority fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@aditya-balachander
aditya-balachander committed Nov 11, 2024
1 parent 0ac2000 commit 730ba6c
Show file tree
Hide file tree
Showing 17 changed files with 1,442 additions and 452 deletions.
4 changes: 4 additions & 0 deletions cumulusci/core/tests/test_datasets_e2e.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ def write_yaml(filename: str, json: Any):
"after": "Insert Account",
}
},
"select_options": {},
},
"Insert Event": {
"sf_object": "Event",
Expand All @@ -316,16 +317,19 @@ def write_yaml(filename: str, json: Any):
"after": "Insert Lead",
}
},
"select_options": {},
},
"Insert Account": {
"sf_object": "Account",
"table": "Account",
"fields": ["Name"],
"select_options": {},
},
"Insert Lead": {
"sf_object": "Lead",
"table": "Lead",
"fields": ["Company", "LastName"],
"select_options": {},
},
}
assert tuple(actual.items()) == tuple(expected.items()), actual.items()
Expand Down
17 changes: 17 additions & 0 deletions ...sks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def test_simple_generate_mapping_from_declarations(self, org_config):
"sf_object": "Account",
"table": "Account",
"fields": ["Name", "Description"],
"select_options": {},
}
}

Expand Down Expand Up @@ -74,11 +75,13 @@ def test_generate_mapping_from_both_kinds_of_declarations(self, org_config):
"sf_object": "Contact",
"table": "Contact",
"fields": ["FirstName", "LastName"],
"select_options": {},
},
"Insert Account": {
"sf_object": "Account",
"table": "Account",
"fields": ["Name", "Description"],
"select_options": {},
},
}.items()
)
Expand Down Expand Up @@ -111,6 +114,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config):
"sf_object": "Account",
"table": "Account",
"fields": ["Name", "Description"],
"select_options": {},
},
"Insert Contact": {
"sf_object": "Contact",
Expand All @@ -119,6 +123,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config):
"lookups": {
"AccountId": {"table": ["Account"], "key_field": "AccountId"}
},
"select_options": {},
},
}

Expand Down Expand Up @@ -157,6 +162,7 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
"sf_object": "Account",
"table": "Account",
"fields": ["Name", "Description"],
"select_options": {},
},
"Insert Contact": {
"sf_object": "Contact",
Expand All @@ -165,11 +171,13 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
"lookups": {
"AccountId": {"table": ["Account"], "key_field": "AccountId"}
},
"select_options": {},
},
"Insert Lead": {
"sf_object": "Lead",
"table": "Lead",
"fields": ["LastName", "Company"],
"select_options": {},
},
"Insert Event": {
"sf_object": "Event",
Expand All @@ -178,6 +186,7 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
"lookups": {
"WhoId": {"table": ["Contact", "Lead"], "key_field": "WhoId"}
},
"select_options": {},
},
}

Expand Down Expand Up @@ -221,6 +230,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups(
},
"sf_object": "Account",
"table": "Account",
"select_options": {},
},
"Insert Contact": {
"sf_object": "Contact",
Expand All @@ -229,6 +239,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups(
"lookups": {
"AccountId": {"table": ["Account"], "key_field": "AccountId"}
},
"select_options": {},
},
}, mf

Expand All @@ -252,11 +263,13 @@ def test_generate_load_mapping__with_load_declarations(self, org_config):
"sf_object": "Account",
"api": DataApi.REST,
"table": "Account",
"select_options": {},
},
"Insert Contact": {
"sf_object": "Contact",
"api": DataApi.BULK,
"table": "Contact",
"select_options": {},
},
}, mf

Expand Down Expand Up @@ -288,24 +301,28 @@ def test_generate_load_mapping__with_upserts(self, org_config):
"Insert Account": {
"sf_object": "Account",
"table": "Account",
"select_options": {},
},
"Upsert Account Name": {
"sf_object": "Account",
"table": "Account",
"action": DataOperationType.UPSERT,
"update_key": ("Name",),
"fields": ["Name"],
"select_options": {},
},
"Etl_Upsert Account AccountNumber_Name": {
"sf_object": "Account",
"table": "Account",
"action": DataOperationType.ETL_UPSERT,
"update_key": ("AccountNumber", "Name"),
"fields": ["AccountNumber", "Name"],
"select_options": {},
},
"Insert Contact": {
"sf_object": "Contact",
"table": "Contact",
"select_options": {},
},
}, mf

Expand Down
167 changes: 85 additions & 82 deletions cumulusci/tasks/bulkdata/load.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,90 @@ def _execute_step(

return step.job_result

def process_lookup_fields(self, mapping, fields, polymorphic_fields):
"""Modify fields and priority fields based on lookup and polymorphic checks."""
for name, lookup in mapping.lookups.items():
if name in fields:
# Get the index of the lookup field before removing it
insert_index = fields.index(name)
# Remove the lookup field from fields
fields.remove(name)

# Do the same for priority fields
lookup_in_priority_fields = False
if name in mapping.select_options.priority_fields:
# Set flag to True
lookup_in_priority_fields = True
# Remove the lookup field from priority fields
del mapping.select_options.priority_fields[name]

# Check if this lookup field is polymorphic
if (
name in polymorphic_fields
and len(polymorphic_fields[name]["referenceTo"]) > 1
):
# Convert to list if string
if not isinstance(lookup.table, list):
lookup.table = [lookup.table]
# Polymorphic field handling
polymorphic_references = lookup.table
relationship_name = polymorphic_fields[name]["relationshipName"]

# Loop through each polymorphic type (e.g., Contact, Lead)
for ref_type in polymorphic_references:
# Find the mapping step for this polymorphic type
lookup_mapping_step = next(
(
step
for step in self.mapping.values()
if step.table == ref_type
),
None,
)
if lookup_mapping_step:
lookup_fields = lookup_mapping_step.get_load_field_list()
# Insert fields in the format {relationship_name}.{ref_type}.{lookup_field}
for field in lookup_fields:
fields.insert(
insert_index,
f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}",
)
insert_index += 1
if lookup_in_priority_fields:
mapping.select_options.priority_fields[
f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}"
] = f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}"

else:
# Non-polymorphic field handling
lookup_table = lookup.table

if isinstance(lookup_table, list):
lookup_table = lookup_table[0]

# Get the mapping step for the non-polymorphic reference
lookup_mapping_step = next(
(
step
for step in self.mapping.values()
if step.table == lookup_table
),
None,
)

if lookup_mapping_step:
relationship_name = polymorphic_fields[name]["relationshipName"]
lookup_fields = lookup_mapping_step.get_load_field_list()

# Insert the new fields at the same position as the removed lookup field
for field in lookup_fields:
fields.insert(insert_index, f"{relationship_name}.{field}")
insert_index += 1
if lookup_in_priority_fields:
mapping.select_options.priority_fields[
f"{relationship_name}.{field}"
] = f"{relationship_name}.{field}"

def configure_step(self, mapping):
"""Create a step appropriate to the action"""
bulk_mode = mapping.bulk_mode or self.bulk_mode or "Parallel"
Expand Down Expand Up @@ -370,85 +454,7 @@ def configure_step(self, mapping):
for field in describe_result["fields"]
if field["type"] == "reference"
}

# Loop through each lookup to get the corresponding fields
for name, lookup in mapping.lookups.items():
if name in fields:
# Get the index of the lookup field before removing it
insert_index = fields.index(name)
# Remove the lookup field from fields
fields.remove(name)

# Check if this lookup field is polymorphic
if (
name in polymorphic_fields
and len(polymorphic_fields[name]["referenceTo"]) > 1
):
# Convert to list if string
if not isinstance(lookup.table, list):
lookup.table = [lookup.table]
# Polymorphic field handling
polymorphic_references = lookup.table
relationship_name = polymorphic_fields[name][
"relationshipName"
]

# Loop through each polymorphic type (e.g., Contact, Lead)
for ref_type in polymorphic_references:
# Find the mapping step for this polymorphic type
lookup_mapping_step = next(
(
step
for step in self.mapping.values()
if step.sf_object == ref_type
),
None,
)

if lookup_mapping_step:
lookup_fields = (
lookup_mapping_step.get_load_field_list()
)
# Insert fields in the format {relationship_name}.{ref_type}.{lookup_field}
for field in lookup_fields:
fields.insert(
insert_index,
f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}",
)
insert_index += 1

else:
# Non-polymorphic field handling
lookup_table = lookup.table

if isinstance(lookup_table, list):
lookup_table = lookup_table[0]

# Get the mapping step for the non-polymorphic reference
lookup_mapping_step = next(
(
step
for step in self.mapping.values()
if step.sf_object == lookup_table
),
None,
)

if lookup_mapping_step:
relationship_name = polymorphic_fields[name][
"relationshipName"
]
lookup_fields = (
lookup_mapping_step.get_load_field_list()
)

# Insert the new fields at the same position as the removed lookup field
for field in lookup_fields:
fields.insert(
insert_index, f"{relationship_name}.{field}"
)
insert_index += 1

self.process_lookup_fields(mapping, fields, polymorphic_fields)
else:
action = mapping.action

Expand Down Expand Up @@ -503,9 +509,6 @@ def _stream_queried_data(self, mapping, local_ids, query):
pkey = row[0]
row = list(row[1:]) + statics

# Replace None values in row with empty strings
row = [value if value is not None else "" for value in row]

if mapping.anchor_date and (date_context[0] or date_context[1]):
row = adjust_relative_dates(
mapping, date_context, row, DataOperationType.INSERT
Expand Down
5 changes: 4 additions & 1 deletion cumulusci/tasks/bulkdata/mapping_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,16 +124,19 @@ def split_update_key(cls, val):
def validate_priority_fields(cls, values):
select_options = values.get("select_options")
fields_ = values.get("fields_", {})
lookups = values.get("lookups", {})

if select_options and select_options.priority_fields:
priority_field_names = set(select_options.priority_fields.keys())
field_names = set(fields_.keys())
lookup_names = set(lookups.keys())

# Check if all priority fields are present in the fields
missing_fields = priority_field_names - field_names
missing_fields = missing_fields - lookup_names
if missing_fields:
raise ValueError(
f"Priority fields {missing_fields} are not present in 'fields'"
f"Priority fields {missing_fields} are not present in 'fields' or 'lookups'"
)

return values
Expand Down
Loading

0 comments on commit 730ba6c

Please sign in to comment.