Skip to content

Commit

Permalink
refactors to bulk import's handling of account names and affiliated-o…
Browse files Browse the repository at this point in the history 
…rg fields + thrivent import fix for CLHS
  • Loading branch information
@brmeyer
brmeyer committed Jul 24, 2023
1 parent 0034cc5 commit 7f11e78
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 66 deletions.
98 changes: 49 additions & 49 deletions src/main/java/com/impactupgrade/nucleus/service/segment/SfdcCrmService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@
import com.sforce.soap.partner.SaveResult;
import com.sforce.soap.partner.sobject.SObject;
import com.sforce.ws.ConnectionException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.stripe.util.CaseInsensitiveMap;

import java.time.ZonedDateTime;
import java.util.ArrayList;
Expand Down Expand Up @@ -1106,7 +1105,7 @@ protected void processBulkImportCoreRecords(List<CrmImportEvent> importEvents) t
account = new SObject("Account");
account.setId(existingAccount.getId());

setBulkImportAccountFields(account, existingAccount, importEvent);
setBulkImportAccountFields(account, existingAccount, importEvent.account, importEvent.raw);
if (!batchUpdateAccounts.contains(existingAccount.getId())) {
batchUpdateAccounts.add(existingAccount.getId());
sfdcClient.batchUpdate(account);
Expand All @@ -1119,7 +1118,7 @@ protected void processBulkImportCoreRecords(List<CrmImportEvent> importEvents) t
account = new SObject("Account");
account.setId(existingAccount.getId());

setBulkImportAccountFields(account, existingAccount, importEvent);
setBulkImportAccountFields(account, existingAccount, importEvent.account, importEvent.raw);
if (!batchUpdateAccounts.contains(existingAccount.getId())) {
batchUpdateAccounts.add(existingAccount.getId());
sfdcClient.batchUpdate(account);
Expand All @@ -1132,7 +1131,7 @@ protected void processBulkImportCoreRecords(List<CrmImportEvent> importEvents) t
// TODO: This was mainly due to CLHS' original FACTS migration that created isolated households or, worse,
// combined grandparents into the student's household. Raiser's Edge has the correct relationships and
// households, so we're using this to override the past.
// account = insertBulkImportAccount(importEvent.contactLastName + " Household", importEvent,
// account = insertBulkImportAccount(importEvent.account, importEvent.raw,
// accountExtRefFieldName, existingAccountsByExtRef, accountMode);
}
}
Expand All @@ -1154,12 +1153,7 @@ protected void processBulkImportCoreRecords(List<CrmImportEvent> importEvents) t
// If we're in the second pass, we already know we need to insert the contact.
if (secondPass) {
if (account == null) {
String accountName = importEvent.account.name;
if (Strings.isNullOrEmpty(accountName)) {
accountName = importEvent.contactLastName + " Household";
}

account = insertBulkImportAccount(accountName, importEvent,
account = insertBulkImportAccount(importEvent.account, importEvent.raw,
accountExtRefFieldName, existingAccountsByExtRef, accountMode);
}

Expand All @@ -1176,7 +1170,7 @@ else if (accountMode && (!contactMode || !contactModeRow) && !Strings.isNullOrEm
}

if (existingAccount == null) {
account = insertBulkImportAccount(importEvent.account.name, importEvent, accountExtRefFieldName, existingAccountsByExtRef, accountMode);
account = insertBulkImportAccount(importEvent.account, importEvent.raw, accountExtRefFieldName, existingAccountsByExtRef, accountMode);
existingAccountsByName.put(importEvent.account.name.toLowerCase(Locale.ROOT), account);
} else {
account = updateBulkImportAccount(existingAccount, importEvent, batchUpdateAccounts, true);
Expand Down Expand Up @@ -1484,7 +1478,7 @@ protected SObject updateBulkImportAccount(SObject existingAccount, CrmImportEven
SObject account = new SObject("Account");
account.setId(existingAccount.getId());

setBulkImportAccountFields(account, existingAccount, importEvent);
setBulkImportAccountFields(account, existingAccount, importEvent.account, importEvent.raw);

if (!bulkUpdateAccounts.contains(account.getId())) {
bulkUpdateAccounts.add(account.getId());
Expand All @@ -1495,8 +1489,8 @@ protected SObject updateBulkImportAccount(SObject existingAccount, CrmImportEven
}

protected SObject insertBulkImportAccount(
String accountName,
CrmImportEvent importEvent,
CrmAccount crmAccount,
CaseInsensitiveMap<String> raw,
Optional<String> accountExtRefFieldName,
Map<String, SObject> existingAccountsByExtRef,
boolean accountImports
Expand All @@ -1510,8 +1504,7 @@ protected SObject insertBulkImportAccount(

SObject account = new SObject("Account");

setField(account, "Name", accountName);
setBulkImportAccountFields(account, null, importEvent);
setBulkImportAccountFields(account, null, crmAccount, raw);

String accountId = sfdcClient.insert(account).getId();
account.setId(accountId);
Expand Down Expand Up @@ -1659,35 +1652,42 @@ protected void setBulkImportContactFields(SObject contact, SObject existingConta

contact.setField("OwnerId", importEvent.contactOwnerId);

setBulkImportCustomFields(contact, existingContact, "Contact", importEvent);
setBulkImportCustomFields(contact, existingContact, "Contact", importEvent.raw);
}

protected void setBulkImportAccountFields(SObject account, SObject existingAccount, CrmImportEvent importEvent)
protected void setBulkImportAccountFields(SObject account, SObject existingAccount, CrmAccount crmAccount, CaseInsensitiveMap<String> raw)
throws ExecutionException {
if (!Strings.isNullOrEmpty(importEvent.account.recordTypeId)) {
account.setField("RecordTypeId", importEvent.account.recordTypeId);
} else if (!Strings.isNullOrEmpty(importEvent.account.recordTypeName)) {
account.setField("RecordTypeId", recordTypeNameToIdCache.get(importEvent.account.recordTypeName));
if (!Strings.isNullOrEmpty(crmAccount.recordTypeId)) {
account.setField("RecordTypeId", crmAccount.recordTypeId);
} else if (!Strings.isNullOrEmpty(crmAccount.recordTypeName)) {
account.setField("RecordTypeId", recordTypeNameToIdCache.get(crmAccount.recordTypeName));
}

String accountName = crmAccount.name;
if (Strings.isNullOrEmpty(accountName)) {
// Likely a household and likely to be overwritten by NPSP's household naming rules.
accountName = "Household";
}
account.setField("Name", accountName);

setField(account, "BillingStreet", importEvent.account.billingAddress.street);
setField(account, "BillingCity", importEvent.account.billingAddress.city);
setField(account, "BillingState", importEvent.account.billingAddress.state);
setField(account, "BillingPostalCode", importEvent.account.billingAddress.postalCode);
setField(account, "BillingCountry", importEvent.account.billingAddress.country);
setField(account, "ShippingStreet", importEvent.account.mailingAddress.street);
setField(account, "ShippingCity", importEvent.account.mailingAddress.city);
setField(account, "ShippingState", importEvent.account.mailingAddress.state);
setField(account, "ShippingPostalCode", importEvent.account.mailingAddress.postalCode);
setField(account, "ShippingCountry", importEvent.account.mailingAddress.country);
setField(account, "BillingStreet", crmAccount.billingAddress.street);
setField(account, "BillingCity", crmAccount.billingAddress.city);
setField(account, "BillingState", crmAccount.billingAddress.state);
setField(account, "BillingPostalCode", crmAccount.billingAddress.postalCode);
setField(account, "BillingCountry", crmAccount.billingAddress.country);
setField(account, "ShippingStreet", crmAccount.mailingAddress.street);
setField(account, "ShippingCity", crmAccount.mailingAddress.city);
setField(account, "ShippingState", crmAccount.mailingAddress.state);
setField(account, "ShippingPostalCode", crmAccount.mailingAddress.postalCode);
setField(account, "ShippingCountry", crmAccount.mailingAddress.country);

account.setField("Description", importEvent.account.description);
account.setField("OwnerId", importEvent.account.ownerId);
account.setField("Phone", importEvent.account.phone);
account.setField("Type", importEvent.account.type);
account.setField("Website", importEvent.account.website);
account.setField("Description", crmAccount.description);
account.setField("OwnerId", crmAccount.ownerId);
account.setField("Phone", crmAccount.phone);
account.setField("Type", crmAccount.type);
account.setField("Website", crmAccount.website);

setBulkImportCustomFields(account, existingAccount, "Account", importEvent);
setBulkImportCustomFields(account, existingAccount, "Account", raw);
}

// TODO: This mostly duplicates the primary import's accounts by-id, by-extref, and by-name. DRY it up?
Expand Down Expand Up @@ -1717,7 +1717,7 @@ protected void importOrgAffiliations(
org = new SObject("Account");
org.setId(existingOrg.getId());

setBulkImportAccountFields(org, existingOrg, importEvent);
setBulkImportAccountFields(org, existingOrg, crmOrg, importEvent.raw);
if (!batchUpdateAccounts.contains(existingOrg.getId())) {
batchUpdateAccounts.add(existingOrg.getId());
sfdcClient.batchUpdate(org);
Expand All @@ -1730,7 +1730,7 @@ protected void importOrgAffiliations(
org = new SObject("Account");
org.setId(existingOrg.getId());

setBulkImportAccountFields(org, existingOrg, importEvent);
setBulkImportAccountFields(org, existingOrg, crmOrg, importEvent.raw);
if (!batchUpdateAccounts.contains(existingOrg.getId())) {
batchUpdateAccounts.add(existingOrg.getId());
sfdcClient.batchUpdate(org);
Expand All @@ -1743,13 +1743,13 @@ protected void importOrgAffiliations(
// TODO: This was mainly due to CLHS' original FACTS migration that created isolated households or, worse,
// combined grandparents into the student's household. Raiser's Edge has the correct relationships and
// households, so we're using this to override the past.
org = insertBulkImportAccount(crmOrg.name, importEvent, orgExtRefFieldName, existingAccountsByExtRef, true);
org = insertBulkImportAccount(crmOrg, importEvent.raw, orgExtRefFieldName, existingAccountsByExtRef, true);
}
} else {
SObject existingOrg = existingAccountsByName.get(crmOrg.name.toLowerCase(Locale.ROOT)).stream().findFirst().orElse(null);

if (existingOrg == null) {
org = insertBulkImportAccount(crmOrg.name, importEvent, orgExtRefFieldName, existingAccountsByExtRef, true);
org = insertBulkImportAccount(crmOrg, importEvent.raw, orgExtRefFieldName, existingAccountsByExtRef, true);
existingAccountsByName.put(crmOrg.name.toLowerCase(Locale.ROOT), org);
} else {
org = updateBulkImportAccount(existingOrg, importEvent, batchUpdateAccounts, true);
Expand Down Expand Up @@ -1852,7 +1852,7 @@ protected void setBulkImportRecurringDonationFields(SObject recurringDonation, S

recurringDonation.setField("OwnerId", importEvent.recurringDonationOwnerId);

setBulkImportCustomFields(recurringDonation, existingRecurringDonation, "Recurring Donation", importEvent);
setBulkImportCustomFields(recurringDonation, existingRecurringDonation, "Recurring Donation", importEvent.raw);
}

protected void setBulkImportOpportunityFields(SObject opportunity, SObject existingOpportunity, CrmImportEvent importEvent)
Expand Down Expand Up @@ -1882,13 +1882,13 @@ protected void setBulkImportOpportunityFields(SObject opportunity, SObject exist

opportunity.setField("OwnerId", importEvent.opportunityOwnerId);

setBulkImportCustomFields(opportunity, existingOpportunity, "Opportunity", importEvent);
setBulkImportCustomFields(opportunity, existingOpportunity, "Opportunity", importEvent.raw);
}

protected void setBulkImportCustomFields(SObject sObject, SObject existingSObject, String type, CrmImportEvent importEvent) {
protected void setBulkImportCustomFields(SObject sObject, SObject existingSObject, String type, CaseInsensitiveMap<String> raw) {
String prefix = type + " Custom ";

importEvent.raw.entrySet().stream().filter(entry -> entry.getKey().startsWith(prefix) && !Strings.isNullOrEmpty(entry.getValue())).forEach(entry -> {
raw.entrySet().stream().filter(entry -> entry.getKey().startsWith(prefix) && !Strings.isNullOrEmpty(entry.getValue())).forEach(entry -> {
String key = entry.getKey().replace(prefix, "");

if (key.startsWith("Append ")) {
Expand Down Expand Up @@ -1940,10 +1940,10 @@ protected void processBulkImportCampaignRecords(List<CrmImportEvent> importEvent

if (!Strings.isNullOrEmpty(importEvent.campaignId)) {
campaign.setId(importEvent.campaignId);
setBulkImportCustomFields(campaign, existingCampaignById.get(importEvent.campaignId), "Campaign", importEvent);
setBulkImportCustomFields(campaign, existingCampaignById.get(importEvent.campaignId), "Campaign", importEvent.raw);
sfdcClient.batchUpdate(campaign);
} else {
setBulkImportCustomFields(campaign, null, "Campaign", importEvent);
setBulkImportCustomFields(campaign, null, "Campaign", importEvent.raw);
sfdcClient.batchInsert(campaign);
}

Expand Down
26 changes: 9 additions & 17 deletions src/main/java/com/impactupgrade/nucleus/util/RaisersEdgeToSalesforce.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ private static void migrate(Environment env) throws Exception {
// }
// sfdcClient.batchFlush();
//
// File giftsFile = new File("/home/brmeyer/Downloads/RE Export June 2023/Gifts-with-Installments-v7-check-ref-number.xlsx");
// File giftsFile = new File("/home/brmeyer/Downloads/Gifts-with-Installments-v7-check-ref-number.xlsx");
// InputStream giftInputStream = new FileInputStream(giftsFile);
// List<Map<String, String>> giftRows = Utils.getExcelData(giftInputStream);

Expand All @@ -394,7 +394,7 @@ private static void migrate(Environment env) throws Exception {
// // TODO: The following completely skips Bulk Upsert!
//
// Map<String, String> campaignNameToId = sfdcClient.getCampaigns().stream()
// .collect(Collectors.toMap(c -> (String) c.getField("Name"), c -> c.getId()));
// .collect(Collectors.toMap(c -> (String) c.getField("Name"), c -> c.getId(), (c1, c2) -> c1));
//
// for (int i = 0; i < giftRows.size(); i++) {
// log.info("processing campaign row {}", i + 2);
Expand Down Expand Up @@ -807,6 +807,10 @@ private static void migrate(Environment env) throws Exception {
// SObject sfdcOpportunity = buildDonation(giftRow, campaignNameToId);
//
// String constituentId = giftRow.get("Gf_CnBio_ID");
// // TODO: CLHS-specific workaround -- one Thrivent constituent had no ID, for some reason
// if (Strings.isNullOrEmpty(constituentId)) {
// constituentId = "thrivent";
// }
// SObject contact = constituentIdToContact.get(constituentId);
// SObject account = constituentIdToAccount.get(constituentId);
// String donorId;
Expand All @@ -818,18 +822,6 @@ private static void migrate(Environment env) throws Exception {
// if (account != null) {
// sfdcOpportunity.setField("AccountId", account.getId());
// donorId = account.getId();
// } else if (!Strings.isNullOrEmpty(giftRow.get("Gf_CnBio_Org_Name"))) {
// List<SObject> accountsByName = sfdcClient.getAccountsByName(giftRow.get("Gf_CnBio_Org_Name")).stream().filter(a -> giftRow.get("Gf_CnBio_Org_Name").equalsIgnoreCase((String) a.getField("Name"))).toList();
// if (accountsByName.size() == 1) {
// sfdcOpportunity.setField("AccountId", accountsByName.get(0).getId());
// donorId = accountsByName.get(0).getId();
// } else if (accountsByName.size() > 1) {
// log.warn("DUPLICATE CONSTITUENTS: {}", giftRow.get("Gf_CnBio_Org_Name"));
// continue;
// } else {
// log.warn("MISSING CONSTITUENT: {}", constituentId);
// continue;
// }
// } else {
// log.warn("MISSING CONSTITUENT: {}", constituentId);
// continue;
Expand All @@ -842,18 +834,18 @@ private static void migrate(Environment env) throws Exception {
// oppInsertsByDonorId.get(donorId).add(sfdcOpportunity);
// }
//
// counter = 1;
// counter = 0;
// int total = oppInsertsByDonorId.size();
// for (Map.Entry<String, List<SObject>> entry : oppInsertsByDonorId.entrySet()) {
// counter++;
// log.info("processing donor {} of {}", counter, total);
//
// for (SObject opp : entry.getValue()) {
//
// // TODO: This really slows things down, but we're running into lock contention if a single contact/account's
// // opportunities are spread across multiple inserts. We could optimize it by looking at current batch sizes
// // and flush only when the next batch will push it over the edge?
// sfdcClient.batchInsert(opp);
//// sfdcClient.batchInsert(opp);
// sfdcClient.insert(opp);
// }
// sfdcClient.batchFlush();
// }
Expand Down

0 comments on commit 7f11e78

Please sign in to comment.