From 7f11e78b6ebac37887a20cb22eb2506ec1efbb93 Mon Sep 17 00:00:00 2001 From: Brett Meyer Date: Mon, 24 Jul 2023 01:03:08 -0400 Subject: [PATCH] refactors to bulk import's handling of account names and affiliated-org fields + thrivent import fix for CLHS --- .../service/segment/SfdcCrmService.java | 98 +++++++++---------- .../nucleus/util/RaisersEdgeToSalesforce.java | 26 ++--- 2 files changed, 58 insertions(+), 66 deletions(-) diff --git a/src/main/java/com/impactupgrade/nucleus/service/segment/SfdcCrmService.java b/src/main/java/com/impactupgrade/nucleus/service/segment/SfdcCrmService.java index 097bb1854..aabe10516 100644 --- a/src/main/java/com/impactupgrade/nucleus/service/segment/SfdcCrmService.java +++ b/src/main/java/com/impactupgrade/nucleus/service/segment/SfdcCrmService.java @@ -34,8 +34,7 @@ import com.sforce.soap.partner.SaveResult; import com.sforce.soap.partner.sobject.SObject; import com.sforce.ws.ConnectionException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.stripe.util.CaseInsensitiveMap; import java.time.ZonedDateTime; import java.util.ArrayList; @@ -1106,7 +1105,7 @@ protected void processBulkImportCoreRecords(List importEvents) t account = new SObject("Account"); account.setId(existingAccount.getId()); - setBulkImportAccountFields(account, existingAccount, importEvent); + setBulkImportAccountFields(account, existingAccount, importEvent.account, importEvent.raw); if (!batchUpdateAccounts.contains(existingAccount.getId())) { batchUpdateAccounts.add(existingAccount.getId()); sfdcClient.batchUpdate(account); @@ -1119,7 +1118,7 @@ protected void processBulkImportCoreRecords(List importEvents) t account = new SObject("Account"); account.setId(existingAccount.getId()); - setBulkImportAccountFields(account, existingAccount, importEvent); + setBulkImportAccountFields(account, existingAccount, importEvent.account, importEvent.raw); if (!batchUpdateAccounts.contains(existingAccount.getId())) { batchUpdateAccounts.add(existingAccount.getId()); sfdcClient.batchUpdate(account); @@ -1132,7 +1131,7 @@ protected void processBulkImportCoreRecords(List importEvents) t // TODO: This was mainly due to CLHS' original FACTS migration that created isolated households or, worse, // combined grandparents into the student's household. Raiser's Edge has the correct relationships and // households, so we're using this to override the past. -// account = insertBulkImportAccount(importEvent.contactLastName + " Household", importEvent, +// account = insertBulkImportAccount(importEvent.account, importEvent.raw, // accountExtRefFieldName, existingAccountsByExtRef, accountMode); } } @@ -1154,12 +1153,7 @@ protected void processBulkImportCoreRecords(List importEvents) t // If we're in the second pass, we already know we need to insert the contact. if (secondPass) { if (account == null) { - String accountName = importEvent.account.name; - if (Strings.isNullOrEmpty(accountName)) { - accountName = importEvent.contactLastName + " Household"; - } - - account = insertBulkImportAccount(accountName, importEvent, + account = insertBulkImportAccount(importEvent.account, importEvent.raw, accountExtRefFieldName, existingAccountsByExtRef, accountMode); } @@ -1176,7 +1170,7 @@ else if (accountMode && (!contactMode || !contactModeRow) && !Strings.isNullOrEm } if (existingAccount == null) { - account = insertBulkImportAccount(importEvent.account.name, importEvent, accountExtRefFieldName, existingAccountsByExtRef, accountMode); + account = insertBulkImportAccount(importEvent.account, importEvent.raw, accountExtRefFieldName, existingAccountsByExtRef, accountMode); existingAccountsByName.put(importEvent.account.name.toLowerCase(Locale.ROOT), account); } else { account = updateBulkImportAccount(existingAccount, importEvent, batchUpdateAccounts, true); @@ -1484,7 +1478,7 @@ protected SObject updateBulkImportAccount(SObject existingAccount, CrmImportEven SObject account = new SObject("Account"); account.setId(existingAccount.getId()); - setBulkImportAccountFields(account, existingAccount, importEvent); + setBulkImportAccountFields(account, existingAccount, importEvent.account, importEvent.raw); if (!bulkUpdateAccounts.contains(account.getId())) { bulkUpdateAccounts.add(account.getId()); @@ -1495,8 +1489,8 @@ protected SObject updateBulkImportAccount(SObject existingAccount, CrmImportEven } protected SObject insertBulkImportAccount( - String accountName, - CrmImportEvent importEvent, + CrmAccount crmAccount, + CaseInsensitiveMap raw, Optional accountExtRefFieldName, Map existingAccountsByExtRef, boolean accountImports @@ -1510,8 +1504,7 @@ protected SObject insertBulkImportAccount( SObject account = new SObject("Account"); - setField(account, "Name", accountName); - setBulkImportAccountFields(account, null, importEvent); + setBulkImportAccountFields(account, null, crmAccount, raw); String accountId = sfdcClient.insert(account).getId(); account.setId(accountId); @@ -1659,35 +1652,42 @@ protected void setBulkImportContactFields(SObject contact, SObject existingConta contact.setField("OwnerId", importEvent.contactOwnerId); - setBulkImportCustomFields(contact, existingContact, "Contact", importEvent); + setBulkImportCustomFields(contact, existingContact, "Contact", importEvent.raw); } - protected void setBulkImportAccountFields(SObject account, SObject existingAccount, CrmImportEvent importEvent) + protected void setBulkImportAccountFields(SObject account, SObject existingAccount, CrmAccount crmAccount, CaseInsensitiveMap raw) throws ExecutionException { - if (!Strings.isNullOrEmpty(importEvent.account.recordTypeId)) { - account.setField("RecordTypeId", importEvent.account.recordTypeId); - } else if (!Strings.isNullOrEmpty(importEvent.account.recordTypeName)) { - account.setField("RecordTypeId", recordTypeNameToIdCache.get(importEvent.account.recordTypeName)); + if (!Strings.isNullOrEmpty(crmAccount.recordTypeId)) { + account.setField("RecordTypeId", crmAccount.recordTypeId); + } else if (!Strings.isNullOrEmpty(crmAccount.recordTypeName)) { + account.setField("RecordTypeId", recordTypeNameToIdCache.get(crmAccount.recordTypeName)); + } + + String accountName = crmAccount.name; + if (Strings.isNullOrEmpty(accountName)) { + // Likely a household and likely to be overwritten by NPSP's household naming rules. + accountName = "Household"; } + account.setField("Name", accountName); - setField(account, "BillingStreet", importEvent.account.billingAddress.street); - setField(account, "BillingCity", importEvent.account.billingAddress.city); - setField(account, "BillingState", importEvent.account.billingAddress.state); - setField(account, "BillingPostalCode", importEvent.account.billingAddress.postalCode); - setField(account, "BillingCountry", importEvent.account.billingAddress.country); - setField(account, "ShippingStreet", importEvent.account.mailingAddress.street); - setField(account, "ShippingCity", importEvent.account.mailingAddress.city); - setField(account, "ShippingState", importEvent.account.mailingAddress.state); - setField(account, "ShippingPostalCode", importEvent.account.mailingAddress.postalCode); - setField(account, "ShippingCountry", importEvent.account.mailingAddress.country); + setField(account, "BillingStreet", crmAccount.billingAddress.street); + setField(account, "BillingCity", crmAccount.billingAddress.city); + setField(account, "BillingState", crmAccount.billingAddress.state); + setField(account, "BillingPostalCode", crmAccount.billingAddress.postalCode); + setField(account, "BillingCountry", crmAccount.billingAddress.country); + setField(account, "ShippingStreet", crmAccount.mailingAddress.street); + setField(account, "ShippingCity", crmAccount.mailingAddress.city); + setField(account, "ShippingState", crmAccount.mailingAddress.state); + setField(account, "ShippingPostalCode", crmAccount.mailingAddress.postalCode); + setField(account, "ShippingCountry", crmAccount.mailingAddress.country); - account.setField("Description", importEvent.account.description); - account.setField("OwnerId", importEvent.account.ownerId); - account.setField("Phone", importEvent.account.phone); - account.setField("Type", importEvent.account.type); - account.setField("Website", importEvent.account.website); + account.setField("Description", crmAccount.description); + account.setField("OwnerId", crmAccount.ownerId); + account.setField("Phone", crmAccount.phone); + account.setField("Type", crmAccount.type); + account.setField("Website", crmAccount.website); - setBulkImportCustomFields(account, existingAccount, "Account", importEvent); + setBulkImportCustomFields(account, existingAccount, "Account", raw); } // TODO: This mostly duplicates the primary import's accounts by-id, by-extref, and by-name. DRY it up? @@ -1717,7 +1717,7 @@ protected void importOrgAffiliations( org = new SObject("Account"); org.setId(existingOrg.getId()); - setBulkImportAccountFields(org, existingOrg, importEvent); + setBulkImportAccountFields(org, existingOrg, crmOrg, importEvent.raw); if (!batchUpdateAccounts.contains(existingOrg.getId())) { batchUpdateAccounts.add(existingOrg.getId()); sfdcClient.batchUpdate(org); @@ -1730,7 +1730,7 @@ protected void importOrgAffiliations( org = new SObject("Account"); org.setId(existingOrg.getId()); - setBulkImportAccountFields(org, existingOrg, importEvent); + setBulkImportAccountFields(org, existingOrg, crmOrg, importEvent.raw); if (!batchUpdateAccounts.contains(existingOrg.getId())) { batchUpdateAccounts.add(existingOrg.getId()); sfdcClient.batchUpdate(org); @@ -1743,13 +1743,13 @@ protected void importOrgAffiliations( // TODO: This was mainly due to CLHS' original FACTS migration that created isolated households or, worse, // combined grandparents into the student's household. Raiser's Edge has the correct relationships and // households, so we're using this to override the past. - org = insertBulkImportAccount(crmOrg.name, importEvent, orgExtRefFieldName, existingAccountsByExtRef, true); + org = insertBulkImportAccount(crmOrg, importEvent.raw, orgExtRefFieldName, existingAccountsByExtRef, true); } } else { SObject existingOrg = existingAccountsByName.get(crmOrg.name.toLowerCase(Locale.ROOT)).stream().findFirst().orElse(null); if (existingOrg == null) { - org = insertBulkImportAccount(crmOrg.name, importEvent, orgExtRefFieldName, existingAccountsByExtRef, true); + org = insertBulkImportAccount(crmOrg, importEvent.raw, orgExtRefFieldName, existingAccountsByExtRef, true); existingAccountsByName.put(crmOrg.name.toLowerCase(Locale.ROOT), org); } else { org = updateBulkImportAccount(existingOrg, importEvent, batchUpdateAccounts, true); @@ -1852,7 +1852,7 @@ protected void setBulkImportRecurringDonationFields(SObject recurringDonation, S recurringDonation.setField("OwnerId", importEvent.recurringDonationOwnerId); - setBulkImportCustomFields(recurringDonation, existingRecurringDonation, "Recurring Donation", importEvent); + setBulkImportCustomFields(recurringDonation, existingRecurringDonation, "Recurring Donation", importEvent.raw); } protected void setBulkImportOpportunityFields(SObject opportunity, SObject existingOpportunity, CrmImportEvent importEvent) @@ -1882,13 +1882,13 @@ protected void setBulkImportOpportunityFields(SObject opportunity, SObject exist opportunity.setField("OwnerId", importEvent.opportunityOwnerId); - setBulkImportCustomFields(opportunity, existingOpportunity, "Opportunity", importEvent); + setBulkImportCustomFields(opportunity, existingOpportunity, "Opportunity", importEvent.raw); } - protected void setBulkImportCustomFields(SObject sObject, SObject existingSObject, String type, CrmImportEvent importEvent) { + protected void setBulkImportCustomFields(SObject sObject, SObject existingSObject, String type, CaseInsensitiveMap raw) { String prefix = type + " Custom "; - importEvent.raw.entrySet().stream().filter(entry -> entry.getKey().startsWith(prefix) && !Strings.isNullOrEmpty(entry.getValue())).forEach(entry -> { + raw.entrySet().stream().filter(entry -> entry.getKey().startsWith(prefix) && !Strings.isNullOrEmpty(entry.getValue())).forEach(entry -> { String key = entry.getKey().replace(prefix, ""); if (key.startsWith("Append ")) { @@ -1940,10 +1940,10 @@ protected void processBulkImportCampaignRecords(List importEvent if (!Strings.isNullOrEmpty(importEvent.campaignId)) { campaign.setId(importEvent.campaignId); - setBulkImportCustomFields(campaign, existingCampaignById.get(importEvent.campaignId), "Campaign", importEvent); + setBulkImportCustomFields(campaign, existingCampaignById.get(importEvent.campaignId), "Campaign", importEvent.raw); sfdcClient.batchUpdate(campaign); } else { - setBulkImportCustomFields(campaign, null, "Campaign", importEvent); + setBulkImportCustomFields(campaign, null, "Campaign", importEvent.raw); sfdcClient.batchInsert(campaign); } diff --git a/src/main/java/com/impactupgrade/nucleus/util/RaisersEdgeToSalesforce.java b/src/main/java/com/impactupgrade/nucleus/util/RaisersEdgeToSalesforce.java index 6b0ffee5c..6ecff8511 100644 --- a/src/main/java/com/impactupgrade/nucleus/util/RaisersEdgeToSalesforce.java +++ b/src/main/java/com/impactupgrade/nucleus/util/RaisersEdgeToSalesforce.java @@ -383,7 +383,7 @@ private static void migrate(Environment env) throws Exception { // } // sfdcClient.batchFlush(); // -// File giftsFile = new File("/home/brmeyer/Downloads/RE Export June 2023/Gifts-with-Installments-v7-check-ref-number.xlsx"); +// File giftsFile = new File("/home/brmeyer/Downloads/Gifts-with-Installments-v7-check-ref-number.xlsx"); // InputStream giftInputStream = new FileInputStream(giftsFile); // List> giftRows = Utils.getExcelData(giftInputStream); @@ -394,7 +394,7 @@ private static void migrate(Environment env) throws Exception { // // TODO: The following completely skips Bulk Upsert! // // Map campaignNameToId = sfdcClient.getCampaigns().stream() -// .collect(Collectors.toMap(c -> (String) c.getField("Name"), c -> c.getId())); +// .collect(Collectors.toMap(c -> (String) c.getField("Name"), c -> c.getId(), (c1, c2) -> c1)); // // for (int i = 0; i < giftRows.size(); i++) { // log.info("processing campaign row {}", i + 2); @@ -807,6 +807,10 @@ private static void migrate(Environment env) throws Exception { // SObject sfdcOpportunity = buildDonation(giftRow, campaignNameToId); // // String constituentId = giftRow.get("Gf_CnBio_ID"); +// // TODO: CLHS-specific workaround -- one Thrivent constituent had no ID, for some reason +// if (Strings.isNullOrEmpty(constituentId)) { +// constituentId = "thrivent"; +// } // SObject contact = constituentIdToContact.get(constituentId); // SObject account = constituentIdToAccount.get(constituentId); // String donorId; @@ -818,18 +822,6 @@ private static void migrate(Environment env) throws Exception { // if (account != null) { // sfdcOpportunity.setField("AccountId", account.getId()); // donorId = account.getId(); -// } else if (!Strings.isNullOrEmpty(giftRow.get("Gf_CnBio_Org_Name"))) { -// List accountsByName = sfdcClient.getAccountsByName(giftRow.get("Gf_CnBio_Org_Name")).stream().filter(a -> giftRow.get("Gf_CnBio_Org_Name").equalsIgnoreCase((String) a.getField("Name"))).toList(); -// if (accountsByName.size() == 1) { -// sfdcOpportunity.setField("AccountId", accountsByName.get(0).getId()); -// donorId = accountsByName.get(0).getId(); -// } else if (accountsByName.size() > 1) { -// log.warn("DUPLICATE CONSTITUENTS: {}", giftRow.get("Gf_CnBio_Org_Name")); -// continue; -// } else { -// log.warn("MISSING CONSTITUENT: {}", constituentId); -// continue; -// } // } else { // log.warn("MISSING CONSTITUENT: {}", constituentId); // continue; @@ -842,18 +834,18 @@ private static void migrate(Environment env) throws Exception { // oppInsertsByDonorId.get(donorId).add(sfdcOpportunity); // } // -// counter = 1; +// counter = 0; // int total = oppInsertsByDonorId.size(); // for (Map.Entry> entry : oppInsertsByDonorId.entrySet()) { // counter++; // log.info("processing donor {} of {}", counter, total); // // for (SObject opp : entry.getValue()) { -// // // TODO: This really slows things down, but we're running into lock contention if a single contact/account's // // opportunities are spread across multiple inserts. We could optimize it by looking at current batch sizes // // and flush only when the next batch will push it over the edge? -// sfdcClient.batchInsert(opp); +//// sfdcClient.batchInsert(opp); +// sfdcClient.insert(opp); // } // sfdcClient.batchFlush(); // }