21367: [Spike] Nightly job to filter and sync contacts

src/main/java/com/impactupgrade/nucleus/App.java

@@ -19,6 +19,7 @@
 import com.impactupgrade.nucleus.controller.ScheduledJobController;
 import com.impactupgrade.nucleus.controller.SfdcController;
 import com.impactupgrade.nucleus.controller.StripeController;
+import com.impactupgrade.nucleus.controller.DataSyncController;
 import com.impactupgrade.nucleus.controller.TwilioController;
 import com.impactupgrade.nucleus.environment.EnvironmentFactory;
 import com.impactupgrade.nucleus.security.SecurityExceptionMapper;
@@ -89,6 +90,7 @@ public void start() throws Exception {
 
     apiConfig.register(backupController());
     apiConfig.register(communicationController());
+    apiConfig.register(dataSyncController());
     apiConfig.register(crmController());
     apiConfig.register(donationFormController());
     apiConfig.register(emailController());
@@ -145,6 +147,7 @@ public void registerServlets(ServletContextHandler context) throws Exception {}
   // Allow orgs to override specific controllers.
   protected BackupController backupController() { return new BackupController(envFactory); }
   protected CommunicationController communicationController() { return new CommunicationController(envFactory); }
+  protected DataSyncController dataSyncController() { return new DataSyncController(envFactory); }
   protected CrmController crmController() { return new CrmController(envFactory); }
   protected DonationFormController donationFormController() { return new DonationFormController(envFactory); }
   protected EmailController emailController() { return new EmailController(envFactory); }

src/main/java/com/impactupgrade/nucleus/client/SfdcClient.java

@@ -611,6 +611,31 @@ protected QueryResult querySmsContacts(String updatedSinceClause, String filter,
     return query(query);
   }
 
+  public List<QueryResult> getDonorContacts(Calendar updatedSince)
+      throws ConnectionException, InterruptedException {
+    List<QueryResult> queryResults = new ArrayList<>();
+
+    String updatedSinceClause = "";
+    if (updatedSince != null) {
+      updatedSinceClause = "SystemModStamp >= " + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").format(updatedSince.getTime());
+    }
+    queryResults.add(queryDonorContacts(updatedSinceClause));
+
+    return queryResults;
+  }
+
+  protected QueryResult queryDonorContacts(String updatedSinceClause) throws ConnectionException, InterruptedException {
+    if (Strings.isNullOrEmpty(updatedSinceClause)) {
+      env.logJobWarn("no filter provided; out of caution, skipping the query to protect API limits");
+      return new QueryResult();
+    }
+    String query = "SELECT " + getFieldsList(CONTACT_FIELDS, env.getConfig().salesforce.customQueryFields.contact, null) + " " +
+        "FROM Contact " +
+        "WHERE " + updatedSinceClause +
+        "AND Account.npo02__TotalOppAmount__c > 0.0";
+    return query(query);
+  }
+
   public List<SObject> searchContacts(ContactSearch contactSearch, String... extraFields)
       throws ConnectionException, InterruptedException {
     List<String> clauses = new ArrayList<>();
@@ -780,6 +805,13 @@ public List<SObject> getDonationsByAccountId(String accountId, String... extraFi
     return queryListAutoPaged(query);
   }
 
+  public List<SObject> getDonationsUpdatedAfter(Calendar updatedSince, String... extraFields) throws ConnectionException, InterruptedException {
+    String updatedSinceClause = "SystemModStamp >= " + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").format(updatedSince.getTime());
+    String query = "select " + getFieldsList(DONATION_FIELDS, env.getConfig().salesforce.customQueryFields.donation, extraFields) + " from Opportunity " +
+        "where " + updatedSinceClause + " AND stageName = 'Closed Won' ORDER BY CloseDate DESC ";
+    return queryListAutoPaged(query);
+  }
+
   public Optional<SObject> getNextPledgedDonationByRecurringDonationId(String recurringDonationId, String... extraFields) throws ConnectionException, InterruptedException {
     // TODO: Using TOMORROW to account for timezone issues -- we can typically get away with that approach
     // since most RDs are monthly...

src/main/java/com/impactupgrade/nucleus/controller/DataSyncController.java

@@ -0,0 +1,67 @@
+package com.impactupgrade.nucleus.controller;
+
+import com.impactupgrade.nucleus.entity.JobStatus;
+import com.impactupgrade.nucleus.entity.JobType;
+import com.impactupgrade.nucleus.environment.Environment;
+import com.impactupgrade.nucleus.environment.EnvironmentFactory;
+import com.impactupgrade.nucleus.service.segment.DataSyncService;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.util.Calendar;
+
+@Path("/data-sync")
+public class DataSyncController {
+
+  protected final EnvironmentFactory environmentFactory;
+
+  public DataSyncController(EnvironmentFactory environmentFactory) {
+    this.environmentFactory = environmentFactory;
+  }
+
+  @GET
+  @Path("/contacts/daily")
+  public Response syncDaily(@QueryParam("syncDays") Integer syncDays, @Context HttpServletRequest request) throws Exception {
+    Environment env = environmentFactory.init(request);
+
+    Calendar lastSync = Calendar.getInstance();
+    // run daily, but setting this high to catch previous misses
+    if (syncDays == null || syncDays <= 0) {
+      syncDays = 3;
+    }
+    lastSync.add(Calendar.DATE, -syncDays);
+
+    Runnable thread = () -> {
+      try {
+        String jobName = "Contacts Sync: Daily";
+        env.startJobLog(JobType.EVENT, null, jobName, "Nucleus Portal");
+        boolean success = true;
+
+        for (DataSyncService dataSyncService : env.allDataSyncServices()) {
+          try {
+            dataSyncService.syncContacts(lastSync);
+            env.logJobInfo("{}: sync contacts done", dataSyncService.name());
+          } catch (Exception e) {
+            env.logJobError("sync contacts failed for {}", dataSyncService.name(), e);
+            env.logJobError(e.getMessage());
+            success = false;
+          }
+        }
+
+        env.endJobLog(success ? JobStatus.DONE : JobStatus.FAILED);
+
+      } catch (Exception e) {
+        env.logJobError("sync contacts failed!", e);
+        env.logJobError(e.getMessage());
+        env.endJobLog(JobStatus.FAILED);
+      }
+    };
+    new Thread(thread).start();
+
+    return Response.ok().build();
+  }
+}

src/main/java/com/impactupgrade/nucleus/environment/Environment.java

@@ -32,6 +32,7 @@
 import com.impactupgrade.nucleus.service.segment.BareCrmService;
 import com.impactupgrade.nucleus.service.segment.PaymentGatewayService;
 import com.impactupgrade.nucleus.service.segment.SegmentService;
+import com.impactupgrade.nucleus.service.segment.DataSyncService;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections4.map.CaseInsensitiveMap;
 import org.apache.http.client.utils.URLEncodedUtils;
@@ -202,6 +203,14 @@ public CommunicationService communicationService(String name) {
     return segmentService(name, CommunicationService.class);
   }
 
+  public DataSyncService dataSyncService(String name) {
+    return segmentService(name, DataSyncService.class);
+  }
+
+  public List<DataSyncService> allDataSyncServices() {
+    return segmentServices(DataSyncService.class);
+  }
+
   public Optional<AccountingPlatformService> accountingPlatformService() {
     if (Strings.isNullOrEmpty(getConfig().accountingPrimary)) {
       return Optional.empty();

src/main/java/com/impactupgrade/nucleus/service/logic/AccountingService.java

@@ -58,7 +58,8 @@ protected void processTransaction(CrmDonation crmDonation) throws Exception {
             return;
         }
 
-        String contactId = _accountingPlatformService.get().updateOrCreateContact(crmContact);
+        String contactId = _accountingPlatformService.get()
+            .updateOrCreateContacts(List.of(crmContact)).stream().findFirst().orElse(null);
         if (!Strings.isNullOrEmpty(contactId)) {
             env.logJobInfo("Upserted contact: {}", contactId);
 

src/main/java/com/impactupgrade/nucleus/service/segment/AccountingPlatformService.java

@@ -8,13 +8,14 @@
 import com.impactupgrade.nucleus.model.CrmContact;
 import com.impactupgrade.nucleus.model.CrmDonation;
 
+import java.util.List;
 import java.util.Optional;
 
 public interface AccountingPlatformService extends SegmentService {
 
     Optional<AccountingTransaction> getTransaction(CrmDonation crmDonation) throws Exception;
 
-    String updateOrCreateContact(CrmContact crmContact) throws Exception;
+    List<String> updateOrCreateContacts(List<CrmContact> crmContacts) throws Exception;
 
     String createTransaction(AccountingTransaction accountingTransaction) throws Exception;
 

src/main/java/com/impactupgrade/nucleus/service/segment/BareCrmService.java

@@ -157,6 +157,11 @@ public void insertDonationDeposit(List<CrmDonation> crmDonations) throws Excepti
 
   }
 
+  @Override
+  public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+    return List.of();
+  }
+
   @Override
   public String insertRecurringDonation(CrmRecurringDonation crmRecurringDonation) throws Exception {
     return null;
@@ -217,6 +222,11 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, Environmen
     return new PagedResults<>();
   }
 
+  @Override
+  public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+    return new PagedResults<>();
+  }
+
   @Override
   public Map<String, String> getContactLists(CrmContactListType listType) throws Exception {
     return Collections.emptyMap();

src/main/java/com/impactupgrade/nucleus/service/segment/BloomerangCrmService.java

@@ -379,6 +379,11 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, Environmen
     return new PagedResults<>();
   }
 
+  @Override
+  public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+    return new PagedResults<>();
+  }
+
   @Override
   public double getDonationsTotal(String filter) throws Exception {
     throw new RuntimeException("not implemented");
@@ -492,6 +497,11 @@ public void insertDonationDeposit(List<CrmDonation> crmDonations) throws Excepti
     // currently no deposit management
   }
 
+  @Override
+  public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+    return List.of();
+  }
+
   @Override
   public Optional<CrmRecurringDonation> getRecurringDonationById(String id) throws Exception {
     Donation recurringDonation = getDonation(id);

src/main/java/com/impactupgrade/nucleus/service/segment/CrmService.java

@@ -122,6 +122,7 @@ default Optional<CrmDonation> getDonationByTransactionId(String transactionId) t
   void updateDonation(CrmDonation crmDonation) throws Exception;
   void refundDonation(CrmDonation crmDonation) throws Exception;
   void insertDonationDeposit(List<CrmDonation> crmDonations) throws Exception;
+  List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception;
 
   // Some CRMs do not have full-blown notions of RDs, so no RD ID. Searching by the payment gateway's
   // subscription is at times the only option.
@@ -189,6 +190,8 @@ default void batchFlush() throws Exception {
   PagedResults<CrmContact> getEmailContacts(Calendar updatedSince, EnvironmentConfig.CommunicationList communicationList) throws Exception;
   PagedResults<CrmAccount> getEmailAccounts(Calendar updatedSince, EnvironmentConfig.CommunicationList communicationList) throws Exception;
   PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, EnvironmentConfig.CommunicationList communicationList) throws Exception;
+  PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception;
+
   // Map<Contact Id, List<Campaign Name>>
   // We pass the whole list of contacts that we're about to sync to this all at once, then let the implementations
   // decide how to implement it in the most performant way. Some APIs may solely allow retrieval one at a time.

src/main/java/com/impactupgrade/nucleus/service/segment/DataSyncService.java

@@ -0,0 +1,10 @@
+package com.impactupgrade.nucleus.service.segment;
+
+import java.util.Calendar;
+
+public interface DataSyncService extends SegmentService {
+
+  void syncContacts(Calendar updatedAfter) throws Exception;
+
+  void syncTransactions(Calendar updatedAfter) throws Exception;
+}

src/main/java/com/impactupgrade/nucleus/service/segment/DynamicsCrmService.java

@@ -113,6 +113,11 @@ public void refundDonation(CrmDonation crmDonation) throws Exception {
     //TODO: howto?
   }
 
+  @Override
+  public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+    return List.of();
+  }
+
   private CrmDonation toCrmDonation(DynamicsCrmClient.Opportunity opportunity) {
     if (opportunity == null) {
       return null;
@@ -169,6 +174,11 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, Environmen
     return new PagedResults<>();
   }
 
+  @Override
+  public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+    return new PagedResults<>();
+  }
+
   @Override
   public PagedResults.ResultSet<CrmContact> queryMoreContacts(String queryLocator) throws Exception {
     return null;

src/main/java/com/impactupgrade/nucleus/service/segment/GoogleSheetCrmService.java

@@ -100,6 +100,11 @@ public void refundDonation(CrmDonation crmDonation) throws Exception {
 
     }
 
+    @Override
+    public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+        return List.of();
+    }
+
     @Override
     public PagedResults.ResultSet<CrmContact> queryMoreContacts(String queryLocator) throws Exception {
         return null;
@@ -135,6 +140,11 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, Environmen
         return new PagedResults<>();
     }
 
+    @Override
+    public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+        return new PagedResults<>();
+    }
+
     @Override
     public double getDonationsTotal(String filter) throws Exception {
         return 0;

src/main/java/com/impactupgrade/nucleus/service/segment/HubSpotCrmService.java

@@ -581,6 +581,11 @@ public void insertDonationDeposit(List<CrmDonation> crmDonations) throws Excepti
     // TODO: Break out a set-fields method? Or just allow this whole thing to be overridden?
   }
 
+  @Override
+  public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+    return List.of();
+  }
+
   @Override
   public String insertRecurringDonation(CrmRecurringDonation crmRecurringDonation) throws Exception {
     // TODO: campaign
@@ -1048,6 +1053,11 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, Environmen
     return PagedResults.unpagedResults(crmContacts);
   }
 
+  @Override
+  public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+    return new PagedResults<>();
+  }
+
   @Override
   public Map<String, String> getContactLists(CrmContactListType listType) throws Exception {
     Map<String, String> listNameToId = new HashMap<>();

src/main/java/com/impactupgrade/nucleus/service/segment/SfdcCrmService.java

@@ -758,6 +758,11 @@ public void insertDonationDeposit(List<CrmDonation> crmDonations) throws Excepti
     }
   }
 
+  @Override
+  public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+    return toCrmDonation(sfdcClient.getDonationsUpdatedAfter(updatedAfter));
+  }
+
   protected void setDonationDepositFields(SObject existingOpportunity, SObject opportunityUpdate,
       CrmDonation crmDonation) throws InterruptedException {
     // If the payment gateway event has a refund ID, this item in the payout was a refund. Mark it as such!
@@ -1042,6 +1047,12 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince,
     return toCrmContactPages(queryResults);
   }
 
+  @Override
+  public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+    List<QueryResult> queryResults = sfdcClient.getDonorContacts(updatedSince);
+    return toCrmContactPages(queryResults);
+  }
+
   @Override
   public Map<String, String> getContactLists(CrmContactListType listType) throws Exception {
     Map<String, String> lists = new HashMap<>();

src/main/java/com/impactupgrade/nucleus/service/segment/SharePointCrmService.java

@@ -353,6 +353,11 @@ public void insertDonationDeposit(List<CrmDonation> crmDonations) throws Excepti
 
     }
 
+    @Override
+    public List<CrmDonation> getDonations(Calendar updatedAfter) throws Exception {
+        return List.of();
+    }
+
     @Override
     public String insertRecurringDonation(CrmRecurringDonation crmRecurringDonation) throws Exception {
         return null;
@@ -413,6 +418,11 @@ public PagedResults<CrmContact> getSmsContacts(Calendar updatedSince, Environmen
         return new PagedResults<>();
     }
 
+    @Override
+    public PagedResults<CrmContact> getDonorContacts(Calendar updatedSince) throws Exception {
+        return new PagedResults<>();
+    }
+
     @Override
     public Map<String, String> getContactLists(CrmContactListType listType) throws Exception {
         return Collections.emptyMap();