21367: [Spike] Nightly job to filter and sync contacts

src/main/java/com/impactupgrade/nucleus/App.java

@@ -19,6 +19,7 @@
 import com.impactupgrade.nucleus.controller.ScheduledJobController;
 import com.impactupgrade.nucleus.controller.SfdcController;
 import com.impactupgrade.nucleus.controller.StripeController;
+import com.impactupgrade.nucleus.controller.DataSyncController;
 import com.impactupgrade.nucleus.controller.TwilioController;
 import com.impactupgrade.nucleus.environment.EnvironmentFactory;
 import com.impactupgrade.nucleus.security.SecurityExceptionMapper;
@@ -89,6 +90,7 @@ public void start() throws Exception {
 
     apiConfig.register(backupController());
     apiConfig.register(communicationController());
+    apiConfig.register(dataSyncController());
     apiConfig.register(crmController());
     apiConfig.register(donationFormController());
     apiConfig.register(emailController());
@@ -145,6 +147,7 @@ public void registerServlets(ServletContextHandler context) throws Exception {}
   // Allow orgs to override specific controllers.
   protected BackupController backupController() { return new BackupController(envFactory); }
   protected CommunicationController communicationController() { return new CommunicationController(envFactory); }
+  protected DataSyncController dataSyncController() { return new DataSyncController(envFactory); }
   protected CrmController crmController() { return new CrmController(envFactory); }
   protected DonationFormController donationFormController() { return new DonationFormController(envFactory); }
   protected EmailController emailController() { return new EmailController(envFactory); }

src/main/java/com/impactupgrade/nucleus/client/SfdcClient.java

@@ -125,7 +125,7 @@ public SfdcClient(Environment env, String username, String password, boolean isS
     }
 
     if (npsp) {
-      ACCOUNT_FIELDS += ", npo02__NumberOfClosedOpps__c, npo02__TotalOppAmount__c, npo02__LastCloseDate__c, npo02__LargestAmount__c, npo02__OppsClosedThisYear__c, npo02__OppAmountThisYear__c, npo02__FirstCloseDate__c";
+      ACCOUNT_FIELDS += ", npo02__NumberOfClosedOpps__c, npo02__TotalOppAmount__c, npo02__LastCloseDate__c, npo02__LargestAmount__c, npo02__OppsClosedThisYear__c, npo02__OppAmountThisYear__c, npo02__FirstCloseDate__c, npe01__One2OneContact__c";
       CONTACT_FIELDS += ", account.npo02__NumberOfClosedOpps__c, account.npo02__TotalOppAmount__c, account.npo02__FirstCloseDate__c, account.npo02__LastCloseDate__c, account.npo02__LargestAmount__c, account.npo02__OppsClosedThisYear__c, account.npo02__OppAmountThisYear__c, npe01__Home_Address__c, npe01__WorkPhone__c, npe01__PreferredPhone__c, npe01__HomeEmail__c, npe01__WorkEmail__c, npe01__AlternateEmail__c, npe01__Preferred_Email__c, HomePhone";
       DONATION_FIELDS += ", npe03__Recurring_Donation__c";
       RECURRINGDONATION_FIELDS = "id, name, npe03__Recurring_Donation_Campaign__c, npe03__Recurring_Donation_Campaign__r.Name, npe03__Next_Payment_Date__c, npe03__Installment_Period__c, npe03__Amount__c, npe03__Open_Ended_Status__c, npe03__Contact__c, npe03__Contact__r.Id, npe03__Contact__r.Name, npe03__Contact__r.Email, npe03__Contact__r.Phone, npe03__Schedule_Type__c, npe03__Date_Established__c, npe03__Organization__c, npe03__Organization__r.Id, npe03__Organization__r.Name, OwnerId, Owner.Id, Owner.IsActive";
@@ -675,6 +675,64 @@ protected QueryResult querySmsContacts(String updatedSinceClause, String filter,
     return query(query);
   }
 
+  public List<QueryResult> getDonorIndividualContacts(Calendar updatedSince, String... extraFields)
+      throws ConnectionException, InterruptedException {
+    List<QueryResult> queryResults = new ArrayList<>();
+
+    String updatedSinceClause = "";
+    if (updatedSince != null) {
+      updatedSinceClause = "SystemModStamp >= " + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").format(updatedSince.getTime());
+    }
+    queryResults.add(queryDonorIndividualContacts(updatedSinceClause, extraFields));
+
+    return queryResults;
+  }
+
+  protected QueryResult queryDonorIndividualContacts(String updatedSinceClause, String... extraFields) throws ConnectionException, InterruptedException {
+    if (Strings.isNullOrEmpty(updatedSinceClause)) {
+      env.logJobWarn("no filter provided; out of caution, skipping the query to protect API limits");
+      return new QueryResult();
+    }
+    Set<String> organizationRecordTypeNames = Set.of("business", "church", "school", "org", "group");
+    String query = "SELECT " + getFieldsList(CONTACT_FIELDS, env.getConfig().salesforce.customQueryFields.contact, extraFields) + " " +
+        "FROM Contact " +
+        "WHERE " + updatedSinceClause +
+        "AND (" + organizationRecordTypeNames.stream()
+          .map(name -> "Account.RecordType.Name NOT LIKE '%" + name + "%'")
+          .collect(Collectors.joining(" AND ")) + ") " +
+        "AND npo02__TotalOppAmount__c > 0.0";
+    return query(query);
+  }
+
+  public List<QueryResult> getDonorOrganizationAccounts(Calendar updatedSince, String... extraFields)
+      throws ConnectionException, InterruptedException {
+    List<QueryResult> queryResults = new ArrayList<>();
+
+    String updatedSinceClause = "";
+    if (updatedSince != null) {
+      updatedSinceClause = "SystemModStamp >= " + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").format(updatedSince.getTime());
+    }
+    queryResults.add(queryDonorOrganizationAccounts(updatedSinceClause, extraFields));
+
+    return queryResults;
+  }
+
+  protected QueryResult queryDonorOrganizationAccounts(String updatedSinceClause, String... extraFields) throws ConnectionException, InterruptedException {
+    if (Strings.isNullOrEmpty(updatedSinceClause)) {
+      env.logJobWarn("no filter provided; out of caution, skipping the query to protect API limits");
+      return new QueryResult();
+    }
+    Set<String> organizationRecordTypeNames = Set.of("business", "church", "school", "org", "group");
+    String query = "SELECT " + getFieldsList(ACCOUNT_FIELDS, env.getConfig().salesforce.customQueryFields.account, extraFields) + " " +
+        "FROM Account " +
+        "WHERE " + updatedSinceClause +
+        "AND (" + organizationRecordTypeNames.stream()
+          .map(name -> "RecordType.Name LIKE '%" + name + "%'")
+          .collect(Collectors.joining(" OR ")) + ") " +
+        "AND npo02__TotalOppAmount__c > 0.0";
+    return query(query);
+  }
+
   public List<SObject> searchContacts(ContactSearch contactSearch, String... extraFields)
       throws ConnectionException, InterruptedException {
     List<String> clauses = new ArrayList<>();
@@ -862,6 +920,15 @@ public List<SObject> getDonationsByAccountId(String accountId, String... extraFi
     return queryListAutoPaged(query);
   }
 
+  public List<SObject> getDonationsUpdatedAfter(Calendar updatedSince, String... extraFields) throws ConnectionException, InterruptedException {
+    // TODO: not allowing updates for now, so only grab what recently closed
+//    String updatedSinceClause = "SystemModStamp >= " + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").format(updatedSince.getTime());
+    String updatedSinceClause = "CloseDate >= " + new SimpleDateFormat("yyyy-MM-dd").format(updatedSince.getTime());
+    String query = "select " + getFieldsList(DONATION_FIELDS, env.getConfig().salesforce.customQueryFields.donation, extraFields) + " from Opportunity " +
+        "where " + updatedSinceClause + " AND stageName = 'Closed Won' ORDER BY CloseDate ASC";
+    return queryListAutoPaged(query);
+  }
+
   public Optional<SObject> getNextPledgedDonationByRecurringDonationId(String recurringDonationId, String... extraFields) throws ConnectionException, InterruptedException {
     // TODO: Using TOMORROW to account for timezone issues -- we can typically get away with that approach
     // since most RDs are monthly...

src/main/java/com/impactupgrade/nucleus/controller/DataSyncController.java

@@ -0,0 +1,109 @@
+package com.impactupgrade.nucleus.controller;
+
+import com.impactupgrade.nucleus.entity.JobStatus;
+import com.impactupgrade.nucleus.entity.JobType;
+import com.impactupgrade.nucleus.environment.Environment;
+import com.impactupgrade.nucleus.environment.EnvironmentFactory;
+import com.impactupgrade.nucleus.service.segment.DataSyncService;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.util.Calendar;
+
+@Path("/data-sync")
+public class DataSyncController {
+
+  protected final EnvironmentFactory environmentFactory;
+
+  public DataSyncController(EnvironmentFactory environmentFactory) {
+    this.environmentFactory = environmentFactory;
+  }
+
+  @GET
+  @Path("/contacts/daily")
+  public Response syncContactsDaily(@QueryParam("syncDays") Integer syncDays, @Context HttpServletRequest request) throws Exception {
+    Environment env = environmentFactory.init(request);
+
+    Calendar lastSync = Calendar.getInstance();
+    // run daily, but setting this high to catch previous misses
+    if (syncDays == null || syncDays <= 0) {
+      syncDays = 3;
+    }
+    lastSync.add(Calendar.DATE, -syncDays);
+
+    Runnable thread = () -> {
+      try {
+        String jobName = "Contacts Sync: Daily";
+        env.startJobLog(JobType.EVENT, null, jobName, "Nucleus Portal");
+        boolean success = true;
+
+        for (DataSyncService dataSyncService : env.allDataSyncServices()) {
+          try {
+            dataSyncService.syncContacts(lastSync);
+            env.logJobInfo("{}: sync contacts done", dataSyncService.name());
+          } catch (Exception e) {
+            env.logJobError("sync contacts failed for {}", dataSyncService.name(), e);
+            env.logJobError(e.getMessage());
+            success = false;
+          }
+        }
+
+        env.endJobLog(success ? JobStatus.DONE : JobStatus.FAILED);
+
+      } catch (Exception e) {
+        env.logJobError("sync contacts failed!", e);
+        env.logJobError(e.getMessage());
+        env.endJobLog(JobStatus.FAILED);
+      }
+    };
+    new Thread(thread).start();
+
+    return Response.ok().build();
+  }
+
+  @GET
+  @Path("/transactions/daily")
+  public Response syncTransactionsDaily(@QueryParam("syncDays") Integer syncDays, @Context HttpServletRequest request) throws Exception {
+    Environment env = environmentFactory.init(request);
+
+    Calendar lastSync = Calendar.getInstance();
+    // run daily, but setting this high to catch previous misses
+    if (syncDays == null || syncDays <= 0) {
+      syncDays = 3;
+    }
+    lastSync.add(Calendar.DATE, -syncDays);
+
+    Runnable thread = () -> {
+      try {
+        String jobName = "Transactions Sync: Daily";
+        env.startJobLog(JobType.EVENT, null, jobName, "Nucleus Portal");
+        boolean success = true;
+
+        for (DataSyncService dataSyncService : env.allDataSyncServices()) {
+          try {
+            dataSyncService.syncTransactions(lastSync);
+            env.logJobInfo("{}: sync transactions done", dataSyncService.name());
+          } catch (Exception e) {
+            env.logJobError("sync transactions failed for {}", dataSyncService.name(), e);
+            env.logJobError(e.getMessage());
+            success = false;
+          }
+        }
+
+        env.endJobLog(success ? JobStatus.DONE : JobStatus.FAILED);
+
+      } catch (Exception e) {
+        env.logJobError("sync transactions failed!", e);
+        env.logJobError(e.getMessage());
+        env.endJobLog(JobStatus.FAILED);
+      }
+    };
+    new Thread(thread).start();
+
+    return Response.ok().build();
+  }
+}

src/main/java/com/impactupgrade/nucleus/controller/PaypalController.java

@@ -119,7 +119,6 @@ private void processEvent(Event event, Environment env) throws Exception {
         //env.donationService().createDonation(paymentGatewayEvent);
         env.logJobInfo("Donation created for '" + eventType + "' event.");
         env.logJobInfo("Processing transaction for '" + eventType + "' event...");
-        //env.accountingService().processTransaction(paymentGatewayEvent);
         env.logJobInfo("Transaction processed for '" + eventType + "' event.");
       }
       case "PAYMENT.CAPTURE.DECLINED" -> {
@@ -170,7 +169,6 @@ private void processEvent(Event event, Environment env) throws Exception {
         //env.donationService().createDonation(paymentGatewayEvent);
         env.logJobInfo("Donation created for '" + eventType + "' event.");
         env.logJobInfo("Processing transaction for '" + eventType + "' event...");
-        //env.accountingService().processTransaction(paymentGatewayEvent);
         env.logJobInfo("Transaction processed for '" + eventType + "' event.");
       }
       case "BILLING.SUBSCRIPTION.CANCELLED" -> {

src/main/java/com/impactupgrade/nucleus/controller/StripeController.java

@@ -130,7 +130,6 @@ public void processEvent(String eventType, StripeObject stripeObject, Environmen
           // must first process the account/contact so they're available
           env.contactService().processDonor(paymentGatewayEvent);
           env.donationService().processDonation(paymentGatewayEvent);
-          env.accountingService().processTransaction(paymentGatewayEvent);
         }
       }
       case "payment_intent.succeeded" -> {
@@ -142,7 +141,6 @@ public void processEvent(String eventType, StripeObject stripeObject, Environmen
         // must first process the account/contact so they're available
         env.contactService().processDonor(paymentGatewayEvent);
         env.donationService().processDonation(paymentGatewayEvent);
-        env.accountingService().processTransaction(paymentGatewayEvent);
       }
       case "charge.failed" -> {
         Charge charge = (Charge) stripeObject;

src/main/java/com/impactupgrade/nucleus/environment/Environment.java

@@ -17,7 +17,6 @@
 import com.impactupgrade.nucleus.client.VirtuousClient;
 import com.impactupgrade.nucleus.entity.JobStatus;
 import com.impactupgrade.nucleus.entity.JobType;
-import com.impactupgrade.nucleus.service.logic.AccountingService;
 import com.impactupgrade.nucleus.service.logic.ActivityService;
 import com.impactupgrade.nucleus.service.logic.ContactService;
 import com.impactupgrade.nucleus.service.logic.DonationService;
@@ -32,6 +31,7 @@
 import com.impactupgrade.nucleus.service.segment.BareCrmService;
 import com.impactupgrade.nucleus.service.segment.PaymentGatewayService;
 import com.impactupgrade.nucleus.service.segment.SegmentService;
+import com.impactupgrade.nucleus.service.segment.DataSyncService;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections4.map.CaseInsensitiveMap;
 import org.apache.http.client.utils.URLEncodedUtils;
@@ -147,7 +147,6 @@ public void setOtherContext(MultivaluedMap<String, String> otherContext) {
   public MessagingService messagingService() { return new MessagingService(this); }
   public NotificationService notificationService() { return new NotificationService(this); }
   public ScheduledJobService scheduledJobService() { return new ScheduledJobService(this); }
-  public AccountingService accountingService() { return new AccountingService(this); }
 
   // segment services
 
@@ -202,6 +201,14 @@ public CommunicationService communicationService(String name) {
     return segmentService(name, CommunicationService.class);
   }
 
+  public DataSyncService dataSyncService(String name) {
+    return segmentService(name, DataSyncService.class);
+  }
+
+  public List<DataSyncService> allDataSyncServices() {
+    return segmentServices(DataSyncService.class);
+  }
+
   public Optional<AccountingPlatformService> accountingPlatformService() {
     if (Strings.isNullOrEmpty(getConfig().accountingPrimary)) {
       return Optional.empty();

src/main/java/com/impactupgrade/nucleus/environment/EnvironmentConfig.java

@@ -223,8 +223,6 @@ public static class Salesforce extends Platform {
 
     public Map<AccountType, String> accountTypeToRecordTypeIds = new HashMap<>();
     public Map<TransactionType, String> transactionTypeToRecordTypeIds = new HashMap<>();
-    @Deprecated
-    public Map<TransactionType, String> paymentEventTypeToRecordTypeIds = transactionTypeToRecordTypeIds;
   }
   public static class SalesforceCustomFields implements Serializable {
     public Set<String> account = new HashSet<>();

src/main/java/com/impactupgrade/nucleus/model/AccountingContact.java

@@ -0,0 +1,12 @@
+package com.impactupgrade.nucleus.model;
+
+public class AccountingContact {
+
+  public String contactId;
+  public String crmContactId;
+
+  public AccountingContact(String contactId, String crmContactId) {
+    this.contactId = contactId;
+    this.crmContactId = crmContactId;
+  }
+}