Skip to content

Commit

Permalink
Code review comments (InP)
Browse files Browse the repository at this point in the history
  • Loading branch information
@VSydor
VSydor committed May 2, 2023
1 parent aac309c commit 2c730ca
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 63 deletions.
63 changes: 53 additions & 10 deletions src/main/java/com/impactupgrade/nucleus/client/MailchimpClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,18 @@
import com.ecwid.maleorang.method.v3_0.lists.merge_fields.EditMergeFieldMethod;
import com.ecwid.maleorang.method.v3_0.lists.merge_fields.GetMergeFieldsMethod;
import com.ecwid.maleorang.method.v3_0.lists.merge_fields.MergeFieldInfo;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.impactupgrade.nucleus.environment.EnvironmentConfig;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -206,17 +210,21 @@ public String updateContactTagsBatch(String listId, List<EmailContact> emailCont
List<String> active = emailContact.activeTags;
List<String> inactive = emailContact.inactiveTags;
ArrayList<MailchimpObject> tags = new ArrayList<>();
for (String activeTag : active) {
MailchimpObject tag = new MailchimpObject();
tag.mapping.put(TAG_STATUS, TAG_ACTIVE);
tag.mapping.put(TAG_NAME, activeTag);
tags.add(tag);
if (CollectionUtils.isNotEmpty(active)) {
for (String activeTag : active) {
MailchimpObject tag = new MailchimpObject();
tag.mapping.put(TAG_STATUS, TAG_ACTIVE);
tag.mapping.put(TAG_NAME, activeTag);
tags.add(tag);
}
}
for (String inactiveTag : inactive) {
MailchimpObject tag = new MailchimpObject();
tag.mapping.put(TAG_STATUS, TAG_INACTIVE);
tag.mapping.put(TAG_NAME, inactiveTag);
tags.add(tag);
if (CollectionUtils.isNotEmpty(inactive)) {
for (String inactiveTag : inactive) {
MailchimpObject tag = new MailchimpObject();
tag.mapping.put(TAG_STATUS, TAG_INACTIVE);
tag.mapping.put(TAG_NAME, inactiveTag);
tags.add(tag);
}
}

EditMemberMethod.AddorRemoveTag editMemberMethod = new EditMemberMethod.AddorRemoveTag(listId, emailContact.email);
Expand Down Expand Up @@ -260,4 +268,39 @@ public String exceptionToString(MailchimpException e) {
}

public record EmailContact(String email, List<String> activeTags, List<String> inactiveTags) {};

@JsonIgnoreProperties(ignoreUnknown = true)
public static final class BatchOperation {
@JsonProperty("status_code")
public Integer status;
@JsonProperty("operation_id")
public String operationId;
public BatchOperationResponse response;
}

@JsonIgnoreProperties(ignoreUnknown = true)
public static final class BatchOperationResponse {
public String id;
@JsonProperty("merge_fields")
public Map<String, String> mergeFields;
@JsonProperty("contact_id")
public String contactId;
@JsonProperty("email_address")
public String email;
@JsonProperty("full_name")
public String fullName;
@JsonProperty("tags_count")
public Integer tagsCount;
@JsonProperty("last_changed")
public Date lastChangedAt;
@JsonProperty("list_id")
public String listId;

// error response fields
public String instance;
public String detail;
public String type;
public String title;
//public Integer status;
}
}
100 changes: 47 additions & 53 deletions src/main/java/com/impactupgrade/nucleus/service/segment/MailchimpEmailService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
import com.ecwid.maleorang.method.v3_0.batches.BatchStatus;
import com.ecwid.maleorang.method.v3_0.lists.members.MemberInfo;
import com.ecwid.maleorang.method.v3_0.lists.merge_fields.MergeFieldInfo;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.impactupgrade.nucleus.client.MailchimpClient;
Expand Down Expand Up @@ -48,7 +51,7 @@ public class MailchimpEmailService extends AbstractEmailService {
private static final Logger log = LogManager.getLogger(MailchimpEmailService.class);

private static final Integer BATCH_REQUEST_OPERATIONS_SIZE = 500;
private static final Integer BATCH_STATUS_RETRY_TIMEOUT_IN_SECONDS = 30;
private static final Integer BATCH_STATUS_RETRY_TIMEOUT_IN_SECONDS = 10;
private static final Integer BATCH_STATUS_MAX_RETRIES = 5;

private final Map<String, String> mergeFieldsNameToTag = new HashMap<>();
Expand Down Expand Up @@ -100,7 +103,6 @@ protected void syncContacts(List<CrmContact> crmContacts, Map<String, List<Strin
MailchimpClient mailchimpClient = new MailchimpClient(mailchimpConfig);

List<CrmContact> contactsToUpsert = new ArrayList<>();
List<CrmContact> contactsToUpdateTags;
List<CrmContact> contactsToArchive = new ArrayList<>();

// transactional is always subscribed
Expand All @@ -120,43 +122,30 @@ protected void syncContacts(List<CrmContact> crmContacts, Map<String, List<Strin
List<MemberInfo> memberInfos = toMemberInfos(emailList, contactsToUpsert, contactsCustomFields);

String upsertBatchId = mailchimpClient.upsertContactsBatch(emailList.id, memberInfos);
String upsertResponse = getBatchResponse(mailchimpClient, mailchimpConfig, upsertBatchId, 0);

List<String> failedEmails = getFailedEmails(upsertResponse);
if (CollectionUtils.isNotEmpty(failedEmails)) {
log.warn("Failed to upsert emails: {}", failedEmails);
contactsToUpdateTags = contactsToUpsert.stream()
.filter(crmContact -> !failedEmails.contains(crmContact.email))
.collect(Collectors.toList());
} else {
contactsToUpdateTags = contactsToUpsert;
}
// Getting batch processing results synchronously to make sure
// all contacts were processed before updating tags
getBatchOperations(mailchimpClient, mailchimpConfig, upsertBatchId, 0);

// if they can't, they're archived, and will be failed to be retrieved for update
Map<String, List<String>> activeTags = getActiveTags(contactsToUpdateTags, crmContactCampaignNames, mailchimpConfig);
List<MailchimpClient.EmailContact> emailContacts = contactsToUpdateTags.stream()
Map<String, List<String>> activeTags = getActiveTags(contactsToUpsert, crmContactCampaignNames, mailchimpConfig);
List<MailchimpClient.EmailContact> emailContacts = contactsToUpsert.stream()
.map(crmContact -> new MailchimpClient.EmailContact(crmContact.email, activeTags.get(crmContact.email), tags.get(crmContact.email)))
.collect(Collectors.toList());

String updateTagsBatchId = updateTagsBatch(emailList.id, emailContacts, mailchimpClient);
getBatchResponse(mailchimpClient, mailchimpConfig, updateTagsBatchId, 0);
updateTagsBatch(emailList.id, emailContacts, mailchimpClient);

// if they can't, they're archived, and will be failed to be retrieved for update
List<String> emailsToArchive = contactsToArchive.stream().map(crmContact -> crmContact.email).collect(Collectors.toList());
mailchimpClient.archiveContactsBatch(emailList.id, emailsToArchive);

} catch (MailchimpException e) {
log.warn("Mailchimp syncContact failed: {}", mailchimpClient.exceptionToString(e));
log.warn("Mailchimp syncContacts failed: {}", mailchimpClient.exceptionToString(e));
} catch (Exception e) {
log.warn("Mailchimp syncContact failed", e);
log.warn("Mailchimp syncContacts failed", e);
}
}

//TODO: move to MailchimpClient lib?
protected String getBatchResponse(MailchimpClient mailchimpClient, EnvironmentConfig.EmailPlatform mailchimpConfig, String batchStatusId, Integer attemptCount) throws Exception {
if (Strings.isNullOrEmpty(batchStatusId)) {
return null;
}
String batchResponse = null;
protected List<MailchimpClient.BatchOperation> getBatchOperations(MailchimpClient mailchimpClient, EnvironmentConfig.EmailPlatform mailchimpConfig, String batchStatusId, Integer attemptCount) throws Exception {
List<MailchimpClient.BatchOperation> batchOperations = null;
if (attemptCount == BATCH_STATUS_MAX_RETRIES) {
log.error("exhausted retries; returning...");
} else {
Expand All @@ -165,18 +154,25 @@ protected String getBatchResponse(MailchimpClient mailchimpClient, EnvironmentCo
log.info("Batch '{}' is not finished. Retrying in {} seconds...", batchStatusId, BATCH_STATUS_RETRY_TIMEOUT_IN_SECONDS);
Thread.sleep(BATCH_STATUS_RETRY_TIMEOUT_IN_SECONDS * 1000);
Integer newAttemptCount = attemptCount + 1;
batchResponse = getBatchResponse(mailchimpClient, mailchimpConfig, batchStatusId, newAttemptCount);
batchOperations = getBatchOperations(mailchimpClient, mailchimpConfig, batchStatusId, newAttemptCount);
} else {
batchResponse = getBatchResponseBody(batchStatus, mailchimpConfig);
log.info("Batch '{}' finished! (finished/total) {}/{}", batchStatus.finished_operations, batchStatus.total_operations);
if (batchStatus.errored_operations > 0) {
log.warn("Errored operations count: {}", batchStatus.errored_operations);
}
String batchResponse = getBatchResponseAsString(batchStatus, mailchimpConfig);
batchOperations = deserializeBatchOperations(batchResponse);
}
}
return batchResponse;
return batchOperations;
}

//TODO: move to MailchimpClient lib?
protected String getBatchResponseBody(BatchStatus batchStatus, EnvironmentConfig.EmailPlatform mailchimpConfig) throws Exception {
String responseBody = null;
protected String getBatchResponseAsString(BatchStatus batchStatus, EnvironmentConfig.EmailPlatform mailchimpConfig) throws Exception {
if (Strings.isNullOrEmpty(batchStatus.response_body_url)) {
return null;
}

String responseString = null;
InputStream inputStream = HttpClient.get(batchStatus.response_body_url, HttpClient.HeaderBuilder.builder()
.authBearerToken(mailchimpConfig.secretKey)
.header("Accept-Encoding", "application/gzip"), InputStream.class);
Expand All @@ -186,39 +182,37 @@ protected String getBatchResponseBody(BatchStatus batchStatus, EnvironmentConfig
TarArchiveEntry tarArchiveEntry;
while ((tarArchiveEntry = (TarArchiveEntry) tarArchiveInputStream.getNextEntry()) != null) {
if (!tarArchiveEntry.isDirectory()) {
responseBody = IOUtils.toString(tarArchiveInputStream, StandardCharsets.UTF_8);
responseString = IOUtils.toString(tarArchiveInputStream, StandardCharsets.UTF_8);
}
}
} catch (Exception e) {
log.error("Failed to get batch response body! {}", e);
}

return responseBody;
return responseString;
}

//TODO: possible to use custom class for response but skipping this for now
// since we only ned to know the status and email
protected List<String> getFailedEmails(String response) {
if (Strings.isNullOrEmpty(response)) {
protected List<MailchimpClient.BatchOperation> deserializeBatchOperations(String batchOperationsString) {
if (Strings.isNullOrEmpty(batchOperationsString)) {
return Collections.emptyList();
}
List<MailchimpClient.BatchOperation> batchOperations = new ArrayList<>();
try {
JSONArray jsonArray = new JSONArray(response);
List<JSONObject> jsonObjects = new ArrayList<>();
for (int i = 0; i < jsonArray.length(); i++) {
jsonObjects.add(jsonArray.getJSONObject(i));
JSONArray jsonArray = new JSONArray(batchOperationsString);
ObjectMapper objectMapper = new ObjectMapper();

for (int i = 0; i< jsonArray.length(); i ++) {
JSONObject batchOperation = jsonArray.getJSONObject(i);
// Response is an escaped string - converting it to json object and back to string to unescape
String response = batchOperation.getString("response");
JSONObject responseObject = new JSONObject(response);
batchOperation.put("response", responseObject);

batchOperations.add(objectMapper.readValue(batchOperation.toString(), new TypeReference<>() {}));
}
return jsonObjects.stream()
.filter(jo -> jo.getInt("status_code") < 200 || jo.getInt("status_code") > 299)
.map(jo -> {
JSONObject responseObject = new JSONObject(jo.getString("response"));
return responseObject.getString("email_address");
})
.collect(Collectors.toList());
} catch (Exception e) {
log.error("Failed to get emails from response! {}", e);
return Collections.emptyList();
} catch (JsonProcessingException e) {
log.warn("Failed to deserialize batch operations! {}", e.getMessage());
}
return batchOperations;
}

protected Map<String, List<String>> getActiveTags(List<CrmContact> crmContacts, Map<String, List<String>> crmContactCampaignNames, EnvironmentConfig.EmailPlatform mailchimpConfig) throws Exception {
Expand Down

0 comments on commit 2c730ca

Please sign in to comment.