Skip to content

Commit

Permalink
pervasive logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@brmeyer
brmeyer committed Jul 23, 2023
1 parent e2964df commit 069eb3b
Show file tree
Hide file tree
Showing 56 changed files with 687 additions and 863 deletions.
12 changes: 6 additions & 6 deletions src/main/java/com/impactupgrade/nucleus/client/DonorWranglerClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public Optional<DwDonor> getContactByEmail(String email) throws Exception {

HttpResponse response = executePost(params);
String responseString = getResponseString(response);
log.info("Donor Wrangler getDonorByEmail response: {}", responseString);
env.logJobInfo("Donor Wrangler getDonorByEmail response: {}", responseString);
JSONObject jsonObject = new JSONObject(responseString);

if (!jsonObject.has("result")) {
Expand All @@ -70,15 +70,15 @@ public Optional<DwDonor> contactSearch(String searchKey, String searchValue) thr

HttpResponse response = executePost(params);
String responseString = getResponseString(response);
log.info("Donor Wrangler contactSearch response: {}", responseString);
env.logJobInfo("Donor Wrangler contactSearch response: {}", responseString);
JSONObject jsonObject = new JSONObject(responseString);
JSONArray jsonArray = jsonObject.getJSONArray("results");

if (jsonArray.isEmpty()) {
return Optional.empty();
} else {
if (jsonArray.length() > 1) {
log.warn("multiple contacts found for {}={}; returning the first", searchKey, searchValue);
env.logJobWarn("multiple contacts found for {}={}; returning the first", searchKey, searchValue);
}
JSONObject jsonDonor = (JSONObject) jsonArray.get(0);
DwDonor donor = DwDonor.fromJson(jsonDonor);
Expand All @@ -104,7 +104,7 @@ public String upsertContact(CrmContact crmContact) throws Exception {

HttpResponse response = executePost(params);
String responseString = getResponseString(response);
log.info("Donor Wrangler addUpdateDonor response: {}", responseString);
env.logJobInfo("Donor Wrangler addUpdateDonor response: {}", responseString);

JSONObject jsonObject = new JSONObject(responseString);

Expand All @@ -124,7 +124,7 @@ public List<DwDonation> getDonationsByDonorId(String donorId) throws Exception {

HttpResponse response = executePost(params);
String responseString = getResponseString(response);
log.info("Donor Wrangler getDonationsByDonorId response: {}", responseString);
env.logJobInfo("Donor Wrangler getDonationsByDonorId response: {}", responseString);
JSONObject jsonObject = new JSONObject(responseString);
JSONArray jsonArray = jsonObject.getJSONArray("results");

Expand Down Expand Up @@ -175,7 +175,7 @@ public String insertDonation(CrmDonation crmDonation) throws Exception {

HttpResponse response = executePost(params);
String responseString = getResponseString(response);
log.info("Donor Wrangler addUpdateDonation response: {}", responseString);
env.logJobInfo("Donor Wrangler addUpdateDonation response: {}", responseString);

JSONObject jsonObject = new JSONObject(responseString);

Expand Down
18 changes: 8 additions & 10 deletions src/main/java/com/impactupgrade/nucleus/client/MSGraphClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@

import com.azure.identity.ClientSecretCredential;
import com.azure.identity.ClientSecretCredentialBuilder;
import com.impactupgrade.nucleus.environment.EnvironmentConfig;
import com.impactupgrade.nucleus.environment.Environment;
import com.microsoft.graph.authentication.TokenCredentialAuthProvider;
import com.microsoft.graph.core.ClientException;
import com.microsoft.graph.models.DriveItem;
import com.microsoft.graph.requests.DriveItemCollectionPage;
import com.microsoft.graph.requests.DriveItemContentStreamRequest;
import com.microsoft.graph.requests.GraphServiceClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.InputStream;
import java.util.LinkedList;
Expand All @@ -19,20 +17,20 @@

public class MSGraphClient {

private static final Logger log = LogManager.getLogger(MSGraphClient.class);

private static final String DEFAULT_SCOPE = "https://graph.microsoft.com/.default";

protected final Environment env;
protected final ClientSecretCredential clientSecretCredential;
protected final TokenCredentialAuthProvider tokenCredentialAuthProvider;
protected final GraphServiceClient graphClient;

public MSGraphClient(EnvironmentConfig.SharePointPlatform sharePoint) {
public MSGraphClient(Environment env) {
this.env = env;
List<String> scopes = List.of(DEFAULT_SCOPE);
this.clientSecretCredential = new ClientSecretCredentialBuilder()
.clientId(sharePoint.clientId)
.clientSecret(sharePoint.clientSecret)
.tenantId(sharePoint.tenantId)
.clientId(env.getConfig().sharePoint.clientId)
.clientSecret(env.getConfig().sharePoint.clientSecret)
.tenantId(env.getConfig().sharePoint.tenantId)
.build();
this.tokenCredentialAuthProvider = new TokenCredentialAuthProvider(scopes, clientSecretCredential);

Expand Down Expand Up @@ -109,7 +107,7 @@ private InputStream getInputStream(DriveItemContentStreamRequest request) {
try {
return request.get();
} catch (ClientException e) {
log.error("Failed to execute request!", e.getMessage());
env.logJobError("Failed to execute request!", e.getMessage());
}
return null;
}
Expand Down
19 changes: 9 additions & 10 deletions src/main/java/com/impactupgrade/nucleus/client/MailchimpClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,9 @@
import com.ecwid.maleorang.method.v3_0.lists.merge_fields.MergeFieldInfo;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.impactupgrade.nucleus.environment.Environment;
import com.impactupgrade.nucleus.environment.EnvironmentConfig;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.text.SimpleDateFormat;
Expand All @@ -37,8 +36,6 @@

public class MailchimpClient {

private static final Logger log = LogManager.getLogger(MailchimpClient.class);

public static final String SUBSCRIBED = "subscribed";
public static final String ARCHIVED = "archived";
public static final String FIRST_NAME = "FNAME";
Expand All @@ -53,9 +50,11 @@ public class MailchimpClient {
public static final String TAG_INACTIVE = "inactive";

protected final com.ecwid.maleorang.MailchimpClient client;
protected final Environment env;

public MailchimpClient(EnvironmentConfig.EmailPlatform mailchimpConfig) {
public MailchimpClient(EnvironmentConfig.EmailPlatform mailchimpConfig, Environment env) {
client = new com.ecwid.maleorang.MailchimpClient(mailchimpConfig.secretKey);
this.env = env;
}

public MemberInfo getContactInfo(String listId, String contactEmail) throws IOException, MailchimpException {
Expand All @@ -78,8 +77,8 @@ public void upsertContact(String listId, MemberInfo contact) throws IOException,
// We're finding that address validation is SUPER picky, especially when it comes to CRMs that combine
// street1 and street2 into a single street. If the upsert fails, try it again without ADDRESS...
if (contact.merge_fields.mapping.containsKey(ADDRESS)) {
log.info("Mailchimp upsertContact failed: {}", error);
log.info("retrying upsertContact without ADDRESS");
env.logJobInfo("Mailchimp upsertContact failed: {}", error);
env.logJobInfo("retrying upsertContact without ADDRESS");
upsertMemberMethod.merge_fields.mapping.remove(ADDRESS);
client.execute(upsertMemberMethod);
} else {
Expand Down Expand Up @@ -118,17 +117,17 @@ public List<MemberInfo> getListMembers(String listId, String status, Calendar si
GetMembersMethod getMembersMethod = new GetMembersMethod(listId);
getMembersMethod.status = status;
getMembersMethod.count = 500; // subjective, but this is timing out periodically -- may need to dial it back further
log.info("retrieving list {} contacts", listId);
env.logJobInfo("retrieving list {} contacts", listId);
if (sinceLastChanged != null) {
getMembersMethod.since_last_changed = sinceLastChanged.getTime();
String formattedDate = new SimpleDateFormat("yyyy-MM-dd").format(sinceLastChanged.getTime());
log.info("retrieving contacts whose information changed after {}", formattedDate);
env.logJobInfo("retrieving contacts whose information changed after {}", formattedDate);
}
GetMembersMethod.Response getMemberResponse = client.execute(getMembersMethod);
List<MemberInfo> members = new ArrayList<>(getMemberResponse.members);
while (getMemberResponse.total_items > members.size()) {
getMembersMethod.offset = members.size();
log.info("retrieving list {} contacts (offset {} of total {})", listId, getMembersMethod.offset, getMemberResponse.total_items);
env.logJobInfo("retrieving list {} contacts (offset {} of total {})", listId, getMembersMethod.offset, getMemberResponse.total_items);
getMemberResponse = client.execute(getMembersMethod);
members.addAll(getMemberResponse.members);
}
Expand Down
6 changes: 1 addition & 5 deletions src/main/java/com/impactupgrade/nucleus/client/MinistryByTextClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import com.impactupgrade.nucleus.environment.EnvironmentConfig;
import com.impactupgrade.nucleus.model.CrmContact;
import com.impactupgrade.nucleus.util.HttpClient;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import javax.ws.rs.core.Form;
import javax.ws.rs.core.GenericType;
Expand All @@ -24,8 +22,6 @@
// TODO: To eventually become a mbt-java-client open source lib?
public class MinistryByTextClient {

private static final Logger log = LogManager.getLogger(MinistryByTextClient.class);

protected static String AUTH_ENDPOINT = "https://login-qa.ministrybytext.com/connect/token";
protected static String API_ENDPOINT_BASE = "https://api-qa.ministrybytext.com/";

Expand Down Expand Up @@ -103,7 +99,7 @@ public static class SubscriberRelation {

protected HttpClient.HeaderBuilder headers() {
if (isAccessTokenInvalid()) {
log.info("Getting new access token...");
env.logJobInfo("Getting new access token...");
TokenResponse tokenResponse = getAccessToken();
accessToken = tokenResponse.accessToken;
Calendar onehour = Calendar.getInstance();
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/com/impactupgrade/nucleus/client/RaiselyClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import static com.impactupgrade.nucleus.util.HttpClient.post;

public class RaiselyClient{
private static final Logger log = LogManager.getLogger(RaiselyClient.class);
private static final String RAISELY_API_URL = "https://api.raisely.com/v3/";
private static final String APPLICATION_JSON = "application/json";

Expand Down Expand Up @@ -53,10 +52,10 @@ protected String getAccessToken() {
String username = env.getConfig().raisely.username;
String password = env.getConfig().raisely.password;

log.info("Getting token...");
env.logJobInfo("Getting token...");
HttpClient.HeaderBuilder headers = HttpClient.HeaderBuilder.builder();
TokenResponse response = post(RAISELY_API_URL + "login", Map.of("requestAdminToken", "true", "username", username, "password", password), APPLICATION_JSON, headers, TokenResponse.class);
log.info("Token: {}", response.token);
env.logJobInfo("Token: {}", response.token);
this._accessToken = response.token;
}

Expand Down
22 changes: 9 additions & 13 deletions src/main/java/com/impactupgrade/nucleus/client/SfdcBulkClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@
import com.sforce.ws.ConnectionException;
import com.sforce.ws.ConnectorConfig;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
Expand All @@ -47,8 +45,6 @@
*/
public class SfdcBulkClient {

private static final Logger log = LogManager.getLogger(SfdcBulkClient.class.getName());

protected final Environment env;

public SfdcBulkClient(Environment env) {
Expand Down Expand Up @@ -82,7 +78,7 @@ public void ownerTransfer(String oldOwnerId, String newOwnerId, String object, S
query += " AND " + whereClause;
}

log.info("retrieving all {} records to transfer; bulk query: {}", object, query);
env.logJobInfo("retrieving all {} records to transfer; bulk query: {}", object, query);

try (
ByteArrayInputStream queryIS = new ByteArrayInputStream(query.getBytes());
Expand All @@ -99,7 +95,7 @@ public void ownerTransfer(String oldOwnerId, String newOwnerId, String object, S
QueryResultList queryResultList = bulkConn.getQueryResultList(queryJob.getId(), queryBatchInfo.getId());
String[] queryResults = queryResultList.getResult();
for (int i = 0; i < queryResults.length; i++) {
log.info("processing query result set {} of {}", i+1, queryResults.length);
env.logJobInfo("processing query result set {} of {}", i+1, queryResults.length);
String queryResultId = queryResults[i];

InputStream queryResultIS = null;
Expand Down Expand Up @@ -182,7 +178,7 @@ public void uploadIWaveFile(File iwaveFile) throws AsyncApiException, Connection
}

private void uploadSpec(JobInfo jobInfo, InputStream specFile, BulkConnection bulkConn) throws AsyncApiException {
log.info("uploading the spec file");
env.logJobInfo("uploading the spec file");
bulkConn.createTransformationSpecFromStream(jobInfo, specFile);
}

Expand All @@ -209,9 +205,9 @@ private void checkUploadResults(JobInfo job, List<BatchInfo> batchInfoList, Bulk
String id = resultInfo.get("Id");
String error = resultInfo.get("Error");
if (success && created) {
log.info("Created row with id " + id);
env.logJobInfo("Created row with id " + id);
} else if (!success) {
log.error("Failed with error: " + error);
env.logJobError("Failed with error: " + error);
}
}
}
Expand Down Expand Up @@ -248,14 +244,14 @@ private void awaitCompletion(JobInfo job, List<BatchInfo> batchInfoList, BulkCon
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {}
log.info("Awaiting results..." + incomplete.size());
env.logJobInfo("Awaiting results..." + incomplete.size());
sleepTime = 10000L;
BatchInfo[] statusList = bulkConn.getBatchInfoList(job.getId()).getBatchInfo();
for (BatchInfo b : statusList) {
if (b.getState() == BatchStateEnum.Completed
|| b.getState() == BatchStateEnum.Failed) {
if (incomplete.remove(b.getId())) {
log.info("BATCH STATUS:\n" + b);
env.logJobInfo("BATCH STATUS:\n" + b);
}
}
}
Expand All @@ -277,7 +273,7 @@ private JobInfo createJob(String sObjectType, OperationEnum operation, BulkConne
job.setOperation(operation);
job.setContentType(ContentType.CSV);
job = bulkConn.createJob(job);
log.info(job);
env.logJobInfo(job.toString());
return job;
}

Expand Down Expand Up @@ -358,7 +354,7 @@ private void createBatch(FileOutputStream tmpOut, File tmpFile, List<BatchInfo>
tmpOut.close();
try (FileInputStream tmpInputStream = new FileInputStream(tmpFile)) {
BatchInfo batchInfo = bulkConn.createBatchFromStream(jobInfo, tmpInputStream);
log.info(batchInfo);
env.logJobInfo(batchInfo.toString());
batchInfos.add(batchInfo);
}
}
Expand Down
Loading

0 comments on commit 069eb3b

Please sign in to comment.