Merge pull request #492 from bcgov/grad-release

Grad release 1.20.0
bcgov · Jun 4, 2024 · fda090b · fda090b
2 parents 9fcd407 + f6c9208
commit fda090b
Show file tree

Hide file tree

Showing 22 changed files with 299 additions and 212 deletions.
diff --git a/api/pom.xml b/api/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>ca.bc.gov.educ</groupId>
     <artifactId>educ-grad-batch-graduation-api</artifactId>
-    <version>1.8.62</version>
+    <version>1.8.63</version>
     <name>educ-grad-batch-graduation-api</name>
     <description>Ministry of Education GRAD BATCH GRADUATION API</description>
 
@@ -52,7 +52,6 @@
         <org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
         <springdoc.version>2.0.2</springdoc.version>
         <log4j.version>2.18.0</log4j.version>
-        <resilience4j.version>2.0.2</resilience4j.version>
         <shedlock.version>4.38.0</shedlock.version>
     </properties>
 
@@ -129,11 +128,6 @@
             <artifactId>modelmapper</artifactId>
             <version>3.1.0</version>
         </dependency>
-        <dependency>
-            <groupId>io.github.resilience4j</groupId>
-            <artifactId>resilience4j-spring-boot2</artifactId>
-            <version>${resilience4j.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-aop</artifactId>

diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java
@@ -892,7 +892,7 @@ public Step masterStepCertRegen(JobRepository jobRepository, PlatformTransaction
     @Bean
     public Step certRegenJobStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SkipSQLTransactionExceptionsListener skipListener) {
         return new StepBuilder("certRegenJobStep", jobRepository)
-                .<UUID, Integer>chunk(1, transactionManager)
+                .<StudentCredentialDistribution, Integer>chunk(1, transactionManager)
                 .reader(itemReaderCertRegen())
                 .processor(itemProcessorCertRegen())
                 .writer(itemWriterCertRegen())

diff --git a/...ava/ca/bc/gov/educ/api/batchgraduation/processor/RunCertificateRegenerationProcessor.java b/...ava/ca/bc/gov/educ/api/batchgraduation/processor/RunCertificateRegenerationProcessor.java
@@ -2,16 +2,21 @@
 
 import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
 import ca.bc.gov.educ.api.batchgraduation.model.GraduationStudentRecordDistribution;
+import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
 import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
+import jakarta.validation.constraints.NotNull;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.batch.item.ItemProcessor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.lang.Nullable;
 
+import java.io.IOException;
 import java.util.UUID;
 
-public class RunCertificateRegenerationProcessor implements ItemProcessor<UUID, Integer> {
+public class RunCertificateRegenerationProcessor implements ItemProcessor<StudentCredentialDistribution, Integer> {
 
 	private static final Logger LOGGER = LoggerFactory.getLogger(RunCertificateRegenerationProcessor.class);
 
@@ -25,27 +30,43 @@ public class RunCertificateRegenerationProcessor implements ItemProcessor<UUID,
 	Long batchId;
 
 	@Override
-	public Integer process(UUID key) throws Exception {
-		GraduationStudentRecordDistribution stuRec = restUtils.getStudentData(key.toString());
-		if (stuRec != null) {
-			LOGGER.info("Processing partitionData: studentID = {}", stuRec.getStudentID());
+	@Nullable
+	public Integer process(@NotNull StudentCredentialDistribution item) throws Exception {
+		if (item.getStudentID() != null) {
+			LOGGER.info("Processing partitionData: studentID = {}", item.getStudentID());
 			summaryDTO.setBatchId(batchId);
 			try {
+				if (StringUtils.isBlank(item.getPen())) {
+					item.setPen(getPenNumber(item.getStudentID()));
+				}
 				summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() + 1L);
-				Integer count = restUtils.runRegenerateStudentCertificate(stuRec.getPen());
+				Integer count = restUtils.runRegenerateStudentCertificate(item.getPen());
 				if (count > 0) {
-					restUtils.updateStudentGradRecord(key, batchId, "CERTREGEN");
+					restUtils.updateStudentGradRecord(item.getStudentID(), batchId, "CERTREGEN");
 				}
 				return count;
 			} catch(Exception e) {
 				LOGGER.error("Unexpected Error: {}", e.getLocalizedMessage());
-				summaryDTO.updateError(key,"GRAD-GRADUATION-API IS DOWN","Graduation API is unavailable at this moment");
+				summaryDTO.updateError(item.getStudentID(), "Unexpected Error in GRAD-GRADUATION-API", e.getLocalizedMessage());
 				summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() - 1L);
-				LOGGER.info("Failed STU-ID:{} Errors:{}",key,summaryDTO.getErrors().size());
+				LOGGER.info("Failed STU-ID:{} Errors:{}",item.getStudentID(),summaryDTO.getErrors().size());
 				return null;
 			}
 		}
+		LOGGER.warn("Skipped STU-ID:null for pen# {},Errors:{}",item.getPen(),summaryDTO.getErrors().size());
 		return null;
 	}
 
+	private String getPenNumber(UUID studentID) throws IOException {
+		GraduationStudentRecordDistribution stuRec = null;
+		try {
+			stuRec = restUtils.getStudentData(studentID.toString());
+		} catch (RuntimeException se) {
+			LOGGER.error("Unknown exception occurred in PEN API: {}", se.getLocalizedMessage());
+		}
+		if (stuRec != null) {
+			return stuRec.getPen();
+		}
+		throw new IOException(String.format("Student not found in PEN API: STU-ID:%s", studentID));
+	}
 }
diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseReader.java
@@ -3,33 +3,38 @@
 import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
 import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
 import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.batch.core.JobExecution;
-import org.springframework.batch.item.ItemReader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 
-import java.util.List;
-import java.util.UUID;
-
-public abstract class BaseReader implements ItemReader<UUID> {
+@Slf4j
+public abstract class BaseReader {
 
     @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
     @Autowired
     RestUtils restUtils;
 
-    @Value("#{stepExecutionContext['index']}")
-    protected Integer nxtStudentForProcessing;
-
-    @Value("#{stepExecutionContext['data']}")
-    protected List<UUID> studentList;
-
     @Value("#{stepExecutionContext['summary']}")
     AlgorithmSummaryDTO summaryDTO;
 
     @Value("#{stepExecution.jobExecution}")
     JobExecution jobExecution;
 
+    protected void aggregate(String summaryContextName) {
+        AlgorithmSummaryDTO totalSummaryDTO = (AlgorithmSummaryDTO)jobExecution.getExecutionContext().get(summaryContextName);
+        if (totalSummaryDTO == null) {
+            totalSummaryDTO = new AlgorithmSummaryDTO();
+            jobExecution.getExecutionContext().put(summaryContextName, totalSummaryDTO);
+        }
+        totalSummaryDTO.setBatchId(summaryDTO.getBatchId());
+        totalSummaryDTO.setReadCount(totalSummaryDTO.getReadCount() + summaryDTO.getReadCount());
+        totalSummaryDTO.setProcessedCount(totalSummaryDTO.getProcessedCount() + summaryDTO.getProcessedCount());
+        totalSummaryDTO.getErrors().putAll(summaryDTO.getErrors());
+    }
+
     protected void fetchAccessToken() {
+        log.info("Fetching the access token from KeyCloak API");
         ResponseObj res = restUtils.getTokenResponseObject();
         if (res != null) {
             summaryDTO.setAccessToken(res.getAccess_token());

diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java
@@ -0,0 +1,38 @@
+package ca.bc.gov.educ.api.batchgraduation.reader;
+
+import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
+import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
+import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import java.util.List;
+import java.util.UUID;
+
+public abstract class BaseStudentReader implements ItemReader<UUID> {
+
+    @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
+    @Autowired
+    RestUtils restUtils;
+
+    @Value("#{stepExecutionContext['index']}")
+    protected Integer nxtStudentForProcessing;
+
+    @Value("#{stepExecutionContext['data']}")
+    protected List<UUID> studentList;
+
+    @Value("#{stepExecutionContext['summary']}")
+    AlgorithmSummaryDTO summaryDTO;
+
+    @Value("#{stepExecution.jobExecution}")
+    JobExecution jobExecution;
+
+    protected void fetchAccessToken() {
+        ResponseObj res = restUtils.getTokenResponseObject();
+        if (res != null) {
+            summaryDTO.setAccessToken(res.getAccess_token());
+        }
+    }
+}
diff --git a/...ava/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java b/...ava/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java
@@ -5,7 +5,7 @@
 
 import java.util.UUID;
 
-public class RecalculateProjectedGradRunErrorReader extends BaseReader {
+public class RecalculateProjectedGradRunErrorReader extends BaseStudentReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateProjectedGradRunErrorReader.class);
 

diff --git a/...a/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java b/...a/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java
@@ -5,7 +5,7 @@
 
 import java.util.UUID;
 
-public class RecalculateProjectedGradRunErrorRetryReader extends BaseReader {
+public class RecalculateProjectedGradRunErrorRetryReader extends BaseStudentReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateProjectedGradRunErrorRetryReader.class);
 

diff --git a/...ain/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java b/...ain/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java
@@ -5,7 +5,7 @@
 
 import java.util.UUID;
 
-public class RecalculateProjectedGradRunReader extends BaseReader {
+public class RecalculateProjectedGradRunReader extends BaseStudentReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateProjectedGradRunReader.class);
 

diff --git a/...rc/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java b/...rc/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java
@@ -5,7 +5,7 @@
 
 import java.util.UUID;
 
-public class RecalculateStudentErrorReader extends BaseReader {
+public class RecalculateStudentErrorReader extends BaseStudentReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateStudentErrorReader.class);
 

diff --git a/...in/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java b/...in/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java
@@ -5,7 +5,7 @@
 
 import java.util.UUID;
 
-public class RecalculateStudentErrorRetryReader extends BaseReader {
+public class RecalculateStudentErrorRetryReader extends BaseStudentReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateStudentErrorRetryReader.class);
 

diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java
@@ -5,7 +5,7 @@
 
 import java.util.UUID;
 
-public class RecalculateStudentReader extends BaseReader {
+public class RecalculateStudentReader extends BaseStudentReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RecalculateStudentReader.class);
 

diff --git a/...main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificatePartitioner.java b/...main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificatePartitioner.java
@@ -13,7 +13,6 @@
 import reactor.core.publisher.Mono;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 import static ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants.SEARCH_REQUEST;
 
@@ -43,37 +42,39 @@ public Map<String, ExecutionContext> partition(int gridSize) {
             if(parallelDTO != null) {
                 credentialList.addAll(parallelDTO.certificateList());
             }
+        } else if (certificateRegenerationRequest.getPens() != null && !certificateRegenerationRequest.getPens().isEmpty()) {
+            certificateRegenerationRequest.getPens().forEach(p -> {
+                StudentCredentialDistribution scd = new StudentCredentialDistribution();
+                scd.setPen(p);
+                scd.setStudentID(getStudentIDByPen(p));
+                if (scd.getStudentID() != null)
+                    credentialList.add(scd);
+            });
         } else {
             credentialList.addAll(getStudentsForUserReqRun(certificateRegenerationRequest));
         }
 
-        Set<UUID> studentSet = new HashSet<>();
         if(!credentialList.isEmpty()) {
-            studentSet = credentialList.stream().map(StudentCredentialDistribution::getStudentID).collect(Collectors.toSet());
-        }
-
-        List<UUID> studentList = new ArrayList<>(studentSet);
-        if(!studentList.isEmpty()) {
             createTotalSummaryDTO("regenCertSummaryDTO");
-            updateBatchJobHistory(createBatchJobHistory(), (long) studentList.size());
-            int partitionSize = studentList.size()/gridSize + 1;
-            List<List<UUID>> partitions = new LinkedList<>();
-            for (int i = 0; i < studentList.size(); i += partitionSize) {
-                partitions.add(studentList.subList(i, Math.min(i + partitionSize, studentList.size())));
+            updateBatchJobHistory(createBatchJobHistory(), (long) credentialList.size());
+            int partitionSize = credentialList.size()/gridSize + 1;
+            List<List<StudentCredentialDistribution>> partitions = new LinkedList<>();
+            for (int i = 0; i < credentialList.size(); i += partitionSize) {
+                partitions.add(credentialList.subList(i, Math.min(i + partitionSize, credentialList.size())));
             }
             Map<String, ExecutionContext> map = new HashMap<>(partitions.size());
             for (int i = 0; i < partitions.size(); i++) {
                 ExecutionContext executionContext = new ExecutionContext();
                 AlgorithmSummaryDTO summaryDTO = new AlgorithmSummaryDTO();
-                List<UUID> data = partitions.get(i);
+                List<StudentCredentialDistribution> data = partitions.get(i);
                 executionContext.put("data", data);
                 summaryDTO.setReadCount(data.size());
                 executionContext.put("summary", summaryDTO);
                 executionContext.put("index",0);
                 String key = "partition" + i;
                 map.put(key, executionContext);
             }
-            LOGGER.info("Found {} in total running on {} partitions",studentList.size(),map.size());
+            LOGGER.info("Found {} in total running on {} partitions",credentialList.size(),map.size());
             return map;
         }
         LOGGER.info("No Certificates are found to process them with null distribution date");
@@ -89,6 +90,15 @@ private List<StudentCredentialDistribution> getStudentsForUserReqRun(Certificate
         }
     }
 
+    private UUID getStudentIDByPen(String pen) {
+        try {
+            String accessToken = restUtils.fetchAccessToken();
+            return restUtils.getStudentIDByPen(pen, accessToken);
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
     @Override
     protected JobExecution getJobExecution() {
         return context;

diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificateReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificateReader.java
@@ -1,41 +1,37 @@
 package ca.bc.gov.educ.api.batchgraduation.reader;
 
-import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
+import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.beans.factory.annotation.Value;
 
-import java.util.UUID;
+import java.util.List;
 
-public class RegenerateCertificateReader extends BaseReader {
+public class RegenerateCertificateReader extends BaseReader implements ItemReader<StudentCredentialDistribution> {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RegenerateCertificateReader.class);
 
+    @Value("#{stepExecutionContext['index']}")
+    private Integer nxtCredentialForProcessing;
+
+    @Value("#{stepExecutionContext['data']}")
+    List<StudentCredentialDistribution> credentialList;
+
     @Override
-    public UUID read() throws Exception {
+    public StudentCredentialDistribution read() throws Exception {
         fetchAccessToken();
-        summaryDTO.setReadCount(studentList.size());
+        summaryDTO.setReadCount(credentialList.size());
 
-        UUID nextStudent = null;
+        StudentCredentialDistribution nextCredential = null;
 
-        if (nxtStudentForProcessing < studentList.size()) {
-            nextStudent = studentList.get(nxtStudentForProcessing);
-            LOGGER.info("StudID:{} - {} of {}", nextStudent, nxtStudentForProcessing + 1, summaryDTO.getReadCount());
-            nxtStudentForProcessing++;
+        if (nxtCredentialForProcessing < credentialList.size()) {
+            nextCredential = credentialList.get(nxtCredentialForProcessing);
+            LOGGER.info("StudID:{} - {} of {}", nextCredential, nxtCredentialForProcessing + 1, summaryDTO.getReadCount());
+            nxtCredentialForProcessing++;
         } else {
             aggregate("regenCertSummaryDTO");
         }
-        return nextStudent;
-    }
-
-    public void aggregate(String summaryContextName) {
-        AlgorithmSummaryDTO totalSummaryDTO = (AlgorithmSummaryDTO)jobExecution.getExecutionContext().get(summaryContextName);
-        if (totalSummaryDTO == null) {
-            totalSummaryDTO = new AlgorithmSummaryDTO();
-            jobExecution.getExecutionContext().put(summaryContextName, totalSummaryDTO);
-        }
-        totalSummaryDTO.setBatchId(summaryDTO.getBatchId());
-        totalSummaryDTO.setReadCount(totalSummaryDTO.getReadCount() + summaryDTO.getReadCount());
-        totalSummaryDTO.setProcessedCount(totalSummaryDTO.getProcessedCount() + summaryDTO.getProcessedCount());
-        totalSummaryDTO.getErrors().putAll(summaryDTO.getErrors());
+        return nextCredential;
     }
 }