From 201c14c3eb9cf8127e001224f065ee5fb6da8d54 Mon Sep 17 00:00:00 2001 From: Chris Ditcher Date: Fri, 23 Aug 2024 13:50:57 -0700 Subject: [PATCH 1/8] Update update-configmap.sh --- tools/config/update-configmap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/config/update-configmap.sh b/tools/config/update-configmap.sh index d9a044fc..28fe1f45 100644 --- a/tools/config/update-configmap.sh +++ b/tools/config/update-configmap.sh @@ -23,6 +23,7 @@ FLB_CONFIG="[SERVICE] Exclude_Path *.gz,*.zip Parser docker Mem_Buf_Limit 20MB + Buffer_Max_Size 1MB [FILTER] Name record_modifier Match * From 336d821c13e050065828507db3e8cb493f50578e Mon Sep 17 00:00:00 2001 From: githubmamatha <106563495+githubmamatha@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:49:56 -0700 Subject: [PATCH 2/8] Update pom.xml --- api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/pom.xml b/api/pom.xml index b5ed33f6..339a16ff 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -6,7 +6,7 @@ ca.bc.gov.educ educ-grad-batch-graduation-api - 1.8.68 + 1.8.69 educ-grad-batch-graduation-api Ministry of Education GRAD BATCH GRADUATION API From 0197311f5aa2becb222c6de2b228d68cb8cbf5f7 Mon Sep 17 00:00:00 2001 From: Jinil Sung Date: Thu, 19 Sep 2024 14:22:47 -0700 Subject: [PATCH 3/8] GRAD2-2968: enable multi-partitioning to regenerate school reports in parallel. (#531) GRAD2-2968: enable multi-partitioning to regenerate school reports in parallel. --- .../config/BatchJobConfig.java | 5 +- ...ReportsCompletionNotificationListener.java | 35 ++++---- .../model/SchoolReportsRegenSummaryDTO.java | 5 +- .../RegenerateSchoolReportsProcessor.java | 31 +++---- .../reader/BaseSchoolReader.java | 11 --- .../reader/EDWSnapshotSchoolReader.java | 5 +- .../RegenerateSchoolReportsPartitioner.java | 81 +++++++++++-------- .../reader/RegenerateSchoolReportsReader.java | 32 +++----- .../writer/RegenerateSchoolReportsWriter.java | 11 +-- 9 files changed, 102 insertions(+), 114 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java index f9e0da7c..ab792dd0 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobConfig.java @@ -1074,7 +1074,7 @@ public RegenerateSchoolReportsWriter itemWriterSchoolReportsRegen() { @Bean public Step schoolReportsRegenJobStep(JobRepository jobRepository, PlatformTransactionManager transactionManager, SkipSQLTransactionExceptionsListener skipListener) { return new StepBuilder("schoolReportsRegenJobStep", jobRepository) - ., List>chunk(1, transactionManager) + .chunk(1, transactionManager) .reader(itemReaderSchoolReportsRegen()) .processor(itemProcessorSchoolReportsRegen()) .writer(itemWriterSchoolReportsRegen()) @@ -1086,10 +1086,11 @@ public Step schoolReportsRegenJobStep(JobRepository jobRepository, PlatformTrans @Bean public Step masterStepSchoolReportsRegen(JobRepository jobRepository, PlatformTransactionManager transactionManager, EducGradBatchGraduationApiConstants constants, SkipSQLTransactionExceptionsListener skipListener) { + int partitionSize = constants.getNumberOfPartitions() / 2; return new StepBuilder("masterStepSchoolReportsRegen", jobRepository) .partitioner(schoolReportsRegenJobStep(jobRepository, transactionManager, skipListener).getName(), partitionerSchoolReportsRegen()) .step(schoolReportsRegenJobStep(jobRepository, transactionManager, skipListener)) - .gridSize(constants.getNumberOfPartitions()) + .gridSize(partitionSize != 0? partitionSize : 1) .taskExecutor(taskExecutor()) .build(); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/RegenSchoolReportsCompletionNotificationListener.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/RegenSchoolReportsCompletionNotificationListener.java index 0093e29c..bcd2f39f 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/RegenSchoolReportsCompletionNotificationListener.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/RegenSchoolReportsCompletionNotificationListener.java @@ -1,18 +1,17 @@ package ca.bc.gov.educ.api.batchgraduation.listener; +import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity; +import ca.bc.gov.educ.api.batchgraduation.model.BaseSummaryDTO; import ca.bc.gov.educ.api.batchgraduation.model.SchoolReportsRegenSummaryDTO; import ca.bc.gov.educ.api.batchgraduation.util.DateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; import org.springframework.batch.item.ExecutionContext; import org.springframework.stereotype.Component; import java.util.Date; -import static ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants.SEARCH_REQUEST; - @Slf4j @Component public class RegenSchoolReportsCompletionNotificationListener extends BaseRegenSchoolReportsCompletionNotificationListener { @@ -22,34 +21,38 @@ public void afterJob(JobExecution jobExecution) { if (jobExecution.getStatus() == BatchStatus.COMPLETED) { long elapsedTimeMillis = getElapsedTimeMillis(jobExecution); log.info("======================================================================================="); - JobParameters jobParameters = jobExecution.getJobParameters(); ExecutionContext jobContext = jobExecution.getExecutionContext(); Long jobExecutionId = jobExecution.getId(); - String jobType = jobParameters.getString("jobType"); - log.info("{} Regen School Reports Job {} completed in {} s with jobExecution status {}", jobType, jobExecutionId, elapsedTimeMillis / 1000, jobExecution.getStatus()); + log.info("Regen School Reports Job {} completed in {} s with jobExecution status {}", jobExecutionId, elapsedTimeMillis / 1000, jobExecution.getStatus()); String status = jobExecution.getStatus().toString(); - Date startTime = DateUtils.toDate(jobExecution.getStartTime()); Date endTime = DateUtils.toDate(jobExecution.getEndTime()); - String jobTrigger = jobParameters.getString("jobTrigger"); SchoolReportsRegenSummaryDTO summaryDTO = (SchoolReportsRegenSummaryDTO) jobContext.get("schoolReportsRegenSummaryDTO"); - String studentSearchRequest = jobParameters.getString(SEARCH_REQUEST, "{}"); // display Summary Details - log.info("Records read : {}", summaryDTO.getReadCount()); + assert summaryDTO != null; + log.info("Records read : {}", summaryDTO.getReadCount()); log.info("Processed count: {}", summaryDTO.getProcessedCount()); log.info(" --------------------------------------------------------------------------------------"); log.info("Errors:{}", summaryDTO.getErrors().size()); + log.info(" --------------------------------------------------------------------------------------"); + summaryDTO.getSchools().forEach(value -> log.debug("School Report regenerated for {}", value.getMincode())); + // save batch job & error history + saveBatchJobHistory(summaryDTO, jobExecutionId, status, endTime); - updateUserSchedulingJobs(jobParameters); + } + } - String jobParametersDTO = buildJobParametersDTO(jobType, studentSearchRequest, null, null); - // save batch job & error history - processBatchJobHistory(summaryDTO, jobExecutionId, status, jobTrigger, jobType, startTime, endTime, jobParametersDTO); - log.info(" --------------------------------------------------------------------------------------"); - summaryDTO.getSchools().forEach((value) -> log.info("School {} number of Regen School Reports : {}", value.getMincode(), value.getNumberOfSchoolReports())); + private void saveBatchJobHistory(BaseSummaryDTO summaryDTO, Long jobExecutionId, String status, Date endTime) { + BatchGradAlgorithmJobHistoryEntity ent = gradBatchHistoryService.getGradAlgorithmJobHistory(jobExecutionId); + if (ent != null) { + ent.setActualStudentsProcessed(summaryDTO.getProcessedCount()); + ent.setFailedStudentsProcessed((int) summaryDTO.getErroredCount()); + ent.setEndTime(DateUtils.toLocalDateTime(endTime)); + ent.setStatus(status); + gradBatchHistoryService.saveGradAlgorithmJobHistory(ent); } } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/model/SchoolReportsRegenSummaryDTO.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/model/SchoolReportsRegenSummaryDTO.java index b2acaa3b..8d498e1f 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/model/SchoolReportsRegenSummaryDTO.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/model/SchoolReportsRegenSummaryDTO.java @@ -2,7 +2,6 @@ import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.List; @@ -11,9 +10,11 @@ @NoArgsConstructor public class SchoolReportsRegenSummaryDTO extends BaseSummaryDTO { + private String reportBatchType; // REGALG or TVRRUN + private List errors = new ArrayList<>(); private List globalList = new ArrayList<>(); private List schools = new ArrayList<>(); - private StudentSearchRequest studentSearchRequest; } + diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/processor/RegenerateSchoolReportsProcessor.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/processor/RegenerateSchoolReportsProcessor.java index ddae55f7..77b3f10d 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/processor/RegenerateSchoolReportsProcessor.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/processor/RegenerateSchoolReportsProcessor.java @@ -7,11 +7,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import java.util.List; -import java.util.stream.Collectors; - @Slf4j -public class RegenerateSchoolReportsProcessor implements ItemProcessor, List> { +public class RegenerateSchoolReportsProcessor implements ItemProcessor { @Autowired RestUtils restUtils; @@ -19,27 +16,19 @@ public class RegenerateSchoolReportsProcessor implements ItemProcessor process(List minCodes) throws Exception { - Long batchId = summaryDTO.getBatchId(); - StudentSearchRequest searchRequest = summaryDTO.getStudentSearchRequest(); - long countRegeneratedSchoolReports = 0l; - List reportTypes = searchRequest.getReportTypes(); + public String process(String minCode) throws Exception { + summaryDTO.setBatchId(batchId); if(log.isDebugEnabled()) { - log.debug("Process Schools: {}", !minCodes.isEmpty() ? String.join(",", minCodes) : summaryDTO.getSchools().stream().map(School::getMincode).collect(Collectors.joining(","))); + log.debug("Processing {} School Report: {} ", summaryDTO.getReportBatchType(), minCode); } - String reportType; - if(reportTypes != null && !reportTypes.isEmpty() && "NONGRADPRJ".compareToIgnoreCase(reportTypes.get(0)) == 0) - reportType = "TVRRUN"; - else - reportType = "REGALG"; - - for (String minCode : minCodes) { - countRegeneratedSchoolReports += restUtils.createAndStoreSchoolReports(minCode, reportType, summaryDTO); - } + long countRegeneratedSchoolReports = restUtils.createAndStoreSchoolReports(minCode, summaryDTO.getReportBatchType(), summaryDTO); - summaryDTO.setProcessedCount(countRegeneratedSchoolReports); - return minCodes; + summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() + countRegeneratedSchoolReports); + return minCode; } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseSchoolReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseSchoolReader.java index 705bb1d8..f5646549 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseSchoolReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseSchoolReader.java @@ -1,7 +1,5 @@ package ca.bc.gov.educ.api.batchgraduation.reader; -import ca.bc.gov.educ.api.batchgraduation.model.EdwSnapshotSchoolSummaryDTO; -import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj; import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils; import org.springframework.batch.core.JobExecution; import org.springframework.batch.item.ItemReader; @@ -22,16 +20,7 @@ public abstract class BaseSchoolReader implements ItemReader { @Value("#{stepExecutionContext['data']}") List schools; - @Value("#{stepExecutionContext['summary']}") - EdwSnapshotSchoolSummaryDTO summaryDTO; - @Value("#{stepExecution.jobExecution}") JobExecution jobExecution; - protected void fetchAccessToken() { - ResponseObj res = restUtils.getTokenResponseObject(); - if (res != null) { - summaryDTO.setAccessToken(res.getAccess_token()); - } - } } \ No newline at end of file diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/EDWSnapshotSchoolReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/EDWSnapshotSchoolReader.java index 7177e06c..afc398ea 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/EDWSnapshotSchoolReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/EDWSnapshotSchoolReader.java @@ -4,16 +4,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.item.ItemReader; +import org.springframework.beans.factory.annotation.Value; public class EDWSnapshotSchoolReader extends BaseSchoolReader implements ItemReader { private static final Logger LOGGER = LoggerFactory.getLogger(EDWSnapshotSchoolReader.class); + @Value("#{stepExecutionContext['summary']}") + EdwSnapshotSchoolSummaryDTO summaryDTO; + @Override public String read() throws Exception { String nextSchool = null; if (nextSchoolForProcessing < schools.size()) { - fetchAccessToken(); nextSchool = schools.get(nextSchoolForProcessing); LOGGER.info("School: {} - {} of {}", nextSchool, nextSchoolForProcessing + 1, summaryDTO.getReadCount()); nextSchoolForProcessing++; diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsPartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsPartitioner.java index 2c8adc80..f712d01b 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsPartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsPartitioner.java @@ -11,8 +11,6 @@ import java.util.*; import java.util.stream.Collectors; -import static ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants.SEARCH_REQUEST; - @Slf4j public class RegenerateSchoolReportsPartitioner extends BasePartitioner { @@ -51,22 +49,19 @@ public Map partition(int gridSize) { } summaryDTO.setBatchId(jobExecution.getId()); - summaryDTO.setStudentSearchRequest(searchRequest); + summaryDTO.setReportBatchType(determineReportBatchType(searchRequest.getReportTypes())); Long totalSchoolReportsCount = 0L; - List reportTypes = searchRequest.getReportTypes(); Long schoolReportsCount = 0L; List finalSchoolDistricts = new ArrayList<>(); - List schoolReportsLite = new ArrayList<>(); + List schoolReportsLite; if(processAllReports) { - if (reportTypes != null && !reportTypes.isEmpty()) { - if ("NONGRADPRJ".compareToIgnoreCase(reportTypes.get(0)) == 0) { - schoolReportsLite = restUtils.getSchoolReportsLiteByReportType("NONGRADPRJ", summaryDTO); - } else { - schoolReportsLite = restUtils.getSchoolReportsLiteByReportType( "GRADREG", summaryDTO); - } + if ("TVRRUN".compareToIgnoreCase(summaryDTO.getReportBatchType()) == 0) { + schoolReportsLite = restUtils.getSchoolReportsLiteByReportType("NONGRADPRJ", summaryDTO); + } else { + schoolReportsLite = restUtils.getSchoolReportsLiteByReportType( "GRADREG", summaryDTO); } if (schoolReportsLite != null && !schoolReportsLite.isEmpty()) { @@ -81,21 +76,19 @@ public Map partition(int gridSize) { totalSchoolReportsCount += finalSchoolDistricts.size(); } else { for (String schoolOfRecord : schoolDistricts) { - if (reportTypes != null && !reportTypes.isEmpty()) { - if ("NONGRADPRJ".compareToIgnoreCase(reportTypes.get(0)) == 0) { - schoolReportsCount += restUtils.getTotalReportsForProcessing(List.of(schoolOfRecord), "NONGRADPRJ", summaryDTO); - } else { - schoolReportsCount += restUtils.getTotalReportsForProcessing(List.of(schoolOfRecord), "GRADREG", summaryDTO); - } - if (schoolReportsCount > 0) { - finalSchoolDistricts.add(schoolOfRecord); - School school = new School(schoolOfRecord); - school.setNumberOfSchoolReports(schoolReportsCount); - summaryDTO.getSchools().add(school); - totalSchoolReportsCount += schoolReportsCount; - } - schoolReportsCount = 0L; + if ("TVRRUN".compareToIgnoreCase(summaryDTO.getReportBatchType()) == 0) { + schoolReportsCount += restUtils.getTotalReportsForProcessing(List.of(schoolOfRecord), "NONGRADPRJ", summaryDTO); + } else { + schoolReportsCount += restUtils.getTotalReportsForProcessing(List.of(schoolOfRecord), "GRADREG", summaryDTO); + } + if (schoolReportsCount > 0) { + finalSchoolDistricts.add(schoolOfRecord); + School school = new School(schoolOfRecord); + school.setNumberOfSchoolReports(schoolReportsCount); + summaryDTO.getSchools().add(school); + totalSchoolReportsCount += schoolReportsCount; } + schoolReportsCount = 0L; } } @@ -103,19 +96,37 @@ public Map partition(int gridSize) { long diff = (endTime - startTime)/1000; log.debug("Total {} schools after filters in {} sec", finalSchoolDistricts.size(), diff); - updateBatchJobHistory(createBatchJobHistory(), totalSchoolReportsCount); summaryDTO.setReadCount(totalSchoolReportsCount); summaryDTO.setProcessedCount(0); - Map map = new HashMap<>(); - ExecutionContext executionContext = new ExecutionContext(); - executionContext.put(SEARCH_REQUEST, searchRequest); - executionContext.put("data", finalSchoolDistricts); - executionContext.put("summary", summaryDTO); - executionContext.put("readCount", 0); - map.put("partition0", executionContext); + if (!finalSchoolDistricts.isEmpty()) { + updateBatchJobHistory(createBatchJobHistory(), totalSchoolReportsCount); + int partitionSize = finalSchoolDistricts.size()/gridSize + 1; + List> partitions = new LinkedList<>(); + for (int i = 0; i < finalSchoolDistricts.size(); i += partitionSize) { + partitions.add(finalSchoolDistricts.subList(i, Math.min(i + partitionSize, finalSchoolDistricts.size()))); + } + Map map = new HashMap<>(partitions.size()); + for (int i = 0; i < partitions.size(); i++) { + ExecutionContext executionContext = new ExecutionContext(); + SchoolReportsRegenSummaryDTO partitionSummaryDTO = new SchoolReportsRegenSummaryDTO(); + partitionSummaryDTO.setReportBatchType(summaryDTO.getReportBatchType()); + List data = partitions.get(i); + executionContext.put("data", data); + partitionSummaryDTO.setReadCount(data.size()); + executionContext.put("summary", partitionSummaryDTO); + executionContext.put("index",0); + String key = "partition" + i; + map.put(key, executionContext); + } + log.info("Found {} in total running on {} partitions",finalSchoolDistricts.size(),map.size()); + return map; + } + log.info("No Schools Found for School Reports Regeneration"); + return new HashMap<>(); + } - log.info("Found {} in total running on 1 partitions", totalSchoolReportsCount); - return map; + private String determineReportBatchType(List reportTypes) { + return reportTypes != null && !reportTypes.isEmpty() && "NONGRADPRJ".compareToIgnoreCase(reportTypes.get(0)) == 0 ? "TVRRUN" : "REGALG"; } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsReader.java index 4b6016c1..b7eb1976 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateSchoolReportsReader.java @@ -2,36 +2,26 @@ import ca.bc.gov.educ.api.batchgraduation.model.SchoolReportsRegenSummaryDTO; import lombok.extern.slf4j.Slf4j; -import org.springframework.batch.core.JobExecution; import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Value; -import java.util.List; - @Slf4j -public class RegenerateSchoolReportsReader implements ItemReader> { - - @Value("#{stepExecutionContext['data']}") - List schools; +public class RegenerateSchoolReportsReader extends BaseSchoolReader implements ItemReader { @Value("#{stepExecutionContext['summary']}") SchoolReportsRegenSummaryDTO summaryDTO; - @Value("#{stepExecution.jobExecution}") - JobExecution jobExecution; - - @Value("#{stepExecutionContext['readCount']}") - Long readCount; - @Override - public List read() throws Exception { - if(readCount > 0) return null; - readCount++; - if(log.isDebugEnabled()) { - log.info("Read schools Codes -> {} of {} schools", readCount, schools.size()); + public String read() throws Exception { + String nextSchool = null; + if (nextSchoolForProcessing < schools.size()) { + nextSchool = schools.get(nextSchoolForProcessing); + log.info("School: {} - {} of {}", nextSchool, nextSchoolForProcessing + 1, summaryDTO.getReadCount()); + nextSchoolForProcessing++; + } else { + aggregate("schoolReportsRegenSummaryDTO"); } - aggregate("schoolReportsRegenSummaryDTO"); - return schools; + return nextSchool; } private void aggregate(String summaryContextName) { @@ -41,7 +31,7 @@ private void aggregate(String summaryContextName) { jobExecution.getExecutionContext().put(summaryContextName, totalSummaryDTO); } totalSummaryDTO.setBatchId(summaryDTO.getBatchId()); - totalSummaryDTO.setReadCount(summaryDTO.getReadCount()); + totalSummaryDTO.setProcessedCount(totalSummaryDTO.getProcessedCount() + summaryDTO.getProcessedCount()); totalSummaryDTO.getGlobalList().addAll(summaryDTO.getGlobalList()); } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/writer/RegenerateSchoolReportsWriter.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/writer/RegenerateSchoolReportsWriter.java index 9e4d076f..03a31fdc 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/writer/RegenerateSchoolReportsWriter.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/writer/RegenerateSchoolReportsWriter.java @@ -3,15 +3,16 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; - -import java.util.List; +import org.springframework.lang.NonNull; @Slf4j -public class RegenerateSchoolReportsWriter implements ItemWriter> { +public class RegenerateSchoolReportsWriter implements ItemWriter { @Override - public void write(Chunk> chunk) throws Exception { - log.info("Regenerate School Reports Writer"); + public void write(@NonNull Chunk list) throws Exception { + if(log.isDebugEnabled()) { + log.info("Regenerate School Reports Writer: chunk size = {}", list.size()); + } } } From 98d80bee9cbc3f53cad1fea336c2642f3cb02476 Mon Sep 17 00:00:00 2001 From: Jinil Sung Date: Thu, 21 Nov 2024 15:52:28 -0800 Subject: [PATCH 4/8] GRAD2-3012: User Request REGALG is fixed to show the correct update user in audit history. GRAD2-3012: User Request REGALG is fixed to show the correct update user in audit history. --- .../api/batchgraduation/controller/JobLauncherController.java | 3 --- .../listener/BaseRunCompletionNotificationListener.java | 2 -- 2 files changed, 5 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/controller/JobLauncherController.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/controller/JobLauncherController.java index aad3e8b0..39ec3b7c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/controller/JobLauncherController.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/controller/JobLauncherController.java @@ -234,9 +234,6 @@ public ResponseEntity launchRegGradSpecialJob(@RequestBody Stu response.setTriggerBy(MANUAL); response.setStartTime(LocalDateTime.now()); response.setStatus(BatchStatusEnum.STARTED.name()); - if(StringUtils.isBlank(studentSearchRequest.getActivityCode())) { - studentSearchRequest.setActivityCode("USERDIST"); - } validateInput(response, studentSearchRequest); if(response.getException() != null) { return ResponseEntity.status(400).body(response); diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/BaseRunCompletionNotificationListener.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/BaseRunCompletionNotificationListener.java index d5f0a2b4..c779304c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/BaseRunCompletionNotificationListener.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/BaseRunCompletionNotificationListener.java @@ -3,7 +3,6 @@ import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity; import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO; import ca.bc.gov.educ.api.batchgraduation.model.ProcessError; -import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj; import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest; import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils; import ca.bc.gov.educ.api.batchgraduation.service.GradBatchHistoryService; @@ -84,7 +83,6 @@ protected void handleSummary(JobExecution jobExecution, String summaryDtoName, b } LOGGER.info(" --------------------------------------------------------------------------------------"); - ResponseObj obj = restUtils.getTokenResponseObject(); if (!isSpecialRun) { updateBackStudentFlagForErroredStudents(summaryDTO.getErrors(), jobType); } From 99a144da0a3df6550b43202f34ff024a42254dce Mon Sep 17 00:00:00 2001 From: Jinil Sung Date: Fri, 22 Nov 2024 11:32:42 -0800 Subject: [PATCH 5/8] GRAD2-3012: task is complete. GRAD2-3012: task is complete. --- ...cialRunCompletionNotificationListener.java | 23 ------------------- .../reader/BasePartitioner.java | 2 +- .../reader/BaseStudentReader.java | 11 +++++++++ ...ecalculateProjectedGradRunErrorReader.java | 1 + ...ulateProjectedGradRunErrorRetryReader.java | 1 + .../RecalculateProjectedGradRunReader.java | 1 + .../reader/RecalculateStudentErrorReader.java | 1 + .../RecalculateStudentErrorRetryReader.java | 1 + .../reader/RecalculateStudentReader.java | 1 + .../reader/RegGradAlgPartitioner.java | 4 ++++ .../reader/RegGradAlgPartitionerRetry.java | 4 ++++ .../reader/SpcRegGradAlgPartitioner.java | 3 +++ .../reader/SpcRegGradAlgPartitionerRetry.java | 4 ++++ .../reader/SpecialGradRunStudentReader.java | 1 + .../reader/SpecialProjectedGradRunReader.java | 1 + .../reader/TvrRunPartitioner.java | 4 ++++ .../reader/TvrRunPartitionerRetry.java | 4 ++++ .../api/batchgraduation/rest/RESTService.java | 20 ++++++++++++---- .../EducGradBatchGraduationApiConstants.java | 1 + 19 files changed, 60 insertions(+), 28 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/SpecialRunCompletionNotificationListener.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/SpecialRunCompletionNotificationListener.java index 44f09da8..a61da3b6 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/SpecialRunCompletionNotificationListener.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/listener/SpecialRunCompletionNotificationListener.java @@ -1,22 +1,16 @@ package ca.bc.gov.educ.api.batchgraduation.listener; -import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest; import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils; import ca.bc.gov.educ.api.batchgraduation.util.JsonTransformer; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.time.ZoneId; import java.util.Date; -import java.util.List; - -import static ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants.SEARCH_REQUEST; @Component public class SpecialRunCompletionNotificationListener extends BaseRunCompletionNotificationListener { @@ -29,8 +23,6 @@ public class SpecialRunCompletionNotificationListener extends BaseRunCompletionN @Autowired JsonTransformer jsonTransformer; - private static final String RUN_BY = "runBy"; - @Override public void afterJob(JobExecution jobExecution) { if (jobExecution.getStatus() == BatchStatus.COMPLETED) { @@ -38,22 +30,7 @@ public void afterJob(JobExecution jobExecution) { LOGGER.info("======================================================================================="); LOGGER.info("Special Job completed in {} s with jobExecution status {}", elapsedTimeMillis/1000, jobExecution.getStatus()); handleSummary(jobExecution, "spcRunAlgSummaryDTO", true); - processGradStudentRecordJobHistory(jobExecution); LOGGER.info("======================================================================================="); } } - - private void processGradStudentRecordJobHistory(JobExecution jobExecution) { - - JobParameters jobParameters = jobExecution.getJobParameters(); - Long batchId = jobExecution.getId(); - String userName = jobParameters.getString(RUN_BY); - - String searchRequest = jobParameters.getString(SEARCH_REQUEST, "{}"); - StudentSearchRequest req = (StudentSearchRequest)jsonTransformer.unmarshall(searchRequest, StudentSearchRequest.class); - - restUtils.updateStudentGradRecordHistory(List.of(), batchId, userName, StringUtils.upperCase(req.getActivityCode())); - } - - } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BasePartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BasePartitioner.java index 621570f7..7b8b344e 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BasePartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BasePartitioner.java @@ -26,7 +26,7 @@ public abstract class BasePartitioner extends SimplePartitioner { private static final Logger LOGGER = LoggerFactory.getLogger(BasePartitioner.class); private static final String RERUN_TYPE = "reRunType"; - private static final String RUN_BY = "runBy"; + protected static final String RUN_BY = "runBy"; private static final String PREV_BATCH_ID = "previousBatchId"; private static final String RERUN_ALL = "RERUN_ALL"; private static final String RERUN_FAILED = "RERUN_FAILED"; diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java index d37c999a..a97a75e6 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BaseStudentReader.java @@ -3,6 +3,8 @@ import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO; import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj; import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils; +import ca.bc.gov.educ.api.batchgraduation.util.ThreadLocalStateUtil; +import org.apache.commons.lang3.StringUtils; import org.springframework.batch.core.JobExecution; import org.springframework.batch.item.ItemReader; import org.springframework.beans.factory.annotation.Autowired; @@ -20,6 +22,9 @@ public abstract class BaseStudentReader implements ItemReader { @Value("#{stepExecutionContext['index']}") protected Integer nxtStudentForProcessing; + @Value("#{stepExecutionContext['runBy']}") + protected String runBy; + @Value("#{stepExecutionContext['data']}") protected List studentList; @@ -35,4 +40,10 @@ protected void fetchAccessToken() { summaryDTO.setAccessToken(res.getAccess_token()); } } + + protected void setUserName() { + if (StringUtils.isNotBlank(runBy) && StringUtils.isBlank(ThreadLocalStateUtil.getCurrentUser())) { + ThreadLocalStateUtil.setCurrentUser(runBy); + } + } } \ No newline at end of file diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java index 30e3966d..9293eb7d 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorReader.java @@ -12,6 +12,7 @@ public class RecalculateProjectedGradRunErrorReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java index edd68426..e17cb48d 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunErrorRetryReader.java @@ -12,6 +12,7 @@ public class RecalculateProjectedGradRunErrorRetryReader extends BaseStudentRead @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java index 3cbca59d..d51253c2 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateProjectedGradRunReader.java @@ -12,6 +12,7 @@ public class RecalculateProjectedGradRunReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java index 6a2a3cc4..a74b5f9c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorReader.java @@ -12,6 +12,7 @@ public class RecalculateStudentErrorReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java index 64c2c31e..810d2249 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentErrorRetryReader.java @@ -12,6 +12,7 @@ public class RecalculateStudentErrorRetryReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java index ae887f4c..c5d8f2b9 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RecalculateStudentReader.java @@ -12,6 +12,7 @@ public class RecalculateStudentReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); summaryDTO.setReadCount(studentList.size()); UUID nextStudent = null; diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitioner.java index be76349c..c5b15f73 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitioner.java @@ -6,6 +6,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Value; @@ -31,6 +32,8 @@ public JobExecution getJobExecution() { public Map partition(int gridSize) { initializeRunType(); BatchGradAlgorithmJobHistoryEntity jobHistory = createBatchJobHistory(); + JobParameters jobParameters = jobExecution.getJobParameters(); + String username = jobParameters.getString(RUN_BY); List studentList; if (runType == RunTypeEnum.NORMAL_JOB_PROCESS) { studentList = restUtils.getStudentsForAlgorithm(); @@ -57,6 +60,7 @@ public Map partition(int gridSize) { summaryDTO.setReadCount(data.size()); executionContext.put("summary", summaryDTO); executionContext.put("index",0); + executionContext.put("runBy", username); String key = "partition" + i; map.put(key, executionContext); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitionerRetry.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitionerRetry.java index dad374d3..cebc76a1 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitionerRetry.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitionerRetry.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Value; @@ -30,6 +31,8 @@ public JobExecution getJobExecution() { @Override public Map partition(int gridSize) { Long batchId = jobExecution.getId(); + JobParameters jobParameters = jobExecution.getJobParameters(); + String username = jobParameters.getString(RUN_BY); List studentList = getInputDataForErroredStudents(batchId); createTotalSummaryDTO("regGradAlgSummaryDTO"); @@ -48,6 +51,7 @@ public Map partition(int gridSize) { summaryDTO.setReadCount(data.size()); executionContext.put("summary", summaryDTO); executionContext.put("index",0); + executionContext.put("runBy", username); String key = "partition" + i; map.put(key, executionContext); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitioner.java index 82df4cc8..d9307e8f 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitioner.java @@ -4,6 +4,7 @@ import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO; import ca.bc.gov.educ.api.batchgraduation.model.RunTypeEnum; import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest; +import ca.bc.gov.educ.api.batchgraduation.util.ThreadLocalStateUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; @@ -37,6 +38,7 @@ public Map partition(int gridSize) { BatchGradAlgorithmJobHistoryEntity jobHistory = createBatchJobHistory(); List studentList; JobParameters jobParameters = jobExecution.getJobParameters(); + String username = jobParameters.getString(RUN_BY); String searchRequest = jobParameters.getString(SEARCH_REQUEST, "{}"); StudentSearchRequest req = (StudentSearchRequest)jsonTransformer.unmarshall(searchRequest, StudentSearchRequest.class); if (runType == RunTypeEnum.NORMAL_JOB_PROCESS) { @@ -64,6 +66,7 @@ public Map partition(int gridSize) { summaryDTO.setReadCount(data.size()); executionContext.put("summary", summaryDTO); executionContext.put("index",0); + executionContext.put("runBy", username); String key = "partition" + i; map.put(key, executionContext); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitionerRetry.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitionerRetry.java index e4eddd53..49d2c634 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitionerRetry.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitionerRetry.java @@ -3,6 +3,7 @@ import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO; import ca.bc.gov.educ.api.batchgraduation.model.RunTypeEnum; import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest; +import ca.bc.gov.educ.api.batchgraduation.util.ThreadLocalStateUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; @@ -34,6 +35,8 @@ public JobExecution getJobExecution() { @Override public Map partition(int gridSize) { Long batchId = jobExecution.getId(); + JobParameters jobParameters = jobExecution.getJobParameters(); + String username = jobParameters.getString(RUN_BY); List studentList = getInputDataForErroredStudents(batchId); createTotalSummaryDTO("spcRunAlgSummaryDTO"); @@ -52,6 +55,7 @@ public Map partition(int gridSize) { summaryDTO.setReadCount(data.size()); executionContext.put("summary", summaryDTO); executionContext.put("index",0); + executionContext.put("runBy", username); String key = "partition" + i; map.put(key, executionContext); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialGradRunStudentReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialGradRunStudentReader.java index ce051e9f..5d1f2884 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialGradRunStudentReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialGradRunStudentReader.java @@ -12,6 +12,7 @@ public class SpecialGradRunStudentReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialProjectedGradRunReader.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialProjectedGradRunReader.java index a12faff6..408035f1 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialProjectedGradRunReader.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpecialProjectedGradRunReader.java @@ -12,6 +12,7 @@ public class SpecialProjectedGradRunReader extends BaseStudentReader { @Override public UUID read() throws Exception { fetchAccessToken(); + setUserName(); UUID nextStudent = null; if (nxtStudentForProcessing < studentList.size()) { diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitioner.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitioner.java index 5f9d2b5f..a8a96458 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitioner.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitioner.java @@ -6,6 +6,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Value; @@ -31,6 +32,8 @@ public JobExecution getJobExecution() { public Map partition(int gridSize) { initializeRunType(); BatchGradAlgorithmJobHistoryEntity jobHistory = createBatchJobHistory(); + JobParameters jobParameters = jobExecution.getJobParameters(); + String username = jobParameters.getString(RUN_BY); List studentList; if (runType == RunTypeEnum.NORMAL_JOB_PROCESS) { studentList = restUtils.getStudentsForProjectedAlgorithm(); @@ -57,6 +60,7 @@ public Map partition(int gridSize) { summaryDTO.setReadCount(data.size()); executionContext.put("summary", summaryDTO); executionContext.put("index",0); + executionContext.put("runBy", username); String key = "partition" + i; map.put(key, executionContext); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitionerRetry.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitionerRetry.java index 6aba6575..a4bb0df8 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitionerRetry.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/TvrRunPartitionerRetry.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; import org.springframework.batch.item.ExecutionContext; import org.springframework.beans.factory.annotation.Value; @@ -30,6 +31,8 @@ public JobExecution getJobExecution() { @Override public Map partition(int gridSize) { Long batchId = jobExecution.getId(); + JobParameters jobParameters = jobExecution.getJobParameters(); + String username = jobParameters.getString(RUN_BY); List studentList = getInputDataForErroredStudents(batchId); createTotalSummaryDTO("tvrRunSummaryDTO"); @@ -48,6 +51,7 @@ public Map partition(int gridSize) { summaryDTO.setReadCount(data.size()); executionContext.put("summary", summaryDTO); executionContext.put("index",0); + executionContext.put("runBy", username); String key = "partition" + i; map.put(key, executionContext); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java index c5e128de..c288e1f7 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java @@ -83,7 +83,10 @@ public T get(String url, Class clazz) { obj = this.batchWebClient .get() .uri(url) - .headers(h -> h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID())) + .headers(h -> { + h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID()); + h.set(EducGradBatchGraduationApiConstants.USERNAME, ThreadLocalStateUtil.getCurrentUser()); + }) .retrieve() // if 5xx errors, throw Service error .onStatus(HttpStatusCode::is5xxServerError, @@ -154,7 +157,10 @@ public T post(String url, Object body, Class clazz) { try { obj = this.batchWebClient.post() .uri(url) - .headers(h -> h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID())) + .headers(h -> { + h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID()); + h.set(EducGradBatchGraduationApiConstants.USERNAME, ThreadLocalStateUtil.getCurrentUser()); + }) .body(BodyInserters.fromValue(body)) .retrieve() .onStatus(HttpStatusCode::is5xxServerError, @@ -192,7 +198,10 @@ public T put(String url, Object body, Class clazz) { try { obj = this.batchWebClient.put() .uri(url) - .headers(h -> h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID())) + .headers(h -> { + h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID()); + h.set(EducGradBatchGraduationApiConstants.USERNAME, ThreadLocalStateUtil.getCurrentUser()); + }) .body(BodyInserters.fromValue(body)) .retrieve() .onStatus(HttpStatusCode::is5xxServerError, @@ -227,7 +236,10 @@ public T delete(String url, Class boundClass) { T obj; try { obj = this.batchWebClient.delete().uri(url) - .headers(h -> h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID())) + .headers(h -> { + h.set(EducGradBatchGraduationApiConstants.CORRELATION_ID, ThreadLocalStateUtil.getCorrelationID()); + h.set(EducGradBatchGraduationApiConstants.USERNAME, ThreadLocalStateUtil.getCurrentUser()); + }) .retrieve().bodyToMono(boundClass).block(); } catch(Exception e) { // catches IOExceptions and the like diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java index dd44be18..871d332b 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java @@ -16,6 +16,7 @@ public class EducGradBatchGraduationApiConstants { public static final String API_ROOT_MAPPING = ""; public static final String API_VERSION = "v1"; public static final String CORRELATION_ID = "correlationID"; + public static final String USERNAME = "username"; public static final String GRAD_BATCH_API_ROOT_MAPPING = "/api/" + API_VERSION + "/batch"; // Manual Run From 52e90489d551c2ae0bd495b1011fad9a397d3fcd Mon Sep 17 00:00:00 2001 From: Jinil Sung Date: Fri, 22 Nov 2024 14:12:00 -0800 Subject: [PATCH 6/8] Fixed sonar complaints. Fixed sonar complaints. --- .../batchgraduation/config/RequestInterceptor.java | 3 +-- .../educ/api/batchgraduation/rest/RESTService.java | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/RequestInterceptor.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/RequestInterceptor.java index 57804272..66c45ddb 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/RequestInterceptor.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/RequestInterceptor.java @@ -43,8 +43,7 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons // username Authentication auth = SecurityContextHolder.getContext().getAuthentication(); - if (auth instanceof JwtAuthenticationToken) { - JwtAuthenticationToken authenticationToken = (JwtAuthenticationToken) auth; + if (auth instanceof JwtAuthenticationToken authenticationToken) { Jwt jwt = (Jwt) authenticationToken.getCredentials(); String username = JwtUtil.getName(jwt); ThreadLocalStateUtil.setCurrentUser(username); diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java index c288e1f7..926d90c2 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/rest/RESTService.java @@ -71,7 +71,7 @@ public T get(String url, Class clazz, String accessToken) { throw new ServiceException(getErrorMessage( url, e.getLocalizedMessage()), - (e instanceof WebClientResponseException) ? ((WebClientResponseException) e).getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), + (e instanceof WebClientResponseException exception) ? exception.getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), e); } return obj; @@ -107,7 +107,7 @@ public T get(String url, Class clazz) { throw new ServiceException(getErrorMessage( url, e.getLocalizedMessage()), - (e instanceof WebClientResponseException) ? ((WebClientResponseException) e).getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), + (e instanceof WebClientResponseException exception) ? exception.getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), e); } return obj; @@ -146,7 +146,7 @@ public T post(String url, Object body, Class clazz, String accessToken) { throw new ServiceException(getErrorMessage( url, e.getLocalizedMessage()), - (e instanceof WebClientResponseException) ? ((WebClientResponseException) e).getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), + (e instanceof WebClientResponseException exception) ? exception.getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), e); } return obj; @@ -177,7 +177,7 @@ public T post(String url, Object body, Class clazz) { throw new ServiceException(getErrorMessage( url, e.getLocalizedMessage()), - (e instanceof WebClientResponseException) ? ((WebClientResponseException) e).getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), + (e instanceof WebClientResponseException exception) ? exception.getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), e); } return obj; @@ -218,7 +218,7 @@ public T put(String url, Object body, Class clazz) { throw new ServiceException(getErrorMessage( url, e.getLocalizedMessage()), - (e instanceof WebClientResponseException) ? ((WebClientResponseException) e).getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), + (e instanceof WebClientResponseException exception) ? exception.getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), e); } return obj; @@ -246,7 +246,7 @@ public T delete(String url, Class boundClass) { throw new ServiceException(getErrorMessage( url, e.getLocalizedMessage()), - (e instanceof WebClientResponseException) ? ((WebClientResponseException) e).getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), + (e instanceof WebClientResponseException exception) ? exception.getStatusCode().value() : HttpStatus.SERVICE_UNAVAILABLE.value(), e); } return obj; From a2e3bba440fe04e27a6e0b83f78072ca318e9bf5 Mon Sep 17 00:00:00 2001 From: githubmamatha <106563495+githubmamatha@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:37:59 -0800 Subject: [PATCH 7/8] ZAPscan version upgraded and added dependabot github actions (#541) --- .github/dependabot.yml | 4 ++++ .../workflows/build.from.developer.branch.deploy.to.dev.yml | 2 +- .github/workflows/build.from.main.branch.deploy.to.dev.yml | 2 +- .github/workflows/build.from.release.branch.deploy.to.dev.yml | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0d28b300..85708f49 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,7 @@ updates: schedule: interval: "daily" target-branch: "grad-release" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/build.from.developer.branch.deploy.to.dev.yml b/.github/workflows/build.from.developer.branch.deploy.to.dev.yml index ad579c75..2026371d 100644 --- a/.github/workflows/build.from.developer.branch.deploy.to.dev.yml +++ b/.github/workflows/build.from.developer.branch.deploy.to.dev.yml @@ -154,6 +154,6 @@ jobs: # now hit it with a zap scan - name: ZAP Scan - uses: zaproxy/action-api-scan@v0.7.0 + uses: zaproxy/action-api-scan@v0.9.0 with: target: 'https://${{ env.REPO_NAME }}-${{ env.OPENSHIFT_NAMESPACE }}.apps.silver.devops.gov.bc.ca/api/v1/api-docs' diff --git a/.github/workflows/build.from.main.branch.deploy.to.dev.yml b/.github/workflows/build.from.main.branch.deploy.to.dev.yml index d192192d..80ad3870 100644 --- a/.github/workflows/build.from.main.branch.deploy.to.dev.yml +++ b/.github/workflows/build.from.main.branch.deploy.to.dev.yml @@ -141,6 +141,6 @@ jobs: # now hit it with a zap scan - name: ZAP Scan - uses: zaproxy/action-api-scan@v0.7.0 + uses: zaproxy/action-api-scan@v0.9.0 with: target: 'https://${{ env.REPO_NAME }}-${{ env.OPENSHIFT_NAMESPACE }}.apps.silver.devops.gov.bc.ca/api/v1/api-docs' diff --git a/.github/workflows/build.from.release.branch.deploy.to.dev.yml b/.github/workflows/build.from.release.branch.deploy.to.dev.yml index 0375760b..e4296d52 100644 --- a/.github/workflows/build.from.release.branch.deploy.to.dev.yml +++ b/.github/workflows/build.from.release.branch.deploy.to.dev.yml @@ -149,6 +149,6 @@ jobs: # now hit it with a zap scan - name: ZAP Scan - uses: zaproxy/action-api-scan@v0.7.0 + uses: zaproxy/action-api-scan@v0.9.0 with: target: 'https://${{ env.REPO_NAME }}-${{ env.OPENSHIFT_NAMESPACE }}.apps.silver.devops.gov.bc.ca/api/v1/api-docs' From 743adfd24cf4e704e173a0368eeda5d087554200 Mon Sep 17 00:00:00 2001 From: githubmamatha <106563495+githubmamatha@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:14:10 -0800 Subject: [PATCH 8/8] Update pom.xml --- api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/pom.xml b/api/pom.xml index 0b45da9e..35c83805 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -6,7 +6,7 @@ ca.bc.gov.educ educ-grad-batch-graduation-api - 1.8.71 + 1.8.72 educ-grad-batch-graduation-api Ministry of Education GRAD BATCH GRADUATION API