From baee716cf63cb51df0489ad3d742df046ca958ea Mon Sep 17 00:00:00 2001 From: Viacheslav Kolesnyk <94473337+viacheslavkol@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:09:36 +0200 Subject: [PATCH] feat(reindex): Implement Reindex status Endpoint (#641) * feat(reindex): Implement Reindex status Endpoint - Implement DB schema - Implement status update trigger on processed items updates - Implement endpoint Implements: MSEARCH-797 --- descriptors/ModuleDescriptor-template.json | 16 ++- .../controller/IndexManagementController.java | 8 ++ .../search/converter/ReindexStatusMapper.java | 11 ++ .../model/reindex/ReindexStatusEntity.java | 34 +++++ .../search/model/types/ReindexStatus.java | 19 +++ .../reindex/ReindexOrchestrationService.java | 4 +- .../reindex/ReindexRangeIndexService.java | 42 +++++- .../reindex/jdbc/ReindexStatusRepository.java | 86 ++++++++++++ .../v4.0/create-reindex-entity-tables.xml | 34 +++++ .../v4.0/update-reindex-status-trigger.sql | 22 ++++ .../examples/result/ReindexStatusResult.yaml | 28 ++++ .../resources/swagger.api/mod-search.yaml | 3 + .../search-index-reindex-status.yaml | 24 ++++ .../schemas/response/reindexStatusItem.yaml | 33 +++++ .../IndexManagementControllerTest.java | 18 +++ .../ReindexOrchestrationServiceTest.java | 25 +++- .../reindex/ReindexRangeIndexServiceTest.java | 74 ++++++++++- .../jdbc/ReindexStatusRepositoryIT.java | 123 ++++++++++++++++++ .../search/support/base/ApiEndpoints.java | 4 + .../resources/sql/populate-reindex-status.sql | 6 + 20 files changed, 603 insertions(+), 11 deletions(-) create mode 100644 src/main/java/org/folio/search/converter/ReindexStatusMapper.java create mode 100644 src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java create mode 100644 src/main/java/org/folio/search/model/types/ReindexStatus.java create mode 100644 src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java create mode 100644 src/main/resources/changelog/changes/v4.0/update-reindex-status-trigger.sql create mode 100644 src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml create mode 100644 src/main/resources/swagger.api/paths/search-index/search-index-reindex-status.yaml create mode 100644 src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml create mode 100644 src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java create mode 100644 src/test/resources/sql/populate-reindex-status.sql diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index 8594cd297..d8db61958 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -4,7 +4,7 @@ "provides": [ { "id": "indices", - "version": "0.7", + "version": "0.8", "handlers": [ { "methods": [ @@ -74,6 +74,15 @@ "modulePermissions": [ "user-tenants.collection.get" ] + }, + { + "methods": [ + "GET" + ], + "pathPattern": "/search/index/instance-records/reindex/status", + "permissionsRequired": [ + "search.index.reindex.status.get" + ] } ] }, @@ -693,6 +702,11 @@ "displayName": "Search - starts inventory reindex operation", "description": "Starts inventory reindex operation" }, + { + "permissionName": "search.index.reindex.status.get", + "displayName": "Search - returns reindex status for entities", + "description": "Returns reindex status for entities" + }, { "permissionName": "search.facets.collection.get", "displayName": "Search - returns facets for a query for given filter options by record type", diff --git a/src/main/java/org/folio/search/controller/IndexManagementController.java b/src/main/java/org/folio/search/controller/IndexManagementController.java index a31daba31..fd75da615 100644 --- a/src/main/java/org/folio/search/controller/IndexManagementController.java +++ b/src/main/java/org/folio/search/controller/IndexManagementController.java @@ -8,12 +8,14 @@ import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.domain.dto.ReindexJob; import org.folio.search.domain.dto.ReindexRequest; +import org.folio.search.domain.dto.ReindexStatusItem; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.domain.dto.UpdateIndexDynamicSettingsRequest; import org.folio.search.domain.dto.UpdateMappingsRequest; import org.folio.search.rest.resource.IndexManagementApi; import org.folio.search.service.IndexService; import org.folio.search.service.ResourceService; +import org.folio.search.service.reindex.ReindexRangeIndexService; import org.springframework.http.ResponseEntity; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.RequestMapping; @@ -30,6 +32,7 @@ public class IndexManagementController implements IndexManagementApi { private final IndexService indexService; + private final ReindexRangeIndexService reindexRangeService; private final ResourceService resourceService; @Override @@ -59,4 +62,9 @@ public ResponseEntity updateIndexDynamicSettings( public ResponseEntity updateMappings(String tenantId, UpdateMappingsRequest request) { return ResponseEntity.ok(indexService.updateMappings(request.getResourceName(), tenantId)); } + + @Override + public ResponseEntity> getReindexStatus(String tenantId) { + return ResponseEntity.ok(reindexRangeService.getReindexStatuses(tenantId)); + } } diff --git a/src/main/java/org/folio/search/converter/ReindexStatusMapper.java b/src/main/java/org/folio/search/converter/ReindexStatusMapper.java new file mode 100644 index 000000000..2137679ed --- /dev/null +++ b/src/main/java/org/folio/search/converter/ReindexStatusMapper.java @@ -0,0 +1,11 @@ +package org.folio.search.converter; + +import org.folio.search.domain.dto.ReindexStatusItem; +import org.folio.search.model.reindex.ReindexStatusEntity; +import org.mapstruct.Mapper; + +@Mapper(componentModel = "spring") +public interface ReindexStatusMapper { + + ReindexStatusItem convert(ReindexStatusEntity entity); +} diff --git a/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java b/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java new file mode 100644 index 000000000..42e59bca4 --- /dev/null +++ b/src/main/java/org/folio/search/model/reindex/ReindexStatusEntity.java @@ -0,0 +1,34 @@ +package org.folio.search.model.reindex; + +import java.sql.Timestamp; +import lombok.Data; +import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.model.types.ReindexStatus; + +@Data +public class ReindexStatusEntity { + + public static final String REINDEX_STATUS_TABLE = "reindex_status"; + public static final String ENTITY_TYPE_COLUMN = "entity_type"; + public static final String STATUS_COLUMN = "status"; + public static final String TOTAL_MERGE_RANGES_COLUMN = "total_merge_ranges"; + public static final String PROCESSED_MERGE_RANGES_COLUMN = "processed_merge_ranges"; + public static final String TOTAL_UPLOAD_RANGES_COLUMN = "total_upload_ranges"; + public static final String PROCESSED_UPLOAD_RANGES_COLUMN = "processed_upload_ranges"; + public static final String START_TIME_MERGE_COLUMN = "start_time_merge"; + public static final String END_TIME_MERGE_COLUMN = "end_time_merge"; + public static final String START_TIME_UPLOAD_COLUMN = "start_time_upload"; + public static final String END_TIME_UPLOAD_COLUMN = "end_time_upload"; + + private final ReindexEntityType entityType; + private final ReindexStatus status; + private int totalMergeRanges; + private int processedMergeRanges; + private int totalUploadRanges; + private int processedUploadRanges; + private Timestamp startTimeMerge; + private Timestamp endTimeMerge; + private Timestamp startTimeUpload; + private Timestamp endTimeUpload; + +} diff --git a/src/main/java/org/folio/search/model/types/ReindexStatus.java b/src/main/java/org/folio/search/model/types/ReindexStatus.java new file mode 100644 index 000000000..af8d28fa9 --- /dev/null +++ b/src/main/java/org/folio/search/model/types/ReindexStatus.java @@ -0,0 +1,19 @@ +package org.folio.search.model.types; + +import lombok.Getter; + +@Getter +public enum ReindexStatus { + + MERGE_IN_PROGRESS("Merge In Progress"), + MERGE_COMPLETED("Merge Completed"), + UPLOAD_IN_PROGRESS("Upload In Progress"), + UPLOAD_COMPLETED("Upload Completed"), + UPLOAD_FAILED("Upload Failed"); + + private final String value; + + ReindexStatus(String value) { + this.value = value; + } +} diff --git a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java index 959c295c1..5330bab53 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java @@ -23,9 +23,11 @@ public boolean process(ReindexRangeIndexEvent event) { var folioIndexOperationResponse = elasticRepository.indexResources(documents); rangeIndexService.updateFinishDate(event); if (folioIndexOperationResponse.getStatus() == FolioIndexOperationResponse.StatusEnum.ERROR) { - // TODO MSEARCH-797 - update status as failed indicating upload has failed + rangeIndexService.setReindexUploadFailed(event.getEntityType()); throw new ReindexException(folioIndexOperationResponse.getErrorMessage()); } + + rangeIndexService.addProcessedUploadRanges(event.getEntityType(), documents.size()); return true; } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexRangeIndexService.java b/src/main/java/org/folio/search/service/reindex/ReindexRangeIndexService.java index 5a560e290..f63678e0b 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexRangeIndexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexRangeIndexService.java @@ -15,17 +15,26 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import org.folio.search.converter.ReindexStatusMapper; +import org.folio.search.domain.dto.ReindexStatusItem; import org.folio.search.domain.dto.ResourceEvent; +import org.folio.search.exception.RequestValidationException; import org.folio.search.model.event.ReindexRangeIndexEvent; import org.folio.search.model.reindex.UploadRangeEntity; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.service.consortium.ConsortiumTenantService; import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; +import org.folio.search.service.reindex.jdbc.ReindexStatusRepository; +import org.folio.spring.integration.XOkapiHeaders; import org.folio.spring.tools.kafka.FolioMessageProducer; import org.springframework.stereotype.Service; @Service public class ReindexRangeIndexService { + static final String REQUEST_NOT_ALLOWED_MSG = + "The request not allowed for member tenant of consortium environment"; + private static final Map RESOURCE_NAME_MAP = Map.of( ReindexEntityType.INSTANCE, INSTANCE_RESOURCE, ReindexEntityType.SUBJECT, INSTANCE_SUBJECT_RESOURCE, @@ -35,11 +44,19 @@ public class ReindexRangeIndexService { private final Map repositories; private final FolioMessageProducer indexRangeEventProducer; + private final ReindexStatusRepository statusRepository; + private final ReindexStatusMapper reindexStatusMapper; + private final ConsortiumTenantService tenantService; public ReindexRangeIndexService(List repositories, - FolioMessageProducer indexRangeEventProducer) { + FolioMessageProducer indexRangeEventProducer, + ReindexStatusRepository statusRepository, ReindexStatusMapper reindexStatusMapper, + ConsortiumTenantService tenantService) { this.repositories = repositories.stream().collect(Collectors.toMap(ReindexJdbcRepository::entityType, identity())); this.indexRangeEventProducer = indexRangeEventProducer; + this.statusRepository = statusRepository; + this.reindexStatusMapper = reindexStatusMapper; + this.tenantService = tenantService; } public void prepareAndSendIndexRanges(ReindexEntityType entityType) { @@ -67,6 +84,29 @@ public void updateFinishDate(ReindexRangeIndexEvent event) { repository.setIndexRangeFinishDate(event.getId(), Timestamp.from(Instant.now())); } + public List getReindexStatuses(String tenantId) { + var centralTenant = tenantService.getCentralTenant(tenantId); + if (centralTenant.isPresent() && !centralTenant.get().equals(tenantId)) { + throw new RequestValidationException(REQUEST_NOT_ALLOWED_MSG, XOkapiHeaders.TENANT, tenantId); + } + + var statuses = statusRepository.getReindexStatuses(); + + return statuses.stream().map(reindexStatusMapper::convert).toList(); + } + + public void setReindexUploadFailed(ReindexEntityType entityType) { + statusRepository.setReindexUploadFailed(entityType); + } + + public void addProcessedMergeRanges(ReindexEntityType entityType, int processedMergeRanges) { + statusRepository.addReindexCounts(entityType, processedMergeRanges, 0); + } + + public void addProcessedUploadRanges(ReindexEntityType entityType, int processedUploadRanges) { + statusRepository.addReindexCounts(entityType, 0, processedUploadRanges); + } + private List prepareEvents(List uploadRanges) { return uploadRanges.stream() .map(range -> { diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java new file mode 100644 index 000000000..be1b4d9bb --- /dev/null +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepository.java @@ -0,0 +1,86 @@ +package org.folio.search.service.reindex.jdbc; + +import static org.folio.search.model.reindex.ReindexStatusEntity.END_TIME_MERGE_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.END_TIME_UPLOAD_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.PROCESSED_MERGE_RANGES_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.PROCESSED_UPLOAD_RANGES_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.REINDEX_STATUS_TABLE; +import static org.folio.search.model.reindex.ReindexStatusEntity.START_TIME_MERGE_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.START_TIME_UPLOAD_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.STATUS_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.TOTAL_MERGE_RANGES_COLUMN; +import static org.folio.search.model.reindex.ReindexStatusEntity.TOTAL_UPLOAD_RANGES_COLUMN; +import static org.folio.search.utils.JdbcUtils.getFullTableName; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.folio.search.model.reindex.ReindexStatusEntity; +import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.model.types.ReindexStatus; +import org.folio.spring.FolioExecutionContext; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +@Repository +@RequiredArgsConstructor +public class ReindexStatusRepository { + + private static final String SELECT_REINDEX_STATUS_BY_REINDEX_ID_SQL = "SELECT * FROM %s;"; + private static final String UPDATE_REINDEX_STATUS_SQL = """ + UPDATE %s + SET status = ?, %s = ? + WHERE entity_type = ?; + """; + private static final String UPDATE_REINDEX_COUNTS_SQL = """ + UPDATE %s + SET processed_merge_ranges = processed_merge_ranges + ?, + processed_upload_ranges = processed_upload_ranges + ? + WHERE entity_type = ?; + """; + + private final FolioExecutionContext context; + private final JdbcTemplate jdbcTemplate; + + + public List getReindexStatuses() { + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = SELECT_REINDEX_STATUS_BY_REINDEX_ID_SQL.formatted(fullTableName); + return jdbcTemplate.query(sql, reindexStatusRowMapper()); + } + + public void setReindexUploadFailed(ReindexEntityType entityType) { + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = UPDATE_REINDEX_STATUS_SQL.formatted(fullTableName, END_TIME_UPLOAD_COLUMN); + + jdbcTemplate.update(sql, ReindexStatus.UPLOAD_FAILED.name(), Timestamp.from(Instant.now()), entityType.name()); + } + + public void addReindexCounts(ReindexEntityType entityType, int processedMergeRanges, + int processedUploadRanges) { + var fullTableName = getFullTableName(context, REINDEX_STATUS_TABLE); + var sql = UPDATE_REINDEX_COUNTS_SQL.formatted(fullTableName); + + jdbcTemplate.update(sql, processedMergeRanges, processedUploadRanges, entityType.name()); + } + + private RowMapper reindexStatusRowMapper() { + return (rs, rowNum) -> { + var reindexStatus = new ReindexStatusEntity( + ReindexEntityType.valueOf(rs.getString(ReindexStatusEntity.ENTITY_TYPE_COLUMN)), + ReindexStatus.valueOf(rs.getString(STATUS_COLUMN)) + ); + reindexStatus.setTotalMergeRanges(rs.getInt(TOTAL_MERGE_RANGES_COLUMN)); + reindexStatus.setProcessedMergeRanges(rs.getInt(PROCESSED_MERGE_RANGES_COLUMN)); + reindexStatus.setTotalUploadRanges(rs.getInt(TOTAL_UPLOAD_RANGES_COLUMN)); + reindexStatus.setProcessedUploadRanges(rs.getInt(PROCESSED_UPLOAD_RANGES_COLUMN)); + reindexStatus.setStartTimeMerge(rs.getTimestamp(START_TIME_MERGE_COLUMN)); + reindexStatus.setEndTimeMerge(rs.getTimestamp(END_TIME_MERGE_COLUMN)); + reindexStatus.setStartTimeUpload(rs.getTimestamp(START_TIME_UPLOAD_COLUMN)); + reindexStatus.setEndTimeUpload(rs.getTimestamp(END_TIME_UPLOAD_COLUMN)); + return reindexStatus; + }; + } +} diff --git a/src/main/resources/changelog/changes/v4.0/create-reindex-entity-tables.xml b/src/main/resources/changelog/changes/v4.0/create-reindex-entity-tables.xml index 3c424cce6..a2b270790 100644 --- a/src/main/resources/changelog/changes/v4.0/create-reindex-entity-tables.xml +++ b/src/main/resources/changelog/changes/v4.0/create-reindex-entity-tables.xml @@ -229,6 +229,33 @@ + + + + + + + + Create reindex_status table + + + + + + + + + + + + + + + + + + + @@ -263,4 +290,11 @@ + + + + + + + diff --git a/src/main/resources/changelog/changes/v4.0/update-reindex-status-trigger.sql b/src/main/resources/changelog/changes/v4.0/update-reindex-status-trigger.sql new file mode 100644 index 000000000..6957d459d --- /dev/null +++ b/src/main/resources/changelog/changes/v4.0/update-reindex-status-trigger.sql @@ -0,0 +1,22 @@ +CREATE OR REPLACE FUNCTION update_reindex_status_trigger() +RETURNS TRIGGER AS $$ +BEGIN + -- update status and end time for merge + IF OLD.status = 'MERGE_IN_PROGRESS' and NEW.total_merge_ranges = NEW.processed_merge_ranges + THEN NEW.status = 'MERGE_COMPLETED'; NEW.end_time_merge = current_timestamp; + ELSE + -- update status and end time for upload + IF OLD.status = 'UPLOAD_IN_PROGRESS' and NEW.total_upload_ranges = NEW.processed_upload_ranges + THEN NEW.status = 'UPLOAD_COMPLETED'; NEW.end_time_upload = current_timestamp; + END IF; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS reindex_status_updated_trigger ON reindex_status CASCADE; +CREATE TRIGGER reindex_status_updated_trigger + BEFORE UPDATE + ON reindex_status + FOR EACH ROW + EXECUTE FUNCTION update_reindex_status_trigger(); diff --git a/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml b/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml new file mode 100644 index 000000000..bc10d89f6 --- /dev/null +++ b/src/main/resources/swagger.api/examples/result/ReindexStatusResult.yaml @@ -0,0 +1,28 @@ +value: + - entityType: 'instance' + status: 'Upload Completed' + totalMergeRanges: 3 + processedMergeRanges: 3 + totalUploadRanges: 2 + processedUploadRanges: 2 + startTimeMerge: '2024-04-01T01:37:34.15755006Z' + endTimeMerge: '2024-04-01T01:37:35.15755006Z' + startTimeUpload: '2024-04-01T01:37:36.15755006Z' + endTimeUpload: '2024-04-01T01:37:37.15755006Z' + - entityType: 'item' + status: 'Merge In Progress' + totalMergeRanges: 3 + processedMergeRanges: 2 + startTimeMerge: '2024-04-01T01:37:34.15755006Z' + - entityType: 'contributor' + status: 'Upload Completed' + totalUploadRanges: 3 + processedUploadRanges: 3 + startTimeUpload: '2024-04-01T01:37:34.15755006Z' + endTimeUpload: '2024-04-01T01:37:35.15755006Z' + - entityType: 'classification' + status: 'Upload Failed' + totalUploadRanges: 2 + processedUploadRanges: 1 + startTimeUpload: '2024-04-01T01:37:36.15755006Z' + endTimeUpload: '2024-04-01T01:37:37.15755006Z' diff --git a/src/main/resources/swagger.api/mod-search.yaml b/src/main/resources/swagger.api/mod-search.yaml index ab9fe6d71..603842ca4 100644 --- a/src/main/resources/swagger.api/mod-search.yaml +++ b/src/main/resources/swagger.api/mod-search.yaml @@ -112,6 +112,9 @@ paths: /search/index/inventory/reindex: $ref: 'paths/search-index/search-index-inventory-reindex.yaml' + /search/index/instance-records/reindex/status: + $ref: 'paths/search-index/search-index-reindex-status.yaml' + /search/config/languages: $ref: 'paths/search-config/search-config-languages.yaml' diff --git a/src/main/resources/swagger.api/paths/search-index/search-index-reindex-status.yaml b/src/main/resources/swagger.api/paths/search-index/search-index-reindex-status.yaml new file mode 100644 index 000000000..6ddcfbbf9 --- /dev/null +++ b/src/main/resources/swagger.api/paths/search-index/search-index-reindex-status.yaml @@ -0,0 +1,24 @@ +get: + operationId: getReindexStatus + summary: Get Reindex Status + description: Get a list of statuses for each resource reindexing + tags: + - index-management + parameters: + - $ref: '../../parameters/x-okapi-tenant-header.yaml' + responses: + '200': + description: 'Reindex statuses by entity type' + content: + application/json: + examples: + ReindexStatusResult: + $ref: '../../examples/result/ReindexStatusResult.yaml' + schema: + type: array + items: + $ref: '../../schemas/response/reindexStatusItem.yaml' + '400': + $ref: '../../responses/badRequestResponse.yaml' + '500': + $ref: '../../responses/internalServerErrorResponse.yaml' diff --git a/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml b/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml new file mode 100644 index 000000000..8b3f1c8ae --- /dev/null +++ b/src/main/resources/swagger.api/schemas/response/reindexStatusItem.yaml @@ -0,0 +1,33 @@ +description: Reindex status item +type: object +properties: + entityType: + type: string + description: Entity type for reindex status item + status: + type: string + description: Reindex status + totalMergeRanges: + type: integer + description: Total merge ranges to process for entity + processedMergeRanges: + type: integer + description: Processed merge ranges for entity + totalUploadRanges: + type: integer + description: Total upload ranges to process for entity + processedUploadRanges: + type: integer + description: Processed upload ranges for entity + startTimeMerge: + type: string + description: Start time of reindex merge phase for entity + endTimeMerge: + type: string + description: End time of reindex merge phase for entity + startTimeUpload: + type: string + description: Start time of reindex upload phase for entity + endTimeUpload: + type: string + description: End time of reindex upload phase for entity diff --git a/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java b/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java index 8c1fcdd51..05cd95493 100644 --- a/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java +++ b/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java @@ -1,6 +1,7 @@ package org.folio.search.controller; import static org.folio.search.support.base.ApiEndpoints.createIndicesPath; +import static org.folio.search.support.base.ApiEndpoints.reindexInstanceRecordsStatus; import static org.folio.search.utils.SearchResponseHelper.getSuccessFolioCreateIndexResponse; import static org.folio.search.utils.SearchResponseHelper.getSuccessIndexOperationResponse; import static org.folio.search.utils.TestUtils.OBJECT_MAPPER; @@ -12,6 +13,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.http.MediaType.APPLICATION_JSON; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -26,11 +28,14 @@ import org.folio.search.domain.dto.IndexDynamicSettings; import org.folio.search.domain.dto.ReindexJob; import org.folio.search.domain.dto.ReindexRequest; +import org.folio.search.domain.dto.ReindexStatusItem; import org.folio.search.domain.dto.UpdateIndexDynamicSettingsRequest; import org.folio.search.domain.dto.UpdateMappingsRequest; import org.folio.search.exception.SearchOperationException; +import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.IndexService; import org.folio.search.service.ResourceService; +import org.folio.search.service.reindex.ReindexRangeIndexService; import org.folio.spring.integration.XOkapiHeaders; import org.folio.spring.testing.type.UnitTest; import org.hibernate.validator.internal.engine.ConstraintViolationImpl; @@ -60,6 +65,8 @@ class IndexManagementControllerTest { private IndexService indexService; @MockBean private ResourceService resourceService; + @MockBean + private ReindexRangeIndexService reindexRangeService; @Test void createIndex_positive() throws Exception { @@ -247,6 +254,17 @@ void reindexInventoryRecords_negative_illegalArgumentException() throws Exceptio .andExpect(jsonPath("$.errors[0].code", is("validation_error"))); } + @Test + void getReindexStatus_positive() throws Exception { + var reindexStatus = new ReindexStatusItem().entityType(ReindexEntityType.INSTANCE.name()); + when(reindexRangeService.getReindexStatuses(TENANT_ID)).thenReturn(List.of(reindexStatus)); + + mockMvc.perform(get(reindexInstanceRecordsStatus()) + .header(XOkapiHeaders.TENANT, TENANT_ID)) + .andExpect(status().isOk()) + .andExpect(jsonPath("[0].entityType", is(reindexStatus.getEntityType()))); + } + private static MockHttpServletRequestBuilder preparePostRequest(String endpoint, String requestBody) { return post(endpoint) .content(requestBody) diff --git a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java index 649bbaae4..ee9a70ff9 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; +import java.util.UUID; import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.exception.ReindexException; @@ -15,6 +16,7 @@ import org.folio.search.model.index.SearchDocumentBody; import org.folio.search.model.types.IndexActionType; import org.folio.search.model.types.IndexingDataFormat; +import org.folio.search.model.types.ReindexEntityType; import org.folio.search.repository.PrimaryResourceRepository; import org.folio.search.service.converter.MultiTenantSearchDocumentConverter; import org.folio.spring.testing.type.UnitTest; @@ -41,10 +43,10 @@ class ReindexOrchestrationServiceTest { @Test void process_shouldProcessSuccessfully() { // Arrange - ReindexRangeIndexEvent event = new ReindexRangeIndexEvent(); - ResourceEvent resourceEvent = new ResourceEvent(); - FolioIndexOperationResponse folioIndexOperationResponse = - new FolioIndexOperationResponse().status(FolioIndexOperationResponse.StatusEnum.SUCCESS); + var event = reindexEvent(); + var resourceEvent = new ResourceEvent(); + var folioIndexOperationResponse = new FolioIndexOperationResponse() + .status(FolioIndexOperationResponse.StatusEnum.SUCCESS); when(rangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, @@ -59,14 +61,15 @@ void process_shouldProcessSuccessfully() { verify(rangeIndexService).fetchRecordRange(event); verify(documentConverter).convert(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); + verify(rangeIndexService).addProcessedUploadRanges(event.getEntityType(), 1); } @Test void process_shouldThrowReindexException_whenElasticSearchReportsError() { // Arrange - ReindexRangeIndexEvent event = new ReindexRangeIndexEvent(); - ResourceEvent resourceEvent = new ResourceEvent(); - FolioIndexOperationResponse folioIndexOperationResponse = new FolioIndexOperationResponse() + var event = reindexEvent(); + var resourceEvent = new ResourceEvent(); + var folioIndexOperationResponse = new FolioIndexOperationResponse() .status(FolioIndexOperationResponse.StatusEnum.ERROR) .errorMessage("Error occurred during indexing."); @@ -81,5 +84,13 @@ void process_shouldThrowReindexException_whenElasticSearchReportsError() { verify(rangeIndexService).fetchRecordRange(event); verify(documentConverter).convert(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); + verify(rangeIndexService).setReindexUploadFailed(event.getEntityType()); + } + + private ReindexRangeIndexEvent reindexEvent() { + var event = new ReindexRangeIndexEvent(); + event.setId(UUID.randomUUID()); + event.setEntityType(ReindexEntityType.INSTANCE); + return event; } } diff --git a/src/test/java/org/folio/search/service/reindex/ReindexRangeIndexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexRangeIndexServiceTest.java index 473cc6dde..b82fbd098 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexRangeIndexServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexRangeIndexServiceTest.java @@ -1,25 +1,38 @@ package org.folio.search.service.reindex; import static org.assertj.core.api.Assertions.assertThat; +import static org.folio.search.service.reindex.ReindexRangeIndexService.REQUEST_NOT_ALLOWED_MSG; import static org.folio.search.utils.SearchUtils.INSTANCE_RESOURCE; +import static org.folio.search.utils.TestConstants.CENTRAL_TENANT_ID; import static org.folio.search.utils.TestConstants.TENANT_ID; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import org.assertj.core.groups.Tuple; +import org.folio.search.converter.ReindexStatusMapper; +import org.folio.search.domain.dto.ReindexStatusItem; import org.folio.search.domain.dto.ResourceEvent; +import org.folio.search.exception.RequestValidationException; import org.folio.search.model.event.ReindexRangeIndexEvent; +import org.folio.search.model.reindex.ReindexStatusEntity; import org.folio.search.model.reindex.UploadRangeEntity; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.model.types.ReindexStatus; +import org.folio.search.service.consortium.ConsortiumTenantService; import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; +import org.folio.search.service.reindex.jdbc.ReindexStatusRepository; +import org.folio.spring.integration.XOkapiHeaders; import org.folio.spring.testing.extension.Random; import org.folio.spring.testing.extension.impl.RandomParametersExtension; import org.folio.spring.testing.type.UnitTest; import org.folio.spring.tools.kafka.FolioMessageProducer; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -33,12 +46,16 @@ class ReindexRangeIndexServiceTest { private @Mock ReindexJdbcRepository repository; private @Mock FolioMessageProducer indexRangeEventProducer; + private @Mock ReindexStatusRepository statusRepository; + private @Mock ReindexStatusMapper reindexStatusMapper; + private @Mock ConsortiumTenantService tenantService; private ReindexRangeIndexService service; @BeforeEach void setUp() { when(repository.entityType()).thenReturn(ReindexEntityType.INSTANCE); - service = new ReindexRangeIndexService(List.of(repository), indexRangeEventProducer); + service = new ReindexRangeIndexService(List.of(repository), indexRangeEventProducer, statusRepository, + reindexStatusMapper, tenantService); } @Test @@ -87,4 +104,59 @@ void fetchRecordRange_positive() { .extracting(ResourceEvent::getTenant, ResourceEvent::getNew, ResourceEvent::getResourceName) .containsExactly(Tuple.tuple(TENANT_ID, mockRecord, INSTANCE_RESOURCE)); } + + @Test + void getReindexStatuses() { + var statusEntities = List.of(new ReindexStatusEntity(ReindexEntityType.INSTANCE, ReindexStatus.MERGE_COMPLETED)); + var expected = List.of(new ReindexStatusItem()); + + when(statusRepository.getReindexStatuses()).thenReturn(statusEntities); + when(reindexStatusMapper.convert(statusEntities.get(0))).thenReturn(expected.get(0)); + + var actual = service.getReindexStatuses(TENANT_ID); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void getReindexStatuses_negative_consortiumMemberTenant() { + when(tenantService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of(CENTRAL_TENANT_ID)); + + var ex = Assertions.assertThrows(RequestValidationException.class, () -> service.getReindexStatuses(TENANT_ID)); + + assertThat(ex.getMessage()).isEqualTo(REQUEST_NOT_ALLOWED_MSG); + assertThat(ex.getKey()).isEqualTo(XOkapiHeaders.TENANT); + assertThat(ex.getValue()).isEqualTo(TENANT_ID); + verifyNoInteractions(statusRepository); + verifyNoInteractions(reindexStatusMapper); + } + + @Test + void setReindexUploadFailed() { + var entityType = ReindexEntityType.INSTANCE; + + service.setReindexUploadFailed(entityType); + + verify(statusRepository).setReindexUploadFailed(entityType); + } + + @Test + void addProcessedMergeRanges() { + var entityType = ReindexEntityType.INSTANCE; + var ranges = 5; + + service.addProcessedMergeRanges(entityType, ranges); + + verify(statusRepository).addReindexCounts(entityType, ranges, 0); + } + + @Test + void addProcessedUploadRanges() { + var entityType = ReindexEntityType.INSTANCE; + var ranges = 5; + + service.addProcessedUploadRanges(entityType, ranges); + + verify(statusRepository).addReindexCounts(entityType, 0, ranges); + } } diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java new file mode 100644 index 000000000..4917e5153 --- /dev/null +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexStatusRepositoryIT.java @@ -0,0 +1,123 @@ +package org.folio.search.service.reindex.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.folio.search.model.types.ReindexEntityType.CLASSIFICATION; +import static org.folio.search.model.types.ReindexEntityType.CONTRIBUTOR; +import static org.folio.search.model.types.ReindexEntityType.INSTANCE; +import static org.folio.search.model.types.ReindexEntityType.SUBJECT; +import static org.folio.search.model.types.ReindexStatus.MERGE_COMPLETED; +import static org.folio.search.model.types.ReindexStatus.UPLOAD_COMPLETED; +import static org.folio.search.model.types.ReindexStatus.UPLOAD_FAILED; +import static org.folio.search.model.types.ReindexStatus.UPLOAD_IN_PROGRESS; +import static org.folio.search.utils.TestConstants.TENANT_ID; +import static org.mockito.Mockito.when; + +import java.sql.Timestamp; +import org.assertj.core.api.Condition; +import org.folio.search.model.reindex.ReindexStatusEntity; +import org.folio.spring.FolioExecutionContext; +import org.folio.spring.FolioModuleMetadata; +import org.folio.spring.testing.extension.EnablePostgres; +import org.folio.spring.testing.type.IntegrationTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.jdbc.JdbcTest; +import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +@IntegrationTest +@JdbcTest +@EnablePostgres +@AutoConfigureJson +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +class ReindexStatusRepositoryIT { + + private @Autowired JdbcTemplate jdbcTemplate; + private @MockBean FolioExecutionContext context; + private ReindexStatusRepository repository; + + @BeforeEach + void setUp() { + repository = new ReindexStatusRepository(context, jdbcTemplate); + when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { + @Override + public String getModuleName() { + return null; + } + + @Override + public String getDBSchemaName(String tenantId) { + return "public"; + } + }); + when(context.getTenantId()).thenReturn(TENANT_ID); + } + + @Test + void getUploadRanges_returnEmptyList_whenNoUploadRangesAndNotPopulate() { + // act + var statuses = repository.getReindexStatuses(); + + // assert + assertThat(statuses).isEmpty(); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void getUploadRanges_returnList() { + // act + var statuses = repository.getReindexStatuses(); + + // assert + assertThat(statuses) + .hasSize(4) + .are(new Condition<>(status -> status.getTotalMergeRanges() == 3 + && "2024-04-01 01:37:34.15755".equals(status.getStartTimeMerge().toString()) + && "2024-04-01 01:37:35.15755".equals(status.getEndTimeMerge().toString()), "common properties match")) + .extracting(ReindexStatusEntity::getEntityType, ReindexStatusEntity::getStatus, + ReindexStatusEntity::getProcessedMergeRanges, ReindexStatusEntity::getTotalUploadRanges, + ReindexStatusEntity::getProcessedUploadRanges, ReindexStatusEntity::getStartTimeUpload, + ReindexStatusEntity::getEndTimeUpload) + .containsExactly(tuple(CONTRIBUTOR, MERGE_COMPLETED, 3, 0, 0, null, null), + tuple(SUBJECT, UPLOAD_IN_PROGRESS, 2, 2, 1, Timestamp.valueOf("2024-04-01 01:37:36.15755"), null), + tuple(INSTANCE, UPLOAD_COMPLETED, 3, 2, 2, Timestamp.valueOf("2024-04-01 01:37:36.15755"), + Timestamp.valueOf("2024-04-01 01:37:37.15755")), + tuple(CLASSIFICATION, UPLOAD_FAILED, 3, 2, 1, Timestamp.valueOf("2024-04-01 01:37:36.15755"), + Timestamp.valueOf("2024-04-01 01:37:37.15755"))); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void setReindexUploadFailed() { + // act + repository.setReindexUploadFailed(SUBJECT); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .hasSize(4) + .filteredOn(reindexStatus -> SUBJECT.equals(reindexStatus.getEntityType())) + .anyMatch(reindexStatus -> UPLOAD_FAILED.equals(reindexStatus.getStatus()) + && reindexStatus.getEndTimeUpload() != null); + } + + @Test + @Sql("/sql/populate-reindex-status.sql") + void addReindexCounts_shouldChangeStatus() { + // act + repository.addReindexCounts(SUBJECT, 0, 1); + + // assert + var statuses = repository.getReindexStatuses(); + assertThat(statuses) + .hasSize(4) + .filteredOn(reindexStatus -> SUBJECT.equals(reindexStatus.getEntityType())) + .anyMatch(reindexStatus -> UPLOAD_COMPLETED.equals(reindexStatus.getStatus()) + && reindexStatus.getEndTimeUpload() != null); + } +} diff --git a/src/test/java/org/folio/search/support/base/ApiEndpoints.java b/src/test/java/org/folio/search/support/base/ApiEndpoints.java index 72eea5f95..b87a61a76 100644 --- a/src/test/java/org/folio/search/support/base/ApiEndpoints.java +++ b/src/test/java/org/folio/search/support/base/ApiEndpoints.java @@ -174,6 +174,10 @@ public static String updateIndexSettingsPath() { return "/search/index/settings"; } + public static String reindexInstanceRecordsStatus() { + return "/search/index/instance-records/reindex/status"; + } + public static String allRecordsSortedBy(String sort, CqlSort order) { return String.format("cql.allRecords=1 sortBy %s/sort.%s", sort, order); } diff --git a/src/test/resources/sql/populate-reindex-status.sql b/src/test/resources/sql/populate-reindex-status.sql new file mode 100644 index 000000000..c8f0afe08 --- /dev/null +++ b/src/test/resources/sql/populate-reindex-status.sql @@ -0,0 +1,6 @@ +INSERT INTO reindex_status (entity_type, status, total_merge_ranges, processed_merge_ranges, total_upload_ranges, processed_upload_ranges, start_time_merge, end_time_merge, start_time_upload, end_time_upload) +VALUES + ('CONTRIBUTOR', 'MERGE_COMPLETED', '3', '3', '0', '0', '2024-04-01T01:37:34.15755006Z', '2024-04-01T01:37:35.15755006Z', null, null), + ('SUBJECT', 'UPLOAD_IN_PROGRESS', '3', '2', '2', '1', '2024-04-01T01:37:34.15755006Z', '2024-04-01T01:37:35.15755006Z', '2024-04-01T01:37:36.15755006Z', null), + ('INSTANCE', 'UPLOAD_COMPLETED', '3', '3', '2', '2', '2024-04-01T01:37:34.15755006Z', '2024-04-01T01:37:35.15755006Z', '2024-04-01T01:37:36.15755006Z', '2024-04-01T01:37:37.15755006Z'), + ('CLASSIFICATION', 'UPLOAD_FAILED', '3', '3', '2', '1', '2024-04-01T01:37:34.15755006Z', '2024-04-01T01:37:35.15755006Z', '2024-04-01T01:37:36.15755006Z', '2024-04-01T01:37:37.15755006Z');