From 0aaaa451e02e63c748ce0f8910aed62b4d4873dd Mon Sep 17 00:00:00 2001 From: vburlachenko Date: Mon, 6 May 2024 19:35:01 +0300 Subject: [PATCH] 1623 - added additional filterState last_runs_statuses, created API for generating SearchFacetsData --- .../controller/DataQualityRunsController.java | 13 ++ .../dto/DataEntityTaskRunStatusDto.java | 36 +++++ .../oddplatform/dto/FacetType.java | 1 + .../oddplatform/mapper/FacetStateMapper.java | 6 + .../mapper/FacetStateMapperImpl.java | 53 ++++++- ...ReactiveDataQualityRunsRepositoryImpl.java | 59 +++++--- .../ReactiveSearchFacetRepository.java | 5 + .../ReactiveSearchFacetRepositoryImpl.java | 132 +++++++++++++----- .../repository/util/FTSConstants.java | 21 ++- .../service/DataQualityRunsService.java | 8 ++ .../service/DataQualityRunsServiceImpl.java | 52 +++++-- .../service/search/SearchServiceImpl.java | 2 + odd-platform-specification/components.yaml | 16 +++ odd-platform-specification/openapi.yaml | 57 ++++++++ 14 files changed, 393 insertions(+), 68 deletions(-) create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTaskRunStatusDto.java diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataQualityRunsController.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataQualityRunsController.java index dbcb2f077..aa1a93e38 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataQualityRunsController.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DataQualityRunsController.java @@ -3,7 +3,9 @@ import java.util.List; import lombok.RequiredArgsConstructor; import org.opendatadiscovery.oddplatform.api.contract.api.DataQualityRunsApi; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRunStatus; import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityResults; +import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData; import org.opendatadiscovery.oddplatform.service.DataQualityRunsService; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; @@ -31,4 +33,15 @@ public Mono> getDataQualityTestsRuns(final Li deNamespaceIds, deDatasourceIds, deOwnerIds, deTitleIds, deTagIds) .map(ResponseEntity::ok); } + + @Override + public Mono> createDataQualityLatestRunsSearch(final List namespaceIds, + final List datasourceIds, + final List ownerIds, + final List tagIds, + final DataEntityRunStatus status, + final ServerWebExchange exchange) { + return service.createDataQualityLatestRunsSearch(namespaceIds, datasourceIds, ownerIds, tagIds, status) + .map(ResponseEntity::ok); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTaskRunStatusDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTaskRunStatusDto.java new file mode 100644 index 000000000..6f9c73c4a --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/DataEntityTaskRunStatusDto.java @@ -0,0 +1,36 @@ +package org.opendatadiscovery.oddplatform.dto; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import lombok.Getter; + +import static java.util.function.Function.identity; + +@Getter +public enum DataEntityTaskRunStatusDto { + SUCCESS(1, "SUCCESS"), + FAILED(2, "FAILED"), + SKIPPED(3, "SKIPPED"), + BROKEN(4, "BROKEN"), + ABORTED(5, "ABORTED"), + UNKNOWN(6, "UNKNOWN"); + + private final short id; + private final String status; + + DataEntityTaskRunStatusDto(final int id, + final String status) { + this.id = (short) id; + this.status = status; + } + + private static final Map MAP = Arrays + .stream(DataEntityTaskRunStatusDto.values()) + .collect(Collectors.toMap(DataEntityTaskRunStatusDto::getStatus, identity())); + + public static Optional findByStatus(final String status) { + return Optional.ofNullable(MAP.get(status)); + } +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java index 527ba3a02..b87ffe39f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/FacetType.java @@ -9,6 +9,7 @@ public enum FacetType { TAGS, GROUPS, STATUSES, + LAST_RUN_STATUSES, DATA_ENTITY; public static FacetType lookup(final String facetType) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapper.java index 749574090..cae1b2ec1 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapper.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapper.java @@ -30,4 +30,10 @@ public interface FacetStateMapper { TermFacetState mapDto(final FacetStateDto state); FacetState mapDto(final List entityClasses, final FacetStateDto state); + + SearchFormData mapToFormData(final List namespaceIds, + final List datasourceIds, + final List ownerIds, + final List tagIds, + final List entityClasses); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java index 846de2b0c..259cf4b3c 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/FacetStateMapperImpl.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import java.time.OffsetDateTime; import java.time.ZoneOffset; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -10,6 +11,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jooq.JSONB; import org.opendatadiscovery.oddplatform.api.contract.model.CountableSearchFilter; @@ -23,6 +25,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.TermFacetState; import org.opendatadiscovery.oddplatform.api.contract.model.TermSearchFormData; import org.opendatadiscovery.oddplatform.api.contract.model.TermSearchFormDataFilters; +import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; import org.opendatadiscovery.oddplatform.dto.FacetStateDto; import org.opendatadiscovery.oddplatform.dto.FacetType; import org.opendatadiscovery.oddplatform.dto.SearchFilterDto; @@ -45,7 +48,8 @@ public class FacetStateMapperImpl implements FacetStateMapper { SearchFormDataFilters::getOwners, FacetType.OWNERS, SearchFormDataFilters::getTags, FacetType.TAGS, SearchFormDataFilters::getGroups, FacetType.GROUPS, - SearchFormDataFilters::getStatuses, FacetType.STATUSES + SearchFormDataFilters::getStatuses, FacetType.STATUSES, + SearchFormDataFilters::getLastRunStatuses, FacetType.LAST_RUN_STATUSES ); private static final Map>, FacetType> @@ -170,7 +174,52 @@ public FacetState mapDto(final List entityClasses, final .owners(getSearchFiltersForFacetType(state, FacetType.OWNERS)) .namespaces(getSearchFiltersForFacetType(state, FacetType.NAMESPACES)) .tags(getSearchFiltersForFacetType(state, FacetType.TAGS)) - .groups(getSearchFiltersForFacetType(state, FacetType.GROUPS)); + .groups(getSearchFiltersForFacetType(state, FacetType.GROUPS)) + .statuses(getSearchFiltersForFacetType(state, FacetType.STATUSES)) + .lastRunStatuses(getSearchFiltersForFacetType(state, FacetType.LAST_RUN_STATUSES)); + } + + @Override + public SearchFormData mapToFormData(final List namespaceIds, + final List datasourceIds, + final List ownerIds, + final List tagIds, + final List entityClasses) { + final SearchFormDataFilters filters = new SearchFormDataFilters() + .namespaces(getFilterStateList(namespaceIds, FacetType.NAMESPACES)) + .datasources(getFilterStateList(datasourceIds, FacetType.DATA_SOURCES)) + .owners(getFilterStateList(ownerIds, FacetType.OWNERS)) + .tags(getFilterStateList(tagIds, FacetType.TAGS)) + .entityClasses(getFilterStateListForEntityClasses(entityClasses)); + + return new SearchFormData().filters(filters).query(""); + } + + private List getFilterStateList(final List filterIds, final FacetType filterType) { + if (CollectionUtils.isEmpty(filterIds)) { + return Collections.emptyList(); + } + + return filterIds.stream() + .map(id -> new SearchFilterState() + .entityId(id) + .entityName(filterType.name()) + .selected(true)) + .collect(Collectors.toList()); + } + + private List getFilterStateListForEntityClasses(final List filterIds) { + if (CollectionUtils.isEmpty(filterIds)) { + return Collections.emptyList(); + } + + return filterIds.stream() + .map(id -> new SearchFilterState() + .entityId(Long.valueOf(id)) + .entityName(DataEntityClassDto.findById(id).orElseThrow(() -> new IllegalArgumentException( + "Unknown data entity type id: %d".formatted(id))).name()) + .selected(true)) + .collect(Collectors.toList()); } private SearchFilterDto mapFilter(final SearchFilterState f, final FacetType type) { diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRunsRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRunsRepositoryImpl.java index 5dc336b9b..605e7e698 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRunsRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveDataQualityRunsRepositoryImpl.java @@ -46,6 +46,7 @@ public class ReactiveDataQualityRunsRepositoryImpl implements ReactiveDataQualit public static final String DATA_QUALITY_TEST_TYPE = "specific_attributes->'DATA_QUALITY_TEST'->'expectation'->>'category'"; public static final String ID = "id"; + public static final String DATA_ENTITY_ID = "data_entity_id"; public static final String ODDRN = "oddrn"; public static final String CATEGORY = "category"; public static final String CATEGORIES_CTE = "categories_cte"; @@ -63,30 +64,9 @@ public class ReactiveDataQualityRunsRepositoryImpl implements ReactiveDataQualit @Override public Flux getLatestDataQualityRunsResults(final DataQualityTestFiltersDto filtersDto) { - final Table> testFilters = generateTestFiltersCte(filtersDto); - final Select> deCategoryCTE = - DSL.select(DATA_ENTITY.ID.as(ID), DATA_ENTITY.ODDRN.as(ODDRN), - field(DATA_QUALITY_TEST_TYPE, String.class).as(CATEGORY)) - .from(DATA_ENTITY, testFilters) - .where(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.JOB.getId()) - .and(field(DATA_QUALITY_TEST_TYPE, String.class).isNotNull())) - .and(testFilters.field(DATA_ENTITY.ID).eq(DATA_ENTITY.ID)); - - final Table> categoriesSubTable = deCategoryCTE.asTable(CATEGORIES_CTE); + final Table> categoriesSubTable = getCTEForLastRuns(filtersDto); final List> fromList = new ArrayList<>(List.of(categoriesSubTable, DATA_ENTITY_TASK_LAST_RUN)); - final List conditionList - = new ArrayList<>(List.of(categoriesSubTable.field(ODDRN, String.class) - .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN))); - - if (shouldAddFiltersForDataEntity(filtersDto)) { - final Table> deFilters = generateDataEntityFiltersCte(filtersDto); - - fromList.addAll(List.of(deFilters, DATA_QUALITY_TEST_RELATIONS)); - - conditionList.add(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN - .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)); - conditionList.add(deFilters.field(DATA_ENTITY.ODDRN).eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN)); - } + final List conditionList = getConditionsForLastRuns(filtersDto, fromList, categoriesSubTable); final SelectSeekStep1, String> query = DSL.select( @@ -195,6 +175,39 @@ public Flux getMonitoredTables(final DataQualityTestFilte .map(item -> item.into(MonitoredtablesRecord.class)); } + private Table> getCTEForLastRuns(final DataQualityTestFiltersDto filtersDto) { + final Table> testFilters = generateTestFiltersCte(filtersDto); + final Select> deCategoryCTE = + DSL.select(DATA_ENTITY.ID.as(ID), DATA_ENTITY.ODDRN.as(ODDRN), + field(DATA_QUALITY_TEST_TYPE, String.class).as(CATEGORY)) + .from(DATA_ENTITY, testFilters) + .where(DATA_ENTITY.TYPE_ID.eq(DataEntityTypeDto.JOB.getId()) + .and(field(DATA_QUALITY_TEST_TYPE, String.class).isNotNull())) + .and(testFilters.field(DATA_ENTITY.ID).eq(DATA_ENTITY.ID)); + + return deCategoryCTE.asTable(CATEGORIES_CTE); + } + + private List getConditionsForLastRuns(final DataQualityTestFiltersDto filtersDto, + final List> fromList, + final Table> categoriesSubTable) { + final List conditionList + = new ArrayList<>(List.of(categoriesSubTable.field(ODDRN, String.class) + .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN))); + + if (shouldAddFiltersForDataEntity(filtersDto)) { + final Table> deFilters = generateDataEntityFiltersCte(filtersDto); + + fromList.addAll(List.of(deFilters, DATA_QUALITY_TEST_RELATIONS)); + + conditionList.add(DATA_QUALITY_TEST_RELATIONS.DATA_QUALITY_TEST_ODDRN + .eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)); + conditionList.add(deFilters.field(DATA_ENTITY.ODDRN).eq(DATA_QUALITY_TEST_RELATIONS.DATASET_ODDRN)); + } + + return conditionList; + } + private SelectConditionStep getDataQualityQuery(final Table> dataEntityCTE, final DataQualityTestFiltersDto filtersDto) { final Table> testFilters = generateTestFiltersCte(filtersDto); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java index e434e9552..5ce651d1f 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepository.java @@ -51,4 +51,9 @@ Mono> getStatusFacetForDataEntity(final String query, final int page, final int size, final FacetStateDto state); + + Mono> getLastRunStatusesFacetForDataEntity(final String query, + final int page, + final int size, + final FacetStateDto state); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java index 91ae3a2da..a1a56e73d 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveSearchFacetRepositoryImpl.java @@ -15,12 +15,15 @@ import org.jooq.Field; import org.jooq.Record; import org.jooq.Record1; +import org.jooq.Record2; import org.jooq.Record3; +import org.jooq.SelectJoinStep; import org.jooq.SelectOrderByStep; import org.jooq.Table; import org.jooq.impl.DSL; import org.opendatadiscovery.oddplatform.dto.DataEntityClassDto; import org.opendatadiscovery.oddplatform.dto.DataEntityStatusDto; +import org.opendatadiscovery.oddplatform.dto.DataEntityTaskRunStatusDto; import org.opendatadiscovery.oddplatform.dto.DataEntityTypeDto; import org.opendatadiscovery.oddplatform.dto.FacetStateDto; import org.opendatadiscovery.oddplatform.dto.FacetType; @@ -44,6 +47,7 @@ import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_STRUCTURE; import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_VERSION; import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_TASK_LAST_RUN; import static org.opendatadiscovery.oddplatform.model.Tables.DATA_SOURCE; import static org.opendatadiscovery.oddplatform.model.Tables.GROUP_ENTITY_RELATIONS; import static org.opendatadiscovery.oddplatform.model.Tables.NAMESPACE; @@ -464,8 +468,91 @@ public Mono> getStatusFacetForDataEntity(final String .select(DATA_ENTITY.STATUS, count(DATA_ENTITY.ID)) .from(DATA_ENTITY); + select = addAdditionalJoinsByState(state, conditions, select); + + final List statusIds = statusIdsByName(query); + if (!statusIds.isEmpty()) { + conditions.add(DATA_ENTITY.STATUS.in(statusIds)); + } + + select + .where(conditions) + .groupBy(DATA_ENTITY.STATUS) + .orderBy(count(DATA_ENTITY.ID).desc()) + .limit(size) + .offset((page - 1) * size); + + final Flux> existingStatuses = jooqReactiveOperations.flux(select) + .map(r -> { + final DataEntityStatusDto status = DataEntityStatusDto.findById(r.component1()) + .orElseThrow(() -> new IllegalArgumentException( + String.format("There's no status with id %d", r.component1()))); + + return Pair.of(statusToSearchFilter(status), r.component2().longValue()); + }); + + final Flux> all = Flux.fromStream(Arrays.stream(DataEntityStatusDto.values())) + .map(s -> Pair.of(statusToSearchFilter(s), 0L)); + + return Flux.concat(existingStatuses, all) + .filter(s -> StringUtils.isEmpty(query) + || StringUtils.containsIgnoreCase(query, s.getLeft().getName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (t1, t2) -> t1 == 0 ? t2 : t1)); + } + + @Override + public Mono> getLastRunStatusesFacetForDataEntity(final String query, + final int page, + final int size, + final FacetStateDto state) { + final List conditions = new ArrayList<>(); + conditions.add(DATA_ENTITY.HOLLOW.isFalse()); + conditions.add(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isNull().or(DATA_ENTITY.EXCLUDE_FROM_SEARCH.isFalse())); + conditions.addAll(getQueryAndEntityClassConditions(state)); + + SelectJoinStep> select = + select(DATA_ENTITY_TASK_LAST_RUN.STATUS, count(DATA_ENTITY_TASK_LAST_RUN.LAST_TASK_RUN_ODDRN)) + .from(DATA_ENTITY_TASK_LAST_RUN) + .join(DATA_ENTITY).on(DATA_ENTITY.ODDRN.eq(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN)); + + select = addAdditionalJoinsByState(state, conditions, select); + + if (StringUtils.isNotBlank(query)) { + conditions.add(DATA_ENTITY_TASK_LAST_RUN.STATUS.containsIgnoreCase(query)); + } + + select + .where(conditions) + .groupBy(DATA_ENTITY_TASK_LAST_RUN.STATUS) + .orderBy(count(DATA_ENTITY_TASK_LAST_RUN.LAST_TASK_RUN_ODDRN).desc()) + .limit(size) + .offset((page - 1) * size); + + final Flux> existingStatuses = jooqReactiveOperations.flux(select) + .map(r -> { + final DataEntityTaskRunStatusDto status = DataEntityTaskRunStatusDto.findByStatus(r.component1()) + .orElseThrow(() -> new IllegalArgumentException( + String.format("There's no last run status with name %s", r.component1()))); + + return Pair.of(lastRunStatusToSearchFilter(status), r.component2().longValue()); + }); + + final Flux> all = Flux.fromStream(Arrays.stream(DataEntityTaskRunStatusDto.values())) + .map(s -> Pair.of(lastRunStatusToSearchFilter(s), 0L)); + + return Flux.concat(existingStatuses, all) + .filter(s -> StringUtils.isEmpty(query) + || StringUtils.containsIgnoreCase(query, s.getLeft().getName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (t1, t2) -> t1 == 0 ? t2 : t1)); + } + + private SelectJoinStep addAdditionalJoinsByState(final FacetStateDto state, + final List conditions, + final SelectJoinStep select) { +// Checkstyle forced me to do that + SelectJoinStep resultSelect = select; if (StringUtils.isNotEmpty(state.getQuery())) { - select.join(SEARCH_ENTRYPOINT).on(SEARCH_ENTRYPOINT.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); + resultSelect.join(SEARCH_ENTRYPOINT).on(SEARCH_ENTRYPOINT.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); conditions.add(jooqFTSHelper.ftsCondition(SEARCH_ENTRYPOINT.SEARCH_VECTOR, state.getQuery())); } @@ -475,12 +562,12 @@ public Mono> getStatusFacetForDataEntity(final String } final Set ownerIds = state.getFacetEntitiesIds(FacetType.OWNERS); if (!CollectionUtils.isEmpty(ownerIds)) { - select.join(OWNERSHIP).on(OWNERSHIP.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); + resultSelect.join(OWNERSHIP).on(OWNERSHIP.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); conditions.add(OWNERSHIP.OWNER_ID.in(ownerIds)); } final Set namespaceIds = state.getFacetEntitiesIds(FacetType.NAMESPACES); if (!CollectionUtils.isEmpty(namespaceIds)) { - select.leftJoin(DATA_SOURCE) + resultSelect.leftJoin(DATA_SOURCE) .on(DATA_SOURCE.ID.eq(DATA_ENTITY.DATA_SOURCE_ID)) .leftJoin(NAMESPACE).on(NAMESPACE.ID.eq(DATA_ENTITY.NAMESPACE_ID)) .or(NAMESPACE.ID.eq(DATA_SOURCE.NAMESPACE_ID)); @@ -488,14 +575,14 @@ public Mono> getStatusFacetForDataEntity(final String } final Set tagIds = state.getFacetEntitiesIds(FacetType.TAGS); if (!CollectionUtils.isEmpty(tagIds)) { - select = select.join(TAG_TO_DATA_ENTITY) + resultSelect = resultSelect.join(TAG_TO_DATA_ENTITY) .on(TAG_TO_DATA_ENTITY.DATA_ENTITY_ID.eq(DATA_ENTITY.ID)); conditions.add(TAG_TO_DATA_ENTITY.TAG_ID.in(tagIds)); } final Set groupIds = state.getFacetEntitiesIds(FacetType.GROUPS); if (!CollectionUtils.isEmpty(groupIds)) { - select = select.join(GROUP_ENTITY_RELATIONS) + resultSelect = resultSelect.join(GROUP_ENTITY_RELATIONS) .on(GROUP_ENTITY_RELATIONS.DATA_ENTITY_ODDRN.eq(DATA_ENTITY.ODDRN)); final var groupOddrns = DSL.select(DATA_ENTITY.ODDRN) @@ -503,34 +590,8 @@ public Mono> getStatusFacetForDataEntity(final String .where(DATA_ENTITY.ID.in(groupIds)); conditions.add(GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(groupOddrns)); } - final List statusIds = statusIdsByName(query); - if (!statusIds.isEmpty()) { - conditions.add(DATA_ENTITY.STATUS.in(statusIds)); - } - select - .where(conditions) - .groupBy(DATA_ENTITY.STATUS) - .orderBy(count(DATA_ENTITY.ID).desc()) - .limit(size) - .offset((page - 1) * size); - - final Flux> existingStatuses = jooqReactiveOperations.flux(select) - .map(r -> { - final DataEntityStatusDto status = DataEntityStatusDto.findById(r.component1()) - .orElseThrow(() -> new IllegalArgumentException( - String.format("There's no status with id %d", r.component1()))); - - return Pair.of(statusToSearchFilter(status), r.component2().longValue()); - }); - - final Flux> all = Flux.fromStream(Arrays.stream(DataEntityStatusDto.values())) - .map(s -> Pair.of(statusToSearchFilter(s), 0L)); - - return Flux.concat(existingStatuses, all) - .filter(s -> StringUtils.isEmpty(query) - || StringUtils.containsIgnoreCase(query, s.getLeft().getName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (t1, t2) -> t1 == 0 ? t2 : t1)); + return resultSelect; } private List typeIdsByName(final String name) { @@ -568,6 +629,13 @@ private SearchFilterId statusToSearchFilter(final DataEntityStatusDto status) { .build(); } + private SearchFilterId lastRunStatusToSearchFilter(final DataEntityTaskRunStatusDto lastRunStatus) { + return SearchFilterId.builder() + .entityId(lastRunStatus.getId()) + .name(lastRunStatus.name()) + .build(); + } + private List getDataEntityDefaultConditions() { final List conditions = new ArrayList<>(); conditions.add(DATA_ENTITY.HOLLOW.isFalse()); diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java index 3b98a5202..7f558eac4 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/util/FTSConstants.java @@ -16,6 +16,7 @@ import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_STRUCTURE; import static org.opendatadiscovery.oddplatform.model.Tables.DATASET_VERSION; import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY; +import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY_TASK_LAST_RUN; import static org.opendatadiscovery.oddplatform.model.Tables.DATA_SOURCE; import static org.opendatadiscovery.oddplatform.model.Tables.GROUP_ENTITY_RELATIONS; import static org.opendatadiscovery.oddplatform.model.Tables.LOOKUP_TABLES; @@ -78,8 +79,8 @@ public class FTSConstants { public static final Map, Condition>> DATA_ENTITY_CONDITIONS = Map.of( - FacetType.ENTITY_CLASSES, filters -> DATA_ENTITY.ENTITY_CLASS_IDS - .contains(extractFilterId(filters).stream().map(Long::intValue).toArray(Integer[]::new)), + FacetType.ENTITY_CLASSES, filters -> DSL.arrayOverlap(DATA_ENTITY.ENTITY_CLASS_IDS, + extractFilterId(filters).stream().map(Long::intValue).toArray(Integer[]::new)), FacetType.DATA_SOURCES, filters -> DATA_ENTITY.DATA_SOURCE_ID.in(extractFilterId(filters)), FacetType.NAMESPACES, filters -> NAMESPACE.ID.in(extractFilterId(filters)), FacetType.TYPES, filters -> DATA_ENTITY.TYPE_ID.in(extractFilterId(filters)), @@ -114,7 +115,15 @@ public class FTSConstants { .where(DATA_ENTITY.ID.in(extractFilterId(filters))); return GROUP_ENTITY_RELATIONS.GROUP_ODDRN.in(groupOddrns); }, - FacetType.STATUSES, filters -> DATA_ENTITY.STATUS.in(extractFilterId(filters)) + FacetType.STATUSES, filters -> DATA_ENTITY.STATUS.in(extractFilterId(filters)), + FacetType.LAST_RUN_STATUSES, filters -> { + final var dataEntities = select(DATA_ENTITY.ID) + .from(DATA_ENTITY_TASK_LAST_RUN, DATA_ENTITY) + .where(DATA_ENTITY_TASK_LAST_RUN.TASK_ODDRN.eq(DATA_ENTITY.ODDRN)) + .and(DATA_ENTITY_TASK_LAST_RUN.STATUS.in(extractFilterValue(filters))); + + return DATA_ENTITY.ID.in(dataEntities); + } ); public static final Map, Condition>> TERM_CONDITIONS = Map.of( @@ -131,4 +140,10 @@ private static List extractFilterId(final List filters) { .map(SearchFilterDto::getEntityId) .collect(Collectors.toList()); } + + private static List extractFilterValue(final List filters) { + return filters.stream() + .map(SearchFilterDto::getEntityName) + .collect(Collectors.toList()); + } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsService.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsService.java index 21357bc89..e14bb3d22 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsService.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsService.java @@ -1,7 +1,9 @@ package org.opendatadiscovery.oddplatform.service; import java.util.List; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRunStatus; import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityResults; +import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData; import reactor.core.publisher.Mono; public interface DataQualityRunsService { @@ -10,4 +12,10 @@ Mono getDataQualityTestsRuns(final List namespaceIds, final List tagIds, final List deNamespaceIds, final List deDatasourceIds, final List deOwnerIds, final List deTitleIds, final List deTagIds); + + Mono createDataQualityLatestRunsSearch(final List namespaceIds, + final List datasourceIds, + final List ownerIds, + final List tagIds, + final DataEntityRunStatus status); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsServiceImpl.java index 51d565c16..1eb2e5ea3 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/DataQualityRunsServiceImpl.java @@ -2,21 +2,32 @@ import java.util.List; import lombok.RequiredArgsConstructor; +import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRunStatus; import org.opendatadiscovery.oddplatform.api.contract.model.DataQualityResults; +import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData; +import org.opendatadiscovery.oddplatform.api.contract.model.SearchFilterState; +import org.opendatadiscovery.oddplatform.api.contract.model.SearchFormData; +import org.opendatadiscovery.oddplatform.dto.DataEntityTaskRunStatusDto; import org.opendatadiscovery.oddplatform.dto.DataQualityTestFiltersDto; import org.opendatadiscovery.oddplatform.mapper.DataQualityCategoryMapper; import org.opendatadiscovery.oddplatform.mapper.DataQualityTestFiltersMapper; +import org.opendatadiscovery.oddplatform.mapper.FacetStateMapper; import org.opendatadiscovery.oddplatform.mapper.TablesDashboardMapper; import org.opendatadiscovery.oddplatform.repository.reactive.ReactiveDataQualityRunsRepository; +import org.opendatadiscovery.oddplatform.service.search.SearchService; import org.springframework.stereotype.Service; import reactor.core.publisher.Mono; +import static org.opendatadiscovery.oddplatform.dto.DataEntityClassDto.DATA_QUALITY_TEST; + @Service @RequiredArgsConstructor public class DataQualityRunsServiceImpl implements DataQualityRunsService { + private final SearchService searchService; private final DataQualityCategoryMapper testsMapper; private final TablesDashboardMapper tablesDashboardMapper; private final DataQualityTestFiltersMapper dataQualityTestFiltersMapper; + private final FacetStateMapper facetStateMapper; private final ReactiveDataQualityRunsRepository dataQualityRunsRepository; @Override @@ -31,14 +42,39 @@ public Mono getDataQualityTestsRuns(final List namespa final List deTitleIds, final List deTagIds) { final DataQualityTestFiltersDto filtersDto - = dataQualityTestFiltersMapper.mapToDto(namespaceIds, datasourceIds, ownerIds, titleIds, tagIds, - deNamespaceIds, deDatasourceIds, deOwnerIds, deTitleIds, deTagIds); + = dataQualityTestFiltersMapper.mapToDto(namespaceIds, datasourceIds, ownerIds, titleIds, tagIds, + deNamespaceIds, deDatasourceIds, deOwnerIds, deTitleIds, deTagIds); return dataQualityRunsRepository.getLatestDataQualityRunsResults(filtersDto) - .collectList() - .zipWith(dataQualityRunsRepository.getLatestTablesHealth(filtersDto).collectList() - .zipWith(dataQualityRunsRepository.getMonitoredTables(filtersDto).collectList())) - .map(item -> new DataQualityResults() - .testResults(testsMapper.mapToDto(item.getT1())) - .tablesDashboard(tablesDashboardMapper.mapToDto(item.getT2().getT1(), item.getT2().getT2()))); + .collectList() + .zipWith(dataQualityRunsRepository.getLatestTablesHealth(filtersDto).collectList() + .zipWith(dataQualityRunsRepository.getMonitoredTables(filtersDto).collectList())) + .map(item -> new DataQualityResults() + .testResults(testsMapper.mapToDto(item.getT1())) + .tablesDashboard(tablesDashboardMapper.mapToDto(item.getT2().getT1(), item.getT2().getT2()))); + } + + @Override + public Mono createDataQualityLatestRunsSearch(final List namespaceIds, + final List datasourceIds, + final List ownerIds, + final List tagIds, + final DataEntityRunStatus status) { + final SearchFormData searchFormData = + facetStateMapper.mapToFormData(namespaceIds, datasourceIds, ownerIds, tagIds, + List.of(DATA_QUALITY_TEST.getId())); + + if (status != null) { + final DataEntityTaskRunStatusDto statusDto = + DataEntityTaskRunStatusDto.findByStatus(status.getValue()) + .orElseThrow(() -> new IllegalArgumentException( + String.format("Status %s was not founded in the system", status.getValue()))); + + searchFormData.getFilters().addLastRunStatusesItem(new SearchFilterState() + .entityId((long) statusDto.getId()) + .entityName(statusDto.getStatus()) + .selected(true)); + } + + return searchService.search(searchFormData); } } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java index a9d5ac483..041a425a5 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/service/search/SearchServiceImpl.java @@ -171,6 +171,8 @@ private Function>> getFacetFetchOp case TYPES -> s -> searchFacetRepository.getTypeFacetForDataEntity(query, page, size, s); case GROUPS -> s -> searchFacetRepository.getGroupFacetForDataEntity(query, page, size, s); case STATUSES -> s -> searchFacetRepository.getStatusFacetForDataEntity(query, page, size, s); + case LAST_RUN_STATUSES -> s -> + searchFacetRepository.getLastRunStatusesFacetForDataEntity(query, page, size, s); }; } diff --git a/odd-platform-specification/components.yaml b/odd-platform-specification/components.yaml index decd7940b..bafc7b755 100644 --- a/odd-platform-specification/components.yaml +++ b/odd-platform-specification/components.yaml @@ -1422,6 +1422,7 @@ components: - types - groups - statuses + - last_run_statuses SearchFilter: type: object @@ -1483,6 +1484,10 @@ components: type: array items: $ref: '#/components/schemas/SearchFilter' + last_run_statuses: + type: array + items: + $ref: '#/components/schemas/SearchFilter' required: - entity_classes @@ -2283,6 +2288,10 @@ components: type: array items: $ref: '#/components/schemas/SearchFilterState' + last_run_statuses: + type: array + items: + $ref: '#/components/schemas/SearchFilterState' required: - filters @@ -4394,6 +4403,13 @@ components: schema: $ref: '#/components/schemas/RelationshipsType' + DataQualityLastRunStatusParam: + name: status + in: query + required: false + schema: + $ref: '#/components/schemas/DataEntityRunStatus' + responses: Deleted: description: The resource has been successfully deleted diff --git a/odd-platform-specification/openapi.yaml b/odd-platform-specification/openapi.yaml index 60e90c010..7f5cec5ec 100644 --- a/odd-platform-specification/openapi.yaml +++ b/odd-platform-specification/openapi.yaml @@ -2086,6 +2086,63 @@ paths: tags: - dataQualityRuns + /api/dataqatests/latestruns/search: + post: + summary: Get Data Quality latest runs search uuid + description: Get Data Quality latest runs search uuid + operationId: createDataQualityLatestRunsSearch + parameters: + - name: namespaceIds + in: query + required: false + schema: + type: array + items: + type: integer + format: int64 + style: form + explode: true + - name: datasourceIds + in: query + required: false + schema: + type: array + items: + type: integer + format: int64 + style: form + explode: true + - name: ownerIds + in: query + required: false + schema: + type: array + items: + type: integer + format: int64 + style: form + explode: true + - name: tagIds + in: query + required: false + schema: + type: array + items: + type: integer + format: int64 + style: form + explode: true + - $ref: './components.yaml/#/components/parameters/DataQualityLastRunStatusParam' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: './components.yaml/#/components/schemas/SearchFacetsData' + tags: + - dataQualityRuns + /api/queryexample: get: summary: List of QueryExamples