From b23b1a49853d39049051c7f77846fa4b35caf79e Mon Sep 17 00:00:00 2001 From: vburlachenko Date: Tue, 23 Jan 2024 13:51:53 +0200 Subject: [PATCH] 1576 - implementation of ingest relationships API and GET Relationships API --- .../oddplatform/dto/RelationshipDto.java | 3 +- .../oddplatform/dto/RelationshipTypeDto.java | 6 ++++ .../mapper/RelationshipMapper.java | 5 ++- .../ReactiveRelationshipsRepositoryImpl.java | 36 +++++-------------- .../RelationshipsIngestionServiceImpl.java | 10 ++++-- .../V0_0_87__create_relation_tables.sql | 1 + 6 files changed, 27 insertions(+), 34 deletions(-) create mode 100644 odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipTypeDto.java diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java index 09904904a..27450ae2c 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java @@ -9,6 +9,5 @@ public record RelationshipDto(RelationshipsPojo relationshipPojo, DataEntityPojo sourceDataEntity, DataEntityPojo targetDataEntity, ErdRelationshipDetailsPojo erdRelationshipDetailsPojo, - GraphRelationshipPojo graphRelationshipPojo, - boolean isErd) { + GraphRelationshipPojo graphRelationshipPojo) { } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipTypeDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipTypeDto.java new file mode 100644 index 000000000..f6b823830 --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipTypeDto.java @@ -0,0 +1,6 @@ +package org.opendatadiscovery.oddplatform.dto; + +public enum RelationshipTypeDto { + ERD, + GRAPH +} diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipMapper.java index 8435d9040..28193bfc1 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipMapper.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipMapper.java @@ -8,6 +8,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DatasetRelationshipType; import org.opendatadiscovery.oddplatform.api.contract.model.PageInfo; import org.opendatadiscovery.oddplatform.dto.RelationshipDto; +import org.opendatadiscovery.oddplatform.dto.RelationshipTypeDto; import org.opendatadiscovery.oddplatform.utils.Page; import org.springframework.stereotype.Component; @@ -38,7 +39,9 @@ public DatasetRelationship mapToDatasetRelationship(final RelationshipDto item) .targetDatasetOddrn(item.relationshipPojo().getTargetDatasetOddrn()) .sourceDataEntityId(item.sourceDataEntity() != null ? item.sourceDataEntity().getId() : null) .targetDataEntityId(item.targetDataEntity() != null ? item.targetDataEntity().getId() : null) - .type(item.isErd() ? DatasetRelationshipType.ERD : DatasetRelationshipType.GRAPH) + .type(RelationshipTypeDto.ERD.name().equals(item.relationshipPojo().getRelationshipType()) + ? DatasetRelationshipType.ERD + : DatasetRelationshipType.GRAPH) .erdRelationship(erdRelationshipMapper.mapPojoToDetails(item.erdRelationshipDetailsPojo())) .graphRelationship(graphRelationshipMapper.mapPojoToDetails(item.graphRelationshipPojo())); } diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveRelationshipsRepositoryImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveRelationshipsRepositoryImpl.java index 81d2e5545..1f29717c0 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveRelationshipsRepositoryImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/repository/reactive/ReactiveRelationshipsRepositoryImpl.java @@ -18,6 +18,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.RelationshipsType; import org.opendatadiscovery.oddplatform.dto.RelationshipDto; import org.opendatadiscovery.oddplatform.dto.RelationshipStatusDto; +import org.opendatadiscovery.oddplatform.dto.RelationshipTypeDto; import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.ErdRelationshipDetailsPojo; import org.opendatadiscovery.oddplatform.model.tables.pojos.GraphRelationshipPojo; @@ -96,12 +97,8 @@ public Flux getRelationsByDatasetIdAndType(final Long dataEntit ); final ResultQuery query = switch (type) { - case ERD -> joinStep.where(DSL.notExists( - select().from(GRAPH_RELATIONSHIP) - .where(GRAPH_RELATIONSHIP.RELATIONSHIP_ID.eq(RELATIONSHIPS.ID)))); - case GRAPH -> joinStep.where(DSL.notExists( - select().from(ERD_RELATIONSHIP_DETAILS) - .where(ERD_RELATIONSHIP_DETAILS.RELATIONSHIP_ID.eq(RELATIONSHIPS.ID)))); + case ERD -> joinStep.where(RELATIONSHIPS.RELATIONSHIP_TYPE.eq(RelationshipTypeDto.ERD.name())); + case GRAPH -> joinStep.where(RELATIONSHIPS.RELATIONSHIP_TYPE.eq(RelationshipTypeDto.GRAPH.name())); default -> joinStep; }; @@ -110,8 +107,7 @@ public Flux getRelationsByDatasetIdAndType(final Long dataEntit r.into(srcDataEntity).into(DataEntityPojo.class), r.into(trgtDataEntity).into(DataEntityPojo.class), r.into(ERD_RELATIONSHIP_DETAILS).into(ErdRelationshipDetailsPojo.class), - r.into(GRAPH_RELATIONSHIP).into(GraphRelationshipPojo.class), - r.get(IS_ERD_RELATION, Boolean.class)) + r.into(GRAPH_RELATIONSHIP).into(GraphRelationshipPojo.class)) ); } @@ -130,8 +126,7 @@ public Mono getRelationshipById(final Long relationshipId) { r.into(srcDataEntity).into(DataEntityPojo.class), r.into(trgtDataEntity).into(DataEntityPojo.class), r.into(ERD_RELATIONSHIP_DETAILS).into(ErdRelationshipDetailsPojo.class), - r.into(GRAPH_RELATIONSHIP).into(GraphRelationshipPojo.class), - r.get(IS_ERD_RELATION, Boolean.class)) + r.into(GRAPH_RELATIONSHIP).into(GraphRelationshipPojo.class)) ); } @@ -163,11 +158,6 @@ public Mono> getRelationships(final Integer page, final In trgtDataEntity.asterisk(), ERD_RELATIONSHIP_DETAILS.asterisk(), GRAPH_RELATIONSHIP.asterisk()) - .select(DSL.exists( - DSL.select().from(ERD_RELATIONSHIP_DETAILS) - .where(ERD_RELATIONSHIP_DETAILS.RELATIONSHIP_ID.eq(relationshipCTE.field(RELATIONSHIPS.ID)))) - .as(IS_ERD_RELATION) - ) .from(relationshipCTE.getName()) .leftJoin(srcDataEntity) .on(relationshipCTE.field(RELATIONSHIPS.SOURCE_DATASET_ODDRN).eq(srcDataEntity.field(DATA_ENTITY.ODDRN))) @@ -180,14 +170,10 @@ public Mono> getRelationships(final Integer page, final In final ResultQuery resultQuery = switch (type) { case ERD -> generalQuery - .where(DSL.notExists( - select().from(GRAPH_RELATIONSHIP) - .where(GRAPH_RELATIONSHIP.RELATIONSHIP_ID.eq(relationshipCTE.field(RELATIONSHIPS.ID))))) + .where(relationshipCTE.field(RELATIONSHIPS.RELATIONSHIP_TYPE).eq(RelationshipTypeDto.ERD.name())) .groupBy(groupByFields); case GRAPH -> generalQuery - .where(DSL.notExists( - select().from(ERD_RELATIONSHIP_DETAILS) - .where(ERD_RELATIONSHIP_DETAILS.RELATIONSHIP_ID.eq(relationshipCTE.field(RELATIONSHIPS.ID))))) + .where(relationshipCTE.field(RELATIONSHIPS.RELATIONSHIP_TYPE).eq(RelationshipTypeDto.GRAPH.name())) .groupBy(groupByFields); default -> generalQuery.groupBy(groupByFields); }; @@ -202,8 +188,7 @@ public Mono> getRelationships(final Integer page, final In r.into(srcDataEntity).into(DataEntityPojo.class), r.into(trgtDataEntity).into(DataEntityPojo.class), r.into(ERD_RELATIONSHIP_DETAILS).into(ErdRelationshipDetailsPojo.class), - r.into(GRAPH_RELATIONSHIP).into(GraphRelationshipPojo.class), - r.get(IS_ERD_RELATION, Boolean.class)), + r.into(GRAPH_RELATIONSHIP).into(GraphRelationshipPojo.class)), fetchCount(inputQuery))); } @@ -214,11 +199,6 @@ private SelectOnConditionStep getRelationshipGenericQuery(final Table handleERDRelations(final List existedPojos final Map erdToCreate = new HashMap<>(); final Map> erdToUpdateMap = new HashMap<>(); - fillUpdateAndCreateMaps(existedPojos, erdMap, erdToUpdateMap, erdToCreate); + fillUpdateAndCreateMaps(existedPojos, erdMap, erdToUpdateMap, erdToCreate, RelationshipTypeDto.ERD); return updateExistedERDRelations(erdToUpdateMap) .then( @@ -115,7 +116,7 @@ private Mono handleGraphRelations(final List existedPoj final Map graphToCreate = new HashMap<>(); final Map> graphToUpdate = new HashMap<>(); - fillUpdateAndCreateMaps(existedPojos, graphMap, graphToUpdate, graphToCreate); + fillUpdateAndCreateMaps(existedPojos, graphMap, graphToUpdate, graphToCreate, RelationshipTypeDto.GRAPH); return updateExistedGraphRelations(graphToUpdate).then(Flux.fromIterable(graphToCreate.entrySet()) .flatMap(element -> relationshipsRepository.create(element.getKey()) @@ -164,10 +165,13 @@ private Mono removeRelationships(final List existedPojo private void fillUpdateAndCreateMaps(final List existedPojos, final Map ingestMap, final Map toUpdateMap, - final Map toCreatteMap) { + final Map toCreatteMap, + final RelationshipTypeDto type) { ingestMap.forEach((key, value) -> { final RelationshipsPojo oldPojo = getExistedRelationByOddrn(existedPojos, key); + key.setRelationshipType(type.name()); + if (oldPojo != null) { key.setId(oldPojo.getId()); toUpdateMap.put(key.getId(), Pair.of(key, value)); diff --git a/odd-platform-api/src/main/resources/db/migration/V0_0_87__create_relation_tables.sql b/odd-platform-api/src/main/resources/db/migration/V0_0_87__create_relation_tables.sql index 04338ae42..45fad5407 100644 --- a/odd-platform-api/src/main/resources/db/migration/V0_0_87__create_relation_tables.sql +++ b/odd-platform-api/src/main/resources/db/migration/V0_0_87__create_relation_tables.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS relationships target_dataset_oddrn varchar(512) NOT NULL, is_manualy_created boolean, relationship_status SMALLINT, + relationship_type varchar(256), last_ingested_at TIMESTAMP WITHOUT TIME ZONE, deleted_at TIMESTAMP WITHOUT TIME ZONE,