Introduce list of relathionships between Data Entities (#1630)

Co-authored-by: vburlachenko <[email protected]>
opendatadiscovery · Feb 20, 2024 · 516a9d1 · 516a9d1
1 parent ff8f4ba
commit 516a9d1
Show file tree

Hide file tree

Showing 40 changed files with 641 additions and 119 deletions.
diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/dto/RelationshipDto.java
@@ -2,11 +2,16 @@
 
 import lombok.Builder;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.DataSourcePojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.NamespacePojo;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.RelationshipsPojo;
 
 @Builder
 public record RelationshipDto(DataEntityPojo dataEntityRelationship,
                               RelationshipsPojo relationshipPojo,
+                              DataSourcePojo dataSourcePojo,
+                              NamespacePojo relationshipNamespacePojo,
+                              NamespacePojo dataSourceNamespacePojo,
                               DataEntityPojo sourceDataEntity,
                               DataEntityPojo targetDataEntity) {
 }
diff --git a/...api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipDetailsMapper.java b/...api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipDetailsMapper.java
@@ -29,8 +29,8 @@ public DataEntityERDRelationshipDetails mapToDataEntityERDRelationshipDetails(
                 relationshipDetailsDto.targetDataEntity() != null ? relationshipDetailsDto.targetDataEntity().getId() :
                     null)
             .type(RelationshipTypeDto.ERD.name().equals(relationshipDetailsDto.relationshipPojo().getRelationshipType())
-                ? DataEntityRelationshipType.ERD
-                : DataEntityRelationshipType.GRAPH)
+                ? DataEntityRelationshipType.ENTITY_RELATIONSHIP
+                : DataEntityRelationshipType.GRAPH_RELATIONSHIP)
             .erdRelationship(
                 erdRelationshipMapper.mapPojoToDetails(relationshipDetailsDto.erdRelationshipDetailsPojo()));
     }
@@ -50,8 +50,8 @@ public DataEntityGraphRelationshipDetails mapToDataEntityGraphRelationshipDetail
                 relationshipDetailsDto.targetDataEntity() != null ? relationshipDetailsDto.targetDataEntity().getId() :
                     null)
             .type(RelationshipTypeDto.ERD.name().equals(relationshipDetailsDto.relationshipPojo().getRelationshipType())
-                ? DataEntityRelationshipType.ERD
-                : DataEntityRelationshipType.GRAPH)
+                ? DataEntityRelationshipType.ENTITY_RELATIONSHIP
+                : DataEntityRelationshipType.GRAPH_RELATIONSHIP)
             .graphRelationship(
                 graphRelationshipMapper.mapPojoToDetails(relationshipDetailsDto.graphRelationshipPojo()));
     }

diff --git a/...atform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipMapper.java b/...atform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/RelationshipMapper.java
@@ -7,6 +7,7 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRelationshipList;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataEntityRelationshipType;
 import org.opendatadiscovery.oddplatform.api.contract.model.PageInfo;
+import org.opendatadiscovery.oddplatform.dto.DataSourceDto;
 import org.opendatadiscovery.oddplatform.dto.RelationshipDto;
 import org.opendatadiscovery.oddplatform.dto.RelationshipTypeDto;
 import org.opendatadiscovery.oddplatform.utils.Page;
@@ -15,6 +16,9 @@
 @RequiredArgsConstructor
 @Component
 public class RelationshipMapper {
+    private final DataSourceSafeMapper dataSourceSafeMapper;
+    private final NamespaceMapper namespaceMapper;
+
     public DataEntityRelationshipList mapListToRelationshipList(final List<RelationshipDto> relationshipDtos) {
         return new DataEntityRelationshipList()
             .items(mapToRelationshipList(relationshipDtos))
@@ -36,9 +40,12 @@ public DataEntityRelationship mapToDatasetRelationship(final RelationshipDto ite
             .targetDatasetOddrn(item.relationshipPojo().getTargetDatasetOddrn())
             .sourceDataEntityId(item.sourceDataEntity() != null ? item.sourceDataEntity().getId() : null)
             .targetDataEntityId(item.targetDataEntity() != null ? item.targetDataEntity().getId() : null)
+            .dataSource(dataSourceSafeMapper.mapDto(new DataSourceDto(item.dataSourcePojo(),
+                item.dataSourceNamespacePojo(), null)))
+            .namespace(namespaceMapper.mapPojo(item.relationshipNamespacePojo()))
             .type(RelationshipTypeDto.ERD.name().equals(item.relationshipPojo().getRelationshipType())
-                ? DataEntityRelationshipType.ERD
-                : DataEntityRelationshipType.GRAPH);
+                ? DataEntityRelationshipType.ENTITY_RELATIONSHIP
+                : DataEntityRelationshipType.GRAPH_RELATIONSHIP);
     }
 
     public DataEntityRelationshipList mapListToRelationshipPage(final Page<RelationshipDto> relationshipDtoPage) {

diff --git a/...scovery/oddplatform/repository/reactive/ReactiveDataEntityRelationshipRepositoryImpl.java b/...scovery/oddplatform/repository/reactive/ReactiveDataEntityRelationshipRepositoryImpl.java
@@ -18,8 +18,12 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.RelationshipsType;
 import org.opendatadiscovery.oddplatform.dto.RelationshipDto;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.DataSourcePojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.NamespacePojo;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.RelationshipsPojo;
 import org.opendatadiscovery.oddplatform.model.tables.records.DataEntityRecord;
+import org.opendatadiscovery.oddplatform.model.tables.records.DataSourceRecord;
+import org.opendatadiscovery.oddplatform.model.tables.records.NamespaceRecord;
 import org.opendatadiscovery.oddplatform.model.tables.records.RelationshipsRecord;
 import org.opendatadiscovery.oddplatform.repository.util.JooqQueryHelper;
 import org.opendatadiscovery.oddplatform.repository.util.JooqReactiveOperations;
@@ -30,6 +34,8 @@
 
 import static org.opendatadiscovery.oddplatform.dto.DataEntityClassDto.DATA_RELATIONSHIP;
 import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY;
+import static org.opendatadiscovery.oddplatform.model.Tables.DATA_SOURCE;
+import static org.opendatadiscovery.oddplatform.model.Tables.NAMESPACE;
 import static org.opendatadiscovery.oddplatform.model.Tables.RELATIONSHIPS;
 
 @Slf4j
@@ -40,6 +46,9 @@ public class ReactiveDataEntityRelationshipRepositoryImpl
     private static final String RELATIONSHIPS_CTE = "relationships_cte";
     private static final String SOURCE_DATA_ENTITY = "source_data_entity";
     private static final String TARGET_DATA_ENTITY = "target_data_entity";
+    private static final String RELATIONSHIP_NAMESPACE = "relationship_namespace";
+    private static final String DATA_SOURCE_NAMESPACE = "data_source_namespace";
+    private static final String DATA_SOURCE_CTE = "data_source_cte";
 
     private final JooqReactiveOperations jooqReactiveOperations;
     private final JooqQueryHelper jooqQueryHelper;
@@ -50,6 +59,9 @@ public Mono<Page<RelationshipDto>> getRelationships(final Integer page, final In
         final Table<RelationshipsRecord> relationships = RELATIONSHIPS.asTable(RELATIONSHIPS_CTE);
         final Table<DataEntityRecord> srcDataEntity = DATA_ENTITY.asTable(SOURCE_DATA_ENTITY);
         final Table<DataEntityRecord> trgtDataEntity = DATA_ENTITY.asTable(TARGET_DATA_ENTITY);
+        final Table<DataSourceRecord> dataSource = DATA_SOURCE.asTable(DATA_SOURCE_CTE);
+        final Table<NamespaceRecord> relationshipNamespace = NAMESPACE.asTable(RELATIONSHIP_NAMESPACE);
+        final Table<NamespaceRecord> dataSourceNamespace = NAMESPACE.asTable(DATA_SOURCE_NAMESPACE);
 
         final List<Condition> conditionList = new ArrayList<>();
 
@@ -71,14 +83,16 @@ public Mono<Page<RelationshipDto>> getRelationships(final Integer page, final In
 
         final List<Field<?>> groupByFields =
             Stream.of(relationshipsDataEntityCTE.fields(), srcDataEntity.fields(),
-                    trgtDataEntity.fields(), relationships.fields())
+                    trgtDataEntity.fields(), relationships.fields(), dataSource.fields(),
+                    relationshipNamespace.fields(), dataSourceNamespace.fields())
                 .flatMap(Arrays::stream)
                 .toList();
 
         final ResultQuery<Record> resultQuery = DSL.with(relationshipsDataEntityCTE.getName())
             .as(relationshipSelect)
             .select(relationshipsDataEntityCTE.fields())
-            .select(relationships.asterisk(), srcDataEntity.asterisk(), trgtDataEntity.asterisk())
+            .select(relationships.asterisk(), srcDataEntity.asterisk(), trgtDataEntity.asterisk(),
+                dataSource.asterisk(), relationshipNamespace.asterisk(), dataSourceNamespace.asterisk())
             .from(relationshipsDataEntityCTE.getName())
             .join(relationships)
             .on(relationshipsDataEntityCTE.field(DATA_ENTITY.ID).eq(relationships.field(RELATIONSHIPS.DATA_ENTITY_ID))
@@ -89,6 +103,13 @@ public Mono<Page<RelationshipDto>> getRelationships(final Integer page, final In
             .on(relationships.field(RELATIONSHIPS.SOURCE_DATASET_ODDRN).eq(srcDataEntity.field(DATA_ENTITY.ODDRN)))
             .leftJoin(trgtDataEntity)
             .on(relationships.field(RELATIONSHIPS.TARGET_DATASET_ODDRN).eq(trgtDataEntity.field(DATA_ENTITY.ODDRN)))
+            .leftJoin(dataSource)
+            .on(relationshipsDataEntityCTE.field(DATA_ENTITY.DATA_SOURCE_ID).eq(dataSource.field(DATA_SOURCE.ID)))
+            .leftJoin(dataSourceNamespace)
+            .on(dataSource.field(DATA_SOURCE.NAMESPACE_ID).eq(dataSourceNamespace.field(NAMESPACE.ID)))
+            .leftJoin(relationshipNamespace)
+            .on(relationshipsDataEntityCTE.field(DATA_ENTITY.NAMESPACE_ID)
+                .eq(relationshipNamespace.field(NAMESPACE.ID)))
             .groupBy(groupByFields);
 
         return jooqReactiveOperations.flux(resultQuery)
@@ -100,6 +121,9 @@ public Mono<Page<RelationshipDto>> getRelationships(final Integer page, final In
                     .dataEntityRelationship(r.into(relationshipsDataEntityCTE).into(DataEntityPojo.class))
                     .sourceDataEntity(r.into(srcDataEntity).into(DataEntityPojo.class))
                     .targetDataEntity(r.into(trgtDataEntity).into(DataEntityPojo.class))
+                    .dataSourcePojo(r.into(dataSource).into(DataSourcePojo.class))
+                    .dataSourceNamespacePojo(r.into(dataSourceNamespace).into(NamespacePojo.class))
+                    .relationshipNamespacePojo(r.into(relationshipNamespace).into(NamespacePojo.class))
                     .build(),
                 jooqReactiveOperations
                     .mono(DSL.selectCount().from(DATA_ENTITY).where(conditionList))

diff --git a/...pendatadiscovery/oddplatform/repository/reactive/ReactiveRelationshipsRepositoryImpl.java b/...pendatadiscovery/oddplatform/repository/reactive/ReactiveRelationshipsRepositoryImpl.java
@@ -12,8 +12,12 @@
 import org.opendatadiscovery.oddplatform.api.contract.model.RelationshipsType;
 import org.opendatadiscovery.oddplatform.dto.RelationshipDto;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.DataEntityPojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.DataSourcePojo;
+import org.opendatadiscovery.oddplatform.model.tables.pojos.NamespacePojo;
 import org.opendatadiscovery.oddplatform.model.tables.pojos.RelationshipsPojo;
 import org.opendatadiscovery.oddplatform.model.tables.records.DataEntityRecord;
+import org.opendatadiscovery.oddplatform.model.tables.records.DataSourceRecord;
+import org.opendatadiscovery.oddplatform.model.tables.records.NamespaceRecord;
 import org.opendatadiscovery.oddplatform.model.tables.records.RelationshipsRecord;
 import org.opendatadiscovery.oddplatform.repository.util.JooqQueryHelper;
 import org.opendatadiscovery.oddplatform.repository.util.JooqReactiveOperations;
@@ -23,6 +27,8 @@
 import reactor.core.publisher.Mono;
 
 import static org.opendatadiscovery.oddplatform.model.Tables.DATA_ENTITY;
+import static org.opendatadiscovery.oddplatform.model.Tables.DATA_SOURCE;
+import static org.opendatadiscovery.oddplatform.model.Tables.NAMESPACE;
 import static org.opendatadiscovery.oddplatform.model.Tables.RELATIONSHIPS;
 
 @Slf4j
@@ -34,6 +40,9 @@ public class ReactiveRelationshipsRepositoryImpl
     private static final String RELATIONSHIPS_DATA_ENTITY = "relationships_data_entity";
     private static final String SOURCE_DATA_ENTITY = "source_data_entity";
     private static final String TARGET_DATA_ENTITY = "target_data_entity";
+    private static final String RELATIONSHIP_NAMESPACE = "relationship_namespace";
+    private static final String DATA_SOURCE_NAMESPACE = "data_source_namespace";
+    private static final String DATA_SOURCE_CTE = "data_source_cte";
 
     public ReactiveRelationshipsRepositoryImpl(final JooqReactiveOperations jooqReactiveOperations,
                                                final JooqQueryHelper jooqQueryHelper,
@@ -61,10 +70,17 @@ public Flux<RelationshipDto> getRelationsByDatasetIdAndType(final Long dataEntit
         final Table<DataEntityRecord> relationshipsDataEntity = DATA_ENTITY.asTable(RELATIONSHIPS_DATA_ENTITY);
         final Table<DataEntityRecord> srcDataEntity = DATA_ENTITY.asTable(SOURCE_DATA_ENTITY);
         final Table<DataEntityRecord> trgtDataEntity = DATA_ENTITY.asTable(TARGET_DATA_ENTITY);
+        final Table<DataSourceRecord> dataSource = DATA_SOURCE.asTable(DATA_SOURCE_CTE);
+        final Table<NamespaceRecord> relationshipNamespace = NAMESPACE.asTable(RELATIONSHIP_NAMESPACE);
+        final Table<NamespaceRecord> dataSourceNamespace = NAMESPACE.asTable(DATA_SOURCE_NAMESPACE);
+
         final SelectOnConditionStep<Record> query = DSL.select(RELATIONSHIPS.asterisk(),
                 relationshipsDataEntity.asterisk(),
                 srcDataEntity.asterisk(),
-                trgtDataEntity.asterisk())
+                trgtDataEntity.asterisk(),
+                dataSource.asterisk(),
+                relationshipNamespace.asterisk(),
+                dataSourceNamespace.asterisk())
             .from(RELATIONSHIPS)
             .leftJoin(srcDataEntity)
             .on(RELATIONSHIPS.SOURCE_DATASET_ODDRN.eq(srcDataEntity.field(DATA_ENTITY.ODDRN)))
@@ -75,7 +91,13 @@ public Flux<RelationshipDto> getRelationsByDatasetIdAndType(final Long dataEntit
                 .or(dataEntityCTE.field(DATA_ENTITY.ODDRN).eq(RELATIONSHIPS.TARGET_DATASET_ODDRN)))
             )
             .join(relationshipsDataEntity)
-            .on(RELATIONSHIPS.DATA_ENTITY_ID.eq(relationshipsDataEntity.field(DATA_ENTITY.ID)));
+            .on(RELATIONSHIPS.DATA_ENTITY_ID.eq(relationshipsDataEntity.field(DATA_ENTITY.ID)))
+            .leftJoin(dataSource)
+            .on(relationshipsDataEntity.field(DATA_ENTITY.DATA_SOURCE_ID).eq(dataSource.field(DATA_SOURCE.ID)))
+            .leftJoin(relationshipNamespace)
+            .on(relationshipsDataEntity.field(DATA_ENTITY.NAMESPACE_ID).eq(relationshipNamespace.field(NAMESPACE.ID)))
+            .leftJoin(dataSourceNamespace)
+            .on(dataSource.field(DATA_SOURCE.NAMESPACE_ID).eq(dataSourceNamespace.field(NAMESPACE.ID)));
 
         ResultQuery<Record> finalQuery = query;
 
@@ -87,19 +109,29 @@ public Flux<RelationshipDto> getRelationsByDatasetIdAndType(final Long dataEntit
             .map(r -> mapToDto(r.into(RELATIONSHIPS).into(RelationshipsPojo.class),
                     r.into(relationshipsDataEntity).into(DataEntityPojo.class),
                     r.into(srcDataEntity).into(DataEntityPojo.class),
-                    r.into(trgtDataEntity).into(DataEntityPojo.class))
+                    r.into(trgtDataEntity).into(DataEntityPojo.class),
+                    r.into(dataSource).into(DataSourcePojo.class),
+                    r.into(dataSourceNamespace).into(NamespacePojo.class),
+                    r.into(relationshipNamespace).into(NamespacePojo.class)
+                )
             );
     }
 
     private RelationshipDto mapToDto(final RelationshipsPojo pojo,
                                      final DataEntityPojo relationshipsDataEntity,
                                      final DataEntityPojo srcDataEntity,
-                                     final DataEntityPojo trgtDataEntity) {
+                                     final DataEntityPojo trgtDataEntity,
+                                     final DataSourcePojo dataSourcePojo,
+                                     final NamespacePojo dataSourceNamespace,
+                                     final NamespacePojo relationshipNamespace) {
         return RelationshipDto.builder()
             .relationshipPojo(pojo)
             .dataEntityRelationship(relationshipsDataEntity)
             .sourceDataEntity(srcDataEntity)
             .targetDataEntity(trgtDataEntity)
+            .dataSourcePojo(dataSourcePojo)
+            .dataSourceNamespacePojo(dataSourceNamespace)
+            .relationshipNamespacePojo(relationshipNamespace)
             .build();
     }
 }
diff --git a/odd-platform-specification/components.yaml b/odd-platform-specification/components.yaml
@@ -3972,13 +3972,19 @@ components:
           type: string
         target_dataset_oddrn:
           type: string
+        data_source:
+          $ref: '#/components/schemas/DataSourceSafe'
+        namespace:
+          $ref: '#/components/schemas/Namespace'
         type:
           $ref: '#/components/schemas/DataEntityRelationshipType'
       required:
+        - id
         - name
         - oddrn
         - source_dataset_oddrn
         - target_dataset_oddrn
+        - data_source
         - type
 
     DataEntityERDRelationshipDetails:
@@ -4004,8 +4010,8 @@ components:
     DataEntityRelationshipType:
       type: string
       enum:
-        - ERD
-        - GRAPH
+        - ENTITY_RELATIONSHIP
+        - GRAPH_RELATIONSHIP
 
     ERDRelationshipDetails:
       type: object

diff --git a/odd-platform-ui/package.json b/odd-platform-ui/package.json
@@ -46,8 +46,8 @@
     "@mui/x-date-pickers": "^5.0.20",
     "@reduxjs/toolkit": "^1.9.7",
     "@tanstack/react-query": "^5.17.19",
-    "@tanstack/react-virtual": "^3.0.1",
     "@tanstack/react-table": "^8.11.2",
+    "@tanstack/react-virtual": "^3.0.1",
     "@uiw/react-md-editor": "^3.25.6",
     "@visx/curve": "^3.3.0",
     "@visx/event": "^3.3.0",
@@ -80,10 +80,10 @@
     "react-infinite-scroll-component": "^6.1.0",
     "react-multi-date-picker": "^4.4.1",
     "react-redux": "^8.1.2",
-    "react-router-dom": "^6.21.2",
+    "react-router-dom": "^6.22.0",
     "react-truncate-markup": "^5.1.2",
     "recharts": "^2.12.0",
-    "styled-components": "^6.1.1",
+    "styled-components": "^6.1.8",
     "use-debounce": "^10.0.0",
     "uuid": "^9.0.1",
     "vanilla-jsoneditor": "^0.7.11"