1575 - implementation of ingest relationships and GET Relationships API

.gitignore

@@ -23,3 +23,4 @@ node_modules/
 /tests/dist/
 /playwright/.cache/
 .tool-versions
+.vscode

docker/demo.yaml

@@ -17,6 +17,9 @@ services:
       - SPRING_DATASOURCE_URL=jdbc:postgresql://database:5432/${POSTGRES_DATABASE}
       - SPRING_DATASOURCE_USERNAME=${POSTGRES_USER}
       - SPRING_DATASOURCE_PASSWORD=${POSTGRES_PASSWORD}
+      - SPRING_CUSTOM-DATASOURCE_URL=jdbc:postgresql://database:5432/${POSTGRES_DATABASE}?schema=lookup_tables_schema
+      - SPRING_CUSTOM-DATASOURCE_USERNAME=${POSTGRES_USER}
+      - SPRING_CUSTOM-DATASOURCE_PASSWORD=${POSTGRES_PASSWORD}
     depends_on:
       - database
     ports:

gradle/libs.versions.toml

@@ -2,8 +2,8 @@
 spring-webflux = '6.0.9'
 reactor-extra = '3.5.1'
 micrometer-registry-prometheus = '1.9.0'
-ingestion-contract-server = '0.1.32'
-oddrn-generator-java = '0.1.20'
+ingestion-contract-server = '0.1.34'
+oddrn-generator-java = '0.1.21'
 odd-integration-manifests = '0.0.6'
 apache-collections = '4.4'
 apache-lang = '3.12.0'
@@ -33,7 +33,7 @@ shedlock-version = '4.42.0'
 json-schema-validator = '1.0.84'
 caffeine = '3.1.1'
 lombok = '1.18.24'
-testcontainers = '1.18.3'
+testcontainers = '1.19.3'
 slf4j-api = '1.7.30'
 logback = '1.2.11'
 easy-random-core = '5.0.0'

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/annotation/ReactiveCustomTransactional.java

@@ -0,0 +1,13 @@
+package org.opendatadiscovery.oddplatform.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.springframework.transaction.annotation.Transactional;
+
+@Target({ElementType.METHOD, ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+@Transactional("customTransactionManager")
+public @interface ReactiveCustomTransactional {
+}

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/manager/AuthorizationManagerType.java

@@ -6,6 +6,9 @@ public enum AuthorizationManagerType {
     DATASET_FIELD,
     TERM,
     QUERY_EXAMPLE,
+    LOOKUP_TABLE,
+    LOOKUP_TABLE_DEFINITION,
+    LOOKUP_TABLE_DATA,
     ALERT,
     DEG
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/manager/ReactiveAuthorizationManagerFactory.java

@@ -13,6 +13,9 @@ public final class ReactiveAuthorizationManagerFactory {
     private static final String DATASET_FIELD_ID = "dataset_field_id";
     private static final String DATA_ENTITY_GROUP_ID = "data_entity_group_id";
     private static final String QUERY_EXAMPLE_ID = "example_id";
+    private static final String LOOKUP_TABLE_ID = "lookup_table_id";
+    private static final String LOOKUP_TABLE_DEFINITION_ID = "column_id";
+    private static final String LOOKUP_TABLE_DATA_ID = "row_id";
     private static final String ALERT_ID = "alert_id";
 
     private ReactiveAuthorizationManagerFactory() {
@@ -46,6 +49,9 @@ private static String resourceExtractorVariableName(final AuthorizationManagerTy
             case ALERT -> ALERT_ID;
             case DEG -> DATA_ENTITY_GROUP_ID;
             case QUERY_EXAMPLE -> QUERY_EXAMPLE_ID;
+            case LOOKUP_TABLE -> LOOKUP_TABLE_ID;
+            case LOOKUP_TABLE_DEFINITION -> LOOKUP_TABLE_DEFINITION_ID;
+            case LOOKUP_TABLE_DATA -> LOOKUP_TABLE_DATA_ID;
             default -> throw new IllegalArgumentException("Unsupported resource type: " + type);
         };
     }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/auth/util/SecurityConstants.java

@@ -43,6 +43,15 @@
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_DELETE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_TOKEN_REGENERATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.DATA_SOURCE_UPDATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_CREATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DATA_CREATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DATA_DELETE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DATA_UPDATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DEFINITION_CREATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DEFINITION_DELETE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DEFINITION_UPDATE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_DELETE;
+import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.LOOKUP_TABLE_UPDATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.NAMESPACE_CREATE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.NAMESPACE_DELETE;
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.NAMESPACE_UPDATE;
@@ -73,6 +82,7 @@
 import static org.opendatadiscovery.oddplatform.dto.policy.PolicyPermissionDto.TERM_UPDATE;
 import static org.springframework.http.HttpMethod.DELETE;
 import static org.springframework.http.HttpMethod.GET;
+import static org.springframework.http.HttpMethod.PATCH;
 import static org.springframework.http.HttpMethod.POST;
 import static org.springframework.http.HttpMethod.PUT;
 
@@ -97,6 +107,8 @@ public final class SecurityConstants {
         new SecurityRule(NO_CONTEXT, new PathPatternParserServerWebExchangeMatcher("/api/terms", POST), TERM_CREATE),
         new SecurityRule(NO_CONTEXT, new PathPatternParserServerWebExchangeMatcher("/api/queryexample", POST),
             QUERY_EXAMPLE_CREATE),
+        new SecurityRule(NO_CONTEXT, new PathPatternParserServerWebExchangeMatcher("/api/referencedata/table", POST),
+            LOOKUP_TABLE_CREATE),
         new SecurityRule(NO_CONTEXT, new PathPatternParserServerWebExchangeMatcher("/api/datasources", POST),
             DATA_SOURCE_CREATE),
         new SecurityRule(NO_CONTEXT,
@@ -290,6 +302,36 @@ DATA_ENTITY, new PathPatternParserServerWebExchangeMatcher(
         new SecurityRule(NO_CONTEXT,
             new PathPatternParserServerWebExchangeMatcher(
                 "/api/queryexample/{example_id}/dataset/{data_entity_id}", DELETE),
-            QUERY_EXAMPLE_DATASET_DELETE)
+            QUERY_EXAMPLE_DATASET_DELETE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher("/api/referencedata/table/{lookup_table_id}", PUT),
+            LOOKUP_TABLE_UPDATE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher("/api/referencedata/table/{lookup_table_id}", DELETE),
+            LOOKUP_TABLE_DELETE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher(
+                "/api/referencedata/table/{lookup_table_id}/columns", POST),
+            LOOKUP_TABLE_DEFINITION_CREATE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher(
+                "/api/referencedata/table/{lookup_table_id}/column/{column_id}", PATCH),
+            LOOKUP_TABLE_DEFINITION_UPDATE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher(
+                "/api/referencedata/table/{lookup_table_id}/column/{column_id}", DELETE),
+            LOOKUP_TABLE_DEFINITION_DELETE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher(
+                "/api/referencedata/table/{lookup_table_id}/data", POST),
+            LOOKUP_TABLE_DATA_CREATE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher(
+                "/api/referencedata/table/{lookup_table_id}/data/{row_id}", PUT),
+            LOOKUP_TABLE_DATA_UPDATE),
+        new SecurityRule(NO_CONTEXT,
+            new PathPatternParserServerWebExchangeMatcher(
+                "/api/referencedata/table/{lookup_table_id}/data/{row_id}", DELETE),
+            LOOKUP_TABLE_DATA_DELETE)
     );
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/config/R2DBCConfiguration.java

@@ -5,18 +5,23 @@
 import io.r2dbc.spi.ConnectionFactories;
 import io.r2dbc.spi.ConnectionFactory;
 import io.r2dbc.spi.ConnectionFactoryOptions;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
 import org.springframework.boot.autoconfigure.r2dbc.R2dbcProperties;
 import org.springframework.boot.context.properties.PropertyMapper;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
 import org.springframework.r2dbc.connection.R2dbcTransactionManager;
+import org.springframework.r2dbc.core.DatabaseClient;
 import org.springframework.transaction.ReactiveTransactionManager;
 
 @Configuration
 public class R2DBCConfiguration {
 
     @Bean(destroyMethod = "dispose")
+    @Primary
     public ConnectionPool connectionFactory(final DataSourceProperties dataSourceProperties,
                                             final R2dbcProperties properties) {
         final String r2dbcUrl = dataSourceProperties.getUrl().replace("jdbc", "r2dbc");
@@ -42,8 +47,59 @@ public ConnectionPool connectionFactory(final DataSourceProperties dataSourcePro
         return new ConnectionPool(builder.build());
     }
 
+    @Bean(destroyMethod = "dispose")
+    @Qualifier("customConnectionPool")
+    public ConnectionPool databaseClientForCustomSchema(
+        @Value("${spring.custom-datasource.url}") final String url,
+        @Value("${spring.custom-datasource.username}") final String username,
+        @Value("${spring.custom-datasource.password}") final String password,
+        final R2dbcProperties properties) {
+        final String r2dbcUrl = url.replace("jdbc", "r2dbc");
+        final ConnectionFactory factory = ConnectionFactories.get(ConnectionFactoryOptions.parse(r2dbcUrl).mutate()
+            .option(ConnectionFactoryOptions.PROTOCOL, "postgresql")
+            .option(ConnectionFactoryOptions.USER, username)
+            .option(ConnectionFactoryOptions.PASSWORD, password)
+            .build());
+
+        final R2dbcProperties.Pool pool = properties.getPool();
+        final PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull();
+        final ConnectionPoolConfiguration.Builder builder = ConnectionPoolConfiguration.builder(factory);
+        map.from(pool.getMaxIdleTime()).to(builder::maxIdleTime);
+        map.from(pool.getMaxLifeTime()).to(builder::maxLifeTime);
+        map.from(pool.getMaxAcquireTime()).to(builder::maxAcquireTime);
+        map.from(pool.getMaxCreateConnectionTime()).to(builder::maxCreateConnectionTime);
+        map.from(pool.getInitialSize()).to(builder::initialSize);
+        map.from(pool.getMaxSize()).to(builder::maxSize);
+        map.from(pool.getValidationQuery()).whenHasText().to(builder::validationQuery);
+        map.from(pool.getValidationDepth()).to(builder::validationDepth);
+        map.from(pool.getMinIdle()).to(builder::minIdle);
+        map.from(pool.getMaxValidationTime()).to(builder::maxValidationTime);
+
+        return new ConnectionPool(builder.build());
+    }
+
+    @Bean
+    public DatabaseClient databaseClient(final ConnectionFactory schema1ConnectionFactory) {
+        return DatabaseClient.create(schema1ConnectionFactory);
+    }
+
+    @Bean
+    @Qualifier("customDataClient")
+    public DatabaseClient databaseClientCustomTables(
+        @Qualifier("customConnectionPool") final ConnectionFactory connectionFactory) {
+        return DatabaseClient.create(connectionFactory);
+    }
+
     @Bean
+    @Primary
     public ReactiveTransactionManager reactiveTransactionManager(final ConnectionFactory connectionFactory) {
         return new R2dbcTransactionManager(connectionFactory);
     }
+
+    @Bean
+    @Qualifier("customTransactionManager")
+    public ReactiveTransactionManager reactiveCustomTransactionManager(
+        @Qualifier("customConnectionPool") final ConnectionFactory connectionFactory) {
+        return new R2dbcTransactionManager(connectionFactory);
+    }
 }

odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/controller/DatasetController.java

@@ -4,7 +4,13 @@
 import org.opendatadiscovery.oddplatform.api.contract.api.DataSetApi;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetStructure;
 import org.opendatadiscovery.oddplatform.api.contract.model.DataSetVersionDiffList;
+import org.opendatadiscovery.oddplatform.api.contract.model.ERDRelationshipDetailsList;
+import org.opendatadiscovery.oddplatform.api.contract.model.GraphRelationshipDetailsList;
+import org.opendatadiscovery.oddplatform.api.contract.model.RelationshipList;
 import org.opendatadiscovery.oddplatform.service.DatasetVersionService;
+import org.opendatadiscovery.oddplatform.service.ERDRelationshipsService;
+import org.opendatadiscovery.oddplatform.service.GraphRelationshipsService;
+import org.opendatadiscovery.oddplatform.service.RelationshipsService;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ServerWebExchange;
@@ -14,6 +20,9 @@
 @RequiredArgsConstructor
 public class DatasetController implements DataSetApi {
     private final DatasetVersionService datasetVersionService;
+    private final RelationshipsService relationshipsService;
+    private final ERDRelationshipsService erdRelationshipsService;
+    private final GraphRelationshipsService graphRelationshipsService;
 
     @Override
     public Mono<ResponseEntity<DataSetStructure>> getDataSetStructureByVersionId(
@@ -44,4 +53,29 @@ public Mono<ResponseEntity<DataSetVersionDiffList>> getDataSetStructureDiff(fina
         return datasetVersionService.getDatasetVersionDiff(dataEntityId, firstVersionId, secondVersionId)
             .map(ResponseEntity::ok);
     }
+
+    @Override
+    public Mono<ResponseEntity<RelationshipList>> getDataSetRelationships(final Long dataEntityId,
+                                                                          final ServerWebExchange exchange) {
+        return relationshipsService.getRelationsByDatasetId(dataEntityId)
+            .map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<ERDRelationshipDetailsList>>
+        getDataSetErdRelationshipsById(final Long dataEntityId,
+                                   final Long relationshipsId,
+                                   final ServerWebExchange exchange) {
+        return erdRelationshipsService.getDataSetErdRelationshipsById(dataEntityId, relationshipsId)
+            .map(ResponseEntity::ok);
+    }
+
+    @Override
+    public Mono<ResponseEntity<GraphRelationshipDetailsList>>
+        getDataSetGraphRelationshipsById(final Long dataEntityId,
+                                     final Long relationshipsId,
+                                     final ServerWebExchange exchange) {
+        return graphRelationshipsService.getDataSetGraphRelationshipsById(relationshipsId)
+            .map(ResponseEntity::ok);
+    }
 }