diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java index c1167bd80..f8f7f0218 100644 --- a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataEntityMapperImpl.java @@ -62,6 +62,7 @@ public class DataEntityMapperImpl implements DataEntityMapper { private static DataEntityClassAndTypeDictionary TYPE_DICTIONARY = null; private final DataSourceMapper dataSourceMapper; + private final DataSourceSafeMapper dataSourceSafeMapper; private final OwnershipMapper ownershipMapper; private final TagMapper tagMapper; private final MetadataFieldValueMapper metadataFieldValueMapper; @@ -86,7 +87,7 @@ public DataEntity mapPojo(final DataEntityDimensionsDto dto) { .entityClasses(entityClasses.stream().map(this::mapEntityClass).toList()) .type(type) .ownership(ownershipMapper.mapDtos(dto.getOwnership())) - .dataSource(dataSourceMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null))) + .dataSource(dataSourceSafeMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null))) .dataEntityGroups(groups); if (entityClasses.contains(DataEntityClassDto.DATA_SET)) { @@ -250,7 +251,7 @@ public DataEntityDetails mapDtoDetails(final DataEntityDetailsDto dto) { .type(type) .status(dataEntityStatusMapper.mapStatus(pojo)) .ownership(ownershipMapper.mapDtos(dto.getOwnership())) - .dataSource(dataSourceMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null))) + .dataSource(dataSourceSafeMapper.mapDto(new DataSourceDto(dto.getDataSource(), dto.getNamespace(), null))) .tags(tagMapper.mapToTagList(dto.getTags())) .metadataFieldValues(metadataFieldValueMapper.mapDtos(dto.getMetadata())) .terms(linkedTerms) diff --git a/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataSourceSafeMapper.java b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataSourceSafeMapper.java new file mode 100644 index 000000000..f7092bd3f --- /dev/null +++ b/odd-platform-api/src/main/java/org/opendatadiscovery/oddplatform/mapper/DataSourceSafeMapper.java @@ -0,0 +1,15 @@ +package org.opendatadiscovery.oddplatform.mapper; + +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceSafe; +import org.opendatadiscovery.oddplatform.dto.DataSourceDto; + +@Mapper( + config = MapperConfig.class, + uses = {NamespaceMapper.class} +) +public interface DataSourceSafeMapper { + @Mapping(source = "dataSource", target = ".") + DataSourceSafe mapDto(DataSourceDto dataSourceDto); +} diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java index 558c12b3a..f76e60d46 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/BaseIngestionTest.java @@ -20,6 +20,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataSetStructure; import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceFormData; +import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceSafe; import org.opendatadiscovery.oddplatform.api.contract.model.MetricSet; import org.opendatadiscovery.oddplatform.api.contract.model.SearchFacetsData; import org.opendatadiscovery.oddplatform.api.contract.model.SearchFilterState; @@ -53,6 +54,12 @@ protected DataSource createDataSource(final DataSourceFormData dataSource) { return ingestDataSource(dataSource); } + protected DataSourceSafe createDataSourceSafeFromDataSource(final DataSource dataSource) { + return new DataSourceSafe(dataSource.getId(), dataSource.getOddrn(), dataSource.getName()) + .description(dataSource.getDescription()) + .namespace(dataSource.getNamespace()); + } + private DataSource ingestDataSource(final DataSourceFormData dataSource) { return webTestClient.post() .uri("/api/datasources") diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/AlertIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/AlertIngestionTest.java index 12b924234..c58427b6e 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/AlertIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/AlertIngestionTest.java @@ -76,7 +76,8 @@ public void backwardsIncompatibleAlertIngestionTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -98,7 +99,8 @@ public void backwardsIncompatibleAlertIngestionTest() { ingestAndAssert(dataEntityList); final var updatedExpectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -152,7 +154,8 @@ public void failedJobAlertIngestionTest() { final long jobId = extractIngestedEntityIdAndAssert(createdDataSource); final DataEntityDetails expectedDataEntityDetails = - IngestionModelMapper.buildExpectedBaseDEDetails(jobId, job, createdDataSource); + IngestionModelMapper.buildExpectedBaseDEDetails(jobId, job, + createDataSourceSafeFromDataSource(createdDataSource)); assertDataEntityDetailsEqual(expectedDataEntityDetails, (expected, actual) -> { assertThat(actual.getSourceList()).isEmpty(); @@ -218,7 +221,8 @@ public void failedDQTestAlertIngestionTest() { .forEach((expectedOddrn, dataEntity) -> { final long entityId = ingestionMap.get(expectedOddrn); final DataEntityDetails expectedDetails = - IngestionModelMapper.buildExpectedBaseDEDetails(entityId, dataEntity, createdDataSource); + IngestionModelMapper.buildExpectedBaseDEDetails(entityId, dataEntity, + createDataSourceSafeFromDataSource(createdDataSource)); if (dataEntity.getDataset() != null) { expectedDetails.setStats(new DataSetStats() diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetFieldIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetFieldIngestionTest.java index fe9f978cb..7a8918c12 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetFieldIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetFieldIngestionTest.java @@ -69,7 +69,8 @@ public void enumValuesComplexTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -155,7 +156,8 @@ public void rewritingInternalEnumStateTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -222,7 +224,8 @@ public void externalStatisticsTagStateIngestionTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetIngestionTest.java index 4b1d4a87f..5e4e108de 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/DatasetIngestionTest.java @@ -55,7 +55,8 @@ public void simpleDatasetIngestionTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -107,7 +108,8 @@ public void changeDatasetStructureTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -142,7 +144,8 @@ public void changeDatasetStructureTest() { ingestAndAssert(dataEntityList); final var updatedExpectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -213,7 +216,8 @@ public void updateHollowDatasetSchemaTest() { assertThat(dataEntityId).isNotNull(); final DataEntityDetails expectedDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(dataEntityId, dataEntity, createdDataSource); + .buildExpectedBaseDEDetails(dataEntityId, dataEntity, + createDataSourceSafeFromDataSource(createdDataSource)); if (dataEntity.getDataset() != null) { expectedDetails.setStats(new DataSetStats() diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java index 7ed494b1b..180626e95 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/LineageIngestionTest.java @@ -100,7 +100,8 @@ public void simpleLineageIngestionTest() { assertThat(dataEntityId).isNotNull(); final DataEntityDetails expectedDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(dataEntityId, dataEntity, createdDataSource); + .buildExpectedBaseDEDetails(dataEntityId, dataEntity, + createDataSourceSafeFromDataSource(createdDataSource)); if (dataEntity.getDataset() != null) { expectedDetails.setStats(new DataSetStats() diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/MetadataIngestionTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/MetadataIngestionTest.java index b291752a7..e06f6fb6b 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/MetadataIngestionTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/MetadataIngestionTest.java @@ -49,7 +49,8 @@ public void simpleMetadataIngestionTest() { final long foundEntityId = extractIngestedEntityIdAndAssert(createdDataSource); final var expectedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -78,7 +79,8 @@ public void simpleMetadataIngestionTest() { ingestAndAssert(dataEntityList); final var expectedUpdatedDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) @@ -95,7 +97,8 @@ public void simpleMetadataIngestionTest() { ingestAndAssert(dataEntityList); final var expectedEmptyMetadataDataEntityDetails = IngestionModelMapper - .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, createdDataSource) + .buildExpectedBaseDEDetails(foundEntityId, datasetToIngest, + createDataSourceSafeFromDataSource(createdDataSource)) .stats(new DataSetStats() .consumersCount(0L) .fieldsCount((long) datasetToIngest.getDataset().getFieldList().size()) diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java index 469a5f248..e04cc9a57 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/api/ingestion/utils/IngestionModelMapper.java @@ -14,7 +14,7 @@ import org.opendatadiscovery.oddplatform.api.contract.model.DataSetField; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetFieldStat; import org.opendatadiscovery.oddplatform.api.contract.model.DataSetFieldType; -import org.opendatadiscovery.oddplatform.api.contract.model.DataSource; +import org.opendatadiscovery.oddplatform.api.contract.model.DataSourceSafe; import org.opendatadiscovery.oddplatform.api.contract.model.DateTimeFieldStat; import org.opendatadiscovery.oddplatform.api.contract.model.EnumValue; import org.opendatadiscovery.oddplatform.api.contract.model.IntegerFieldStat; @@ -33,7 +33,7 @@ public class IngestionModelMapper { public static DataEntityDetails buildExpectedBaseDEDetails(final long entityId, final DataEntity dataEntity, - final DataSource dataSource) { + final DataSourceSafe dataSource) { return new DataEntityDetails() .id(entityId) .sourceCreatedAt(dataEntity.getCreatedAt()) diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java index 6c90ffd14..40f250c43 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/mapper/LineageMapperTest.java @@ -56,6 +56,9 @@ void setUp() { new DateTimeMapperImpl() ) ), + new DataSourceSafeMapperImpl( + new NamespaceMapperImpl() + ), new OwnershipMapperImpl( new OwnerMapperImpl(), new TitleMapperImpl() diff --git a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java index a4e55bf29..37234fbcf 100644 --- a/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java +++ b/odd-platform-api/src/test/java/org/opendatadiscovery/oddplatform/service/LineageServiceTest.java @@ -20,6 +20,7 @@ import org.opendatadiscovery.oddplatform.mapper.DataEntityRunMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DataEntityStatusMapper; import org.opendatadiscovery.oddplatform.mapper.DataSourceMapperImpl; +import org.opendatadiscovery.oddplatform.mapper.DataSourceSafeMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DatasetFieldApiMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DatasetVersionMapperImpl; import org.opendatadiscovery.oddplatform.mapper.DateTimeMapperImpl; @@ -81,6 +82,9 @@ void setUp() { new DateTimeMapperImpl() ) ), + new DataSourceSafeMapperImpl( + new NamespaceMapperImpl() + ), new OwnershipMapperImpl( new OwnerMapperImpl(), new TitleMapperImpl() diff --git a/odd-platform-specification/components.yaml b/odd-platform-specification/components.yaml index ed0ce7ecf..3b43a3249 100644 --- a/odd-platform-specification/components.yaml +++ b/odd-platform-specification/components.yaml @@ -530,7 +530,7 @@ components: ownership: $ref: '#/components/schemas/OwnershipList' data_source: - $ref: '#/components/schemas/DataSource' + $ref: '#/components/schemas/DataSourceSafe' entity_classes: type: array items: @@ -1210,6 +1210,25 @@ components: - oddrn - name + DataSourceSafe: + type: object + properties: + id: + type: integer + format: int64 + namespace: + $ref: '#/components/schemas/Namespace' + oddrn: + type: string + name: + type: string + description: + type: string + required: + - id + - oddrn + - name + DataSourceList: type: object properties: @@ -1462,8 +1481,6 @@ components: $ref: '#/components/schemas/BinaryFieldStat' datetime_stats: $ref: '#/components/schemas/DateTimeFieldStat' - required: - - name BooleanFieldStat: type: object