diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7f0dfbd82b..71ef34294f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,7 +29,7 @@ updates: - "dependency" - "gradle" commit-message: - prefix: "[DCJ-400-gradle]" + prefix: "[DT-400-gradle]" ignore: # Google API dependencies use a version format that Dependabot wrongly interprets as semver. - dependency-name: "com.google.apis:*" diff --git a/README.md b/README.md index 9126073783..7708f1f364 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,13 @@ See [started guide](docs/jade-getting-started.md) for information on running con ### Run TDR locally To run TDR locally: -`./scripts/run local` +`./scripts/run start_local` To run TDR in docker: -`./scripts/run docker` +`./scripts/run start_docker` To run TDR locally and wait for debugger to attach on port 5005: -`./scripts/run local --debug-jvm` +`./scripts/run start_local --debug-jvm` To have the code hot reload, enable automatic builds in intellij, go to: `Preferences -> Build, Execution, Deployment -> Compiler` diff --git a/src/main/java/bio/terra/app/configuration/WebConfig.java b/src/main/java/bio/terra/app/configuration/WebConfig.java index af805ee885..a5815cdd15 100644 --- a/src/main/java/bio/terra/app/configuration/WebConfig.java +++ b/src/main/java/bio/terra/app/configuration/WebConfig.java @@ -36,6 +36,6 @@ public void configurePathMatch(PathMatchConfigurer configurer) { public void addResourceHandlers(ResourceHandlerRegistry registry) { registry .addResourceHandler("/webjars/swagger-ui-dist/**") - .addResourceLocations("classpath:/META-INF/resources/webjars/swagger-ui-dist/4.3.0/"); + .addResourceLocations("classpath:/META-INF/resources/webjars/swagger-ui-dist/5.18.2/"); } } diff --git a/src/main/java/bio/terra/common/PdaoLoadStatistics.java b/src/main/java/bio/terra/common/PdaoLoadStatistics.java index e988153b9b..4e6270ff50 100644 --- a/src/main/java/bio/terra/common/PdaoLoadStatistics.java +++ b/src/main/java/bio/terra/common/PdaoLoadStatistics.java @@ -1,5 +1,6 @@ package bio.terra.common; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.cloud.bigquery.JobStatistics; import java.time.Instant; @@ -9,7 +10,11 @@ public class PdaoLoadStatistics { private final Instant startTime; private final Instant endTime; - public PdaoLoadStatistics(long badRecords, long rowCount, Instant startTime, Instant endTime) { + public PdaoLoadStatistics( + @JsonProperty("badRecords") long badRecords, + @JsonProperty("rowCount") long rowCount, + @JsonProperty("startTime") Instant startTime, + @JsonProperty("endTime") Instant endTime) { this.badRecords = badRecords; this.rowCount = rowCount; this.startTime = startTime; diff --git a/src/main/java/bio/terra/service/dataset/SchemaValidationContext.java b/src/main/java/bio/terra/service/dataset/SchemaValidationContext.java new file mode 100644 index 0000000000..16fd43a897 --- /dev/null +++ b/src/main/java/bio/terra/service/dataset/SchemaValidationContext.java @@ -0,0 +1,395 @@ +package bio.terra.service.dataset; + +import bio.terra.common.PdaoConstant; +import bio.terra.common.ValidationUtils; +import bio.terra.model.AssetModel; +import bio.terra.model.AssetTableModel; +import bio.terra.model.ColumnModel; +import bio.terra.model.DatePartitionOptionsModel; +import bio.terra.model.IntPartitionOptionsModel; +import bio.terra.model.RelationshipModel; +import bio.terra.model.TableDataType; +import bio.terra.model.TableModel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.springframework.validation.Errors; + +/** + * SchemaValidationContext represents shared functionality used to validate schemas for datasets. + * Much of this functionality is shared between DatasetRequestValidator (used for creating dataset + * schemas) and DatasetSchemaUpdateValidator (used for updating dataset schemas). + */ +public class SchemaValidationContext { + + private static final String PRIMARY_KEY = "PrimaryKey"; + + private final Map> tableColumnMap = new HashMap<>(); + private final Map> tableArrayColumns = new HashMap<>(); + private final Set relationshipNameSet = new HashSet<>(); + private final String fieldName; + + SchemaValidationContext(String fieldName) { + this.fieldName = fieldName; + } + + static SchemaValidationContext forUpdate() { + return new SchemaValidationContext("changes"); + } + + static SchemaValidationContext forCreate() { + return new SchemaValidationContext("schema"); + } + + String getFieldName() { + return fieldName; + } + + void addTable(String tableName, List columns) { + HashSet colNames = new HashSet<>(); + HashSet arrayCols = new HashSet<>(); + + for (ColumnModel col : columns) { + colNames.add(col.getName()); + if (col.isArrayOf()) { + arrayCols.add(col.getName()); + } + } + + tableColumnMap.put(tableName, colNames); + tableArrayColumns.put(tableName, arrayCols); + } + + void addRelationship(String relationshipName) { + relationshipNameSet.add(relationshipName); + } + + boolean isValidTable(String tableName) { + return tableColumnMap.containsKey(tableName); + } + + boolean isValidTableColumn(String tableName, String columnName) { + return isValidTable(tableName) && tableColumnMap.get(tableName).contains(columnName); + } + + boolean isArrayColumn(String tableName, String columnName) { + return isValidTableColumn(tableName, columnName) + && tableArrayColumns.get(tableName).contains(columnName); + } + + boolean isValidRelationship(String relationshipName) { + return relationshipNameSet.contains(relationshipName); + } + + public void validateTable(TableModel table, Errors errors) { + String tableName = table.getName(); + List columns = table.getColumns(); + List primaryKeyList = table.getPrimaryKey(); + List columnNames = new ArrayList<>(); + if (columns.isEmpty()) { + errors.rejectValue( + getFieldName(), + "IncompleteSchemaDefinition", + "Each table must contain at least one column"); + } else { + columns.stream().map(ColumnModel::getName).forEach(columnNames::add); + } + + if (tableName != null) { + validateDataTypes(columns, errors); + + if (ValidationUtils.hasDuplicates(columnNames)) { + List duplicates = ValidationUtils.findDuplicates(columnNames); + errors.rejectValue( + getFieldName(), + "DuplicateColumnNames", + String.format("Duplicate columns: %s", String.join(", ", duplicates))); + } + if (primaryKeyList != null && !new HashSet<>(columnNames).containsAll(primaryKeyList)) { + List missingKeys = new ArrayList<>(primaryKeyList); + missingKeys.removeAll(columnNames); + errors.rejectValue( + getFieldName(), + "MissingPrimaryKeyColumn", + String.format("Expected column(s): %s", String.join(", ", missingKeys))); + } + for (ColumnModel columnModel : table.getColumns()) { + if (primaryKeyList != null && primaryKeyList.contains(columnModel.getName())) { + validateColumnType(errors, columnModel, PRIMARY_KEY); + } + validateColumnMode(errors, columnModel); + } + + addTable(tableName, columns); + } + + TableModel.PartitionModeEnum mode = table.getPartitionMode(); + DatePartitionOptionsModel dateOptions = table.getDatePartitionOptions(); + IntPartitionOptionsModel intOptions = table.getIntPartitionOptions(); + + if (mode == TableModel.PartitionModeEnum.DATE) { + if (dateOptions == null) { + errors.rejectValue( + getFieldName(), + "MissingDatePartitionOptions", + "datePartitionOptions must be specified when using 'date' partitionMode"); + } else { + validateDatePartitionOptions(dateOptions, columns, errors); + } + } else if (dateOptions != null) { + errors.rejectValue( + getFieldName(), + "InvalidDatePartitionOptions", + "datePartitionOptions can only be specified when using 'date' partitionMode"); + } + + if (mode == TableModel.PartitionModeEnum.INT) { + if (intOptions == null) { + errors.rejectValue( + getFieldName(), + "MissingIntPartitionOptions", + "intPartitionOptions must be specified when using 'int' partitionMode"); + } else { + validateIntPartitionOptions(intOptions, columns, errors); + } + } else if (intOptions != null) { + errors.rejectValue( + getFieldName(), + "InvalidIntPartitionOptions", + "intPartitionOptions can only be specified when using 'int' partitionMode"); + } + } + + private void validateDataTypes(List columns, Errors errors) { + List invalidColumns = new ArrayList<>(); + for (ColumnModel column : columns) { + // spring defaults user input not belonging to the TableDataType enum to null + if (column.getDatatype() == null) { + invalidColumns.add(column); + } + } + if (!invalidColumns.isEmpty()) { + errors.rejectValue( + getFieldName(), + "InvalidDatatype", + "invalid datatype in table column(s): " + + invalidColumns.stream().map(ColumnModel::getName).collect(Collectors.joining(", ")) + + ", DataTypes must be lowercase, valid DataTypes are " + + Arrays.toString(TableDataType.values())); + } + } + + // Primary Keys and Foreign Keys cannot be filerefs or dirrefs and Primary keys cannot be arrays + private void validateColumnType(Errors errors, ColumnModel columnModel, String keyType) { + if (keyType.equals(PRIMARY_KEY) && columnModel.isArrayOf()) { + rejectKey(errors, keyType, columnModel.getName(), "array"); + } + + Set invalidTypes = Set.of(TableDataType.DIRREF, TableDataType.FILEREF); + if (columnModel.getDatatype() != null && invalidTypes.contains(columnModel.getDatatype())) { + rejectKey(errors, keyType, columnModel.getName(), columnModel.getDatatype().toString()); + } + if (PRIMARY_KEY.equals(keyType) && Boolean.FALSE.equals(columnModel.isRequired())) { + errors.rejectValue( + getFieldName(), + "OptionalPrimaryKeyColumn", + String.format("A %s column cannot be marked as not required", PRIMARY_KEY)); + } + } + + private void validateColumnMode(Errors errors, ColumnModel columnModel) { + // Explicitly check if isRequired is true to avoid a null pointer exception. + // isArrayOf has a default value set in the open-api spec so it does not require + // the same handling. + if (Boolean.TRUE.equals(columnModel.isRequired()) && columnModel.isArrayOf()) { + errors.rejectValue( + getFieldName(), + "InvalidColumnMode", + String.format("Array column %s cannot be marked as required", columnModel.getName())); + } + } + + private void validateDatePartitionOptions( + DatePartitionOptionsModel options, List columns, Errors errors) { + String targetColumn = options.getColumn(); + + if (targetColumn == null) { + errors.rejectValue(getFieldName(), "MissingDatePartitionColumnName"); + } else if (!targetColumn.equals(PdaoConstant.PDAO_INGEST_DATE_COLUMN_ALIAS)) { + Optional matchingColumn = + columns.stream().filter(c -> targetColumn.equals(c.getName())).findFirst(); + + if (matchingColumn.isPresent()) { + TableDataType colType = matchingColumn.get().getDatatype(); + + if (colType != TableDataType.DATE && colType != TableDataType.TIMESTAMP) { + errors.rejectValue( + getFieldName(), + "InvalidDatePartitionColumnType", + "partitionColumn in datePartitionOptions must refer to a DATE or TIMESTAMP column"); + } + } else { + errors.rejectValue( + getFieldName(), "InvalidDatePartitionColumnName", "No such column: " + targetColumn); + } + } + } + + private void validateIntPartitionOptions( + IntPartitionOptionsModel options, List columns, Errors errors) { + String targetColumn = options.getColumn(); + + if (targetColumn == null) { + errors.rejectValue(getFieldName(), "MissingIntPartitionColumnName"); + } else { + Optional matchingColumn = + columns.stream().filter(c -> targetColumn.equals(c.getName())).findFirst(); + + if (matchingColumn.isPresent()) { + TableDataType colType = matchingColumn.get().getDatatype(); + + if (colType != TableDataType.INTEGER && colType != TableDataType.INT64) { + errors.rejectValue( + getFieldName(), + "InvalidIntPartitionColumnType", + "partitionColumn in intPartitionOptions must refer to an INTEGER or INT64 column"); + } + } else { + errors.rejectValue( + getFieldName(), "InvalidIntPartitionColumnName", "No such column: " + targetColumn); + } + } + + Long min = options.getMin(); + Long max = options.getMax(); + Long interval = options.getInterval(); + + if (min == null || max == null || interval == null) { + errors.rejectValue( + getFieldName(), + "MissingIntPartitionOptions", + "intPartitionOptions must specify min, max, and interval"); + } else { + if (max <= min) { + errors.rejectValue( + getFieldName(), + "InvalidIntPartitionRange", + "Max partition value must be larger than min partition value"); + } + if (interval <= 0) { + errors.rejectValue( + getFieldName(), "InvalidIntPartitionInterval", "Partition interval must be >= 1"); + } + if (max > min && interval > 0 && (max - min) / interval > 4000L) { + errors.rejectValue( + getFieldName(), + "TooManyIntPartitions", + "Cannot configure more than 4K partitions through min, max, and interval"); + } + } + } + + private void rejectKey(Errors errors, String keyType, String columnName, String type) { + errors.rejectValue( + getFieldName(), + String.format("Invalid%s", keyType), + String.format("%s %s cannot be a column with %s type", keyType, columnName, type)); + } + + void validateRelationship( + RelationshipModel relationship, List tables, Errors errors) { + ArrayList> validationErrors = + ValidationUtils.getRelationshipValidationErrors(relationship, tables); + validationErrors.forEach(e -> rejectValues(errors, e)); + + String relationshipName = relationship.getName(); + if (relationshipName != null) { + addRelationship(relationshipName); + } + } + + private void rejectValues(Errors errors, Map errorMap) { + for (var entry : errorMap.entrySet()) { + var errorCode = entry.getKey(); + var errorMessage = entry.getValue(); + errors.rejectValue(getFieldName(), errorCode, errorMessage); + } + } + + private void validateAssetTable(AssetTableModel assetTable, Errors errors) { + + String tableName = assetTable.getName(); + List columnNames = assetTable.getColumns(); + if (tableName != null && columnNames != null) { + // An empty list acts like a wildcard to include all columns from a table in the asset + // specification. + if (columnNames.isEmpty()) { + if (!isValidTable(tableName)) { + errors.rejectValue( + getFieldName(), "InvalidAssetTable", "Invalid asset table: " + tableName); + } + } else { + columnNames.forEach( + (columnName) -> { + if (!isValidTableColumn(tableName, columnName)) { + errors.rejectValue( + getFieldName(), + "InvalidAssetTableColumn", + "Invalid asset table: " + tableName + " column: " + columnName); + } + }); + } + } + } + + void validateAsset(AssetModel asset, Errors errors) { + List assetTables = asset.getTables(); + + String rootTable = asset.getRootTable(); + String rootColumn = asset.getRootColumn(); + + if (assetTables != null) { + boolean hasRootTable = false; + for (AssetTableModel assetTable : assetTables) { + validateAssetTable(assetTable, errors); + if (assetTable.getName().equals(rootTable)) { + if (!isValidTableColumn(rootTable, rootColumn)) { + errors.rejectValue( + getFieldName(), + "InvalidRootColumn", + "Invalid root table column. Table: " + rootTable + " Column: " + rootColumn); + } else if (isArrayColumn(rootTable, rootColumn)) { + errors.rejectValue( + getFieldName(), + "InvalidArrayRootColumn", + "Invalid use of array column as asset root. Table: " + + rootTable + + " Column: " + + rootColumn); + } + hasRootTable = true; + } + } + if (!hasRootTable) { + errors.rejectValue(getFieldName(), "NoRootTable"); + } + } + + List follows = asset.getFollow(); + if (follows != null) { + if (follows.stream().anyMatch(relationshipName -> !isValidRelationship(relationshipName))) { + errors.rejectValue(getFieldName(), "InvalidFollowsRelationship"); + } + } + // TODO: There is another validation that can be done here to make sure that the graph is + // connected that has + // been left out to avoid complexity before we know if we're going keep using this or not. + } +} diff --git a/src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStep.java b/src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStep.java index 286ab06639..2bf5c0b24c 100644 --- a/src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStep.java +++ b/src/main/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStep.java @@ -1,15 +1,20 @@ package bio.terra.service.dataset.flight.create; +import bio.terra.common.exception.ErrorReportException; import bio.terra.service.auth.iam.IamService; import bio.terra.service.dataset.flight.DatasetWorkingMapKeys; import bio.terra.stairway.FlightContext; import bio.terra.stairway.FlightMap; import bio.terra.stairway.Step; import bio.terra.stairway.StepResult; +import bio.terra.stairway.StepStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** The step is only meant to be invoked for GCP backed datasets. */ public class CreateDatasetRegisterIngestServiceAccountStep implements Step { - + private static final Logger logger = + LoggerFactory.getLogger(CreateDatasetRegisterIngestServiceAccountStep.class); private final IamService iamService; public CreateDatasetRegisterIngestServiceAccountStep(IamService iamService) { @@ -22,7 +27,18 @@ public StepResult doStep(FlightContext context) throws InterruptedException { String datasetServiceAccount = workingMap.get(DatasetWorkingMapKeys.SERVICE_ACCOUNT_EMAIL, String.class); - iamService.registerUser(datasetServiceAccount); + try { + iamService.registerUser(datasetServiceAccount); + } catch (ErrorReportException e) { + // This is the super class type of IamExceptions (i.e. IAmNotFoundException) and ApiExceptions + // Catch transient errors from Sam, where the service account created in the previous step + // may not be ready to use yet. Do not catch InterruptedExceptions. + logger.warn( + String.format( + "Service account, %s, may not be ready to use yet. Retrying.", datasetServiceAccount), + e); + return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, e); + } return StepResult.getStepResultSuccess(); } diff --git a/src/main/java/bio/terra/service/dataset/flight/create/DatasetCreateFlight.java b/src/main/java/bio/terra/service/dataset/flight/create/DatasetCreateFlight.java index 24aa4e8883..6123ec82f3 100644 --- a/src/main/java/bio/terra/service/dataset/flight/create/DatasetCreateFlight.java +++ b/src/main/java/bio/terra/service/dataset/flight/create/DatasetCreateFlight.java @@ -102,7 +102,9 @@ public DatasetCreateFlight(FlightMap inputParameters, Object applicationContext) // Create the service account to use to ingest data and register it in Terra if (datasetRequest.isDedicatedIngestServiceAccount()) { addStep(new CreateDatasetCreateIngestServiceAccountStep(resourceService, datasetRequest)); - addStep(new CreateDatasetRegisterIngestServiceAccountStep(iamService)); + addStep( + new CreateDatasetRegisterIngestServiceAccountStep(iamService), + getDefaultExponentialBackoffRetryRule()); } } diff --git a/src/main/java/bio/terra/service/filedata/azure/tables/TableDirectoryDao.java b/src/main/java/bio/terra/service/filedata/azure/tables/TableDirectoryDao.java index afc9bdb88d..c442c883a3 100644 --- a/src/main/java/bio/terra/service/filedata/azure/tables/TableDirectoryDao.java +++ b/src/main/java/bio/terra/service/filedata/azure/tables/TableDirectoryDao.java @@ -440,7 +440,7 @@ public void addEntriesToSnapshot( // Store the batch of entries. This will override existing entries, // but that is not the typical case and it is lower cost just overwrite // rather than retrieve to avoid the write. - batchStoreDirectoryEntry(snapshotTableServiceClient, snapshotId, snapshotEntries); + storeDirectoryEntries(snapshotTableServiceClient, snapshotId, snapshotEntries); return null; })); @@ -498,4 +498,15 @@ void batchStoreDirectoryEntry( createEntityForPath(tableClient, snapshotId, tableName, snapshotEntry))) .toList()); } + + // Store the directory entries in the snapshot table sequentially, without a thread pool + void storeDirectoryEntries( + TableServiceClient snapshotTableServiceClient, + UUID snapshotId, + List snapshotEntries) { + String tableName = StorageTableName.SNAPSHOT.toTableName(snapshotId); + TableClient tableClient = snapshotTableServiceClient.getTableClient(tableName); + snapshotEntries.forEach( + snapshotEntry -> createEntityForPath(tableClient, snapshotId, tableName, snapshotEntry)); + } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 7862a56435..42234e607e 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -145,7 +145,6 @@ azure.retryTimeoutSeconds=3600 rbs.enabled=false rbs.poolId=datarepo_v1 rbs.instanceUrl=https://buffer.tools.integ.envs.broadinstitute.org -rbs.clientCredentialFilePath=/tmp/buffer-client-sa-account.json ecm.basePath=https://externalcreds.dsde-dev.broadinstitute.org ecm.rasIssuer=https://stsstg.nih.gov usermetrics.appId=datarepo diff --git a/src/main/resources/db/changelog.xml b/src/main/resources/db/changelog.xml index 98a9ef5fd7..a5d11e9e80 100644 --- a/src/main/resources/db/changelog.xml +++ b/src/main/resources/db/changelog.xml @@ -90,4 +90,5 @@ + diff --git a/src/main/resources/db/changesets/20241127_asset_column_asset_id_index.yaml b/src/main/resources/db/changesets/20241127_asset_column_asset_id_index.yaml new file mode 100644 index 0000000000..93383ecad0 --- /dev/null +++ b/src/main/resources/db/changesets/20241127_asset_column_asset_id_index.yaml @@ -0,0 +1,11 @@ +databaseChangeLog: + - changeSet: + id: asset_column_asset_id_index + author: rjohanek + changes: + - createIndex: + indexName: asset_column_asset_id_idx + tableName: asset_column + columns: + - column: + name: asset_id diff --git a/src/main/resources/templates/index.html b/src/main/resources/templates/index.html index d4754f376d..3cca45f92f 100644 --- a/src/main/resources/templates/index.html +++ b/src/main/resources/templates/index.html @@ -121,7 +121,7 @@ ac.classList.add('hidden'); } else { // Set the correct client ID value using the native input component (needed for newer react versions used by Swagger UI) - var clientIdInput = ac.querySelector('#client_id'); + var clientIdInput = ac.querySelector('#client_id_authorizationCode'); if (clientIdInput) { var nativeInputValueSetter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set; nativeInputValueSetter.call(clientIdInput, clientIds[scheme]); @@ -197,4 +197,4 @@ } - \ No newline at end of file + diff --git a/src/test/java/bio/terra/service/dataset/DatasetSchemaUpdateValidatorTest.java b/src/test/java/bio/terra/service/dataset/DatasetSchemaUpdateValidatorTest.java index 837d4c1981..9e36f2aaa0 100644 --- a/src/test/java/bio/terra/service/dataset/DatasetSchemaUpdateValidatorTest.java +++ b/src/test/java/bio/terra/service/dataset/DatasetSchemaUpdateValidatorTest.java @@ -17,8 +17,11 @@ import bio.terra.model.ColumnModel; import bio.terra.model.DatasetSchemaUpdateModel; import bio.terra.model.DatasetSchemaUpdateModelChanges; +import bio.terra.model.DatePartitionOptionsModel; import bio.terra.model.ErrorModel; +import bio.terra.model.IntPartitionOptionsModel; import bio.terra.model.TableDataType; +import bio.terra.model.TableModel; import bio.terra.service.auth.iam.IamService; import bio.terra.service.filedata.FileService; import bio.terra.service.job.JobService; @@ -43,6 +46,7 @@ @ContextConfiguration( classes = { DatasetSchemaUpdateValidator.class, + DatasetRequestValidator.class, DatasetsApiController.class, ApiValidationExceptionHandler.class }) @@ -61,23 +65,25 @@ class DatasetSchemaUpdateValidatorTest { @MockBean private IngestRequestValidator ingestRequestValidator; @MockBean private AssetModelValidator assetModelValidator; @MockBean private DataDeletionRequestValidator dataDeletionRequestValidator; - @MockBean private DatasetRequestValidator datasetRequestValidator; @BeforeEach void setup() throws Exception { when(ingestRequestValidator.supports(any())).thenReturn(true); - when(datasetRequestValidator.supports(any())).thenReturn(true); when(assetModelValidator.supports(any())).thenReturn(true); when(dataDeletionRequestValidator.supports(any())).thenReturn(true); } private ErrorModel expectBadDatasetUpdateRequest(DatasetSchemaUpdateModel datasetRequest) throws Exception { + return expectBadDatasetUpdateRequest(TestUtils.mapToJson(datasetRequest)); + } + + private ErrorModel expectBadDatasetUpdateRequest(String datasetRequest) throws Exception { MvcResult result = mvc.perform( post("/api/repository/v1/datasets/{id}/updateSchema", UUID.randomUUID()) .contentType(MediaType.APPLICATION_JSON) - .content(TestUtils.mapToJson(datasetRequest))) + .content(datasetRequest)) .andExpect(status().is4xxClientError()) .andReturn(); @@ -106,7 +112,7 @@ void testSchemaUpdateWithDuplicateTables() throws Exception { newTableName, List.of(newTableColumnName))))); ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); assertThat( - "Required column throws error", + "Duplicate table throws error", errorModel.getErrorDetail().get(0), containsString("DuplicateTableNames")); } @@ -150,4 +156,343 @@ void testSchemaUpdateWithDuplicateRelationships() throws Exception { errorModel.getErrorDetail().get(0), containsString("DuplicateRelationshipNames")); } + + @Test + void testNoColumns() throws Exception { + String newTableName = "new_table"; + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("No columns with new table") + .changes( + new DatasetSchemaUpdateModelChanges() + .addTables(List.of(DatasetFixtures.tableModel(newTableName, List.of())))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "No columns with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("IncompleteSchemaDefinition")); + } + + @Test + void testInvalidDatatypeColumns() throws Exception { + // Load bad data type from JSON b/c couldn't set it in the model + String json = TestUtils.loadJson("update-schema-bad-data-type.json"); + ErrorModel errorModel = expectBadDatasetUpdateRequest(json); + String responseBody = String.join(", ", errorModel.getErrorDetail()); + assertThat( + "Invalid DataTypes are logged and returned", + responseBody, + containsString( + "invalid datatype in table column(s): bad_column, " + + "DataTypes must be lowercase, valid DataTypes are [string, boolean, bytes, date, datetime, dirref, fileref, " + + "float, float64, integer, int64, numeric, record, text, time, timestamp]")); + } + + @Test + void testDuplicateColumns() throws Exception { + String newTableName = "new_table"; + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Duplicate columns with new table") + .changes( + new DatasetSchemaUpdateModelChanges() + .addTables( + List.of( + DatasetFixtures.tableModel( + newTableName, List.of("column1", "column1"))))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Duplicate column with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("DuplicateColumnNames")); + } + + @Test + void testMissingPrimaryKeyColumn() throws Exception { + String newTableName = "new_table"; + String primaryKeyColumnName = "primary_key"; + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of("column1")); + tableModel.addPrimaryKeyItem(primaryKeyColumnName); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Missing primary key column with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Missing primary key column with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("MissingPrimaryKeyColumn")); + } + + @Test + void testOptionalPrimaryKeyColumn() throws Exception { + String newTableName = "new_table"; + String primaryKeyColumnName = "primary_key"; + ColumnModel newColumn = + DatasetFixtures.columnModel(primaryKeyColumnName, TableDataType.STRING, false, false); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addPrimaryKeyItem(primaryKeyColumnName); + tableModel.addColumnsItem(newColumn); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Optional primary key column with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Optional primary key column with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("OptionalPrimaryKeyColumn")); + } + + @Test + void testInvalidColumnMode() throws Exception { + String newTableName = "new_table"; + ColumnModel newColumn = + DatasetFixtures.columnModel("column1", TableDataType.STRING, false, true); + newColumn.arrayOf(true); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem(newColumn); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid Column Mode with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid Column Mode with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidColumnMode")); + } + + @Test + void testMissingDatePartitionOptions() throws Exception { + String newTableName = "new_table"; + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of("column1")); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.DATE); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Missing date partition options with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Missing date partition options with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("MissingDatePartitionOptions")); + } + + @Test + void testMissingDatePartitionColumnName() throws Exception { + String newTableName = "new_table"; + DatePartitionOptionsModel datePartitionOptions = new DatePartitionOptionsModel(); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of("column1")); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.DATE); + tableModel.setDatePartitionOptions(datePartitionOptions); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Missing date partition column name with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Missing date partition column name with new table throws error", + // first error is NotNull error for column, second is MissingDatePartitionColumnName + errorModel.getErrorDetail().get(1), + containsString("MissingDatePartitionColumnName")); + } + + @Test + void testInvalidDatePartitionColumnType() throws Exception { + String newTableName = "new_table"; + String newColumnName = "column1"; + DatePartitionOptionsModel datePartitionOptions = + new DatePartitionOptionsModel().column(newColumnName); + ColumnModel newColumn = + DatasetFixtures.columnModel(newColumnName, TableDataType.STRING, false, true); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem(newColumn); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.DATE); + tableModel.setDatePartitionOptions(datePartitionOptions); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid date partition column type with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid date partition column type with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidDatePartitionColumnType")); + } + + @Test + void testInvalidDatePartitionColumnName() throws Exception { + String newTableName = "new_table"; + DatePartitionOptionsModel datePartitionOptions = + new DatePartitionOptionsModel().column("column1"); + ColumnModel newColumn = + DatasetFixtures.columnModel("column2", TableDataType.STRING, false, true); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem(newColumn); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.DATE); + tableModel.setDatePartitionOptions(datePartitionOptions); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid date partition column name with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid date partition column name with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidDatePartitionColumnName")); + } + + @Test + void testInvalidDatePartitionOptions() throws Exception { + String newTableName = "new_table"; + DatePartitionOptionsModel datePartitionOptions = + new DatePartitionOptionsModel().column("column1"); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem( + DatasetFixtures.columnModel("column1", TableDataType.INTEGER, false, false)); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.INT); + tableModel.setDatePartitionOptions(datePartitionOptions); + IntPartitionOptionsModel intPartitionOptions = new IntPartitionOptionsModel(); + intPartitionOptions.min(1L); + intPartitionOptions.max(10L); + intPartitionOptions.interval(1L); + intPartitionOptions.column("column1"); + tableModel.setIntPartitionOptions(intPartitionOptions); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid date partition options with mode Int with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid date partition options with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidDatePartitionOptions")); + } + + @Test + void testMissingIntPartitionOptions() throws Exception { + String newTableName = "new_table"; + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of("column1")); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.INT); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Missing int partition options with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Missing int partition options with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("MissingIntPartitionOptions")); + } + + @Test + void testMissingIntPartitionColumnName() throws Exception { + String newTableName = "new_table"; + IntPartitionOptionsModel intPartitionOptions = new IntPartitionOptionsModel(); + intPartitionOptions.min(1L); + intPartitionOptions.max(10L); + intPartitionOptions.interval(1L); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of("column1")); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.INT); + tableModel.setIntPartitionOptions(intPartitionOptions); + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Missing int partition column name with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Missing int partition column name with new table throws error", + // first error is NotNull error for column, second is MissingIntPartitionColumnName + errorModel.getErrorDetail().get(1), + containsString("MissingIntPartitionColumnName")); + } + + @Test + void testInvalidIntPartitionColumnType() throws Exception { + String newTableName = "new_table"; + String newColumnName = "column1"; + IntPartitionOptionsModel intPartitionOptions = + new IntPartitionOptionsModel().column(newColumnName); + intPartitionOptions.min(1L); + intPartitionOptions.max(10L); + intPartitionOptions.interval(1L); + ColumnModel newColumn = + DatasetFixtures.columnModel(newColumnName, TableDataType.STRING, false, true); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem(newColumn); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.INT); + tableModel.setIntPartitionOptions(intPartitionOptions); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid int partition column type with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid int partition column type with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidIntPartitionColumnType")); + } + + @Test + void testInvalidIntPartitionColumnName() throws Exception { + String newTableName = "new_table"; + IntPartitionOptionsModel intPartitionOptions = new IntPartitionOptionsModel().column("column1"); + intPartitionOptions.min(1L); + intPartitionOptions.max(10L); + intPartitionOptions.interval(1L); + ColumnModel newColumn = + DatasetFixtures.columnModel("column2", TableDataType.INTEGER, false, true); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem(newColumn); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.INT); + tableModel.setIntPartitionOptions(intPartitionOptions); + + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid int partition column name with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid int partition column name with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidIntPartitionColumnName")); + } + + @Test + void testInvalidIntPartitionOptions() throws Exception { + String newTableName = "new_table"; + IntPartitionOptionsModel intPartitionOptions = new IntPartitionOptionsModel(); + intPartitionOptions.min(1L); + intPartitionOptions.max(10L); + intPartitionOptions.interval(1L); + intPartitionOptions.column("column1"); + DatePartitionOptionsModel datePartitionOptions = + new DatePartitionOptionsModel().column("column1"); + TableModel tableModel = DatasetFixtures.tableModel(newTableName, List.of()); + tableModel.addColumnsItem( + DatasetFixtures.columnModel("column1", TableDataType.DATE, false, false)); + tableModel.setPartitionMode(TableModel.PartitionModeEnum.DATE); + tableModel.setDatePartitionOptions(datePartitionOptions); + tableModel.setIntPartitionOptions(intPartitionOptions); + DatasetSchemaUpdateModel updateModel = + new DatasetSchemaUpdateModel() + .description("Invalid int partition options with mode Date with new table") + .changes(new DatasetSchemaUpdateModelChanges().addTables(List.of(tableModel))); + ErrorModel errorModel = expectBadDatasetUpdateRequest(updateModel); + assertThat( + "Invalid int partition options with new table throws error", + errorModel.getErrorDetail().get(0), + containsString("InvalidIntPartitionOptions")); + } } diff --git a/src/test/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStepTest.java b/src/test/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStepTest.java new file mode 100644 index 0000000000..84e24e6119 --- /dev/null +++ b/src/test/java/bio/terra/service/dataset/flight/create/CreateDatasetRegisterIngestServiceAccountStepTest.java @@ -0,0 +1,65 @@ +package bio.terra.service.dataset.flight.create; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import bio.terra.app.controller.exception.ApiException; +import bio.terra.common.category.Unit; +import bio.terra.service.auth.iam.IamService; +import bio.terra.service.auth.iam.exception.IamUnauthorizedException; +import bio.terra.service.dataset.flight.DatasetWorkingMapKeys; +import bio.terra.stairway.FlightContext; +import bio.terra.stairway.FlightMap; +import bio.terra.stairway.StepStatus; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@Tag(Unit.TAG) +@ExtendWith(MockitoExtension.class) +class CreateDatasetRegisterIngestServiceAccountStepTest { + @Mock private IamService iamService; + @Mock private FlightContext flightContext; + private CreateDatasetRegisterIngestServiceAccountStep step; + + @BeforeEach + void setup() { + step = new CreateDatasetRegisterIngestServiceAccountStep(iamService); + FlightMap workingMap = new FlightMap(); + workingMap.put(DatasetWorkingMapKeys.SERVICE_ACCOUNT_EMAIL, "email"); + when(flightContext.getWorkingMap()).thenReturn(workingMap); + } + + @Test + void doStep() throws InterruptedException { + assertThat(step.doStep(flightContext).getStepStatus(), equalTo(StepStatus.STEP_RESULT_SUCCESS)); + verify(iamService).registerUser("email"); + } + + @Test + void doStep_RetryIAmException() throws InterruptedException { + doThrow(new IamUnauthorizedException("Unauthorized")).when(iamService).registerUser("email"); + assertThat( + step.doStep(flightContext).getStepStatus(), equalTo(StepStatus.STEP_RESULT_FAILURE_RETRY)); + } + + @Test + void doStep_ApiException() throws InterruptedException { + doThrow(new ApiException("ApiException")).when(iamService).registerUser("email"); + assertThat( + step.doStep(flightContext).getStepStatus(), equalTo(StepStatus.STEP_RESULT_FAILURE_RETRY)); + } + + @Test + void doStep_NullPointerException() { + doThrow(new NullPointerException()).when(iamService).registerUser("email"); + assertThrows(NullPointerException.class, () -> step.doStep(flightContext)); + } +} diff --git a/src/test/java/bio/terra/service/dataset/flight/ingest/IngestUtilsTest.java b/src/test/java/bio/terra/service/dataset/flight/ingest/IngestUtilsTest.java index 75f8ddaa32..c6c58100d7 100644 --- a/src/test/java/bio/terra/service/dataset/flight/ingest/IngestUtilsTest.java +++ b/src/test/java/bio/terra/service/dataset/flight/ingest/IngestUtilsTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.Mockito.when; +import bio.terra.common.PdaoLoadStatistics; import bio.terra.common.category.Unit; import bio.terra.model.IngestRequestModel; import bio.terra.model.IngestRequestModel.FormatEnum; @@ -20,6 +21,7 @@ import bio.terra.stairway.FlightMap; import bio.terra.stairway.ShortUUID; import com.azure.storage.blob.BlobUrlParts; +import java.time.Instant; import java.util.UUID; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; @@ -193,4 +195,17 @@ private FlightMap createFlightMap(IngestRequestModel.UpdateStrategyEnum updateSt inputParameters.put(JobMapKeys.REQUEST.getKeyName(), ingestRequest); return inputParameters; } + + @Test + void putIngestStatistics() { + when(context.getWorkingMap()).thenReturn(new FlightMap()); + PdaoLoadStatistics ingestStatistics = + new PdaoLoadStatistics(123, 456, Instant.EPOCH, Instant.EPOCH); + IngestUtils.putIngestStatistics(context, ingestStatistics); + PdaoLoadStatistics actual = IngestUtils.getIngestStatistics(context); + assertEquals(ingestStatistics.getRowCount(), actual.getRowCount()); + assertEquals(ingestStatistics.getBadRecords(), actual.getBadRecords()); + assertEquals(ingestStatistics.getStartTime(), actual.getStartTime()); + assertEquals(ingestStatistics.getEndTime(), actual.getEndTime()); + } } diff --git a/src/test/java/bio/terra/service/filedata/azure/tables/TableDirectoryDaoTest.java b/src/test/java/bio/terra/service/filedata/azure/tables/TableDirectoryDaoTest.java index 24f9d478d9..f50c9edb9f 100644 --- a/src/test/java/bio/terra/service/filedata/azure/tables/TableDirectoryDaoTest.java +++ b/src/test/java/bio/terra/service/filedata/azure/tables/TableDirectoryDaoTest.java @@ -2,11 +2,12 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; import bio.terra.common.EmbeddedDatabaseTest; @@ -16,6 +17,7 @@ import bio.terra.service.filedata.google.firestore.FireStoreDirectoryEntry; import bio.terra.service.resourcemanagement.azure.AzureAuthService; import com.azure.core.http.rest.PagedIterable; +import com.azure.core.http.rest.Response; import com.azure.data.tables.TableClient; import com.azure.data.tables.TableServiceClient; import com.azure.data.tables.models.TableEntity; @@ -25,10 +27,9 @@ import java.util.List; import java.util.UUID; import java.util.stream.Stream; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -36,17 +37,16 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; -@RunWith(SpringRunner.class) @SpringBootTest @AutoConfigureMockMvc @ActiveProfiles({"google", "unittest"}) -@Category(Unit.class) +@Tag(Unit.TAG) @EmbeddedDatabaseTest -public class TableDirectoryDaoTest { +class TableDirectoryDaoTest { private static final String FULL_PATH = "/directory/file.json"; private static final UUID DATASET_ID = UUID.randomUUID(); + private static final UUID SNAPSHOT_ID = UUID.randomUUID(); private static final String PARTITION_KEY = DATASET_ID + " _dr_ directory"; private static final String ROW_KEY = " _dr_ directory file.json"; private static final String NONEXISTENT_PATH = "/directory/nonexistent.json"; @@ -60,8 +60,8 @@ public class TableDirectoryDaoTest { @MockBean private TableClient tableClient; @Autowired private TableDirectoryDao dao; - @Before - public void setUp() { + @BeforeEach + void setUp() { dao = spy(dao); when(authService.getTableServiceClient(any(), any(), any())).thenReturn(tableServiceClient); when(tableServiceClient.getTableClient(any())).thenReturn(tableClient); @@ -90,7 +90,7 @@ public void setUp() { } @Test - public void testRetrieveByPath() { + void testRetrieveByPath() { when(tableClient.getEntity(PARTITION_KEY, ROW_KEY)).thenReturn(entity); FireStoreDirectoryEntry response = dao.retrieveByPath( @@ -98,7 +98,7 @@ public void testRetrieveByPath() { DATASET_ID, StorageTableName.DATASET.toTableName(DATASET_ID), FULL_PATH); - assertEquals("The same entry is returned", directoryEntry, response); + assertThat("The same entry is returned", directoryEntry, equalTo(response)); when(tableClient.getEntity(PARTITION_KEY, NONEXISTENT_ROW_KEY)) .thenThrow(TableServiceException.class); @@ -108,11 +108,11 @@ public void testRetrieveByPath() { DATASET_ID, StorageTableName.DATASET.toTableName(DATASET_ID), NONEXISTENT_PATH); - assertNull("The entry does not exist", nonExistentEntry); + assertThat("The entry does not exist", nonExistentEntry, is(nullValue())); } @Test - public void testRetrieveByFileId() { + void testRetrieveByFileId() { PagedIterable mockPagedIterable = mock(PagedIterable.class); Iterator mockIterator = mock(Iterator.class); when(mockIterator.hasNext()).thenReturn(true, false); @@ -136,7 +136,7 @@ public void testRetrieveByFileId() { } @Test - public void testRetrieveByFileIdNotFound() { + void testRetrieveByFileIdNotFound() { PagedIterable mockPagedIterable = mock(PagedIterable.class); Iterator mockIterator = mock(Iterator.class); when(mockIterator.hasNext()).thenReturn(false); @@ -146,11 +146,11 @@ public void testRetrieveByFileIdNotFound() { FireStoreDirectoryEntry response = dao.retrieveById( tableServiceClient, StorageTableName.DATASET.toTableName(DATASET_ID), "nonexistentId"); - assertNull("The entry does not exist", response); + assertThat("The entry does not exist", response, is(nullValue())); } @Test - public void validateRefIdsFindsMissingRecords() { + void validateRefIdsFindsMissingRecords() { PagedIterable mockPagedIterable = mock(PagedIterable.class); Iterator mockIterator = mock(Iterator.class); when(mockIterator.hasNext()).thenReturn(false); @@ -160,19 +160,32 @@ public void validateRefIdsFindsMissingRecords() { String missingId = UUID.randomUUID().toString(); List refIds = List.of(missingId); List response = dao.validateRefIds(tableServiceClient, DATASET_ID, refIds); - assertEquals(response.get(0), missingId); + assertThat(response.get(0), equalTo(missingId)); } @Test - public void testEnumerateDirectory() { + void testEnumerateDirectory() { PagedIterable mockPagedIterable = mock(PagedIterable.class); - Stream mockStream = List.of(entity).stream(); + Stream mockStream = Stream.of(entity); when(mockPagedIterable.stream()).thenReturn(mockStream); when(tableClient.listEntities(any(), any(), any())).thenReturn(mockPagedIterable); List response = dao.enumerateDirectory( tableServiceClient, StorageTableName.DATASET.toTableName(DATASET_ID), FILE_ID); - assertEquals(response.get(0), directoryEntry); + assertThat(response.get(0), equalTo(directoryEntry)); + } + + @Test + void storeDirectoryEntries() { + FireStoreDirectoryEntry entry1 = new FireStoreDirectoryEntry().name("file1").path("path1"); + FireStoreDirectoryEntry entry2 = new FireStoreDirectoryEntry().name("file2").path("path2"); + Response responseMock = mock(Response.class); + when(responseMock.getStatusCode()).thenReturn(200); + when(tableClient.upsertEntityWithResponse(any(), any(), any(), any())).thenReturn(responseMock); + + dao.storeDirectoryEntries(tableServiceClient, SNAPSHOT_ID, List.of(entry1, entry2)); + + Mockito.verify(tableClient, times(2)).upsertEntityWithResponse(any(), any(), any(), any()); } } diff --git a/src/test/resources/update-schema-bad-data-type.json b/src/test/resources/update-schema-bad-data-type.json new file mode 100644 index 0000000000..f941b08757 --- /dev/null +++ b/src/test/resources/update-schema-bad-data-type.json @@ -0,0 +1,25 @@ +{ + "description":"Test invalid data type", + "changes":{ + "addTables":[ + { + "name":"new_table", + "columns":[ + { + "name":"bad_column", + "datatype":"badDataType", + "array_of":false, + "required":false + } + ], + "primaryKey":null, + "partitionMode":"none", + "datePartitionOptions":null, + "intPartitionOptions":null, + "rowCount":null + } + ], + "addColumns":null, + "addRelationships":null + } +} diff --git a/tools/rbs-bearer-token.sh b/tools/rbs-bearer-token.sh old mode 100755 new mode 100644 index f4a25b72d0..d0fab07666 --- a/tools/rbs-bearer-token.sh +++ b/tools/rbs-bearer-token.sh @@ -55,4 +55,4 @@ printf "\n" echo "Use the above bearer token here: ${RBS_SWAGGER_URL}" echo "With pool Id ${RBS_POOL_ID} (Note: this may be out of date. Get the latest from the RBS repo: https://github.com/DataBiosphere/terra-resource-buffer/tree/master/src/main/resources/config)" printf "\n" -echo "NOTE: Make sure to log back in with your desired user (i.e. gcloud auth login OR glcoud auth set account )" \ No newline at end of file +echo "NOTE: Make sure to log back in with your desired user (i.e. gcloud auth login OR glcoud auth set account )"