diff --git a/packages/client/hmi-client/src/services/project.ts b/packages/client/hmi-client/src/services/project.ts index 35f39e1474..e192d4b0cb 100644 --- a/packages/client/hmi-client/src/services/project.ts +++ b/packages/client/hmi-client/src/services/project.ts @@ -8,7 +8,14 @@ import { b64EncodeUnicode } from '@/utils/binary'; import DatasetIcon from '@/assets/svg/icons/dataset.svg?component'; import { Component } from 'vue'; import * as EventService from '@/services/event'; -import { AssetType, EventType, PermissionRelationships, Project } from '@/types/Types'; +import { + AssetType, + Code, + EventType, + PermissionRelationships, + Project, + ProjectAsset +} from '@/types/Types'; /** * Create a project @@ -23,9 +30,14 @@ async function create( userId: Project['userId'] = '' ): Promise { try { - const response = await API.post( - `/projects?name=${name}&description=${description}&userId=${userId}` - ); + const project: Project = { + name, + description, + userId, + projectAssets: [] as ProjectAsset[], + codeAssets: [] as Code[] + }; + const response = await API.post(`/projects`, project); const { status, data } = response; if (status !== 201) return null; return data ?? null; diff --git a/packages/client/hmi-client/src/types/Types.ts b/packages/client/hmi-client/src/types/Types.ts index 66c062a9fb..4e77eff771 100644 --- a/packages/client/hmi-client/src/types/Types.ts +++ b/packages/client/hmi-client/src/types/Types.ts @@ -165,7 +165,6 @@ export interface Dataset extends TerariumAsset { metadata?: any; source?: string; grounding?: Grounding; - project?: Project; } export interface DatasetColumn { @@ -405,7 +404,6 @@ export interface Project extends TerariumAsset { */ projectAssets: ProjectAsset[]; codeAssets: Code[]; - datasetAssets: Dataset[]; metadata?: { [index: string]: string }; publicProject?: boolean; userPermission?: string; diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java index 356ba3d2c2..7342250487 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetController.java @@ -148,7 +148,7 @@ public ResponseEntity> getDatasets( if (query == null) { return ResponseEntity.ok(datasetService.getAssets(page, pageSize)); } else { - return ResponseEntity.ok(datasetService.searchAssets(page, pageSize, query)); + return ResponseEntity.ok(datasetService.getAssets(page, pageSize, query)); } } catch (final IOException e) { @@ -221,7 +221,7 @@ public ResponseEntity getDataset(@PathVariable("id") final UUID id) { } return dataset.map(ResponseEntity::ok) .orElseGet(() -> ResponseEntity.notFound().build()); - } catch (final Exception e) { + } catch (final IOException e) { final String error = "Unable to get dataset"; log.error(error, e); throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error); @@ -476,7 +476,7 @@ public ResponseEntity getDownloadURL( if (dataset.isEmpty()) { throw new ResponseStatusException(org.springframework.http.HttpStatus.NOT_FOUND, "Dataset not found"); } - } catch (final Exception e) { + } catch (final IOException e) { final String error = "Unable to get dataset"; log.error(error, e); throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java index 4f2a84fe0f..9de7226258 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectController.java @@ -36,7 +36,6 @@ import software.uncharted.terarium.hmiserver.models.dataservice.AssetType; import software.uncharted.terarium.hmiserver.models.dataservice.ResponseDeleted; import software.uncharted.terarium.hmiserver.models.dataservice.code.Code; -import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; import software.uncharted.terarium.hmiserver.models.dataservice.project.Project; import software.uncharted.terarium.hmiserver.models.dataservice.project.ProjectAsset; import software.uncharted.terarium.hmiserver.models.permissions.PermissionGroup; @@ -46,7 +45,6 @@ import software.uncharted.terarium.hmiserver.service.CurrentUserService; import software.uncharted.terarium.hmiserver.service.UserService; import software.uncharted.terarium.hmiserver.service.data.CodeService; -import software.uncharted.terarium.hmiserver.service.data.DatasetService; import software.uncharted.terarium.hmiserver.service.data.ITerariumAssetService; import software.uncharted.terarium.hmiserver.service.data.ProjectAssetService; import software.uncharted.terarium.hmiserver.service.data.ProjectService; @@ -80,29 +78,10 @@ public class ProjectController { final CodeService codeService; - final DatasetService datasetService; - final UserService userService; final ObjectMapper objectMapper; - static final String WELCOME_MESSAGE = - """ -
-

Hey there!

-

This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:

-
-
    -
  • Upload stuff: Upload documents, models, code or datasets with the green button in the bottom left corner.
    • -
  • Explore and add: Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.
    • -
  • Build a model: Create a model that fits just what you need.
    • -
  • Create a workflow: Connect resources with operators so you can focus on the science and not the plumbing.
    • -
-
-

Feel free to erase this text and make it your own.

-
- """; - // -------------------------------------------------------------------------- // Basic Project Operations // -------------------------------------------------------------------------- @@ -381,15 +360,26 @@ public ResponseEntity deleteProject(@PathVariable("id") final U }) @PostMapping @Secured(Roles.USER) - public ResponseEntity createProject( - @RequestParam("name") final String name, - @RequestParam("description") final String description, - @RequestParam("userId") final String userId) { - Project project = (Project) - new Project().setUserId(userId).setDescription(description).setName(name); - - project.setOverviewContent(WELCOME_MESSAGE.getBytes()); - + public ResponseEntity createProject(@RequestBody Project project) { + if (project.getOverviewContent() == null) { + final String welcomeMessage = + """ +
+

Hey there!

+

This is your project overview page. Use this space however you like. Not sure where to start? Here are some things you can try:

+
+
    +
  • Upload stuff: Upload documents, models, code or datasets with the green button in the bottom left corner.
    • +
  • Explore and add: Use the project selector in the top nav to switch to the Explorer where you can find documents, models and datasets that you can add to your project.
    • +
  • Build a model: Create a model that fits just what you need.
    • +
  • Create a workflow: Connect resources with operators so you can focus on the science and not the plumbing.
    • +
+
+

Feel free to erase this text and make it your own.

+
+ """; + project.setOverviewContent(welcomeMessage.getBytes()); + } project = projectService.createProject(project); try { @@ -517,22 +507,6 @@ public ResponseEntity createAsset( code.get().setProject(project.get()); codeService.updateAsset(code.get()); - } else if (assetType.equals(AssetType.DATASET)) { - - final Optional dataset = datasetService.getAsset(assetId); - if (dataset.isEmpty()) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Dataset Asset does not exist"); - } - - if (project.get().getDatasetAssets() == null) - project.get().setDatasetAssets(new ArrayList<>()); - if (project.get().getDatasetAssets().contains(dataset.get())) { - throw new ResponseStatusException( - HttpStatus.CONFLICT, "Dataset Asset already exists on project"); - } - - dataset.get().setProject(project.get()); - datasetService.updateAsset(dataset.get()); } // double check that this asset is not already a part of this project, and if it @@ -596,26 +570,20 @@ public ResponseEntity deleteAsset( final RebacProject rebacProject = new RebacProject(projectId, reBACService); if (rebacUser.canWrite(rebacProject)) { - /* TODO: At the end of the Postgres migration we will be getting rid of ProjectAsset and instead - projects will directly hold a reference to the assets associated with them. During this - transition we need to properly create the relationships when users add assets to their - projects. However the exact API may not look like this in the end, and in fact may be - directly in the controllers for these assets and not in this ProjectController - */ if (assetType.equals(AssetType.CODE)) { + /* TODO: At the end of the Postgres migration we will be getting rid of ProjectAsset and instead + projects will directly hold a reference to the assets associated with them. During this + transition we need to properly create the relationships when users add assets to their + projects. However the exact API may not look like this in the end, and in fact may be + directly in the controllers for these assets and not in this ProjectController + */ + final Optional deletedCode = codeService.deleteAsset(assetId); if (deletedCode.isEmpty() || deletedCode.get().getDeletedOn() == null) { throw new ResponseStatusException( HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete code asset"); } - } else if (assetType.equals(AssetType.DATASET)) { - - final Optional deletedDataset = datasetService.deleteAsset(assetId); - if (deletedDataset.isEmpty() || deletedDataset.get().getDeletedOn() == null) { - throw new ResponseStatusException( - HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete dataset asset"); - } } final boolean deleted = projectAssetService.deleteByAssetId(projectId, assetType, assetId); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java index 6065e5f016..b1784161c7 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/Grounding.java @@ -2,14 +2,10 @@ import java.io.Serial; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; -import org.hibernate.annotations.JdbcTypeCode; -import org.hibernate.type.SqlTypes; import software.uncharted.terarium.hmiserver.annotations.TSModel; import software.uncharted.terarium.hmiserver.annotations.TSOptional; @@ -23,29 +19,9 @@ public class Grounding implements Serializable { private static final long serialVersionUID = 302308407252037615L; /** Ontological identifier per DKG */ - @JdbcTypeCode(SqlTypes.JSON) private List identifiers; /** (Optional) Additional context that informs the grounding */ @TSOptional - @JdbcTypeCode(SqlTypes.JSON) private Map context; - - @Override - public Grounding clone() { - - final Grounding clone = new Grounding(); - if (this.identifiers != null) { - clone.identifiers = new ArrayList<>(); - clone.identifiers.addAll(this.identifiers); - } - if (this.context != null) { - clone.context = new HashMap<>(); - for (final String key : this.context.keySet()) { - clone.context.put(key, context.get(key)); - } - } - - return clone; - } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java index 16df798914..a3b53e1ae7 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/Dataset.java @@ -1,17 +1,9 @@ package software.uncharted.terarium.hmiserver.models.dataservice.dataset; import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.databind.JsonNode; -import jakarta.persistence.Column; -import jakarta.persistence.Convert; -import jakarta.persistence.ElementCollection; -import jakarta.persistence.Entity; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; import java.io.Serial; import java.sql.Timestamp; -import java.util.ArrayList; import java.util.List; import lombok.Data; import lombok.EqualsAndHashCode; @@ -20,113 +12,59 @@ import software.uncharted.terarium.hmiserver.annotations.TSOptional; import software.uncharted.terarium.hmiserver.models.TerariumAsset; import software.uncharted.terarium.hmiserver.models.dataservice.Grounding; -import software.uncharted.terarium.hmiserver.models.dataservice.JsonConverter; -import software.uncharted.terarium.hmiserver.models.dataservice.ObjectConverter; -import software.uncharted.terarium.hmiserver.models.dataservice.project.Project; /** Represents a dataset document from TDS */ @EqualsAndHashCode(callSuper = true) @Data @Accessors(chain = true) @TSModel -@Entity public class Dataset extends TerariumAsset { @Serial private static final long serialVersionUID = 6927286281160755696L; - /** UserId of the user who created the dataset */ @TSOptional - @Column(length = 255) private String userId; /** ESGF id of the dataset. This will be null for datasets that are not from ESGF */ @TSOptional - @Column(length = 255) private String esgfId; /** (Optional) data source date */ @TSOptional @JsonAlias("data_source_date") - @Column(columnDefinition = "TIMESTAMP WITH TIME ZONE") private Timestamp dataSourceDate; /** (Optional) list of file names associated with the dataset */ @TSOptional @JsonAlias("file_names") - @ElementCollection - @Column(length = 1024) private List fileNames; - @ManyToOne - @JoinColumn(name = "project_id") - @JsonBackReference - @TSOptional - private Project project; - + /** + * (Optional) Url from which the dataset can be downloaded/fetched TODO: IS THIS NEEDED? IS THIS FROM OLD TDS? + * https://github.com/DARPA-ASKEM/terarium/issues/3194 + */ @TSOptional @JsonAlias("dataset_url") - @Column(length = 1024) private String datasetUrl; /** (Optional) List of urls from which the dataset can be downloaded/fetched. Used for ESGF datasets */ @TSOptional - @Column(length = 1024) - @ElementCollection private List datasetUrls; /** Information regarding the columns that make up the dataset */ @TSOptional - @Convert(converter = ObjectConverter.class) - @Column(columnDefinition = "text") private List columns; /** (Optional) Unformatted metadata about the dataset */ @TSOptional - @Convert(converter = JsonConverter.class) - @Column(columnDefinition = "text") private JsonNode metadata; /** (Optional) Source of dataset */ @TSOptional - @Column(columnDefinition = "text") private String source; /** (Optional) Grounding of ontological concepts related to the dataset as a whole */ @TSOptional - @Convert(converter = ObjectConverter.class) - @Column(columnDefinition = "text") private Grounding grounding; - - @Override - public Dataset clone() { - final Dataset clone = new Dataset(); - super.cloneSuperFields(clone); - - clone.userId = this.userId; - clone.esgfId = this.esgfId; - clone.dataSourceDate = this.dataSourceDate; - if (fileNames != null) { - clone.fileNames = new ArrayList<>(); - clone.fileNames.addAll(fileNames); - } - clone.datasetUrl = this.datasetUrl; - if (datasetUrls != null) { - clone.datasetUrls = new ArrayList<>(); - clone.datasetUrls.addAll(datasetUrls); - } - - if (columns != null) { - clone.columns = new ArrayList<>(); - for (final DatasetColumn column : columns) { - clone.columns.add(column.clone()); - } - } - - if (this.metadata != null) clone.metadata = this.metadata.deepCopy(); - clone.source = this.source; - if (this.grounding != null) clone.grounding = this.grounding.clone(); - - return clone; - } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java index d95d861a8d..9b66ce54a3 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/dataset/DatasetColumn.java @@ -1,17 +1,10 @@ package software.uncharted.terarium.hmiserver.models.dataservice.dataset; import com.fasterxml.jackson.annotation.JsonAlias; -import jakarta.persistence.Column; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.Data; import lombok.experimental.Accessors; -import org.hibernate.annotations.JdbcTypeCode; -import org.hibernate.type.SqlTypes; import software.uncharted.terarium.hmiserver.annotations.TSModel; import software.uncharted.terarium.hmiserver.annotations.TSOptional; import software.uncharted.terarium.hmiserver.models.dataservice.Grounding; @@ -23,73 +16,34 @@ public class DatasetColumn { /** Name of the column */ - @Column(length = 255) private String name; /** * Datatype. One of: unknown, boolean, string, char, integer, int, float, double, timestamp, datetime, date, time */ @JsonAlias("data_type") - @Enumerated(EnumType.STRING) private ColumnType dataType; /** (Optional) String that describes the formatting of the value */ @TSOptional @JsonAlias("format_str") - @Column(length = 255) private String formatStr; /** Column annotations from the MIT data profiling tool */ - @Column(columnDefinition = "text") private List annotations; /** (Optional) Unformatted metadata about the dataset */ @TSOptional - @JdbcTypeCode(SqlTypes.JSON) private Map metadata; /** (Optional) Grounding of ontological concepts related to the column */ @TSOptional - @JdbcTypeCode(SqlTypes.JSON) private Grounding grounding; @TSOptional - @Column(columnDefinition = "text") private String description; - public void updateMetadata(final Map metadata) { - if (this.metadata == null) { - this.metadata = metadata; - } else { - this.metadata.putAll(metadata); - } - } - - @Override - public DatasetColumn clone() { - final DatasetColumn clone = new DatasetColumn(); - - clone.name = this.name; - clone.dataType = this.dataType; - clone.formatStr = this.formatStr; - if (this.annotations != null) { - clone.annotations = new ArrayList<>(); - clone.annotations.addAll(this.annotations); - } - - if (this.metadata != null) { - clone.metadata = new HashMap<>(); - clone.metadata.putAll(this.metadata); - } - - if (this.grounding != null) clone.grounding = this.grounding.clone(); - - clone.description = this.description; - - return clone; - } - - public enum ColumnType { + enum ColumnType { @JsonAlias("unknown") UNKNOWN, @JsonAlias("boolean") @@ -115,4 +69,12 @@ public enum ColumnType { @JsonAlias("time") TIME } + + public void updateMetadata(final Map metadata) { + if (this.metadata == null) { + this.metadata = metadata; + } else { + this.metadata.putAll(metadata); + } + } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java index 4680a3c14b..b941226793 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/project/Project.java @@ -21,7 +21,6 @@ import software.uncharted.terarium.hmiserver.annotations.TSOptional; import software.uncharted.terarium.hmiserver.models.TerariumAsset; import software.uncharted.terarium.hmiserver.models.dataservice.code.Code; -import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; @EqualsAndHashCode(callSuper = true) @Data @@ -66,13 +65,6 @@ public class Project extends TerariumAsset { @JsonManagedReference private List codeAssets = new ArrayList<>(); - @OneToMany(mappedBy = "project") - @Where(clause = "deleted_on IS NULL") - @Schema(accessMode = Schema.AccessMode.READ_ONLY) - @ToString.Exclude - @JsonManagedReference - private List datasetAssets = new ArrayList<>(); - @TSOptional @Transient @Schema(accessMode = Schema.AccessMode.READ_ONLY, defaultValue = "{}") diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/DatasetRepository.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/DatasetRepository.java deleted file mode 100644 index a0b2f89b51..0000000000 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/repository/data/DatasetRepository.java +++ /dev/null @@ -1,7 +0,0 @@ -package software.uncharted.terarium.hmiserver.repository.data; - -import java.util.UUID; -import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; -import software.uncharted.terarium.hmiserver.repository.PSCrudSoftDeleteRepository; - -public interface DatasetRepository extends PSCrudSoftDeleteRepository {} diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java index 78401fc81b..7ac35464c6 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DataMigrationESToPG.java @@ -48,7 +48,6 @@ public class DataMigrationESToPG { private final WorkflowService workflowService; private final SimulationService simulationService; private final CodeService codeService; - private final DatasetService datasetService; @PersistenceContext private EntityManager entityManager; @@ -142,8 +141,7 @@ void migrateFromEsToPg(final ElasticsearchService elasticService) throws IOExcep return List.of( new MigrationConfig<>(workflowService, elasticConfig.getWorkflowIndex()), new MigrationConfig<>(simulationService, elasticConfig.getSimulationIndex()), - new MigrationConfig<>(codeService, elasticConfig.getCodeIndex()), - new MigrationConfig<>(datasetService, elasticConfig.getDatasetIndex())); + new MigrationConfig<>(codeService, elasticConfig.getCodeIndex())); // TODO: Write a script to properly sync the old ProjectAsset to the new PG data } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java index 0fce792841..07df6c10f7 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/DatasetService.java @@ -5,21 +5,19 @@ import software.uncharted.terarium.hmiserver.configuration.Config; import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration; import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; -import software.uncharted.terarium.hmiserver.repository.data.DatasetRepository; import software.uncharted.terarium.hmiserver.service.elasticsearch.ElasticsearchService; import software.uncharted.terarium.hmiserver.service.s3.S3ClientService; @Service -public class DatasetService extends TerariumAssetServiceWithSearch { +public class DatasetService extends S3BackedAssetService { public DatasetService( - final Config config, final ElasticsearchConfiguration elasticConfig, + final Config config, final ElasticsearchService elasticService, final ProjectAssetService projectAssetService, - final S3ClientService s3ClientService, - final DatasetRepository repository) { - super(config, elasticConfig, elasticService, projectAssetService, s3ClientService, repository, Dataset.class); + final S3ClientService s3ClientService) { + super(elasticConfig, config, elasticService, projectAssetService, s3ClientService, Dataset.class); } @Override @@ -33,9 +31,4 @@ protected String getAssetPath() { protected String getAssetIndex() { return elasticConfig.getDatasetIndex(); } - - @Override - public String getAssetAlias() { - return elasticConfig.getDatasetAlias(); - } } diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java index b0f90cc8fc..b6a99fe3ad 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/data/ModelService.java @@ -143,8 +143,7 @@ public List getAssets(final Integer page, final Integer pageSize) throws @Observed(name = "function_profile") public Model createAsset(final Model asset) throws IOException { // Make sure that the model framework is set to lowercase - if (asset.getHeader() != null && asset.getHeader().getSchemaName() != null) - asset.getHeader().setSchemaName(asset.getHeader().getSchemaName().toLowerCase()); + asset.getHeader().setSchemaName(asset.getHeader().getSchemaName().toLowerCase()); // Set default value for model parameters (0.0) if (asset.getSemantics() != null diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java index deb855f4f1..51add8e8bf 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/DatasetControllerTests.java @@ -26,10 +26,12 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import software.uncharted.terarium.hmiserver.TerariumApplicationTests; +import software.uncharted.terarium.hmiserver.configuration.ElasticsearchConfiguration; import software.uncharted.terarium.hmiserver.configuration.MockUser; import software.uncharted.terarium.hmiserver.models.dataservice.PresignedURL; import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; import software.uncharted.terarium.hmiserver.service.data.DatasetService; +import software.uncharted.terarium.hmiserver.service.elasticsearch.ElasticsearchService; public class DatasetControllerTests extends TerariumApplicationTests { @@ -39,14 +41,20 @@ public class DatasetControllerTests extends TerariumApplicationTests { @Autowired private DatasetService datasetService; + @Autowired + private ElasticsearchService elasticService; + + @Autowired + private ElasticsearchConfiguration elasticConfig; + @BeforeEach public void setup() throws IOException { - datasetService.setupIndexAndAliasAndEnsureEmpty(); + elasticService.createOrEnsureIndexIsEmpty(elasticConfig.getDatasetIndex()); } @AfterEach public void teardown() throws IOException { - datasetService.teardownIndexAndAlias(); + elasticService.deleteIndex(elasticConfig.getDatasetIndex()); } @Test diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ModelControllerTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ModelControllerTests.java index 2c3b8a8763..fb74f58fff 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ModelControllerTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ModelControllerTests.java @@ -52,7 +52,8 @@ public void testItCanCreateModel() throws Exception { .setName("test-name") .setModelSchema("test-schema") .setModelVersion("0.1.2") - .setDescription("test-description")); + .setDescription("test-description") + .setSchemaName("petrinet")); mockMvc.perform(MockMvcRequestBuilders.post("/models") .with(csrf()) @@ -70,7 +71,8 @@ public void testItCanGetModel() throws Exception { .setName("test-name") .setModelSchema("test-schema") .setModelVersion("0.1.2") - .setDescription("test-description"))); + .setDescription("test-description") + .setSchemaName("petrinet"))); mockMvc.perform(MockMvcRequestBuilders.get("/models/" + model.getId()).with(csrf())) .andExpect(status().isOk()); @@ -85,7 +87,8 @@ public void testItCanUpdateModel() throws Exception { .setName("test-name") .setModelSchema("test-schema") .setModelVersion("0.1.2") - .setDescription("test-description"))); + .setDescription("test-description") + .setSchemaName("petrinet"))); mockMvc.perform(MockMvcRequestBuilders.put("/models/" + model.getId()) .with(csrf()) @@ -103,7 +106,8 @@ public void testItCanDeleteModel() throws Exception { .setName("test-name") .setModelSchema("test-schema") .setModelVersion("0.1.2") - .setDescription("test-description"))); + .setDescription("test-description") + .setSchemaName("petrinet"))); mockMvc.perform(MockMvcRequestBuilders.delete("/models/" + model.getId()) .with(csrf())) @@ -121,7 +125,8 @@ public void testItCanGetModelDescription() throws Exception { .setName("test-name") .setModelSchema("test-schema") .setModelVersion("0.1.2") - .setDescription("test-description"))); + .setDescription("test-description") + .setSchemaName("petrinet"))); mockMvc.perform(MockMvcRequestBuilders.get("/models/" + model.getId() + "/descriptions") .with(csrf())) @@ -137,7 +142,8 @@ public void testItCanGetModelDescriptions() throws Exception { .setName("test-name") .setModelSchema("test-schema") .setModelVersion("0.1.2") - .setDescription("test-description"))); + .setDescription("test-description") + .setSchemaName("petrinet"))); mockMvc.perform(MockMvcRequestBuilders.get("/models/descriptions").with(csrf())) .andExpect(status().isOk()); diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java index c0146dac17..4dd7d2cd37 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/dataservice/ProjectControllerTests.java @@ -60,9 +60,13 @@ public void teardown() throws IOException { @WithUserDetails(MockUser.URSULA) public void testItCanCreateProject() throws Exception { - mockMvc.perform(MockMvcRequestBuilders.post("/projects?name=test&userId=abc123&description=desc") + final Project project = (Project) new Project().setName("test-name").setDescription("test-description"); + project.setUserId(MockUser.URSULA); + + mockMvc.perform(MockMvcRequestBuilders.post("/projects") .with(csrf()) - .contentType("application/json")) + .contentType("application/json") + .content(objectMapper.writeValueAsString(project))) .andExpect(status().isCreated()); } diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/data/DatasetServiceTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/data/DatasetServiceTests.java deleted file mode 100644 index 40eb2e219a..0000000000 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/data/DatasetServiceTests.java +++ /dev/null @@ -1,252 +0,0 @@ -package software.uncharted.terarium.hmiserver.service.data; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.test.context.support.WithUserDetails; -import software.uncharted.terarium.hmiserver.TerariumApplicationTests; -import software.uncharted.terarium.hmiserver.configuration.MockUser; -import software.uncharted.terarium.hmiserver.models.dataservice.Grounding; -import software.uncharted.terarium.hmiserver.models.dataservice.Identifier; -import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; -import software.uncharted.terarium.hmiserver.models.dataservice.dataset.DatasetColumn; - -@Slf4j -public class DatasetServiceTests extends TerariumApplicationTests { - - @Autowired - private ObjectMapper mapper; - - @Autowired - private DatasetService datasetService; - - @BeforeEach - public void setup() throws IOException { - datasetService.setupIndexAndAliasAndEnsureEmpty(); - } - - @AfterEach - public void teardown() throws IOException { - datasetService.teardownIndexAndAlias(); - } - - static Dataset createDataset() throws Exception { - return createDataset("A"); - } - - static Dataset createDataset(final String key) throws Exception { - - final Grounding grounding = new Grounding(); - grounding.setContext(new HashMap<>()); - grounding.getContext().put("hello", "world-" + key); - grounding.getContext().put("foo", "bar-" + key); - grounding.setIdentifiers(new ArrayList<>()); - grounding.getIdentifiers().add(new Identifier("curie", "maria")); - - final DatasetColumn column1 = new DatasetColumn() - .setName("Title") - .setDataType(DatasetColumn.ColumnType.STRING) - .setDescription("hello world") - .setGrounding(grounding); - final DatasetColumn column2 = new DatasetColumn() - .setName("Value") - .setDataType(DatasetColumn.ColumnType.FLOAT) - .setDescription("3.1415926") - .setGrounding(grounding); - - final Dataset dataset = new Dataset(); - dataset.setName("test-dataset-name-" + key); - dataset.setDescription("test-dataset-description-" + key); - dataset.setColumns(new ArrayList<>()); - dataset.getColumns().add(column1); - dataset.getColumns().add(column2); - dataset.setGrounding(grounding); - dataset.setPublicAsset(true); - - return dataset; - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanCreateDataset() throws Exception { - - final Dataset before = (Dataset) createDataset().setId(UUID.randomUUID()); - final Dataset after = datasetService.createAsset(before); - - Assertions.assertEquals(before.getId(), after.getId()); - Assertions.assertNotNull(after.getId()); - Assertions.assertNotNull(after.getCreatedOn()); - Assertions.assertEquals(after.getColumns().size(), 2); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCantCreateDuplicates() throws Exception { - - final Dataset dataset = (Dataset) createDataset().setId(UUID.randomUUID()); - - datasetService.createAsset(dataset); - - try { - datasetService.createAsset(dataset); - Assertions.fail("Should have thrown an exception"); - } catch (final IllegalArgumentException e) { - Assertions.assertTrue(e.getMessage().contains("already exists")); - } - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanGetDatasets() throws Exception { - - datasetService.createAsset(createDataset("0")); - datasetService.createAsset(createDataset("1")); - datasetService.createAsset(createDataset("2")); - - final List datasets = datasetService.getAssets(0, 3); - - Assertions.assertEquals(3, datasets.size()); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanGetDataset() throws Exception { - - final Dataset dataset = datasetService.createAsset(createDataset()); - - final Dataset fetchedDataset = datasetService.getAsset(dataset.getId()).get(); - - Assertions.assertEquals(dataset, fetchedDataset); - Assertions.assertEquals(dataset.getId(), fetchedDataset.getId()); - Assertions.assertEquals(dataset.getCreatedOn(), fetchedDataset.getCreatedOn()); - Assertions.assertEquals(dataset.getUpdatedOn(), fetchedDataset.getUpdatedOn()); - Assertions.assertEquals(dataset.getDeletedOn(), fetchedDataset.getDeletedOn()); - Assertions.assertEquals(dataset.getGrounding(), fetchedDataset.getGrounding()); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanUpdateDataset() throws Exception { - - final Dataset dataset = datasetService.createAsset(createDataset()); - dataset.setName("new name"); - - final Dataset updatedDataset = datasetService.updateAsset(dataset).orElseThrow(); - - Assertions.assertEquals(dataset, updatedDataset); - Assertions.assertNotNull(updatedDataset.getUpdatedOn()); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanDeleteDataset() throws Exception { - - final Dataset dataset = datasetService.createAsset(createDataset()); - - datasetService.deleteAsset(dataset.getId()); - - final Optional deleted = datasetService.getAsset(dataset.getId()); - - Assertions.assertTrue(deleted.isEmpty()); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanCloneDataset() throws Exception { - - Dataset dataset = createDataset(); - dataset = datasetService.createAsset(dataset); - - final Dataset cloned = datasetService.cloneAsset(dataset.getId()); - - Assertions.assertNotEquals(dataset.getId(), cloned.getId()); - Assertions.assertEquals(dataset.getGrounding(), cloned.getGrounding()); - Assertions.assertEquals(dataset.getColumns(), cloned.getColumns()); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanExportAndImportDataset() throws Exception { - - Dataset dataset = createDataset(); - dataset = datasetService.createAsset(dataset); - - final byte[] exported = datasetService.exportAsset(dataset.getId()); - - final Dataset imported = datasetService.importAsset(exported); - - Assertions.assertNotEquals(dataset.getId(), imported.getId()); - Assertions.assertEquals(dataset.getName(), imported.getName()); - Assertions.assertEquals(dataset.getDescription(), imported.getDescription()); - Assertions.assertEquals(dataset.getGrounding(), imported.getGrounding()); - Assertions.assertEquals(dataset.getColumns(), imported.getColumns()); - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanSearchAssets() throws Exception { - - final int NUM = 32; - - List datasets = new ArrayList<>(); - for (int i = 0; i < NUM; i++) { - datasets.add(createDataset(String.valueOf(i))); - } - datasets = datasetService.createAssets(datasets); - - final List results = datasetService.searchAssets(0, NUM, null); - - Assertions.assertEquals(NUM, results.size()); - - for (int i = 0; i < results.size(); i++) { - Assertions.assertEquals(datasets.get(i).getName(), results.get(i).getName()); - Assertions.assertEquals( - datasets.get(i).getDescription(), results.get(i).getDescription()); - Assertions.assertEquals( - datasets.get(i).getGrounding(), results.get(i).getGrounding()); - Assertions.assertEquals( - datasets.get(i).getCreatedOn().toInstant().getEpochSecond(), - results.get(i).getCreatedOn().toInstant().getEpochSecond()); - Assertions.assertEquals( - datasets.get(i).getUpdatedOn().toInstant().getEpochSecond(), - results.get(i).getUpdatedOn().toInstant().getEpochSecond()); - Assertions.assertEquals( - datasets.get(i).getDeletedOn(), results.get(i).getDeletedOn()); - } - } - - @Test - @WithUserDetails(MockUser.URSULA) - public void testItCanSyncToNewIndex() throws Exception { - - final int NUM = 32; - - final List datasets = new ArrayList<>(); - for (int i = 0; i < NUM; i++) { - datasets.add(createDataset(String.valueOf(i))); - } - datasetService.createAssets(datasets); - - final String currentIndex = datasetService.getCurrentAssetIndex(); - - Assertions.assertEquals(NUM, datasetService.searchAssets(0, NUM, null).size()); - - datasetService.syncAllAssetsToNewIndex(true); - - final String newIndex = datasetService.getCurrentAssetIndex(); - - Assertions.assertEquals(NUM, datasetService.searchAssets(0, NUM, null).size()); - - Assertions.assertNotEquals(currentIndex, newIndex); - } -}