diff --git a/hopsworks-UT/src/test/java/io/hops/hopsworks/TestSubResources.java b/hopsworks-UT/src/test/java/io/hops/hopsworks/TestSubResources.java new file mode 100644 index 0000000000..a97e0ec738 --- /dev/null +++ b/hopsworks-UT/src/test/java/io/hops/hopsworks/TestSubResources.java @@ -0,0 +1,146 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks; + +import io.hops.hopsworks.common.dataset.util.DatasetPath; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.scanners.TypeAnnotationsScanner; + +import javax.enterprise.context.RequestScoped; +import javax.persistence.Entity; +import javax.ws.rs.Path; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.HashSet; +import java.util.Set; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * This test will ensure that sub resources do not contain entity class properties. + * 1. Sub resources are request scoped and so should not cache anything. + * 2. If an entity class is passed to a sub resource that means it needs to be fetched from database before the sub + * resource is called. This will bypass any filters that are set to intercept methods on the sub resource. + *

+ * So if we have a project entity in a sub resource, and we pass it from a parent resource the project entity will be + * fetched from database before the user, that is making the request, is authenticated and verified if he/she has access + * to the project. + */ +public class TestSubResources { + Set> subResourceClasses = new HashSet<>(); + Set> resourceClasses = new HashSet<>(); + Set> entityClasses = new HashSet<>(); + + @Before + public void beforeTest() { + Reflections reflections = new Reflections("io.hops.hopsworks", new SubTypesScanner(false), + new TypeAnnotationsScanner()); + for (Class c : reflections.getTypesAnnotatedWith(Path.class, true)) { + if (c.getName().startsWith("io.hops.hopsworks.api")) { + resourceClasses.add(c); + } + } + for (Class c : reflections.getTypesAnnotatedWith(RequestScoped.class, true)) { + if (c.getName().startsWith("io.hops.hopsworks.api")) { + subResourceClasses.add(c); + } + } + + for (Class c : reflections.getTypesAnnotatedWith(Entity.class, true)) { + if (c.getName().startsWith("io.hops.hopsworks.persistence.entity")) { + entityClasses.add(c); + } + } + + } + + @Test + public void testIfClassesAreFound() { + System.out.println("SubResources: " + subResourceClasses.size()); + assertFalse("Sub Resource classes should not be empty.", subResourceClasses.isEmpty()); + System.out.println("Entity classes: " + entityClasses.size()); + assertFalse("Entity classes should not be empty.", entityClasses.isEmpty()); + } + + @Test + public void testSubResourceAreFinal() { + for (Class c : resourceClasses) { + if (!Modifier.isFinal(c.getModifiers())) { + System.out.println("Api Resources should be final. Inheriting a resource class will force the swagger " + + "documentation to be too general. Offending classes " + c.getName()); + } +// assertTrue("Api Resources should be final. Inheriting a resource class will force the swagger " + +// "documentation to be too general. Offending classes " + c.getName(), +// Modifier.isFinal(c.getModifiers())); + } + for (Class c : subResourceClasses) { + if (!Modifier.isFinal(c.getModifiers())) { + System.out.println("Api Resources should be final. Inheriting a resource class will force the swagger " + + "documentation to be too general. Offending classes " + c.getName()); + } +// assertTrue("Api Resources should be final. Inheriting a resource class will force the swagger " + +// "documentation to be too general. Offending classes " + c.getName(), +// Modifier.isFinal(c.getModifiers())); + } + } + + @Test + public void testEntityClassesInFields() { + for (Class c : subResourceClasses) { + Set> entityClassesInFields = containsEntity(c); + if (!entityClassesInFields.isEmpty()) { + System.out.println("Class " + c.getName() + " contains entity class fields. "); + System.out.println(entityClassesInFields); + } + assertTrue("SubResources should not contain entity classes fields that query the database.", + entityClassesInFields.isEmpty()); + } + } + + @Test + public void testDatasetPathClassesInFields() { + for (Class c : subResourceClasses) { + if (containsClass(c, DatasetPath.class)) { + System.out.println("Class " + c.getName() + " contains DatasetPath in fields. "); + Assert.fail("SubResources should not contain any Class that queries the database. Found DatasetPath"); + } + } + } + + private Set> containsEntity(Class c) { + Set> entityClassesInFields = new HashSet<>(); + for (Field field : c.getDeclaredFields()) { + if (entityClasses.contains(field.getType())) { + entityClassesInFields.add(field.getType()); + } + } + return entityClassesInFields; + } + + private boolean containsClass(Class c, Class c1) { + for (Field field : c.getDeclaredFields()) { + if (field.getType().equals(c1)) { + return true; + } + } + return false; + } +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/activities/ProjectActivitiesResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/activities/ProjectActivitiesResource.java index ccea34b9e0..8f84fb31f7 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/activities/ProjectActivitiesResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/activities/ProjectActivitiesResource.java @@ -17,14 +17,14 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.project.ProjectFacade; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.ActivitiesException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.project.Project; -import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -42,53 +42,25 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; -import java.util.logging.Level; import java.util.logging.Logger; @Api(value = "Project Activities Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ProjectActivitiesResource { +public class ProjectActivitiesResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(ProjectActivitiesResource.class.getName()); @EJB private ActivitiesBuilder activitiesBuilder; @EJB - private ProjectFacade projectFacade; - - private Integer projectId; - private String projectName; + private ProjectController projectController; public ProjectActivitiesResource() { } - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProjectById() throws ProjectException { - Project project = projectFacade.find(this.projectId); - if (project == null) { - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + projectId); - } - return project; - } - - private Project getProjectByName() throws ProjectException { - Project project = projectFacade.findByName(this.projectName); - if (project == null) { - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectName: " + - projectName); - } - return project; - } - - private Project getProject() throws ProjectException { - return this.projectId != null ? getProjectById() : getProjectByName(); + @Override + protected ProjectController getProjectController() { + return projectController; } @GET diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/airflow/AirflowService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/airflow/AirflowService.java index 1eca909852..c8e0909ba8 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/airflow/AirflowService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/airflow/AirflowService.java @@ -19,12 +19,13 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.common.airflow.AirflowController; import io.hops.hopsworks.common.airflow.AirflowDagDTO; -import io.hops.hopsworks.common.dao.project.ProjectFacade; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.AirflowException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -45,31 +46,20 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Airflow related endpoints") -public class AirflowService { +public class AirflowService extends ProjectSubResource { @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private JWTHelper jwtHelper; @EJB private AirflowController airflowController; - private Integer projectId; - // No @EJB annotation for Project, it's injected explicitly in ProjectService. - private Project project; - - - // Audience for Airflow JWTs - private static final String[] JWT_AUDIENCE = new String[]{Audience.API}; public AirflowService() { } - public void setProjectId(Integer projectId) { - this.projectId = projectId; - this.project = this.projectFacade.find(projectId); - } - - public Integer getProjectId() { - return projectId; + @Override + protected ProjectController getProjectController() { + return projectController; } @POST @@ -80,9 +70,9 @@ public Integer getProjectId() { @ApiOperation(value = "Generate an Airflow Python DAG file from a DAG definition") public Response composeDAG(AirflowDagDTO dagDefinition, @Context HttpServletRequest req, - @Context SecurityContext sc) throws AirflowException { + @Context SecurityContext sc) throws AirflowException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); - airflowController.composeDAG(project, user, dagDefinition); + airflowController.composeDAG(getProject(), user, dagDefinition); return Response.ok().build(); } -} \ No newline at end of file +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/AlertResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/AlertResource.java index 4f7ddcdbfd..c35aeeb0cf 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/AlertResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/AlertResource.java @@ -27,13 +27,13 @@ import io.hops.hopsworks.api.alert.silence.SilenceResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.AlertException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -64,7 +64,7 @@ @RequestScoped @Api(value = "Alert Resource") @TransactionAttribute(TransactionAttributeType.NEVER) -public class AlertResource { +public class AlertResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(AlertResource.class.getName()); @@ -81,24 +81,9 @@ public class AlertResource { @Inject private ReceiverResource receiverResource; - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + @Override + protected ProjectController getProjectController() { + return projectController; } @GET @@ -169,19 +154,19 @@ public Response createAlerts(PostableAlertDTOs alerts, @Context UriInfo uriInfo, @Path("silences") public SilenceResource silence() { - this.silenceResource.setProjectId(this.projectId); + this.silenceResource.setProjectId(getProjectId()); return silenceResource; } @Path("receivers") public ReceiverResource receiver() { - this.receiverResource.setProjectId(this.projectId); + this.receiverResource.setProjectId(getProjectId()); return receiverResource; } @Path("routes") public RouteResource route() { - this.routeResource.setProjectId(this.projectId); + this.routeResource.setProjectId(getProjectId()); return routeResource; } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/receiver/ReceiverResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/receiver/ReceiverResource.java index 256db6b0a2..81b7501f52 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/receiver/ReceiverResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/receiver/ReceiverResource.java @@ -32,6 +32,7 @@ import io.hops.hopsworks.api.alert.silence.SilenceDTO; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.project.ProjectController; @@ -70,8 +71,7 @@ @Api(value = "Alert Receiver Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ReceiverResource { - +public class ReceiverResource extends ProjectSubResource { @EJB private ReceiverBuilder receiverBuilder; @EJB @@ -81,24 +81,9 @@ public class ReceiverResource { @EJB private AMClient alertManager; - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + @Override + protected ProjectController getProjectController() { + return projectController; } @GET diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/route/RouteResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/route/RouteResource.java index 5d0ef0b211..4cde7116af 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/route/RouteResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/route/RouteResource.java @@ -26,13 +26,13 @@ import io.hops.hopsworks.alerting.exceptions.AlertManagerNoSuchElementException; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.AlertException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -63,7 +63,7 @@ @Api(value = "Route Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class RouteResource { +public class RouteResource extends ProjectSubResource { @EJB private RouteBuilder routeBuilder; @@ -72,24 +72,9 @@ public class RouteResource { @EJB private AlertManagerConfiguration alertManagerConfiguration; - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + @Override + protected ProjectController getProjectController() { + return projectController; } @GET diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/silence/SilenceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/silence/SilenceResource.java index e8debdeac4..7194bbf455 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/silence/SilenceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/alert/silence/SilenceResource.java @@ -25,6 +25,7 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.project.ProjectController; @@ -62,7 +63,7 @@ @Api(value = "Silence Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class SilenceResource { +public class SilenceResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(SilenceResource.class.getName()); @@ -75,24 +76,9 @@ public class SilenceResource { @EJB private JWTHelper jWTHelper; - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + @Override + protected ProjectController getProjectController() { + return projectController; } @GET diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetResource.java index 5aa8d9f827..f4f495f9ef 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetResource.java @@ -23,6 +23,7 @@ import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.DownloadService; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.api.util.UploadService; @@ -87,10 +88,10 @@ @Api(value = "Dataset Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class DatasetResource { - +public class DatasetResource extends ProjectSubResource { + private static final Logger LOGGER = Logger.getLogger(DatasetResource.class.getName()); - + @EJB private DatasetController datasetController; @EJB @@ -120,26 +121,11 @@ public class DatasetResource { @EJB private Settings settings; - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); - } - private void checkIfDataOwner(Project project, Users user) throws DatasetException { if (!projectTeamFacade.findCurrentRole(project, user).equals(AllowedRoles.DATA_OWNER)) { throw new DatasetException(RESTCodes.DatasetErrorCode.DATASET_ACCESS_PERMISSION_DENIED, Level.FINE); @@ -493,20 +479,22 @@ public Response delete(@PathParam("path") String path, @Path("/download") public DownloadService download() { - this.downloadService.setProjectId(this.projectId); + this.downloadService.setProjectId(this.getProjectId()); return this.downloadService; } - + @Path("upload/{path: .+}") public UploadService upload(@PathParam("path") String path, @QueryParam("type") DatasetType datasetType) { - this.uploader.setParams(this.projectId, path, datasetType); + this.uploader.setProjectId(getProjectId()); + this.uploader.setDatasetType(datasetType); + this.uploader.setPath(path); return this.uploader; } - + @Path("tags") public DatasetTagsResource tags() throws ProjectException { - this.tagsResource.setParams(getProject()); + this.tagsResource.setProjectId(getProjectId()); return this.tagsResource; } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetSubResource.java new file mode 100644 index 0000000000..478b03a17f --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/DatasetSubResource.java @@ -0,0 +1,31 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.dataset; + +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.persistence.entity.dataset.DatasetType; + +public abstract class DatasetSubResource extends ProjectSubResource { + private DatasetType datasetType; + + public DatasetType getDatasetType() { + return datasetType; + } + + public void setDatasetType(DatasetType datasetType) { + this.datasetType = datasetType; + } +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/tags/DatasetTagsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/tags/DatasetTagsResource.java index a6e2597217..0146a4353d 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/tags/DatasetTagsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/dataset/tags/DatasetTagsResource.java @@ -15,24 +15,26 @@ */ package io.hops.hopsworks.api.dataset.tags; -import io.hops.hopsworks.api.tags.TagBuilder; -import io.hops.hopsworks.common.tags.TagControllerIface; -import io.hops.hopsworks.common.tags.TagsDTO; -import io.hops.hopsworks.api.tags.TagsExpansionBeanParam; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.api.tags.TagBuilder; +import io.hops.hopsworks.api.tags.TagsExpansionBeanParam; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.dataset.util.DatasetPath; import io.hops.hopsworks.common.featurestore.metadata.AttachMetadataResult; +import io.hops.hopsworks.common.project.ProjectController; +import io.hops.hopsworks.common.tags.TagControllerIface; +import io.hops.hopsworks.common.tags.TagsDTO; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.dataset.DatasetType; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; @@ -66,7 +68,7 @@ @Api(value = "Dataset Tags Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class DatasetTagsResource { +public class DatasetTagsResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(DatasetTagsResource.class.getName()); @EJB @@ -77,15 +79,12 @@ public class DatasetTagsResource { private TagControllerIface tagsController; @EJB private TagBuilder tagsBuilder; + @EJB + private ProjectController projectController; - private Project project; - - public void setProject(Project project) { - this.project = project; - } - - public void setParams(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Create or update tag for a dataset", response = TagsDTO.class) @@ -102,8 +101,8 @@ public Response putTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam("datasetType") DatasetType datasetType, @ApiParam(value = "Value to set for the tag") String value) - throws DatasetException, MetadataException, FeatureStoreMetadataException { - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, datasetType); + throws DatasetException, MetadataException, FeatureStoreMetadataException, ProjectException { + DatasetPath datasetPath = datasetHelper.getDatasetPath(getProject(), path, datasetType); Users user = jWTHelper.getUserPrincipal(sc); AttachMetadataResult result = tagsController.upsert(user, datasetPath, schemaName, value); @@ -129,9 +128,10 @@ public Response putTag(@Context SecurityContext sc, @Context UriInfo uriInfo, public Response bulkPutTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam("datasetType") DatasetType datasetType, + @Context HttpServletRequest req, TagsDTO tags) - throws DatasetException, MetadataException, FeatureStoreMetadataException { - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, datasetType); + throws DatasetException, MetadataException, FeatureStoreMetadataException, ProjectException { + DatasetPath datasetPath = datasetHelper.getDatasetPath(getProject(), path, datasetType); Users user = jWTHelper.getUserPrincipal(sc); AttachMetadataResult result; @@ -170,8 +170,8 @@ public Response getTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam("datasetType") DatasetType datasetType, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws DatasetException, FeatureStoreMetadataException, MetadataException { - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, datasetType); + throws DatasetException, FeatureStoreMetadataException, MetadataException, ProjectException { + DatasetPath datasetPath = datasetHelper.getDatasetPath(getProject(), path, datasetType); Users user = jWTHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); resourceRequest.setExpansions(tagsExpansionBeanParam.getResources()); @@ -195,8 +195,8 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @PathParam("path") String path, @QueryParam("datasetType") DatasetType datasetType, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws DatasetException, FeatureStoreMetadataException, MetadataException { - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, datasetType); + throws DatasetException, FeatureStoreMetadataException, MetadataException, ProjectException { + DatasetPath datasetPath = datasetHelper.getDatasetPath(getProject(), path, datasetType); Users user = jWTHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); @@ -218,8 +218,8 @@ public Response deleteTags(@Context SecurityContext sc, @Context HttpServletRequest req, @PathParam("path") String path, @QueryParam("datasetType") DatasetType datasetType) - throws DatasetException, MetadataException { - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, datasetType); + throws DatasetException, MetadataException, ProjectException { + DatasetPath datasetPath = datasetHelper.getDatasetPath(getProject(), path, datasetType); Users user = jWTHelper.getUserPrincipal(sc); tagsController.deleteAll(user, datasetPath); @@ -242,8 +242,8 @@ public Response deleteTag(@Context SecurityContext sc, @PathParam("schemaName") String schemaName, @PathParam("path") String path, @QueryParam("datasetType") DatasetType datasetType) - throws DatasetException, MetadataException { - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, datasetType); + throws DatasetException, MetadataException, ProjectException { + DatasetPath datasetPath = datasetHelper.getDatasetPath(getProject(), path, datasetType); Users user = jWTHelper.getUserPrincipal(sc); tagsController.delete(user, datasetPath, schemaName); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsResource.java index cf3f471ad7..7282d5317b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsResource.java @@ -24,12 +24,14 @@ import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; import io.hops.hopsworks.api.jwt.JWTHelper; -import io.hops.hopsworks.api.util.Pagination; +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.api.util.Pagination;; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.dataset.DatasetController; import io.hops.hopsworks.common.hdfs.Utils; import io.hops.hopsworks.common.hdfs.inode.InodeController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO; import io.hops.hopsworks.common.python.environment.EnvironmentController; import io.hops.hopsworks.common.util.AccessController; @@ -77,13 +79,15 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ExperimentsResource { +public class ExperimentsResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(ExperimentsResource.class.getName()); @EJB private ProjectFacade projectFacade; @EJB + private ProjectController projectController; + @EJB private ExperimentsBuilder experimentsBuilder; @EJB private JWTHelper jwtHelper; @@ -104,10 +108,9 @@ public class ExperimentsResource { @EJB private Settings settings; - private Project project; - public ExperimentsResource setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get a list of all experiments for this project", response = ExperimentDTO.class) @@ -119,7 +122,8 @@ public ExperimentsResource setProjectId(Integer projectId) { public Response getAll( @BeanParam Pagination pagination, @BeanParam ExperimentsBeanParam experimentsBeanParam, - @Context UriInfo uriInfo, @Context SecurityContext sc) throws ExperimentsException { + @Context HttpServletRequest req, + @Context UriInfo uriInfo, @Context SecurityContext sc) throws ExperimentsException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXPERIMENTS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); @@ -127,7 +131,7 @@ public Response getAll( resourceRequest.setSort(experimentsBeanParam.getSortBySet()); resourceRequest.setExpansions(experimentsBeanParam.getExpansions().getResources()); Users user = jwtHelper.getUserPrincipal(sc); - ExperimentDTO dto = experimentsBuilder.build(uriInfo, resourceRequest, project, user); + ExperimentDTO dto = experimentsBuilder.build(uriInfo, resourceRequest, getProject(), user); return Response.ok().entity(dto).build(); } @@ -142,10 +146,13 @@ public Response get ( @PathParam("id") String id, @BeanParam ExpansionBeanParam expansions, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) - throws ExperimentsException, DatasetException, ProvenanceException, MetadataException, GenericException { + throws ExperimentsException, DatasetException, ProvenanceException, MetadataException, GenericException, + ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXPERIMENTS); resourceRequest.setExpansions(expansions.getResources()); + Project project = getProject(); ProvStateDTO fileState = experimentsController.getExperiment(project, id); Users user = jwtHelper.getUserPrincipal(sc); if(fileState != null) { @@ -190,10 +197,10 @@ public Response post ( } Users user = jwtHelper.getUserPrincipal(sc); - Project experimentProject = project; + Project experimentProject = getProject(); switch(type) { case INIT: { - String experimentPath = Utils.getProjectPath(project.getName()) + Settings.HOPS_EXPERIMENTS_DATASET + + String experimentPath = Utils.getProjectPath(experimentProject.getName()) + Settings.HOPS_EXPERIMENTS_DATASET + "/" + id + "/" + Settings.ENVIRONMENT_FILE; experimentDTO.setEnvironment(environmentController.exportEnv(experimentProject, user, experimentPath)); try { @@ -234,6 +241,7 @@ public Response post ( private Project getModelsProjectAndCheckAccess(ExperimentDTO experimentDTO) throws ProjectException, DatasetException, GenericException { + Project project = getProject(); Project modelProject; if (experimentDTO.getModelProjectName() == null) { modelProject = project; @@ -268,8 +276,9 @@ public Response delete ( @QueryParam("endpointId") Integer parentProjId, @Context HttpServletRequest req, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws DatasetException { + @Context SecurityContext sc) throws DatasetException, ProjectException { Users hopsworksUser = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); if(parentProjId != null) { Project parentProject = projectFacade.find(parentProjId); experimentsController.delete(hopsworksUser, project, parentProject, id); @@ -278,18 +287,22 @@ public Response delete ( } return Response.noContent().build(); } - + @ApiOperation(value = "TensorBoard sub-resource", tags = {"TensorBoardResource"}) @Path("{id}/tensorboard") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) - public TensorBoardResource tensorboard(@PathParam("id") String id) { - return this.tensorBoardResource.setProject(project, id); + public TensorBoardResource tensorboard(@PathParam("id") String id) throws ProjectException { + this.tensorBoardResource.setProjectId(getProjectId()); + this.tensorBoardResource.setExperimentId(id); + return this.tensorBoardResource; } @ApiOperation(value = "Results sub-resource", tags = {"ExperimentResultsResource"}) @Path("{id}/results") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) - public ExperimentResultsResource results(@PathParam("id") String id) { - return this.resultsResource.setProject(project, id); + public ExperimentResultsResource results(@PathParam("id") String id) throws ProjectException { + this.resultsResource.setProjectId(getProjectId()); + this.resultsResource.setExperimentId(id); + return this.resultsResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsSubResource.java new file mode 100644 index 0000000000..f610e42717 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/ExperimentsSubResource.java @@ -0,0 +1,30 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.experiments; + +import io.hops.hopsworks.api.project.ProjectSubResource; + +public abstract class ExperimentsSubResource extends ProjectSubResource { + private String experimentId; + + public String getExperimentId() { + return experimentId; + } + + public void setExperimentId(String experimentId) { + this.experimentId = experimentId; + } +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/results/ExperimentResultsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/results/ExperimentResultsResource.java index 415853f54c..cdc3a53d2e 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/results/ExperimentResultsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/results/ExperimentResultsResource.java @@ -15,6 +15,7 @@ */ package io.hops.hopsworks.api.experiments.results; +import io.hops.hopsworks.api.experiments.ExperimentsSubResource; import io.hops.hopsworks.api.experiments.dto.results.ExperimentResultSummaryDTO; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; @@ -22,9 +23,10 @@ import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.ExperimentsException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.ApiOperation; @@ -32,6 +34,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.GET; import javax.ws.rs.Produces; @@ -44,22 +47,15 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ExperimentResultsResource { - - private Project project; - private String experimentId; +public class ExperimentResultsResource extends ExperimentsSubResource { @EJB private ExperimentResultsBuilder experimentResultsBuilder; - - public ExperimentResultsResource setProject(Project project, String experimentId) { - this.project = project; - this.experimentId = experimentId; - return this; - } - - public Project getProject() { - return project; + @EJB + private ProjectController projectController; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get results information", response = ExperimentResultSummaryDTO.class) @@ -70,14 +66,16 @@ public Project getProject() { @FeatureFlagRequired(requiredFeatureFlags = {FeatureFlags.DATA_SCIENCE_PROFILE}) public Response getResults(@Context UriInfo uriInfo, @BeanParam Pagination pagination, + @Context HttpServletRequest req, @BeanParam ExperimentResultsBeanParam experimentResultsBeanParam, - @Context SecurityContext sc) throws ExperimentsException { + @Context SecurityContext sc) throws ExperimentsException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.RESULTS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(experimentResultsBeanParam.getSortBySet()); - ExperimentResultSummaryDTO dto = experimentResultsBuilder.build(uriInfo, resourceRequest, project, experimentId); + ExperimentResultSummaryDTO dto = + experimentResultsBuilder.build(uriInfo, resourceRequest, getProject(), getExperimentId()); if(dto == null) { throw new ExperimentsException(RESTCodes.ExperimentsErrorCode.RESULTS_NOT_FOUND, Level.FINE); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/tensorboard/TensorBoardResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/tensorboard/TensorBoardResource.java index 1c05251d1b..7a30108969 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/tensorboard/TensorBoardResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/experiments/tensorboard/TensorBoardResource.java @@ -17,6 +17,7 @@ package io.hops.hopsworks.api.experiments.tensorboard; import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; +import io.hops.hopsworks.api.experiments.ExperimentsSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; @@ -26,6 +27,7 @@ import io.hops.hopsworks.api.project.util.PathValidator; import io.hops.hopsworks.common.dao.tensorflow.config.TensorBoardDTO; import io.hops.hopsworks.common.experiments.tensorboard.TensorBoardController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.TensorBoardException; @@ -40,6 +42,7 @@ import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; import javax.persistence.PersistenceException; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -59,7 +62,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class TensorBoardResource { +public class TensorBoardResource extends ExperimentsSubResource { @EJB private TensorBoardController tensorBoardController; @@ -68,17 +71,11 @@ public class TensorBoardResource { @EJB private JWTHelper jWTHelper; - private Project project; - private String experimentId; - - public TensorBoardResource setProject(Project project, String experimentId) { - this.project = project; - this.experimentId = experimentId; - return this; - } - - public Project getProject() { - return project; + @EJB + private ProjectController projectController; + @Override + protected ProjectController getProjectController() { + return projectController; } private final static Logger LOGGER = Logger.getLogger(TensorBoardResource.class.getName()); @@ -89,10 +86,11 @@ public Project getProject() { @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @FeatureFlagRequired(requiredFeatureFlags = {FeatureFlags.DATA_SCIENCE_PROFILE}) - public Response getTensorBoard(@Context SecurityContext sc) throws TensorBoardException { + public Response getTensorBoard(@Context HttpServletRequest req, + @Context SecurityContext sc) throws TensorBoardException, ProjectException { try { Users user = jWTHelper.getUserPrincipal(sc); - TensorBoardDTO tbDTO = tensorBoardController.getTensorBoard(project, user); + TensorBoardDTO tbDTO = tensorBoardController.getTensorBoard(getProject(), user); if(tbDTO == null) { throw new TensorBoardException(RESTCodes.TensorBoardErrorCode.TENSORBOARD_NOT_FOUND, Level.FINE); } @@ -109,13 +107,15 @@ public Response getTensorBoard(@Context SecurityContext sc) throws TensorBoardEx @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @FeatureFlagRequired(requiredFeatureFlags = {FeatureFlags.DATA_SCIENCE_PROFILE}) - public Response startTensorBoard(@Context SecurityContext sc, @Context UriInfo uriInfo) throws DatasetException, + public Response startTensorBoard(@Context SecurityContext sc, + @Context HttpServletRequest req, + @Context UriInfo uriInfo) throws DatasetException, ProjectException, TensorBoardException, UnsupportedEncodingException, ServiceDiscoveryException { - - DsPath dsPath = pathValidator.validatePath(this.project, "Experiments/" + experimentId); + Project project = getProject(); + DsPath dsPath = pathValidator.validatePath(project, "Experiments/" + getExperimentId()); String fullPath = dsPath.getFullPath().toString(); Users user = jWTHelper.getUserPrincipal(sc); - TensorBoardDTO tensorBoardDTO = tensorBoardController.startTensorBoard(experimentId, project, user, fullPath); + TensorBoardDTO tensorBoardDTO = tensorBoardController.startTensorBoard(getExperimentId(), project, user, fullPath); waitForTensorBoardLoaded(tensorBoardDTO); UriBuilder builder = uriInfo.getAbsolutePathBuilder(); @@ -127,13 +127,15 @@ public Response startTensorBoard(@Context SecurityContext sc, @Context UriInfo u @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @FeatureFlagRequired(requiredFeatureFlags = {FeatureFlags.DATA_SCIENCE_PROFILE}) - public Response stopTensorBoard(@Context SecurityContext sc) throws TensorBoardException { + public Response stopTensorBoard(@Context HttpServletRequest req, + @Context SecurityContext sc) throws TensorBoardException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); TensorBoardDTO tbDTO = tensorBoardController.getTensorBoard(project, user); if (tbDTO == null) { return Response.noContent().build(); } - tensorBoardController.cleanup(this.project, user); + tensorBoardController.cleanup(project, user); return Response.noContent().build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreCommonSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreCommonSubResource.java new file mode 100644 index 0000000000..797c8b45fd --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreCommonSubResource.java @@ -0,0 +1,182 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.featurestore; + +import com.google.common.base.Strings; +import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; +import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.project.Project; + +/** + * When using this always check if trainingDataset returns a value before testing the feature view + * b/c a training dataset can set feature view name and version. But if it is a sub resource of feature view + * it should never set trainingDatasetId and trainingDatasetVersion. + */ +public abstract class FeaturestoreCommonSubResource extends FeaturestoreSubResource { + private String featureViewName; + private Integer featureViewVersion; + + private Integer featureGroupId; + private Integer trainingDatasetId; + private Integer trainingDatasetVersion; + + public String getFeatureViewName() { + return featureViewName; + } + + public void setFeatureViewName(String featureViewName) { + this.featureViewName = featureViewName; + } + + public Integer getFeatureViewVersion() { + return featureViewVersion; + } + + public void setFeatureViewVersion(Integer featureViewVersion) { + this.featureViewVersion = featureViewVersion; + } + + public Integer getFeatureGroupId() { + return featureGroupId; + } + + public void setFeatureGroupId(Integer featureGroupId) { + this.featureGroupId = featureGroupId; + } + + public Integer getTrainingDatasetId() { + return trainingDatasetId; + } + + public void setTrainingDatasetId(Integer trainingDatasetId) { + this.trainingDatasetId = trainingDatasetId; + } + + public Integer getTrainingDatasetVersion() { + return trainingDatasetVersion; + } + + public void setTrainingDatasetVersion(Integer trainingDatasetVersion) { + this.trainingDatasetVersion = trainingDatasetVersion; + } + + public void setFeatureView(String name, Integer version) { + this.featureViewName = name; + this.featureViewVersion = version; + } + + public FeatureView getFeatureView(Project project) throws FeaturestoreException, ProjectException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return getFeatureViewController().getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, + getFeaturestore(project)); + } + + public FeatureView getFeatureView(Featurestore featurestore) throws FeaturestoreException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return getFeatureViewController().getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, + featurestore); + } + + public FeatureView getFeatureView() throws ProjectException, FeaturestoreException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return getFeatureViewController().getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, + getFeaturestore()); + } + + public Featuregroup getFeaturegroup(Project project) throws FeaturestoreException, ProjectException { + if (featureGroupId == null) { + return null; + } + return getFeaturegroupController().getFeaturegroupById(getFeaturestore(project), featureGroupId); + } + + public Featuregroup getFeaturegroup(Featurestore featurestore) throws FeaturestoreException { + if (featureGroupId == null) { + return null; + } + return getFeaturegroupController().getFeaturegroupById(featurestore, featureGroupId); + } + + public Featuregroup getFeaturegroup() throws ProjectException, FeaturestoreException { + if (featureGroupId == null) { + return null; + } + return getFeaturegroupController().getFeaturegroupById(getFeaturestore(), featureGroupId); + } + + public TrainingDataset getTrainingDataset(Project project) throws FeaturestoreException, ProjectException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(getFeaturestore(project), trainingDatasetId); + } + if (trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(getFeatureView(project), + trainingDatasetVersion); + } + return null; + } + + public TrainingDataset getTrainingDataset(Featurestore featurestore) throws FeaturestoreException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(featurestore, trainingDatasetId); + } + if (trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(getFeatureView(featurestore), + trainingDatasetVersion); + } + return null; + } + + public TrainingDataset getTrainingDataset(Featurestore featurestore, FeatureView featureView) + throws FeaturestoreException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(featurestore, trainingDatasetId); + } + if (featureView != null && trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(featureView, + trainingDatasetVersion); + } + + return null; + } + + public TrainingDataset getTrainingDataset() throws ProjectException, FeaturestoreException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(getFeaturestore(), trainingDatasetId); + } + if (trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(getFeatureView(), + trainingDatasetVersion); + } + return null; + } + + protected abstract FeatureViewController getFeatureViewController(); + protected abstract TrainingDatasetController getTrainingDatasetController(); + protected abstract FeaturegroupController getFeaturegroupController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java index fcb6f3e68c..0aed8dbaa9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java @@ -18,6 +18,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.featurestore.datavalidationv2.greatexpectations.GreatExpectationResource; import io.hops.hopsworks.api.featurestore.featuregroup.FeaturegroupService; import io.hops.hopsworks.api.featurestore.featureview.FeatureViewService; @@ -28,7 +29,7 @@ import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.NoCacheResponse; import io.hops.hopsworks.api.kafka.KafkaResource; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.FeaturestoreDTO; @@ -40,7 +41,6 @@ import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.hops.hopsworks.restutils.RESTCodes; @@ -75,7 +75,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Featurestore service", description = "A service that manages project's feature stores") -public class FeaturestoreService { +public class FeaturestoreService extends ProjectSubResource { @EJB private NoCacheResponse noCacheResponse; @@ -106,15 +106,9 @@ public class FeaturestoreService { @Inject private KafkaResource kafkaResource; - private Project project; - - /** - * Set the project of the featurestore (provided by parent resource) - * - * @param projectId the id of the project - */ - public void setProjectId(Integer projectId) throws ProjectException { - this.project = projectController.findProjectById(projectId); + @Override + protected ProjectController getProjectController() { + return projectController; } /** @@ -141,9 +135,9 @@ public Response getFeaturestores( allowableValues = "include_shared=false,include_shared=true", defaultValue = "true") Boolean includeShared - ) - throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { List featurestores; + Project project = getProject(); if (includeShared) { featurestores = featurestoreController.getFeaturestoresForProject(project); } else { @@ -175,12 +169,12 @@ public Response getFeaturestores( public Response getFeaturestore(@ApiParam(value = "Id of the featurestore", required = true) @PathParam("featurestoreId") Integer featurestoreId, @Context HttpServletRequest req, - @Context SecurityContext sc) throws FeaturestoreException { + @Context SecurityContext sc) throws FeaturestoreException, ProjectException { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } FeaturestoreDTO featurestoreDTO = - featurestoreController.getFeaturestoreDTOForProjectWithId(project, featurestoreId); + featurestoreController.getFeaturestoreDTOForProjectWithId(getProject(), featurestoreId); GenericEntity featurestoreDTOGeneric = new GenericEntity(featurestoreDTO) { }; @@ -232,10 +226,10 @@ public Response getFeaturestoreSettings(@Context HttpServletRequest req, @Contex public Response getFeaturestoreByName(@ApiParam(value = "Id of the featurestore", required = true) @PathParam("name") String name, @Context HttpServletRequest req, - @Context SecurityContext sc) throws FeaturestoreException { + @Context SecurityContext sc) throws FeaturestoreException, ProjectException { verifyNameProvided(name); FeaturestoreDTO featurestoreDTO = - featurestoreController.getFeaturestoreForProjectWithName(project, name); + featurestoreController.getFeaturestoreForProjectWithName(getProject(), name); GenericEntity featurestoreDTOGeneric = new GenericEntity(featurestoreDTO) { }; @@ -251,11 +245,8 @@ public Response getFeaturestoreByName(@ApiParam(value = "Id of the featurestore" */ @Path("/{featurestoreId}/featuregroups") public FeaturegroupService featuregroupService(@PathParam("featurestoreId") Integer featurestoreId, - @Context SecurityContext sc) throws FeaturestoreException { - featuregroupService.setProject(project); - if (featurestoreId == null) { - throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); - } + @Context SecurityContext sc) { + featuregroupService.setProjectId(getProjectId()); featuregroupService.setFeaturestoreId(featurestoreId); return featuregroupService; } @@ -268,24 +259,22 @@ public FeaturegroupService featuregroupService(@PathParam("featurestoreId") Inte * @throws FeaturestoreException */ @Path("/{featurestoreId}/trainingdatasets") - public TrainingDatasetService trainingDatasetService(@PathParam("featurestoreId") Integer featurestoreId) - throws FeaturestoreException { - trainingDatasetService.setProject(project); + public TrainingDatasetService trainingDatasetService(@PathParam("featurestoreId") Integer featurestoreId) { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } + trainingDatasetService.setProjectId(getProjectId()); trainingDatasetService.setFeaturestoreId(featurestoreId); return trainingDatasetService; } @Path("/{featurestoreId}/featureview") - public FeatureViewService featureViewService(@PathParam("featurestoreId") Integer featurestoreId) - throws FeaturestoreException { - featureViewService.setProject(project); + public FeatureViewService featureViewService(@PathParam("featurestoreId") Integer featurestoreId) { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } - featureViewService.setFeaturestore(featurestoreId); + featureViewService.setProjectId(getProjectId()); + featureViewService.setFeaturestoreId(featurestoreId); return featureViewService; } @@ -298,18 +287,19 @@ public FeatureViewService featureViewService(@PathParam("featurestoreId") Intege */ @Path("/{featurestoreId}/storageconnectors") public FeaturestoreStorageConnectorService storageConnectorService( - @PathParam("featurestoreId") Integer featurestoreId) throws FeaturestoreException { - featurestoreStorageConnectorService.setProject(project); + @PathParam("featurestoreId") Integer featurestoreId) { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } + featurestoreStorageConnectorService.setProjectId(getProjectId()); featurestoreStorageConnectorService.setFeaturestoreId(featurestoreId); return featurestoreStorageConnectorService; } - + @Path("/query") public FsQueryConstructorResource constructQuery() { - return fsQueryConstructorResource.setProject(project); + fsQueryConstructorResource.setProjectId(getProjectId()); + return fsQueryConstructorResource; } @Path("keywords") @@ -323,10 +313,10 @@ public FsQueryConstructorResource constructQuery() { @ApiOperation(value = "Get available keywords for the featurestore", response = KeywordDTO.class) public Response getUsedKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, - @Context UriInfo uriInfo) { + @Context UriInfo uriInfo) throws ProjectException { List keywords = keywordCtrl.getAllKeywords(); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); - KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, keywords); + KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, getProject(), keywords); return Response.ok().entity(dto).build(); } @@ -349,14 +339,12 @@ private void verifyNameProvided(String featureStoreName) { * @throws FeaturestoreException */ @Path("/{featurestoreId}/transformationfunctions") - public TransformationFunctionResource transformationResource( - @PathParam("featurestoreId") Integer featurestoreId) throws FeaturestoreException { - this.transformationFunctionResource.setProject(project); + public TransformationFunctionResource transformationResource(@PathParam("featurestoreId") Integer featurestoreId) { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } - this.transformationFunctionResource.setFeaturestore( - featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId)); + this.transformationFunctionResource.setProjectId(getProjectId()); + this.transformationFunctionResource.setFeaturestoreId(featurestoreId); return transformationFunctionResource; } @@ -368,14 +356,12 @@ public TransformationFunctionResource transformationResource( * @throws FeaturestoreException */ @Path("/{featurestoreId}/greatexpectations") - public GreatExpectationResource greatExpectationResource( - @PathParam("featurestoreId") Integer featurestoreId) throws FeaturestoreException { - this.greatExpectationResource.setProject(project); + public GreatExpectationResource greatExpectationResource(@PathParam("featurestoreId") Integer featurestoreId) { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } - this.greatExpectationResource.setFeaturestore( - featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId)); + this.greatExpectationResource.setProjectId(getProjectId()); + this.greatExpectationResource.setFeaturestoreId(featurestoreId); return greatExpectationResource; } @@ -387,14 +373,13 @@ public GreatExpectationResource greatExpectationResource( * @throws FeaturestoreException */ @Path("{featurestoreId}/kafka") - public KafkaResource kafkaResource(@PathParam("featurestoreId") Integer featurestoreId) - throws FeaturestoreException { + public KafkaResource kafkaResource(@PathParam("featurestoreId") Integer featurestoreId) { if (featurestoreId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ID_NOT_PROVIDED.getMessage()); } - //This call verifies that the project have access to the featurestoreId provided - Featurestore featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId); - this.kafkaResource.setProject(featurestore.getProject()); + //FeaturestoreId is set to verifies that the project have access to the featurestoreId provided + this.kafkaResource.setFeaturestoreId(featurestoreId); + this.kafkaResource.setProjectId(getProjectId()); return this.kafkaResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreSubResource.java new file mode 100644 index 0000000000..875c3a6fa5 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreSubResource.java @@ -0,0 +1,47 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.featurestore; + +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.project.Project; + +public abstract class FeaturestoreSubResource extends ProjectSubResource { + + private Integer featurestoreId; + + public Integer getFeaturestoreId() { + return featurestoreId; + } + + public void setFeaturestoreId(Integer featurestoreId) { + this.featurestoreId = featurestoreId; + } + + public Featurestore getFeaturestore(Project project) throws ProjectException, FeaturestoreException { + return getFeaturestoreController().getFeaturestoreForProjectWithId(project, featurestoreId); + } + + public Featurestore getFeaturestore() throws ProjectException, FeaturestoreException { + Project project = getProject(); + return getFeaturestoreController().getFeaturestoreForProjectWithId(project, featurestoreId); + } + + protected abstract FeaturestoreController getFeaturestoreController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FsQueryConstructorResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FsQueryConstructorResource.java index 3196667dd6..55f1ed6482 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FsQueryConstructorResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FsQueryConstructorResource.java @@ -16,16 +16,18 @@ package io.hops.hopsworks.api.featurestore; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.common.featurestore.query.FsQueryDTO; import io.hops.hopsworks.common.featurestore.query.QueryDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; @@ -35,6 +37,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.PUT; import javax.ws.rs.Produces; @@ -47,18 +50,18 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Query constructor service") -public class FsQueryConstructorResource { +public class FsQueryConstructorResource extends ProjectSubResource { @EJB private FsQueryBuilder fsQueryBuilder; @EJB private JWTHelper jWTHelper; + @EJB + private ProjectController projectController; - private Project project; - - public FsQueryConstructorResource setProject(Project project) { - this.project = project; - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } /** @@ -80,13 +83,15 @@ public FsQueryConstructorResource setProject(Project project) { @ApiKeyRequired(acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Construct the SQL query to join the requested features", response = FsQueryDTO.class) - public Response constructQuery(@Context SecurityContext sc, @Context UriInfo uriInfo, - QueryDTO queryDto) throws FeaturestoreException, ServiceException { + public Response constructQuery(@Context SecurityContext sc, + @Context UriInfo uriInfo, + @Context HttpServletRequest req, + QueryDTO queryDto) throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); if (queryDto == null) { throw new IllegalArgumentException("Please submit a query to construct"); } - FsQueryDTO fsQueryDTO = fsQueryBuilder.build(uriInfo, project, user, queryDto); + FsQueryDTO fsQueryDTO = fsQueryBuilder.build(uriInfo, getProject(), user, queryDto); return Response.ok().entity(fsQueryDTO).build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/activities/ActivityResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/activities/ActivityResource.java index 77a55dce40..37021c1d63 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/activities/ActivityResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/activities/ActivityResource.java @@ -16,6 +16,8 @@ package io.hops.hopsworks.api.featurestore.activities; +import io.hops.hopsworks.api.featurestore.FeaturestoreCommonSubResource; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; @@ -25,8 +27,10 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.ActivitiesException; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; @@ -42,6 +46,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.GET; import javax.ws.rs.Produces; @@ -54,7 +59,7 @@ @Api(value = "Feature store activity Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ActivityResource { +public class ActivityResource extends FeaturestoreCommonSubResource { @EJB private FeaturegroupController featuregroupController; @@ -67,32 +72,37 @@ public class ActivityResource { @EJB private JWTHelper jwtHelper; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - private TrainingDataset trainingDataset; - private FeatureView featureView; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - public void setProject(Project project) { - this.project = project; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureGroupId(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } - public void setTrainingDatasetById(Integer trainingDatasetId) throws FeaturestoreException { - this.trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); + @Override + protected TrainingDatasetController getTrainingDatasetController() { + return trainingDatasetController; } - public void setFeatureViewByNameAndVersion(String name, Integer version) throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } + @GET @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "List activities on this entity") @@ -104,7 +114,8 @@ public void setFeatureViewByNameAndVersion(String name, Integer version) throws public Response get(@BeanParam Pagination pagination, @BeanParam ActivitiesBeanParam activitiesBeanParam, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws FeaturestoreException, ActivitiesException { + @Context HttpServletRequest req, + @Context SecurityContext sc) throws FeaturestoreException, ActivitiesException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ACTIVITIES); @@ -113,7 +124,11 @@ public Response get(@BeanParam Pagination pagination, resourceRequest.setSort(activitiesBeanParam.getSortBySet()); resourceRequest.setFilter(activitiesBeanParam.getFilterBySet()); resourceRequest.setExpansions(activitiesBeanParam.getExpansions().getResources()); - + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featureStore); + Featuregroup featuregroup = getFeaturegroup(featureStore); + FeatureView featureView = getFeatureView(featureStore); ActivityDTO dto = null; if (featuregroup != null) { dto = activityBuilder.build(uriInfo, resourceRequest, project, user, featuregroup); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java index 2d2fec4524..2e4c758165 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/code/CodeResource.java @@ -17,19 +17,23 @@ package io.hops.hopsworks.api.featurestore.code; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.code.CodeActions; import io.hops.hopsworks.common.featurestore.code.CodeController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; import io.hops.hopsworks.common.jupyter.NotebookConversion; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.HopsSecurityException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.exceptions.UserException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -64,7 +68,7 @@ @Api(value = "Feature store code Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class CodeResource { +public class CodeResource extends FeatureGroupSubResource { @EJB private CodeBuilder codeBuilder; @@ -76,26 +80,35 @@ public class CodeResource { private FeaturegroupController featuregroupController; @EJB private TrainingDatasetController trainingDatasetController; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - private TrainingDataset trainingDataset; - - public void setProject(Project project) { - this.project = project; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureStore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureGroupId(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } + private Integer trainingDatasetId; - public void setTrainingDatasetId(Integer trainingDatasetId) throws FeaturestoreException { - this.trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); + public void setTrainingDatasetId(Integer trainingDatasetId) { + this.trainingDatasetId = trainingDatasetId; + } + private TrainingDataset getTrainingDataset(Featurestore featurestore) throws FeaturestoreException { + if (trainingDatasetId != null) { + return trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); + } + return null; } @GET @@ -109,7 +122,8 @@ public void setTrainingDatasetId(Integer trainingDatasetId) throws FeaturestoreE public Response get(@BeanParam Pagination pagination, @BeanParam CodeBeanParam codeBeanParam, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws FeaturestoreException, ServiceException { + @Context HttpServletRequest req, + @Context SecurityContext sc) throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); @@ -119,7 +133,10 @@ public Response get(@BeanParam Pagination pagination, resourceRequest.setSort(codeBeanParam.getSortBySet()); resourceRequest.setFilter(codeBeanParam.getFilterSet()); resourceRequest.setField(codeBeanParam.getFieldSet()); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); + Featuregroup featuregroup = getFeaturegroup(featurestore); CodeDTO dto; if (featuregroup != null) { dto = codeBuilder.build(uriInfo, resourceRequest, project, user, featurestore, featuregroup, @@ -145,7 +162,8 @@ public Response get(@BeanParam Pagination pagination, @BeanParam CodeBeanParam codeBeanParam, @PathParam("codeId") Integer codeId, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws FeaturestoreException, ServiceException { + @Context HttpServletRequest req, + @Context SecurityContext sc) throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); @@ -155,7 +173,10 @@ public Response get(@BeanParam Pagination pagination, resourceRequest.setSort(codeBeanParam.getSortBySet()); resourceRequest.setFilter(codeBeanParam.getFilterSet()); resourceRequest.setField(codeBeanParam.getFieldSet()); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); + Featuregroup featuregroup = getFeaturegroup(featurestore); CodeDTO dto; if (featuregroup != null) { dto = codeBuilder.build(uriInfo, resourceRequest, project, user, featuregroup, codeId, @@ -183,13 +204,18 @@ public Response post(@Context UriInfo uriInfo, @QueryParam("type") CodeActions.RunType type, @QueryParam("databricksClusterId") String databricksClusterId, CodeDTO codeDTO) - throws FeaturestoreException, DatasetException, HopsSecurityException, ServiceException, UserException { + throws FeaturestoreException, DatasetException, HopsSecurityException, ServiceException, UserException, + ProjectException { Users user = jWTHelper.getUserPrincipal(sc); String databricksNotebook = null; byte[] databricksArchive = null; + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); + Featuregroup featuregroup = getFeaturegroup(featurestore); CodeDTO dto; if (featuregroup != null) { FeaturestoreCode featurestoreCode = codeController.registerCode(project, user, codeDTO.getCommitTime(), diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/commit/CommitResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/commit/CommitResource.java index a4da001b82..c77a6d6ee0 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/commit/CommitResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/commit/CommitResource.java @@ -16,19 +16,21 @@ package io.hops.hopsworks.api.featurestore.commit; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featuregroup.cached.FeatureGroupCommitController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; -import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; - import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.FeatureGroupCommit; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; @@ -55,7 +57,7 @@ @Api(value = "Feature Group commit Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class CommitResource { +public class CommitResource extends FeatureGroupSubResource { @EJB private CommitBuilder commitBuilder; @@ -66,23 +68,27 @@ public class CommitResource { @EJB private JWTHelper jwtHelper; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - public CommitResource setProject(Project project) { - this.project = project; - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(this.featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } + @POST @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @@ -94,8 +100,10 @@ public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response commitFeatureGroup(@Context UriInfo uriInfo, CommitDTO commitDTO, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); FeatureGroupCommit featureGroupCommit = featureGroupCommitController.createHudiFeatureGroupCommit(user, featuregroup, commitDTO.getCommitTime(), commitDTO.getRowsUpdated(), commitDTO.getRowsInserted(), @@ -117,14 +125,14 @@ public Response getFeatureGroupCommit(@Context UriInfo uriInfo, @BeanParam Pagination pagination, @BeanParam CommitBeanParam commitBeanParam, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMITS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(commitBeanParam.getSortBySet()); resourceRequest.setFilter(commitBeanParam.getFilter()); - - CommitDTO builtCommitDTO = commitBuilder.build(uriInfo, resourceRequest, project, featuregroup); + Project project = getProject(); + CommitDTO builtCommitDTO = commitBuilder.build(uriInfo, resourceRequest, project, getFeaturegroup(project)); return Response.ok().entity(builtCommitDTO).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureGroupValidationAlertResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureGroupValidationAlertResource.java index 1c9ffa9222..621b28d572 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureGroupValidationAlertResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureGroupValidationAlertResource.java @@ -22,8 +22,13 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.datavalidation.FeatureGroupAlertFacade; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.alert.FeatureGroupAlert; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; +import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.ApiOperation; @@ -74,13 +79,17 @@ public Response createOrUpdate( @Context HttpServletRequest req, @Context - SecurityContext sc) throws FeaturestoreException { - featureStoreAlertValidation.validateEntityType(getEntityType(), this.featuregroup, this.featureView); + SecurityContext sc) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validateEntityType(getEntityType(), featuregroup, featureView); if (dto == null) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ALERT_ILLEGAL_ARGUMENT, Level.FINE, Constants.NO_PAYLOAD); } - FeatureGroupAlert featureGroupAlert = featureGroupAlertFacade.findByFeatureGroupAndId(this.featuregroup, id); + FeatureGroupAlert featureGroupAlert = featureGroupAlertFacade.findByFeatureGroupAndId(featuregroup, id); featureStoreAlertValidation.validateUpdate(featureGroupAlert, dto.getStatus(), featuregroup); featureGroupAlert = featureStoreAlertController.updateAlert(dto, featureGroupAlert, project); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureStoreAlertResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureStoreAlertResource.java index d63f53e036..3141313dee 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureStoreAlertResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidation/alert/FeatureStoreAlertResource.java @@ -15,7 +15,7 @@ */ package io.hops.hopsworks.api.featurestore.datavalidation.alert; -import io.hops.hopsworks.alert.dao.AlertReceiverFacade; +import com.google.common.base.Strings; import io.hops.hopsworks.alert.exception.AlertManagerAccessControlException; import io.hops.hopsworks.alert.exception.AlertManagerUnreachableException; import io.hops.hopsworks.alerting.api.alert.dto.Alert; @@ -25,28 +25,32 @@ import io.hops.hopsworks.api.alert.AlertDTO; import io.hops.hopsworks.api.alert.FeatureStoreAlertController; import io.hops.hopsworks.api.alert.FeatureStoreAlertValidation; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.featurestore.featureview.FeatureViewAlertBuilder; import io.hops.hopsworks.api.featurestore.featureview.FeatureViewAlertDTO; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.project.alert.ProjectAlertsDTO; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.alert.AlertController; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.api.RestDTO; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.datavalidation.FeatureGroupAlertFacade; -import io.hops.hopsworks.common.featurestore.featuremonitoring.alert.FeatureMonitoringAlertController; +import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewAlertFacade; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.AlertException; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.featureview.alert.FeatureViewAlert; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidation.alert.FeatureGroupAlert; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.alert.FeatureViewAlert; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.hops.hopsworks.restutils.RESTCodes; @@ -77,7 +81,7 @@ @Api(value = "FeatureStoreAlert Resource") @TransactionAttribute(TransactionAttributeType.NEVER) -public abstract class FeatureStoreAlertResource { +public abstract class FeatureStoreAlertResource extends FeaturestoreSubResource { private static final Logger LOGGER = Logger.getLogger(FeatureStoreAlertResource.class.getName()); @EJB @@ -89,40 +93,60 @@ public abstract class FeatureStoreAlertResource { @EJB private AlertBuilder alertBuilder; @EJB - private AlertReceiverFacade alertReceiverFacade; - @EJB private FeatureViewController featureViewController; @EJB private FeatureViewAlertBuilder featureViewAlertBuilder; @EJB protected FeatureViewAlertFacade featureViewAlertFacade; @EJB - protected FeatureMonitoringAlertController featureMonitoringAlertController; - @EJB protected FeatureStoreAlertController featureStoreAlertController; @EJB protected FeatureStoreAlertValidation featureStoreAlertValidation; - protected Featuregroup featuregroup; - protected FeatureView featureView; - protected Featurestore featureStore; - protected Project project; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; + @EJB + private FeaturegroupController featuregroupController; + + private String featureViewName; + private Integer featureViewVersion; + private Integer featureGroupId; - public void setFeatureGroup(Featuregroup featuregroup) { - this.featuregroup = featuregroup; + public void setFeatureView(String name, Integer version) { + this.featureViewName = name; + this.featureViewVersion = version; } - public void setFeatureView(String name, Integer version, Featurestore featurestore) throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); + public void setFeatureGroupId(Integer featureGroupId) { + this.featureGroupId = featureGroupId; } - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + protected FeatureView getFeatureView(Featurestore featurestore) throws FeaturestoreException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return featureViewController.getByNameVersionAndFeatureStore(this.featureViewName, this.featureViewVersion, + featurestore); } - public void setProject(Project project) { - this.project = project; + protected Featuregroup getFeatureGroup(Featurestore featurestore) throws FeaturestoreException { + if (featureGroupId != null) { + return featuregroupController.getFeaturegroupById(featurestore, this.featureGroupId); + } + return null; + } + + @Override + protected ProjectController getProjectController() { + return projectController; + } + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } + protected abstract ResourceRequest.Name getEntityType(); @GET @@ -142,21 +166,25 @@ public Response get( @Context HttpServletRequest req, @Context - SecurityContext sc) throws FeaturestoreException { + SecurityContext sc) throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(featureGroupAlertBeanParam.getSortBySet()); resourceRequest.setFilter(featureGroupAlertBeanParam.getFilter()); - featureStoreAlertValidation.validateEntityType(getEntityType(), this.featuregroup, this.featureView); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validateEntityType(getEntityType(), featuregroup, featureView); FeatureGroupAlertDTO dto; FeatureViewAlertDTO featureViewAlertDto; if (getEntityType().equals(ResourceRequest.Name.FEATUREGROUPS)) { - dto = featureGroupAlertBuilder.buildItems(uriInfo, resourceRequest, this.featuregroup); + dto = featureGroupAlertBuilder.buildItems(uriInfo, resourceRequest, featuregroup); return Response.ok().entity(dto).build(); } else { featureViewAlertDto = featureViewAlertBuilder.buildMany(uriInfo, resourceRequest, - featureStoreAlertController.retrieveManyAlerts(resourceRequest, this.featureView)); + featureStoreAlertController.retrieveManyAlerts(resourceRequest, featureView)); return Response.ok().entity(featureViewAlertDto).build(); } } @@ -178,15 +206,19 @@ public Response getById( HttpServletRequest req, @Context SecurityContext sc) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); - featureStoreAlertValidation.validateEntityType(getEntityType(), this.featuregroup, this.featureView); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validateEntityType(getEntityType(), featuregroup, featureView); if (getEntityType().equals(ResourceRequest.Name.FEATUREGROUPS)) { - FeatureGroupAlertDTO dto = featureGroupAlertBuilder.build(uriInfo, resourceRequest, this.featuregroup, id); + FeatureGroupAlertDTO dto = featureGroupAlertBuilder.build(uriInfo, resourceRequest, featuregroup, id); return Response.ok().entity(dto).build(); } else { FeatureViewAlertDTO dto = featureViewAlertBuilder.buildFeatureViewAlertDto(uriInfo, resourceRequest, - featureStoreAlertController.retrieveSingleAlert(id, this.featureView)); + featureStoreAlertController.retrieveSingleAlert(id, featureView)); return Response.ok().entity(dto).build(); } } @@ -226,9 +258,13 @@ public Response create(PostableFeatureStoreAlerts dto, @Context HttpServletRequest req, @Context - SecurityContext sc) throws FeaturestoreException, AlertException { + SecurityContext sc) throws FeaturestoreException, AlertException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); - featureStoreAlertValidation.validateEntityType(getEntityType(), this.featuregroup, this.featureView); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validateEntityType(getEntityType(), featuregroup, featureView); if (getEntityType().equals(ResourceRequest.Name.FEATUREGROUPS)) { FeatureGroupAlertDTO featureGroupAlertDTO = createAlert(dto, bulk, uriInfo, resourceRequest); return Response.created(featureGroupAlertDTO.getHref()).entity(featureGroupAlertDTO).build(); @@ -242,7 +278,7 @@ public Response create(PostableFeatureStoreAlerts dto, private FeatureGroupAlertDTO createAlert(PostableFeatureStoreAlerts featureGroupAlertDTO, Boolean bulk, - UriInfo uriInfo, ResourceRequest resourceRequest) throws FeaturestoreException { + UriInfo uriInfo, ResourceRequest resourceRequest) throws FeaturestoreException, ProjectException { FeatureGroupAlertDTO dto; if (bulk) { featureStoreAlertValidation.validateBulk(featureGroupAlertDTO); @@ -257,12 +293,15 @@ private FeatureGroupAlertDTO createAlert(PostableFeatureStoreAlerts featureGroup return dto; } - private FeatureGroupAlertDTO createAlert(PostableFeatureStoreAlerts dto, UriInfo uriInfo, - ResourceRequest resourceRequest) throws FeaturestoreException { - featureStoreAlertValidation.validate(dto, this.featuregroup, this.featureView); + ResourceRequest resourceRequest) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validate(dto, featuregroup, featureView); FeatureGroupAlert featureGroupAlert = - featureStoreAlertController.persistFeatureGroupEntityValues(dto, this.featuregroup); + featureStoreAlertController.persistFeatureGroupEntityValues(dto, featuregroup); featureStoreAlertController.createRoute(project,featureGroupAlert); return featureGroupAlertBuilder.buildItems(uriInfo, resourceRequest, featureGroupAlert); } @@ -283,8 +322,9 @@ public Response getTestById( @Context HttpServletRequest req, @Context - SecurityContext sc) throws AlertException { + SecurityContext sc) throws AlertException, ProjectException { List alerts; + Project project = getProject(); FeatureGroupAlert featureGroupAlert=null; FeatureViewAlert featureViewAlert=null; try { @@ -331,17 +371,21 @@ public Response deleteById( @Context HttpServletRequest req, @Context - SecurityContext sc) throws FeaturestoreException { - featureStoreAlertValidation.validateEntityType(getEntityType(), this.featuregroup, this.featureView); + SecurityContext sc) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validateEntityType(getEntityType(), featuregroup, featureView); if (getEntityType().equals(ResourceRequest.Name.FEATUREGROUPS)) { - FeatureGroupAlert featureGroupAlert = featureGroupAlertFacade.findByFeatureGroupAndId(this.featuregroup, id); + FeatureGroupAlert featureGroupAlert = featureGroupAlertFacade.findByFeatureGroupAndId(featuregroup, id); if (featureGroupAlert != null) { featureStoreAlertController.deleteRoute(featureGroupAlert, project); featureGroupAlertFacade.remove(featureGroupAlert); } return Response.noContent().build(); } else { - FeatureViewAlert featureViewAlert = featureViewAlertFacade.findByFeatureViewAndId(this.featureView, id); + FeatureViewAlert featureViewAlert = featureViewAlertFacade.findByFeatureViewAndId(featureView, id); if (featureViewAlert != null) { featureStoreAlertController.deleteRoute(featureViewAlert, project); featureViewAlertFacade.remove(featureViewAlert); @@ -351,7 +395,7 @@ public Response deleteById( } private FeatureViewAlertDTO createFeatureViewAlert(PostableFeatureStoreAlerts paDTO, Boolean bulk, - UriInfo uriInfo, ResourceRequest resourceRequest) throws FeaturestoreException { + UriInfo uriInfo, ResourceRequest resourceRequest) throws FeaturestoreException, ProjectException { FeatureViewAlertDTO dto; if (bulk) { featureStoreAlertValidation.validateBulk(paDTO); @@ -367,11 +411,15 @@ private FeatureViewAlertDTO createFeatureViewAlert(PostableFeatureStoreAlerts pa } private FeatureViewAlertDTO createFeatureViewAlert(PostableFeatureStoreAlerts dto, UriInfo uriInfo, - ResourceRequest resourceRequest) throws FeaturestoreException { - featureStoreAlertValidation.validate(dto, this.featuregroup, this.featureView); + ResourceRequest resourceRequest) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validate(dto, featuregroup, featureView); FeatureViewAlert featureViewAlert; FeatureViewAlertDTO fvDTO; - featureViewAlert = featureStoreAlertController.persistFeatureViewEntityValues(dto, this.featureView); + featureViewAlert = featureStoreAlertController.persistFeatureViewEntityValues(dto, featureView); fvDTO = featureViewAlertBuilder.buildFeatureViewAlertDto(uriInfo, resourceRequest, featureViewAlert); featureStoreAlertController.createRoute(project,featureViewAlert); return fvDTO; diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/expectations/ExpectationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/expectations/ExpectationResource.java index d991d591f0..8f6730f3af 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/expectations/ExpectationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/expectations/ExpectationResource.java @@ -16,17 +16,20 @@ package io.hops.hopsworks.api.featurestore.datavalidationv2.expectations; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.datavalidationv2.suites.ExpectationSuiteSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.datavalidationv2.expectations.ExpectationController; import io.hops.hopsworks.common.featurestore.datavalidationv2.expectations.ExpectationDTO; import io.hops.hopsworks.common.featurestore.datavalidationv2.suites.ExpectationSuiteController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidationv2.Expectation; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidationv2.ExpectationSuite; @@ -44,8 +47,8 @@ import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; -import javax.ws.rs.PUT; import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -58,7 +61,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Expectation resource") -public class ExpectationResource { +public class ExpectationResource extends ExpectationSuiteSubResource { @EJB private FeaturegroupController featuregroupController; @@ -70,26 +73,29 @@ public class ExpectationResource { private ExpectationController expectationController; @EJB private JWTHelper jWTHelper; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - private ExpectationSuite expectationSuite; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } - public void setExpectationSuite(Integer expectationSuiteId) { - this.expectationSuite = expectationSuiteController.getExpectationSuiteById(expectationSuiteId); + @Override + protected ExpectationSuiteController getExpectationSuiteController() { + return expectationSuiteController; } /** @@ -114,11 +120,12 @@ public Response get( @Context UriInfo uriInfo, @PathParam("expectationId") - Integer expectationId) throws FeaturestoreException { + Integer expectationId) throws FeaturestoreException, ProjectException { Expectation expectation = expectationController.getExpectationById(expectationId); - + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); return Response.ok() - .entity(expectationBuilder.build(uriInfo, project, featuregroup, expectationSuite, expectation)) + .entity(expectationBuilder.build(uriInfo, project, featuregroup, getExpectationSuite(), expectation)) .build(); } @@ -145,10 +152,12 @@ public Response createExpectation( HttpServletRequest req, @Context UriInfo uriInfo, - ExpectationDTO expectationDTO) throws FeaturestoreException { + ExpectationDTO expectationDTO) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); + ExpectationSuite expectationSuite = getExpectationSuite(); boolean logActivity = true; boolean verifyInput = true; Expectation expectation = expectationController.createOrUpdateExpectation( @@ -186,10 +195,12 @@ public Response updateExpectation( UriInfo uriInfo, @PathParam("expectationId") Integer expectationId, - ExpectationDTO expectationDTO) throws FeaturestoreException { + ExpectationDTO expectationDTO) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); + ExpectationSuite expectationSuite = getExpectationSuite(); boolean logActivity = true; boolean verifyInput = true; Expectation expectation = expectationController.createOrUpdateExpectation( @@ -251,9 +262,9 @@ public Response getExpectationByExpectationSuite( @Context HttpServletRequest req, @Context - UriInfo uriInfo) throws FeaturestoreException { - - ExpectationDTO dtos = expectationBuilder.build(uriInfo, project, featuregroup, expectationSuite); + UriInfo uriInfo) throws FeaturestoreException, ProjectException { + Project project = getProject(); + ExpectationDTO dtos = expectationBuilder.build(uriInfo, project, getFeaturegroup(project), getExpectationSuite()); return Response.ok().entity(dtos).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/greatexpectations/GreatExpectationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/greatexpectations/GreatExpectationResource.java index 1518e086e8..92e72f23e8 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/greatexpectations/GreatExpectationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/greatexpectations/GreatExpectationResource.java @@ -16,9 +16,14 @@ package io.hops.hopsworks.api.featurestore.datavalidationv2.greatexpectations; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.project.ProjectController; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.project.Project; @@ -43,20 +48,24 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Great expectation resource", description = "A service to retrieve all support great expectations") -public class GreatExpectationResource { +public class GreatExpectationResource extends FeaturestoreSubResource { @EJB GreatExpectationBuilder greatExpectationBuilder; - private Project project; - private Featurestore featurestore; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } @ApiOperation(value = "Fetch all validation report from feature group", @@ -74,8 +83,9 @@ public Response getAll( @Context HttpServletRequest req, @Context - UriInfo uriInfo) { - + UriInfo uriInfo) throws ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); GreatExpectationDTO dtos = greatExpectationBuilder.build(uriInfo, project, featurestore); return Response.ok().entity(dtos).build(); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportResource.java index dd10cfddc6..12849315f4 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportResource.java @@ -16,18 +16,21 @@ package io.hops.hopsworks.api.featurestore.datavalidationv2.reports; -import io.hops.hopsworks.common.featurestore.datavalidationv2.reports.ValidationReportController; -import io.hops.hopsworks.common.featurestore.datavalidationv2.reports.ValidationReportDTO; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.datavalidationv2.reports.ValidationReportController; +import io.hops.hopsworks.common.featurestore.datavalidationv2.reports.ValidationReportDTO; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidationv2.ValidationReport; import io.hops.hopsworks.persistence.entity.project.Project; @@ -43,10 +46,10 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.PUT; import javax.ws.rs.Path; -import javax.ws.rs.DELETE; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.Context; @@ -58,7 +61,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Validation report resource") -public class ValidationReportResource { +public class ValidationReportResource extends FeatureGroupSubResource { @EJB private FeaturegroupController featuregroupController; @@ -68,21 +71,24 @@ public class ValidationReportResource { private ValidationReportBuilder validationReportBuilder; @EJB private JWTHelper jWTHelper; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } /** @@ -108,12 +114,12 @@ public Response getById( @Context UriInfo uriInfo, @PathParam("validationReportId") - Integer validationReportId) throws FeaturestoreException { + Integer validationReportId) throws FeaturestoreException, ProjectException { ValidationReport validationReport = validationReportController.getValidationReportById(validationReportId); - + Project project = getProject(); ValidationReportDTO dto = validationReportBuilder.build( - uriInfo, project, featuregroup, validationReport); + uriInfo, project, getFeaturegroup(project), validationReport); return Response.ok().entity(dto).build(); } @@ -142,15 +148,16 @@ public Response getAll( @Context HttpServletRequest req, @Context - UriInfo uriInfo) { + UriInfo uriInfo) throws ProjectException, FeaturestoreException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.VALIDATIONREPORT); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(validationReportBeanParam.getSortBySet()); resourceRequest.setFilter(validationReportBeanParam.getFilter()); - - ValidationReportDTO dtos = validationReportBuilder.build(uriInfo, resourceRequest, project, featuregroup); + Project project = getProject(); + ValidationReportDTO dtos = + validationReportBuilder.build(uriInfo, resourceRequest, project, getFeaturegroup(project)); return Response.ok().entity(dtos).build(); } @@ -178,10 +185,11 @@ public Response createValidationReport( HttpServletRequest req, @Context UriInfo uriInfo, - ValidationReportDTO validationReportDTO) throws FeaturestoreException { - - Users user = jWTHelper.getUserPrincipal(sc); + ValidationReportDTO validationReportDTO) throws FeaturestoreException, ProjectException { + Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); ValidationReport validationReport = validationReportController.createValidationReport( user, featuregroup, validationReportDTO); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/results/ValidationResultResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/results/ValidationResultResource.java index eeea8f45c2..d77e78a44f 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/results/ValidationResultResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/results/ValidationResultResource.java @@ -16,18 +16,19 @@ package io.hops.hopsworks.api.featurestore.datavalidationv2.results; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.datavalidationv2.results.ValidationResultDTO; -import io.hops.hopsworks.common.featurestore.datavalidationv2.suites.ExpectationSuiteController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; @@ -52,28 +53,29 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Expectation resource") -public class ValidationResultResource { +public class ValidationResultResource extends FeatureGroupSubResource { @EJB private FeaturegroupController featuregroupController; @EJB - private ExpectationSuiteController expectationSuiteController; - @EJB private ValidationResultBuilder validationResultBuilder; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } /** @@ -105,16 +107,17 @@ public Response getAll( @Context UriInfo uriInfo, @PathParam("expectationId") - Integer expectationId) throws FeaturestoreException { + Integer expectationId) throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.VALIDATIONRESULT); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(validationResultBeanParam.getSortBySet()); resourceRequest.setFilter(validationResultBeanParam.getFilter()); + Project project = getProject(); ValidationResultDTO dtos = - validationResultBuilder.buildHistory(uriInfo, resourceRequest, project, featuregroup, expectationId); + validationResultBuilder.buildHistory(uriInfo, resourceRequest, project, getFeaturegroup(project), expectationId); return Response.ok().entity(dtos).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteResource.java index c4b58d715e..a1e83a333e 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteResource.java @@ -16,18 +16,21 @@ package io.hops.hopsworks.api.featurestore.datavalidationv2.suites; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.featurestore.datavalidationv2.expectations.ExpectationResource; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jobs.JobDTO; import io.hops.hopsworks.api.jobs.JobsBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.app.FsJobManagerController; import io.hops.hopsworks.common.featurestore.datavalidationv2.suites.ExpectationSuiteController; import io.hops.hopsworks.common.featurestore.datavalidationv2.suites.ExpectationSuiteDTO; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.JobException; @@ -69,7 +72,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Expectation suite resource") -public class ExpectationSuiteResource { +public class ExpectationSuiteResource extends FeatureGroupSubResource { @EJB private JWTHelper jWTHelper; @@ -86,20 +89,24 @@ public class ExpectationSuiteResource { @Inject private ExpectationResource expectationResource; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } /** @@ -121,7 +128,9 @@ public Response get( @Context HttpServletRequest req, @Context - UriInfo uriInfo) throws FeaturestoreException { + UriInfo uriInfo) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); ExpectationSuite expectationSuite = expectationSuiteController.getExpectationSuite(featuregroup); return Response.ok() .entity(expectationSuiteBuilder.build(uriInfo, project, featuregroup, expectationSuite)) @@ -152,8 +161,9 @@ public Response createExpectationSuite( HttpServletRequest req, @Context UriInfo uriInfo, - ExpectationSuiteDTO expectationSuiteDTO) throws FeaturestoreException { - + ExpectationSuiteDTO expectationSuiteDTO) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); if (expectationSuiteController.getExpectationSuite(featuregroup) != null) { throw new FeaturestoreException( RESTCodes.FeaturestoreErrorCode.EXPECTATION_SUITE_ALREADY_EXISTS, @@ -197,10 +207,11 @@ public Response updateExpectationSuite( UriInfo uriInfo, @PathParam("expectationSuiteId") Integer expectationSuiteId, - ExpectationSuiteDTO expectationSuiteDTO) throws FeaturestoreException { + ExpectationSuiteDTO expectationSuiteDTO) throws FeaturestoreException, ProjectException { - Users user = jWTHelper.getUserPrincipal(sc); - + Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); ExpectationSuite expectationSuite = expectationSuiteController.createOrUpdateExpectationSuite( user, featuregroup, expectationSuiteDTO); @@ -238,10 +249,11 @@ public Response updateMetadataExpectationSuite( UriInfo uriInfo, @PathParam("expectationSuiteId") Integer expectationSuiteId, - ExpectationSuiteDTO expectationSuiteDTO) throws FeaturestoreException { + ExpectationSuiteDTO expectationSuiteDTO) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); // Expectations list of the DTO is discarded boolean verifyInput = true; boolean logActivity = true; @@ -275,11 +287,11 @@ public Response deleteExpectationSuite( @Context UriInfo uriInfo, @PathParam("expectationSuiteId") - Integer expectationSuiteId) { + Integer expectationSuiteId) throws ProjectException, FeaturestoreException { Users user = jWTHelper.getUserPrincipal(sc); - expectationSuiteController.deleteExpectationSuite(user, featuregroup); + expectationSuiteController.deleteExpectationSuite(user, getFeaturegroup()); return Response.noContent().build(); } @@ -303,7 +315,9 @@ public Response compute( SecurityContext sc) throws FeaturestoreException, ServiceException, JobException, ProjectException, GenericException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeaturegroup(featurestore); if (expectationSuiteController.getExpectationSuite(featuregroup) == null) { throw new FeaturestoreException( RESTCodes.FeaturestoreErrorCode.NO_EXPECTATION_SUITE_ATTACHED_TO_THIS_FEATUREGROUP, @@ -318,16 +332,15 @@ public Response compute( ///////////////////////////////////////// //// Single Expectation Service //////////////////////////////////////// - + @Path("/{expectationSuiteId: [0-9]+}/expectations") public ExpectationResource expectationResource( @PathParam("expectationSuiteId") - Integer expectationSuiteId) - throws FeaturestoreException { - this.expectationResource.setProject(project); - this.expectationResource.setFeaturestore(featurestore); - this.expectationResource.setFeatureGroup(featuregroup.getId()); - this.expectationResource.setExpectationSuite(expectationSuiteId); + Integer expectationSuiteId) { + this.expectationResource.setProjectId(getProjectId()); + this.expectationResource.setFeaturestoreId(getFeaturestoreId()); + this.expectationResource.setFeatureGroupId(getFeatureGroupId()); + this.expectationResource.setExpectationSuiteId(expectationSuiteId); return expectationResource; } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteSubResource.java new file mode 100644 index 0000000000..8dc7da0afa --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/suites/ExpectationSuiteSubResource.java @@ -0,0 +1,38 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.featurestore.datavalidationv2.suites; + +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; +import io.hops.hopsworks.common.featurestore.datavalidationv2.suites.ExpectationSuiteController; +import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.datavalidationv2.ExpectationSuite; + +public abstract class ExpectationSuiteSubResource extends FeatureGroupSubResource { + private Integer expectationSuiteId; + + public Integer getExpectationSuiteId() { + return expectationSuiteId; + } + + public void setExpectationSuiteId(Integer expectationSuiteId) { + this.expectationSuiteId = expectationSuiteId; + } + + public ExpectationSuite getExpectationSuite() { + return getExpectationSuiteController().getExpectationSuiteById(expectationSuiteId); + } + + protected abstract ExpectationSuiteController getExpectationSuiteController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringConfigurationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringConfigurationResource.java index f775de3828..1ddb5f9426 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringConfigurationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringConfigurationResource.java @@ -19,6 +19,7 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import javax.ejb.EJB; @@ -33,20 +34,24 @@ public class FeatureGroupFeatureMonitoringConfigurationResource extends FeatureM @EJB private FeaturegroupController featuregroupController; - private Featuregroup featureGroup; + private Integer featureGroupId; /** * Sets the feature group of the tag resource * * @param featureGroupId */ - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featureGroup = featuregroupController.getFeaturegroupById(featureStore, featureGroupId); + public void setFeatureGroupId(Integer featureGroupId) throws FeaturestoreException { + this.featureGroupId = featureGroupId; + } + + private Featuregroup getFeatureGroup() throws ProjectException, FeaturestoreException { + return featuregroupController.getFeaturegroupById(getFeaturestore(), featureGroupId); } @Override protected Integer getItemId() { - return featureGroup.getId(); + return this.featureGroupId; } @Override @@ -55,12 +60,12 @@ protected ResourceRequest.Name getItemType() { } @Override - protected String getItemName() { - return featureGroup.getName(); + protected String getItemName() throws ProjectException, FeaturestoreException { + return getFeatureGroup().getName(); } @Override - protected Integer getItemVersion() { - return featureGroup.getVersion(); + protected Integer getItemVersion() throws ProjectException, FeaturestoreException { + return getFeatureGroup().getVersion(); } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringResultResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringResultResource.java index f26feb8b47..e25345b13b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringResultResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupFeatureMonitoringResultResource.java @@ -18,6 +18,7 @@ import io.hops.hopsworks.api.featurestore.featuremonitoring.result.FeatureMonitoringResultResource; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import javax.ejb.EJB; @@ -32,14 +33,18 @@ public class FeatureGroupFeatureMonitoringResultResource extends FeatureMonitori @EJB private FeaturegroupController featuregroupController; - private Featuregroup featureGroup; + private Integer featureGroupId; - /** - * Sets the feature group of the tag resource - * - * @param featureGroupId - */ - public void setFeatureGroup(Integer featureGroupId) throws FeaturestoreException { - this.featureGroup = featuregroupController.getFeaturegroupById(featureStore, featureGroupId); + public void setFeatureGroupId(Integer featureGroupId) throws FeaturestoreException { + this.featureGroupId = featureGroupId; + } + + private Featuregroup getFeatureGroup() throws ProjectException, FeaturestoreException { + return featuregroupController.getFeaturegroupById(getFeaturestore(), featureGroupId); + } + + @Override + protected void check() throws ProjectException, FeaturestoreException { + getFeatureGroup(); } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupPreviewResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupPreviewResource.java index 6612e48c30..eaf34bb853 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupPreviewResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupPreviewResource.java @@ -17,17 +17,19 @@ package io.hops.hopsworks.api.featurestore.featuregroup; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featuregroup.cached.FeatureGroupStorage; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.HopsSecurityException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; @@ -50,7 +52,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class FeatureGroupPreviewResource { +public class FeatureGroupPreviewResource extends FeatureGroupSubResource { @EJB private PreviewBuilder previewBuilder; @@ -60,22 +62,24 @@ public class FeatureGroupPreviewResource { private JWTHelper jwtHelper; @EJB private Settings settings; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - - public FeatureGroupPreviewResource setProject(Project project) { - this.project = project; - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroupId(int featureGroupId) throws FeaturestoreException { - featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } @ApiOperation(value = "Get feature group preview", response = PreviewDTO.class) @@ -88,9 +92,10 @@ public void setFeatureGroupId(int featureGroupId) throws FeaturestoreException { public Response getPreview(@BeanParam FeatureGroupPreviewBeanParam featureGroupPreviewBeanParam, @Context HttpServletRequest req, @Context UriInfo uriInfo, @Context SecurityContext sc) - throws FeaturestoreException, HopsSecurityException { + throws FeaturestoreException, HopsSecurityException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); // validate user input if (featureGroupPreviewBeanParam.getLimit() != null && ( featureGroupPreviewBeanParam.getLimit() < 0 || diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupSubResource.java new file mode 100644 index 0000000000..e576790019 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeatureGroupSubResource.java @@ -0,0 +1,59 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.featurestore.featuregroup; + +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; +import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; +import io.hops.hopsworks.persistence.entity.project.Project; + +public abstract class FeatureGroupSubResource extends FeaturestoreSubResource { + private Integer featureGroupId; + + public Integer getFeatureGroupId() { + return featureGroupId; + } + + public void setFeatureGroupId(Integer featureGroupId) { + this.featureGroupId = featureGroupId; + } + + public Featuregroup getFeaturegroup(Project project) throws FeaturestoreException, ProjectException { + if (featureGroupId != null) { + return getFeaturegroupController().getFeaturegroupById(getFeaturestore(project), featureGroupId); + } + return null; + } + + public Featuregroup getFeaturegroup(Featurestore featurestore) throws FeaturestoreException { + if (featureGroupId != null) { + return getFeaturegroupController().getFeaturegroupById(featurestore, featureGroupId); + } + return null; + } + + public Featuregroup getFeaturegroup() throws ProjectException, FeaturestoreException { + if (featureGroupId != null) { + return getFeaturegroupController().getFeaturegroupById(getFeaturestore(), featureGroupId); + } + return null; + } + + protected abstract FeaturegroupController getFeaturegroupController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java index 1a91fb9a77..a89b7b80c9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuregroup/FeaturegroupService.java @@ -17,6 +17,8 @@ package io.hops.hopsworks.api.featurestore.featuregroup; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.featurestore.activities.ActivityResource; import io.hops.hopsworks.api.featurestore.code.CodeResource; import io.hops.hopsworks.api.featurestore.commit.CommitResource; @@ -28,16 +30,14 @@ import io.hops.hopsworks.api.featurestore.keyword.FeatureGroupKeywordResource; import io.hops.hopsworks.api.featurestore.statistics.StatisticsResource; import io.hops.hopsworks.api.featurestore.tag.FeatureGroupTagResource; -import io.hops.hopsworks.api.filter.JWTNotRequired; -import io.hops.hopsworks.api.jobs.JobDTO; -import io.hops.hopsworks.api.jobs.JobsBuilder; -import io.hops.hopsworks.api.provenance.FeatureGroupProvenanceResource; -import io.hops.hopsworks.common.featurestore.featuregroup.stream.DeltaStreamerJobConf; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.filter.JWTNotRequired; import io.hops.hopsworks.api.filter.NoCacheResponse; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.jobs.JobDTO; +import io.hops.hopsworks.api.jobs.JobsBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.provenance.FeatureGroupProvenanceResource; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.OptionDTO; @@ -46,6 +46,8 @@ import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO; import io.hops.hopsworks.common.featurestore.featuregroup.IngestionJob; +import io.hops.hopsworks.common.featurestore.featuregroup.stream.DeltaStreamerJobConf; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeaturestoreException; @@ -110,7 +112,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Featuregroup service", description = "A service that manages a feature store's feature groups") -public class FeaturegroupService { +public class FeaturegroupService extends FeaturestoreSubResource { @EJB private NoCacheResponse noCacheResponse; @@ -158,29 +160,19 @@ public class FeaturegroupService { private FeatureGroupFeatureMonitoringResultResource featureMonitoringResultResource; @EJB private Settings settings; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; private static final Logger LOGGER = Logger.getLogger(FeaturegroupService.class.getName()); - /** - * Set the project of the featurestore (provided by parent resource) - * - * @param project the project where the featurestore resides - */ - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** - * Sets the featurestore of the featuregroups (provided by parent resource) - * - * @param featurestoreId id of the featurestore - * @throws FeaturestoreException - */ - public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreException { - //This call verifies that the project have access to the featurestoreId provided - this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId); + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } /** @@ -226,8 +218,10 @@ public Response getFeaturegroupsForFeaturestore( HttpServletRequest req, @Context SecurityContext sc) - throws FeaturestoreException, ServiceException { + throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(featureGroupBeanParam); List featuregroups = featuregroupController .getFeaturegroupsForFeaturestore(featurestore, project, user, convertToQueryParam(resourceRequest)); @@ -259,6 +253,8 @@ public Response createFeaturegroup(@Context SecurityContext sc, throws FeaturestoreException, ServiceException, KafkaException, SchemaException, ProjectException, UserException, GenericException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); if(featuregroupDTO == null) { throw new IllegalArgumentException("Input JSON for creating a new Feature Group cannot be null"); } @@ -295,9 +291,13 @@ public Response createFeaturegroup(@Context SecurityContext sc, @ApiOperation(value = "Get specific featuregroup from a specific featurestore", response = FeaturegroupDTO.class) public Response getFeatureGroup(@ApiParam(value = "Id of the featuregroup", required = true) - @PathParam("featuregroupId") Integer featuregroupId, @Context SecurityContext sc) - throws FeaturestoreException, ServiceException { + @PathParam("featuregroupId") Integer featuregroupId, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); verifyIdProvided(featuregroupId); FeaturegroupDTO featuregroupDTO = featuregroupController.getFeaturegroupWithIdAndFeaturestore(featurestore, featuregroupId, project, user); @@ -326,7 +326,8 @@ public Response getFeatureGroupForOnlinefs( @PathParam("featuregroupId") Integer featuregroupId, @Context HttpServletRequest req, @Context SecurityContext sc) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { + Featurestore featurestore = getFeaturestore(); verifyIdProvided(featuregroupId); Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); FeaturegroupDTO featuregroupDTO = new FeaturegroupDTO(featuregroup); @@ -357,9 +358,13 @@ public Response getFeatureGroupForOnlinefs( public Response getFeatureGroup(@ApiParam(value = "Name of the feature group", required = true) @PathParam("name") String name, @ApiParam(value = "Filter by a specific version") - @QueryParam("version") Integer version, @Context SecurityContext sc) - throws FeaturestoreException, ServiceException { + @QueryParam("version") Integer version, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); verifyNameProvided(name); List featuregroupDTO; if (version == null) { @@ -395,9 +400,11 @@ public Response deleteFeatureGroup(@Context SecurityContext sc, @Context HttpServletRequest req, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId) - throws FeaturestoreException, ServiceException, SchemaException, KafkaException { + throws FeaturestoreException, ServiceException, SchemaException, KafkaException, ProjectException { verifyIdProvided(featuregroupId); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); //Verify that the user has the data-owner role or is the creator of the featuregroup Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); try { @@ -441,6 +448,8 @@ public Response deleteFeaturegroupContents(@Context SecurityContext sc, GenericException { verifyIdProvided(featuregroupId); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); //Verify that the user has the data-owner role or is the creator of the featuregroup Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); try { @@ -497,6 +506,8 @@ public Response updateFeaturegroup(@Context SecurityContext sc, } verifyIdProvided(featuregroupId); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); FeaturegroupDTO updatedFeaturegroupDTO = null; if(updateMetadata) { @@ -555,10 +566,11 @@ public Response ingestionJob(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId, IngestionJobConf ingestionJobConf) - throws DatasetException, HopsSecurityException, FeaturestoreException, JobException { + throws DatasetException, HopsSecurityException, FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); verifyIdProvided(featuregroupId); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); Map dataOptions = null; if (ingestionJobConf.getDataOptions() != null) { @@ -578,98 +590,88 @@ public Response ingestionJob(@Context SecurityContext sc, IngestionJobDTO ingestionJobDTO = ingestionJobBuilder.build(uriInfo, project, featuregroup, ingestionJob); return Response.ok().entity(ingestionJobDTO).build(); } - + @Path("/{featuregroupId}/preview") public FeatureGroupPreviewResource getFeatureGroupPreview( - @ApiParam(value = "Id of the featuregroup") @PathParam("featuregroupId") Integer featuregroupId) - throws FeaturestoreException { - FeatureGroupPreviewResource fgPreviewResource = featureGroupPreviewResource.setProject(project); - fgPreviewResource.setFeaturestore(featurestore); - fgPreviewResource.setFeatureGroupId(featuregroupId); - return fgPreviewResource; + @ApiParam(value = "Id of the featuregroup") @PathParam("featuregroupId") Integer featuregroupId) { + featureGroupPreviewResource.setProjectId(getProjectId()); + featureGroupPreviewResource.setFeaturestoreId(getFeaturestoreId()); + featureGroupPreviewResource.setFeatureGroupId(featuregroupId); + return featureGroupPreviewResource; } @Path("/{featureGroupId}/statistics") - public StatisticsResource statistics(@PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.statisticsResource.setProject(project); - this.statisticsResource.setFeaturestore(featurestore); - this.statisticsResource.setFeatureGroupById(featureGroupId); + public StatisticsResource statistics(@PathParam("featureGroupId") Integer featureGroupId) { + this.statisticsResource.setProjectId(getProjectId()); + this.statisticsResource.setFeaturestoreId(getFeaturestoreId()); + this.statisticsResource.setFeatureGroupId(featureGroupId); return statisticsResource; } @Path("/{featureGroupId}/code") - public CodeResource code(@PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.codeResource.setProject(project); - this.codeResource.setFeatureStore(featurestore); + public CodeResource code(@PathParam("featureGroupId") Integer featureGroupId) { + this.codeResource.setProjectId(getProjectId()); + this.codeResource.setFeaturestoreId(getFeaturestoreId()); this.codeResource.setFeatureGroupId(featureGroupId); return codeResource; } - + @Path("/{featureGroupId}/provenance") public FeatureGroupProvenanceResource provenance(@PathParam("featureGroupId") Integer featureGroupId) { - this.provenanceResource.setProject(project); - this.provenanceResource.setFeatureStore(featurestore); + this.provenanceResource.setProjectId(getProjectId()); + this.provenanceResource.setFeaturestoreId(getFeaturestoreId()); this.provenanceResource.setFeatureGroupId(featureGroupId); return provenanceResource; } @Path("/{featureGroupId}/expectationsuite") - public ExpectationSuiteResource expectationSuite(@PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.expectationSuiteResource.setProject(project); - this.expectationSuiteResource.setFeaturestore(featurestore); - this.expectationSuiteResource.setFeatureGroup(featureGroupId); + public ExpectationSuiteResource expectationSuite(@PathParam("featureGroupId") Integer featureGroupId) { + this.expectationSuiteResource.setProjectId(getProjectId()); + this.expectationSuiteResource.setFeaturestoreId(getFeaturestoreId()); + this.expectationSuiteResource.setFeatureGroupId(featureGroupId); return expectationSuiteResource; } @Path("/{featureGroupId}/validationreport") - public ValidationReportResource validationReport(@PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.validationReportResource.setProject(project); - this.validationReportResource.setFeaturestore(featurestore); - this.validationReportResource.setFeatureGroup(featureGroupId); + public ValidationReportResource validationReport(@PathParam("featureGroupId") Integer featureGroupId) { + this.validationReportResource.setProjectId(getProjectId()); + this.validationReportResource.setFeaturestoreId(getFeaturestoreId()); + this.validationReportResource.setFeatureGroupId(featureGroupId); return validationReportResource; } @Path("/{featureGroupId}/commits") public CommitResource timetravel ( - @ApiParam(value = "Id of the featuregroup") @PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.commitResource.setProject(project); - this.commitResource.setFeaturestore(featurestore); - this.commitResource.setFeatureGroup(featureGroupId); + @ApiParam(value = "Id of the featuregroup") @PathParam("featureGroupId") Integer featureGroupId) { + this.commitResource.setProjectId(getProjectId()); + this.commitResource.setFeaturestoreId(getFeaturestoreId()); + this.commitResource.setFeatureGroupId(featureGroupId); return commitResource; } - @Path("/{featureGroupId}/keywords") public FeatureGroupKeywordResource keywords ( - @ApiParam(value = "Id of the featuregroup") @PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.featureGroupKeywordResource.setProject(project); - this.featureGroupKeywordResource.setFeaturestore(featurestore); + @ApiParam(value = "Id of the featuregroup") @PathParam("featureGroupId") Integer featureGroupId) { + this.featureGroupKeywordResource.setProjectId(getProjectId()); + this.featureGroupKeywordResource.setFeaturestoreId(getFeaturestoreId()); this.featureGroupKeywordResource.setFeatureGroupId(featureGroupId); return featureGroupKeywordResource; } - + @Path("/{featureGroupId}/activity") public ActivityResource activity(@ApiParam(value = "Id of the feature group") - @PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - this.activityResource.setProject(project); - this.activityResource.setFeaturestore(featurestore); + @PathParam("featureGroupId") Integer featureGroupId) { + this.activityResource.setProjectId(getProjectId()); + this.activityResource.setFeaturestoreId(getFeaturestoreId()); this.activityResource.setFeatureGroupId(featureGroupId); return this.activityResource; } - + @Path("/{featureGroupId}/alerts") - public FeatureStoreAlertResource alerts(@PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { - featureGroupDataValidationAlertResource.setFeatureGroup( - featuregroupController.getFeaturegroupById(featurestore, featureGroupId)); - featureGroupDataValidationAlertResource.setProject(project); + public FeatureStoreAlertResource alerts(@PathParam("featureGroupId") Integer featureGroupId) { + featureGroupDataValidationAlertResource.setProjectId(getProjectId()); + featureGroupDataValidationAlertResource.setFeaturestoreId(getFeaturestoreId()); + featureGroupDataValidationAlertResource.setFeatureGroupId(featureGroupId); return featureGroupDataValidationAlertResource; } @@ -689,11 +691,11 @@ public Response deltaStreamerJob(@Context SecurityContext sc, @ApiParam(value = "Id of the featuregroup", required = true) @PathParam("featuregroupId") Integer featuregroupId, DeltaStreamerJobConf deltaStreamerJobConf) - throws FeaturestoreException, JobException { + throws FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); verifyIdProvided(featuregroupId); - - Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featuregroupId); + Project project = getProject(); + Featuregroup featuregroup = featuregroupController.getFeaturegroupById(getFeaturestore(project), featuregroupId); Jobs deltaStreamerJob = fsJobManagerController.setupHudiDeltaStreamerJob(project, user, featuregroup, deltaStreamerJobConf); @@ -704,13 +706,11 @@ public Response deltaStreamerJob(@Context SecurityContext sc, @Path("/{featureGroupId}/tags") public FeatureGroupTagResource tags(@ApiParam(value = "Id of the feature group") - @PathParam("featureGroupId") Integer featureGroupId) - throws FeaturestoreException { + @PathParam("featureGroupId") Integer featureGroupId) { verifyIdProvided(featureGroupId); - this.tagResource.setProject(project); - this.tagResource.setFeatureStore(featurestore); - Featuregroup featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); - this.tagResource.setFeatureGroup(featuregroup); + this.tagResource.setProjectId(getProjectId()); + this.tagResource.setFeaturestoreId(getFeaturestoreId()); + this.tagResource.setFeatureGroupId(featureGroupId); return this.tagResource; } @@ -732,7 +732,9 @@ public Response importFeatureGroup(@Context SecurityContext sc, throws FeaturestoreException, JobException, ProjectException, ServiceException, GenericException { Users user = jWTHelper.getUserPrincipal(sc); - Jobs importDataJob = fsJobManagerController.setupImportFgJob(project, user, featurestore, importFgJobConf); + Project project = getProject(); + Jobs importDataJob = + fsJobManagerController.setupImportFgJob(project, user, getFeaturestore(project), importFgJobConf); JobDTO jobDTO = jobsBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.JOBS), importDataJob); return Response.created(jobDTO.getHref()).entity(jobDTO).build(); } @@ -742,13 +744,10 @@ public Response importFeatureGroup(@Context SecurityContext sc, //////////////////////////////////////// @Path("/{featureGroupId}/validationresult") - public ValidationResultResource validationResultResource( - @PathParam("featureGroupId") - Integer featureGroupId - ) throws FeaturestoreException { - this.validationResultResource.setProject(project); - this.validationResultResource.setFeaturestore(featurestore); - this.validationResultResource.setFeatureGroup(featureGroupId); + public ValidationResultResource validationResultResource(@PathParam("featureGroupId") Integer featureGroupId) { + this.validationResultResource.setProjectId(getProjectId()); + this.validationResultResource.setFeaturestoreId(getFeaturestoreId()); + this.validationResultResource.setFeatureGroupId(featureGroupId); return validationResultResource; } @@ -763,9 +762,9 @@ public FeatureGroupFeatureMonitoringConfigurationResource featureGroupFeatureMon Level.FINE ); } - this.featureMonitoringConfigurationResource.setProject(project); - this.featureMonitoringConfigurationResource.setFeatureStore(featurestore); - this.featureMonitoringConfigurationResource.setFeatureGroup(featureGroupId); + this.featureMonitoringConfigurationResource.setProjectId(getProjectId()); + this.featureMonitoringConfigurationResource.setFeaturestoreId(getFeaturestoreId()); + this.featureMonitoringConfigurationResource.setFeatureGroupId(featureGroupId); return featureMonitoringConfigurationResource; } @@ -779,9 +778,9 @@ public FeatureGroupFeatureMonitoringResultResource featureMonitoringResultResour Level.FINE ); } - this.featureMonitoringResultResource.setProject(project); - this.featureMonitoringResultResource.setFeatureStore(featurestore); - this.featureMonitoringResultResource.setFeatureGroup(featureGroupId); + this.featureMonitoringResultResource.setProjectId(getProjectId()); + this.featureMonitoringResultResource.setFeaturestoreId(getFeaturestoreId()); + this.featureMonitoringResultResource.setFeatureGroupId(featureGroupId); return featureMonitoringResultResource; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/config/FeatureMonitoringConfigurationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/config/FeatureMonitoringConfigurationResource.java index 51eec42646..d6fbe5a156 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/config/FeatureMonitoringConfigurationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/config/FeatureMonitoringConfigurationResource.java @@ -15,19 +15,22 @@ */ package io.hops.hopsworks.api.featurestore.featuremonitoring.config; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; -import io.hops.hopsworks.api.jobs.JobsBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featuremonitoring.config.FeatureMonitoringConfigurationController; import io.hops.hopsworks.common.featurestore.featuremonitoring.config.FeatureMonitoringConfigurationDTO; import io.hops.hopsworks.common.featurestore.featuremonitoring.config.FeatureMonitoringConfigurationInputValidation; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.JobException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; @@ -61,7 +64,7 @@ @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature Monitoring Configuration resource") -public abstract class FeatureMonitoringConfigurationResource { +public abstract class FeatureMonitoringConfigurationResource extends FeaturestoreSubResource { @EJB private FeatureMonitoringConfigurationController featureMonitoringConfigurationController; @@ -72,29 +75,30 @@ public abstract class FeatureMonitoringConfigurationResource { @EJB private FeatureMonitoringConfigurationBuilder featureMonitoringConfigurationBuilder; @EJB - private JobsBuilder jobsBuilder; - @EJB private JWTHelper jWTHelper; @EJB private FeatureMonitoringConfigurationInputValidation featureMonitoringConfigurationInputValidation; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - protected Project project; - protected Featurestore featureStore; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - protected abstract Integer getItemId(); + protected abstract Integer getItemId() throws ProjectException, FeaturestoreException; protected abstract ResourceRequest.Name getItemType(); // Added to init engine class for FV only - protected abstract String getItemName(); - protected abstract Integer getItemVersion(); + protected abstract String getItemName() throws ProjectException, FeaturestoreException; + protected abstract Integer getItemVersion() throws ProjectException, FeaturestoreException; /** * Endpoint to fetch the feature monitoring configuration with a given id. @@ -118,14 +122,14 @@ public Response getById(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Id of the Feature Monitoring Configuration", required = true) @PathParam("configId") Integer configId - ) throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { FeatureMonitoringConfiguration config = featureMonitoringConfigurationController.getFeatureMonitoringConfigurationByConfigId(configId); FeatureMonitoringConfigurationDTO dto = featureMonitoringConfigurationBuilder.build( - uriInfo, featureStore, getItemType(), getItemId(), getItemName(), getItemVersion(), config); + uriInfo, getFeaturestore(), getItemType(), getItemId(), getItemName(), getItemVersion(), config); return Response.ok().entity(dto).build(); } @@ -152,8 +156,8 @@ public Response getByName(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Name of the Feature Monitoring Configuration", required = true) @PathParam("name") String name - ) throws FeaturestoreException { - + ) throws FeaturestoreException, ProjectException { + Featurestore featureStore = getFeaturestore(); FeatureMonitoringConfiguration config = featureMonitoringConfigurationController.getFeatureMonitoringConfigurationByEntityAndName( featureStore, getItemType(), getItemId(), name); @@ -187,8 +191,8 @@ public Response getByFeatureName(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Name of the feature", required = true) @PathParam("featureName")String featureName - ) throws FeaturestoreException { - + ) throws FeaturestoreException, ProjectException { + Featurestore featureStore = getFeaturestore(); List configs = featureMonitoringConfigurationController.getFeatureMonitoringConfigurationByEntityAndFeatureName(featureStore, getItemType(), getItemId(), featureName); @@ -219,8 +223,8 @@ public Response getByFeatureName(@Context SecurityContext sc, public Response getByEntityId(@Context SecurityContext sc, @Context HttpServletRequest req, @Context UriInfo uriInfo - ) throws FeaturestoreException { - + ) throws FeaturestoreException, ProjectException { + Featurestore featureStore = getFeaturestore(); List configs = featureMonitoringConfigurationController.getFeatureMonitoringConfigurationByEntity( featureStore, getItemType(), getItemId()); @@ -252,9 +256,10 @@ public Response createFeatureMonitoringConfiguration(@Context SecurityContext sc @Context HttpServletRequest req, @Context UriInfo uriInfo, FeatureMonitoringConfigurationDTO configDTO - ) throws FeaturestoreException, JobException { + ) throws FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); Featuregroup featureGroup = null; FeatureView featureView = null; if (getItemType() == ResourceRequest.Name.FEATUREGROUPS) { @@ -297,8 +302,9 @@ public Response updateFeatureMonitoringConfiguration(@Context SecurityContext sc @Context UriInfo uriInfo, @PathParam("configId") Integer configId, FeatureMonitoringConfigurationDTO configDTO - ) throws FeaturestoreException, JobException { - + ) throws FeaturestoreException, JobException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); Featuregroup featureGroup = null; FeatureView featureView = null; if (getItemType() == ResourceRequest.Name.FEATUREGROUPS) { diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/result/FeatureMonitoringResultResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/result/FeatureMonitoringResultResource.java index 0ab121b68a..acfbcb183d 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/result/FeatureMonitoringResultResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featuremonitoring/result/FeatureMonitoringResultResource.java @@ -15,17 +15,21 @@ */ package io.hops.hopsworks.api.featurestore.featuremonitoring.result; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dao.AbstractFacade; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featuremonitoring.config.FeatureMonitoringConfigurationController; import io.hops.hopsworks.common.featurestore.featuremonitoring.result.FeatureMonitoringResultController; import io.hops.hopsworks.common.featurestore.featuremonitoring.result.FeatureMonitoringResultDTO; import io.hops.hopsworks.common.featurestore.featuremonitoring.result.FeatureMonitoringResultInputValidation; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuremonitoring.result.FeatureMonitoringResult; @@ -55,7 +59,7 @@ @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature Monitoring Result resource") -public abstract class FeatureMonitoringResultResource { +public abstract class FeatureMonitoringResultResource extends FeaturestoreSubResource { @EJB private FeatureMonitoringResultController featureMonitoringResultController; @@ -66,17 +70,23 @@ public abstract class FeatureMonitoringResultResource { @EJB private FeatureMonitoringConfigurationController featureMonitoringConfigurationController; - protected Project project; - protected Featurestore featureStore; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } + protected abstract void check() throws ProjectException, FeaturestoreException; + /** * Endpoint to fetch a list of feature monitoring results attached to a Feature. * @@ -102,7 +112,7 @@ public Response getAllByConfigId(@BeanParam Pagination pagination, @Context UriInfo uriInfo, @ApiParam(value = "Id of the configuration", required = true) @PathParam("configId") Integer configId - ) throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.FEATURE_MONITORING); resourceRequest.setOffset(pagination.getOffset()); @@ -110,7 +120,9 @@ public Response getAllByConfigId(@BeanParam Pagination pagination, resourceRequest.setSort(featureMonitoringResultBeanParam.getSortBySet()); resourceRequest.setFilter(featureMonitoringResultBeanParam.getFilter()); resourceRequest.setExpansions(featureMonitoringResultBeanParam.getExpansion().getResources()); - + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + check(); AbstractFacade.CollectionInfo results = featureMonitoringResultController.getAllFeatureMonitoringResultByConfigId( pagination.getOffset(), pagination.getLimit(), featureMonitoringResultBeanParam.getSortBySet(), @@ -144,7 +156,9 @@ public Response getSingle(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Id of the result", required = true) @PathParam("resultId") Integer resultId - ) throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); FeatureMonitoringResult result = featureMonitoringResultController.getFeatureMonitoringResultById(resultId); FeatureMonitoringResultDTO dtos = featureMonitoringResultBuilder.buildWithStats( @@ -174,8 +188,9 @@ public Response getSingle(@Context SecurityContext sc, public Response createFeatureMonitoringResult(@Context SecurityContext sc, @Context HttpServletRequest req, @Context UriInfo uriInfo, - FeatureMonitoringResultDTO resultDTO) throws FeaturestoreException { - + FeatureMonitoringResultDTO resultDTO) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); featureMonitoringResultInputValidation.validateOnCreate(resultDTO); FeatureMonitoringResult result = featureMonitoringResultBuilder.buildFromDTO(resultDTO); result.setFeatureMonitoringConfig( diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitorAlertResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitorAlertResource.java index bdeb7808ea..6eadcc7ca6 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitorAlertResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitorAlertResource.java @@ -23,8 +23,13 @@ import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.featurestore.featureview.alert.FeatureViewAlert; +import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.ApiOperation; @@ -74,13 +79,17 @@ public Response createOrUpdate( @Context HttpServletRequest req, @Context - SecurityContext sc) throws FeaturestoreException { - featureStoreAlertValidation.validateEntityType(getEntityType(), this.featuregroup, this.featureView); + SecurityContext sc) throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeatureGroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + featureStoreAlertValidation.validateEntityType(getEntityType(), featuregroup, featureView); if (dto == null) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ALERT_ILLEGAL_ARGUMENT, Level.FINE, Constants.NO_PAYLOAD); } - FeatureViewAlert featureViewAlert = featureViewAlertFacade.findByFeatureViewAndId(this.featureView, id); + FeatureViewAlert featureViewAlert = featureViewAlertFacade.findByFeatureViewAndId(featureView, id); featureStoreAlertValidation.validateUpdate(featureViewAlert, dto.getStatus(), featureView); featureViewAlert = featureStoreAlertController.updateAlert(dto, featureViewAlert, project); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringConfigurationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringConfigurationResource.java index 141d6aee65..7892731dd1 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringConfigurationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringConfigurationResource.java @@ -19,6 +19,7 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import javax.ejb.EJB; @@ -30,7 +31,8 @@ @TransactionAttribute(TransactionAttributeType.NEVER) public class FeatureViewFeatureMonitoringConfigurationResource extends FeatureMonitoringConfigurationResource { - private FeatureView featureView; + private String featureViewName; + private Integer featureViewVersion; @EJB private FeatureViewController featureViewController; @@ -41,12 +43,18 @@ public class FeatureViewFeatureMonitoringConfigurationResource extends FeatureMo * @param version */ public void setFeatureView(String name, Integer version) throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featureStore); + this.featureViewName = name; + this.featureViewVersion = version; + } + + protected FeatureView getFeatureView() throws ProjectException, FeaturestoreException { + return featureViewController.getByNameVersionAndFeatureStore(this.featureViewName, this.featureViewVersion, + getFeaturestore()); } @Override - protected Integer getItemId() { - return featureView.getId(); + protected Integer getItemId() throws ProjectException, FeaturestoreException { + return getFeatureView().getId(); } @Override @@ -55,12 +63,14 @@ protected ResourceRequest.Name getItemType() { } @Override - protected String getItemName() { - return featureView.getName(); + protected String getItemName() throws ProjectException, FeaturestoreException { + return getFeatureView().getName(); } @Override - protected Integer getItemVersion() { - return featureView.getVersion(); + protected Integer getItemVersion() throws ProjectException, FeaturestoreException { + return getFeatureView().getVersion(); } + + } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringResultResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringResultResource.java index 1172627ded..e65a8ec73a 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringResultResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewFeatureMonitoringResultResource.java @@ -18,6 +18,7 @@ import io.hops.hopsworks.api.featurestore.featuremonitoring.result.FeatureMonitoringResultResource; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import javax.ejb.EJB; @@ -29,10 +30,10 @@ @TransactionAttribute(TransactionAttributeType.NEVER) public class FeatureViewFeatureMonitoringResultResource extends FeatureMonitoringResultResource { - private FeatureView featureView; @EJB private FeatureViewController featureViewController; - + private String featureViewName; + private Integer featureViewVersion; /** * Sets the feature view of the feature monitoring ressource * @@ -40,6 +41,17 @@ public class FeatureViewFeatureMonitoringResultResource extends FeatureMonitorin * @param version of the Feature View */ public void setFeatureView(String name, Integer version) throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featureStore); + this.featureViewName = name; + this.featureViewVersion = version; + } + + protected FeatureView getFeatureView() throws ProjectException, FeaturestoreException { + return featureViewController.getByNameVersionAndFeatureStore(this.featureViewName, this.featureViewVersion, + getFeaturestore()); + } + + @Override + protected void check() throws ProjectException, FeaturestoreException { + getFeatureView(); } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java index c7253cbd8f..bf71b88dcc 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewResource.java @@ -17,21 +17,25 @@ package io.hops.hopsworks.api.featurestore.featureview; import com.google.api.client.util.Sets; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.JWTNotRequired; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dao.QueryParam; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewDTO; import io.hops.hopsworks.common.featurestore.featureview.ServingKeyDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.JobException; import io.hops.hopsworks.exceptions.MetadataException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; @@ -72,7 +76,7 @@ @RequestScoped @Api(value = "Feature View Resource") @TransactionAttribute(TransactionAttributeType.NEVER) -public class FeatureViewResource { +public class FeatureViewResource extends FeaturestoreSubResource { @EJB private JWTHelper jWTHelper; @@ -81,8 +85,20 @@ public class FeatureViewResource { @EJB private FeatureViewBuilder featureViewBuilder; - private Project project; - private Featurestore featurestore; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; + + @Override + protected ProjectController getProjectController() { + return projectController; + } + + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } public FeatureViewResource() { } @@ -104,12 +120,13 @@ public Response create( @Context UriInfo uriInfo, FeatureViewDTO featureViewDTO) throws FeaturestoreException, ServiceException, IOException, - FeatureStoreMetadataException, MetadataException, DatasetException { + FeatureStoreMetadataException, MetadataException, DatasetException, ProjectException { if (featureViewDTO == null) { throw new IllegalArgumentException("Input JSON for creating a new Feature View cannot be null"); } Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); FeatureView featureView = featureViewBuilder.convertFromDTO(project, featurestore, user, featureViewDTO); featureView = featureViewController.createFeatureView(project, user, featureView, featurestore); @@ -141,8 +158,10 @@ public Response getAll( @BeanParam FeatureViewBeanParam param ) throws FeaturestoreException, ServiceException, MetadataException, DatasetException, - FeatureStoreMetadataException { + FeatureStoreMetadataException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(param); List featureViews = featureViewController.getByFeatureStore(featurestore, convertToQueryParam(resourceRequest)); @@ -174,8 +193,10 @@ public Response getByName( @PathParam("name") String name ) throws FeaturestoreException, ServiceException, MetadataException, DatasetException, - FeatureStoreMetadataException { + FeatureStoreMetadataException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(param); List featureViews = featureViewController.getByNameAndFeatureStore(name, featurestore, convertToQueryParam(resourceRequest)); @@ -210,8 +231,10 @@ public Response getByNameVersion( @PathParam("version") Integer version ) throws FeaturestoreException, ServiceException, MetadataException, DatasetException, - FeatureStoreMetadataException { + FeatureStoreMetadataException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(param); FeatureView featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); @@ -244,8 +267,10 @@ public Response getServingKeys( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); FeatureView featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); List servingKeys = featureViewController.getServingKeys(project, user, featureView); @@ -270,8 +295,10 @@ public Response deleteName( @ApiParam(value = "Name of the feature view", required = true) @PathParam("name") String name - ) throws FeaturestoreException, JobException { + ) throws FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); featureViewController.delete(user, project, featurestore, name); return Response.ok().build(); @@ -296,8 +323,10 @@ public Response deleteNameVersion( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException, JobException { + ) throws FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); featureViewController.delete(user, project, featurestore, name, version); return Response.ok().build(); @@ -329,11 +358,13 @@ public Response update( @PathParam("version") Integer version, FeatureViewDTO featureViewDTO) throws FeaturestoreException, ServiceException, - FeatureStoreMetadataException, MetadataException, DatasetException { + FeatureStoreMetadataException, MetadataException, DatasetException, ProjectException { if (featureViewDTO == null) { throw new IllegalArgumentException("Input JSON for updating a Feature View cannot be null"); } Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(param); FeatureView featureView = featureViewController.update(user, project, featurestore, name, version, featureViewDTO.getDescription()); @@ -343,7 +374,6 @@ public Response update( .build(); } - // This endpoint is necassary for onlinefs notification system // (can't use Provenance since onlinefs doesn’t have a user) @GET @@ -374,14 +404,6 @@ public Response getAllFeatureViewsByFeatureGroup( return Response.ok().entity(parentFeatureViewDTO).build(); } - public void setProject(Project project) { - this.project = project; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } - private ResourceRequest makeResourceRequest(FeatureViewBeanParam param) { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.FEATUREVIEW); resourceRequest.setOffset(param.getPagination().getOffset()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewService.java index 597142c89e..6cabd6f3b3 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewService.java @@ -16,6 +16,7 @@ package io.hops.hopsworks.api.featurestore.featureview; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.featurestore.activities.ActivityResource; import io.hops.hopsworks.api.featurestore.keyword.FeatureViewKeywordResource; import io.hops.hopsworks.api.featurestore.preparestatement.PreparedStatementResource; @@ -26,14 +27,12 @@ import io.hops.hopsworks.api.featurestore.transformation.TransformationResource; import io.hops.hopsworks.api.provenance.FeatureViewProvenanceResource; import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiParam; -import java.util.logging.Level; import javax.ejb.EJB; import javax.ejb.TransactionAttribute; @@ -42,11 +41,12 @@ import javax.inject.Inject; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import java.util.logging.Level; @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature View service") -public class FeatureViewService { +public class FeatureViewService extends FeaturestoreSubResource { @Inject private FeatureViewResource featureViewResource; @@ -78,14 +78,23 @@ public class FeatureViewService { private Settings settings; @Inject private FeatureViewFeatureMonitorAlertResource featureViewFeatureMonitorAlertResource; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; + @Override + protected ProjectController getProjectController() { + return projectController; + } + + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } @Path("") public FeatureViewResource featureViewResource() { - this.featureViewResource.setProject(project); - this.featureViewResource.setFeaturestore(featurestore); + this.featureViewResource.setProjectId(getProjectId()); + this.featureViewResource.setFeaturestoreId(getFeaturestoreId()); return this.featureViewResource; } @@ -97,13 +106,28 @@ public TrainingDatasetResource trainingDatasetResource( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - this.trainingDatasetResource.setProject(project); - this.trainingDatasetResource.setFeaturestore(featurestore); + ) { + this.trainingDatasetResource.setProjectId(getProjectId()); + this.trainingDatasetResource.setFeaturestoreId(getFeaturestoreId()); this.trainingDatasetResource.setFeatureView(featureViewName, version); return this.trainingDatasetResource; } - + + @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/tags") + public FeatureViewTagResource tags( + @ApiParam(value = "Name of the feature view", required = true) + @PathParam("name") + String featureViewName, + @ApiParam(value = "Version of the feature view", required = true) + @PathParam("version") + Integer version + ) { + this.tagResource.setProjectId(getProjectId()); + this.tagResource.setFeaturestoreId(getFeaturestoreId()); + this.tagResource.setFeatureView(featureViewName, version); + return this.tagResource; + } + @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/query") public QueryResource query( @ApiParam(value = "Name of the feature view", required = true) @@ -112,13 +136,13 @@ public QueryResource query( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - this.queryResource.setProject(project); - this.queryResource.setFeaturestore(featurestore); + ) { + this.queryResource.setProjectId(getProjectId()); + this.queryResource.setFeaturestoreId(getFeaturestoreId()); this.queryResource.setFeatureView(featureViewName, version); return this.queryResource; } - + @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/keywords") public FeatureViewKeywordResource keywords( @ApiParam(value = "Name of the feature view", required = true) @@ -127,13 +151,13 @@ public FeatureViewKeywordResource keywords( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - this.featureViewKeywordResource.setProject(project); - this.featureViewKeywordResource.setFeaturestore(featurestore); + ) { + this.featureViewKeywordResource.setProjectId(getProjectId()); + this.featureViewKeywordResource.setFeaturestoreId(getFeaturestoreId()); this.featureViewKeywordResource.setFeatureView(featureViewName, version); return this.featureViewKeywordResource; } - + @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/activity") public ActivityResource activity( @ApiParam(value = "Id of the feature view", required = true) @@ -142,10 +166,10 @@ public ActivityResource activity( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - this.activityResource.setProject(project); - this.activityResource.setFeaturestore(featurestore); - this.activityResource.setFeatureViewByNameAndVersion(featureViewName, version); + ) { + this.activityResource.setProjectId(getProjectId()); + this.activityResource.setFeaturestoreId(getFeaturestoreId()); + this.activityResource.setFeatureView(featureViewName, version); return this.activityResource; } @@ -156,13 +180,13 @@ public TransformationResource transformation( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - this.transformationResource.setProject(project); - this.transformationResource.setFeaturestore(featurestore); + ) { + this.transformationResource.setProjectId(getProjectId()); + this.transformationResource.setFeaturestoreId(getFeaturestoreId()); this.transformationResource.setFeatureView(featureViewName, version); return this.transformationResource; } - + @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/preparedstatement") public PreparedStatementResource preparedStatement( @ApiParam(value = "Name of the feature view", required = true) @@ -171,33 +195,23 @@ public PreparedStatementResource preparedStatement( @ApiParam(value = "Version of the feature view", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - this.preparedStatementResource.setProject(project); - this.preparedStatementResource.setFeatureStore(featurestore); + ) { + this.preparedStatementResource.setProjectId(getProjectId()); + this.preparedStatementResource.setFeaturestoreId(getFeaturestoreId()); this.preparedStatementResource.setFeatureView(featureViewName, version); return this.preparedStatementResource; } - public void setProject(Project project) { - this.project = project; - } - - public void setFeaturestore(Integer featurestoreId) throws FeaturestoreException { - //This call verifies that the project have access to the featurestoreId provided - this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId); - } - @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/provenance") public FeatureViewProvenanceResource provenance( @PathParam("name") String name, @PathParam("version") Integer version) { - this.provenanceResource.setProject(project); - this.provenanceResource.setFeatureStore(featurestore); - this.provenanceResource.setFeatureViewName(name); - this.provenanceResource.setFeatureViewVersion(version); + this.provenanceResource.setProjectId(getProjectId()); + this.provenanceResource.setFeaturestoreId(getFeaturestoreId()); + this.provenanceResource.setFeatureView(name, version); return provenanceResource; } - + @Path("/{name: [a-z0-9_]*(?=[a-z])[a-z0-9_]+}/version/{version: [0-9]+}/statistics") public StatisticsResource statistics( @ApiParam(value = "Name of the feature view", required = true) @@ -207,9 +221,9 @@ public StatisticsResource statistics( @PathParam("version") Integer version ) throws FeaturestoreException { - this.statisticsResource.setProject(project); - this.statisticsResource.setFeaturestore(featurestore); - this.statisticsResource.setFeatureViewByNameAndVersion(name, version); + this.statisticsResource.setProjectId(getProjectId()); + this.statisticsResource.setFeaturestoreId(getFeaturestoreId()); + this.statisticsResource.setFeatureView(name, version); return statisticsResource; } @@ -228,8 +242,8 @@ public FeatureViewFeatureMonitoringConfigurationResource featureMonitoringConfig Level.FINE ); } - this.featureMonitoringConfigurationResource.setProject(project); - this.featureMonitoringConfigurationResource.setFeatureStore(featurestore); + this.featureMonitoringConfigurationResource.setProjectId(getProjectId()); + this.featureMonitoringConfigurationResource.setFeaturestoreId(getFeaturestoreId()); this.featureMonitoringConfigurationResource.setFeatureView(name, version); return this.featureMonitoringConfigurationResource; } @@ -249,8 +263,8 @@ public FeatureViewFeatureMonitoringResultResource featureMonitoringResultResourc Level.FINE ); } - this.featureMonitoringResultResource.setProject(project); - this.featureMonitoringResultResource.setFeatureStore(featurestore); + this.featureMonitoringResultResource.setProjectId(getProjectId()); + this.featureMonitoringResultResource.setFeaturestoreId(getFeaturestoreId()); this.featureMonitoringResultResource.setFeatureView(name, version); return this.featureMonitoringResultResource; } @@ -270,9 +284,9 @@ public FeatureViewFeatureMonitorAlertResource featureViewFeatureMonitorAlertReso Level.FINE ); } - featureViewFeatureMonitorAlertResource.setFeatureStore(featurestore); - featureViewFeatureMonitorAlertResource.setFeatureView(featureViewName, version, featurestore); - featureViewFeatureMonitorAlertResource.setProject(project); + this.featureViewFeatureMonitorAlertResource.setProjectId(getProjectId()); + this.featureViewFeatureMonitorAlertResource.setFeaturestoreId(getFeaturestoreId()); + this.featureViewFeatureMonitorAlertResource.setFeatureView(featureViewName, version); return featureViewFeatureMonitorAlertResource; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewSubResource.java new file mode 100644 index 0000000000..18202d5e76 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/featureview/FeatureViewSubResource.java @@ -0,0 +1,64 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.featurestore.featureview; + +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; +import io.hops.hopsworks.persistence.entity.project.Project; + +public abstract class FeatureViewSubResource extends FeaturestoreSubResource { + private String name; + private Integer version; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } + + public void setFeatureView(String name, Integer version) { + this.name = name; + this.version = version; + } + + public FeatureView getFeatureView(Project project) throws FeaturestoreException, ProjectException { + return getFeatureViewController().getByNameVersionAndFeatureStore(name, version, getFeaturestore(project)); + } + + public FeatureView getFeatureView(Featurestore featurestore) throws FeaturestoreException { + return getFeatureViewController().getByNameVersionAndFeatureStore(name, version, featurestore); + } + + public FeatureView getFeatureView() throws ProjectException, FeaturestoreException { + return getFeatureViewController().getByNameVersionAndFeatureStore(name, version, getFeaturestore()); + } + + protected abstract FeatureViewController getFeatureViewController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureGroupKeywordResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureGroupKeywordResource.java index 60f6212370..b8066306dd 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureGroupKeywordResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureGroupKeywordResource.java @@ -17,15 +17,18 @@ import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.featurestore.FeaturestoreKeywordBuilder; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.keyword.KeywordDTO; import io.hops.hopsworks.common.featurestore.metadata.FeatureStoreKeywordControllerIface; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; @@ -55,28 +58,31 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature Group Keywords Resource") -public class FeatureGroupKeywordResource { +public class FeatureGroupKeywordResource extends FeatureGroupSubResource { @EJB private FeaturegroupController featuregroupController; @EJB private FeaturestoreKeywordBuilder featurestoreKeywordBuilder; @Inject private FeatureStoreKeywordControllerIface keywordCtrl; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeatureGroupId(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } @GET @@ -89,8 +95,10 @@ public void setFeatureGroupId(Integer featureGroupId) throws FeaturestoreExcepti allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, - @Context UriInfo uriInfo) { + @Context UriInfo uriInfo) throws ProjectException, FeaturestoreException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); List keywords = keywordCtrl.getKeywords(featuregroup); KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, featuregroup, keywords); return Response.ok().entity(dto).build(); @@ -108,8 +116,10 @@ public Response getKeywords(@Context SecurityContext sc, public Response replaceKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, @Context UriInfo uriInfo, KeywordDTO keywordDTO) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); KeywordDTO dto; List updatedKeywords = keywordCtrl.replaceKeywords(featuregroup, new HashSet<>(keywordDTO.getKeywords())); dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, featuregroup, updatedKeywords); @@ -127,8 +137,10 @@ public Response deleteKeywords(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @QueryParam("keyword") String keyword) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featuregroup featuregroup = getFeaturegroup(project); keywordCtrl.deleteKeyword(featuregroup, keyword); List updatedKeywords = keywordCtrl.getKeywords(featuregroup); KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, featuregroup, updatedKeywords); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureViewKeywordResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureViewKeywordResource.java index fdd1667eeb..5767d872d6 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureViewKeywordResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/FeatureViewKeywordResource.java @@ -17,13 +17,17 @@ import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.featurestore.FeaturestoreKeywordBuilder; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.keyword.KeywordDTO; import io.hops.hopsworks.common.featurestore.metadata.FeatureStoreKeywordControllerIface; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; @@ -55,7 +59,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature View Keywords Resource") -public class FeatureViewKeywordResource { +public class FeatureViewKeywordResource extends FeatureViewSubResource { @EJB private FeatureViewController featureViewController; @EJB @@ -63,20 +67,22 @@ public class FeatureViewKeywordResource { @Inject private FeatureStoreKeywordControllerIface keywordCtrl; - private Project project; - private Featurestore featurestore; - private FeatureView featureView; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - - public void setFeatureView(String name, Integer version) throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } @GET @@ -89,8 +95,11 @@ public void setFeatureView(String name, Integer version) throws FeaturestoreExce allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, - @Context UriInfo uriInfo) { + @Context UriInfo uriInfo) throws ProjectException, FeaturestoreException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featurestore); List keywords = keywordCtrl.getKeywords(featureView); KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, featureView, keywords); return Response.ok().entity(dto).build(); @@ -108,8 +117,11 @@ public Response getKeywords(@Context SecurityContext sc, public Response replaceKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, @Context UriInfo uriInfo, KeywordDTO keywordDTO) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featurestore); List updatedKeywords = keywordCtrl.replaceKeywords(featureView, new HashSet<>(keywordDTO.getKeywords())); KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, featureView, updatedKeywords); return Response.ok().entity(dto).build(); @@ -126,8 +138,11 @@ public Response deleteKeywords(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @QueryParam("keyword") String keyword) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featurestore); keywordCtrl.deleteKeyword(featureView, keyword); List updatedKeywords = keywordCtrl.getKeywords(featureView); KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, featureView, updatedKeywords); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/TrainingDatasetKeywordResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/TrainingDatasetKeywordResource.java index e8f49d2918..93c4031e9d 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/TrainingDatasetKeywordResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/keyword/TrainingDatasetKeywordResource.java @@ -17,13 +17,20 @@ import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.featurestore.FeaturestoreKeywordBuilder; +import io.hops.hopsworks.api.featurestore.trainingdataset.TrainingDatasetSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.keyword.KeywordDTO; import io.hops.hopsworks.common.featurestore.metadata.FeatureStoreKeywordControllerIface; +import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; @@ -53,21 +60,37 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Training Dataset Keyword Resource") -public class TrainingDatasetKeywordResource { +public class TrainingDatasetKeywordResource extends TrainingDatasetSubResource { @EJB private FeaturestoreKeywordBuilder featurestoreKeywordBuilder; @Inject private FeatureStoreKeywordControllerIface keywordCtrl; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private FeatureViewController featureViewController; + @EJB + private TrainingDatasetController trainingDatasetController; - private Project project; - private TrainingDataset trainingDataset; + @Override + protected ProjectController getProjectController() { + return projectController; + } - public void setProject(Project project) { - this.project = project; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } - public void setTrainingDataset(TrainingDataset trainingDataset) { - this.trainingDataset = trainingDataset; + @Override + protected TrainingDatasetController getTrainingDatasetController() { + return trainingDatasetController; } @GET @@ -80,9 +103,12 @@ public void setTrainingDataset(TrainingDataset trainingDataset) { allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, - @Context UriInfo uriInfo) { + @Context UriInfo uriInfo) throws ProjectException, FeaturestoreException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); List keywords = keywordCtrl.getKeywords(trainingDataset); KeywordDTO dto = featurestoreKeywordBuilder.build(uriInfo, resourceRequest, project, trainingDataset, keywords); return Response.ok().entity(dto).build(); @@ -100,7 +126,10 @@ public Response getKeywords(@Context SecurityContext sc, public Response replaceKeywords(@Context SecurityContext sc, @Context HttpServletRequest req, @Context UriInfo uriInfo, KeywordDTO keywordDTO) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); List updatedKeywords = keywordCtrl.replaceKeywords(trainingDataset, new HashSet<>(keywordDTO.getKeywords())); @@ -120,7 +149,10 @@ public Response deleteKeywords(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @QueryParam("keyword") String keyword) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KEYWORDS); keywordCtrl.deleteKeyword(trainingDataset, keyword); List updatedKeywords = keywordCtrl.getKeywords(trainingDataset); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/preparestatement/PreparedStatementResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/preparestatement/PreparedStatementResource.java index 385ede1b95..f8c7e10807 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/preparestatement/PreparedStatementResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/preparestatement/PreparedStatementResource.java @@ -16,18 +16,21 @@ package io.hops.hopsworks.api.featurestore.preparestatement; -import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.featurestore.trainingdataset.PreparedStatementBuilder; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; @@ -51,7 +54,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class PreparedStatementResource { +public class PreparedStatementResource extends FeatureViewSubResource { @EJB private JWTHelper jWTHelper; @@ -60,22 +63,25 @@ public class PreparedStatementResource { @EJB private PreparedStatementBuilder preparedStatementBuilder; - private Project project; - private Featurestore featurestore; - private FeatureView featureView; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - - public void setFeatureStore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - - public void setFeatureView(String name, Integer version) throws FeaturestoreException { - featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } + @ApiOperation(value = "Get prepared statements used to generate model serving vector from feature view query", response = ServingPreparedStatementDTO.class) @GET @@ -100,11 +106,13 @@ public Response getPreparedStatements( @QueryParam("inference_helper_columns") @DefaultValue("false") boolean inference_helper_columns) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ServingPreparedStatementDTO servingPreparedStatementDTO = preparedStatementBuilder.build(uriInfo, - new ResourceRequest(ResourceRequest.Name.PREPAREDSTATEMENTS), project, user, featurestore, featureView, batch, - inference_helper_columns); + new ResourceRequest(ResourceRequest.Name.PREPAREDSTATEMENTS), project, user, featurestore, + getFeatureView(featurestore), batch, inference_helper_columns); return Response.ok().entity(servingPreparedStatementDTO).build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/query/QueryResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/query/QueryResource.java index fb509f26ae..4db2744f41 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/query/QueryResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/query/QueryResource.java @@ -16,22 +16,24 @@ package io.hops.hopsworks.api.featurestore.query; -import io.hops.hopsworks.api.featurestore.FsQueryBuilder; -import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.query.FsQueryDTO; import io.hops.hopsworks.common.featurestore.query.Query; import io.hops.hopsworks.common.featurestore.query.QueryBuilder; import io.hops.hopsworks.common.featurestore.query.QueryController; import io.hops.hopsworks.common.featurestore.query.QueryDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; @@ -56,7 +58,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class QueryResource { +public class QueryResource extends FeatureViewSubResource { @EJB private FeatureViewController featureViewController; @@ -65,13 +67,24 @@ public class QueryResource { @EJB private JWTHelper jWTHelper; @EJB - private FsQueryBuilder fsQueryBuilder; - @EJB private QueryBuilder queryBuilder; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - private Featurestore featurestore; - private FeatureView featureView; - private Project project; + @Override + protected ProjectController getProjectController() { + return projectController; + } + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; + } @ApiOperation(value = "Return batch query with given event time.", response = FsQueryDTO.class) @@ -123,10 +136,12 @@ public Response constructBatchQuery( @ApiParam(value = "Training data version") @QueryParam("td_version") Integer trainingDataVersion - ) throws FeaturestoreException, ServiceException { + ) throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); Query query = queryController.constructBatchQuery( - featureView, project, user, startTime, endTime, withLabel, withPrimaryKeys, withEventTime, + getFeatureView(featurestore), project, user, startTime, endTime, withLabel, withPrimaryKeys, withEventTime, inferenceHelperColumns, trainingHelperColumns, isHiveEngine, trainingDataVersion); return Response.ok().entity(queryBuilder.build(query, featurestore, project, user)).build(); } @@ -145,22 +160,13 @@ public Response getQuery( SecurityContext sc, @Context HttpServletRequest req - ) throws FeaturestoreException, ServiceException { + ) throws FeaturestoreException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - Query query = queryController.makeQuery(featureView, project, user, true, false, false, true, true, false); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Query query = + queryController.makeQuery(getFeatureView(featurestore), project, user, true, false, false, true, true, false); QueryDTO queryDTO = queryBuilder.build(query, featurestore, project, user); return Response.ok().entity(queryDTO).build(); } - - public void setFeatureView(String name, Integer version) throws FeaturestoreException { - featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); - } - - public void setProject(Project project) { - this.project = project; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/statistics/StatisticsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/statistics/StatisticsResource.java index f06b5dae3c..86bb9f56cb 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/statistics/StatisticsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/statistics/StatisticsResource.java @@ -16,14 +16,16 @@ package io.hops.hopsworks.api.featurestore.statistics; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreCommonSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jobs.JobDTO; import io.hops.hopsworks.api.jobs.JobsBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.app.FsJobManagerController; import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; @@ -34,6 +36,7 @@ import io.hops.hopsworks.common.featurestore.statistics.StatisticsFilters; import io.hops.hopsworks.common.featurestore.statistics.StatisticsInputValidation; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; @@ -85,7 +88,7 @@ @Api(value = "Feature store statistics Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class StatisticsResource { +public class StatisticsResource extends FeaturestoreCommonSubResource { @EJB private StatisticsBuilder statisticsBuilder; @@ -108,42 +111,34 @@ public class StatisticsResource { @EJB private JobsBuilder jobsBuilder; - private Project project; - private Featurestore featurestore; - private Featuregroup featuregroup; - private FeatureView featureView; - private TrainingDataset trainingDataset; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - public void setProject(Project project) { - this.project = project; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureGroupById(Integer featureGroupId) throws FeaturestoreException { - this.featuregroup = featuregroupController.getFeaturegroupById(featurestore, featureGroupId); - } - - public void setFeatureViewByNameAndVersion(String featureViewName, Integer featureViewVersion) - throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore( - featureViewName, featureViewVersion, featurestore); + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } - - public void setTrainingDatasetByVersion(FeatureView featureView, Integer trainingDatasetVersion) - throws FeaturestoreException { - this.trainingDataset = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, - trainingDatasetVersion); - } - - public void setTrainingDatasetById(Integer trainingDatasetId) throws FeaturestoreException { - this.trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); + + @Override + protected TrainingDatasetController getTrainingDatasetController() { + return trainingDatasetController; } - public void setTrainingDataset(TrainingDataset trainingDataset) { - this.trainingDataset = trainingDataset; + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } @GET @@ -163,7 +158,7 @@ public Response get(@BeanParam Pagination pagination, // backwards compatibility @ApiParam(value = "for_transformation", example = "false") @QueryParam("for_transformation") Boolean forTransformation) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); // backwards compatibility @@ -180,20 +175,24 @@ public Response get(@BeanParam Pagination pagination, resourceRequest.setField(statisticsBeanParam.getFieldSet()); StatisticsFilters filters = new StatisticsFilters((Set)statisticsBeanParam.getFilterSet()); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeaturegroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); StatisticsDTO dto; if (featuregroup != null) { // feature group statistics statisticsInputValidation.validateStatisticsFiltersForFeatureGroup((Set) statisticsBeanParam.getFilterSet()); statisticsInputValidation.validateGetForFeatureGroup(featuregroup, filters); dto = statisticsBuilder.build(uriInfo, resourceRequest, project, user, featurestore, featuregroup, featureNames); - } else if (featureView != null) { - statisticsInputValidation.validateStatisticsFiltersForFeatureView((Set) statisticsBeanParam.getFilterSet()); - statisticsInputValidation.validateGetForFeatureView(featureView, filters); - dto = statisticsBuilder.build(uriInfo, resourceRequest, project, user, featurestore, featureView, featureNames); - } else { // training dataset statistics + } else if (trainingDataset != null) { statisticsInputValidation.validateStatisticsFiltersForTrainingDataset((Set)statisticsBeanParam.getFilterSet()); dto = statisticsBuilder.build(uriInfo, resourceRequest, project, user, featurestore, trainingDataset, featureNames); + } else { + statisticsInputValidation.validateStatisticsFiltersForFeatureView((Set) statisticsBeanParam.getFilterSet()); + statisticsInputValidation.validateGetForFeatureView(featureView, filters); + dto = statisticsBuilder.build(uriInfo, resourceRequest, project, user, featurestore, featureView, featureNames); } return Response.ok().entity(dto).build(); } @@ -211,17 +210,21 @@ public Response register(@Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc, StatisticsDTO statisticsDTO) - throws FeaturestoreException, DatasetException, HopsSecurityException, IOException { + throws FeaturestoreException, DatasetException, HopsSecurityException, IOException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - - StatisticsDTO dto; + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeaturegroup(featurestore); + FeatureView featureView = getFeatureView(featurestore); + TrainingDataset trainingDataset = getTrainingDataset(featurestore, featureView); + StatisticsDTO dto = null; if (featuregroup != null) { - dto = registerFeatureGroupStatistics(user, uriInfo, statisticsDTO); + dto = registerFeatureGroupStatistics(project, featuregroup, user, uriInfo, statisticsDTO); + } else if (trainingDataset != null) { + dto = registerTrainingDatasetStatistics(project, trainingDataset, user, uriInfo, statisticsDTO); } else if (featureView != null) { - dto = registerFeatureViewStatistics(user, uriInfo, statisticsDTO); - } else { - dto = registerTrainingDatasetStatistics(user, uriInfo, statisticsDTO); + dto = registerFeatureViewStatistics(project, featureView, user, uriInfo, statisticsDTO); } return Response.ok().entity(dto).build(); } @@ -241,13 +244,18 @@ public Response compute(@Context UriInfo uriInfo, @Context SecurityContext sc) throws FeaturestoreException, ServiceException, JobException, ProjectException, GenericException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + Featuregroup featuregroup = getFeaturegroup(featurestore); + TrainingDataset trainingDataset = getTrainingDataset(featurestore); Jobs job = fsJobManagerController.setupStatisticsJob(project, user, featurestore, featuregroup, trainingDataset); JobDTO jobDTO = jobsBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.JOBS), job); return Response.created(jobDTO.getHref()).entity(jobDTO).build(); } - private StatisticsDTO registerFeatureGroupStatistics(Users user, UriInfo uriInfo, StatisticsDTO statisticsDTO) - throws FeaturestoreException, IOException, DatasetException, HopsSecurityException { + private StatisticsDTO registerFeatureGroupStatistics(Project project, Featuregroup featuregroup, Users user, + UriInfo uriInfo, StatisticsDTO statisticsDTO) throws FeaturestoreException, IOException, DatasetException, + HopsSecurityException { statisticsInputValidation.validateRegisterForFeatureGroup(featuregroup, statisticsDTO); Collection stats = featureDescriptiveStatisticsBuilder.buildManyFromContentOrDTO( statisticsDTO.getFeatureDescriptiveStatistics(), statisticsDTO.getContent()); @@ -259,8 +267,9 @@ private StatisticsDTO registerFeatureGroupStatistics(Users user, UriInfo uriInfo return statisticsBuilder.build(uriInfo, resourceRequest, project, user, featuregroup, featureGroupStatistics); } - private StatisticsDTO registerFeatureViewStatistics(Users user, UriInfo uriInfo, StatisticsDTO statisticsDTO) - throws FeaturestoreException, IOException, DatasetException, HopsSecurityException { + private StatisticsDTO registerFeatureViewStatistics(Project project, FeatureView featureView, Users user, + UriInfo uriInfo, StatisticsDTO statisticsDTO) throws FeaturestoreException, IOException, DatasetException, + HopsSecurityException { statisticsInputValidation.validateRegisterForFeatureView(featureView, statisticsDTO); Collection stats = featureDescriptiveStatisticsBuilder.buildManyFromContentOrDTO( statisticsDTO.getFeatureDescriptiveStatistics(), statisticsDTO.getContent()); @@ -272,8 +281,9 @@ private StatisticsDTO registerFeatureViewStatistics(Users user, UriInfo uriInfo, return statisticsBuilder.build(uriInfo, resourceRequest, project, user, featureView, featureViewStatistics); } - private StatisticsDTO registerTrainingDatasetStatistics(Users user, UriInfo uriInfo, StatisticsDTO statisticsDTO) - throws FeaturestoreException, IOException, DatasetException, HopsSecurityException { + private StatisticsDTO registerTrainingDatasetStatistics(Project project, TrainingDataset trainingDataset, Users user, + UriInfo uriInfo, StatisticsDTO statisticsDTO) throws FeaturestoreException, IOException, DatasetException, + HopsSecurityException { statisticsInputValidation.validateRegisterForTrainingDataset(trainingDataset, statisticsDTO); // register training dataset statistics as a file in hopsfs TrainingDatasetStatistics statistics; diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java index e07d0d22d7..630c2416d9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java @@ -17,22 +17,22 @@ package io.hops.hopsworks.api.featurestore.storageconnector; import com.google.common.base.Strings; -import io.hops.hadoop.shaded.javax.servlet.http.HttpServletRequest; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.NoCacheResponse; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.provenance.StorageConnectorProvenanceResource; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.featurestore.FeaturestoreController; -import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController; -import io.hops.hopsworks.common.featurestore.storageconnectors.connectionChecker.ConnectionChecker; -import io.hops.hopsworks.common.featurestore.storageconnectors.connectionChecker.ConnectionCheckerDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO; import io.hops.hopsworks.common.featurestore.storageconnectors.StorageConnectorUtil; +import io.hops.hopsworks.common.featurestore.storageconnectors.connectionChecker.ConnectionChecker; +import io.hops.hopsworks.common.featurestore.storageconnectors.connectionChecker.ConnectionCheckerDTO; import io.hops.hopsworks.common.kafka.KafkaBrokers; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.ProjectException; @@ -53,6 +53,7 @@ import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -81,7 +82,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "StorageConnector service", description = "A service that manages a feature store's storage connectors") -public class FeaturestoreStorageConnectorService { +public class FeaturestoreStorageConnectorService extends FeaturestoreSubResource { @EJB private NoCacheResponse noCacheResponse; @@ -94,37 +95,23 @@ public class FeaturestoreStorageConnectorService { @EJB private Settings settings; @EJB - private OnlineFeaturestoreController onlineFeaturestoreController; - @EJB private StorageConnectorUtil storageConnectorUtil; @EJB private ConnectionChecker connectionChecker; @Inject private StorageConnectorProvenanceResource provenanceResource; - private Project project; - private Featurestore featurestore; + @EJB + private ProjectController projectController; - /** - * Set the project of the featurestore (provided by parent resource) - * - * @param project - * the project where the featurestore resides - */ - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** - * Sets the featurestore of the featuregroups (provided by parent resource) - * - * @param featurestoreId - * id of the featurestore - * @throws FeaturestoreException - */ - public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreException { - //This call verifies that the project have access to the featurestoreId provided - this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId); + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } /** @@ -142,8 +129,10 @@ public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreExcepti public Response getStorageConnectors(@BeanParam StorageConnectorBeanParam storageConnectorBeanParam, @Context SecurityContext sc, @Context HttpServletRequest req) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = makeResourceRequest(storageConnectorBeanParam); List featurestoreStorageConnectorDTOS = storageConnectorController .getConnectorsForFeaturestore(user, project, featurestore, convertToQueryParam(resourceRequest)); @@ -158,16 +147,20 @@ public Response getStorageConnectors(@BeanParam StorageConnectorBeanParam storag @Path("{connectorName}") @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) - @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) + @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, + allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) + @ApiKeyRequired(acceptedScopes = {ApiScope.FEATURESTORE}, + allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Get a storage connector with a specific name and from a featurestore", response = FeaturestoreStorageConnectorDTO.class) public Response getStorageConnector(@Context SecurityContext sc, + @Context HttpServletRequest req, @ApiParam(value = "Name of the storage connector", required = true) @PathParam("connectorName") String connectorName) - throws FeaturestoreException { - + throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); verifyStorageConnectorName(connectorName); FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = storageConnectorController.getConnectorDTOWithName(user, project, featurestore, connectorName); @@ -200,6 +193,8 @@ public Response createNewStorageConnector(@Context SecurityContext sc, "null"); } Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); FeaturestoreStorageConnectorDTO createdFeaturestoreStorageConnectorDTO = storageConnectorController .createStorageConnector(user, project, featurestore, featurestoreStorageConnectorDTO); GenericEntity featurestoreStorageConnectorDTOGenericEntity = @@ -224,10 +219,13 @@ public Response createNewStorageConnector(@Context SecurityContext sc, @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) @ApiOperation(value = "Delete storage connector with a specific name and type from a featurestore") public Response deleteStorageConnector(@Context SecurityContext sc, + @Context HttpServletRequest req, @ApiParam(value = "name of the storage connector", required = true) @PathParam("connectorName") String connectorName) - throws UserException, FeaturestoreException { + throws UserException, FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); storageConnectorController.deleteConnectorWithName(user, project, connectorName, featurestore); return Response.ok().build(); } @@ -259,6 +257,8 @@ public Response updateStorageConnector(@ApiParam(value = "Name of the storage co "null"); } Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); verifyStorageConnectorName(connectorName); storageConnectorController.updateStorageConnector(user, project, featurestore, featurestoreStorageConnectorInputDTO, connectorName); @@ -289,13 +289,15 @@ public Response updateStorageConnector(@ApiParam(value = "Name of the storage co allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Get online featurestore storage connector for this feature store", response = FeaturestoreStorageConnectorDTO.class) - public Response getOnlineFeaturestoreStorageConnector(@Context SecurityContext sc) throws FeaturestoreException { + public Response getOnlineFeaturestoreStorageConnector(@Context SecurityContext sc, + @Context HttpServletRequest req) + throws FeaturestoreException, ProjectException { if (!settings.isOnlineFeaturestore()) { throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ONLINE_NOT_ENABLED, Level.FINE, "Online Featurestore is not enabled for this Hopsworks cluster."); } - // continue even if featureStoreDb does not exist, see HWORKS-919 Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); FeaturestoreStorageConnectorDTO featurestoreJdbcConnectorDTO = storageConnectorController.getOnlineFeaturestoreConnector(user, project); GenericEntity featurestoreStorageConnectorDTOGenericEntity = @@ -325,11 +327,11 @@ public Response getFeatureStoreKafkaConnector(@Context SecurityContext sc, @Context HttpServletRequest req, @QueryParam("external") @DefaultValue("false") Boolean externalListeners) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { KafkaBrokers.BrokerProtocol brokerProtocol = Boolean.TRUE.equals(externalListeners) ? KafkaBrokers.BrokerProtocol.EXTERNAL: KafkaBrokers.BrokerProtocol.INTERNAL; FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO = - storageConnectorController.getKafkaConnector(featurestore, brokerProtocol); + storageConnectorController.getKafkaConnector(getFeaturestore(), brokerProtocol); GenericEntity featurestoreStorageConnectorDTOGenericEntity = new GenericEntity(featurestoreStorageConnectorDTO) {}; @@ -395,14 +397,14 @@ private void verifyStorageConnectorName(String connectorName) { @ApiKeyRequired(acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Test storage connector to data source") - public Response testConnection(@Context SecurityContext sc, - FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) - throws FeaturestoreException { + public Response testConnection(@Context SecurityContext sc, @Context HttpServletRequest req, + FeaturestoreStorageConnectorDTO featurestoreStorageConnectorDTO) throws FeaturestoreException, ProjectException { if (featurestoreStorageConnectorDTO == null) { throw new IllegalArgumentException("Input storage connector DTO cannot be null"); } Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ConnectionCheckerDTO resultDto = connectionChecker.checkConnection(user, project, featurestore, featurestoreStorageConnectorDTO); @@ -414,8 +416,8 @@ public Response testConnection(@Context SecurityContext sc, public StorageConnectorProvenanceResource provenance( @ApiParam(value = "Name of the storage connector", required = true) @PathParam("connectorName") String connectorName) { - this.provenanceResource.setProject(project); - this.provenanceResource.setFeatureStore(featurestore); + this.provenanceResource.setProjectId(getProjectId()); + this.provenanceResource.setFeaturestoreId(getFeaturestoreId()); this.provenanceResource.setConnectorName(connectorName); return this.provenanceResource; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureGroupTagResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureGroupTagResource.java index af606aa18b..12fc5c9a4b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureGroupTagResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureGroupTagResource.java @@ -16,15 +16,20 @@ package io.hops.hopsworks.api.featurestore.tag; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.tags.TagsExpansionBeanParam; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.featurestore.metadata.AttachMetadataResult; import io.hops.hopsworks.common.featurestore.metadata.FeatureStoreTagControllerIface; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.tags.TagsDTO; -import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; @@ -63,42 +68,32 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature Group Tags resource") -public class FeatureGroupTagResource { +public class FeatureGroupTagResource extends FeatureGroupSubResource { @Inject private FeatureStoreTagControllerIface tagController; @EJB private FeatureStoreTagBuilder tagBuilder; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; + @EJB + private FeaturegroupController featuregroupController; - private Project project; - private Featurestore featureStore; - private Featuregroup featureGroup; - - /** - * Set the project of the tag resource (provided by parent resource) - * - * @param project the project where the tag operations will be performed - */ - public void setProject(Project project) { - this.project = project; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - /** - * Sets the feature store of the tag resource - * - * @param featureStore - */ - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturegroupController getFeaturegroupController() { + return featuregroupController; } - /** - * Sets the feature group of the tag resource - * - * @param featureGroup - */ - public void setFeatureGroup(Featuregroup featureGroup) { - this.featureGroup = featureGroup; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Create or update one tag", response = TagsDTO.class) @@ -116,7 +111,10 @@ public Response putTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @ApiParam(value = "Value to set for the tag") String value) - throws FeatureStoreMetadataException, FeaturestoreException { + throws FeatureStoreMetadataException, FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + Featuregroup featureGroup = getFeaturegroup(featureStore); AttachMetadataResult result = tagController.upsertTag(featureGroup, name, value); TagsDTO dto = tagBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.TAGS), project.getId(), featureStore.getId(), ResourceRequest.Name.FEATUREGROUPS, featureGroup.getId(), result.getItems()); @@ -140,8 +138,10 @@ public Response putTag(@Context SecurityContext sc, public Response putTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, TagsDTO tagsDTO) - throws FeatureStoreMetadataException, FeaturestoreException { - + throws FeatureStoreMetadataException, FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + Featuregroup featureGroup = getFeaturegroup(featureStore); Map tags; if(tagsDTO.getItems() == null || tagsDTO.getItems().isEmpty()) { tags = new HashMap<>(); @@ -171,7 +171,10 @@ public Response putTags(@Context SecurityContext sc, @Context UriInfo uriInfo, public Response getTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws FeatureStoreMetadataException { + throws FeatureStoreMetadataException, ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + Featuregroup featureGroup = getFeaturegroup(featureStore); Map result = tagController.getTags(featureGroup); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); resourceRequest.setExpansions(tagsExpansionBeanParam.getResources()); @@ -193,7 +196,10 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws FeatureStoreMetadataException { + throws FeatureStoreMetadataException, ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + Featuregroup featureGroup = getFeaturegroup(featureStore); Optional result = tagController.getTag(featureGroup, name); if(result.isPresent()) { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); @@ -219,8 +225,8 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteTags(@Context SecurityContext sc, @Context HttpServletRequest req) - throws FeaturestoreException { - tagController.deleteTags(featureGroup); + throws FeaturestoreException, ProjectException { + tagController.deleteTags(getFeaturegroup()); return Response.noContent().build(); } @@ -236,8 +242,8 @@ public Response deleteTags(@Context SecurityContext sc, public Response deleteTag(@Context SecurityContext sc, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name) - throws FeaturestoreException, FeatureStoreMetadataException { - tagController.deleteTag(featureGroup, name); + throws FeaturestoreException, FeatureStoreMetadataException, ProjectException { + tagController.deleteTag(getFeaturegroup(), name); return Response.noContent().build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureViewTagResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureViewTagResource.java index 3de4082dc2..1d75d35aab 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureViewTagResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/FeatureViewTagResource.java @@ -16,16 +16,20 @@ package io.hops.hopsworks.api.featurestore.tag; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.tags.TagsExpansionBeanParam; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.metadata.AttachMetadataResult; import io.hops.hopsworks.common.featurestore.metadata.FeatureStoreTagControllerIface; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.tags.TagsDTO; -import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; @@ -64,45 +68,31 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature View Tags resource") -public class FeatureViewTagResource { +public class FeatureViewTagResource extends FeatureViewSubResource { @Inject private FeatureStoreTagControllerIface tagController; @EJB private FeatureStoreTagBuilder tagBuilder; - - private Project project; - private Featurestore featureStore; - private FeatureView featureView; @EJB private FeatureViewController featureViewController; - /** - * Set the project of the tag resource (provided by parent resource) - * - * @param project the project where the tag operations will be performed - */ - public void setProject(Project project) { - this.project = project; - } + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - /** - * Sets the feature store of the tag resource - * - * @param featureStore - */ - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected ProjectController getProjectController() { + return projectController; } - - /** - * Sets the feature view of the tag resource - * - * @param name - * @param version - */ - public void setFeatureView(String name, Integer version) throws FeaturestoreException { - this.featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featureStore); + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } @ApiOperation(value = "Create or update one tag", response = TagsDTO.class) @@ -120,7 +110,10 @@ public Response putTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @ApiParam(value = "Value to set for the tag") String value) - throws FeatureStoreMetadataException, FeaturestoreException { + throws FeatureStoreMetadataException, FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featureStore); AttachMetadataResult result = tagController.upsertTag(featureView, name, value); TagsDTO dto = tagBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.TAGS), project.getId(), featureStore.getId(), ResourceRequest.Name.FEATUREVIEW, featureView.getId(), result.getItems()); @@ -144,8 +137,10 @@ public Response putTag(@Context SecurityContext sc, public Response putTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, TagsDTO tagsDTO) - throws FeatureStoreMetadataException, FeaturestoreException { - + throws FeatureStoreMetadataException, FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featureStore); Map tags; if(tagsDTO.getItems() == null || tagsDTO.getItems().isEmpty()) { tags = new HashMap<>(); @@ -175,7 +170,10 @@ public Response putTags(@Context SecurityContext sc, @Context UriInfo uriInfo, public Response getTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws FeatureStoreMetadataException { + throws FeatureStoreMetadataException, ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featureStore); Map result = tagController.getTags(featureView); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); resourceRequest.setExpansions(tagsExpansionBeanParam.getResources()); @@ -197,7 +195,10 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws FeatureStoreMetadataException { + throws FeatureStoreMetadataException, ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + FeatureView featureView = getFeatureView(featureStore); Optional result = tagController.getTag(featureView, name); if(result.isPresent()) { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); @@ -223,8 +224,8 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteTags(@Context SecurityContext sc, @Context HttpServletRequest req) - throws FeaturestoreException { - tagController.deleteTags(featureView); + throws FeaturestoreException, ProjectException { + tagController.deleteTags(getFeatureView()); return Response.noContent().build(); } @@ -240,8 +241,8 @@ public Response deleteTags(@Context SecurityContext sc, public Response deleteTag(@Context SecurityContext sc, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name) - throws FeaturestoreException, FeatureStoreMetadataException { - tagController.deleteTag(featureView, name); + throws FeaturestoreException, FeatureStoreMetadataException, ProjectException { + tagController.deleteTag(getFeatureView(), name); return Response.noContent().build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/TrainingDatasetTagResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/TrainingDatasetTagResource.java index f5dd23f94d..2c1485d006 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/TrainingDatasetTagResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/tag/TrainingDatasetTagResource.java @@ -16,15 +16,21 @@ package io.hops.hopsworks.api.featurestore.tag; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.trainingdataset.TrainingDatasetSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.tags.TagsExpansionBeanParam; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.metadata.AttachMetadataResult; import io.hops.hopsworks.common.featurestore.metadata.FeatureStoreTagControllerIface; +import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.tags.TagsDTO; -import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.metadata.FeatureStoreTag; @@ -63,42 +69,39 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Training Dataset Tags resource") -public class TrainingDatasetTagResource { +public class TrainingDatasetTagResource extends TrainingDatasetSubResource { @Inject private FeatureStoreTagControllerIface tagController; @EJB private FeatureStoreTagBuilder tagBuilder; - private Project project; - private Featurestore featureStore; - private TrainingDataset trainingDataset; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private FeatureViewController featureViewController; + @EJB + private TrainingDatasetController trainingDatasetController; - /** - * Set the project of the tag resource (provided by parent resource) - * - * @param project the project where the tag operations will be performed - */ - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** - * Sets the feature store of the tag resource - * - * @param featureStore - */ - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } - /** - * Sets the training dataset of the tag resource - * - * @param trainingDataset - */ - public void setTrainingDataset(TrainingDataset trainingDataset) { - this.trainingDataset = trainingDataset; + @Override + protected TrainingDatasetController getTrainingDatasetController() { + return trainingDatasetController; } @ApiOperation(value = "Create or update one tag", response = TagsDTO.class) @@ -116,7 +119,10 @@ public Response putTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @ApiParam(value = "Value to set for the tag") String value) - throws FeatureStoreMetadataException, FeaturestoreException { + throws FeatureStoreMetadataException, FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featureStore); AttachMetadataResult result = tagController.upsertTag(trainingDataset, name, value); TagsDTO dto = tagBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.TAGS), project.getId(), featureStore.getId(), ResourceRequest.Name.TRAININGDATASETS, trainingDataset.getId(), result.getItems()); @@ -140,8 +146,10 @@ public Response putTag(@Context SecurityContext sc, public Response putTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, TagsDTO tagsDTO) - throws FeatureStoreMetadataException, FeaturestoreException { - + throws FeatureStoreMetadataException, FeaturestoreException, ProjectException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featureStore); Map tags; if(tagsDTO.getItems() == null || tagsDTO.getItems().isEmpty()) { tags = new HashMap<>(); @@ -171,7 +179,10 @@ public Response putTags(@Context SecurityContext sc, @Context UriInfo uriInfo, public Response getTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws FeatureStoreMetadataException { + throws FeatureStoreMetadataException, ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featureStore); Map result = tagController.getTags(trainingDataset); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); resourceRequest.setExpansions(tagsExpansionBeanParam.getResources()); @@ -193,7 +204,10 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws FeatureStoreMetadataException { + throws FeatureStoreMetadataException, ProjectException, FeaturestoreException { + Project project = getProject(); + Featurestore featureStore = getFeaturestore(project); + TrainingDataset trainingDataset = getTrainingDataset(featureStore); Optional result = tagController.getTag(trainingDataset, name); if(result.isPresent()) { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); @@ -219,8 +233,8 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteTags(@Context SecurityContext sc, @Context HttpServletRequest req) - throws FeaturestoreException { - tagController.deleteTags(trainingDataset); + throws FeaturestoreException, ProjectException { + tagController.deleteTags(getTrainingDataset()); return Response.noContent().build(); } @@ -236,8 +250,8 @@ public Response deleteTags(@Context SecurityContext sc, public Response deleteTag(@Context SecurityContext sc, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name) - throws FeaturestoreException, FeatureStoreMetadataException { - tagController.deleteTag(trainingDataset, name); + throws FeaturestoreException, FeatureStoreMetadataException, ProjectException { + tagController.deleteTag(getTrainingDataset(), name); return Response.noContent().build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetResource.java index 8fe75261a0..b063823094 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetResource.java @@ -15,33 +15,35 @@ */ package io.hops.hopsworks.api.featurestore.trainingdataset; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.featurestore.keyword.TrainingDatasetKeywordResource; -import io.hops.hopsworks.api.featurestore.tag.TrainingDatasetTagResource; -import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.api.featurestore.statistics.StatisticsResource; +import io.hops.hopsworks.api.featurestore.tag.TrainingDatasetTagResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jobs.JobDTO; import io.hops.hopsworks.api.jobs.JobsBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.provenance.TrainingDatasetProvenanceResource; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.OptionDTO; import io.hops.hopsworks.common.featurestore.app.FsJobManagerController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.JobException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ProjectException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; import io.hops.hopsworks.persistence.entity.jobs.description.Jobs; import io.hops.hopsworks.persistence.entity.project.Project; @@ -80,7 +82,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class TrainingDatasetResource { +public class TrainingDatasetResource extends FeatureViewSubResource { @Inject private StatisticsResource statisticsResource; @@ -103,9 +105,23 @@ public class TrainingDatasetResource { @Inject private TrainingDatasetProvenanceResource provenanceResource; - private Featurestore featurestore; - private FeatureView featureView; - private Project project; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; + + @Override + protected ProjectController getProjectController() { + return projectController; + } + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; + } @POST @Produces(MediaType.APPLICATION_JSON) @@ -124,10 +140,13 @@ public Response create( @Context UriInfo uriInfo, TrainingDatasetDTO trainingDatasetDTO) - throws FeaturestoreException, IOException, ServiceException { + throws FeaturestoreException, IOException, ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TrainingDatasetDTO createdTrainingDatasetDTO = - trainingDatasetController.createTrainingDataset(user, project, featurestore, featureView, trainingDatasetDTO); + trainingDatasetController.createTrainingDataset(user, project, featurestore, getFeatureView(featurestore), + trainingDatasetDTO); return Response.created(uriInfo.getRequestUri()).entity(createdTrainingDatasetDTO).build(); } @@ -149,10 +168,12 @@ public Response getAll( @BeanParam TrainingDatasetExpansionBeanParam param ) throws FeaturestoreException, ServiceException, MetadataException, DatasetException, FeatureStoreMetadataException, - IOException { + IOException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); List trainingDatasets = - trainingDatasetController.getTrainingDatasetByFeatureView(featureView); + trainingDatasetController.getTrainingDatasetByFeatureView(getFeatureView(featurestore)); ResourceRequest resourceRequest = makeResourceRequest(param); TrainingDatasetDTO trainingDatasetDTO = trainingDatasetDTOBuilder.build(user, project, trainingDatasets, uriInfo, resourceRequest); @@ -181,9 +202,12 @@ public Response getByVersion( @PathParam("version") Integer version ) throws FeaturestoreException, ServiceException, MetadataException, DatasetException, FeatureStoreMetadataException, - IOException { + IOException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + TrainingDataset trainingDataset = + trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(getFeatureView(featurestore), version); ResourceRequest resourceRequest = makeResourceRequest(param); TrainingDatasetDTO trainingDatasetDTO = trainingDatasetDTOBuilder.build(user, project, trainingDataset, uriInfo, @@ -210,9 +234,11 @@ public Response delete( SecurityContext sc, @Context HttpServletRequest req - ) throws FeaturestoreException, JobException { + ) throws FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - trainingDatasetController.delete(user, project, featurestore, featureView); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + trainingDatasetController.delete(user, project, featurestore, getFeatureView(featurestore)); return Response.ok().build(); } @@ -233,9 +259,11 @@ public Response deleteByVersion( @ApiParam(value = "training dataset version") @PathParam("version") Integer version - ) throws FeaturestoreException, JobException { + ) throws FeaturestoreException, JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - trainingDatasetController.delete(user, project, featurestore, featureView, version); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + trainingDatasetController.delete(user, project, featurestore, getFeatureView(featurestore), version); return Response.ok().build(); } @@ -253,9 +281,11 @@ public Response deleteDataOnly( SecurityContext sc, @Context HttpServletRequest req - ) throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - trainingDatasetController.deleteDataOnly(user, project, featurestore, featureView); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + trainingDatasetController.deleteDataOnly(user, project, featurestore, getFeatureView(featurestore)); return Response.ok().build(); } @@ -275,9 +305,11 @@ public Response deleteDataOnlyByVersion( @ApiParam(value = "training dataset version") @PathParam("version") Integer version - ) throws FeaturestoreException { + ) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - trainingDatasetController.deleteDataOnly(user, project, featurestore, featureView, version); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); + trainingDatasetController.deleteDataOnly(user, project, featurestore, getFeatureView(featurestore), version); return Response.ok().build(); } @@ -302,14 +334,17 @@ public Response updateTrainingDataset(@Context SecurityContext sc, @PathParam("version") Integer version, TrainingDatasetDTO trainingDatasetDTO) - throws FeaturestoreException, ServiceException { + throws FeaturestoreException, ServiceException, ProjectException { if (trainingDatasetDTO == null) { throw new IllegalArgumentException("Input JSON for updating a Training Dataset cannot be null"); } Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); trainingDatasetDTO.setVersion(version); - TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, + TrainingDataset trainingDataset = + trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(getFeatureView(featurestore), version); trainingDatasetDTO.setId(trainingDataset.getId()); TrainingDatasetDTO oldTrainingDatasetDTO = trainingDatasetController.convertTrainingDatasetToDTO(user, project, @@ -331,13 +366,11 @@ public Response updateTrainingDataset(@Context SecurityContext sc, @Path("/version/{version: [0-9]+}/keywords") public TrainingDatasetKeywordResource keywords( @ApiParam(value = "Version of the training dataset", required = true) - @PathParam("version") - Integer version - ) throws FeaturestoreException { - this.trainingDatasetKeywordResource.setProject(project); - TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion( - featureView, version); - this.trainingDatasetKeywordResource.setTrainingDataset(trainingDataset); + @PathParam("version") Integer version) { + this.trainingDatasetKeywordResource.setProjectId(getProjectId()); + this.trainingDatasetKeywordResource.setFeaturestoreId(getFeaturestoreId()); + this.trainingDatasetKeywordResource.setFeatureView(getName(), getVersion()); + this.trainingDatasetKeywordResource.setTrainingDatasetVersion(version); return this.trainingDatasetKeywordResource; } @@ -346,11 +379,12 @@ public StatisticsResource statistics( @ApiParam(value = "Version of the training dataset", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - statisticsResource.setProject(project); - statisticsResource.setFeaturestore(featurestore); - statisticsResource.setTrainingDatasetByVersion(featureView, version); - return statisticsResource; + ) { + this.statisticsResource.setProjectId(getProjectId()); + this.statisticsResource.setFeaturestoreId(getFeaturestoreId()); + this.statisticsResource.setFeatureView(getName(), getVersion()); + this.statisticsResource.setTrainingDatasetVersion(version); + return this.statisticsResource; } @Path("/version/{version: [0-9]+}/tags") @@ -358,12 +392,11 @@ public TrainingDatasetTagResource tags( @ApiParam(value = "Version of the training dataset", required = true) @PathParam("version") Integer version - ) throws FeaturestoreException { - tagResource.setProject(project); - tagResource.setFeatureStore(featurestore); - TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion( - featureView, version); - tagResource.setTrainingDataset(trainingDataset); + ) { + tagResource.setProjectId(getProjectId()); + tagResource.setFeaturestoreId(getFeaturestoreId()); + tagResource.setFeatureView(getName(), getVersion()); + tagResource.setTrainingDatasetVersion(version); return tagResource; } @@ -390,31 +423,21 @@ public Response compute(@Context UriInfo uriInfo, writeOptions = trainingDatasetJobConf.getWriteOptions() .stream().collect(Collectors.toMap(OptionDTO::getName, OptionDTO::getValue)); } - - Jobs job = fsJobManagerController.setupTrainingDatasetJob(project, user, featureView, trainingDatasetVersion, + Project project = getProject(); + Jobs job = + fsJobManagerController.setupTrainingDatasetJob(project, user, getFeatureView(project), trainingDatasetVersion, trainingDatasetJobConf.getOverwrite(), writeOptions, trainingDatasetJobConf.getSparkJobConfiguration()); JobDTO jobDTO = jobsBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.JOBS), job); return Response.created(jobDTO.getHref()).entity(jobDTO).build(); } - public void setFeatureView(String name, Integer version) throws FeaturestoreException { - featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); - } - - public void setProject(Project project) { - this.project = project; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } - @Path("/version/{version: [0-9]+}/provenance") public TrainingDatasetProvenanceResource provenance(@ApiParam(value = "Id of the training dataset") @PathParam("version") Integer trainingDatasetVersion) { - this.provenanceResource.setProject(project); - this.provenanceResource.setFeatureView(featureView); + this.provenanceResource.setProjectId(getProjectId()); + this.provenanceResource.setFeaturestoreId(getFeaturestoreId()); + this.provenanceResource.setFeatureView(getName(), getVersion()); this.provenanceResource.setTrainingDatasetVersion(trainingDatasetVersion); return this.provenanceResource; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetService.java index d600ac0972..dffa4cb3ed 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetService.java @@ -17,6 +17,8 @@ package io.hops.hopsworks.api.featurestore.trainingdataset; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.featurestore.FsQueryBuilder; import io.hops.hopsworks.api.featurestore.activities.ActivityResource; import io.hops.hopsworks.api.featurestore.code.CodeResource; @@ -27,20 +29,20 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.NoCacheResponse; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jobs.JobDTO; import io.hops.hopsworks.api.jobs.JobsBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.OptionDTO; import io.hops.hopsworks.common.featurestore.app.FsJobManagerController; import io.hops.hopsworks.common.featurestore.query.FsQueryDTO; import io.hops.hopsworks.common.featurestore.query.ServingPreparedStatementDTO; -import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.user.activity.ActivityFacade; -import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; import io.hops.hopsworks.common.featurestore.transformationFunction.TransformationFunctionAttachedDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; @@ -95,7 +97,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "TrainingDataset service", description = "A service that manages a feature store's training datasets") -public class TrainingDatasetService { +public class TrainingDatasetService extends FeaturestoreSubResource { @EJB private NoCacheResponse noCacheResponse; @@ -127,28 +129,17 @@ public class TrainingDatasetService { private TransformationFunctionBuilder transformationFunctionBuilder; @Inject private TrainingDatasetTagResource tagResource; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; - - /** - * Set the project of the featurestore (provided by parent resource) - * - * @param project the project where the featurestore resides - */ - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** - * Sets the featurestore of the training datasets (provided by parent resource) - * - * @param featurestoreId id of the featurestore - * @throws FeaturestoreException - */ - public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreException { - //This call verifies that the project have access to the featurestoreId provided - this.featurestore = featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId); + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } /** @@ -165,8 +156,11 @@ public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreExcepti @ApiOperation(value = "Get the list of training datasets for a featurestore", response = TrainingDatasetDTO.class, responseContainer = "List") public Response getAll(@Context SecurityContext sc, - @Context HttpServletRequest req) throws ServiceException, FeaturestoreException { + @Context HttpServletRequest req) + throws ServiceException, FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); List trainingDatasetDTOs = trainingDatasetController.getTrainingDatasetsForFeaturestore(user, project, featurestore); GenericEntity> trainingDatasetsGeneric = @@ -174,6 +168,7 @@ public Response getAll(@Context SecurityContext sc, return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(trainingDatasetsGeneric).build(); } + /** * Endpoint for creating a new trainingDataset * @@ -193,15 +188,17 @@ public Response getAll(@Context SecurityContext sc, public Response create(@Context SecurityContext sc, @Context HttpServletRequest req, TrainingDatasetDTO trainingDatasetDTO) - throws FeaturestoreException, IOException, ServiceException { - if(trainingDatasetDTO == null){ + throws FeaturestoreException, IOException, ServiceException, ProjectException { + if (trainingDatasetDTO == null) { throw new IllegalArgumentException("Input JSON for creating a new Training Dataset cannot be null"); } Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TrainingDatasetDTO createdTrainingDatasetDTO = trainingDatasetController.createTrainingDataset(user, project, featurestore, trainingDatasetDTO); GenericEntity createdTrainingDatasetDTOGeneric = - new GenericEntity(createdTrainingDatasetDTO) {}; + new GenericEntity(createdTrainingDatasetDTO) {}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED).entity(createdTrainingDatasetDTOGeneric) .build(); } @@ -229,10 +226,12 @@ public Response create(@Context SecurityContext sc, public Response getById(@Context HttpServletRequest req, @ApiParam(value = "Id of the training dataset", required = true) @PathParam("trainingdatasetid") Integer trainingdatasetid, @Context SecurityContext sc) - throws FeaturestoreException, ServiceException { + throws FeaturestoreException, ServiceException, ProjectException { verifyIdProvided(trainingdatasetid); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TrainingDatasetDTO trainingDatasetDTO = trainingDatasetController .getTrainingDatasetWithIdAndFeaturestore(user, project, featurestore, trainingdatasetid); GenericEntity trainingDatasetGeneric = @@ -262,10 +261,12 @@ public Response getByName(@Context HttpServletRequest req, @PathParam("name") String name, @ApiParam(value = "Filter by a specific version") @QueryParam("version") Integer version, @Context SecurityContext sc) - throws FeaturestoreException, ServiceException { + throws FeaturestoreException, ServiceException, ProjectException { verifyNameProvided(name); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); List trainingDatasetDTO; if (version == null) { trainingDatasetDTO = @@ -302,9 +303,11 @@ public Response getByName(@Context HttpServletRequest req, public Response delete(@Context HttpServletRequest req, @Context SecurityContext sc, @ApiParam(value = "Id of the training dataset", required = true) @PathParam("trainingdatasetid") Integer trainingdatasetid) - throws FeaturestoreException, JobException { + throws FeaturestoreException, JobException, ProjectException { verifyIdProvided(trainingdatasetid); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); String trainingDsName = trainingDatasetController.delete(user, project, featurestore, trainingdatasetid); activityFacade.persistActivity(ActivityFacade.DELETED_TRAINING_DATASET + trainingDsName, project, user, ActivityFlag.SERVICE); @@ -339,15 +342,17 @@ public Response updateTrainingDataset(@Context SecurityContext sc, @QueryParam("updateMetadata") @DefaultValue("false") Boolean updateMetadata, @ApiParam(value = "updateStatsConfig", example = "true") @QueryParam("updateStatsConfig") @DefaultValue("false") - Boolean updateStatsConfig, + Boolean updateStatsConfig, TrainingDatasetDTO trainingDatasetDTO) - throws FeaturestoreException, ServiceException { - if(trainingDatasetDTO == null){ + throws FeaturestoreException, ServiceException, ProjectException { + if (trainingDatasetDTO == null) { throw new IllegalArgumentException("Input JSON for updating a Training Dataset cannot be null"); } verifyIdProvided(trainingdatasetid); trainingDatasetDTO.setId(trainingdatasetid); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TrainingDatasetDTO oldTrainingDatasetDTO = trainingDatasetController .getTrainingDatasetWithIdAndFeaturestore(user, project, featurestore, trainingdatasetid); @@ -366,19 +371,17 @@ public Response updateTrainingDataset(@Context SecurityContext sc, } @Path("/{trainingDatasetId}/statistics") - public StatisticsResource statistics(@PathParam("trainingDatasetId") Integer trainingDatasetId) - throws FeaturestoreException { - this.statisticsResource.setProject(project); - this.statisticsResource.setFeaturestore(featurestore); - this.statisticsResource.setTrainingDatasetById(trainingDatasetId); + public StatisticsResource statistics(@PathParam("trainingDatasetId") Integer trainingDatasetId) { + this.statisticsResource.setProjectId(getProjectId()); + this.statisticsResource.setFeaturestoreId(getFeaturestoreId()); + this.statisticsResource.setTrainingDatasetId(trainingDatasetId); return statisticsResource; } @Path("/{trainingDatasetId}/code") - public CodeResource code(@PathParam("trainingDatasetId") Integer trainingDatasetId) - throws FeaturestoreException { - this.codeResource.setProject(project); - this.codeResource.setFeatureStore(featurestore); + public CodeResource code(@PathParam("trainingDatasetId") Integer trainingDatasetId) { + this.codeResource.setProjectId(getProjectId()); + this.codeResource.setFeaturestoreId(getFeaturestoreId()); this.codeResource.setTrainingDatasetId(trainingDatasetId); return codeResource; } @@ -406,10 +409,11 @@ public Response getQuery( @ApiParam(value = "get query in hive format", example = "true") @QueryParam("hiveQuery") @DefaultValue("false") - boolean isHiveQuery) throws FeaturestoreException, ServiceException { + boolean isHiveQuery) throws FeaturestoreException, ServiceException, ProjectException { verifyIdProvided(trainingdatasetid); Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); FsQueryDTO fsQueryDTO = fsQueryBuilder.build( uriInfo, project, user, featurestore, trainingdatasetid, withLabel, isHiveQuery); return Response.ok().entity(fsQueryDTO).build(); @@ -433,6 +437,8 @@ public Response compute(@Context UriInfo uriInfo, throws FeaturestoreException, ServiceException, JobException, ProjectException, GenericException { verifyIdProvided(trainingDatasetId); Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); Map writeOptions = null; @@ -453,11 +459,10 @@ public Response compute(@Context UriInfo uriInfo, @Path("/{trainingDatasetId}/keywords") public TrainingDatasetKeywordResource keywords( - @ApiParam(value = "Id of the training dataset") @PathParam("trainingDatasetId") Integer trainingDatasetId) - throws FeaturestoreException { - this.trainingDatasetKeywordResource.setProject(project); - TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); - this.trainingDatasetKeywordResource.setTrainingDataset(trainingDataset); + @ApiParam(value = "Id of the training dataset") @PathParam("trainingDatasetId") Integer trainingDatasetId) { + this.trainingDatasetKeywordResource.setProjectId(getProjectId()); + this.trainingDatasetKeywordResource.setFeaturestoreId(getFeaturestoreId()); + this.trainingDatasetKeywordResource.setTrainingDatasetId(trainingDatasetId); return trainingDatasetKeywordResource; } @@ -465,9 +470,9 @@ public TrainingDatasetKeywordResource keywords( public ActivityResource activity(@ApiParam(value = "Id of the training dataset") @PathParam("trainingDatasetId") Integer trainingDatasetId) throws FeaturestoreException { - this.activityResource.setProject(project); - this.activityResource.setFeaturestore(featurestore); - this.activityResource.setTrainingDatasetById(trainingDatasetId); + this.activityResource.setProjectId(getProjectId()); + this.activityResource.setFeaturestoreId(getFeaturestoreId()); + this.activityResource.setTrainingDatasetId(trainingDatasetId); return this.activityResource; } @@ -511,10 +516,11 @@ public Response getPreparedStatements(@Context SecurityContext sc, @PathParam("trainingdatasetid") Integer trainingDatsetId, @ApiParam(value = "get batch serving vectors", example = "false") @QueryParam("batch") @DefaultValue("false") boolean batch) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { verifyIdProvided(trainingDatsetId); Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ServingPreparedStatementDTO servingPreparedStatementDTO = preparedStatementBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.PREPAREDSTATEMENTS), project, user, featurestore, trainingDatsetId, batch); @@ -535,11 +541,12 @@ public Response getTransformationFunction(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Id of the trainingdatasetid", required = true) @PathParam("trainingdatasetid") Integer trainingDatasetId) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { if (trainingDatasetId == null) { throw new IllegalArgumentException(RESTCodes.FeaturestoreErrorCode.TRAINING_DATASET_ID_NOT_PROVIDED.getMessage()); } - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); Users user = jWTHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TRANSFORMATIONFUNCTIONS); @@ -550,13 +557,11 @@ public Response getTransformationFunction(@Context SecurityContext sc, @Path("/{trainingDatasetId}/tags") public TrainingDatasetTagResource tags(@ApiParam(value = "Id of the training dataset") - @PathParam("trainingDatasetId") Integer trainingDatasetId) - throws FeaturestoreException { + @PathParam("trainingDatasetId") Integer trainingDatasetId) { verifyIdProvided(trainingDatasetId); - this.tagResource.setProject(project); - this.tagResource.setFeatureStore(featurestore); - TrainingDataset trainingDataset = trainingDatasetController.getTrainingDatasetById(featurestore, trainingDatasetId); - this.tagResource.setTrainingDataset(trainingDataset); + this.tagResource.setProjectId(getProjectId()); + this.tagResource.setFeaturestoreId(getFeaturestoreId()); + this.tagResource.setTrainingDatasetId(trainingDatasetId); return this.tagResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetSubResource.java new file mode 100644 index 0000000000..b6d32954e3 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/trainingdataset/TrainingDatasetSubResource.java @@ -0,0 +1,128 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.featurestore.trainingdataset; + +import com.google.common.base.Strings; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; +import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; +import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDataset; +import io.hops.hopsworks.persistence.entity.project.Project; + +public abstract class TrainingDatasetSubResource extends FeaturestoreSubResource { + private String featureViewName; + private Integer featureViewVersion; + private Integer trainingDatasetId; + private Integer trainingDatasetVersion; + + public String getFeatureViewName() { + return featureViewName; + } + + public void setFeatureViewName(String featureViewName) { + this.featureViewName = featureViewName; + } + + public Integer getFeatureViewVersion() { + return featureViewVersion; + } + + public void setFeatureViewVersion(Integer featureViewVersion) { + this.featureViewVersion = featureViewVersion; + } + + public Integer getTrainingDatasetId() { + return trainingDatasetId; + } + + public void setTrainingDatasetId(Integer trainingDatasetId) { + this.trainingDatasetId = trainingDatasetId; + } + + public Integer getTrainingDatasetVersion() { + return trainingDatasetVersion; + } + + public void setTrainingDatasetVersion(Integer trainingDatasetVersion) { + this.trainingDatasetVersion = trainingDatasetVersion; + } + + public void setFeatureView(String name, Integer version) { + this.featureViewName = name; + this.featureViewVersion = version; + } + + public FeatureView getFeatureView(Project project) throws FeaturestoreException, ProjectException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return getFeatureViewController().getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, + getFeaturestore(project)); + } + + public FeatureView getFeatureView(Featurestore featurestore) throws FeaturestoreException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return getFeatureViewController().getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, + featurestore); + } + + public FeatureView getFeatureView() throws ProjectException, FeaturestoreException { + if (Strings.isNullOrEmpty(featureViewName) || featureViewVersion == null) { + return null; + } + return getFeatureViewController().getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, + getFeaturestore()); + } + + public TrainingDataset getTrainingDataset(Project project) throws FeaturestoreException, ProjectException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(getFeaturestore(project), trainingDatasetId); + } else if (trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(getFeatureView(project), + trainingDatasetVersion); + } + return null; + } + + public TrainingDataset getTrainingDataset(Featurestore featurestore) throws FeaturestoreException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(featurestore, trainingDatasetId); + } else if (trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(getFeatureView(featurestore), + trainingDatasetVersion); + } + return null; + } + + public TrainingDataset getTrainingDataset() throws ProjectException, FeaturestoreException { + if (trainingDatasetId != null) { + return getTrainingDatasetController().getTrainingDatasetById(getFeaturestore(), trainingDatasetId); + } else if (trainingDatasetVersion != null) { + return getTrainingDatasetController().getTrainingDatasetByFeatureViewAndVersion(getFeatureView(), + trainingDatasetVersion); + } + return null; + } + + protected abstract FeatureViewController getFeatureViewController(); + protected abstract TrainingDatasetController getTrainingDatasetController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformation/TransformationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformation/TransformationResource.java index 545dee6e5d..0f3e83add5 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformation/TransformationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformation/TransformationResource.java @@ -16,19 +16,20 @@ package io.hops.hopsworks.api.featurestore.transformation; -import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.featurestore.transformationFunction.TransformationFunctionBuilder; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.transformationFunction.TransformationFunctionAttachedDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; -import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; @@ -49,20 +50,33 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Transformation Resource") -public class TransformationResource { +public class TransformationResource extends FeatureViewSubResource { @EJB private FeatureViewController featureViewController; @EJB - private TrainingDatasetController trainingDatasetController; - @EJB private TransformationFunctionBuilder transformationFunctionBuilder; @EJB private JWTHelper jWTHelper; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; + + @Override + protected ProjectController getProjectController() { + return projectController; + } + + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; + } + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; + } - private Project project; - private Featurestore featurestore; - private FeatureView featureView; @GET @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @@ -79,24 +93,13 @@ public Response getTransformationFunction( @Context UriInfo uriInfo ) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TRANSFORMATIONFUNCTIONS); + Project project = getProject(); TransformationFunctionAttachedDTO transformationFunctionAttachedDTO = - transformationFunctionBuilder.build(uriInfo, resourceRequest, user, project, featureView); + transformationFunctionBuilder.build(uriInfo, resourceRequest, user, project, getFeatureView(project)); return Response.ok().entity(transformationFunctionAttachedDTO).build(); } - - public void setFeatureView(String name, Integer version) throws FeaturestoreException { - featureView = featureViewController.getByNameVersionAndFeatureStore(name, version, featurestore); - } - - public void setProject(Project project) { - this.project = project; - } - - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; - } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformationFunction/TransformationFunctionResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformationFunction/TransformationFunctionResource.java index 21dc3b0ceb..3f44594f1c 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformationFunction/TransformationFunctionResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/transformationFunction/TransformationFunctionResource.java @@ -16,16 +16,20 @@ package io.hops.hopsworks.api.featurestore.transformationFunction; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetDTO; -import io.hops.hopsworks.common.featurestore.transformationFunction.TransformationFunctionDTO; import io.hops.hopsworks.common.featurestore.transformationFunction.TransformationFunctionController; +import io.hops.hopsworks.common.featurestore.transformationFunction.TransformationFunctionDTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.transformationFunction.TransformationFunction; @@ -46,7 +50,6 @@ import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; - import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -61,7 +64,7 @@ @Api(value = "Feature Store Transformation Function Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class TransformationFunctionResource { +public class TransformationFunctionResource extends FeaturestoreSubResource { @EJB private TransformationFunctionBuilder transformationFunctionBuilder; @@ -69,16 +72,19 @@ public class TransformationFunctionResource { private TransformationFunctionController transformationFunctionController; @EJB private JWTHelper jWTHelper; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private ProjectController projectController; - private Project project; - private Featurestore featurestore; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeaturestore(Featurestore featurestore) { - this.featurestore = featurestore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } @GET @@ -95,10 +101,12 @@ public Response get( @BeanParam TransformationFunctionsBeanParam transformationFunctionsBeanParam, @Context SecurityContext sc, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @QueryParam("name") String name, - @QueryParam("version") Integer version) throws FeaturestoreException { + @QueryParam("version") Integer version) throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TRANSFORMATIONFUNCTIONS); resourceRequest.setSort(transformationFunctionsBeanParam.getSortBySet()); resourceRequest.setFilter(transformationFunctionsBeanParam.getFilter()); @@ -121,11 +129,13 @@ public Response get( @ApiOperation(value = "Register transformation function in to a featurestore", response = TransformationFunctionDTO.class) public Response attach(@Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc, TransformationFunctionDTO transformationFunctionDTO) - throws IOException, FeaturestoreException { + throws IOException, FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); TransformationFunction transformationFunction = transformationFunctionController.register(user, project, featurestore, transformationFunctionDTO); @@ -156,8 +166,10 @@ public Response delete(@Context SecurityContext sc, @Context HttpServletRequest req, @ApiParam(value = "Id of the transformation function dataset", required = true) @PathParam("transformationFunctionId") Integer transformationFunctionId) - throws FeaturestoreException { + throws FeaturestoreException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); transformationFunctionController.delete(project, featurestore, user, transformationFunctionId); return Response.ok().build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java index e2aa8c04cb..fb690cfd48 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitResource.java @@ -31,9 +31,9 @@ import io.hops.hopsworks.api.git.repository.GitRepositoryDTO; import io.hops.hopsworks.api.git.repository.RepositoryBeanParam; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.git.BranchCommits; import io.hops.hopsworks.common.git.CloneCommandConfiguration; import io.hops.hopsworks.common.git.GitBranchAction; @@ -43,9 +43,11 @@ import io.hops.hopsworks.common.git.GitRepositoryAction; import io.hops.hopsworks.common.git.RepositoryActionCommandConfiguration; import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.GitOpException; import io.hops.hopsworks.exceptions.HopsSecurityException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.git.GitOpExecution; import io.hops.hopsworks.persistence.entity.git.GitRepository; @@ -82,11 +84,11 @@ @Api(value = "Git Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class GitResource { +public class GitResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(GitResource.class.getName()); @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private JWTHelper jWTHelper; @EJB @@ -106,12 +108,11 @@ public class GitResource { @EJB private GitCommandConfigurationValidator commandConfigurationValidator; - private Project project; - public GitResource(){} - public void setProjectId(Integer projectId) { - this.project = this.projectFacade.find(projectId); + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get all the repositories in a project", response = GitRepositoryDTO.class) @@ -122,8 +123,9 @@ public void setProjectId(Integer projectId) { @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response gitRepositories(@Context UriInfo uriInfo, @Context SecurityContext sc, + @Context HttpServletRequest req, @BeanParam Pagination pagination, - @BeanParam RepositoryBeanParam repositoryBeanParam) { + @BeanParam RepositoryBeanParam repositoryBeanParam) throws ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.REPOSITORY); resourceRequest.setExpansions(repositoryBeanParam.getExpansions().getResources()); @@ -131,7 +133,7 @@ public Response gitRepositories(@Context UriInfo uriInfo, resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setFilter(repositoryBeanParam.getFilter()); resourceRequest.setSort(repositoryBeanParam.getSortBySet()); - GitRepositoryDTO repositories = gitRepositoryBuilder.build(uriInfo, resourceRequest, project, hopsworksUser); + GitRepositoryDTO repositories = gitRepositoryBuilder.build(uriInfo, resourceRequest, getProject(), hopsworksUser); return Response.ok().entity(repositories).build(); } @@ -144,11 +146,14 @@ public Response gitRepositories(@Context UriInfo uriInfo, @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response gitRepository(@PathParam("repositoryId") Integer repositoryId, @Context SecurityContext sc, + @Context HttpServletRequest req, @Context UriInfo uriInfo, - @BeanParam RepositoryBeanParam repositoryBeanParam) throws GitOpException { + @BeanParam RepositoryBeanParam repositoryBeanParam) + throws GitOpException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.REPOSITORY); resourceRequest.setExpansions(repositoryBeanParam.getExpansions().getResources()); Users hopsworksUser = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); GitRepository gitRepository = commandConfigurationValidator.verifyRepository(project, hopsworksUser, repositoryId); GitRepositoryDTO dto = gitRepositoryBuilder.build(uriInfo, resourceRequest, project, gitRepository); return Response.ok().entity(dto).build(); @@ -167,9 +172,9 @@ public Response clone(CloneCommandConfiguration commandDTO, @Context HttpServletRequest req, @Context UriInfo uriInfo, @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException, HopsSecurityException, - IllegalArgumentException, DatasetException { + IllegalArgumentException, DatasetException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - GitOpExecution execution = gitController.clone(commandDTO, project, hopsworksUser); + GitOpExecution execution = gitController.clone(commandDTO, getProject(), hopsworksUser); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); GitOpExecutionDTO dto = executionBuilder.build(uriInfo, resourceRequest, execution); @@ -189,11 +194,12 @@ public Response executeRepositoryAction(@PathParam("repositoryId") Integer repos @QueryParam("action") GitRepositoryAction action, RepositoryActionCommandConfiguration configuration, @Context SecurityContext sc, + @Context HttpServletRequest req, @Context UriInfo uriInfo, @BeanParam ExecutionBeanParam executionBeanParam) - throws GitOpException, HopsSecurityException, IllegalArgumentException { + throws GitOpException, HopsSecurityException, IllegalArgumentException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - GitOpExecution execution = gitController.executeRepositoryAction(configuration, project, hopsworksUser, action, + GitOpExecution execution = gitController.executeRepositoryAction(configuration, getProject(), hopsworksUser, action, repository); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); @@ -210,9 +216,12 @@ public Response executeRepositoryAction(@PathParam("repositoryId") Integer repos @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response getRepositoryBranches(@Context UriInfo uriInfo, @Context SecurityContext sc, + @Context HttpServletRequest req, @BeanParam Pagination pagination, - @PathParam("repositoryId") Integer repositoryId) throws GitOpException { + @PathParam("repositoryId") Integer repositoryId) + throws GitOpException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); GitRepository repository = commandConfigurationValidator.verifyRepository(project, hopsworksUser, repositoryId); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.BRANCH); resourceRequest.setOffset(pagination.getOffset()); @@ -237,12 +246,12 @@ public Response branch(@PathParam("repositoryId") Integer repositoryId, @Context SecurityContext sc, @Context UriInfo uriInfo, @BeanParam ExecutionBeanParam executionBeanParam) - throws IllegalArgumentException, GitOpException, HopsSecurityException { + throws IllegalArgumentException, GitOpException, HopsSecurityException, ProjectException { if (action == null) { throw new IllegalArgumentException(RESTCodes.GitOpErrorCode.INVALID_BRANCH_ACTION.getMessage()); } Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - GitOpExecution execution = gitController.executeBranchAction(action, project, hopsworksUser, repositoryId, + GitOpExecution execution = gitController.executeBranchAction(action, getProject(), hopsworksUser, repositoryId, branchName, commit); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); @@ -260,13 +269,15 @@ public Response branch(@PathParam("repositoryId") Integer repositoryId, @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response getBranchCommits(@Context UriInfo uriInfo, @Context SecurityContext sc, + @Context HttpServletRequest req, @BeanParam Pagination pagination, @PathParam("repositoryId") Integer repositoryId, - @PathParam("branchName") String branchName) throws GitOpException { + @PathParam("branchName") String branchName) throws GitOpException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMIT); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); Users hopsworksUser = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); GitRepository repository = commandConfigurationValidator.verifyRepository(project, hopsworksUser, repositoryId); GitCommitDTO commits = gitCommitsBuilder.build(uriInfo, resourceRequest, project, repository, branchName); return Response.ok().entity(commits).build(); @@ -284,9 +295,9 @@ public Response updateBranchCommits(@PathParam("repositoryId") Integer repositor @PathParam("branchName") String branchName, BranchCommits commits, @Context HttpServletRequest req, - @Context SecurityContext sc) throws GitOpException { + @Context SecurityContext sc) throws GitOpException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - gitController.updateBranchCommits(project, hopsworksUser, commits, repositoryId, branchName); + gitController.updateBranchCommits(getProject(), hopsworksUser, commits, repositoryId, branchName); return Response.ok().build(); } @@ -299,15 +310,16 @@ public Response updateBranchCommits(@PathParam("repositoryId") Integer repositor @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response remotes(@PathParam("repositoryId") Integer repositoryId, @QueryParam("action") GitRemotesAction action, @QueryParam("url") String remoteUrl, - @QueryParam("name") String remoteName, @Context SecurityContext sc, + @QueryParam("name") String remoteName, + @Context SecurityContext sc, @Context UriInfo uriInfo, @BeanParam ExecutionBeanParam executionBeanParam) - throws GitOpException, HopsSecurityException, IllegalArgumentException { + throws GitOpException, HopsSecurityException, IllegalArgumentException, ProjectException { if (action == null) { throw new IllegalArgumentException(RESTCodes.GitOpErrorCode.INVALID_REMOTES_ACTION.getMessage()); } Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - GitOpExecution execution = gitController.addOrDeleteRemote(action, project, hopsworksUser, repositoryId, + GitOpExecution execution = gitController.addOrDeleteRemote(action, getProject(), hopsworksUser, repositoryId, remoteName, remoteUrl); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); @@ -323,10 +335,12 @@ public Response remotes(@PathParam("repositoryId") Integer repositoryId, @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response getRepositoryRemotes(@Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc, @PathParam("repositoryId") Integer repositoryId) - throws GitOpException { + throws GitOpException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); GitRepository repository = commandConfigurationValidator.verifyRepository(project, hopsworksUser, repositoryId); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.REMOTE); GitRepositoryRemoteDTO dto = gitRepositoryRemoteBuilder.build(uriInfo, resourceRequest, project, repository); @@ -341,13 +355,16 @@ public Response getRepositoryRemotes(@Context UriInfo uriInfo, @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response getRepositoryRemote(@Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc, @PathParam("repositoryId") Integer repositoryId, - @PathParam("remoteName") String remoteName) throws GitOpException { + @PathParam("remoteName") String remoteName) + throws GitOpException, ProjectException { if (Strings.isNullOrEmpty(remoteName)) { throw new IllegalArgumentException("Remote name is empty"); } Users hopsworksUser = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); GitRepository repository = commandConfigurationValidator.verifyRepository(project, hopsworksUser, repositoryId); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.REMOTE); GitRepositoryRemoteDTO dto = gitRepositoryRemoteBuilder.build(uriInfo, resourceRequest, project, repository, @@ -368,11 +385,11 @@ public Response getRepositoryRemote(@Context UriInfo uriInfo, public Response fileCheckout(@Context UriInfo uriInfo, @PathParam("repositoryId") Integer repositoryId, @Context SecurityContext sc, - GitFileCheckout files, - @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException, - HopsSecurityException { + @Context HttpServletRequest req, + @BeanParam ExecutionBeanParam executionBeanParam, + GitFileCheckout files) throws GitOpException, HopsSecurityException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - GitOpExecution execution = gitController.fileCheckout(project, hopsworksUser, repositoryId, files.getFiles()); + GitOpExecution execution = gitController.fileCheckout(getProject(), hopsworksUser, repositoryId, files.getFiles()); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); GitOpExecutionDTO dto = executionBuilder.build(uriInfo, resourceRequest, execution); @@ -388,15 +405,16 @@ public Response fileCheckout(@Context UriInfo uriInfo, @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response delete(@Context UriInfo uriInfo, @PathParam("repositoryId") Integer repositoryId, @Context SecurityContext sc, - @Context HttpServletRequest req) throws GitOpException { + @Context HttpServletRequest req) throws GitOpException, ProjectException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); - gitController.deleteRepository(project, hopsworksUser, repositoryId); + gitController.deleteRepository(getProject(), hopsworksUser, repositoryId); return Response.noContent().build(); } @Path("/repository/{repositoryId}/execution") - public GitExecutionResource gitExecution(@PathParam("repositoryId") Integer repositoryId) throws GitOpException { - GitRepository repository = commandConfigurationValidator.verifyRepository(project, repositoryId); - return this.gitExecutionResource.setRepository(repository); + public GitExecutionResource gitExecution(@PathParam("repositoryId") Integer repositoryId) { + this.gitExecutionResource.setProjectId(getProjectId()); + this.gitExecutionResource.setRepositoryId(repositoryId); + return this.gitExecutionResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitSubResource.java new file mode 100644 index 0000000000..ec123523a0 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/GitSubResource.java @@ -0,0 +1,30 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.git; + +import io.hops.hopsworks.api.project.ProjectSubResource; + +public abstract class GitSubResource extends ProjectSubResource { + private Integer repositoryId; + + public Integer getRepositoryId() { + return repositoryId; + } + + public void setRepositoryId(Integer repositoryId) { + this.repositoryId = repositoryId; + } +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/execution/GitExecutionResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/execution/GitExecutionResource.java index f048ecceb1..8da52cf519 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/execution/GitExecutionResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/git/execution/GitExecutionResource.java @@ -15,16 +15,20 @@ */ package io.hops.hopsworks.api.git.execution; +import io.hops.hadoop.shaded.javax.servlet.http.HttpServletRequest; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.git.GitSubResource; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.git.GitCommandExecutionStateUpdateDTO; import io.hops.hopsworks.common.git.GitExecutionController; import io.hops.hopsworks.common.git.util.GitCommandConfigurationValidator; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.GitOpException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.git.GitOpExecution; import io.hops.hopsworks.persistence.entity.git.GitRepository; @@ -54,7 +58,7 @@ @Api(value = "Git Execution Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class GitExecutionResource { +public class GitExecutionResource extends GitSubResource { @EJB private GitExecutionController executionController; @EJB @@ -63,14 +67,12 @@ public class GitExecutionResource { private JWTHelper jwtHelper; @EJB private GitCommandConfigurationValidator gitCommandConfigurationValidator; + @EJB + private ProjectController projectController; - private GitRepository gitRepository; - private Project project; - - public GitExecutionResource setRepository(GitRepository gitRepository) { - this.gitRepository = gitRepository; - this.project = gitRepository.getProject(); - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get all executions in a repository", response = GitOpExecutionDTO.class) @@ -80,10 +82,13 @@ public GitExecutionResource setRepository(GitRepository gitRepository) { @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response getRepositoryExecutions(@Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc, @BeanParam ExecutionBeanParam executionBeanParam, - @BeanParam Pagination pagination) throws GitOpException { + @BeanParam Pagination pagination) throws GitOpException, ProjectException { Users hopsworksUser = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); + GitRepository gitRepository = gitCommandConfigurationValidator.verifyRepository(project, getRepositoryId()); gitCommandConfigurationValidator.verifyRepository(project, hopsworksUser, gitRepository.getId()); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); @@ -100,10 +105,15 @@ public Response getRepositoryExecutions(@Context UriInfo uriInfo, @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.GIT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response getExecution(@PathParam("executionId") Integer executionId, @Context SecurityContext sc, + public Response getExecution(@PathParam("executionId") Integer executionId, + @Context HttpServletRequest req, + @Context SecurityContext sc, @Context UriInfo uriInfo, - @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException { + @BeanParam ExecutionBeanParam executionBeanParam) throws GitOpException, + ProjectException { Users hopsworksUser = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); + GitRepository gitRepository = gitCommandConfigurationValidator.verifyRepository(project, getRepositoryId()); gitCommandConfigurationValidator.verifyRepository(project, hopsworksUser, gitRepository.getId()); GitOpExecution executionObj = executionController.getExecutionInRepository(gitRepository, executionId); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); @@ -124,11 +134,12 @@ public Response updateGitCommandExecutionState(@PathParam("executionId") Integer @PathParam("repositoryId") Integer repositoryId, GitCommandExecutionStateUpdateDTO stateUpdateDTO, @Context SecurityContext sc, + @Context HttpServletRequest req, @Context UriInfo uriInfo, @BeanParam ExecutionBeanParam executionBeanParam) - throws GitOpException, IllegalArgumentException { + throws GitOpException, IllegalArgumentException, ProjectException { Users hopsworksUser = jwtHelper.getUserPrincipal(sc); - GitOpExecution newExec = executionController.updateGitExecutionState(project,hopsworksUser, stateUpdateDTO, + GitOpExecution newExec = executionController.updateGitExecutionState(getProject(), hopsworksUser, stateUpdateDTO, repositoryId, executionId); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTION); resourceRequest.setExpansions(executionBeanParam.getExpansions().getResources()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobSubResource.java new file mode 100644 index 0000000000..a81f197f70 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobSubResource.java @@ -0,0 +1,40 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.jobs; + +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.common.jobs.JobController; +import io.hops.hopsworks.exceptions.JobException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.jobs.description.Jobs; + +public abstract class JobSubResource extends ProjectSubResource { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Jobs getJob() throws ProjectException, JobException { + return getJobController().getJob(getProject(), name); + } + + protected abstract JobController getJobController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsResource.java index 0fb488cf68..2c868a43b9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/JobsResource.java @@ -17,24 +17,26 @@ package io.hops.hopsworks.api.jobs; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jobs.alert.JobAlertsResource; import io.hops.hopsworks.api.jobs.executions.ExecutionsResource; import io.hops.hopsworks.api.jobs.scheduler.JobScheduleV2Resource; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dao.jobhistory.YarnApplicationAttemptStateFacade; import io.hops.hopsworks.common.dao.jobs.description.JobFacade; import io.hops.hopsworks.common.dao.jobs.description.YarnAppUrlsDTO; -import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.jobs.JobController; import io.hops.hopsworks.common.jobs.execution.ExecutionController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.HopsUtils; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.JobException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.jobs.configuration.JobConfiguration; import io.hops.hopsworks.persistence.entity.jobs.configuration.JobType; @@ -77,10 +79,10 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class JobsResource { - +public class JobsResource extends ProjectSubResource { + private static final Logger LOGGER = Logger.getLogger(JobsResource.class.getName()); - + @EJB private JobFacade jobFacade; @Inject @@ -92,7 +94,7 @@ public class JobsResource { @EJB private YarnApplicationAttemptStateFacade yarnApplicationAttemptStateFacade; @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private JWTHelper jWTHelper; @EJB @@ -101,12 +103,12 @@ public class JobsResource { private JobAlertsResource jobAlertsResource; @Inject private JobScheduleV2Resource scheduleResourceV2; - private Project project; - public JobsResource setProject(Integer projectId) { - this.project = projectFacade.find(projectId); - return this; + + @Override + protected ProjectController getProjectController() { + return projectController; } - + @ApiOperation(value = "Get a list of all jobs for this project", response = JobDTO.class) @GET @Produces(MediaType.APPLICATION_JSON) @@ -118,18 +120,18 @@ public Response getAll( @BeanParam Pagination pagination, @BeanParam JobsBeanParam jobsBeanParam, @Context HttpServletRequest req, - @Context UriInfo uriInfo, @Context SecurityContext sc) { + @Context UriInfo uriInfo, @Context SecurityContext sc) throws ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.JOBS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(jobsBeanParam.getSortBySet()); resourceRequest.setFilter(jobsBeanParam.getFilter()); resourceRequest.setExpansions(jobsBeanParam.getExpansions().getResources()); - - JobDTO dto = jobsBuilder.build(uriInfo, resourceRequest, project); + + JobDTO dto = jobsBuilder.build(uriInfo, resourceRequest, getProject()); return Response.ok().entity(dto).build(); } - + @ApiOperation(value = "Get the job with requested ID", response = JobDTO.class) @GET @Path("{name}") @@ -142,8 +144,8 @@ public Response getJob(@PathParam("name") String name, @BeanParam JobsBeanParam jobsBeanParam, @Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws JobException { - Jobs job = jobController.getJob(project, name); + @Context SecurityContext sc) throws JobException, ProjectException { + Jobs job = jobController.getJob(getProject(), name); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.JOBS); resourceRequest.setExpansions(jobsBeanParam.getExpansions().getResources()); JobDTO dto = jobsBuilder.build(uriInfo, resourceRequest, job); @@ -158,11 +160,12 @@ public Response getJob(@PathParam("name") String name, @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response put ( - @ApiParam(value = "Job configuration parameters", required = true) JobConfiguration config, - @ApiParam(value = "name", required = true) @PathParam("name") String name, - @Context SecurityContext sc, - @Context UriInfo uriInfo) throws JobException { + public Response put( + @ApiParam(value = "Job configuration parameters", required = true) JobConfiguration config, + @ApiParam(value = "name", required = true) @PathParam("name") String name, + @Context SecurityContext sc, + @Context HttpServletRequest req, + @Context UriInfo uriInfo) throws JobException, ProjectException { if (config == null) { throw new IllegalArgumentException("Job configuration was not provided."); } @@ -174,6 +177,7 @@ public Response put ( if (!HopsUtils.jobNameValidator(config.getAppName(), Settings.FILENAME_DISALLOWED_CHARS)) { throw new JobException(RESTCodes.JobErrorCode.JOB_NAME_INVALID, Level.FINE, "job name: " + config.getAppName()); } + Project project = getProject(); //Check if job with same name exists so we update it instead of creating it Response.Status status = Response.Status.CREATED; Jobs job = jobFacade.findByProjectAndName(project, config.getAppName()); @@ -198,11 +202,12 @@ public Response put ( @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response delete( - @ApiParam(value = "id", required = true) @PathParam("name") String name, - @Context SecurityContext sc, - @Context UriInfo uriInfo) throws JobException { + @ApiParam(value = "id", required = true) @PathParam("name") String name, + @Context SecurityContext sc, + @Context HttpServletRequest req, + @Context UriInfo uriInfo) throws JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - Jobs job = jobController.getJob(project, name); + Jobs job = jobController.getJob(getProject(), name); switch (job.getJobType()) { case SPARK: @@ -222,18 +227,20 @@ public Response delete( @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) - @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) - @ApiKeyRequired( acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) + @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) + @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response updateSchedule(ScheduleDTO schedule, - @PathParam("name") String name, - @Context SecurityContext sc, - @Context UriInfo uriInfo) throws JobException { - - if(schedule == null){ + @PathParam("name") String name, + @Context SecurityContext sc, + @Context HttpServletRequest req, + @Context UriInfo uriInfo) throws JobException, ProjectException { + + if (schedule == null) { throw new IllegalArgumentException("Schedule parameter was not provided."); } + Project project = getProject(); Jobs job = jobController.getJob(project, name); - + Users user = jWTHelper.getUserPrincipal(sc); jobController.updateSchedule(project, job, schedule, user); return Response.noContent().build(); @@ -253,26 +260,27 @@ public Response updateSchedule(ScheduleDTO schedule, @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) - @ApiKeyRequired( acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response unscheduleJob(@PathParam("name") String name, @Context SecurityContext sc) throws JobException { - if(Strings.isNullOrEmpty(name)) { + @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) + public Response unscheduleJob(@PathParam("name") String name, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws JobException, ProjectException { + if (Strings.isNullOrEmpty(name)) { throw new IllegalArgumentException("job name was not provided or it was not set."); } - Jobs job = jobFacade.findByProjectAndName(project, name); - if(job == null){ + Jobs job = jobFacade.findByProjectAndName(getProject(), name); + if (job == null) { throw new JobException(RESTCodes.JobErrorCode.JOB_NOT_FOUND, Level.FINEST); } + jobController.unscheduleJob(job); return Response.noContent().build(); } @Path("{name}/schedule/v2") - public JobScheduleV2Resource scheduleV2(@PathParam("name") String name) throws JobException { - Jobs job = jobFacade.findByProjectAndName(project, name); - if (job == null) { - throw new JobException(RESTCodes.JobErrorCode.JOB_NOT_FOUND, Level.FINEST, "Job name: " + name); - } - return this.scheduleResourceV2.setJob(job); + public JobScheduleV2Resource scheduleV2(@PathParam("name") String name) { + this.scheduleResourceV2.setProjectId(getProjectId()); + this.scheduleResourceV2.setName(name); + return this.scheduleResourceV2; } @ApiOperation(value = "Inspect user program and return a JobConfiguration", @@ -285,12 +293,13 @@ public JobScheduleV2Resource scheduleV2(@PathParam("name") String name) throws J @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response inspect ( - @ApiParam (value = "job type", example = "spark") @PathParam("jobtype") JobType jobtype, - @ApiParam(value = "path", example = "/Projects/demo_spark_admin000/Resources/spark-examples.jar", - required = true) @QueryParam("path") String path, - @Context SecurityContext sc) throws JobException { + @ApiParam(value = "job type", example = "spark") @PathParam("jobtype") JobType jobtype, + @ApiParam(value = "path", example = "/Projects/demo_spark_admin000/Resources/spark-examples.jar", + required = true) @QueryParam("path") String path, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - JobConfiguration config = jobController.inspectProgram(path, project, user, jobtype); + JobConfiguration config = jobController.inspectProgram(path, getProject(), user, jobtype); return Response.ok().entity(config).build(); } @@ -304,31 +313,26 @@ public Response inspect ( @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) - public Response configuration (@ApiParam (value = "job type", example = "spark") - @PathParam("jobtype") JobType jobtype, - @Context SecurityContext sc) throws JobException { - JobConfiguration config = jobController.getConfiguration(project, jobtype, true); + public Response configuration(@ApiParam(value = "job type", example = "spark") + @PathParam("jobtype") JobType jobtype, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws JobException, ProjectException { + JobConfiguration config = jobController.getConfiguration(getProject(), jobtype, true); return Response.ok().entity(config).build(); } - + @Path("{name}/executions") public ExecutionsResource executions(@PathParam("name") String name) throws JobException { - Jobs job = jobFacade.findByProjectAndName(project, name); - if (job == null) { - throw new JobException(RESTCodes.JobErrorCode.JOB_NOT_FOUND, Level.FINEST, "job name:" + name); - } else { - return this.executions.setJob(job); - } + this.executions.setName(name); + this.executions.setProjectId(getProjectId()); + return this.executions; } - + @Path("{name}/alerts") public JobAlertsResource alerts(@PathParam("name") String name) throws JobException { - Jobs job = jobFacade.findByProjectAndName(project, name); - if (job == null) { - throw new JobException(RESTCodes.JobErrorCode.JOB_NOT_FOUND, Level.FINEST, "job name:" + name); - } else { - return this.jobAlertsResource.setJob(job); - } + this.jobAlertsResource.setName(name); + this.jobAlertsResource.setProjectId(getProjectId()); + return this.jobAlertsResource; } public enum Action { @@ -352,11 +356,13 @@ public enum Action { @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) - public Response getJobUI(@PathParam("appId") String appId, @PathParam("isLivy") String isLivy, - @Context SecurityContext sc) throws JobException { - executionController.checkAccessRight(appId, project); + public Response getJobUI(@PathParam("appId") String appId, + @PathParam("isLivy") String isLivy, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws JobException, ProjectException { + executionController.checkAccessRight(appId, getProject()); List urls = new ArrayList<>(); - + try { String trackingUrl = yarnApplicationAttemptStateFacade.findTrackingUrlByAppId(appId); if (trackingUrl != null && !trackingUrl.isEmpty()) { @@ -385,8 +391,10 @@ public Response getJobUI(@PathParam("appId") String appId, @PathParam("isLivy") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) public Response getTensorBoardUrls(@PathParam("appId") String appId, - @Context SecurityContext sc) - throws JobException { + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws JobException, ProjectException { + Project project = getProject(); executionController.checkAccessRight(appId, project); List urls = new ArrayList<>(); Users user = jWTHelper.getUserPrincipal(sc); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/alert/JobAlertsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/alert/JobAlertsResource.java index a397a3e998..382c75ff78 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/alert/JobAlertsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/alert/JobAlertsResource.java @@ -29,16 +29,20 @@ import io.hops.hopsworks.alerting.exceptions.AlertManagerResponseException; import io.hops.hopsworks.api.alert.AlertBuilder; import io.hops.hopsworks.api.alert.AlertDTO; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.jobs.JobSubResource; import io.hops.hopsworks.api.project.alert.ProjectAlertsDTO; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.alert.AlertController; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dao.jobs.description.JobAlertsFacade; +import io.hops.hopsworks.common.jobs.JobController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.AlertException; import io.hops.hopsworks.exceptions.JobException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.alertmanager.AlertReceiver; import io.hops.hopsworks.persistence.entity.jobs.description.JobAlert; @@ -81,7 +85,7 @@ @Api(value = "JobAlerts Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class JobAlertsResource { +public class JobAlertsResource extends JobSubResource { private static final Logger LOGGER = Logger.getLogger(JobAlertsResource.class.getName()); @@ -95,12 +99,19 @@ public class JobAlertsResource { private AlertBuilder alertBuilder; @EJB private AlertReceiverFacade alertReceiverFacade; + @EJB + private JobController jobController; + @EJB + private ProjectController projectController; - private Jobs job; + @Override + protected ProjectController getProjectController() { + return projectController; + } - public JobAlertsResource setJob(Jobs job) { - this.job = job; - return this; + @Override + protected JobController getJobController() { + return jobController; } @GET @@ -111,13 +122,13 @@ public JobAlertsResource setJob(Jobs job) { @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response get(@BeanParam Pagination pagination, @BeanParam JobAlertsBeanParam jobalertsBeanParam, @Context HttpServletRequest req, - @Context UriInfo uriInfo, @Context SecurityContext sc) { + @Context UriInfo uriInfo, @Context SecurityContext sc) throws ProjectException, JobException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(jobalertsBeanParam.getSortBySet()); resourceRequest.setFilter(jobalertsBeanParam.getFilter()); - JobAlertsDTO dto = jobalertsBuilder.buildItems(uriInfo, resourceRequest, job); + JobAlertsDTO dto = jobalertsBuilder.buildItems(uriInfo, resourceRequest, getJob()); return Response.ok().entity(dto).build(); } @@ -131,9 +142,9 @@ public Response get(@BeanParam Pagination pagination, @BeanParam JobAlertsBeanPa public Response getById(@PathParam("id") Integer id, @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) - throws JobException { + throws JobException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); - JobAlertsDTO dto = jobalertsBuilder.build(uriInfo, resourceRequest, job, id); + JobAlertsDTO dto = jobalertsBuilder.build(uriInfo, resourceRequest, getJob(), id); return Response.ok().entity(dto).build(); } @@ -161,7 +172,8 @@ public Response getAvailableServices(@Context UriInfo uriInfo, public Response createOrUpdate(@PathParam("id") Integer id, JobAlertsDTO jobAlertsDTO, @Context HttpServletRequest req, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws JobException { + @Context SecurityContext sc) throws JobException, ProjectException { + Jobs job = getJob(); JobAlert jobAlert = jobalertsFacade.findByJobAndId(job, id); if (jobAlert == null) { throw new JobException(RESTCodes.JobErrorCode.JOB_ALERT_NOT_FOUND, Level.FINE, @@ -204,14 +216,14 @@ public Response create(PostableJobAlerts jobAlertsDTO, @QueryParam("bulk") @DefaultValue("false") Boolean bulk, @Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws JobException { + @Context SecurityContext sc) throws JobException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ALERTS); JobAlertsDTO dto = createAlert(jobAlertsDTO, bulk, uriInfo, resourceRequest); return Response.created(dto.getHref()).entity(dto).build(); } private JobAlertsDTO createAlert(PostableJobAlerts jobAlertsDTO, Boolean bulk, UriInfo uriInfo, - ResourceRequest resourceRequest) throws JobException { + ResourceRequest resourceRequest) throws JobException, ProjectException { JobAlertsDTO dto; if (bulk) { validateBulk(jobAlertsDTO); @@ -239,9 +251,10 @@ private void validateBulk(PostableJobAlerts jobAlertsDTO) throws JobException { } } - private JobAlertsDTO createAlert(PostableJobAlerts jobAlertsDTO, UriInfo uriInfo, ResourceRequest resourceRequest) - throws JobException { - validate(jobAlertsDTO); + private JobAlertsDTO createAlert(PostableJobAlerts jobAlertsDTO, UriInfo uriInfo, + ResourceRequest resourceRequest) throws JobException, ProjectException { + Jobs job = getJob(); + validate(jobAlertsDTO, job); JobAlert jobAlert = new JobAlert(); jobAlert.setStatus(jobAlertsDTO.getStatus()); jobAlert.setSeverity(jobAlertsDTO.getSeverity()); @@ -255,7 +268,7 @@ private JobAlertsDTO createAlert(PostableJobAlerts jobAlertsDTO, UriInfo uriInfo return jobalertsBuilder.buildItems(uriInfo, resourceRequest, jobAlert); } - private void validate(PostableJobAlerts jobAlertsDTO) throws JobException { + private void validate(PostableJobAlerts jobAlertsDTO, Jobs job) throws JobException { if (jobAlertsDTO == null) { throw new JobException(RESTCodes.JobErrorCode.JOB_ALERT_ILLEGAL_ARGUMENT, Level.FINE, "No payload."); } @@ -284,7 +297,8 @@ public Response getTestById(@PathParam("id") Integer id, @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) - throws AlertException { + throws AlertException, ProjectException, JobException { + Jobs job = getJob(); JobAlert jobAlert = jobalertsFacade.findByJobAndId(job, id); List alerts; try { @@ -310,8 +324,8 @@ public Response getTestById(@PathParam("id") Integer id, public Response deleteById(@PathParam("id") Integer id, @Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws JobException { - JobAlert jobAlert = jobalertsFacade.findByJobAndId(job, id); + @Context SecurityContext sc) throws JobException, ProjectException { + JobAlert jobAlert = jobalertsFacade.findByJobAndId(getJob(), id); if (jobAlert != null) { deleteRoute(jobAlert); jobalertsFacade.remove(jobAlert); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsResource.java index 9adf5f9f3f..437b685323 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/executions/ExecutionsResource.java @@ -17,14 +17,17 @@ package io.hops.hopsworks.api.jobs.executions; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.jobs.JobSubResource; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.jobs.JobController; import io.hops.hopsworks.common.jobs.JobLogDTO; import io.hops.hopsworks.common.jobs.execution.ExecutionController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.JobException; @@ -62,7 +65,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ExecutionsResource { +public class ExecutionsResource extends JobSubResource { @Inject private ExecutionController executionController; @@ -70,16 +73,21 @@ public class ExecutionsResource { private ExecutionsBuilder executionsBuilder; @EJB private Settings settings; - - @EJB private JWTHelper jWTHelper; + @EJB + private JobController jobController; + @EJB + private ProjectController projectController; - private Jobs job; + @Override + protected ProjectController getProjectController() { + return projectController; + } - public ExecutionsResource setJob(Jobs job) { - this.job = job; - return this; + @Override + protected JobController getJobController() { + return jobController; } @ApiOperation(value = "Get a list of executions for the job.", response = ExecutionDTO.class) @@ -93,8 +101,10 @@ public ExecutionsResource setJob(Jobs job) { public Response getExecutions( @BeanParam Pagination pagination, @BeanParam ExecutionsBeanParam executionsBeanParam, - @Context UriInfo uriInfo, @Context SecurityContext sc) { - + @Context UriInfo uriInfo, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProjectException, JobException { + ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTIONS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); @@ -102,7 +112,7 @@ public Response getExecutions( resourceRequest.setFilter(executionsBeanParam.getFilter()); resourceRequest.setExpansions(executionsBeanParam.getExpansions().getResources()); - ExecutionDTO dto = executionsBuilder.build(uriInfo, resourceRequest, job); + ExecutionDTO dto = executionsBuilder.build(uriInfo, resourceRequest, getJob()); return Response.ok().entity(dto).build(); } @@ -119,9 +129,9 @@ public Response getExecution(@ApiParam(value = "execution id", required = true) @BeanParam ExecutionsBeanParam executionsBeanParam, @Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws JobException { + @Context SecurityContext sc) throws JobException, ProjectException { //If requested execution does not belong to job - Execution execution = executionController.authorize(job, id); + Execution execution = executionController.authorize(getJob(), id); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.EXECUTIONS); resourceRequest.setExpansions(executionsBeanParam.getExpansions().getResources()); ExecutionDTO dto = executionsBuilder.build(uriInfo, resourceRequest, execution); @@ -163,9 +173,9 @@ public Response startExecution( @ApiParam(value = "Arguments for executing the job") String args, @Context SecurityContext sc, @Context UriInfo uriInfo) throws JobException, GenericException, ServiceException, ProjectException { - - Users user = jWTHelper.getUserPrincipal(sc); + Users user = jWTHelper.getUserPrincipal(sc); + Jobs job = getJob(); // run job as job owner if user is airflow if (user.getUsername().equals(settings.getAirflowUser())) { user = job.getCreator(); @@ -192,10 +202,12 @@ public Response startExecution( @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired( acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response delete(@ApiParam(value = "execution id", required = true) @PathParam("id") Integer id, - @Context UriInfo uriInfo, @Context SecurityContext sc) throws JobException { + @Context HttpServletRequest req, + @Context UriInfo uriInfo, + @Context SecurityContext sc) throws JobException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); //If requested execution does not belong to job - Execution execution = executionController.authorize(job, id); + Execution execution = executionController.authorize(getJob(), id); executionController.delete(execution, user); return Response.noContent().build(); } @@ -211,8 +223,8 @@ public Response getLog( @PathParam("id") Integer id, @PathParam("type") JobLogDTO.LogType type, @Context HttpServletRequest req, - @Context SecurityContext sc) throws JobException { - Execution execution = executionController.authorize(job, id); + @Context SecurityContext sc) throws JobException, ProjectException { + Execution execution = executionController.authorize(getJob(), id); JobLogDTO dto = executionController.getLog(execution, type); return Response.ok().entity(dto).build(); } @@ -229,8 +241,8 @@ public Response retryLog( @PathParam("id") Integer id, @PathParam("type") JobLogDTO.LogType type, @Context HttpServletRequest req, - @Context SecurityContext sc) throws JobException { - Execution execution = executionController.authorize(job, id); + @Context SecurityContext sc) throws JobException, ProjectException { + Execution execution = executionController.authorize(getJob(), id); JobLogDTO dto = executionController.retryLogAggregation(execution, type); return Response.ok().entity(dto).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/scheduler/JobScheduleV2Resource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/scheduler/JobScheduleV2Resource.java index 962dd6bc1d..2e55d02ec9 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/scheduler/JobScheduleV2Resource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jobs/scheduler/JobScheduleV2Resource.java @@ -16,14 +16,17 @@ package io.hops.hopsworks.api.jobs.scheduler; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.jobs.JobSubResource; +import io.hops.hopsworks.common.jobs.JobController; import io.hops.hopsworks.common.jobs.scheduler.JobScheduleV2Controller; import io.hops.hopsworks.common.jobs.scheduler.JobScheduleV2DTO; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.JobException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.jobs.description.Jobs; import io.hops.hopsworks.persistence.entity.jobs.scheduler.JobScheduleV2; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; @@ -49,13 +52,26 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Job Schedule resource") -public class JobScheduleV2Resource { +public class JobScheduleV2Resource extends JobSubResource { - private Jobs job; @EJB private JobScheduleV2Controller jobScheduleController; @EJB private JobScheduleV2Builder jobScheduleBuilder; + @EJB + private JobController jobController; + @EJB + private ProjectController projectController; + + @Override + protected ProjectController getProjectController() { + return projectController; + } + + @Override + protected JobController getJobController() { + return jobController; + } @ApiOperation(value = "Get job's schedule.", response = JobScheduleV2DTO.class) @GET @@ -71,8 +87,8 @@ public Response getSchedule( @Context HttpServletRequest req, @Context - UriInfo uriInfo) throws JobException { - JobScheduleV2 jobSchedule = jobScheduleController.getScheduleByJobId(job.getId()); + UriInfo uriInfo) throws JobException, ProjectException { + JobScheduleV2 jobSchedule = jobScheduleController.getScheduleByJobId(getJob().getId()); JobScheduleV2DTO scheduleDTO = jobScheduleBuilder.build(uriInfo, jobSchedule); return Response.ok().entity(scheduleDTO).build(); } @@ -92,9 +108,9 @@ public Response createSchedule(JobScheduleV2DTO scheduleDTO, @Context HttpServletRequest req, @Context - UriInfo uriInfo) { + UriInfo uriInfo) throws ProjectException, JobException { JobScheduleV2 jobSchedule = jobScheduleController.createSchedule( - jobScheduleBuilder.validateAndConvertOnCreate(job, scheduleDTO) + jobScheduleBuilder.validateAndConvertOnCreate(getJob(), scheduleDTO) ); scheduleDTO = jobScheduleBuilder.build(uriInfo, jobSchedule); return Response.created(scheduleDTO.getHref()).entity(scheduleDTO).build(); @@ -110,11 +126,13 @@ public Response createSchedule(JobScheduleV2DTO scheduleDTO, @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response updateSchedule(JobScheduleV2DTO scheduleDTO, - @Context SecurityContext sc, - @Context HttpServletRequest req, - @Context UriInfo uriInfo) - throws JobException { - JobScheduleV2 schedule = jobScheduleBuilder.validateAndConvertOnUpdate(job, scheduleDTO); + @Context + SecurityContext sc, + @Context + HttpServletRequest req, + @Context + UriInfo uriInfo) throws JobException, ProjectException { + JobScheduleV2 schedule = jobScheduleBuilder.validateAndConvertOnUpdate(getJob(), scheduleDTO); schedule = jobScheduleController.updateSchedule(schedule); scheduleDTO = jobScheduleBuilder.build(uriInfo, schedule); return Response.ok().entity(scheduleDTO).build(); @@ -133,14 +151,9 @@ public Response updateSchedule(JobScheduleV2DTO scheduleDTO, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteSchedule( @Context HttpServletRequest req, - @Context SecurityContext sc) { - jobScheduleController.deleteSchedule(job.getId()); + @Context SecurityContext sc) throws ProjectException, JobException { + jobScheduleController.deleteSchedule(getJob().getId()); return Response.noContent().build(); } - public JobScheduleV2Resource setJob(Jobs job) { - this.job = job; - return this; - } - } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java index 084c507a8c..a9cd6762b2 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java @@ -43,11 +43,11 @@ import io.hops.hopsworks.api.filter.NoCacheResponse; import io.hops.hopsworks.api.jobs.executions.MonitoringUrlBuilder; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.common.dao.jobs.quota.YarnProjectsQuotaFacade; import io.hops.hopsworks.common.dao.jupyter.JupyterSettingsFacade; import io.hops.hopsworks.common.dao.jupyter.config.JupyterDTO; import io.hops.hopsworks.common.dao.jupyter.config.JupyterFacade; -import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.hdfs.HdfsUsersController; import io.hops.hopsworks.common.hdfs.Utils; import io.hops.hopsworks.common.jobs.spark.SparkController; @@ -59,6 +59,7 @@ import io.hops.hopsworks.common.jupyter.RemoteFSDriverType; import io.hops.hopsworks.common.livy.LivyController; import io.hops.hopsworks.common.livy.LivyMsg; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.system.job.SystemJobStatus; import io.hops.hopsworks.common.user.UsersController; import io.hops.hopsworks.common.util.Settings; @@ -119,13 +120,13 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class JupyterService { +public class JupyterService extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(JupyterService.class.getName()); private static final String[] JUPYTER_JWT_AUD = new String[]{Audience.API}; @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private NoCacheResponse noCacheResponse; @Inject @@ -157,22 +158,10 @@ public class JupyterService { @EJB private NotebookBuilder notebookBuilder; - private Integer projectId; - // No @EJB annotation for Project, it's injected explicitly in ProjectService. - private Project project; - - public JupyterService() { - } - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - this.project = this.projectFacade.find(projectId); - } - - public Integer getProjectId() { - return projectId; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** * Launches a Jupyter notebook server for this project-specific user * @@ -183,9 +172,10 @@ public Integer getProjectId() { @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response getAllNotebookServersInProject(@Context SecurityContext sc) throws ServiceException { + public Response getAllNotebookServersInProject(@Context HttpServletRequest req, @Context SecurityContext sc) + throws ServiceException, ProjectException { - Collection servers = project.getJupyterProjectCollection(); + Collection servers = getProject().getJupyterProjectCollection(); if (servers == null) { throw new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_FOUND, Level.FINE); @@ -203,9 +193,11 @@ public Response getAllNotebookServersInProject(@Context SecurityContext sc) thro @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response livySessions(@Context SecurityContext sc) { + public Response livySessions(@Context SecurityContext sc, + @Context HttpServletRequest req) throws ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - List sessions = livyController.getLivySessionsForProjectUser(this.project, user); + Project project = getProject(); + List sessions = livyController.getLivySessionsForProjectUser(project, user); List livySessionDTOS = new ArrayList<>(); for (LivyMsg.Session session : sessions) { LivySessionDTO dto = new LivySessionDTO(session); @@ -222,9 +214,11 @@ public Response livySessions(@Context SecurityContext sc) { @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response stopLivySession(@PathParam("appId") String appId, @Context SecurityContext sc) { + public Response stopLivySession(@PathParam("appId") String appId, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - jupyterController.stopSession(project, user, appId); + jupyterController.stopSession(getProject(), user, appId); return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).build(); } @@ -240,9 +234,10 @@ public Response stopLivySession(@PathParam("appId") String appId, @Context Secur @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response settings(@Context SecurityContext sc, - @Context HttpServletRequest req) { + @Context HttpServletRequest req) throws ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); JupyterSettings js = jupyterSettingsFacade.findByProjectUser(project, user.getEmail()); RemoteFSDriverType driver = settings.getJupyterRemoteFsManager(); // if we are using hopsfsmount the base dir is the project path @@ -265,9 +260,10 @@ public Response settings(@Context SecurityContext sc, @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) - public Response isRunning(@Context HttpServletRequest req, @Context SecurityContext sc) throws ServiceException { + public Response isRunning(@Context HttpServletRequest req, @Context SecurityContext sc) + throws ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - JupyterProject jp = jupyterFacade.findByProjectUser(project, user) + JupyterProject jp = jupyterFacade.findByProjectUser(getProject(), user) .orElseThrow(() -> new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_FOUND, Level.FINE)); // Check to make sure the jupyter notebook server is running @@ -293,10 +289,11 @@ public Response isRunning(@Context HttpServletRequest req, @Context SecurityCont @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response startNotebookServer(JupyterSettings jupyterSettings, @Context HttpServletRequest req, - @Context SecurityContext sc, @Context UriInfo uriInfo) throws ProjectException, - HopsSecurityException, ServiceException, GenericException, JobException { + @Context SecurityContext sc, @Context UriInfo uriInfo) + throws ProjectException, HopsSecurityException, ServiceException, GenericException, JobException { Users hopsworksUser = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); //The JupyterSettings bean is serialized without the Project and User when attaching it to the notebook as xattr. // .We need to put the user object when we are launching Jupyter from the notebook. The Project object is set // from in the front-end @@ -383,8 +380,9 @@ public Response startNotebookServer(JupyterSettings jupyterSettings, @Context Ht @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response stopNotebookServer(@Context HttpServletRequest req, - @Context SecurityContext sc) throws ServiceException { + @Context SecurityContext sc) throws ServiceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); JupyterProject jp = jupyterFacade.findByProjectUser(project, user) .orElseThrow(() -> new ServiceException(RESTCodes.ServiceErrorCode.JUPYTER_SERVERS_NOT_FOUND, Level.FINE)); @@ -398,16 +396,17 @@ public Response stopNotebookServer(@Context HttpServletRequest req, @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response convertIPythonNotebook(@PathParam("path") String path, @Context HttpServletRequest req, - @Context SecurityContext sc) throws ServiceException { + @Context SecurityContext sc) throws ServiceException, ProjectException { if (path.startsWith("/")) { path = path.replaceFirst("/", ""); } - String ipynbPath = Utils.getProjectPath(this.project.getName()) + path; + Project project = getProject(); + String ipynbPath = Utils.getProjectPath(project.getName()) + "/" + path; int extensionIndex = ipynbPath.lastIndexOf(".ipynb"); StringBuilder pathBuilder = new StringBuilder(ipynbPath.substring(0, extensionIndex)).append(".py"); String pyAppPath = pathBuilder.toString(); Users user = jWTHelper.getUserPrincipal(sc); - NotebookConversion conversionType = jupyterController.getNotebookConversionType(ipynbPath, user, this.project); + NotebookConversion conversionType = jupyterController.getNotebookConversionType(ipynbPath, user, project); SystemJobStatus status = jupyterController.convertIPythonNotebook(project, user, ipynbPath, pyAppPath, conversionType); return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(status).build(); @@ -421,14 +420,14 @@ public Response convertIPythonNotebook(@PathParam("path") String path, @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response updateNotebookServer(JupyterSettings jupyterSettings, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException { Users user = jWTHelper.getUserPrincipal(sc); jupyterSettingsFacade.update(jupyterSettings); - jupyterController.updateExpirationDate(project, user, jupyterSettings); + jupyterController.updateExpirationDate(getProject(), user, jupyterSettings); return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(jupyterSettings).build(); } - + @GET @Path("recent") @Produces(MediaType.APPLICATION_JSON) @@ -436,11 +435,12 @@ public Response updateNotebookServer(JupyterSettings jupyterSettings, @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response recent(@QueryParam("count") @DefaultValue("5") @Min(0) @Max(100) int count, @Context UriInfo uriInfo, @Context SecurityContext sc, @Context HttpServletRequest req) - throws OpenSearchException { - NotebookDTO dto = notebookBuilder.build(uriInfo, project, count); + throws OpenSearchException, ProjectException { + NotebookDTO dto = notebookBuilder.build(uriInfo, getProject(), count); return Response.ok().entity(dto).build(); } + @ApiOperation(value = "Attach a jupyter configuration to the notebook.", notes = "The notebook is passed as the " + "kernelId from jupyter.") @PUT @@ -452,14 +452,15 @@ public Response recent(@QueryParam("count") @DefaultValue("5") @Min(0) @Max(100) public Response attachJupyterConfigurationToNotebook(@PathParam("hdfsUsername") String hdfsUsername, @PathParam("kernelId") String kernelId, @Context HttpServletRequest req, - @Context SecurityContext sc) throws ServiceException { + @Context SecurityContext sc) + throws ServiceException, ProjectException { Optional user = usersController.findByUsername(hdfsUsersController.getUserName(hdfsUsername)); if (!user.isPresent()) { throw new ServiceException(RESTCodes.ServiceErrorCode .WRONG_HDFS_USERNAME_PROVIDED_FOR_ATTACHING_JUPYTER_CONFIGURATION_TO_NOTEBOOK, Level.FINE, "HDFS username provided does not exist."); } - jupyterController.attachJupyterConfigurationToNotebook(project, user.get(), kernelId); + jupyterController.attachJupyterConfigurationToNotebook(getProject(), user.get(), kernelId); return Response.ok().build(); } } \ No newline at end of file diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/kafka/KafkaResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/kafka/KafkaResource.java index 222fdd0b59..6318ed6304 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/kafka/KafkaResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/kafka/KafkaResource.java @@ -39,11 +39,12 @@ package io.hops.hopsworks.api.kafka; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.kafka.topics.TopicsBeanParam; import io.hops.hopsworks.api.kafka.topics.TopicsBuilder; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dao.kafka.KafkaClusterInfoDTO; @@ -54,14 +55,17 @@ import io.hops.hopsworks.common.dao.kafka.schemas.CompatibilityLevel; import io.hops.hopsworks.common.dao.kafka.schemas.SchemaRegistryError; import io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; import io.hops.hopsworks.common.featurestore.storageconnectors.kafka.FeatureStoreKafkaConnectorDTO; import io.hops.hopsworks.common.kafka.KafkaController; import io.hops.hopsworks.common.kafka.SchemasController; import io.hops.hopsworks.common.kafka.SubjectsCompatibilityController; import io.hops.hopsworks.common.kafka.SubjectsController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.KafkaException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.SchemaException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.project.Project; @@ -95,7 +99,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class KafkaResource { +public class KafkaResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(KafkaResource.class.getName()); @@ -113,16 +117,32 @@ public class KafkaResource { private FeaturestoreStorageConnectorController storageConnectorController; @EJB private KafkaClusterInfoBuilder kafkaClusterInfoBuilder; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - private Project project; + private Integer featurestoreId; public KafkaResource() { } - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - + + public void setFeaturestoreId(Integer featurestoreId) { + this.featurestoreId = featurestoreId; + } + + private void verifyAccessToFeaturestore(Project project) throws FeaturestoreException { + //This call verifies that the project have access to the featurestoreId provided + if (featurestoreId != null) { + featurestoreController.getFeaturestoreForProjectWithId(project, featurestoreId); + } + } + @ApiOperation(value = "Retrieve Kafka broker endpoints.") @GET @Path("/clusterinfo") @@ -134,7 +154,9 @@ public void setProject(Project project) { allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getBrokers(@Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws FeaturestoreException { + @Context SecurityContext sc) throws FeaturestoreException, ProjectException { + Project project = getProject(); + verifyAccessToFeaturestore(project); FeatureStoreKafkaConnectorDTO connectorDTO = storageConnectorController.getKafkaConnector(project); KafkaClusterInfoDTO dto = kafkaClusterInfoBuilder.build(uriInfo, project, Arrays.asList(connectorDTO.getBootstrapServers().split(","))); @@ -154,12 +176,14 @@ public Response getTopics(@Context UriInfo uriInfo, @BeanParam Pagination pagination, @BeanParam TopicsBeanParam topicsBeanParam, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.KAFKA); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setSort(topicsBeanParam.getSortBySet()); resourceRequest.setFilter(topicsBeanParam.getFilter()); + Project project = getProject(); + verifyAccessToFeaturestore(project); TopicDTO dto = topicsBuilder.build(uriInfo, resourceRequest, project); return Response.ok().entity(dto).build(); } @@ -176,7 +200,10 @@ public Response getTopics(@Context UriInfo uriInfo, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response createTopic(TopicDTO topicDto, @Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) + throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); kafkaController.createTopic(project, topicDto); URI uri = uriInfo.getAbsolutePathBuilder().path(topicDto.getName()).build(); topicDto.setHref(uri); @@ -194,7 +221,9 @@ public Response createTopic(TopicDTO topicDto, @Context UriInfo uriInfo, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response removeTopic(@PathParam("topic") String topicName, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); kafkaController.removeTopicFromProject(project, topicName); return Response.noContent().build(); } @@ -211,7 +240,9 @@ public Response removeTopic(@PathParam("topic") String topicName, public Response getTopic(@Context UriInfo uriInfo, @PathParam("topic") String topicName, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); PartitionDetailsDTO dto = topicsBuilder.buildTopicDetails(uriInfo, project, topicName); return Response.ok().entity(dto).build(); } @@ -228,7 +259,9 @@ public Response getTopic(@Context UriInfo uriInfo, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "AGENT", "HOPS_SERVICE_USER"}) public Response getSchema(@PathParam("id") Integer id, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { SubjectDTO dto = schemasController.findSchemaById(project, id); return Response.ok().entity(dto).build(); @@ -249,10 +282,13 @@ public Response getSchema(@PathParam("id") Integer id, @ApiKeyRequired(acceptedScopes = {ApiScope.KAFKA}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getSubjects(@Context SecurityContext sc, - @Context HttpServletRequest req) { + @Context HttpServletRequest req) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); List subjects = subjectsController.getSubjects(project); String array = Arrays.toString(subjects.toArray()); - GenericEntity entity = new GenericEntity(array) {}; + GenericEntity entity = new GenericEntity(array) { + }; return Response.ok().entity(entity).build(); } @@ -268,7 +304,9 @@ public Response getSubjects(@Context SecurityContext sc, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response checkIfSubjectRegistered(@PathParam("subject") String subject, SubjectDTO dto, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { SubjectDTO res = subjectsController.checkIfSchemaRegistered(project, subject, dto.getSchema()); return Response.ok().entity(res).build(); @@ -290,11 +328,15 @@ public Response checkIfSubjectRegistered(@PathParam("subject") String subject, S allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteSubject(@PathParam("subject") String subject, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) + throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { List versions = subjectsController.deleteSubject(project, subject); String array = Arrays.toString(versions.toArray()); - GenericEntity entity = new GenericEntity(array) {}; + GenericEntity entity = new GenericEntity(array) { + }; return Response.ok().entity(entity).build(); } catch (SchemaException e) { SchemaRegistryError error = @@ -314,10 +356,14 @@ public Response deleteSubject(@PathParam("subject") String subject, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteSubjectsVersion(@PathParam("subject") String subject, @PathParam("version") String version, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) + throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { Integer deleted = subjectsController.deleteSubjectsVersion(project, subject, version); - GenericEntity entity = new GenericEntity(deleted) {}; + GenericEntity entity = new GenericEntity(deleted) { + }; return Response.ok().entity(entity).build(); } catch (SchemaException e) { SchemaRegistryError error = @@ -338,7 +384,10 @@ public Response deleteSubjectsVersion(@PathParam("subject") String subject, @Pat allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response postNewSchema(@PathParam("subject") String subject, SubjectDTO dto, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) + throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { SubjectDTO res = subjectsController.registerNewSubject(project, subject, dto.getSchema(), false); return Response.ok().entity(res).build(); @@ -361,7 +410,9 @@ public Response postNewSchema(@PathParam("subject") String subject, SubjectDTO d allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getSubjectVersions(@PathParam("subject") String subject, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { List versions = subjectsController.getSubjectVersions(project, subject); String array = Arrays.toString(versions.toArray()); @@ -387,7 +438,9 @@ public Response getSubjectVersions(@PathParam("subject") String subject, public Response getSubjectDetails(@PathParam("subject") String subject, @PathParam("version") String version, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { SubjectDTO dto = subjectsController.getSubjectDetails(project, subject, version); return Response.ok().entity(dto).build(); @@ -411,10 +464,13 @@ public Response getSubjectDetails(@PathParam("subject") String subject, public Response getSchema(@PathParam("subject") String subject, @PathParam("version") String version, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { SubjectDTO dto = subjectsController.getSubjectDetails(project, subject, version); - GenericEntity entity = new GenericEntity(dto.getSchema()) {}; + GenericEntity entity = new GenericEntity(dto.getSchema()) { + }; return Response.ok().entity(entity).build(); } catch (SchemaException e) { SchemaRegistryError error = @@ -423,6 +479,7 @@ public Response getSchema(@PathParam("subject") String subject, } } + @ApiOperation(value = "Check if schema is compatible with a specific subject version.") @POST @Path("/compatibility/subjects/{subject}/versions/{version}") @@ -436,10 +493,12 @@ public Response getSchema(@PathParam("subject") String subject, public Response checkSchemaCompatibility(@PathParam("subject") String subject, @PathParam("version") String version, SubjectDTO dto, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { CompatibilityCheck isCompatible = - subjectsController.checkIfSchemaCompatible(project, subject, version, dto.getSchema()); + subjectsController.checkIfSchemaCompatible(project, subject, version, dto.getSchema()); return Response.ok().entity(isCompatible).build(); } catch (SchemaException e) { SchemaRegistryError error = @@ -458,7 +517,10 @@ public Response checkSchemaCompatibility(@PathParam("subject") String subject, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.KAFKA}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) - public Response getProjectCompatibility(@Context HttpServletRequest req, @Context SecurityContext sc) { + public Response getProjectCompatibility(@Context HttpServletRequest req, @Context SecurityContext sc) + throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { CompatibilityLevel dto = subjectsCompatibilityController.getProjectCompatibilityLevel(project); return Response.ok().entity(dto).build(); @@ -477,7 +539,11 @@ public Response getProjectCompatibility(@Context HttpServletRequest req, @Contex @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER}) @JWTRequired(acceptedTokens = {Audience.API, Audience.JOB}, allowedUserRoles = {"HOPS_ADMIN"}) @ApiKeyRequired(acceptedScopes = {ApiScope.KAFKA}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) - public Response setProjectCompatibility(Compatibility dto, @Context SecurityContext sc) { + public Response setProjectCompatibility(Compatibility dto, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { Compatibility result = subjectsCompatibilityController.setProjectCompatibility(project, dto); return Response.ok().entity(result).build(); @@ -500,7 +566,9 @@ public Response setProjectCompatibility(Compatibility dto, @Context SecurityCont allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getSubjectCompatibility(@PathParam("subject") String subject, @Context HttpServletRequest req, - @Context SecurityContext sc) { + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { CompatibilityLevel dto = subjectsCompatibilityController.getSubjectCompatibility(project, subject); return Response.ok().entity(dto).build(); @@ -520,7 +588,10 @@ public Response getSubjectCompatibility(@PathParam("subject") String subject, @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN"}) @ApiKeyRequired(acceptedScopes = {ApiScope.KAFKA}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response setSubjectCompatibility(@PathParam("subject") String subject, Compatibility dto, - @Context SecurityContext sc){ + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { Compatibility result = subjectsCompatibilityController.setSubjectCompatibility(project, subject, dto); return Response.ok().entity(result).build(); @@ -542,7 +613,10 @@ public Response setSubjectCompatibility(@PathParam("subject") String subject, Co allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getTopicSubject(@PathParam("topic") String topic, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) + throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); SubjectDTO subjectDTO = kafkaController.getSubjectForTopic(project, topic); return Response.ok().entity(subjectDTO).build(); } @@ -559,7 +633,10 @@ public Response getTopicSubject(@PathParam("topic") String topic, public Response updateSubjectVersion(@PathParam("topic") String topic, @PathParam("subject") String subject, @PathParam("version") Integer version, @Context HttpServletRequest req, - @Context SecurityContext sc) throws KafkaException { + @Context SecurityContext sc) + throws KafkaException, ProjectException, FeaturestoreException { + Project project = getProject(); + verifyAccessToFeaturestore(project); try { kafkaController.updateTopicSubjectVersion(project, topic, subject, version); return Response.ok().build(); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/metadata/XAttrsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/metadata/XAttrsResource.java index 9e39475035..5d49c658ae 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/metadata/XAttrsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/metadata/XAttrsResource.java @@ -19,15 +19,17 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.project.ProjectFacade; +import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.DistributedFsService; import io.hops.hopsworks.common.hdfs.HdfsUsersController; -import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.hdfs.xattrs.XAttrsController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.dataset.DatasetType; import io.hops.hopsworks.persistence.entity.project.Project; @@ -40,6 +42,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -62,10 +65,10 @@ @Api(value = "Extended Attributes Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class XAttrsResource { - +public class XAttrsResource extends ProjectSubResource { + @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private JWTHelper jWTHelper; @EJB @@ -78,33 +81,31 @@ public class XAttrsResource { private HdfsUsersController hdfsUsersController; @EJB private DatasetHelper datasetHelper; - - private Project project; - - public void setProject(Integer projectId) { - this.project = projectFacade.find(projectId); + + @Override + protected ProjectController getProjectController() { + return projectController; } - - - @ApiOperation( value = "Create or Update an extended attribute for a path.", response = XAttrDTO.class) + + @ApiOperation(value = "Create or Update an extended attribute for a path.", response = XAttrDTO.class) @PUT @Path("{path: .+}") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) - public Response put( - @Context SecurityContext sc, @Context UriInfo uriInfo, - @PathParam("path") String path, - @QueryParam("pathType") @DefaultValue("DATASET") DatasetType pathType, - @QueryParam("name") String xattrName, - String metaObj) - throws DatasetException, MetadataException { + public Response put(@Context SecurityContext sc, @Context UriInfo uriInfo, + @Context HttpServletRequest req, + @PathParam("path") String path, + @QueryParam("pathType") @DefaultValue("DATASET") DatasetType pathType, + @QueryParam("name") String xattrName, + String metaObj) + throws DatasetException, MetadataException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - + Project project = getProject(); Response.Status status = Response.Status.OK; String inodePath = datasetHelper.getDatasetPathIfFileExist(project, path, pathType).getFullPath().toString(); - if(xattrsController.addXAttr(project, user, inodePath, xattrName, metaObj)){ + if (xattrsController.addXAttr(project, user, inodePath, xattrName, metaObj)) { status = Response.Status.CREATED; } @@ -127,11 +128,12 @@ public Response put( @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) public Response get(@Context SecurityContext sc, @Context UriInfo uriInfo, - @PathParam("path") String path, - @QueryParam("pathType") @DefaultValue("DATASET") DatasetType pathType, - @QueryParam("name") String xattrName) - throws DatasetException, MetadataException { + @PathParam("path") String path, + @QueryParam("pathType") @DefaultValue("DATASET") DatasetType pathType, + @QueryParam("name") String xattrName) + throws DatasetException, MetadataException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); Map result = new HashMap<>(); DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUsersController.getHdfsUserName(project, user)); @@ -162,11 +164,12 @@ public Response get(@Context SecurityContext sc, @Context UriInfo uriInfo, @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER}) @JWTRequired(acceptedTokens={Audience.API}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) public Response delete(@Context SecurityContext sc, - @PathParam("path") String path, - @QueryParam("pathType") @DefaultValue("DATASET") DatasetType pathType, - @QueryParam("name") String xattrName) - throws DatasetException, MetadataException { + @PathParam("path") String path, + @QueryParam("pathType") @DefaultValue("DATASET") DatasetType pathType, + @QueryParam("name") String xattrName) + throws DatasetException, MetadataException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); String inodePath = datasetHelper.getDatasetPathIfFileExist(project, path, pathType).getFullPath().toString(); xattrsController.removeXAttr(project, user, inodePath, xattrName); return Response.status(Response.Status.NO_CONTENT).build(); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryResource.java index 5710b7b90a..991c377466 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryResource.java @@ -16,9 +16,9 @@ package io.hops.hopsworks.api.modelregistry; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; import io.hops.hopsworks.api.jwt.JWTHelper; @@ -26,13 +26,15 @@ import io.hops.hopsworks.api.modelregistry.models.ModelsController; import io.hops.hopsworks.api.modelregistry.models.ModelsResource; import io.hops.hopsworks.api.modelregistry.models.dto.ModelDTO; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.project.ProjectFacade; +import io.hops.hopsworks.common.project.ProjectController; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ModelRegistryException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; @@ -45,6 +47,7 @@ import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -58,10 +61,10 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ModelRegistryResource { +public class ModelRegistryResource extends ProjectSubResource { @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private ModelRegistryBuilder modelRegistryBuilder; @EJB @@ -71,10 +74,9 @@ public class ModelRegistryResource { @EJB private JWTHelper jwtHelper; - private Project project; - public ModelRegistryResource setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get a list of all model registries accessible for this project", @@ -91,14 +93,15 @@ public Response getAll( @BeanParam ModelRegistryBeanParam modelRegistryBeanParam, @BeanParam Pagination pagination, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, ModelRegistryException, FeatureStoreMetadataException, - MetadataException { + MetadataException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELREGISTRIES); resourceRequest.setExpansions(modelRegistryBeanParam.getExpansions().getResources()); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); - ModelRegistryDTO dto = modelRegistryBuilder.build(uriInfo, resourceRequest, user, project); + ModelRegistryDTO dto = modelRegistryBuilder.build(uriInfo, resourceRequest, user, getProject()); return Response.ok().entity(dto).build(); } @@ -118,9 +121,11 @@ public Response get ( @BeanParam ModelRegistryBeanParam modelRegistryBeanParam, @BeanParam Pagination pagination, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, ModelRegistryException, FeatureStoreMetadataException, - MetadataException { + MetadataException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELREGISTRIES); resourceRequest.setExpansions(modelRegistryBeanParam.getExpansions().getResources()); Project modelRegistryProject = @@ -138,11 +143,11 @@ public Response get ( */ @Path("/{modelRegistryId}/models") public ModelsResource modelsResource(@PathParam("modelRegistryId") Integer modelRegistryId, - @Context SecurityContext sc) throws ModelRegistryException { - modelsResource.setProject(project); + @Context SecurityContext sc) { if (modelRegistryId == null) { throw new IllegalArgumentException(RESTCodes.ModelRegistryErrorCode.MODEL_REGISTRY_ID_NOT_PROVIDED.getMessage()); } + modelsResource.setProjectId(getProjectId()); modelsResource.setModelRegistryId(modelRegistryId); return modelsResource; } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryTagResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryTagResource.java index 93b63c16ba..96d256b12c 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryTagResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/ModelRegistryTagResource.java @@ -15,25 +15,29 @@ */ package io.hops.hopsworks.api.modelregistry; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.modelregistry.models.ModelRegistrySubResource; +import io.hops.hopsworks.api.modelregistry.models.ModelsController; import io.hops.hopsworks.api.modelregistry.models.tags.ModelRegistryTagUri; import io.hops.hopsworks.api.tags.TagBuilder; import io.hops.hopsworks.api.tags.TagsExpansionBeanParam; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dataset.util.DatasetPath; import io.hops.hopsworks.common.featurestore.metadata.AttachMetadataResult; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.tags.TagControllerIface; import io.hops.hopsworks.common.tags.TagsDTO; import io.hops.hopsworks.exceptions.DatasetException; -import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; +import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; @@ -64,7 +68,7 @@ @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Tags resource") -public abstract class ModelRegistryTagResource { +public abstract class ModelRegistryTagResource extends ModelRegistrySubResource { @Inject private TagControllerIface tagController; @@ -73,29 +77,23 @@ public abstract class ModelRegistryTagResource { @EJB private JWTHelper jwtHelper; - protected Project project; - protected Project modelRegistry; + @EJB + private ProjectController projectController; + @EJB + private ModelsController modelsController; - /** - * Set the project of the tag resource (provided by parent resource) - * - * @param project the project where the tag operations will be performed - */ - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** - * Sets the model registry of the tag resource - * - * @param modelRegistry - */ - public void setModelRegistry(Project modelRegistry) { - this.modelRegistry = modelRegistry; + @Override + protected ModelsController getModelsController() { + return modelsController; } - protected abstract DatasetPath getDatasetPath() throws DatasetException; - protected abstract String getItemId(); + protected abstract DatasetPath getDatasetPath() throws DatasetException, ProjectException, ModelRegistryException; + protected abstract String getItemId() throws ProjectException, ModelRegistryException; protected abstract ResourceRequest.Name getItemType(); @ApiOperation(value = "Create or update one tag", response = TagsDTO.class) @@ -114,11 +112,13 @@ public Response putTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @ApiParam(value = "Value to set for the tag") String value) - throws MetadataException, FeatureStoreMetadataException, DatasetException { + throws MetadataException, FeatureStoreMetadataException, DatasetException, ProjectException, + ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); AttachMetadataResult result = tagController.upsert(user, getDatasetPath(), name, value); - ModelRegistryTagUri tagUri = new ModelRegistryTagUri(uriInfo, modelRegistry, getItemType(), getItemId()); + ModelRegistryTagUri tagUri = + new ModelRegistryTagUri(uriInfo, getModelRegistryProject(), getItemType(), getItemId()); TagsDTO dto = tagBuilder.build(tagUri, getDatasetPath(), result.getItems()); UriBuilder builder = uriInfo.getAbsolutePathBuilder(); @@ -142,7 +142,8 @@ public Response putTag(@Context SecurityContext sc, public Response bulkPutTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, TagsDTO tags) - throws MetadataException, FeatureStoreMetadataException, DatasetException { + throws MetadataException, FeatureStoreMetadataException, DatasetException, ProjectException, + ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); AttachMetadataResult result; @@ -156,7 +157,8 @@ public Response bulkPutTags(@Context SecurityContext sc, @Context UriInfo uriInf } result = tagController.upsertAll(user, getDatasetPath(), newTags); } - ModelRegistryTagUri tagUri = new ModelRegistryTagUri(uriInfo, modelRegistry, getItemType(), getItemId()); + ModelRegistryTagUri tagUri = + new ModelRegistryTagUri(uriInfo, getModelRegistryProject(), getItemType(), getItemId()); TagsDTO dto = tagBuilder.build(tagUri, getDatasetPath(), result.getItems()); UriBuilder builder = uriInfo.getAbsolutePathBuilder(); @@ -179,12 +181,14 @@ public Response bulkPutTags(@Context SecurityContext sc, @Context UriInfo uriInf public Response getTags(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws DatasetException, MetadataException, FeatureStoreMetadataException { + throws DatasetException, MetadataException, FeatureStoreMetadataException, ProjectException, + ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); resourceRequest.setExpansions(tagsExpansionBeanParam.getResources()); - ModelRegistryTagUri tagUri = new ModelRegistryTagUri(uriInfo, modelRegistry, getItemType(), getItemId()); + ModelRegistryTagUri tagUri = + new ModelRegistryTagUri(uriInfo, getModelRegistryProject(), getItemType(), getItemId()); TagsDTO dto = tagBuilder.build(tagUri, resourceRequest, user, getDatasetPath()); return Response.status(Response.Status.OK).entity(dto).build(); } @@ -203,12 +207,14 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name, @BeanParam TagsExpansionBeanParam tagsExpansionBeanParam) - throws DatasetException, MetadataException, FeatureStoreMetadataException { + throws DatasetException, MetadataException, FeatureStoreMetadataException, ProjectException, + ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.TAGS); resourceRequest.setExpansions(tagsExpansionBeanParam.getResources()); - ModelRegistryTagUri tagUri = new ModelRegistryTagUri(uriInfo, modelRegistry, getItemType(), getItemId()); + ModelRegistryTagUri tagUri = + new ModelRegistryTagUri(uriInfo, getModelRegistryProject(), getItemType(), getItemId()); TagsDTO dto = tagBuilder.buildAsMap(tagUri, resourceRequest, user, getDatasetPath(), name); return Response.status(Response.Status.OK).entity(dto).build(); } @@ -224,7 +230,7 @@ public Response getTag(@Context SecurityContext sc, @Context UriInfo uriInfo, @FeatureFlagRequired(requiredFeatureFlags = {FeatureFlags.DATA_SCIENCE_PROFILE}) public Response deleteTags(@Context SecurityContext sc, @Context HttpServletRequest req) - throws DatasetException, MetadataException { + throws DatasetException, MetadataException, ProjectException, ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); tagController.deleteAll(user, getDatasetPath()); @@ -244,7 +250,7 @@ public Response deleteTags(@Context SecurityContext sc, public Response deleteTag(@Context SecurityContext sc, @Context HttpServletRequest req, @ApiParam(value = "Name of the tag", required = true) @PathParam("name") String name) - throws DatasetException, MetadataException { + throws DatasetException, MetadataException, ProjectException, ModelRegistryException { Users user = jwtHelper.getUserPrincipal(sc); tagController.delete(user, getDatasetPath(), name); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelRegistrySubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelRegistrySubResource.java new file mode 100644 index 0000000000..e2d411c128 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelRegistrySubResource.java @@ -0,0 +1,62 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.modelregistry.models; + +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; +import io.hops.hopsworks.persistence.entity.project.Project; + +public abstract class ModelRegistrySubResource extends ProjectSubResource { + private Integer modelRegistryId; + + private String modelId; + + public Integer getModelRegistryId() { + return modelRegistryId; + } + + public void setModelRegistryId(Integer modelRegistryId) { + this.modelRegistryId = modelRegistryId; + } + + public String getModelId() { + return modelId; + } + + public void setModelId(String modelId) { + this.modelId = modelId; + } + + public Project getModelRegistryProject(Project project) throws ModelRegistryException { + return getModelsController().verifyModelRegistryAccess(project, modelRegistryId).getParentProject(); + } + + public Project getModelRegistryProject() throws ProjectException, ModelRegistryException { + return getModelsController().verifyModelRegistryAccess(getProject(), modelRegistryId).getParentProject(); + } + + public ModelVersion getModelVersion(Project modelRegistryProject) throws ModelRegistryException { + return getModelsController().getModel(modelRegistryProject, modelId); + } + + public ModelVersion getModelVersion() throws ProjectException, ModelRegistryException { + return getModelsController().getModel(getModelRegistryProject(), modelId); + } + + protected abstract ModelsController getModelsController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java index a5ea9a4f7c..ebccd75849 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/ModelsResource.java @@ -15,9 +15,9 @@ */ package io.hops.hopsworks.api.modelregistry.models; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; import io.hops.hopsworks.api.jwt.JWTHelper; @@ -27,8 +27,10 @@ import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.hdfs.DistributedFsService; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.CryptoPasswordNotFoundException; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.JobException; import io.hops.hopsworks.exceptions.KafkaException; @@ -37,7 +39,6 @@ import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.PythonException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.exceptions.ServingException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -75,7 +76,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ModelsResource { +public class ModelsResource extends ModelRegistrySubResource { private static final Logger LOGGER = Logger.getLogger(ModelsResource.class.getName()); @@ -91,28 +92,21 @@ public class ModelsResource { private ModelUtils modelUtils; @Inject private ModelTagResource tagResource; + @Inject private ModelProvenanceResource provenanceResource; - private Project userProject; - - private Project modelRegistryProject; + @EJB + private ProjectController projectController; - public ModelsResource setProject(Project project) { - this.userProject = project; - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } - /** - * Sets the model registry of the models - * - * @param modelRegistryId id of the model registry - * @throws ModelRegistryException - */ - public void setModelRegistryId(Integer modelRegistryId) throws ModelRegistryException { - //This call verifies that the project have access to the modelRegistryId provided - this.modelRegistryProject = modelsController.verifyModelRegistryAccess(userProject, - modelRegistryId).getParentProject(); + @Override + protected ModelsController getModelsController() { + return modelsController; } @ApiOperation(value = "Get a list of all models for this project", response = ModelDTO.class) @@ -128,16 +122,20 @@ public Response getAll( @BeanParam Pagination pagination, @BeanParam ModelsBeanParam modelsBeanParam, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) - throws ModelRegistryException, GenericException, FeatureStoreMetadataException, MetadataException { + throws ModelRegistryException, GenericException, FeatureStoreMetadataException, MetadataException, + ProjectException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); resourceRequest.setFilter(modelsBeanParam.getFilter()); - resourceRequest.setSort(modelsBeanParam.getSortBySet()); resourceRequest.setExpansions(modelsBeanParam.getExpansions().getResources()); - ModelDTO dto = modelsBuilder.build(uriInfo, resourceRequest, user, userProject, modelRegistryProject); + resourceRequest.setSort(modelsBeanParam.getSortBySet()); + Project userProject = getProject(); + ModelDTO dto = + modelsBuilder.build(uriInfo, resourceRequest, user, userProject, getModelRegistryProject(userProject)); return Response.ok().entity(dto).build(); } @@ -155,12 +153,15 @@ public Response get ( @PathParam("id") String id, @BeanParam ModelsBeanParam modelsBeanParam, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) - throws ProvenanceException, ModelRegistryException, DatasetException, GenericException, - FeatureStoreMetadataException, MetadataException { + throws ProvenanceException, ModelRegistryException, DatasetException, GenericException, + FeatureStoreMetadataException, MetadataException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.MODELS); resourceRequest.setExpansions(modelsBeanParam.getExpansions().getResources()); + Project userProject = getProject(); + Project modelRegistryProject = getModelRegistryProject(userProject); ModelVersion modelVersion = modelsController.getModel(modelRegistryProject, id); if(modelVersion != null) { @@ -190,12 +191,14 @@ public Response delete ( @Context HttpServletRequest req, @Context UriInfo uriInfo, @Context SecurityContext sc) - throws DatasetException, ProvenanceException, ModelRegistryException, KafkaException, ServingException, - CryptoPasswordNotFoundException { + throws DatasetException, ProvenanceException, ModelRegistryException, KafkaException, ServingException, + CryptoPasswordNotFoundException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project userProject = getProject(); + ModelVersion modelVersion = modelsController.getModel(userProject, id); if(modelVersion != null) { - modelsController.delete(user, userProject, modelRegistryProject, modelVersion); + modelsController.delete(user, userProject, getModelRegistryProject(userProject), modelVersion); } return Response.noContent().build(); } @@ -225,6 +228,8 @@ public Response put(@PathParam("id") String id, } modelUtils.validateModelName(modelDTO); Users user = jwtHelper.getUserPrincipal(sc); + Project userProject = getProject(); + Project modelRegistryProject = getModelRegistryProject(userProject); Project modelProject = modelUtils.getModelsProjectAndCheckAccess(modelDTO, userProject); Project experimentProject = modelUtils.getExperimentProjectAndCheckAccess(modelDTO, userProject); ModelsController.Accessor accessor = null; @@ -249,10 +254,9 @@ public Response put(@PathParam("id") String id, public ModelTagResource tags(@ApiParam(value = "Id of the model", required = true) @PathParam("id") String id) throws ModelRegistryException, ProvenanceException { - this.tagResource.setProject(userProject); - this.tagResource.setModelRegistry(modelRegistryProject); - ModelVersion modelVersion = modelsController.getModel(modelRegistryProject, id); - this.tagResource.setModel(modelVersion); + this.tagResource.setProjectId(getProjectId()); + this.tagResource.setModelRegistryId(getModelRegistryId()); + this.tagResource.setModelId(id); return this.tagResource; } @@ -260,10 +264,9 @@ public ModelTagResource tags(@ApiParam(value = "Id of the model", required = tru public ModelProvenanceResource provenance(@ApiParam(value = "Id of the model", required = true) @PathParam("id") String id) throws ModelRegistryException, ProvenanceException { - this.provenanceResource.setAccessProject(userProject); - this.provenanceResource.setModelRegistry(modelRegistryProject); - ModelVersion modelVersion = modelsController.getModel(modelRegistryProject, id); - this.provenanceResource.setModelVersion(modelVersion); + this.provenanceResource.setProjectId(getProjectId()); + this.provenanceResource.setModelRegistryId(getModelRegistryId()); + this.provenanceResource.setModelId(id); return this.provenanceResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/tags/ModelTagResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/tags/ModelTagResource.java index a9cc79b983..ba5fc730de 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/tags/ModelTagResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/modelregistry/models/tags/ModelTagResource.java @@ -21,8 +21,11 @@ import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.dataset.util.DatasetPath; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.persistence.entity.dataset.DatasetType; import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; +import io.hops.hopsworks.persistence.entity.project.Project; import javax.ejb.EJB; import javax.ejb.TransactionAttribute; @@ -37,26 +40,22 @@ public class ModelTagResource extends ModelRegistryTagResource { private ModelUtils modelUtils; @EJB private DatasetHelper datasetHelper; - - private ModelVersion modelVersion; - - /** - * Sets the model version for the tag resource - * - * @param modelVersion - */ - public void setModel(ModelVersion modelVersion) { - this.modelVersion = modelVersion; - } + @Override - protected DatasetPath getDatasetPath() throws DatasetException { + protected DatasetPath getDatasetPath() throws DatasetException, ProjectException, ModelRegistryException { + Project project = getProject(); + Project modelRegistry = getModelRegistryProject(project); + ModelVersion modelVersion = getModelVersion(modelRegistry); return datasetHelper.getDatasetPath(project, modelUtils.getModelFullPath(modelRegistry, modelVersion.getModel().getName(), modelVersion.getVersion()), DatasetType.DATASET); } @Override - protected String getItemId() { + protected String getItemId() throws ModelRegistryException, ProjectException { + Project project = getProject(); + Project modelRegistry = getModelRegistryProject(project); + ModelVersion modelVersion = getModelVersion(modelRegistry); return modelVersion.getMlId(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/opensearch/OpenSearchResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/opensearch/OpenSearchResource.java index 1f44a1d3b5..087b69d490 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/opensearch/OpenSearchResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/opensearch/OpenSearchResource.java @@ -16,20 +16,24 @@ package io.hops.hopsworks.api.opensearch; import com.google.common.base.Strings; +import io.hops.hopsworks.api.filter.AllowedProjectRoles; +import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.opensearch.featurestore.OpenSearchFeaturestoreBuilder; import io.hops.hopsworks.api.opensearch.featurestore.OpenSearchFeaturestoreDTO; import io.hops.hopsworks.api.opensearch.featurestore.OpenSearchFeaturestoreRequest; -import io.hops.hopsworks.api.filter.AllowedProjectRoles; -import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.common.opensearch.FeaturestoreDocType; -import io.hops.hopsworks.exceptions.OpenSearchException; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.exceptions.GenericException; +import io.hops.hopsworks.exceptions.OpenSearchException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.restutils.RESTCodes; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; + +import javax.ejb.EJB; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; @@ -49,22 +53,18 @@ @Api(value = "OpenSearch Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class OpenSearchResource { - +public class OpenSearchResource extends ProjectSubResource { + @Inject private OpenSearchFeaturestoreBuilder openSearchFeaturestoreBuilder; + @EJB + private ProjectController projectController; - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; + @Override + protected ProjectController getProjectController() { + return projectController; } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - + /** * Searches for content inside all project accesible featurestores. Hits 'featurestore' index *

@@ -101,7 +101,7 @@ public Response featurestoreSearch( } OpenSearchFeaturestoreDTO dto = openSearchFeaturestoreBuilder.build( - new OpenSearchFeaturestoreRequest(searchTerm, docType, from, size), projectId); + new OpenSearchFeaturestoreRequest(searchTerm, docType, from, size), getProjectId()); return Response.ok().entity(dto).build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectMembersService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectMembersService.java index a9fdf18af3..7e5c99204e 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectMembersService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectMembersService.java @@ -56,7 +56,6 @@ import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.HopsSecurityException; import io.hops.hopsworks.exceptions.JobException; -import io.hops.hopsworks.exceptions.KafkaException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.exceptions.TensorBoardException; @@ -95,7 +94,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ProjectMembersService { +public class ProjectMembersService extends ProjectSubResource { @EJB private ProjectController projectController; @@ -110,20 +109,15 @@ public class ProjectMembersService { @EJB private DatasetHelper datasetHelper; - private Integer projectId; - public ProjectMembersService() { } - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - public Integer getProjectId() { - return projectId; + + @Override + protected ProjectController getProjectController() { + return projectController; } - private final static Logger logger = Logger.getLogger( - ProjectMembersService.class. - getName()); + private final static Logger logger = Logger.getLogger(ProjectMembersService.class.getName()); @GET @Produces(MediaType.APPLICATION_JSON) @@ -131,7 +125,7 @@ public Integer getProjectId() { allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER}) public Response findMembersByProjectID(@Context HttpServletRequest req, @Context SecurityContext sc) { - List list = projectController.findProjectTeamById(this.projectId); + List list = projectController.findProjectTeamById(getProjectId()); GenericEntity> projects = new GenericEntity>(list) { }; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(projects).build(); @@ -143,9 +137,9 @@ public Response findMembersByProjectID(@Context HttpServletRequest req, @Context allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER}) public Response addMembers(MembersDTO members, @Context HttpServletRequest req, @Context SecurityContext sc) - throws KafkaException, ProjectException, UserException, FeaturestoreException { + throws ProjectException, FeaturestoreException { - Project project = projectController.findProjectById(this.projectId); + Project project = getProject(); RESTApiJsonResponse json = new RESTApiJsonResponse(); List failedMembers = null; Users user = jWTHelper.getUserPrincipal(sc); @@ -191,8 +185,8 @@ public Response addMembers(MembersDTO members, @Context HttpServletRequest req, public Response updateRoleByEmail(@PathParam("email") String email, @FormParam("role") String role, @Context HttpServletRequest req, @Context SecurityContext sc) - throws ProjectException, UserException, FeaturestoreException, IOException, KafkaException { - Project project = projectController.findProjectById(this.projectId); + throws ProjectException, UserException, FeaturestoreException, IOException { + Project project = getProject(); RESTApiJsonResponse json = new RESTApiJsonResponse(); Users user = jWTHelper.getUserPrincipal(sc); if (email == null) { @@ -220,7 +214,7 @@ public Response removeMembersByID(@PathParam("email") String email, @Context SecurityContext sc) throws ProjectException, ServiceException, HopsSecurityException, UserException, GenericException, IOException, JobException, TensorBoardException, FeaturestoreException { - Project project = projectController.findProjectById(this.projectId); + Project project = getProject(); RESTApiJsonResponse json = new RESTApiJsonResponse(); Users reqUser = jWTHelper.getUserPrincipal(sc); if (email == null) { @@ -254,8 +248,8 @@ public Response getDatasetMembers(@PathParam("name") String dsName, @QueryParam("type") DatasetType datasetType, @Context HttpServletRequest req, @Context SecurityContext sc) - throws ProjectException, DatasetException { - Project project = projectController.findProjectById(this.projectId); + throws ProjectException, DatasetException { + Project project = getProject(); String path = Utils.getProjectPath(project.getName()) + dsName; DatasetPath dp = datasetHelper.getDatasetPath(project, path, datasetType); Collection membersCol = accessCtrl.getExtendedMembers(dp.getDataset()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java index bbf4ea35aa..7e3ec9c889 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java @@ -41,14 +41,13 @@ import io.hops.hopsworks.api.activities.ProjectActivitiesResource; import io.hops.hopsworks.api.airflow.AirflowService; import io.hops.hopsworks.api.alert.AlertResource; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.dataset.DatasetResource; -import io.hops.hopsworks.api.opensearch.OpenSearchResource; import io.hops.hopsworks.api.experiments.ExperimentsResource; import io.hops.hopsworks.api.featurestore.FeaturestoreService; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.NoCacheResponse; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.git.GitResource; import io.hops.hopsworks.api.jobs.JobsResource; import io.hops.hopsworks.api.jupyter.JupyterService; @@ -56,8 +55,9 @@ import io.hops.hopsworks.api.kafka.KafkaResource; import io.hops.hopsworks.api.metadata.XAttrsResource; import io.hops.hopsworks.api.modelregistry.ModelRegistryResource; -import io.hops.hopsworks.api.project.jobconfig.DefaultJobConfigurationResource; +import io.hops.hopsworks.api.opensearch.OpenSearchResource; import io.hops.hopsworks.api.project.alert.ProjectAlertsResource; +import io.hops.hopsworks.api.project.jobconfig.DefaultJobConfigurationResource; import io.hops.hopsworks.api.provenance.ProjectProvenanceResource; import io.hops.hopsworks.api.python.PythonResource; import io.hops.hopsworks.api.serving.ServingService; @@ -87,11 +87,11 @@ import io.hops.hopsworks.common.util.AccessController; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.DatasetException; -import io.hops.hopsworks.exceptions.OpenSearchException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.HopsSecurityException; import io.hops.hopsworks.exceptions.KafkaException; +import io.hops.hopsworks.exceptions.OpenSearchException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.SchemaException; @@ -596,7 +596,8 @@ public DatasetResource datasetResource(@PathParam("projectId") Integer id) { @Path("{projectId}/jobs") public JobsResource jobs(@PathParam("projectId") Integer projectId) { - return this.jobs.setProject(projectId); + this.jobs.setProjectId(projectId); + return this.jobs; } @GET @@ -622,7 +623,7 @@ public Response getCurrentMultiplicator(@PathParam("projectId") Integer id) { }; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(multiplicators).build(); } - + @POST @Path("{projectId}/downloadCert") @Produces(MediaType.APPLICATION_JSON) @@ -667,11 +668,10 @@ public Response client(@PathParam("projectId") Integer id, @Context HttpServletR @Path("{projectId}/kafka") public KafkaResource kafka(@PathParam("projectId") Integer id) throws ProjectException { - Project project = projectController.findProjectById(id); - this.kafka.setProject(project); + this.kafka.setProjectId(id); return this.kafka; } - + @Path("{projectId}/jupyter") public JupyterService jupyter(@PathParam("projectId") Integer id) { this.jupyter.setProjectId(id); @@ -736,10 +736,10 @@ public FeaturestoreService featurestoreService(@PathParam("projectId") Integer p @Path("{projectId}/xattrs") public XAttrsResource xattrs(@PathParam("projectId") Integer projectId) { - this.xattrs.setProject(projectId); + this.xattrs.setProjectId(projectId); return xattrs; } - + @Path("{projectId}/provenance") public ProjectProvenanceResource provenance(@PathParam("projectId") Integer id) { this.provenance.setProjectId(id); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectSubResource.java new file mode 100644 index 0000000000..01078640c7 --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectSubResource.java @@ -0,0 +1,56 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.project; + +import io.hops.hopsworks.common.project.ProjectController; +import io.hops.hopsworks.exceptions.ProjectException; +import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.restutils.RESTCodes; + +import java.util.logging.Level; + +public abstract class ProjectSubResource { + + private Integer projectId; + private String projectName; + + public Integer getProjectId() { + return projectId; + } + + public void setProjectId(Integer projectId) { + this.projectId = projectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Project getProject() throws ProjectException { + if (this.projectId != null) { + return getProjectController().findProjectById(this.projectId); + } else if (this.projectName != null) { + return getProjectController().findProjectByName(this.projectName); + } + throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + } + + protected abstract ProjectController getProjectController(); +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/alert/ProjectAlertsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/alert/ProjectAlertsResource.java index e681a352a2..4ffce2fdb0 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/alert/ProjectAlertsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/alert/ProjectAlertsResource.java @@ -31,15 +31,16 @@ import io.hops.hopsworks.api.alert.AlertBuilder; import io.hops.hopsworks.api.alert.AlertDTO; import io.hops.hopsworks.api.alert.FeatureStoreAlertController; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.featurestore.datavalidation.alert.FeatureGroupAlertBuilder; import io.hops.hopsworks.api.featurestore.datavalidation.alert.FeatureGroupAlertDTO; import io.hops.hopsworks.api.featurestore.featureview.FeatureViewAlertBuilder; import io.hops.hopsworks.api.featurestore.featureview.FeatureViewAlertDTO; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jobs.alert.JobAlertsBuilder; import io.hops.hopsworks.api.jobs.alert.JobAlertsDTO; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.alert.AlertController; import io.hops.hopsworks.common.api.ResourceRequest; @@ -90,7 +91,7 @@ @Api(value = "Project Alerts Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class ProjectAlertsResource { +public class ProjectAlertsResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(ProjectAlertsResource.class.getName()); @@ -114,27 +115,12 @@ public class ProjectAlertsResource { private FeatureViewAlertBuilder featureViewAlertBuilder; @EJB private FeatureStoreAlertController featureStoreAlertController; - - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + @Override + protected ProjectController getProjectController() { + return projectController; } - + @GET @Produces(MediaType.APPLICATION_JSON) @ApiOperation(value = "Get all alerts.", response = ProjectAlertsDTO.class) diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationResource.java index 708fd9350e..bf14c8a661 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/jobconfig/DefaultJobConfigurationResource.java @@ -16,12 +16,12 @@ package io.hops.hopsworks.api.project.jobconfig; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; -import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.util.HopsUtils; import io.hops.hopsworks.exceptions.ProjectException; @@ -39,6 +39,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -58,23 +59,19 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class DefaultJobConfigurationResource { +public class DefaultJobConfigurationResource extends ProjectSubResource { private static final Logger LOGGER = Logger.getLogger(DefaultJobConfigurationResource.class.getName()); - @EJB - private ProjectFacade projectFacade; - @EJB private ProjectController projectController; @EJB private DefaultJobConfigurationBuilder defaultJobConfigurationBuilder; - private Project project; - public DefaultJobConfigurationResource setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get all the default job configurations", response = DefaultJobConfigurationDTO.class) @@ -94,7 +91,7 @@ public Response get(@BeanParam Pagination pagination, resourceRequest.setSort(defaultJobConfigurationBeanParam.getSortBySet()); resourceRequest.setFilter(defaultJobConfigurationBeanParam.getFilter()); DefaultJobConfigurationDTO defaultJobConfigurationDTO = - this.defaultJobConfigurationBuilder.build(uriInfo, resourceRequest, this.project); + this.defaultJobConfigurationBuilder.build(uriInfo, resourceRequest, getProject()); if(defaultJobConfigurationDTO == null) { throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_DEFAULT_JOB_CONFIG_NOT_FOUND, Level.FINEST); } @@ -113,7 +110,7 @@ public Response getByType(@PathParam("type") JobType jobType, @Context SecurityContext sc) throws ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.JOBCONFIG); DefaultJobConfiguration defaultJobConfiguration = - projectController.getProjectDefaultJobConfiguration(project, jobType); + projectController.getProjectDefaultJobConfiguration(getProject(), jobType); if(defaultJobConfiguration == null) { throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_DEFAULT_JOB_CONFIG_NOT_FOUND, Level.FINEST); } else { @@ -140,14 +137,14 @@ public Response put ( Response.Status status = Response.Status.CREATED; HopsUtils.validateJobConfigurationType(config, type); - + Project project = getProject(); DefaultJobConfiguration currentConfig = projectController.getProjectDefaultJobConfiguration(project, type); if(currentConfig != null) { status = Response.Status.OK; } DefaultJobConfiguration defaultConfig = - projectController.createOrUpdateDefaultJobConfig(this.project, config, type, currentConfig); + projectController.createOrUpdateDefaultJobConfig(project, config, type, currentConfig); DefaultJobConfigurationDTO defaultJobConfigurationDTO = this.defaultJobConfigurationBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.JOBCONFIG), @@ -169,8 +166,9 @@ public Response put ( @JWTRequired(acceptedTokens={Audience.API, Audience.JOB}, allowedUserRoles={"HOPS_ADMIN", "HOPS_USER"}) @ApiKeyRequired(acceptedScopes = {ApiScope.JOB}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"}) public Response delete(@PathParam("type") JobType type, - @Context SecurityContext sc) { - projectController.removeProjectDefaultJobConfiguration(this.project, type); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProjectException { + projectController.removeProjectDefaultJobConfiguration(getProject(), type); return Response.noContent().build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java index 46328c88e6..50dd4c89a2 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureGroupProvenanceResource.java @@ -16,6 +16,7 @@ package io.hops.hopsworks.api.provenance; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featuregroup.FeatureGroupSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; @@ -33,6 +34,7 @@ import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController; import io.hops.hopsworks.common.hdfs.Utils; import io.hops.hopsworks.common.hdfs.inode.InodeController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitControllerIface; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitLink; import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; @@ -43,11 +45,11 @@ import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.dataset.Dataset; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.Featuregroup; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; import io.hops.hopsworks.persistence.entity.project.Project; @@ -76,7 +78,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature Group Explicit Provenance Resource") -public class FeatureGroupProvenanceResource { +public class FeatureGroupProvenanceResource extends FeatureGroupSubResource { @EJB private FeaturestoreController featurestoreController; @EJB @@ -96,26 +98,28 @@ public class FeatureGroupProvenanceResource { private JWTHelper jwtHelper; @EJB private DatasetHelper datasetHelper; + @EJB + private ProjectController projectController; - private Project project; - - private Featurestore featureStore; - private Integer featureGroupId; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - - public void setFeatureGroupId(Integer featureGroupId) { - this.featureGroupId = featureGroupId; + + @Override + protected FeaturegroupController getFeaturegroupController() { + return featureGroupController; } - private DatasetPath getFeaturestoreDatasetPath() throws FeaturestoreException, DatasetException { - Dataset featurestore = featurestoreController.getProjectFeaturestoreDataset(featureStore.getProject()); + + private DatasetPath getFeaturestoreDatasetPath() throws FeaturestoreException, DatasetException, ProjectException { + Project project = getProject(); + Dataset featurestore = featurestoreController.getProjectFeaturestoreDataset(project); Path featurestorePath = Utils.getDatasetPath(featurestore, settings); Inode featurestoreInode = inodeController.getInodeAtPath(featurestorePath.toString()); return datasetHelper.getTopLevelDatasetPath(project, featurestore, featurestoreInode); @@ -139,11 +143,12 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException, ModelRegistryException { + FeatureStoreMetadataException, IOException, ModelRegistryException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); - Featuregroup fg = featureGroupController.getFeaturegroupById(featureStore, featureGroupId); + Featuregroup fg = featureGroupController.getFeaturegroupById(getFeaturestore(project), getFeatureGroupId()); ProvExplicitLink provenance = provCtrl.featureGroupLinks(project, fg, pagination.getUpstreamLvls(), pagination.getDownstreamLvls()); ProvExplicitLinkDTO result = linksBuilder.build(uriInfo, resourceRequest, project, user, provenance); @@ -161,9 +166,9 @@ public Response status(@BeanParam ProvUsageBeanParams params, @Context HttpServletRequest req, @Context SecurityContext sc) throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException, - FeaturestoreException { + FeaturestoreException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); - Featuregroup featureGroup = featureGroupController.getFeaturegroupById(featureStore, featureGroupId); + Featuregroup featureGroup = featureGroupController.getFeaturegroupById(getFeaturestore(), getFeatureGroupId()); String fgProvenanceId = featureGroup.getName() + "_" + featureGroup.getVersion(); ProvArtifactUsageParentDTO status = usageBuilder.buildAccessible(uriInfo, user, getFeaturestoreDatasetPath(), fgProvenanceId, params.getUsageType()); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java index 14e6fb1436..5f038c9e8b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/FeatureViewProvenanceResource.java @@ -16,6 +16,7 @@ package io.hops.hopsworks.api.provenance; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.featureview.FeatureViewSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; @@ -29,7 +30,9 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.dataset.util.DatasetPath; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitControllerIface; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitLink; import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; @@ -39,11 +42,11 @@ import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.dataset.DatasetType; -import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; @@ -71,7 +74,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Feature View Explicit Provenance Resource") -public class FeatureViewProvenanceResource { +public class FeatureViewProvenanceResource extends FeatureViewSubResource { @EJB private FeatureViewController featureViewController; @@ -85,32 +88,29 @@ public class FeatureViewProvenanceResource { private JWTHelper jwtHelper; @EJB private DatasetHelper datasetHelper; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - private Project project; - - private Featurestore featureStore; - private String featureViewName; - private Integer featureViewVersion; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - - public void setFeatureViewName(String featureViewName) { - this.featureViewName = featureViewName; + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } - - public void setFeatureViewVersion(Integer featureViewVersion) { - this.featureViewVersion = featureViewVersion; - } - - private DatasetPath getFeaturestoreDatasetPath() throws FeaturestoreException, DatasetException { + + private DatasetPath getFeaturestoreDatasetPath() throws FeaturestoreException, DatasetException, ProjectException { + Project project = getProject(); FeatureView featureView - = featureViewController.getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, featureStore); + = featureViewController.getByNameVersionAndFeatureStore(getName(), getVersion(), getFeaturestore(project)); return datasetHelper.getDatasetPath(project, featureViewController.getLocation(featureView), DatasetType.DATASET); } @@ -133,12 +133,13 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException, ModelRegistryException { + FeatureStoreMetadataException, IOException, ModelRegistryException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); FeatureView fv - = featureViewController.getByNameVersionAndFeatureStore(featureViewName, featureViewVersion, featureStore); + = featureViewController.getByNameVersionAndFeatureStore(getName(), getVersion(), getFeaturestore(project)); ProvExplicitLink provenance = provCtrl.featureViewLinks(project, fv, pagination.getUpstreamLvls(), pagination.getDownstreamLvls()); ProvExplicitLinkDTO result = linksBuilder.build(uriInfo, resourceRequest, project, user, provenance); @@ -156,9 +157,9 @@ public Response status(@BeanParam ProvUsageBeanParams params, @Context HttpServletRequest req, @Context SecurityContext sc) throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException, - FeaturestoreException { + FeaturestoreException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); - String fvProvenanceId = featureViewName + "_" + featureViewVersion; + String fvProvenanceId = getName() + "_" + getVersion(); ProvArtifactUsageParentDTO status = usageBuilder.buildAccessible(uriInfo, user, getFeaturestoreDatasetPath(), fvProvenanceId, params.getUsageType()); return Response.ok().entity(status).build(); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java index ad284381de..82c332c9fb 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ModelProvenanceResource.java @@ -19,11 +19,14 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.modelregistry.models.ModelRegistrySubResource; +import io.hops.hopsworks.api.modelregistry.models.ModelsController; import io.hops.hopsworks.api.provenance.explicit.ExplicitProvenanceExpansionBeanParam; import io.hops.hopsworks.api.provenance.explicit.ProvExplicitLinksBuilder; import io.hops.hopsworks.api.provenance.explicit.dto.ProvExplicitLinkDTO; import io.hops.hopsworks.api.provenance.ops.ProvLinksBeanParams; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitControllerIface; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitLink; import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; @@ -33,6 +36,7 @@ import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.models.version.ModelVersion; @@ -62,27 +66,27 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Model Explicit Provenance Resource") -public class ModelProvenanceResource { +public class ModelProvenanceResource extends ModelRegistrySubResource { @EJB private JWTHelper jwtHelper; @Inject private ProvExplicitControllerIface provCtrl; @EJB private ProvExplicitLinksBuilder linksBuilder; - private Project accessProject; - private Project modelRegistry; - private ModelVersion modelVersion; - - public void setAccessProject(Project project) { - this.accessProject = project; - } + @EJB + private ProjectController projectController; + @EJB + private ModelsController modelsController; - public void setModelRegistry(Project project) { - this.modelRegistry = project; + private String modelId; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setModelVersion(ModelVersion modelVersion){ - this.modelVersion = modelVersion; + @Override + protected ModelsController getModelsController() { + return modelsController; } @GET @@ -103,8 +107,11 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException, ModelRegistryException { + FeatureStoreMetadataException, IOException, ModelRegistryException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project accessProject = getProject(); + Project modelRegistry = getModelRegistryProject(accessProject); + ModelVersion modelVersion = getModelVersion(modelRegistry); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); ProvExplicitLink provenance = provCtrl.modelLinks(accessProject, modelVersion, diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProjectProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProjectProvenanceResource.java index 116e90bdd0..b3b289ea59 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProjectProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProjectProvenanceResource.java @@ -15,17 +15,19 @@ */ package io.hops.hopsworks.api.provenance; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.dataset.DatasetAccessType; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.provenance.ops.ProvLinksBeanParams; import io.hops.hopsworks.api.provenance.ops.ProvLinksBuilder; import io.hops.hopsworks.api.provenance.ops.ProvOpsBeanParams; import io.hops.hopsworks.api.provenance.ops.ProvOpsBuilder; -import io.hops.hopsworks.api.provenance.ops.ProvUsageBuilder; import io.hops.hopsworks.api.provenance.ops.ProvUsageBeanParams; +import io.hops.hopsworks.api.provenance.ops.ProvUsageBuilder; +import io.hops.hopsworks.api.provenance.ops.dto.ProvArtifactUsageParentDTO; import io.hops.hopsworks.api.provenance.state.ProvStateBeanParams; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.dao.project.ProjectFacade; @@ -35,21 +37,22 @@ import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.hdfs.Utils; import io.hops.hopsworks.common.hdfs.inode.InodeController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.core.HopsFSProvenanceController; import io.hops.hopsworks.common.provenance.core.dto.ProvDatasetDTO; import io.hops.hopsworks.common.provenance.core.dto.ProvTypeDTO; -import io.hops.hopsworks.api.provenance.ops.dto.ProvArtifactUsageParentDTO; -import io.hops.hopsworks.common.provenance.state.ProvStateBuilder; import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; import io.hops.hopsworks.common.provenance.ops.dto.ProvOpsDTO; +import io.hops.hopsworks.common.provenance.state.ProvStateBuilder; import io.hops.hopsworks.common.provenance.state.dto.ProvStateDTO; import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ProvenanceException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.dataset.Dataset; import io.hops.hopsworks.persistence.entity.hdfs.inode.Inode; @@ -85,9 +88,9 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Project Provenance Service", description = "Project Provenance Service") -public class ProjectProvenanceResource { +public class ProjectProvenanceResource extends ProjectSubResource { private static final Logger logger = Logger.getLogger(ProjectProvenanceResource.class.getName()); - + @EJB private ProjectFacade projectFacade; @EJB @@ -112,17 +115,14 @@ public class ProjectProvenanceResource { private Settings settings; @EJB private InodeController inodeController; + @EJB + private ProjectController projectController; - private Project project; - - public void setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); - } - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - + @GET @Produces(MediaType.APPLICATION_JSON) @AllowedProjectRoles({AllowedProjectRoles.ANYONE}) @@ -130,12 +130,13 @@ public void setProject(Project project) { @ApiKeyRequired(acceptedScopes = {ApiScope.PROJECT}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Get the Provenance Type of PROJECT/DATASET", response = ProvTypeDTO.class) - public Response getProvenanceStatus( - @QueryParam("type") @DefaultValue("PROJECT") TypeOf typeOf, - @Context SecurityContext sc) - throws ProvenanceException { + public Response getProvenanceStatus(@QueryParam("type") @DefaultValue("PROJECT") TypeOf typeOf, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws ProvenanceException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - switch(typeOf) { + Project project = getProject(); + switch (typeOf) { case PROJECT: ProvTypeDTO status = fsProvenanceCtrl.getProjectProvType(user, project); return Response.ok().entity(status).build(); @@ -163,14 +164,14 @@ public enum TypeOf { allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "State Provenance query endpoint", response = ProvStateDTO.class) public Response getFileStates( - @BeanParam - ProvStateBeanParams params, - @BeanParam Pagination pagination, - @Context HttpServletRequest req) throws ProvenanceException { - ProvStateDTO result = stateBuilder.build(project, params, pagination); + @BeanParam ProvStateBeanParams params, + @BeanParam Pagination pagination, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProvenanceException, ProjectException { + ProvStateDTO result = stateBuilder.build(getProject(), params, pagination); return Response.ok().entity(result).build(); } - + @GET @Path("ops") @Produces(MediaType.APPLICATION_JSON) @@ -180,15 +181,15 @@ public Response getFileStates( allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Operations Provenance query endpoint", response = ProvOpsDTO.class) public Response getFileOps( - @BeanParam ProvOpsBeanParams params, - @BeanParam Pagination pagination, - @Context HttpServletRequest req, - @Context SecurityContext sc, - @Context UriInfo uriInfo) throws ProvenanceException, GenericException { - ProvOpsDTO result = opsBuilder.build(project, params, pagination); + @BeanParam ProvOpsBeanParams params, + @BeanParam Pagination pagination, + @Context HttpServletRequest req, + @Context SecurityContext sc, + @Context UriInfo uriInfo) throws ProvenanceException, GenericException, ProjectException { + ProvOpsDTO result = opsBuilder.build(getProject(), params, pagination); return Response.ok().entity(result).build(); } - + @GET @Path("links") @Produces(MediaType.APPLICATION_JSON) @@ -200,13 +201,14 @@ public Response getFileOps( "link feature groups/training datasets/experiments/models through their application ids", response = ProvLinksDTO.class) public Response getLinks( - @BeanParam ProvLinksBeanParams params, - @BeanParam Pagination pagination, - @Context HttpServletRequest req) throws ProvenanceException, GenericException { - ProvLinksDTO result = linksBuilder.build(project, params, pagination); + @BeanParam ProvLinksBeanParams params, + @BeanParam Pagination pagination, + @Context HttpServletRequest req, + @Context SecurityContext sc) throws ProvenanceException, GenericException, ProjectException { + ProvLinksDTO result = linksBuilder.build(getProject(), params, pagination); return Response.ok().entity(result).build(); } - + @GET @Path("usage") @Produces(MediaType.APPLICATION_JSON) @@ -216,23 +218,27 @@ public Response getLinks( allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Artifact usage", response = ProvArtifactUsageParentDTO.class) public Response usage( - @QueryParam("artifact_id") String artifactId, - @QueryParam("endpoint_id") Integer endpointId, - @QueryParam("artifact_type") DatasetAccessType accessType, - @BeanParam ProvUsageBeanParams params, - @Context UriInfo uriInfo, - @Context SecurityContext sc) - throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException { + @QueryParam("artifact_id") String artifactId, + @QueryParam("endpoint_id") Integer endpointId, + @QueryParam("artifact_type") DatasetAccessType accessType, + @BeanParam ProvUsageBeanParams params, + @Context UriInfo uriInfo, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException, + ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - if(artifactId == null) { + Project project = getProject(); + if (artifactId == null) { throw new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.FINE, - "artifactId id cannot be null"); + "artifactId id cannot be null"); } Project targetProject = project; - if(endpointId != null) { - targetProject = projectFacade.findById(endpointId).orElseThrow( - () -> new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.FINE, "target project not found") - ); + if (endpointId != null) { + targetProject = + projectFacade.findById(endpointId).orElseThrow(() -> + new GenericException(RESTCodes.GenericErrorCode.ILLEGAL_ARGUMENT, Level.FINE, "target project not found") + ); } Dataset targetEndpoint; if (accessType != null) { diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProvArtifactResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProvArtifactResource.java index d3add09016..7636fafe10 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProvArtifactResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/ProvArtifactResource.java @@ -18,25 +18,24 @@ import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; -import io.hops.hopsworks.api.provenance.ops.ProvUsageBuilder; import io.hops.hopsworks.api.provenance.ops.ProvUsageBeanParams; +import io.hops.hopsworks.api.provenance.ops.ProvUsageBuilder; import io.hops.hopsworks.api.provenance.ops.dto.ProvArtifactUsageParentDTO; -import io.hops.hopsworks.common.dataset.util.DatasetPath; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ProvenanceException; -import io.hops.hopsworks.exceptions.FeatureStoreMetadataException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; -import io.hops.hopsworks.persistence.entity.user.Users; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; + import javax.ejb.EJB; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -56,13 +55,11 @@ public class ProvArtifactResource { @EJB private JWTHelper jWTHelper; - private Project userProject; - private DatasetPath targetEndpoint; + private String path; private String artifactId; - public void setContext(Project userProject, DatasetPath targetEndpoint) { - this.userProject = userProject; - this.targetEndpoint = targetEndpoint; + public void setPath(String path) { + this.path = path; } public void setArtifactId(String name, Integer version) { @@ -75,14 +72,15 @@ public void setArtifactId(String name, Integer version) { @AllowedProjectRoles({AllowedProjectRoles.DATA_SCIENTIST, AllowedProjectRoles.DATA_OWNER}) @JWTRequired(acceptedTokens = {Audience.API}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) @ApiOperation(value = "Artifact usage", response = ProvArtifactUsageParentDTO.class) - public Response status( - @BeanParam ProvUsageBeanParams params, - @Context UriInfo uriInfo, - @Context SecurityContext sc) - throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException { - Users user = jWTHelper.getUserPrincipal(sc); - ProvArtifactUsageParentDTO status = usageBuilder.buildAccessible(uriInfo, user, targetEndpoint, artifactId, - params.getUsageType()); - return Response.ok().entity(status).build(); + public Response status(@BeanParam ProvUsageBeanParams params, + @Context UriInfo uriInfo, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException { +// Users user = jWTHelper.getUserPrincipal(sc); +// DatasetPath targetEndpoint = +// ProvArtifactUsageParentDTO status = usageBuilder.buildAccessible(uriInfo, user, targetEndpoint, artifactId, +// params.getUsageType()); + return Response.ok().build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/StorageConnectorProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/StorageConnectorProvenanceResource.java index 477574bd0a..dd8d195ef0 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/StorageConnectorProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/StorageConnectorProvenanceResource.java @@ -16,6 +16,7 @@ package io.hops.hopsworks.api.provenance; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.FeaturestoreSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; @@ -24,7 +25,9 @@ import io.hops.hopsworks.api.provenance.explicit.dto.ProvExplicitLinkDTO; import io.hops.hopsworks.api.provenance.ops.ProvLinksBeanParams; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitControllerIface; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitLink; import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; @@ -34,6 +37,7 @@ import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.featurestore.Featurestore; @@ -64,7 +68,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Storage Connector Explicit Provenance Resource") -public class StorageConnectorProvenanceResource { +public class StorageConnectorProvenanceResource extends FeaturestoreSubResource { @EJB private JWTHelper jwtHelper; @Inject @@ -73,17 +77,21 @@ public class StorageConnectorProvenanceResource { private ProvExplicitLinksBuilder linksBuilder; @EJB private FeaturestoreStorageConnectorController storageConnectorController; - private Project project; - private Featurestore featureStore; - private String connectorName; + @EJB + private ProjectController projectController; + @EJB + private FeaturestoreController featurestoreController; - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureStore(Featurestore featureStore) { - this.featureStore = featureStore; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } + private String connectorName; public void setConnectorName(String connectorName) { this.connectorName = connectorName; @@ -106,13 +114,15 @@ public Response getLinks( @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) - throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException, ModelRegistryException { + throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, + FeatureStoreMetadataException, IOException, ModelRegistryException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); + Featurestore featurestore = getFeaturestore(project); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); FeaturestoreConnector connector = - storageConnectorController.getConnectorWithName(user, project, featureStore, connectorName); + storageConnectorController.getConnectorWithName(user, project, featurestore, connectorName); ProvExplicitLink provenance = provCtrl.storageConnectorLinks(project, connector, pagination.getUpstreamLvls(), pagination.getDownstreamLvls()); ProvExplicitLinkDTO result = linksBuilder.build(uriInfo, resourceRequest, project, user, provenance); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java index f3181cbb41..1c611e3b2b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/provenance/TrainingDatasetProvenanceResource.java @@ -16,6 +16,7 @@ package io.hops.hopsworks.api.provenance; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.featurestore.trainingdataset.TrainingDatasetSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; @@ -29,7 +30,10 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.dataset.util.DatasetPath; +import io.hops.hopsworks.common.featurestore.FeaturestoreController; +import io.hops.hopsworks.common.featurestore.featureview.FeatureViewController; import io.hops.hopsworks.common.featurestore.trainingdatasets.TrainingDatasetController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitControllerIface; import io.hops.hopsworks.common.provenance.explicit.ProvExplicitLink; import io.hops.hopsworks.common.provenance.ops.dto.ProvLinksDTO; @@ -39,6 +43,7 @@ import io.hops.hopsworks.exceptions.GenericException; import io.hops.hopsworks.exceptions.MetadataException; import io.hops.hopsworks.exceptions.ModelRegistryException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ProvenanceException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -71,7 +76,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Training Dataset Explicit Provenance Resource") -public class TrainingDatasetProvenanceResource { +public class TrainingDatasetProvenanceResource extends TrainingDatasetSubResource { @EJB private TrainingDatasetController trainingDatasetController; @@ -85,27 +90,37 @@ public class TrainingDatasetProvenanceResource { private JWTHelper jwtHelper; @EJB private DatasetHelper datasetHelper; + @EJB + private FeaturestoreController featurestoreController; + @EJB + private FeatureViewController featureViewController; + @EJB + private ProjectController projectController; - private Project project; - - private FeatureView featureView; - private Integer trainingDatasetVersion; - - public void setProject(Project project) { - this.project = project; + @Override + protected ProjectController getProjectController() { + return projectController; } - public void setFeatureView(FeatureView featureView) { - this.featureView = featureView; + @Override + protected FeaturestoreController getFeaturestoreController() { + return featurestoreController; } - - public void setTrainingDatasetVersion(Integer trainingDatasetVersion) { - this.trainingDatasetVersion = trainingDatasetVersion; + @Override + protected FeatureViewController getFeatureViewController() { + return featureViewController; } - - private DatasetPath getTrainingDatasetPath() throws FeaturestoreException, DatasetException { + + @Override + protected TrainingDatasetController getTrainingDatasetController() { + return trainingDatasetController; + } + + private DatasetPath getTrainingDatasetPath(Project project, FeatureView featureView) + throws FeaturestoreException, DatasetException { TrainingDataset trainingDataset - = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, trainingDatasetVersion); + = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, + getTrainingDatasetVersion()); return datasetHelper.getDatasetPath(project, trainingDataset.getTagPath(), DatasetType.DATASET); } @@ -127,12 +142,14 @@ public Response getLinks( @Context HttpServletRequest req, @Context SecurityContext sc) throws GenericException, FeaturestoreException, DatasetException, ServiceException, MetadataException, - FeatureStoreMetadataException, IOException, ModelRegistryException { + FeatureStoreMetadataException, IOException, ModelRegistryException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.PROVENANCE); resourceRequest.setExpansions(explicitProvenanceExpansionBeanParam.getResources()); TrainingDataset td - = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, trainingDatasetVersion); + = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(getFeatureView(project), + getTrainingDatasetVersion()); ProvExplicitLink provenance = provCtrl.trainingDatasetLinks(project, td, pagination.getUpstreamLvls(), pagination.getDownstreamLvls()); ProvExplicitLinkDTO result = linksBuilder.build(uriInfo, resourceRequest, project, user, provenance); @@ -150,12 +167,15 @@ public Response status(@BeanParam ProvUsageBeanParams params, @Context HttpServletRequest req, @Context SecurityContext sc) throws ProvenanceException, GenericException, DatasetException, MetadataException, FeatureStoreMetadataException, - FeaturestoreException { + FeaturestoreException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); + Project project = getProject(); + FeatureView featureView = getFeatureView(project); TrainingDataset trainingDataset - = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, trainingDatasetVersion); + = trainingDatasetController.getTrainingDatasetByFeatureViewAndVersion(featureView, getTrainingDatasetVersion()); String tdProvenanceId = trainingDataset.getName() + "_" + trainingDataset.getVersion(); - ProvArtifactUsageParentDTO status = usageBuilder.buildAccessible(uriInfo, user, getTrainingDatasetPath(), + ProvArtifactUsageParentDTO status = + usageBuilder.buildAccessible(uriInfo, user, getTrainingDatasetPath(project, featureView), tdProvenanceId, params.getUsageType()); return Response.ok().entity(status).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/PythonResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/PythonResource.java index cc510be327..53f5c51c71 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/PythonResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/PythonResource.java @@ -38,12 +38,13 @@ */ package io.hops.hopsworks.api.python; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.python.environment.EnvironmentResource; -import io.hops.hopsworks.common.dao.project.ProjectFacade; -import io.hops.hopsworks.persistence.entity.project.Project; +import io.hops.hopsworks.common.project.ProjectController; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; + import javax.ejb.EJB; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; @@ -54,24 +55,24 @@ @Api(value = "Python") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class PythonResource { +public class PythonResource extends ProjectSubResource { @Inject private EnvironmentResource environmentResource; @EJB - private ProjectFacade projectFacade; - - private Project project; + private ProjectController projectController; - public void setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Python environment sub-resource") @Path("/environments") public EnvironmentResource environment() { - return this.environmentResource.setProject(project); + this.environmentResource.setProjectId(getProjectId()); + return this.environmentResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/conflicts/EnvironmentConflictsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/conflicts/EnvironmentConflictsResource.java index 388fe81167..fe42707ed4 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/conflicts/EnvironmentConflictsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/conflicts/EnvironmentConflictsResource.java @@ -16,11 +16,14 @@ package io.hops.hopsworks.api.python.conflicts; import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.python.environment.EnvironmentSubResource; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.python.environment.EnvironmentController; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.PythonException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.project.Project; @@ -32,6 +35,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.GET; import javax.ws.rs.Produces; @@ -45,24 +49,18 @@ @Api(value = "Python Environment Conflicts Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class EnvironmentConflictsResource { +public class EnvironmentConflictsResource extends EnvironmentSubResource { @EJB private ConflictBuilder conflictBuilder; @EJB private EnvironmentController environmentController; + @EJB + private ProjectController projectController; - private Project project; - private String pythonVersion; - - public EnvironmentConflictsResource setProject(Project project, String pythonVersion) { - this.project = project; - this.pythonVersion = pythonVersion; - return this; - } - - public Project getProject() { - return project; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get conflicts for this environment") @@ -75,12 +73,13 @@ public Project getProject() { allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response get(@BeanParam ConflictBeanParam environmentConflictBeanParam, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) - throws IOException, ServiceDiscoveryException, PythonException { + throws IOException, ServiceDiscoveryException, PythonException, ProjectException { ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.CONFLICTS); resourceRequest.setFilter(environmentConflictBeanParam.getFilter()); - - environmentController.checkCondaEnabled(project, pythonVersion, true); + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); ConflictDTO dto = conflictBuilder.build(uriInfo, resourceRequest, project); return Response.ok().entity(dto).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentResource.java index 713d400bcc..e7a90ab549 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentResource.java @@ -17,10 +17,11 @@ import com.google.common.base.Strings; import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.project.ProjectSubResource; import io.hops.hopsworks.api.project.util.DsPath; import io.hops.hopsworks.api.project.util.PathValidator; import io.hops.hopsworks.api.python.conflicts.EnvironmentConflictsResource; @@ -29,6 +30,7 @@ import io.hops.hopsworks.api.python.library.LibraryResource; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.hdfs.inode.InodeController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.python.environment.EnvironmentController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.ProjectException; @@ -47,6 +49,7 @@ import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -68,7 +71,7 @@ @Api(value = "Python Environments Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class EnvironmentResource { +public class EnvironmentResource extends ProjectSubResource { @EJB private EnvironmentController environmentController; @@ -88,19 +91,16 @@ public class EnvironmentResource { private EnvironmentHistoryResource environmentHistoryResource; @EJB private EnvironmentBuilder environmentBuilder; + @EJB + private ProjectController projectController; - private Project project; - - public EnvironmentResource setProject(Project project) { - this.project = project; - return this; - } - - public Project getProject() { - return project; + @Override + protected ProjectController getProjectController() { + return projectController; } - private ResourceRequest getResourceRequest(EnvironmentExpansionBeanParam expansions) throws PythonException { + private ResourceRequest getResourceRequest(EnvironmentExpansionBeanParam expansions, Project project) + throws PythonException { if (project.getPythonEnvironment() == null) { throw new PythonException(RESTCodes.PythonErrorCode.ANACONDA_ENVIRONMENT_NOT_FOUND, Level.FINE); } @@ -111,9 +111,9 @@ private ResourceRequest getResourceRequest(EnvironmentExpansionBeanParam expansi return resourceRequest; } - private EnvironmentDTO buildEnvDTO(UriInfo uriInfo, EnvironmentExpansionBeanParam expansions, String version) - throws PythonException, IOException, ServiceDiscoveryException { - ResourceRequest resourceRequest = getResourceRequest(expansions); + private EnvironmentDTO buildEnvDTO(UriInfo uriInfo, EnvironmentExpansionBeanParam expansions, Project project, + String version) throws PythonException, IOException, ServiceDiscoveryException{ + ResourceRequest resourceRequest = getResourceRequest(expansions, project); return environmentBuilder.build(uriInfo, resourceRequest, project, version); } @@ -127,8 +127,11 @@ private EnvironmentDTO buildEnvDTO(UriInfo uriInfo, EnvironmentExpansionBeanPara allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getAll(@BeanParam EnvironmentExpansionBeanParam expansions, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException, IOException, ServiceDiscoveryException { - ResourceRequest resourceRequest = getResourceRequest(expansions); + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws PythonException, IOException, ServiceDiscoveryException, ProjectException { + Project project = getProject(); + ResourceRequest resourceRequest = getResourceRequest(expansions, project); EnvironmentDTO dto = environmentBuilder.buildItems(uriInfo, resourceRequest, project); return Response.ok().entity(dto).build(); } @@ -145,12 +148,15 @@ public Response getAll(@BeanParam EnvironmentExpansionBeanParam expansions, public Response get(@PathParam("version") String version, @BeanParam EnvironmentExpansionBeanParam expansions, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException, IOException, ServiceDiscoveryException { + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, IOException, + ServiceDiscoveryException, ProjectException { + Project project = getProject(); if (project.getPythonEnvironment() == null || !project.getPythonEnvironment().getPythonVersion().equals(version)) { throw new PythonException(RESTCodes.PythonErrorCode.ANACONDA_ENVIRONMENT_NOT_FOUND, Level.FINE); } - EnvironmentDTO dto = buildEnvDTO(uriInfo, expansions, version); + EnvironmentDTO dto = buildEnvDTO(uriInfo, expansions, project, version); return Response.ok().entity(dto).build(); } @@ -167,18 +173,20 @@ public Response get(@PathParam("version") String version, public Response post(@PathParam("version") String version, @QueryParam("action") EnvironmentDTO.Operation action, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException, ServiceException, IOException, - ServiceDiscoveryException { + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws PythonException, ServiceException, IOException, ServiceDiscoveryException, ProjectException { EnvironmentDTO dto; Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); switch ((action != null) ? action : EnvironmentDTO.Operation.CREATE) { case EXPORT: environmentController.exportEnv(project, user, environmentController.getDockerImageEnvironmentFile(project)); - dto = buildEnvDTO(uriInfo, null, version); + dto = buildEnvDTO(uriInfo, null, project, version); return Response.ok().entity(dto).build(); case CREATE: environmentController.createEnv(project, user); - dto = buildEnvDTO(uriInfo,null, version); + dto = buildEnvDTO(uriInfo,null, project, version); return Response.created(dto.getHref()).entity(dto).build(); default: throw new WebApplicationException(RESTCodes.ServiceErrorCode.OPERATION_NOT_SUPPORTED.getMessage(), @@ -196,13 +204,15 @@ public Response post(@PathParam("version") String version, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response postImport(EnvironmentImportDTO environmentImportDTO, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException, ServiceException, DatasetException, - IOException, ProjectException, ServiceDiscoveryException { + @Context SecurityContext sc) + throws PythonException, ServiceException, DatasetException, IOException, ProjectException, + ServiceDiscoveryException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); String version = environmentController.createProjectDockerImageFromImport( - getYmlPath(environmentImportDTO.getPath()), + getYmlPath(environmentImportDTO.getPath(), project), environmentImportDTO.getInstallJupyter(), user, project); - EnvironmentDTO dto = buildEnvDTO(uriInfo,null, version); + EnvironmentDTO dto = buildEnvDTO(uriInfo,null, project, version); return Response.created(dto.getHref()).entity(dto).build(); } @@ -217,16 +227,18 @@ public Response postImport(EnvironmentImportDTO environmentImportDTO, @ApiKeyRequired(acceptedScopes = {ApiScope.PYTHON_LIBRARIES}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response delete(@PathParam("version") String version, - @Context SecurityContext sc) throws PythonException { - environmentController.removeEnvironment(project); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, ProjectException { + environmentController.removeEnvironment(getProject()); return Response.noContent().build(); } - private String getYmlPath(String path) throws DatasetException, ProjectException, UnsupportedEncodingException { + private String getYmlPath(String path, Project project) throws DatasetException, ProjectException, + UnsupportedEncodingException { if(Strings.isNullOrEmpty(path)) { return null; } - DsPath ymlPath = pathValidator.validatePath(this.project, path); + DsPath ymlPath = pathValidator.validatePath(project, path); ymlPath.validatePathExists(inodeController, false); org.apache.hadoop.fs.Path fullPath = ymlPath.getFullPath(); return fullPath.toString(); @@ -236,27 +248,35 @@ private String getYmlPath(String path) throws DatasetException, ProjectException @Path("{version}/libraries") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) public LibraryResource libraries(@PathParam("version") String version) { - return this.libraryResource.setProjectAndVersion(project, version); + this.libraryResource.setProjectId(getProjectId()); + this.libraryResource.setPythonVersion(version); + return this.libraryResource; } - + @ApiOperation(value = "Python opStatus sub-resource", tags = {"EnvironmentCommandsResource"}) @Path("{version}/commands") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) public EnvironmentCommandsResource commands(@PathParam("version") String version) { - return this.environmentCommandsResource.setProject(project, version); + this.environmentCommandsResource.setProjectId(getProjectId()); + this.environmentCommandsResource.setPythonVersion(version); + return this.environmentCommandsResource; } @ApiOperation(value = "Python conflicts sub-resource", tags = {"EnvironmentConflictsResource"}) @Path("{version}/conflicts") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) public EnvironmentConflictsResource conflicts(@PathParam("version") String version) { - return this.environmentConflictsResource.setProject(project, version); + this.environmentConflictsResource.setProjectId(getProjectId()); + this.environmentConflictsResource.setPythonVersion(version); + return this.environmentConflictsResource; } - + @ApiOperation(value = "Environment history subresource", tags = {"EnvironmentHistoryResource"}) @Path("{version}/history") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) public EnvironmentHistoryResource environmentHistory(@PathParam("version") String version) { - return this.environmentHistoryResource.init(project, version); + this.environmentHistoryResource.setProjectId(getProjectId()); + this.environmentHistoryResource.setPythonVersion(version); + return this.environmentHistoryResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentSubResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentSubResource.java new file mode 100644 index 0000000000..403892b24a --- /dev/null +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/EnvironmentSubResource.java @@ -0,0 +1,30 @@ +/* + * This file is part of Hopsworks + * Copyright (C) 2024, Hopsworks AB. All rights reserved + * + * Hopsworks is free software: you can redistribute it and/or modify it under the terms of + * the GNU Affero General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License along with this program. + * If not, see . + */ +package io.hops.hopsworks.api.python.environment; + +import io.hops.hopsworks.api.project.ProjectSubResource; + +public abstract class EnvironmentSubResource extends ProjectSubResource { + private String pythonVersion; + + public String getPythonVersion() { + return pythonVersion; + } + + public void setPythonVersion(String pythonVersion) { + this.pythonVersion = pythonVersion; + } +} diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/EnvironmentCommandsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/EnvironmentCommandsResource.java index 26b1d47695..773f3651b8 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/EnvironmentCommandsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/EnvironmentCommandsResource.java @@ -15,17 +15,20 @@ */ package io.hops.hopsworks.api.python.environment.command; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.python.command.CommandBeanParam; import io.hops.hopsworks.api.python.command.CommandBuilder; import io.hops.hopsworks.api.python.command.CommandDTO; +import io.hops.hopsworks.api.python.environment.EnvironmentSubResource; import io.hops.hopsworks.api.python.environment.command.custom.EnvironmentCustomCommandsResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.python.commands.CommandsController; import io.hops.hopsworks.common.python.environment.EnvironmentController; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.PythonException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.project.Project; @@ -55,7 +58,7 @@ @Api(value = "Python Environment Commands Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class EnvironmentCommandsResource { +public class EnvironmentCommandsResource extends EnvironmentSubResource { @EJB private CommandsController commandsController; @@ -65,16 +68,12 @@ public class EnvironmentCommandsResource { private EnvironmentController environmentController; @Inject private EnvironmentCustomCommandsResource environmentCustomCommandsResource; + @EJB + private ProjectController projectController; - private Project project; - private String pythonVersion; - public EnvironmentCommandsResource setProject(Project project, String pythonVersion) { - this.project = project; - this.pythonVersion = pythonVersion; - return this; - } - public Project getProject() { - return project; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get commands for this environment") @@ -87,9 +86,11 @@ public Project getProject() { allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response get(@BeanParam Pagination pagination, @BeanParam CommandBeanParam environmentsCommandBeanParam, + @Context HttpServletRequest req, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, false); + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), false); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMANDS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); @@ -110,8 +111,10 @@ public Response get(@BeanParam Pagination pagination, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getByName(@PathParam("commandId") Integer commandId, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, false); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), false); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMANDS); CommandDTO dto = commandBuilder.build(uriInfo, resourceRequest, project, commandId); return Response.ok().entity(dto).build(); @@ -125,8 +128,9 @@ public Response getByName(@PathParam("commandId") Integer commandId, @ApiKeyRequired(acceptedScopes = {ApiScope.PYTHON_LIBRARIES}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteAll(@Context SecurityContext sc, - @Context HttpServletRequest req) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, false); + @Context HttpServletRequest req) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), false); commandsController.deleteCommands(project); return Response.noContent().build(); } @@ -140,8 +144,9 @@ public Response deleteAll(@Context SecurityContext sc, @ApiKeyRequired(acceptedScopes = {ApiScope.PYTHON_LIBRARIES}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response deleteCommand(@Context SecurityContext sc, @PathParam("commandId") Integer commandId, - @Context HttpServletRequest req) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, false); + @Context HttpServletRequest req) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), false); commandsController.deleteCommand(project,commandId); return Response.noContent().build(); } @@ -154,8 +159,10 @@ public Response deleteCommand(@Context SecurityContext sc, @PathParam("commandId @ApiKeyRequired(acceptedScopes = {ApiScope.PYTHON_LIBRARIES}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response update(@Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, false); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), false); commandsController.retryFailedCondaEnvOps(project); return Response.noContent().build(); } @@ -164,6 +171,8 @@ public Response update(@Context UriInfo uriInfo, @Path("custom") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) public EnvironmentCustomCommandsResource commands() { - return this.environmentCustomCommandsResource.setProject(project, pythonVersion); + this.environmentCustomCommandsResource.setProjectId(getProjectId()); + this.environmentCustomCommandsResource.setPythonVersion(getPythonVersion()); + return this.environmentCustomCommandsResource; } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/custom/EnvironmentCustomCommandsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/custom/EnvironmentCustomCommandsResource.java index db1dcbb79d..cc417e9950 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/custom/EnvironmentCustomCommandsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/command/custom/EnvironmentCustomCommandsResource.java @@ -15,17 +15,20 @@ */ package io.hops.hopsworks.api.python.environment.command.custom; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.api.python.command.CommandBuilder; import io.hops.hopsworks.api.python.command.CommandDTO; +import io.hops.hopsworks.api.python.environment.EnvironmentSubResource; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.python.commands.custom.CustomCommandsController; import io.hops.hopsworks.common.python.commands.custom.CustomCommandsSettings; import io.hops.hopsworks.common.python.environment.EnvironmentController; import io.hops.hopsworks.exceptions.DatasetException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.PythonException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -44,18 +47,17 @@ import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Produces; -import javax.ws.rs.core.UriInfo; import javax.ws.rs.core.Context; -import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; - +import javax.ws.rs.core.SecurityContext; +import javax.ws.rs.core.UriInfo; import java.io.IOException; @Api(value = "Python Environment Custom Commands Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class EnvironmentCustomCommandsResource { +public class EnvironmentCustomCommandsResource extends EnvironmentSubResource { @EJB private EnvironmentController environmentController; @EJB @@ -64,14 +66,12 @@ public class EnvironmentCustomCommandsResource { private JWTHelper jwtHelper; @EJB private CommandBuilder commandBuilder; + @EJB + private ProjectController projectController; - private Project project; - private String pythonVersion; - - public EnvironmentCustomCommandsResource setProject(Project project, String pythonVersion) { - this.project = project; - this.pythonVersion = pythonVersion; - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Build the environment with custom bash commands") @@ -86,13 +86,14 @@ public EnvironmentCustomCommandsResource setProject(Project project, String pyth public Response build(CustomCommandsSettings commandsSettings, @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) - throws PythonException, IOException, DatasetException, ServiceException { - environmentController.checkCondaEnabled(project, pythonVersion, false); + throws PythonException, IOException, DatasetException, ServiceException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), false); Users user = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMANDS); environmentController.checkCondaEnvExists(project, user); CondaCommands cc = customCommandsController.buildEnvWithCustomCommands(project, user, commandsSettings, - pythonVersion); + getPythonVersion()); CommandDTO dto = commandBuilder.build(uriInfo, resourceRequest, cc); return Response.ok().entity(dto).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/history/EnvironmentHistoryResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/history/EnvironmentHistoryResource.java index 7f9007b98e..b8277acb06 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/history/EnvironmentHistoryResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/environment/history/EnvironmentHistoryResource.java @@ -15,15 +15,17 @@ */ package io.hops.hopsworks.api.python.environment.history; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.python.environment.EnvironmentSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; @@ -48,24 +50,18 @@ @Api(value = "Python Environments History Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class EnvironmentHistoryResource { +public class EnvironmentHistoryResource extends EnvironmentSubResource { @EJB private EnvironmentHistoryBuilder environmentHistoryBuilder; @EJB private JWTHelper jwtHelper; + @EJB + private ProjectController projectController; - private Project project; - private String version; - - public EnvironmentHistoryResource init(Project project, String version) { - this.project = project; - this.version = version; - return this; - } - - public Project getProject() { - return project; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get the environment history for this project", response = EnvironmentHistoryDTO.class) @@ -81,7 +77,7 @@ public Response getAll(@PathParam("version") String version, @Context HttpServletRequest req, @Context SecurityContext sc, @BeanParam EnvironmentHistoryBeanParam environmentHistoryBeanParam, - @BeanParam Pagination pagination) throws ServiceException { + @BeanParam Pagination pagination) throws ServiceException, ProjectException { Users hopsworksUser = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ENVIRONMENT_HISTORY); resourceRequest.setExpansions(environmentHistoryBeanParam.getExpansions().getResources()); @@ -90,7 +86,7 @@ public Response getAll(@PathParam("version") String version, resourceRequest.setFilter(environmentHistoryBeanParam.getFilter()); resourceRequest.setSort(environmentHistoryBeanParam.getSortBySet()); EnvironmentHistoryDTO dto = - environmentHistoryBuilder.build(uriInfo, resourceRequest, project, hopsworksUser, version); + environmentHistoryBuilder.build(uriInfo, resourceRequest, getProject(), hopsworksUser, version); return Response.ok().entity(dto).build(); } @@ -107,12 +103,12 @@ public Response getEnvironmentDelta(@PathParam("buildId") Integer buildId, @Context HttpServletRequest req, @Context UriInfo uriInfo, @BeanParam EnvironmentHistoryBeanParam environmentHistoryBeanParam) - throws ServiceException { + throws ServiceException, ProjectException { Users hopsworksUser = jwtHelper.getUserPrincipal(sc); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.ENVIRONMENT_HISTORY); resourceRequest.setExpansions(environmentHistoryBeanParam.getExpansions().getResources()); EnvironmentHistoryDTO dto = - environmentHistoryBuilder.build(uriInfo, resourceRequest, project, hopsworksUser, buildId); + environmentHistoryBuilder.build(uriInfo, resourceRequest, getProject(), hopsworksUser, buildId); return Response.ok().entity(dto).build(); } } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/LibraryResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/LibraryResource.java index 4b045daa1f..bf0192cd79 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/LibraryResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/LibraryResource.java @@ -16,10 +16,11 @@ package io.hops.hopsworks.api.python.library; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.jwt.JWTHelper; +import io.hops.hopsworks.api.python.environment.EnvironmentSubResource; import io.hops.hopsworks.api.python.library.command.LibraryCommandsResource; import io.hops.hopsworks.api.python.library.search.LibrarySearchBuilder; import io.hops.hopsworks.api.python.library.search.LibrarySearchDTO; @@ -27,6 +28,7 @@ import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.dataset.DatasetController; import io.hops.hopsworks.common.hdfs.HdfsUsersController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.python.commands.CommandsController; import io.hops.hopsworks.common.python.environment.EnvironmentController; import io.hops.hopsworks.common.python.library.LibraryController; @@ -35,6 +37,7 @@ import io.hops.hopsworks.common.util.Settings; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.GenericException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.PythonException; import io.hops.hopsworks.exceptions.ServiceException; import io.hops.hopsworks.jwt.annotation.JWTRequired; @@ -74,7 +77,7 @@ @Api(value = "Python Environment Library Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class LibraryResource { +public class LibraryResource extends EnvironmentSubResource { @EJB private LibraryController libraryController; @@ -96,21 +99,15 @@ public class LibraryResource { private EnvironmentController environmentController; @EJB private JWTHelper jwtHelper; + @EJB + private ProjectController projectController; private static final Pattern VALIDATION_PATTERN = Pattern.compile("[a-zA-Z0-9_\\-\\.\\]\\[]+"); private static final Pattern CHANNEL_PATTERN = Pattern.compile("[a-zA-Z0-9_\\-:/~?&\\.]+"); - - private Project project; - private String pythonVersion; - - public LibraryResource setProjectAndVersion(Project project, String pythonVersion) { - this.project = project; - this.pythonVersion = pythonVersion; - return this; - } - public Project getProject() { - return project; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get the python libraries installed in this environment", response = LibraryDTO.class) @@ -124,8 +121,10 @@ public Project getProject() { public Response get(@BeanParam Pagination pagination, @BeanParam LibrariesBeanParam librariesBeanParam, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, true); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.LIBRARIES); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); @@ -150,9 +149,10 @@ public Response get(@BeanParam Pagination pagination, public Response getByName(@PathParam("library") String library, @BeanParam LibraryExpansionBeanParam expansions, @Context UriInfo uriInfo, @Context HttpServletRequest req, - @Context SecurityContext sc) throws PythonException { + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); validatePattern(library); - environmentController.checkCondaEnabled(project, pythonVersion, true); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); PythonDep dep = libraryController.getPythonDep(library, project); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.LIBRARIES); if (expansions != null) { @@ -174,10 +174,11 @@ public Response getByName(@PathParam("library") String library, @BeanParam Libra public Response uninstall(@Context SecurityContext sc, @Context HttpServletRequest req, @PathParam("library") String library) - throws ServiceException, GenericException, PythonException { + throws ServiceException, GenericException, PythonException, ProjectException { + Project project = getProject(); validatePattern(library); Users user = jwtHelper.getUserPrincipal(sc); - environmentController.checkCondaEnabled(project, pythonVersion, true); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); if (settings.getImmutablePythonLibraryNames().contains(library)) { throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_DEP_REMOVE_FORBIDDEN, Level.INFO, @@ -205,10 +206,11 @@ public Response install(LibrarySpecification librarySpecification, @Context UriInfo uriInfo, @Context HttpServletRequest req, @Context SecurityContext sc) - throws ServiceException, GenericException, PythonException, DatasetException { + throws ServiceException, GenericException, PythonException, DatasetException, ProjectException { Users user = jwtHelper.getUserPrincipal(sc); - environmentController.checkCondaEnabled(project, pythonVersion, true); + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); PackageSource packageSource = librarySpecification.getPackageSource(); if (packageSource == null) { @@ -229,7 +231,7 @@ public Response install(LibrarySpecification librarySpecification, case WHEEL: case REQUIREMENTS_TXT: case ENVIRONMENT_YAML: - validateBundledDependency(user, librarySpecification); + validateBundledDependency(user, librarySpecification, project); break; case GIT: validateGitURL(librarySpecification.getDependencyUrl()); @@ -263,10 +265,12 @@ public Response search(@PathParam("search") String search, @QueryParam("query") String query, @QueryParam("channel") String channel, @Context UriInfo uriInfo, + @Context HttpServletRequest req, @Context SecurityContext sc) - throws ServiceException, PythonException { + throws ServiceException, PythonException, ProjectException { + Project project = getProject(); validatePattern(query); - environmentController.checkCondaEnabled(project, pythonVersion, true); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); LibrarySearchDTO librarySearchDTO; PackageSource packageSource = PackageSource.fromString(search); switch (packageSource) { @@ -287,7 +291,9 @@ public Response search(@PathParam("search") String search, @Path("{library}/commands") @AllowedProjectRoles({AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST}) public LibraryCommandsResource libraryCommandsResource() { - return this.libraryCommandsResource.setProject(project, pythonVersion); + this.libraryCommandsResource.setProjectId(getProjectId()); + this.libraryCommandsResource.setPythonVersion(getPythonVersion()); + return this.libraryCommandsResource; } private void validatePattern(String element) throws IllegalArgumentException { @@ -325,7 +331,7 @@ private void validateLibrary(LibrarySpecification librarySpecification, String l } } - private void validateBundledDependency(Users user, LibrarySpecification librarySpecification) + private void validateBundledDependency(Users user, LibrarySpecification librarySpecification, Project project) throws DatasetException, PythonException { String dependencyUrl = librarySpecification.getDependencyUrl(); diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/command/LibraryCommandsResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/command/LibraryCommandsResource.java index 27fc1f72a1..9dd3e202dc 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/command/LibraryCommandsResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/python/library/command/LibraryCommandsResource.java @@ -15,16 +15,19 @@ */ package io.hops.hopsworks.api.python.library.command; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.python.command.CommandBeanParam; import io.hops.hopsworks.api.python.command.CommandBuilder; import io.hops.hopsworks.api.python.command.CommandDTO; +import io.hops.hopsworks.api.python.environment.EnvironmentSubResource; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.python.commands.CommandsController; import io.hops.hopsworks.common.python.environment.EnvironmentController; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.PythonException; import io.hops.hopsworks.jwt.annotation.JWTRequired; import io.hops.hopsworks.persistence.entity.project.Project; @@ -36,6 +39,7 @@ import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; import javax.enterprise.context.RequestScoped; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BeanParam; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -52,7 +56,7 @@ @Api(value = "Python Environment Library Commands Resource") @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class LibraryCommandsResource { +public class LibraryCommandsResource extends EnvironmentSubResource { @EJB private CommandsController commandsController; @@ -60,14 +64,12 @@ public class LibraryCommandsResource { private CommandBuilder commandBuilder; @EJB private EnvironmentController environmentController; + @EJB + private ProjectController projectController; - private Project project; - private String pythonVersion; - - public LibraryCommandsResource setProject(Project project, String pythonVersion) { - this.project = project; - this.pythonVersion = pythonVersion; - return this; + @Override + protected ProjectController getProjectController() { + return projectController; } @ApiOperation(value = "Get all commands for this library", response = CommandDTO.class) @@ -81,8 +83,10 @@ public LibraryCommandsResource setProject(Project project, String pythonVersion) public Response get(@PathParam("library") String library, @BeanParam Pagination pagination, @BeanParam CommandBeanParam libraryCommandBeanParam, - @Context UriInfo uriInfo, @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, true); + @Context HttpServletRequest req, + @Context UriInfo uriInfo, @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMANDS); resourceRequest.setOffset(pagination.getOffset()); resourceRequest.setLimit(pagination.getLimit()); @@ -102,9 +106,11 @@ public Response get(@PathParam("library") String library, @ApiKeyRequired(acceptedScopes = {ApiScope.PYTHON_LIBRARIES}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response getByName(@PathParam("library") String library, @PathParam("commandId") Integer commandId, + @Context HttpServletRequest req, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, true); + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.COMMANDS); CommandDTO dto = commandBuilder.build(uriInfo, resourceRequest, project, library, commandId); return Response.ok().entity(dto).build(); @@ -119,8 +125,10 @@ public Response getByName(@PathParam("library") String library, @PathParam("comm allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response update(@PathParam("library") String library, @Context UriInfo uriInfo, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, true); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); commandsController.retryFailedCondaLibraryOps(project, library); return Response.noContent().build(); } @@ -133,8 +141,10 @@ public Response update(@PathParam("library") String library, @ApiKeyRequired(acceptedScopes = {ApiScope.PYTHON_LIBRARIES}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER", "HOPS_SERVICE_USER"}) public Response delete(@PathParam("library") String library, - @Context SecurityContext sc) throws PythonException { - environmentController.checkCondaEnabled(project, pythonVersion, true); + @Context HttpServletRequest req, + @Context SecurityContext sc) throws PythonException, ProjectException { + Project project = getProject(); + environmentController.checkCondaEnabled(project, getPythonVersion(), true); commandsController.deleteCommands(project, library); return Response.noContent().build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/ServingService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/ServingService.java index b57f0b0b1b..d9a4e13a9b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/ServingService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/ServingService.java @@ -17,14 +17,15 @@ package io.hops.hopsworks.api.serving; import com.google.common.base.Strings; -import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.filter.NoCacheResponse; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; import io.hops.hopsworks.api.jwt.JWTHelper; -import io.hops.hopsworks.common.dao.project.ProjectFacade; +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.security.QuotaEnforcementException; import io.hops.hopsworks.common.security.QuotasEnforcement; import io.hops.hopsworks.common.serving.ServingController; @@ -82,7 +83,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "TensorFlow Serving service", description = "Manage Serving instances") -public class ServingService { +public class ServingService extends ProjectSubResource { @Inject private ServingController servingController; @@ -91,18 +92,18 @@ public class ServingService { @EJB private NoCacheResponse noCacheResponse; @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private JWTHelper jWTHelper; @EJB private QuotasEnforcement quotasEnforcement; - private Project project; public ServingService(){ } - - public void setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); + + @Override + protected ProjectController getProjectController() { + return projectController; } @GET @@ -117,13 +118,14 @@ public void setProjectId(Integer projectId) { response = ServingView.class, responseContainer = "List") public Response getAll( - @QueryParam("model") String modelName, - @QueryParam("modelVersion") Integer modelVersion, - @QueryParam("status") ServingStatusEnum status, - @QueryParam("name") String servingName, - @Context SecurityContext sc) - throws ServingException, KafkaException, CryptoPasswordNotFoundException { - + @QueryParam("model") String modelName, + @QueryParam("modelVersion") Integer modelVersion, + @QueryParam("status") ServingStatusEnum status, + @QueryParam("name") String servingName, + @Context HttpServletRequest req, + @Context SecurityContext sc) + throws ServingException, KafkaException, CryptoPasswordNotFoundException, ProjectException { + Project project = getProject(); // if filter by name, return a single serving if (!Strings.isNullOrEmpty(servingName)) { ServingWrapper servingWrapper = servingController.get(project, servingName); @@ -169,14 +171,15 @@ public Response getAll( @ApiOperation(value = "Get info about a serving instance for the project", response = ServingView.class) public Response get( @Context SecurityContext sc, + @Context HttpServletRequest req, @ApiParam(value = "Id of the Serving instance", required = true) @PathParam("servingId") Integer servingId) - throws ServingException, KafkaException, CryptoPasswordNotFoundException { + throws ServingException, KafkaException, CryptoPasswordNotFoundException, ProjectException { if (servingId == null) { throw new IllegalArgumentException("servingId was not provided"); } - ServingWrapper servingWrapper = servingController.get(project, servingId); + ServingWrapper servingWrapper = servingController.get(getProject(), servingId); if (servingWrapper == null) { throw new ServingException(RESTCodes.ServingErrorCode.INSTANCE_NOT_FOUND, Level.FINE); } @@ -201,12 +204,12 @@ public Response delete( @Context HttpServletRequest req, @Context SecurityContext sc, @ApiParam(value = "Id of the serving instance", required = true) @PathParam("servingId") Integer servingId) - throws ServingException { + throws ServingException, ProjectException { if (servingId == null) { throw new IllegalArgumentException("servingId was not provided"); } - servingController.delete(project, servingId); + servingController.delete(getProject(), servingId); return Response.ok().build(); } @@ -229,6 +232,7 @@ public Response put( InterruptedException, ExecutionException, UnsupportedEncodingException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); if (serving == null) { throw new IllegalArgumentException("serving was not provided"); } @@ -267,9 +271,10 @@ public Response startOrStop( @ApiParam(value = "ID of the Serving instance to start/stop", required = true) @PathParam("servingId") Integer servingId, @ApiParam(value = "Action", required = true) @QueryParam("action") ServingCommands servingCommand) - throws ServingException { + throws ServingException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); + Project project = getProject(); if (servingId == null) { throw new IllegalArgumentException("servingId was not provided"); } @@ -311,7 +316,7 @@ public Response get( @Context HttpServletRequest req, @ApiParam(value = "Id of the Serving instance", required = true) @PathParam("servingId") Integer servingId) - throws ServingException, KafkaException, CryptoPasswordNotFoundException { + throws ServingException, KafkaException, CryptoPasswordNotFoundException, ProjectException { if (servingId == null) { throw new IllegalArgumentException("servingId was not provided"); @@ -320,7 +325,7 @@ public Response get( throw new IllegalArgumentException("component not valid, possible values are predictor or transformer"); } - List logs = servingController.getLogs(project, servingId, component, tailingLines); + List logs = servingController.getLogs(getProject(), servingId, component, tailingLines); GenericEntity> logsEntity = new GenericEntity>(logs){}; return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK) .entity(logsEntity) diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/inference/InferenceResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/inference/InferenceResource.java index f35f2892f7..f16177421a 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/inference/InferenceResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/serving/inference/InferenceResource.java @@ -17,20 +17,21 @@ package io.hops.hopsworks.api.serving.inference; import com.google.common.base.Strings; +import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; -import io.hops.hopsworks.api.auth.key.ApiKeyRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlagRequired; import io.hops.hopsworks.api.filter.featureFlags.FeatureFlags; -import io.hops.hopsworks.common.dao.project.ProjectFacade; +import io.hops.hopsworks.api.project.ProjectSubResource; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.serving.inference.InferenceController; import io.hops.hopsworks.common.serving.inference.InferenceEndpoint; import io.hops.hopsworks.common.serving.inference.InferenceVerb; import io.hops.hopsworks.exceptions.ApiKeyException; import io.hops.hopsworks.exceptions.InferenceException; +import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.exceptions.ServingException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -63,19 +64,18 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) @Api(value = "Model inference service", description = "Handles inference requests for ML models") -public class InferenceResource { +public class InferenceResource extends ProjectSubResource { @EJB private InferenceController inferenceController; @EJB - private ProjectFacade projectFacade; - - private Project project; + private ProjectController projectController; private final static Logger logger = Logger.getLogger(InferenceResource.class.getName()); - public void setProjectId(Integer projectId) { - this.project = projectFacade.find(projectId); + @Override + protected ProjectController getProjectController() { + return projectController; } @POST @@ -93,15 +93,17 @@ public Response infer( @ApiParam(value = "Name of the model to query", required = true) @PathParam("modelName") String modelName, @ApiParam(value = "Version of the model to query") @PathParam("version") String modelVersion, @ApiParam(value = "Type of query") @PathParam("verb") InferenceVerb verb, + @Context HttpServletRequest req, @Context SecurityContext sc, - @Context HttpHeaders httpHeaders, String inferenceRequestJson) throws InferenceException, ApiKeyException { + @Context HttpHeaders httpHeaders, String inferenceRequestJson) + throws InferenceException, ApiKeyException, ProjectException { Integer version = null; if (!Strings.isNullOrEmpty(modelVersion)) { version = Integer.valueOf(modelVersion.split("/")[2]); } String authHeader = httpHeaders.getRequestHeader(HttpHeaders.AUTHORIZATION).get(0); - String inferenceResult = inferenceController.infer(project, sc.getUserPrincipal().getName(), modelName, version, - verb, inferenceRequestJson, authHeader); + String inferenceResult = inferenceController.infer(getProject(), sc.getUserPrincipal().getName(), modelName, + version, verb, inferenceRequestJson, authHeader); return Response.ok().entity(inferenceResult).build(); } diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/tags/TagSchemasResource.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/tags/TagSchemasResource.java index 30cda387f4..95417ac11b 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/tags/TagSchemasResource.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/tags/TagSchemasResource.java @@ -15,8 +15,8 @@ */ package io.hops.hopsworks.api.tags; -import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.util.Pagination; import io.hops.hopsworks.common.api.ResourceRequest; import io.hops.hopsworks.common.tags.SchemaDTO; diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/DownloadService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/DownloadService.java index 9934748d08..e5c3f2f759 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/DownloadService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/DownloadService.java @@ -40,16 +40,17 @@ package io.hops.hopsworks.api.util; import com.auth0.jwt.interfaces.DecodedJWT; -import io.hops.hopsworks.api.filter.JWTNotRequired; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; -import io.hops.hopsworks.common.dataset.util.DatasetHelper; -import io.hops.hopsworks.common.dataset.util.DatasetPath; +import io.hops.hopsworks.api.dataset.DatasetSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; +import io.hops.hopsworks.api.filter.JWTNotRequired; import io.hops.hopsworks.api.jwt.JWTHelper; import io.hops.hopsworks.common.constants.message.ResponseMessages; import io.hops.hopsworks.common.dao.user.UserFacade; import io.hops.hopsworks.common.dataset.DatasetController; +import io.hops.hopsworks.common.dataset.util.DatasetHelper; +import io.hops.hopsworks.common.dataset.util.DatasetPath; import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps; import io.hops.hopsworks.common.hdfs.DistributedFsService; import io.hops.hopsworks.common.hdfs.HdfsUsersController; @@ -72,6 +73,8 @@ import io.swagger.annotations.ApiOperation; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.javatuples.Pair; import javax.ejb.EJB; import javax.ejb.TransactionAttribute; @@ -91,12 +94,9 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.hadoop.fs.permission.FsPermission; -import org.javatuples.Pair; - @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class DownloadService { +public class DownloadService extends DatasetSubResource { private static final Logger LOGGER = Logger.getLogger(DownloadService.class.getName()); @@ -122,23 +122,9 @@ public class DownloadService { public DownloadService() { } - private Integer projectId; - private String projectName; - - public void setProjectId(Integer projectId) { - this.projectId = projectId; - } - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - private Project getProject() throws ProjectException { - if (this.projectId != null) { - return projectController.findProjectById(this.projectId); - } else if (this.projectName != null) { - return projectController.findProjectByName(this.projectName); - } - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE); + @Override + protected ProjectController getProjectController() { + return projectController; } @GET diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/UploadService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/UploadService.java index 7ac1a61473..391e3f0728 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/UploadService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/util/UploadService.java @@ -39,19 +39,19 @@ package io.hops.hopsworks.api.util; import io.hops.hopsworks.api.auth.key.ApiKeyRequired; +import io.hops.hopsworks.api.dataset.DatasetSubResource; import io.hops.hopsworks.api.filter.AllowedProjectRoles; import io.hops.hopsworks.api.filter.Audience; import io.hops.hopsworks.api.jwt.JWTHelper; -import io.hops.hopsworks.common.dao.project.ProjectFacade; import io.hops.hopsworks.common.dataset.util.DatasetHelper; import io.hops.hopsworks.common.dataset.util.DatasetPath; import io.hops.hopsworks.common.hdfs.HdfsUsersController; +import io.hops.hopsworks.common.project.ProjectController; import io.hops.hopsworks.common.upload.FlowInfo; import io.hops.hopsworks.common.upload.UploadController; import io.hops.hopsworks.exceptions.DatasetException; import io.hops.hopsworks.exceptions.ProjectException; import io.hops.hopsworks.jwt.annotation.JWTRequired; -import io.hops.hopsworks.persistence.entity.dataset.DatasetType; import io.hops.hopsworks.persistence.entity.project.Project; import io.hops.hopsworks.persistence.entity.user.Users; import io.hops.hopsworks.persistence.entity.user.security.apiKey.ApiScope; @@ -80,7 +80,7 @@ @RequestScoped @TransactionAttribute(TransactionAttributeType.NEVER) -public class UploadService { +public class UploadService extends DatasetSubResource { private static final Logger LOGGER = Logger.getLogger(UploadService.class.getName()); @@ -91,58 +91,29 @@ public class UploadService { @EJB private HdfsUsersController hdfsUsersBean; @EJB - private ProjectFacade projectFacade; + private ProjectController projectController; @EJB private UploadController uploadController; private String path; - private DatasetType datasetType; private String username; - - private Integer projectId; - private String projectName; - - private Project getProjectById() throws ProjectException { - Project project = projectFacade.find(this.projectId); - if (project == null) { - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + projectId); - } - return project; - } - - private Project getProjectByName() throws ProjectException { - Project project = projectFacade.findByName(this.projectName); - if (project == null) { - throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectName: " + - projectName); - } - return project; - } - - private Project getProject() throws ProjectException { - return this.projectId != null ? getProjectById() : getProjectByName(); - } public UploadService() { } - - public void setParams(Integer projectId, String path, DatasetType datasetType) { - this.projectId = projectId; + public void setPath(String path) { this.path = path; - this.datasetType = datasetType; } - - public void setParams(String projectName, String path, DatasetType datasetType) { - this.projectName = projectName; - this.path = path; - this.datasetType = datasetType; + + @Override + protected ProjectController getProjectController() { + return projectController; } - + private void configureUploader(SecurityContext sc) throws DatasetException, ProjectException { Users user = jWTHelper.getUserPrincipal(sc); - Project project =this.getProject(); + Project project = getProject(); this.username = hdfsUsersBean.getHdfsUserName(project, user); - DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, this.datasetType); + DatasetPath datasetPath = datasetHelper.getDatasetPath(project, path, getDatasetType()); this.path = datasetPath.getFullPath().toString(); }