[HOPSWORKS-1847] Online feature store storage connectors should not b…

…e stored in the database (#691)

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/FeaturestoreService.java

@@ -37,7 +37,6 @@
 import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupController;
 import io.hops.hopsworks.common.featurestore.featuregroup.FeaturegroupDTO;
 import io.hops.hopsworks.common.featurestore.importjob.FeaturegroupImportJobDTO;
-import io.hops.hopsworks.common.featurestore.online.OnlineFeaturestoreController;
 import io.hops.hopsworks.common.featurestore.settings.FeaturestoreClientSettingsDTO;
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorController;
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO;
@@ -108,8 +107,6 @@ public class FeaturestoreService {
   private JWTHelper jWTHelper;
   @EJB
   private Settings settings;
-  @EJB
-  private OnlineFeaturestoreController onlineFeaturestoreController;
   @Inject
   private FeaturegroupService featuregroupService;
   @Inject
@@ -278,20 +275,6 @@ public Response getFeaturestoreId(@Context SecurityContext sc, @PathParam("featu
     FeaturestoreJdbcConnectorDTO onlineFeaturestoreConnector = null;
     FeaturestoreClientSettingsDTO featurestoreClientSettingsDTO = new FeaturestoreClientSettingsDTO();
     featurestoreClientSettingsDTO.setOnlineFeaturestoreEnabled(settings.isOnlineFeaturestore());
-    if(settings.isOnlineFeaturestore()
-      && onlineFeaturestoreController.checkIfDatabaseExists(
-          onlineFeaturestoreController.getOnlineFeaturestoreDbName(featurestore.getProject()))) {
-      Users user = jWTHelper.getUserPrincipal(sc);
-      String dbUsername = onlineFeaturestoreController.onlineDbUsername(project, user);
-      String dbName = onlineFeaturestoreController.getOnlineFeaturestoreDbName(project);
-      onlineFeaturestoreConnector =
-        featurestoreStorageConnectorController.getOnlineFeaturestoreConnector(user, project,
-          dbUsername, featurestore, dbName);
-      featurestoreDTO.setMysqlServerEndpoint(settings.getFeaturestoreJdbcUrl());
-      featurestoreDTO.setOnlineFeaturestoreSize(onlineFeaturestoreController.getDbSize(featurestore));
-      featurestoreDTO.setOnlineFeaturestoreName(featurestore.getProject().getName());
-      featurestoreDTO.setOnlineEnabled(true);
-    }
     FeaturestoreMetadataDTO featurestoreMetadataDTO =
       new FeaturestoreMetadataDTO(featurestoreDTO, featuregroups, trainingDatasets,
         featurestoreClientSettingsDTO, storageConnectors, onlineFeaturestoreConnector);

hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/storageconnector/FeaturestoreStorageConnectorService.java

@@ -130,7 +130,7 @@ public void setFeaturestoreId(Integer featurestoreId) throws FeaturestoreExcepti
   @ApiKeyRequired( acceptedScopes = {ApiScope.FEATURESTORE}, allowedUserRoles = {"HOPS_ADMIN", "HOPS_USER"})
   @ApiOperation(value = "Get all storage connectors of a feature store",
     response = FeaturestoreStorageConnectorDTO.class, responseContainer = "List")
-  public Response getStorageConnectors(@Context SecurityContext sc) {
+  public Response getStorageConnectors(@Context SecurityContext sc) throws FeaturestoreException {
     List<FeaturestoreStorageConnectorDTO> featurestoreStorageConnectorDTOS =
       featurestoreStorageConnectorController.getAllStorageConnectorsForFeaturestore(featurestore);
     GenericEntity<List<FeaturestoreStorageConnectorDTO>> featurestoreStorageConnectorsGeneric =
@@ -157,7 +157,8 @@ public Response getStorageConnectors(@Context SecurityContext sc) {
     response = FeaturestoreStorageConnectorDTO.class, responseContainer = "List")
   public Response getStorageConnectorsOfType(
     @ApiParam(value = "storage connector type", example = "JDBC")
-    @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, @Context SecurityContext sc) {
+    @PathParam("connectorType") FeaturestoreStorageConnectorType connectorType, @Context SecurityContext sc)
+      throws FeaturestoreException {
     verifyStorageConnectorType(connectorType);
     List<FeaturestoreStorageConnectorDTO> featurestoreStorageConnectorDTOS =
       featurestoreStorageConnectorController.getAllStorageConnectorsForFeaturestoreWithType(featurestore,

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/FeaturestoreController.java

@@ -253,7 +253,7 @@ public Featurestore createProjectFeatureStore(Project project, Users user, Strin
    * @param featurestore the featurestore entity
    * @return a DTO representation of the featurestore
    */
-  public FeaturestoreDTO convertFeaturestoreToDTO(Featurestore featurestore) {
+  private FeaturestoreDTO convertFeaturestoreToDTO(Featurestore featurestore) {
     String hiveDbDescription = featurestoreFacade.getHiveDatabaseDescription(featurestore.getHiveDbId());
     FeaturestoreDTO featurestoreDTO = new FeaturestoreDTO(featurestore);
     featurestoreDTO.setFeaturestoreDescription(hiveDbDescription);
@@ -270,7 +270,7 @@ public FeaturestoreDTO convertFeaturestoreToDTO(Featurestore featurestore) {
       if (settings.isOnlineFeaturestore() &&
           onlineFeaturestoreController.checkIfDatabaseExists(
               onlineFeaturestoreController.getOnlineFeaturestoreDbName(featurestore.getProject()))) {
-        featurestoreDTO.setMysqlServerEndpoint(settings.getFeaturestoreJdbcUrl());
+        featurestoreDTO.setMysqlServerEndpoint(onlineFeaturestoreController.getJdbcURL());
         featurestoreDTO.setOnlineFeaturestoreSize(onlineFeaturestoreController.getDbSize(featurestore));
         featurestoreDTO.setOnlineFeaturestoreName(featurestore.getProject().getName());
         featurestoreDTO.setOnlineEnabled(true);

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/online/OnlineFeaturestoreController.java

@@ -15,6 +15,7 @@
  */
 package io.hops.hopsworks.common.featurestore.online;
 
+import com.logicalclocks.servicediscoverclient.exceptions.ServiceDiscoveryException;
 import io.hops.hopsworks.common.dao.user.UserFacade;
 import io.hops.hopsworks.common.dao.user.security.secrets.SecretsFacade;
 import io.hops.hopsworks.common.featurestore.FeaturestoreConstants;
@@ -23,6 +24,7 @@
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO;
 import io.hops.hopsworks.common.featurestore.storageconnectors.jdbc.FeaturestoreJdbcConnectorController;
 import io.hops.hopsworks.common.hdfs.HdfsUsersController;
+import io.hops.hopsworks.common.hosts.ServiceDiscoveryController;
 import io.hops.hopsworks.common.security.secrets.SecretsController;
 import io.hops.hopsworks.common.util.Settings;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
@@ -78,6 +80,8 @@ public class OnlineFeaturestoreController {
   private HdfsUsersController hdfsUsersController;
   @EJB
   private UserFacade userFacade;
+  @EJB
+  private ServiceDiscoveryController serviceDiscoveryController;
 
   @PostConstruct
   public void init() {
@@ -108,20 +112,16 @@ private Connection initConnection(String databaseName, Project project, Users us
       throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_ONLINE_SECRETS_ERROR,
         Level.SEVERE, "Problem getting secrets for the JDBC connection to the online FS");
     }
-    jdbcString = settings.getFeaturestoreJdbcUrl() + databaseName;
     try {
-      return DriverManager.getConnection(jdbcString, dbUsername, password);
-    } catch (SQLException e) {
-      LOGGER.log(Level.SEVERE,
-        "Error initiating MySQL JDBC connection to online feature store for user: " + user.getEmail() + " error:"
-          + e);
+      return DriverManager.getConnection(getJdbcURL(databaseName), dbUsername, password);
+    } catch (SQLException | ServiceDiscoveryException e) {
       throw new FeaturestoreException(
         RESTCodes.FeaturestoreErrorCode.COULD_NOT_INITIATE_MYSQL_CONNECTION_TO_ONLINE_FEATURESTORE, Level.SEVERE,
         "project: " + project.getName() + ", database: " + databaseName + ", db user:" + dbUsername +
                 ", jdbcString: " + jdbcString, e.getMessage(), e);
     }
   }
-  
+
   /**
    * Runs a update/create SQL query against an online featurestore database, impersonating the user making the request
    *
@@ -425,23 +425,25 @@ public void removeOnlineFeaturestoreUser(Featurestore featurestore, Users user)
     }
 
     String dbUser = onlineDbUsername(featurestore.getProject().getName(), user.getUsername());
+    String connectorName = dbUser + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX;
+
     SecretId id = new SecretId(user.getUid(), dbUser);
     secretsFacade.deleteSecret(id);
     try {
       onlineFeaturestoreFacade.removeOnlineFeaturestoreUser(dbUser);
+
+      List<FeaturestoreStorageConnectorDTO> jdbcConnectors =
+          featurestoreJdbcConnectorController.getJdbcConnectorsForFeaturestore(featurestore);
+      for (FeaturestoreStorageConnectorDTO storageConnector: jdbcConnectors) {
+        if (storageConnector.getName().equalsIgnoreCase(connectorName)) {
+          featurestoreJdbcConnectorController.removeFeaturestoreJdbcConnector(storageConnector.getId());
+        }
+      }
     } catch (Exception e) {
       throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ERROR_DELETING_ONLINE_FEATURESTORE_USER,
         Level.SEVERE, "An error occurred when trying to delete the MySQL database user for an online feature store",
         e.getMessage(), e);
     }
-    String connectorName = dbUser + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX;
-    List<FeaturestoreStorageConnectorDTO> jdbcConnectors =
-      featurestoreJdbcConnectorController.getJdbcConnectorsForFeaturestore(featurestore);
-    for (FeaturestoreStorageConnectorDTO storageConnector: jdbcConnectors) {
-      if (storageConnector.getName().equalsIgnoreCase(connectorName)) {
-        featurestoreJdbcConnectorController.removeFeaturestoreJdbcConnector(storageConnector.getId());
-      }
-    }
   }
 
 
@@ -474,4 +476,15 @@ private String generateRandomUserPw() {
   public Boolean checkIfDatabaseExists(String dbName) {
     return onlineFeaturestoreFacade.checkIfDatabaseExists(dbName);
   }
+
+  public String getJdbcURL() throws ServiceDiscoveryException {
+    return getJdbcURL("");
+  }
+
+  private String getJdbcURL(String dbName) throws ServiceDiscoveryException {
+    return "jdbc:mysql://" + serviceDiscoveryController
+        .getAnyAddressOfServiceWithDNS(ServiceDiscoveryController.HopsworksService.MYSQL).getAddress()
+        + ":3306"
+        + "/" + dbName;
+  }
 }

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/FeaturestoreStorageConnectorController.java

@@ -61,7 +61,8 @@ public class FeaturestoreStorageConnectorController {
    * @param featurestore the featurestore to query
    * @return List of JSON/XML DTOs of the storage connectors
    */
-  public List<FeaturestoreStorageConnectorDTO> getAllStorageConnectorsForFeaturestore(Featurestore featurestore) {
+  public List<FeaturestoreStorageConnectorDTO> getAllStorageConnectorsForFeaturestore(Featurestore featurestore)
+      throws FeaturestoreException{
     List<FeaturestoreStorageConnectorDTO> featurestoreStorageConnectorDTOS = new ArrayList<>();
     featurestoreStorageConnectorDTOS.addAll(
         featurestoreJdbcConnectorController.getJdbcConnectorsForFeaturestore(featurestore));
@@ -79,7 +80,8 @@ public List<FeaturestoreStorageConnectorDTO> getAllStorageConnectorsForFeaturest
    * @return List of JSON/XML DTOs of the storage connectors
    */
   public List<FeaturestoreStorageConnectorDTO> getAllStorageConnectorsForFeaturestoreWithType(
-      Featurestore featurestore, FeaturestoreStorageConnectorType featurestoreStorageConnectorType) {
+      Featurestore featurestore, FeaturestoreStorageConnectorType featurestoreStorageConnectorType)
+      throws FeaturestoreException {
     switch(featurestoreStorageConnectorType) {
       case S3:
         return featurestoreS3ConnectorController.getS3ConnectorsForFeaturestore(featurestore);

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorController.java

@@ -22,7 +22,7 @@
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorDTO;
 import io.hops.hopsworks.common.featurestore.storageconnectors.FeaturestoreStorageConnectorType;
 import io.hops.hopsworks.common.hive.HiveController;
-import io.hops.hopsworks.common.util.Settings;
+import io.hops.hopsworks.common.hosts.ServiceDiscoveryController;
 import io.hops.hopsworks.exceptions.FeaturestoreException;
 import io.hops.hopsworks.persistence.entity.featurestore.Featurestore;
 import io.hops.hopsworks.persistence.entity.featurestore.storageconnector.jdbc.FeaturestoreJdbcConnector;
@@ -43,7 +43,7 @@ public class FeaturestoreJdbcConnectorController {
   @EJB
   private FeaturestoreJdbcConnectorFacade featurestoreJdbcConnectorFacade;
   @EJB
-  private Settings settings;
+  private ServiceDiscoveryController serviceDiscoveryController;
   @EJB
   private HiveController hiveController;
 
@@ -289,9 +289,17 @@ private void verifyUserInput(Featurestore featurestore, FeaturestoreJdbcConnecto
    * @param featurestore featurestore to query for jdbc connectors
    * @return list of XML/JSON DTOs of the jdbc connectors
    */
-  public List<FeaturestoreStorageConnectorDTO> getJdbcConnectorsForFeaturestore(Featurestore featurestore) {
-    List<FeaturestoreJdbcConnector> jdbcConnectors = featurestoreJdbcConnectorFacade.findByFeaturestore(featurestore);
-    return jdbcConnectors.stream().map(FeaturestoreJdbcConnectorDTO::new).collect(Collectors.toList());
+  public List<FeaturestoreStorageConnectorDTO> getJdbcConnectorsForFeaturestore(Featurestore featurestore)
+      throws FeaturestoreException {
+    try {
+      return featurestoreJdbcConnectorFacade.findByFeaturestore(featurestore).stream()
+          .map(FeaturestoreJdbcConnectorDTO::new)
+          .map(this::replaceOnlineFsConnectorUrl)
+          .collect(Collectors.toList());
+    } catch (RuntimeException e) {
+      throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.STORAGE_CONNECTOR_GET_ERROR, Level.SEVERE,
+          "Error resolving MySQL DNS name", e.getMessage(), e);
+    }
   }
 
   /**
@@ -304,7 +312,12 @@ public List<FeaturestoreStorageConnectorDTO> getJdbcConnectorsForFeaturestore(Fe
   public FeaturestoreJdbcConnectorDTO getJdbcConnectorWithIdAndFeaturestore(Featurestore featurestore, Integer id)
       throws FeaturestoreException {
     FeaturestoreJdbcConnector featurestoreJdbcConnector = verifyJdbcConnectorId(id, featurestore);
-    return new FeaturestoreJdbcConnectorDTO(featurestoreJdbcConnector);
+    try {
+      return replaceOnlineFsConnectorUrl(new FeaturestoreJdbcConnectorDTO(featurestoreJdbcConnector));
+    } catch (RuntimeException e) {
+      throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.STORAGE_CONNECTOR_GET_ERROR, Level.SEVERE,
+          "Error resolving MySQL DNS name", e.getMessage(), e);
+    }
   }
 
   /**
@@ -319,14 +332,13 @@ public FeaturestoreJdbcConnectorDTO getJdbcConnectorWithIdAndFeaturestore(Featur
   public FeaturestoreJdbcConnectorDTO createJdbcConnectorForOnlineFeaturestore(String onlineDbUsername,
     Featurestore featurestore, String dbName) throws FeaturestoreException {
     String connectorName = onlineDbUsername + FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_SUFFIX;
-    List<FeaturestoreStorageConnectorDTO> featurestoreConnectors = getJdbcConnectorsForFeaturestore(featurestore);
-    for (FeaturestoreStorageConnectorDTO storageConnector: featurestoreConnectors) {
-      if(connectorName.equalsIgnoreCase(storageConnector.getName())){
-        throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE,
+
+    if (featurestoreJdbcConnectorFacade.findByName(connectorName).isPresent()) {
+      throw new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.ILLEGAL_STORAGE_CONNECTOR_NAME, Level.FINE,
           "a storage connector with that name already exists");
-      }
     }
-    String connectionString = settings.getFeaturestoreJdbcUrl() + dbName;
+
+    String connectionString = "jdbc:mysql://mysql.service.consul:3306/" + dbName;
     String arguments = FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_PASSWORD_ARG + "=" +
       FeaturestoreConstants.ONLINE_FEATURE_STORE_CONNECTOR_PASSWORD_TEMPLATE + "," +
       FeaturestoreConstants.ONLINE_FEATURE_STORE_JDBC_USER_ARG + "=" + onlineDbUsername;
@@ -341,4 +353,18 @@ public FeaturestoreJdbcConnectorDTO createJdbcConnectorForOnlineFeaturestore(Str
     return createFeaturestoreJdbcConnector(featurestore, featurestoreJdbcConnectorDTO);
   }
 
+  private FeaturestoreJdbcConnectorDTO replaceOnlineFsConnectorUrl(FeaturestoreJdbcConnectorDTO jdbcConnectorDTO) {
+    String connectionString = "";
+    try {
+      connectionString = jdbcConnectorDTO.getConnectionString().replace("mysql.service.consul",
+          serviceDiscoveryController.getAnyAddressOfServiceWithDNS(ServiceDiscoveryController.HopsworksService.MYSQL)
+              .getAddress());
+    } catch (ServiceDiscoveryException e) {
+      throw new RuntimeException(e);
+    }
+    jdbcConnectorDTO.setConnectionString(connectionString);
+
+    return jdbcConnectorDTO;
+  }
+
 }

hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/storageconnectors/jdbc/FeaturestoreJdbcConnectorFacade.java

@@ -29,6 +29,7 @@
 import javax.persistence.TypedQuery;
 import javax.validation.ConstraintViolationException;
 import java.util.List;
+import java.util.Optional;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -92,6 +93,16 @@ public FeaturestoreJdbcConnector findByIdAndFeaturestore(Integer id, Featurestor
     }
   }
 
+  public Optional<FeaturestoreJdbcConnector> findByName(String name) {
+    try {
+      return Optional.of(em.createNamedQuery("FeaturestoreJdbcConnector.findByName", FeaturestoreJdbcConnector.class)
+          .setParameter("name", name)
+          .getSingleResult());
+    } catch (NoResultException e) {
+      return Optional.empty();
+    }
+  }
+
   /**
    * Updates a jdbc connector
    *

hopsworks-common/src/main/java/io/hops/hopsworks/common/hosts/ServiceDiscoveryController.java

@@ -62,7 +62,8 @@ public enum HopsworksService {
     HOPSWORKS_APP("hopsworks.glassfish"),
     JUPYTER_LOGSTASH("jupyter.logstash"),
     REGISTRY("registry"),
-    CONSUL_SERVER("consul");
+    CONSUL_SERVER("consul"),
+    MYSQL("mysql");
 
     private String name;
     HopsworksService(String name) {

hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java

@@ -310,7 +310,6 @@ public class Settings implements Serializable {
   /* -------------------- Featurestore --------------- */
   private static final String VARIABLE_FEATURESTORE_DEFAULT_QUOTA = "featurestore_default_quota";
   private static final String VARIABLE_FEATURESTORE_DEFAULT_STORAGE_FORMAT = "featurestore_default_storage_format";
-  private static final String VARIABLE_FEATURESTORE_JDBC_URL = "featurestore_jdbc_url";
   private static final String VARIABLE_ONLINE_FEATURESTORE = "featurestore_online_enabled";
   private static final String VARIABLE_FG_PREVIEW_LIMIT = "fg_preview_limit";
   private static final String VARIABLE_ONLINE_FEATURESTORE_TS = "featurestore_online_tablespace";
@@ -701,7 +700,6 @@ private void populateCache() {
       FEATURESTORE_DB_DEFAULT_QUOTA = setStrVar(VARIABLE_FEATURESTORE_DEFAULT_QUOTA, FEATURESTORE_DB_DEFAULT_QUOTA);
       FEATURESTORE_DB_DEFAULT_STORAGE_FORMAT =
           setStrVar(VARIABLE_FEATURESTORE_DEFAULT_STORAGE_FORMAT, FEATURESTORE_DB_DEFAULT_STORAGE_FORMAT);
-      FEATURESTORE_JDBC_URL = setStrVar(VARIABLE_FEATURESTORE_JDBC_URL, FEATURESTORE_JDBC_URL);
       ONLINE_FEATURESTORE = setBoolVar(VARIABLE_ONLINE_FEATURESTORE, ONLINE_FEATURESTORE);
       ONLINE_FEATURESTORE_TS = setStrVar(VARIABLE_ONLINE_FEATURESTORE_TS, ONLINE_FEATURESTORE_TS);
 
@@ -3552,13 +3550,6 @@ public Boolean isHopsUtilInsecure() {
     return isCloud() || isLocalHost();
   }
 
-  private String FEATURESTORE_JDBC_URL = "jdbc:mysql://" + HOPSWORKS_IP + ":3306/";
-
-  public synchronized String getFeaturestoreJdbcUrl() {
-    checkCache();
-    return FEATURESTORE_JDBC_URL;
-  }
-
   private Boolean REQUESTS_VERIFY = false;
 
   /**

hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/featurestore/storageconnector/jdbc/FeaturestoreJdbcConnector.java

@@ -43,6 +43,8 @@
       "fsjdbc"),
     @NamedQuery(name = "FeaturestoreJdbcConnector.findById",
         query = "SELECT fsjdbc FROM FeaturestoreJdbcConnector fsjdbc WHERE fsjdbc.id = :id"),
+    @NamedQuery(name = "FeaturestoreJdbcConnector.findByName",
+        query = "SELECT fsjdbc FROM FeaturestoreJdbcConnector fsjdbc WHERE fsjdbc.name = :name"),
     @NamedQuery(name = "FeaturestoreJdbcConnector.findByFeaturestore", query = "SELECT fsjdbc " +
         "FROM FeaturestoreJdbcConnector fsjdbc WHERE fsjdbc.featurestore = :featurestore"),
     @NamedQuery(name = "FeaturestoreJdbcConnector.findByFeaturestoreAndId", query = "SELECT fsjdbc " +