[HWORKS-888] Support for gRPC protocol in Python model deployments

python/hsml/client/exceptions.py

@@ -71,6 +71,13 @@ class ModelServingException(Exception):
     ERROR_CODE_DEPLOYMENT_NOT_RUNNING = 250001
 
 
+class InternalClientError(TypeError):
+    """Raised when internal client cannot be initialized due to missing arguments."""
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
 class ExternalClientError(TypeError):
     """Raised when external client cannot be initialized due to missing arguments."""
 

python/hsml/client/hopsworks/base.py

@@ -105,7 +105,7 @@ def _close(self):
         """Closes a client. Can be implemented for clean up purposes, not mandatory."""
         self._connected = False
 
-    def replace_public_host(self, url):
+    def _replace_public_host(self, url):
         """replace hostname to public hostname set in HOPSWORKS_PUBLIC_HOST"""
         ui_url = url._replace(netloc=os.environ[self.HOPSWORKS_PUBLIC_HOST])
         return ui_url

python/hsml/client/hopsworks/external.py

@@ -77,7 +77,7 @@ def _get_project_info(self, project_name):
         """
         return self._send_request("GET", ["project", "getProjectInfo", project_name])
 
-    def replace_public_host(self, url):
+    def _replace_public_host(self, url):
         """no need to replace as we are already in external client"""
         return url
 

python/hsml/client/istio/base.py

@@ -17,12 +17,12 @@
 import os
 from abc import abstractmethod
 
-from hsml.client import base, exceptions
+from hsml.client import base
+from hsml.client.istio.grpc.inference_client import GRPCInferenceServerClient
 
 
 class Client(base.Client):
     SERVING_API_KEY = "SERVING_API_KEY"
-    ISTIO_ENDPOINT = "ISTIO_ENDPOINT"
     HOPSWORKS_PUBLIC_HOST = "HOPSWORKS_PUBLIC_HOST"
 
     BASE_PATH_PARAMS = []
@@ -80,17 +80,18 @@ def _get_host_port_pair(self):
         host, port = endpoint.split(":")
         return host, port
 
-    def _get_serving_api_key(self):
-        """Retrieve serving API key from environment variable."""
-        if self.SERVING_API_KEY not in os.environ:
-            raise exceptions.ExternalClientError("Serving API key not found")
-        return os.environ[self.SERVING_API_KEY]
-
     def _close(self):
         """Closes a client. Can be implemented for clean up purposes, not mandatory."""
         self._connected = False
 
-    def replace_public_host(self, url):
+    def _replace_public_host(self, url):
         """replace hostname to public hostname set in HOPSWORKS_PUBLIC_HOST"""
         ui_url = url._replace(netloc=os.environ[self.HOPSWORKS_PUBLIC_HOST])
         return ui_url
+
+    def _create_grpc_channel(self, service_hostname: str) -> GRPCInferenceServerClient:
+        return GRPCInferenceServerClient(
+            url=self._host + ":" + str(self._port),
+            channel_args=(("grpc.ssl_target_name_override", service_hostname),),
+            serving_api_key=self._auth._token,
+        )

python/hsml/client/istio/external.py

@@ -48,7 +48,7 @@ def _close(self):
         """Closes a client."""
         self._connected = False
 
-    def replace_public_host(self, url):
+    def _replace_public_host(self, url):
         """no need to replace as we are already in external client"""
         return url
 

python/hsml/client/istio/grpc/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

python/hsml/client/istio/grpc/errors.py

@@ -0,0 +1,30 @@
+# Copyright 2022 The KServe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This implementation has been borrowed from the kserve/kserve repository
+# https://github.com/kserve/kserve/blob/release-0.11/python/kserve/kserve/errors.py
+
+
+class InvalidInput(ValueError):
+    """
+    Exception class indicating invalid input arguments.
+    HTTP Servers should return HTTP_400 (Bad Request).
+    """
+
+    def __init__(self, reason):
+        self.reason = reason
+
+    def __str__(self):
+        return self.reason

python/hsml/client/istio/grpc/exceptions.py

@@ -0,0 +1,123 @@
+# Copyright 2023 The KServe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# coding: utf-8
+
+# This implementation has been borrowed from kserve/kserve repository
+# https://github.com/kserve/kserve/blob/release-0.11/python/kserve/kserve/exceptions.py
+
+import six
+
+
+class OpenApiException(Exception):
+    """The base exception class for all OpenAPIExceptions"""
+
+
+class ApiTypeError(OpenApiException, TypeError):
+    def __init__(self, msg, path_to_item=None, valid_classes=None, key_type=None):
+        """Raises an exception for TypeErrors
+
+        Args:
+            msg (str): the exception message
+
+        Keyword Args:
+            path_to_item (list): a list of keys an indices to get to the
+                                 current_item
+                                 None if unset
+            valid_classes (tuple): the primitive classes that current item
+                                   should be an instance of
+                                   None if unset
+            key_type (bool): False if our value is a value in a dict
+                             True if it is a key in a dict
+                             False if our item is an item in a list
+                             None if unset
+        """
+        self.path_to_item = path_to_item
+        self.valid_classes = valid_classes
+        self.key_type = key_type
+        full_msg = msg
+        if path_to_item:
+            full_msg = "{0} at {1}".format(msg, render_path(path_to_item))
+        super(ApiTypeError, self).__init__(full_msg)
+
+
+class ApiValueError(OpenApiException, ValueError):
+    def __init__(self, msg, path_to_item=None):
+        """
+        Args:
+            msg (str): the exception message
+
+        Keyword Args:
+            path_to_item (list) the path to the exception in the
+                received_data dict. None if unset
+        """
+
+        self.path_to_item = path_to_item
+        full_msg = msg
+        if path_to_item:
+            full_msg = "{0} at {1}".format(msg, render_path(path_to_item))
+        super(ApiValueError, self).__init__(full_msg)
+
+
+class ApiKeyError(OpenApiException, KeyError):
+    def __init__(self, msg, path_to_item=None):
+        """
+        Args:
+            msg (str): the exception message
+
+        Keyword Args:
+            path_to_item (None/list) the path to the exception in the
+                received_data dict
+        """
+        self.path_to_item = path_to_item
+        full_msg = msg
+        if path_to_item:
+            full_msg = "{0} at {1}".format(msg, render_path(path_to_item))
+        super(ApiKeyError, self).__init__(full_msg)
+
+
+class ApiException(OpenApiException):
+    def __init__(self, status=None, reason=None, http_resp=None):
+        if http_resp:
+            self.status = http_resp.status
+            self.reason = http_resp.reason
+            self.body = http_resp.data
+            self.headers = http_resp.getheaders()
+        else:
+            self.status = status
+            self.reason = reason
+            self.body = None
+            self.headers = None
+
+    def __str__(self):
+        """Custom error messages for exception"""
+        error_message = "({0})\n" "Reason: {1}\n".format(self.status, self.reason)
+        if self.headers:
+            error_message += "HTTP response headers: {0}\n".format(self.headers)
+
+        if self.body:
+            error_message += "HTTP response body: {0}\n".format(self.body)
+
+        return error_message
+
+
+def render_path(path_to_item):
+    """Returns a string representation of a path"""
+    result = ""
+    for pth in path_to_item:
+        if isinstance(pth, six.integer_types):
+            result += "[{0}]".format(pth)
+        else:
+            result += "['{0}']".format(pth)
+    return result

python/hsml/client/istio/grpc/inference_client.py

@@ -0,0 +1,75 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import grpc
+
+from hsml.client.istio.utils.infer_type import InferRequest, InferResponse
+from hsml.client.istio.grpc.proto.grpc_predict_v2_pb2_grpc import (
+    GRPCInferenceServiceStub,
+)
+
+
+class GRPCInferenceServerClient:
+    def __init__(
+        self,
+        url,
+        serving_api_key,
+        channel_args=None,
+    ):
+        if channel_args is not None:
+            channel_opt = channel_args
+        else:
+            channel_opt = [
+                ("grpc.max_send_message_length", -1),
+                ("grpc.max_receive_message_length", -1),
+            ]
+
+        # Authentication is done via API Key in the Authorization header
+        self._channel = grpc.insecure_channel(url, options=channel_opt)
+        self._client_stub = GRPCInferenceServiceStub(self._channel)
+        self._serving_api_key = serving_api_key
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
+    def __del__(self):
+        """It is called during object garbage collection."""
+        self.close()
+
+    def close(self):
+        """Close the client. Future calls to server will result in an Error."""
+        self._channel.close()
+
+    def infer(self, infer_request: InferRequest, headers=None, client_timeout=None):
+        headers = {} if headers is None else headers
+        headers["authorization"] = "ApiKey " + self._serving_api_key
+        metadata = headers.items()
+
+        # convert the InferRequest to a ModelInferRequest message
+        request = infer_request.to_grpc()
+
+        try:
+            # send request
+            model_infer_response = self._client_stub.ModelInfer(
+                request=request, metadata=metadata, timeout=client_timeout
+            )
+        except grpc.RpcError as rpc_error:
+            raise rpc_error
+
+        # convert back the ModelInferResponse message to InferResponse
+        return InferResponse.from_grpc(model_infer_response)

python/hsml/client/istio/grpc/proto/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2024 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#