Merge branch 'master' into spark-materialize-map-in-pandas

tokoko · Feb 13, 2024 · 5e3b180 · 5e3b180
2 parents a45f6a8 + 4e450ad
commit 5e3b180
Show file tree

Hide file tree

Showing 56 changed files with 781 additions and 187 deletions.
diff --git a/Makefile b/Makefile
@@ -310,7 +310,7 @@ format-python:
 	cd ${ROOT_DIR}/sdk/python; python -m black --target-version py38 feast tests
 
 lint-python:
-	cd ${ROOT_DIR}/sdk/python; python -m mypy
+	cd ${ROOT_DIR}/sdk/python; python -m mypy --exclude=/tests/ --follow-imports=skip feast
 	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ --check-only
 	cd ${ROOT_DIR}/sdk/python; python -m flake8 feast/ tests/
 	cd ${ROOT_DIR}/sdk/python; python -m black --check feast tests

diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
@@ -99,6 +99,7 @@
   * [MySQL (contrib)](reference/online-stores/mysql.md)
   * [Rockset (contrib)](reference/online-stores/rockset.md)
   * [Hazelcast (contrib)](reference/online-stores/hazelcast.md)
+  * [ScyllaDB (contrib)](reference/online-stores/scylladb.md)
 * [Providers](reference/providers/README.md)
   * [Local](reference/providers/local.md)
   * [Google Cloud Platform](reference/providers/google-cloud-platform.md)

diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md
@@ -54,4 +54,6 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli
 [hazelcast.md](hazelcast.md)
 {% endcontent-ref %}
 
-
+{% content-ref url="scylladb.md" %}
+[scylladb.md](scylladb.md)
+{% endcontent-ref %}
diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md
@@ -0,0 +1,94 @@
+# ScyllaDB Cloud online store
+
+## Description
+
+ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast.
+
+The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time.
+
+## Getting started
+
+Install Feast with Cassandra support:
+```bash
+pip install "feast[cassandra]"
+```
+
+Create a new Feast project:
+```bash
+feast init REPO_NAME -t cassandra
+```
+
+### Example (ScyllaDB)
+
+{% code title="feature_store.yaml" %}
+```yaml
+project: scylla_feature_repo
+registry: data/registry.db
+provider: local
+online_store:
+    type: cassandra
+    hosts:
+        - 172.17.0.2
+    keyspace: feast
+    username: scylla
+    password: password
+```
+{% endcode %}
+
+### Example (ScyllaDB Cloud)
+
+{% code title="feature_store.yaml" %}
+```yaml
+project: scylla_feature_repo
+registry: data/registry.db
+provider: local
+online_store:
+    type: cassandra
+    hosts:
+        - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud
+        - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud
+        - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud
+    keyspace: feast
+    username: scylla
+    password: password
+```
+{% endcode %}
+
+
+The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig).
+For a full explanation of configuration options please look at file
+`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`.
+
+Storage specifications can be found at `docs/specs/online_store_format.md`.
+
+## Functionality Matrix
+
+The set of functionality supported by online stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the Cassandra plugin.
+
+|                                                           | Cassandra |
+| :-------------------------------------------------------- | :-------- |
+| write feature values to the online store                  | yes       |
+| read feature values from the online store                 | yes       |
+| update infrastructure (e.g. tables) in the online store   | yes       |
+| teardown infrastructure (e.g. tables) in the online store | yes       |
+| generate a plan of infrastructure changes                 | yes       |
+| support for on-demand transforms                          | yes       |
+| readable by Python SDK                                    | yes       |
+| readable by Java                                          | no        |
+| readable by Go                                            | no        |
+| support for entityless feature views                      | yes       |
+| support for concurrent writing to the same key            | no        |
+| support for ttl (time to live) at retrieval               | no        |
+| support for deleting expired data                         | no        |
+| collocated by feature view                                | yes       |
+| collocated by feature service                             | no        |
+| collocated by entity key                                  | no        |
+
+To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix).
+
+## Resources
+
+* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/)
+* [ScyllaDB website](https://www.scylladb.com/)
+* [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/)
diff --git a/protos/feast/registry/RegistryServer.proto b/protos/feast/registry/RegistryServer.proto
@@ -0,0 +1,230 @@
+syntax = "proto3";
+
+package feast.registry;
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/empty.proto";
+import "feast/core/Registry.proto";
+import "feast/core/Entity.proto";
+import "feast/core/DataSource.proto";
+import "feast/core/FeatureView.proto";
+import "feast/core/RequestFeatureView.proto";
+import "feast/core/StreamFeatureView.proto";
+import "feast/core/OnDemandFeatureView.proto";
+import "feast/core/FeatureService.proto";
+import "feast/core/SavedDataset.proto";
+import "feast/core/ValidationProfile.proto";
+import "feast/core/InfraObject.proto";
+
+service RegistryServer{
+  // Entity RPCs
+  rpc GetEntity (GetEntityRequest) returns (feast.core.Entity) {}
+  rpc ListEntities (ListEntitiesRequest) returns (ListEntitiesResponse) {}
+
+  // DataSource RPCs
+  rpc GetDataSource (GetDataSourceRequest) returns (feast.core.DataSource) {}
+  rpc ListDataSources (ListDataSourcesRequest) returns (ListDataSourcesResponse) {}
+
+  // FeatureView RPCs
+  rpc GetFeatureView (GetFeatureViewRequest) returns (feast.core.FeatureView) {}
+  rpc ListFeatureViews (ListFeatureViewsRequest) returns (ListFeatureViewsResponse) {}
+
+  // RequestFeatureView RPCs
+  rpc GetRequestFeatureView (GetRequestFeatureViewRequest) returns (feast.core.RequestFeatureView) {}
+  rpc ListRequestFeatureViews (ListRequestFeatureViewsRequest) returns (ListRequestFeatureViewsResponse) {}
+
+  // StreamFeatureView RPCs
+  rpc GetStreamFeatureView (GetStreamFeatureViewRequest) returns (feast.core.StreamFeatureView) {}
+  rpc ListStreamFeatureViews (ListStreamFeatureViewsRequest) returns (ListStreamFeatureViewsResponse) {}
+
+  // OnDemandFeatureView RPCs
+  rpc GetOnDemandFeatureView (GetOnDemandFeatureViewRequest) returns (feast.core.OnDemandFeatureView) {}
+  rpc ListOnDemandFeatureViews (ListOnDemandFeatureViewsRequest) returns (ListOnDemandFeatureViewsResponse) {}
+
+  // FeatureService RPCs
+  rpc GetFeatureService (GetFeatureServiceRequest) returns (feast.core.FeatureService) {}
+  rpc ListFeatureServices (ListFeatureServicesRequest) returns (ListFeatureServicesResponse) {}
+
+  // SavedDataset RPCs
+  rpc GetSavedDataset (GetSavedDatasetRequest) returns (feast.core.SavedDataset) {}
+  rpc ListSavedDatasets (ListSavedDatasetsRequest) returns (ListSavedDatasetsResponse) {}
+
+  // ValidationReference RPCs
+  rpc GetValidationReference (GetValidationReferenceRequest) returns (feast.core.ValidationReference) {}
+  rpc ListValidationReferences (ListValidationReferencesRequest) returns (ListValidationReferencesResponse) {}
+
+  rpc ListProjectMetadata (ListProjectMetadataRequest) returns (ListProjectMetadataResponse) {}
+  rpc GetInfra (GetInfraRequest) returns (feast.core.Infra) {}
+  rpc Refresh (RefreshRequest) returns (google.protobuf.Empty) {}
+  rpc Proto (google.protobuf.Empty) returns (feast.core.Registry) {}
+
+}
+
+message RefreshRequest {
+  string project = 1;
+}
+
+message GetInfraRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListProjectMetadataRequest {
+  string project = 1;
+  bool allow_cache = 2;    
+}
+
+message ListProjectMetadataResponse {
+  repeated feast.core.ProjectMetadata project_metadata = 1;     
+}
+
+message GetEntityRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListEntitiesRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListEntitiesResponse {
+    repeated feast.core.Entity entities = 1;
+}
+
+// DataSources
+
+message GetDataSourceRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListDataSourcesRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListDataSourcesResponse {
+    repeated feast.core.DataSource data_sources = 1;
+}
+
+// FeatureViews
+
+message GetFeatureViewRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListFeatureViewsRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListFeatureViewsResponse {
+    repeated feast.core.FeatureView feature_views = 1;
+}
+
+// RequestFeatureView
+
+message GetRequestFeatureViewRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListRequestFeatureViewsRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListRequestFeatureViewsResponse {
+    repeated feast.core.RequestFeatureView request_feature_views = 1;
+}
+
+// StreamFeatureView
+
+message GetStreamFeatureViewRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListStreamFeatureViewsRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListStreamFeatureViewsResponse {
+    repeated feast.core.StreamFeatureView stream_feature_views = 1;
+}
+
+// OnDemandFeatureView
+
+message GetOnDemandFeatureViewRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListOnDemandFeatureViewsRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListOnDemandFeatureViewsResponse {
+    repeated feast.core.OnDemandFeatureView on_demand_feature_views = 1;
+}
+
+// FeatureServices
+
+message GetFeatureServiceRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListFeatureServicesRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListFeatureServicesResponse {
+    repeated feast.core.FeatureService feature_services = 1;
+}
+
+// SavedDataset
+
+message GetSavedDatasetRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListSavedDatasetsRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListSavedDatasetsResponse {
+    repeated feast.core.SavedDataset saved_datasets = 1;
+}
+
+// ValidationReference
+
+message GetValidationReferenceRequest {
+    string name = 1;
+    string project = 2;
+    bool allow_cache = 3;
+}
+
+message ListValidationReferencesRequest {
+    string project = 1;
+    bool allow_cache = 2;
+}
+
+message ListValidationReferencesResponse {
+    repeated feast.core.ValidationReference validation_references = 1;
+}
diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py
@@ -25,7 +25,10 @@
 from pygments import formatters, highlight, lexers
 
 from feast import utils
-from feast.constants import DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT
+from feast.constants import (
+    DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT,
+    DEFAULT_REGISTRY_SERVER_PORT,
+)
 from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError
 from feast.feature_view import FeatureView
 from feast.infra.contrib.grpc_server import get_grpc_server
@@ -753,6 +756,22 @@ def serve_transformations_command(ctx: click.Context, port: int):
     store.serve_transformations(port)
 
 
+@cli.command("serve_registry")
+@click.option(
+    "--port",
+    "-p",
+    type=click.INT,
+    default=DEFAULT_REGISTRY_SERVER_PORT,
+    help="Specify a port for the server",
+)
+@click.pass_context
+def serve_registry_command(ctx: click.Context, port: int):
+    """Start a registry server locally on a given port."""
+    store = create_feature_store(ctx)
+
+    store.serve_registry(port)
+
+
 @cli.command("validate")
 @click.option(
     "--feature-service",

diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py
@@ -44,5 +44,8 @@
 # Default FTS port
 DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569
 
+# Default registry server port
+DEFAULT_REGISTRY_SERVER_PORT = 6570
+
 # Environment variable for feature server docker image tag
 DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG"