From f2b45a21c6720db91fb5578954fab04b7f414236 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 2 Nov 2022 17:29:20 -0700 Subject: [PATCH 1/2] wip Signed-off-by: Kevin Su --- cookbook/core/extend_flyte/data_persistence.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 cookbook/core/extend_flyte/data_persistence.py diff --git a/cookbook/core/extend_flyte/data_persistence.py b/cookbook/core/extend_flyte/data_persistence.py new file mode 100644 index 000000000..585375cc9 --- /dev/null +++ b/cookbook/core/extend_flyte/data_persistence.py @@ -0,0 +1,11 @@ +""" +.. _extend-flyte-persistence-plugin: + +########################## +Writing Persistence Plugin +########################## + +Currently flytekit uses `awscli `__ or `gcsutil `__ +to upload/download + +""" From 2ececf7426878b153713aea24edfb741846467a6 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 7 Nov 2022 14:32:12 -0800 Subject: [PATCH 2/2] Add doc for data persistence plugin Signed-off-by: Kevin Su --- .../core/extend_flyte/data_persistence.py | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/cookbook/core/extend_flyte/data_persistence.py b/cookbook/core/extend_flyte/data_persistence.py index 585375cc9..91b87d0ec 100644 --- a/cookbook/core/extend_flyte/data_persistence.py +++ b/cookbook/core/extend_flyte/data_persistence.py @@ -2,10 +2,70 @@ .. _extend-flyte-persistence-plugin: ########################## -Writing Persistence Plugin +Writing Custom Persistence Plugin ########################## -Currently flytekit uses `awscli `__ or `gcsutil `__ -to upload/download +Currently, flytekit uses `awscli `__ or `gcsutil `__ +by default to download the task's input and upload the task's output from the blob store. You may, however, want to write data to another storage service, such as HDFS or FTP. Flytekit provides a flexible way to extend and register a new persistence plugin. -""" +Let us try to recap, why we should implement a persistence plugin + +- Speed up blob store operations. +- Download the task input from a file system that Flyte doesn't support. +- You have a custom authentication workflow in your own SDK + +Add Custom Persistence Plugin +============================= +1. Extend `DataPersistence `__ and implement the interface. +2. Register it to the `DataPersistencePlugins `__ or overwrite the existing persistence plugin. + +Example +======= +Implement all the abstract functions in ``DataPersistence`` so that flytekit will know how to access and connect to your file systems. + +.. code-block:: python + + class CustomPersistence(DataPersistence):: + + def __init__(self, name: str, default_prefix: typing.Optional[str] = None, **kwargs): + self._name = name + self._default_prefix = default_prefix + + @property + def name(self) -> str: + return self._name + + @property + def default_prefix(self) -> typing.Optional[str]: + return self._default_prefix + + def listdir(self, path: str, recursive: bool = False) -> typing.Generator[str, None, None]: + # Returns true if the given path exists, else false + raise UnsupportedPersistenceOp(f"Listing a directory is not supported by the persistence plugin {self.name}") + + def exists(self, path: str) -> bool: + # Returns true if the given path exists, else false + pass + + def get(self, from_path: str, to_path: str, recursive: bool = False): + # Retrieves data from from_path and writes to the given to_path (to_path is locally accessible) + pass + + def put(self, from_path: str, to_path: str, recursive: bool = False): + # Stores data from from_path and writes to the given to_path (from_path is locally accessible) + pass + + def construct_path(self, add_protocol: bool, add_prefix: bool, *paths: str) -> str: + # if add_protocol is true then is prefixed else + # Constructs a path in the format *args + # delim is dependent on the storage medium. + # each of the args is joined with the delim + pass + + DataPersistencePlugins.register_plugin("cf://", CustomPersistence) + + +Here is also an `example `__ +of adding a custom plugin by leveraging fsspec. Flytekit will use fsspec to do blob operation by default if the ``flytekit-data-fsspec`` plugin is installed. + +""" \ No newline at end of file