From bd2f9a825903c4ce10408974e45cf22fd16caa3d Mon Sep 17 00:00:00 2001 From: Nikki Everett Date: Tue, 11 Jun 2024 13:51:13 -0500 Subject: [PATCH] remove mmcloud plugin (#5468) Signed-off-by: nikki everett --- docs/conf.py | 2 + docs/deprecated_integrations/index.md | 3 - .../mmcloud_plugin/index.md | 98 ------------------- .../mmcloud_plugin/mmcloud_plugin_example.md | 30 ------ 4 files changed, 2 insertions(+), 131 deletions(-) delete mode 100644 docs/deprecated_integrations/mmcloud_plugin/index.md delete mode 100644 docs/deprecated_integrations/mmcloud_plugin/mmcloud_plugin_example.md diff --git a/docs/conf.py b/docs/conf.py index c7c1bcd75a..ae4400dd31 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -105,6 +105,8 @@ "flytesnacks/examples/mmcloud_plugin/mmcloud_plugin_example": "../../../deprecated_integrations/mmcloud_plugin/mmcloud_plugin_example.html", "flytesnacks/examples/snowflake_plugin/index": "../../../deprecated_integrations/snowflake_plugin/index.html", "flytesnacks/examples/snowflake_plugin/snowflake_plugin_example": "../../../deprecated_integrations/snowflake_plugin/snowflake_plugin_example.html", + "deprecated_integrations/mmcloud_plugin/index": "../../flytesnacks/examples/mmcloud_agent/index.html", + "deprecated_integrations/mmcloud_plugin/mmcloud_plugin_example": "../../flytesnacks/examples/mmcloud_agent/index.html" } diff --git a/docs/deprecated_integrations/index.md b/docs/deprecated_integrations/index.md index ef11ffb0db..a623278388 100644 --- a/docs/deprecated_integrations/index.md +++ b/docs/deprecated_integrations/index.md @@ -8,8 +8,6 @@ - Deprecated BigQuery plugin. * - {doc}`Databricks ` - Deprecated Databricks plugin. -* - {doc}`Memory Machine Cloud ` - - Deprecated MemVerge Memory Machine Cloud plugin. * - {doc}`Snowflake ` - Deprecated Snowflake plugin. ``` @@ -20,6 +18,5 @@ bigquery_plugin/index databricks_plugin/index -mmcloud_plugin/index snowflake_plugin/index ``` diff --git a/docs/deprecated_integrations/mmcloud_plugin/index.md b/docs/deprecated_integrations/mmcloud_plugin/index.md deleted file mode 100644 index 9539a82a08..0000000000 --- a/docs/deprecated_integrations/mmcloud_plugin/index.md +++ /dev/null @@ -1,98 +0,0 @@ -```{eval-rst} -.. tags:: AWS, GCP, AliCloud, Integration, Advanced -``` - -(mmcloud_plugin)= - -# Memory Machine Cloud plugin - -```{note} - -This is a legacy implementation of the Memory Machine Cloud integration. We recommend using the {ref}`Memory Machine Cloud agent ` instead. - -``` - -[MemVerge](https://memverge.com/) [Memory Machine Cloud](https://www.mmcloud.io/) (MMCloud)—available on AWS, GCP, and AliCloud—empowers users to continuously optimize cloud resources during runtime, safely execute stateful tasks on spot instances, and monitor resource usage in real time. These capabilities make it an excellent fit for long-running batch workloads. Flyte can be integrated with MMCloud, allowing you to execute Flyte tasks using MMCloud. - -## Installation - -To install the plugin, run the following command: - -```{eval-rst} -.. prompt:: bash - - pip install flytekitplugins-mmcloud -``` - -To get started with MMCloud, see the [MMCloud user guide](https://docs.memverge.com/mmce/current/userguide/olh/index.html). - -## Flyte deployment configuration - -The MMCloud plugin is [enabled in FlytePropeller's configuration](https://docs.flyte.org/en/latest/deployment/plugins/memverge/mmcloud.html). - -## Getting Started - -This plugin allows executing `PythonFunctionTask` using MMCloud without changing any function code. - -```{eval-rst} -.. testcode:: awsbatch-quickstart - from flytekitplugins.mmcloud import MMCloudConfig - - @task(task_config=MMCloudConfig()) - def to_str(i: int) -> str: - return str(i) -``` - -[Resource](https://docs.flyte.org/en/latest/user_guide/productionizing/customizing_task_resources.html) (cpu and mem) requests and limits, [container](https://docs.flyte.org/en/latest/user_guide/customizing_dependencies/multiple_images_in_a_workflow.html) images, and environment variable specifications are supported. - -[ImageSpec](https://docs.flyte.org/en/latest/user_guide/customizing_dependencies/imagespec.html#image-spec-example) may be used to define images to run tasks. - -### Credentials - -The following [secrets](https://docs.flyte.org/en/latest/user_guide/productionizing/secrets.html) are required to be defined for the agent server: -* `mmc_address`: MMCloud OpCenter address -* `mmc_username`: MMCloud OpCenter username -* `mmc_password`: MMCloud OpCenter password - -### Defaults - -Compute resources: -* If only requests are specified, there are no limits. -* If only limits are specified, the requests are equal to the limits. -* If neither resource requests nor limits are specified, the default requests used for job submission are `cpu="1"` and `mem="1Gi"`, and there are no limits. - -### Agent Image - -Install `flytekitplugins-mmcloud` in the agent image. - -A `float` binary (obtainable via the OpCenter) is required. Copy it to the agent image `PATH`. - -Sample `Dockerfile` for building an agent image: -```dockerfile -FROM python:3.11-slim-bookworm - -WORKDIR /root -ENV PYTHONPATH /root - -# flytekit will autoload the agent if package is installed. -RUN pip install flytekitplugins-mmcloud -COPY float /usr/local/bin/float - -# For flytekit versions <= v1.10.2, use pyflyte serve. -# CMD pyflyte serve --port 8000 -# For flytekit versions > v1.10.2, use pyflyte serve agent. -CMD pyflyte serve agent --port 8000 -``` - -## Example usage - -For a usage example, see the {doc}`Memory Machine Cloud example` page. - - - -```{toctree} -:maxdepth: -1 -:hidden: - -mmcloud_plugin_example -``` diff --git a/docs/deprecated_integrations/mmcloud_plugin/mmcloud_plugin_example.md b/docs/deprecated_integrations/mmcloud_plugin/mmcloud_plugin_example.md deleted file mode 100644 index 51e54a9e96..0000000000 --- a/docs/deprecated_integrations/mmcloud_plugin/mmcloud_plugin_example.md +++ /dev/null @@ -1,30 +0,0 @@ -(mmcloud_plugin_example)= -# Memory Machine Cloud - -This example shows how to use the MMCloud plugin to execute tasks on [MemVerge Memory Machine Cloud][mmcloud]. - -```{note} -To clone and run the example code on this page, see the [Flytesnacks repo][flytesnacks]. -``` - -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/mmcloud_plugin/mmcloud_plugin/mmcloud_plugin_example.py -:caption: mmcloud_plugin/mmcloud_plugin_example.py -:lines: 1-2 -``` - -`MMCloudConfig` configures `MMCloudTask`. Tasks specified with `MMCloudConfig` will be executed using MMCloud. Tasks will be executed with requests `cpu="1"` and `mem="1Gi"` by default. - -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/mmcloud_plugin/mmcloud_plugin/mmcloud_plugin_example.py -:caption: mmcloud_plugin/mmcloud_plugin_example.py -:lines: 8-15 -``` - -[Resource](https://docs.flyte.org/en/latest/user_guide/productionizing/customizing_task_resources.html) (cpu and mem) requests and limits, [container](https://docs.flyte.org/en/latest/user_guide/customizing_dependencies/multiple_images_in_a_workflow.html) images, and [environment](https://docs.flyte.org/en/latest/api/flytekit/generated/flytekit.task.html) variable specifications are supported. - -```{rli} https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/mmcloud_plugin/mmcloud_plugin/mmcloud_plugin_example.py -:caption: mmcloud_plugin/mmcloud_plugin_example.py -:lines: 18-32 -``` - -[mmcloud]: https://www.mmcloud.io/ -[flytesnacks]: https://github.com/flyteorg/flytesnacks/tree/master/examples/mmcloud_plugin