Merge branch 'master' into feature/datacatalog-cache-deletion

Signed-off-by: Paul Dittamo <[email protected]>

Makefile

@@ -37,6 +37,7 @@ kustomize:
 .PHONY: helm
 helm: ## Generate K8s Manifest from Helm Charts.
 	bash script/generate_helm.sh
+	make -C docker/sandbox-bundled manifests
 
 .PHONY: release_automation
 release_automation:

charts/flyte-core/README.md

@@ -194,6 +194,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteagent.enabled | bool | `false` |  |
 | flyteagent.plugin_config.plugins.agentService.defaultAgent.endpoint | string | `"dns:///flyteagent.flyte.svc.cluster.local:8000"` |  |
 | flyteagent.plugin_config.plugins.agentService.defaultAgent.insecure | bool | `true` |  |
+| flyteagent.podLabels | object | `{}` | Labels for flyteagent pods |
 | flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment |
 | flyteconsole.enabled | bool | `true` |  |
 | flyteconsole.ga.enabled | bool | `false` |  |

charts/flyte-core/values.yaml

@@ -279,6 +279,8 @@ flyteagent:
         defaultAgent:
           endpoint: "dns:///flyteagent.flyte.svc.cluster.local:8000"
           insecure: true
+  # -- Labels for flyteagent pods
+  podLabels: {}
 
 #
 # FLYTEPROPELLER SETTINGS

docker/sandbox-bundled/Makefile

@@ -15,12 +15,15 @@ flyte: FLYTECONSOLE_VERSION := latest
 flyte:
 	$(foreach arch,amd64 arm64,$(call FLYTE_BINARY_BUILD,$(arch)))
 
-.PHONY: manifests
-manifests:
-	mkdir -p manifests
+.PHONY: dep_update
+dep_update:
 	helm dependency update ../../charts/flyteagent
 	helm dependency update ../../charts/flyte-binary
 	helm dependency update ../../charts/flyte-sandbox
+
+.PHONY: manifests
+manifests:
+	mkdir -p manifests
 	kustomize build \
 		--enable-helm \
 		--load-restrictor=LoadRestrictionsNone \
@@ -35,7 +38,7 @@ manifests:
 		kustomize/complete-agent > manifests/complete-agent.yaml
 
 .PHONY: build
-build: flyte manifests
+build: flyte dep_update manifests
 	[ -n "$(shell docker buildx ls | awk '/^flyte-sandbox / {print $$1}')" ] || \
 		docker buildx create --name flyte-sandbox \
 		--driver docker-container --driver-opt image=moby/buildkit:master \

docs/conf.py

@@ -339,7 +339,8 @@
     "source_regex_mapping": REPLACE_PATTERNS,
     "list_table_toc": [
        "flytesnacks/tutorials",
-        "flytesnacks/integrations",
+       "flytesnacks/integrations",
+       "flytesnacks/deprecated_integrations"
     ],
     "dev_build": bool(int(os.environ.get("MONODOCS_DEV_BUILD", 1))),
 }

docs/core_use_cases/analytics.md

@@ -173,7 +173,7 @@ and [DBT](https://github.com/flyteorg/flytekit/tree/master/plugins/flytekit-dbt)
 integrations.
 
 If you need to connect to a database, Flyte provides first-party
-support for {ref}`AWS Athena <aws-athena>`, {ref}`Google Bigquery <big-query>`,
-{ref}`Snowflake <plugins-snowflake>`, {ref}`SQLAlchemy <sql_alchemy>`, and
+support for {ref}`AWS Athena <aws-athena>`, {ref}`Google Bigquery <bigquery_agent>`,
+{ref}`Snowflake <snowflake_agent>`, {ref}`SQLAlchemy <sql_alchemy>`, and
 {ref}`SQLite3 <integrations_sql_sqlite3>`.
 ```

docs/core_use_cases/data_engineering.md

@@ -170,6 +170,6 @@ and [DBT](https://github.com/flyteorg/flytekit/tree/master/plugins/flytekit-dbt)
 integrations.
 
 For database connectors, Flyte provides first-party support for {ref}`AWS Athena <aws-athena>`,
-{ref}`Google Bigquery <big-query>`, {ref}`Snowflake <plugins-snowflake>`,
+{ref}`Google BigQuery <bigquery_agent>`, {ref}`Snowflake <snowflake_agent>`,
 {ref}`SQLAlchemy <sql_alchemy>`, and {ref}`SQLite3 <integrations_sql_sqlite3>`.
 ```

docs/deployment/agents/airflow.rst

@@ -0,0 +1,97 @@
+.. _deployment-agent-setup-airflow:
+
+Airflow agent
+=================
+
+This guide provides an overview of how to set up the Airflow agent in your Flyte deployment.
+Please note that the you don't need an Airflow cluster to run the Airflow tasks, since Flytekit will
+automatically compile Airflow tasks to Flyte tasks and execute them on the Flyte cluster.
+
+Specify agent configuration
+----------------------------
+
+.. tabs::
+
+    .. group-tab:: Flyte binary
+
+      Edit the relevant YAML file to specify the agent.
+
+      .. code-block:: bash
+
+        kubectl edit configmap flyte-sandbox-config -n flyte
+
+      .. code-block:: yaml
+        :emphasize-lines: 7,11,16
+
+        tasks:
+          task-plugins:
+            enabled-plugins:
+              - container
+              - sidecar
+              - k8s-array
+              - agent-service
+            default-for-task-types:
+              - container: container
+              - container_array: k8s-array
+              - airflow: agent-service
+
+        plugins:
+          agent-service:
+            supportedTaskTypes:
+            - airflow
+
+    .. group-tab:: Flyte core
+
+      Create a file named ``values-override.yaml`` and add the following configuration to it.
+
+      .. code-block:: yaml
+
+        configmap:
+          enabled_plugins:
+            # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig)
+            tasks:
+              # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig)
+              task-plugins:
+                # -- [Enabled Plugins](https://pkg.go.dev/github.com/flyteorg/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend
+                enabled-plugins:
+                  - container
+                  - sidecar
+                  - k8s-array
+                  - agent-service
+                default-for-task-types:
+                  container: container
+                  sidecar: sidecar
+                  container_array: k8s-array
+                  airflow: agent-service
+            plugins:
+              agent-service:
+                supportedTaskTypes:
+                - airflow
+
+
+Upgrade the Flyte Helm release
+------------------------------
+
+.. tabs::
+
+  .. group-tab:: Flyte binary
+
+    .. code-block:: bash
+
+      helm upgrade <RELEASE_NAME> flyteorg/flyte-binary -n <YOUR_NAMESPACE> --values <YOUR_YAML_FILE>
+
+    Replace ``<RELEASE_NAME>`` with the name of your release (e.g., ``flyte-backend``),
+    ``<YOUR_NAMESPACE>`` with the name of your namespace (e.g., ``flyte``),
+    and ``<YOUR_YAML_FILE>`` with the name of your YAML file.
+
+  .. group-tab:: Flyte core
+
+    .. code-block:: bash
+
+      helm upgrade <RELEASE_NAME> flyte/flyte-core -n <YOUR_NAMESPACE> --values values-override.yaml
+
+    Replace ``<RELEASE_NAME>`` with the name of your release (e.g., ``flyte``)
+
+    and ``<YOUR_NAMESPACE>`` with the name of your namespace (e.g., ``flyte``).
+
+For Airflow agent on the Flyte cluster, see `Airflow agent <https://docs.flyte.org/en/latest/flytesnacks/examples/airflow_agent/index.html>`_.

docs/deployment/agents/bigquery.rst

@@ -1,6 +1,6 @@
 .. _deployment-agent-setup-bigquery:
 
-Google BigQuery Agent
+Google BigQuery agent
 ======================
 
 This guide provides an overview of setting up BigQuery agent in your Flyte deployment.
@@ -103,4 +103,4 @@ Upgrade the Flyte Helm release
 
     and ``<YOUR_NAMESPACE>`` with the name of your namespace (e.g., ``flyte``).
 
-For BigQuery plugin on the Flyte cluster, please refer to `BigQuery Plugin Example <https://docs.flyte.org/en/latest/flytesnacks/examples/bigquery_plugin/bigquery.html>`_
+For BigQuery agent on the Flyte cluster, see `BigQuery agent <https://docs.flyte.org/en/latest/flytesnacks/examples/bigquery_integration/bigquery_agent.html>`_.

docs/deployment/agents/databricks.rst

@@ -1,6 +1,6 @@
 .. _deployment-agent-setup-databricks:
 
-Databricks Agent
+Databricks agent
 =================
 
 This guide provides an overview of how to set up Databricks agent in your Flyte deployment.
@@ -291,4 +291,4 @@ Wait for the upgrade to complete. You can check the status of the deployment pod
 
   kubectl get pods -n flyte
 
-For databricks plugin on the Flyte cluster, please refer to `Databricks Plugin Example <https://docs.flyte.org/en/latest/flytesnacks/examples/databricks_plugin/index.html>`_
+For Databricks agent on the Flyte cluster, see `Databricks agent <https://docs.flyte.org/en/latest/flytesnacks/examples/databricks_integration/databricks_agent.html>`_.

docs/deployment/agents/index.md

@@ -2,31 +2,40 @@
 
 # Agent Setup
 
-.. tags:: Agent, Integration, Data, Advanced
+```{tags} Agent, Integration, Data, Advanced
+```
+
+To set configure your Flyte deployment for agents, see the documentation below.
 
-Discover the process of setting up Agents for Flyte.
+:::{note}
+If you are using a managed deployment of Flyte, you will need to contact your deployment administrator to configure agents in your deployment.
+:::
 
 ```{list-table}
 :header-rows: 0
 :widths: 20 30
 
-* - {ref}`Bigquery Agent <deployment-agent-setup-bigquery>`
-  - Guide to setting up the Bigquery agent.
+* - {ref}`Airflow Agent <deployment-agent-setup-airflow>`
+  - Configuring your Flyte deployment for the Airflow agent
+* - {ref}`Databricks Agent <deployment-agent-setup-databricks>`
+  - Configuring your Flyte deployment for the Databricks agent.
+* - {ref}`Google BigQuery Agent <deployment-agent-setup-bigquery>`
+  - Configuring your Flyte deployment for the BigQuery agent.
 * - {ref}`MMCloud Agent <deployment-agent-setup-mmcloud>`
-  - Guide to setting up the MMCloud agent.
+  - Configuring your Flyte deployment for the MMCloud agent.
 * - {ref}`Sensor Agent <deployment-agent-setup-sensor>`
-  - Guide to setting up the Sensor agent.
-* - {ref}`Databricks Agent <deployment-agent-setup-databricks>`
-  - Guide to setting up the Databricks agent.
+  - Configuring your Flyte deployment for the sensor agent.
 ```
 
 ```{toctree}
 :maxdepth: 1
 :name: Agent setup
 :hidden:
 
+airflow
+databricks
 bigquery
 mmcloud
-databricks
 sensor
+snowflake
 ```

docs/deployment/agents/mmcloud.rst

@@ -118,4 +118,4 @@ Wait for the upgrade to complete. You can check the status of the deployment pod
 
   kubectl get pods -n flyte
 
-For MMCloud plugin on the Flyte cluster, please refer to `Memory Machine Cloud Plugin Example <https://docs.flyte.org/en/latest/flytesnacks/examples/mmcloud_plugin/index.html>`_
+For MMCloud agent on the Flyte cluster, see `MMCloud agent <https://docs.flyte.org/en/latest/flytesnacks/examples/mmcloud_integration/mmcloud_agent.html>`_.

docs/deployment/agents/sensor.rst

@@ -1,13 +1,13 @@
 .. _deployment-agent-setup-sensor:
 
-Sensor Agent
+Sensor agent
 =================
 
-Sensor enables users to continuously check for a file or a condition to be met periodically.
+The `sensor agent <https://docs.flyte.org/en/latest/flytesnacks/examples/sensor/index.html>`_ enables users to continuously check for a file or a condition to be met periodically.
 
 When the condition is met, the sensor will complete.
 
-This guide provides an overview of how to set up Sensor in your Flyte deployment.
+This guide provides an overview of how to set up the sensor agent in your Flyte deployment.
 
 Spin up a cluster
 -----------------
@@ -43,7 +43,7 @@ Spin up a cluster
 Specify agent configuration
 ----------------------------
 
-Enable the Sensor agent by adding the following config to the relevant YAML file(s):
+Enable the sensor agent by adding the following config to the relevant YAML file(s):
 
 .. tabs::
 
@@ -77,7 +77,7 @@ Enable the Sensor agent by adding the following config to the relevant YAML file
 
     .. group-tab:: Flyte core
 
-      Create a file named ``values-override.yaml`` and add the following configuration to it.
+      Create a file named ``values-override.yaml`` and add the following configuration to it:
 
       .. code-block:: yaml