test

Signed-off-by: Kevin Su <[email protected]>

.github/workflows/end2end.yml

@@ -31,34 +31,10 @@ jobs:
       - uses: actions/setup-python@v3
         with:
           python-version: 3.11
-      - id: load-docker-cache-datacatalog
-        uses: actions/cache@v3
-        with:
-          path: /tmp/tmp/docker-images-datacatalog
-          key: ${{ inputs.cache_key }}-datacatalog
-      - id: load-docker-cache-flyteadmin
-        uses: actions/cache@v3
-        with:
-          path: /tmp/tmp/docker-images-flyteadmin
-          key: ${{ inputs.cache_key }}-flyteadmin
-      - id: load-docker-cache-flytecopilot
-        uses: actions/cache@v3
-        with:
-          path: /tmp/tmp/docker-images-flytecopilot
-          key: ${{ inputs.cache_key }}-flytecopilot
-      - id: load-docker-cache-flytepropeller
-        uses: actions/cache@v3
-        with:
-          path: /tmp/tmp/docker-images-flytepropeller
-          key: ${{ inputs.cache_key }}-flytepropeller
       - name: Create Sandbox Cluster
         run: |
-          cp /tmp/tmp/docker-images-datacatalog/snapshot-datacatalog.tar snapshot-datacatalog.tar
-          cp /tmp/tmp/docker-images-flyteadmin/snapshot-flyteadmin.tar snapshot-flyteadmin.tar
-          cp /tmp/tmp/docker-images-flytecopilot/snapshot-flytecopilot.tar snapshot-flytecopilot.tar
-          cp /tmp/tmp/docker-images-flytepropeller/snapshot-flytepropeller.tar snapshot-flytepropeller.tar
           flytectl config init
-          flytectl demo start --source=$(pwd)
+          flytectl demo start --source=$(pwd) --image ghcr.io/${{ github.repository_owner }}/flyte-sandbox-bundled:latest
       - name: Prime docker cache
         run: |
           flytectl demo exec -- docker load -i /root/snapshot-datacatalog.tar
@@ -98,26 +74,6 @@ jobs:
           project: flytesnacks
           version: ${{ env.FLYTESNACKS_VERSION }}
           domain: development
-      # - name: Pre Upgrade Tests
-      #   if: ${{ github.event.repository.name == 'flyteadmin' }}
-      #   env:
-      #     PRIORITIES: "${{ inputs.priorities }}"
-      #   run: |
-      #     make end2end_execute
-      - name: Upgrade Helm charts
-        run: |
-          flytectl demo exec -- helm repo add flyteorg https://flyteorg.github.io/flyte
-          flytectl demo exec -- helm repo update
-          flytectl demo exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest
-          # TODO(monorepo): the following commands are not correct.
-          # we have to separate the calls to replace the images into different commands.
-          # flytectl demo exec -- helm upgrade flyte -n flyte-core --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte-core -f /flyteorg/share/flyte/values-sandbox.yaml --wait \
-          #   --set datacatalog.image.repository=${{ github.repository_owner }}/datacalog,datacatalog.image.tag=latest
-          #   --set flyteadmin.image.repository=${{ github.repository_owner }}/datacalog,flyteadmin.image.tag=latest
-          #   --set flytecopilot.image.repository=${{ github.repository_owner }}/datacalog,flytecopilot.image.tag=latest
-          #   --set flytepropeller.image.repository=${{ github.repository_owner }}/datacalog,flytepropeller.image.tag=latest
-
-          flytectl demo exec -- k3s kubectl get pods -n flyte -oyaml
       - name: Post Upgrade Tests
         env:
           PRIORITIES: "${{ inputs.priorities }}"

boilerplate/flyte/end2end/run-tests.py

@@ -22,10 +22,12 @@ def execute_workflow(
     workflow_name,
     inputs,
     cluster_pool_name: Optional[str] = None,
+    wait: bool = False,
 ):
     print(f"Fetching workflow={workflow_name} and version={version}")
     wf = remote.fetch_workflow(name=workflow_name, version=version)
-    return remote.execute(wf, inputs=inputs, wait=False, cluster_pool=cluster_pool_name)
+    print(f"Executing workflow={workflow_name} and version={version}")
+    return remote.execute(wf, inputs=inputs, wait=wait, cluster_pool=cluster_pool_name)
 
 
 def executions_finished(
@@ -64,6 +66,7 @@ def schedule_workflow_groups(
     terminate_workflow_on_failure: bool,
     parsed_manifest: List[dict],
     cluster_pool_name: Optional[str] = None,
+    parallel: bool = True,
 ) -> Dict[str, bool]:
     """
     Schedule workflows executions for all workflow groups and return True if all executions succeed, otherwise
@@ -79,7 +82,7 @@ def schedule_workflow_groups(
         if workflow_group_item:
             workflows = workflow_group_item[0]["examples"]
         executions_by_wfgroup[wf_group] = [
-            execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name)
+            execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name, not parallel)
             for workflow in workflows
         ]
 
@@ -135,6 +138,7 @@ def run(
     test_project_name: str,
     test_project_domain: str,
     cluster_pool_name: Optional[str] = None,
+    parallel: bool = True,
 ) -> List[Dict[str, str]]:
     remote = FlyteRemote(
         Config.auto(config_file=config_file_path),
@@ -182,6 +186,7 @@ def run(
         terminate_workflow_on_failure,
         parsed_manifest,
         cluster_pool_name,
+        parallel
     )
 
     for workflow_group, succeeded in results_by_wfgroup.items():
@@ -241,6 +246,12 @@ def run(
     is_flag=False,
     help="Name of domain in project to run functional tests on",
 )
+@click.option(
+    "--parallel",
+    is_flag=True,
+    default=False,
+    help="Run tests in parallel if set to True",
+)
 @click.argument(
     "cluster_pool_name",
     required=False,
@@ -255,6 +266,7 @@ def cli(
     terminate_workflow_on_failure,
     test_project_name,
     test_project_domain,
+    parallel,
     cluster_pool_name,
 ):
     print(f"return_non_zero_on_failure={return_non_zero_on_failure}")
@@ -266,6 +278,7 @@ def cli(
         test_project_name,
         test_project_domain,
         cluster_pool_name,
+        parallel
     )
 
     # Write a json object in its own line describing the result of this run to stdout