increase shard count (test)

.github/workflows/integration-test.yml

@@ -1,55 +1,50 @@
+---
 name: Integration Tests
-
 on:
   workflow_call:
     inputs:
       os:
-        description: 'OS'
+        description: OS
         type: string
         required: true
       python-version:
-        description: 'Python version'
+        description: Python version
         type: string
         required: true
       test_environment:
-        description: 'The test environment'
+        description: The test environment
         type: string
         required: true
       enable_tmate:
-        description: 'Enable tmate session for debugging'
+        description: Enable tmate session for debugging
         type: string
         required: false
-        default: "never"
+        default: never
       tmate_timeout:
-        description: 'Timeout for tmate session (minutes)'
+        description: Timeout for tmate session (minutes)
         type: number
         required: false
         default: 30
-
   workflow_dispatch:
     inputs:
       os:
-        description: 'OS'
+        description: OS
         type: choice
-        options: 
+        options:
           - ubuntu-latest
           - macos-latest
           - windows-latest
           - ubuntu-dind-runners
         required: false
-        default: 'ubuntu-latest'
+        default: ubuntu-latest
       python-version:
-        description: 'Python version'
+        description: Python version
         type: choice
-        options:
-          - '3.8'
-          - '3.9'
-          - '3.10'
-          - '3.11'
+        options: ['3.8', '3.9', '3.10', '3.11']
         required: false
         default: '3.8'
       test_environment:
-        description: 'The test environment'
+        description: The test environment
         type: choice
         options:
           # Default ZenML deployments
@@ -68,43 +63,34 @@ on:
           - docker-server-airflow-orchestrator-mysql
           - docker-server-airflow-orchestrator-mariadb
         required: false
-        default: 'default'
+        default: default
       enable_tmate:
-        description: 'Enable tmate session for debugging'
+        description: Enable tmate session for debugging
         type: choice
-        options:
-          - 'no'
-          - 'on-failure'
-          - 'always'
-          - 'before-tests'
+        options: [no, on-failure, always, before-tests]
         required: false
         default: 'no'
       tmate_timeout:
-        description: 'Timeout for tmate session (minutes)'
+        description: Timeout for tmate session (minutes)
         type: number
         required: false
         default: 30
-
 jobs:
   integration-and-template-tests:
     name: integration-and-template-tests
     runs-on: ${{ inputs.os }}
     strategy:
       fail-fast: false
       matrix:
-        shard: [
-             0,  1,  2,  3,
-             4,  5,  6,  7,
-        ]
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
     env:
       ZENML_DEBUG: 1
       ZENML_ANALYTICS_OPT_IN: false
-      PYTHONIOENCODING: 'utf-8'
+      PYTHONIOENCODING: utf-8
       # on MAC OS, we need to set this environment variable
       # to fix problems with the fork() calls (see this thread
       # for more information: http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html)
       OBJC_DISABLE_INITIALIZE_FORK_SAFETY: 'YES'
-
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_US_EAST_1_ENV_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_US_EAST_1_ENV_SECRET_ACCESS_KEY }}
       AWS_US_EAST_1_SERVER_URL: ${{ secrets.AWS_US_EAST_1_SERVER_URL }}
@@ -114,11 +100,9 @@ jobs:
       GCP_US_EAST4_SERVER_USERNAME: ${{ secrets.GCP_US_EAST4_SERVER_USERNAME }}
       GCP_US_EAST4_SERVER_PASSWORD: ${{ secrets.GCP_US_EAST4_SERVER_PASSWORD }}
     if: ${{ ! startsWith(github.event.head_commit.message, 'GitBook:') }}
-
     defaults:
       run:
         shell: bash
-
     steps:
       - name: Maximize space for Docker
         uses: easimon/maximize-build-space@v10
@@ -129,34 +113,31 @@ jobs:
           remove-android: 'true'
           remove-haskell: 'true'
           build-mount-path: /var/lib/docker/
-        if: inputs.os=='ubuntu-latest' && (contains(inputs.test_environment, 'docker') || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment, 'airflow') || contains(inputs.test_environment, 'kubernetes'))
-
+        if: inputs.os=='ubuntu-latest' && (contains(inputs.test_environment, 'docker')
+          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
+          'airflow') || contains(inputs.test_environment, 'kubernetes'))
       - name: Reload Docker
-        run:
-          sudo systemctl restart docker
-        if: inputs.os=='ubuntu-latest' && (contains(inputs.test_environment, 'docker') || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment, 'airflow') || contains(inputs.test_environment, 'kubernetes'))
-
+        run: sudo systemctl restart docker
+        if: inputs.os=='ubuntu-latest' && (contains(inputs.test_environment, 'docker')
+          || contains(inputs.test_environment, 'kubeflow') || contains(inputs.test_environment,
+          'airflow') || contains(inputs.test_environment, 'kubernetes'))
       - uses: actions/[email protected]
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           role-to-assume: ${{ secrets.AWS_US_EAST_1_ENV_ROLE_ARN }}
           aws-region: us-east-1
         if: contains(inputs.test_environment, 'aws')
-
       - name: Configure GCP credentials
-        uses: 'google-github-actions/auth@v2'
+        uses: google-github-actions/auth@v2
         with:
-          credentials_json: '${{ secrets.GCP_US_EAST4_ENV_CREDENTIALS }}'
+          credentials_json: ${{ secrets.GCP_US_EAST4_ENV_CREDENTIALS }}
         if: contains(inputs.test_environment, 'gcp')
-
-      - name: 'Set up gcloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v1'
+      - name: Set up gcloud SDK
+        uses: google-github-actions/setup-gcloud@v1
         with:
-          install_components: 'gke-gcloud-auth-plugin'
+          install_components: gke-gcloud-auth-plugin
         if: contains(inputs.test_environment, 'gcp')
-
       - name: Setup environment
         uses: ./.github/actions/setup_environment
         with:
@@ -165,23 +146,18 @@ jobs:
           os: ${{ inputs.os }}
           runners_cache_access_key_id: ${{ secrets.RUNNERS_CACHE_ACCESS_KEY_ID }}
           runners_cache_secret_access_key: ${{ secrets.RUNNERS_CACHE_SECRET_ACCESS_KEY }}
-
       - name: Install docker-compose for non-default environments
         if: inputs.test_environment != 'default'
         run: pip install docker-compose
-
       - name: Install Linux System Dependencies
         if: inputs.os=='ubuntu-latest'
         run: sudo apt install graphviz
-
       - name: Install MacOS System Dependencies
         if: runner.os=='macOS'
         run: brew install graphviz
-
       - name: Install Windows System Dependencies
         if: runner.os=='Windows'
         run: choco install graphviz
-
       - name: Install Docker and Colima on MacOS
         if: runner.os=='macOS'
         run: |
@@ -197,67 +173,56 @@ jobs:
 
           # This is required for the Docker Python SDK to work
           sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
-
       - name: Install kubectl on Linux
         run: |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
           sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
         if: inputs.os=='ubuntu-latest'
-
       - name: Install kubectl on MacOS
         run: |
           curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/darwin/amd64/kubectl"
           sudo install -o root -g 0 -m 0755 kubectl /usr/local/bin/kubectl
         if: runner.os=='macOS'
-
       - name: Install K3D
         run: |
           curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
         if: runner.os!='Windows' && contains(inputs.test_environment, 'kubeflow')
-
       - name: Login to Amazon ECR
         id: login-ecr
         run: |
           aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 715803424590.dkr.ecr.us-east-1.amazonaws.com
         if: contains(inputs.test_environment, 'aws')
-
       - name: Login to Amazon EKS
         id: login-eks
         run: |
           aws eks --region us-east-1 update-kubeconfig --name zenml-ci-cluster --alias zenml-ci-aws-us-east-1
         if: contains(inputs.test_environment, 'aws')
-
       - name: Login to Google ECR
         run: |
           gcloud auth configure-docker --project zenml-ci
         if: contains(inputs.test_environment, 'gcp')
-
       - name: Login to Google GKE
         uses: google-github-actions/get-gke-credentials@v2
         with:
           cluster_name: zenml-ci-cluster
           location: us-east4
           project_id: zenml-ci
         if: contains(inputs.test_environment, 'gcp')
-
       - name: Setup tmate session before tests
         if: ${{ inputs.enable_tmate == 'before-tests' }}
         uses: mxschmitt/[email protected]
         with:
           timeout-minutes: ${{ inputs.tmate_timeout }}
-
       - name: Test
         run: |
           bash scripts/test-coverage-xml.sh integration ${{ inputs.test_environment }} $SHARD
-
       - name: Setup tmate session after tests
         if: ${{ inputs.enable_tmate == 'always' || (inputs.enable_tmate == 'on-failure' && failure()) }}
         uses: mxschmitt/[email protected]
         with:
           timeout-minutes: ${{ inputs.tmate_timeout }}
-
       - name: Verify Python Env unaffected
-        run: |
+        run: |-
           zenml integration list
           pip list
           pip check || true