ci: Refactor the workflows (#322)

.github/workflows/check-release.yml

@@ -31,16 +31,33 @@ jobs:
       package_name: ${{ steps.version_check.outputs.package_name }}
       package_version: ${{ steps.version_check.outputs.package_version }}
 
-  test:
+  airflow-test:
     needs: check-version
-    if: ${{ needs.check-version.outputs.new_release == 'true' }}
-    uses: ./.github/workflows/check-plugin.yml
-    with:
-      plugin: ${{ needs.check-version.outputs.package_name }}
+    if: ${{ needs.check-version.outputs.new_release == 'true' &&  needs.check-version.outputs.package_name == 'kedro-airflow' }}
+    uses: ./.github/workflows/kedro-airflow.yml
+
+  datasets-test:
+    needs: check-version
+    if: ${{ needs.check-version.outputs.new_release == 'true'  &&  needs.check-version.outputs.package_name == 'kedro-datasets' }}
+    uses: ./.github/workflows/kedro-datasets.yml
+
+  docker-test:
+    needs: check-version
+    if: ${{ needs.check-version.outputs.new_release == 'true'  &&  needs.check-version.outputs.package_name == 'kedro-docker' }}
+    uses: ./.github/workflows/kedro-docker.yml
+
+  telemetry-test:
+    needs: check-version
+    if: ${{ needs.check-version.outputs.new_release == 'true'  &&  needs.check-version.outputs.package_name == 'kedro-telemetry' }}
+    uses: ./.github/workflows/kedro-telemetry.yml
 
   build-publish:
-    needs: [check-version, test]
-    if: ${{ needs.check-version.outputs.new_release == 'true' }}
+    needs: [ check-version, airflow-test, docker-test, datasets-test, telemetry-test ]
+    if: |
+      always() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled') &&
+      needs.check-version.outputs.new_release == 'true'
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3

.github/workflows/e2e-tests.yml

@@ -0,0 +1,43 @@
+name: Run e2e tests
+
+on:
+  workflow_call:
+    inputs:
+      plugin:
+        type: string
+      os:
+        type: string
+      python-version:
+        type: string
+
+jobs:
+  e2e-tests:
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install dependencies
+        run: |
+          cd ${{ inputs.plugin }}
+          pip install git+https://github.com/kedro-org/kedro@main
+          pip install ".[test]"
+      - name: pip freeze
+        run: pip freeze
+      - name: Run end to end tests
+        # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run`
+        # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241
+        shell: 'script -q -e -c "bash {0}"'
+        run: make plugin=${{ inputs.plugin }} e2e-tests

.github/workflows/kedro-airflow.yml

@@ -1,6 +1,7 @@
-name: Run checks on Kedro-Airflow
+name: Run checks on kedro-airflow
 
 on:
+  workflow_call:
   push:
     branches:
       - main
@@ -15,12 +16,33 @@ on:
       - "kedro-datasets/**"
       - "kedro-docker/**"
       - "kedro-telemetry/**"
-  schedule:
-    # Run every day at midnight (UTC time)
-    - cron: '0 0 * * *'
 
 jobs:
-  airflow-test:
-    uses: ./.github/workflows/check-plugin.yml
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-airflow
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+    with:
+      plugin: kedro-airflow
+      os: ubuntu-latest
+      python-version: "3.8"
+
+  e2e-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      plugin: kedro-airflow
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}

.github/workflows/kedro-datasets.yml

@@ -1,6 +1,7 @@
-name: Run checks on Kedro-Datasets
+name: Run checks on kedro-datasets
 
 on:
+  workflow_call:
   push:
     branches:
       - main
@@ -15,12 +16,46 @@ on:
       - "kedro-airflow/**"
       - "kedro-docker/**"
       - "kedro-telemetry/**"
-  schedule:
-    # Run every day at midnight (UTC time)
-    - cron: '0 0 * * *'
 
 jobs:
-  datasets-test:
-    uses: ./.github/workflows/check-plugin.yml
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-datasets
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+    with:
+      plugin: kedro-datasets
+      os: ubuntu-latest
+      python-version: "3.11"
+
+  RTD-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: kedro-datasets-ubuntu-latest-python-"3.8"
+          restore-keys: kedro-datasets
+      - name: Install dependencies
+        run: |
+          cd kedro-datasets
+          pip install ".[docs]"
+          pip install ".[test]"
+      - name: RTD build for kedro-datasets
+        run: |
+          make rtd