[DPE-2651] Juju3 pipelines / Testing secrets (#333)

* Juju3 pipelines

* New ci + tox.ini env vars to indicate Juju version

* pyproject.toml, poetry.lock -- adding pytest-mock, ops -> integration group, etc.

* Fixtures to enforce correct Juju version recognition for secrets

* Test changes adapted to Juju2+3

* Tests: Juju2 vs Juju3 (secrets vs databag)

* New unittests for internal secrets

* NO databag usage!

* data_platform_libs/data_interfaces on v19

* Updating CONTRIBUTING doc

* Defaults set to Juju3

* Free up disk space on runner

.github/workflows/ci.yaml

@@ -19,6 +19,9 @@ jobs:
     uses: canonical/data-platform-workflows/.github/workflows/[email protected]
 
   unit-test:
+    strategy:
+      matrix:
+        juju-version:  ["2.9", "3.1"]
     name: Unit test charm
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -31,6 +34,10 @@ jobs:
           pipx install poetry
       - name: Run tests
         run: tox run -e unit
+        env:
+          # This env var is only to indicate Juju version to "simulate" in the unit tests
+          # No libjuju is being actually used in unit testing
+          LIBJUJU_VERSION_SPECIFIER: ${{ matrix.juju-version }}
       - name: Upload Coverage to Codecov
         uses: codecov/codecov-action@v3
 
@@ -98,7 +105,35 @@ jobs:
           # Update whenever charmcraft.yaml is changed
           - series: jammy
             bases-index: 0
-    name: (GH hosted) ${{ matrix.groups.job_name }} | ${{ matrix.ubuntu-versions.series }}
+        juju-snap-channel:  ["2.9/stable", "3.1/stable"]
+        include:
+          - juju-snap-channel: "3.1/stable"
+            agent-version: "3.1.6"
+            libjuju-version:  "3.2.0.1"
+          - juju-snap-channel: "2.9/stable"
+            agent-version: "2.9.45"
+            libjuju-version:  "2.9.44.1"
+        exclude:
+          # Disabling HA tests, as long as we want to have a limited pipeline on Juju3
+          - juju-snap-channel: "3.1/stable"
+            groups:
+              job_name: "high_availability/test_replication.py | group 1"
+          - juju-snap-channel: "3.1/stable"
+            groups:
+              job_name: "high_availability/test_replication.py | group 2"
+          - juju-snap-channel: "3.1/stable"
+            groups:
+              job_name: "high_availability/test_replication.py | group 3"
+          - juju-snap-channel: "3.1/stable"
+            groups:
+              job_name: "high_availability/test_self_healing.py | group 1"
+          - juju-snap-channel: "3.1/stable"
+            groups:
+              job_name: "high_availability/test_upgrade.py | group 1"                                                                                                                                           
+          - juju-snap-channel: "3.1/stable"
+            groups:
+              job_name: "high_availability/test_upgrade_from_stable.py | group 1"
+    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.groups.job_name }} | ${{ matrix.ubuntu-versions.series }}
     needs:
       - lint
       - unit-test
@@ -117,11 +152,22 @@ jobs:
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: lxd
-          bootstrap-options: "--agent-version 2.9.43"
+          bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
+          juju-channel: ${{ matrix.juju-snap-channel }}
+      - name: Update python-libjuju version
+        if: ${{ matrix.libjuju-version == '2.9.44.1' }}
+        run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
       - name: Download packed charm(s)
         uses: actions/download-artifact@v3
         with:
           name: ${{ needs.build.outputs.artifact-name }}
+      - name: Free up disk space
+        run: |
+          # From https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Select test stability level
         id: select-test-stability
         run: |
@@ -136,6 +182,7 @@ jobs:
       - name: Run integration tests
         run: tox run -e integration -- "${{ matrix.groups.path_to_test_file }}" --group="${{ matrix.groups.group_number }}" -m '${{ steps.select-test-stability.outputs.mark_expression }}' --mysql-charm-series="${{ matrix.ubuntu-versions.series }}" --mysql-charm-bases-index="${{ matrix.ubuntu-versions.bases-index }}"
         env:
+          LIBJUJU_VERSION_SPECIFIER: ${{ matrix.libjuju-version }}
           SECRETS_FROM_GITHUB: |
             {
               "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",

CONTRIBUTING.md

@@ -44,6 +44,13 @@ tox run -e integration   # integration tests
 tox                      # runs 'lint' and 'unit' environments
 ```
 
+Tests by default are using Juju 3. In case tests are to be run against Juju 3, the following
+environment variable should be defined with a valid `juju` Python library version:
+
+```
+export LIBJUJU_VERSION_SPECIFIER=2.9.44.1
+```
+
 ## Build charm
 
 Build the charm in this git repository using: