[DPE-6282] update poetry, workflows, and more (#529)

# Issue

Multiple CI workflows are failing and we are not up to date with [the
latest
standards](https://chat.canonical.com/canonical/pl/wekd7r4eqtdn7mxhgxgej8zzsr)
for Carl to move us to charmcraft 3.

# Solution

to fix build
- update to poetry 2
- use new workflow v24.0.6
- use correct charm version file
- use new DP pipelines

.github/workflows/ci.yaml

@@ -16,7 +16,7 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.6
 
   unit-test:
     name: Unit test charm
@@ -34,14 +34,16 @@ jobs:
 
   terraform-test:
     name: Terraform - Lint and Simple Deployment
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     timeout-minutes: 120
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-
+      - name: Install terraform snap
+        run: |
+          sudo snap install terraform --channel=latest/stable --classic
       - name: lint charm module
         working-directory: ./terraform
         run: |
@@ -101,7 +103,7 @@ jobs:
 
   check-terraform:
     name: Check Terraform
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./terraform
@@ -138,7 +140,7 @@ jobs:
           - tests/integration/sharding_tests/application
           - tests/integration/relation_tests/new_relations/application-charm
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
     with:
       path-to-charm-directory: ${{ matrix.path }}
 
@@ -158,7 +160,7 @@ jobs:
       - lint
       - unit-test
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.6
     with:
       artifact-prefix: packed-charm-cache-false # TODO revert to "packed-charm-cache-true" when cache re-enabled
       cloud: lxd

.github/workflows/release.yaml

@@ -14,7 +14,7 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
     with:
       artifact-prefix: temp-release-packed-charm  # TODO: remove after caching re-enabled on PR
 
@@ -23,7 +23,7 @@ jobs:
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v21.0.0
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6
     with:
       channel: 6/edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}

charmcraft.yaml

@@ -14,6 +14,10 @@ parts:
     charm-strict-dependencies: true
     override-build: |
       rustup default stable
+      # Convert subset of poetry.lock to requirements.txt
+      curl -sSL https://install.python-poetry.org | python3 -
+      /root/.local/bin/poetry self add poetry-plugin-export
+      /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt
       craftctl default
     build-snaps:
       - rustup
@@ -27,7 +31,10 @@ parts:
   version_data:
     plugin: dump
     source: .
+    stage:
+      # Exclude requirements.txt file during staging
+      # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2
+      - -requirements.txt
     prime:
       - charm_version
-      - charm_internal_version
       - workload_version

pyproject.toml

@@ -1,12 +1,6 @@
 [tool.poetry]
-name = "mongodb"
-version = "0.0.1"
-description = ""
-license = "Apache-2.0"
-authors = []
-readme = "README.md"
-homepage = "https://charmhub.io/mongodb?channel=6/edge"
-repository = "https://github.com/canonical/mongodb-operator"
+package-mode = false
+requires-poetry = ">=2.0.0"
 
 [tool.poetry.dependencies]
 python = "^3.10.12"
@@ -16,7 +10,7 @@ pymongo = "^4.7.3"
 tenacity = "^8.2.3"
 pyyaml = "^6.0.1"
 jinja2 = "^3.1.3"
-poetry-core = "^1.9.0"
+poetry-core = "^2.0"
 data-platform-helpers = "^0.1.3"
 pyOpenSSL = "^24.2.1"
 
@@ -28,6 +22,9 @@ jsonschema = "^4.22.0"  # tls_certificates lib v3
 cosl = "^0.0.16"  # loki_push_api
 pyOpenSSL = "^24.2.1"
 
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"
+
 [tool.poetry.group.format]
 optional = true
 
@@ -70,13 +67,12 @@ pytest = "^8.1.1"
 pytest-asyncio = "^0.21.1"
 pytest-mock = "^3.14.0"
 pytest-operator = "^0.36.0"
-pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
-pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
-pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/github_secrets"}
-allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v21.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
+pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
+pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
+pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/github_secrets"}
+allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
 
 [build-system]
-requires = ["poetry-core>=1.9.0"]
 build-backend = "poetry.core.masonry.api"
 
 # Testing tools configuration

src/config.py

@@ -21,7 +21,7 @@ class Config:
     MONGODB_SNAP_DATA_DIR = "/var/snap/charmed-mongodb/current"
     MONGOD_CONF_DIR = f"{MONGODB_SNAP_DATA_DIR}/etc/mongod"
     MONGOD_CONF_FILE_PATH = f"{MONGOD_CONF_DIR}/mongod.conf"
-    CHARM_INTERNAL_VERSION_FILE = "charm_internal_version"
+    CHARM_INTERNAL_VERSION_FILE = "charm_version"
     SNAP_PACKAGES = [("charmed-mongodb", "6/edge", 123)]
 
     MONGODB_COMMON_PATH = Path("/var/snap/charmed-mongodb/common")

tests/integration/upgrade/test_sharding_rollback.py

@@ -129,7 +129,7 @@ async def test_rollback_on_shard_and_config_server(
     new_charm = await ops_test.build_charm(".")
     await run_upgrade_sequence(ops_test, CONFIG_SERVER_APP_NAME, new_charm=new_charm)
 
-    with open("charm_internal_version", mode="r") as fd:
+    with open("charm_version", mode="r") as fd:
         revision = fd.read().strip()
 
     # Wait for statuses to settle down

tox.ini

@@ -31,22 +31,23 @@ allowlist_externals =
     mv
     rm
 commands_pre =
-    python -c 'import pathlib; import shutil; import subprocess; git_hash=subprocess.run(["git", "describe", "--always", "--dirty"], capture_output=True, check=True, encoding="utf-8").stdout; file = pathlib.Path("charm_internal_version"); file.write_text(f"{git_hash}")'
+    python -c 'import pathlib; import shutil; import subprocess; git_hash=subprocess.run(["git", "describe", "--always", "--dirty"], capture_output=True, check=True, encoding="utf-8").stdout; file = pathlib.Path("charm_version"); shutil.copy(file, pathlib.Path("charm_version.backup")); version = file.read_text().strip(); file.write_text(f"{version}+{git_hash}")'
+    poetry self add poetry-plugin-export
     poetry export --only main,charm-libs --output requirements.txt
 commands =
     build-production: charmcraft pack {posargs}
     build-dev: charmcraftcache pack {posargs}
 commands_post =
     mv requirements.txt requirements-last-build.txt
-    rm charm_internal_version
+    rm charm_version
 
 
 [testenv:format]
 description = Apply coding style standards to code
 commands_pre =
     poetry install --only format
 commands =
-    poetry lock --no-update
+    poetry lock 
     poetry run isort {[vars]all_path}
     poetry run black {[vars]all_path}
 
@@ -73,22 +74,18 @@ allowlist_externals =
     {[testenv]allowlist_externals}
     rm 
 commands_pre =
-    python -c 'import pathlib; import shutil; import subprocess; git_hash=subprocess.run(["git", "describe", "--always", "--dirty"], capture_output=True, check=True, encoding="utf-8").stdout; file = pathlib.Path("charm_internal_version"); file.write_text(f"{git_hash}")'
+    python -c 'import pathlib; import shutil; import subprocess; git_hash=subprocess.run(["git", "describe", "--always", "--dirty"], capture_output=True, check=True, encoding="utf-8").stdout; file = pathlib.Path("charm_version"); shutil.copy(file, pathlib.Path("charm_version.backup")); version = file.read_text().strip(); file.write_text(f"{version}+{git_hash}")'
     poetry install --only main,charm-libs,unit
 commands =
     poetry run coverage run --source={[vars]src_path},{[vars]lib_path} \
     -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit
     poetry run coverage report
     poetry run coverage xml
 commands_post =
-    rm charm_internal_version
+    rm charm_version
 
 [testenv:integration]
 description = Run integration tests
-set_env =
-    {[testenv]set_env}
-    # Workaround for https://github.com/python-poetry/poetry/issues/6958
-    POETRY_INSTALLER_PARALLEL = false
 pass_env =
     CI
     GITHUB_OUTPUT