add airflow deployment from github actions (#973)

incubateur-ademe · Oct 23, 2024 · d90d030 · d90d030
1 parent 25a7f73
commit d90d030
Show file tree

Hide file tree

Showing 12 changed files with 288 additions and 103 deletions.
diff --git a/.github/workflows/airflow_cd_preprod.yml b/.github/workflows/airflow_cd_preprod.yml
@@ -0,0 +1,43 @@
+name: "[Airflow] [🟠 Preprod] Déploiement continu"
+
+on:
+  workflow_dispatch:
+  push:
+    paths: ["dags*", "airflow*"]
+    branches:
+      - main
+
+env:
+  CLEVER_TOKEN: ${{ secrets.CLEVER_TOKEN }}
+  CLEVER_SECRET: ${{ secrets.CLEVER_SECRET }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  run_airflow_tests:
+    uses: ./.github/workflows/airflow_tests.yml
+
+  deploy_dags:
+    needs: [run_airflow_tests]
+    uses: ./.github/workflows/airflow_deploy_dags.yml
+    with:
+      S3_BUCKET_DESTINATION: s3://preprod-data-dags/
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AIRFLOW_PREPROD_S3_ACCESS_KEY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AIRFLOW_PREPROD_S3_SECRET_KEY }}
+
+  deploy_airflow:
+    needs: [deploy_dags]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: 47ng/[email protected]
+        with:
+          appID: ${{ secrets.AIRFLOW_WEBSERVER_PREPROD_APP_ID }}
+      - uses: 47ng/[email protected]
+        with:
+          appID: ${{ secrets.AIRFLOW_SCHEDULER_PREPROD_APP_ID }}
diff --git a/.github/workflows/airflow_cd_prod.yml b/.github/workflows/airflow_cd_prod.yml
@@ -0,0 +1,43 @@
+name: "[Airflow] [🔴 Prod] Déploiement continu"
+
+on:
+  workflow_dispatch:
+  push:
+    paths: ["dags*", "airflow*"]
+    tags:
+      - v*
+
+env:
+  CLEVER_TOKEN: ${{ secrets.CLEVER_TOKEN }}
+  CLEVER_SECRET: ${{ secrets.CLEVER_SECRET }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  run_airflow_tests:
+    uses: ./.github/workflows/airflow_tests.yml
+
+  deploy_dags:
+    needs: [run_airflow_tests]
+    uses: ./.github/workflows/airflow_deploy_dags.yml
+    with:
+      S3_BUCKET_DESTINATION: s3://prod-dags/
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AIRFLOW_PROD_S3_ACCESS_KEY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AIRFLOW_PROD_S3_SECRET_KEY }}
+
+  deploy_airflow:
+    needs: [run_airflow_tests]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: 47ng/[email protected]
+        with:
+          appID: ${{ secrets.AIRFLOW_WEBSERVER_PROD_APP_ID }}
+      - uses: 47ng/[email protected]
+        with:
+          appID: ${{ secrets.AIRFLOW_SCHEDULER_PROD_APP_ID }}
diff --git a/.github/workflows/airflow_deploy_dags.yml b/.github/workflows/airflow_deploy_dags.yml
@@ -0,0 +1,36 @@
+name: "_[Airflow] Déploiement des DAGs"
+
+on:
+  workflow_call:
+    inputs:
+      S3_BUCKET_DESTINATION:
+        required: true
+        type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  S3_HOST: https://cellar-c2.services.clever-cloud.com
+  FOLDER_SOURCE: dags
+
+jobs:
+  deploy_dags:
+    name: "Déploiement des DAGs dans Clever Cloud"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install AWS CLI dependencies
+        run: |
+          pip install awscli
+      - name: Push to S3 bucket production
+        run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --delete --quiet ${{ env.FOLDER_SOURCE }} ${{ inputs.S3_BUCKET_DESTINATION }}
diff --git a/.github/workflows/airflow_tests.yml b/.github/workflows/airflow_tests.yml
@@ -0,0 +1,112 @@
+name: "_[Airflow] Tests"
+
+on:
+  workflow_call:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  unit_tests:
+    name: "Tests unitaires des DAGs"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          cache: "pip"
+          python-version: "3.12.1"
+      - name: Install Dependencies
+        run: pip install --require-hashes --no-deps -r airflow-requirements.txt -r dev-requirements.txt
+      # TODO: voir pourquoi on ne peut pas lancer les tests ici
+      # 1. On avait d'abord une erreur pydantic
+      # J'ai donc instalé la dépendance
+      # ERROR: while parsing the following warning configuration:
+
+      #   ignore::pydantic.warnings.PydanticDeprecatedSince20
+
+      # This error occurred:
+
+      # Traceback (most recent call last):
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 1917, in parse_warning_filter
+      #     category: type[Warning] = _resolve_warning_category(category_)
+      #                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 1955, in _resolve_warning_category
+      #     m = __import__(module, None, None, [klass])
+      #         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      # ModuleNotFoundError: No module named 'pydantic'
+      #
+      #
+      #
+      # Puis j'ai eu de nouvelles erreurs car pytests lit le pyproject.toml
+      # Traceback (most recent call last):
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/bin/pytest", line 8, in <module>
+      #     sys.exit(console_main())
+      #              ^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 201, in console_main
+      #     code = main()
+      #            ^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 156, in main
+      #     config = _prepareconfig(args, plugins)
+      #              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 341, in _prepareconfig
+      #     config = pluginmanager.hook.pytest_cmdline_parse(
+      #              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_hooks.py", line 513, in __call__
+      #     return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
+      #            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_manager.py", line 120, in _hookexec
+      #     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
+      #            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 139, in _multicall
+      #     raise exception.with_traceback(exception.__traceback__)
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 122, in _multicall
+      #     teardown.throw(exception)  # type: ignore[union-attr]
+      #     ^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/helpconfig.py", line 105, in pytest_cmdline_parse
+      #     config = yield
+      #              ^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 103, in _multicall
+      #     res = hook_impl.function(*args)
+      #           ^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 1140, in pytest_cmdline_parse
+      #     self.parse(args)
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 1494, in parse
+      #     self._preparse(args, addopts=addopts)
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/config/__init__.py", line 1398, in _preparse
+      #     self.hook.pytest_load_initial_conftests(
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_hooks.py", line 513, in __call__
+      #     return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
+      #            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_manager.py", line 120, in _hookexec
+      #     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
+      #            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 139, in _multicall
+      #     raise exception.with_traceback(exception.__traceback__)
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 122, in _multicall
+      #     teardown.throw(exception)  # type: ignore[union-attr]
+      #     ^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/warnings.py", line 151, in pytest_load_initial_conftests
+      #     return (yield)
+      #             ^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 122, in _multicall
+      #     teardown.throw(exception)  # type: ignore[union-attr]
+      #     ^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/_pytest/capture.py", line 154, in pytest_load_initial_conftests
+      #     yield
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pluggy/_callers.py", line 103, in _multicall
+      #     res = hook_impl.function(*args)
+      #           ^^^^^^^^^^^^^^^^^^^^^^^^^
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pytest_django/plugin.py", line 356, in pytest_load_initial_conftests
+      #     with _handle_import_error(_django_project_scan_outcome):
+      #   File "/Users/fabienlefrapper/.local/share/mise/installs/python/3.12.5/lib/python3.12/contextlib.py", line 158, in __exit__
+      #     self.gen.throw(value)
+      #   File "/Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets/.venv/lib/python3.12/site-packages/pytest_django/plugin.py", line 184, in _handle_import_error
+      #     raise ImportError(msg) from None
+      # ImportError: No module named 'decouple'
+
+      # pytest-django found a Django project in /Users/fabienlefrapper/Developer/beta.gouv.fr/quefairedemesobjets (it contains manage.py) and added it to the Python path.
+      # If this is wrong, add "django_find_project = false" to pytest.ini and explicitly manage your Python path.
+      # - name: Run unit tests
+      #   run: pytest dags_unit_tests
diff --git a/.github/workflows/push_pr.yml → .github/workflows/ci.yml b/.github/workflows/push_pr.yml → .github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: Intégration continue
+name: "[LVAO] Intégration continue"
 
 on:
   pull_request:
@@ -10,4 +10,4 @@ defaults:
 
 jobs:
   run_test:
-    uses: ./.github/workflows/run_tests.yml
+    uses: ./.github/workflows/lvao_tests.yml
diff --git a/.github/workflows/lvao_cd_preprod.yml b/.github/workflows/lvao_cd_preprod.yml
@@ -0,0 +1,27 @@
+name: "[LVAO] [🟠 Preprod] Déploiement continu"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  run_test:
+    uses: ./.github/workflows/lvao_tests.yml
+
+  deploy:
+    name: Deploy on Scalingo
+    needs: [run_test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: kolok/deploy-to-scalingo@v1
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          known-host: ssh.osc-fr1.scalingo.com
+          app-name: quefairedemesobjets-preprod
diff --git a/.github/workflows/push_version_tags.yml → .github/workflows/lvao_cd_prod.yml b/.github/workflows/push_version_tags.yml → .github/workflows/lvao_cd_prod.yml
@@ -1,4 +1,4 @@
-name: "[Prod] Déploiement continu"
+name: "[LVAO] [🔴 Prod] Déploiement continu"
 
 on:
   push:
@@ -18,10 +18,10 @@ defaults:
 
 jobs:
   run_test:
-    uses: ./.github/workflows/run_tests.yml
+    uses: ./.github/workflows/lvao_tests.yml
 
   create-release:
-    name: "Create Release"
+    name: Create GitHub Release
     needs: [run_test]
     runs-on: "ubuntu-latest"
     steps:
@@ -31,7 +31,7 @@ jobs:
           prerelease: false
 
   deploy:
-    name: Deploy to Production
+    name: Deploy on Scalingo
     needs: [run_test]
     runs-on: ubuntu-latest
     steps:
@@ -41,16 +41,3 @@ jobs:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
           known-host: ssh.osc-fr1.scalingo.com
           app-name: quefairedemesobjets
-
-  deploy_dags:
-    name: Copy local folder to Production s3 bucket
-    needs: [run_test]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Install AWS CLI dependencies
-        run: |
-          pip install awscli
-      - name: Push to S3 bucket production
-        run: aws --endpoint-url ${{ env.S3_HOST }} s3 sync --delete --quiet ${{ env.FOLDER_SOURCE }} ${{ env.S3_BUCKET_DESTINATION }}
diff --git a/.github/workflows/run_tests.yml → .github/workflows/lvao_tests.yml b/.github/workflows/run_tests.yml → .github/workflows/lvao_tests.yml
@@ -1,3 +1,5 @@
+name: "_[LVAO] Tests"
+
 on:
   workflow_call:
 
@@ -6,14 +8,14 @@ defaults:
     shell: bash
 
 jobs:
-  python:
+  unit_tests:
     runs-on: ubuntu-latest
     services:
       postgres:
         image: postgis/postgis:15-3.3-alpine
         env:
           POSTGRES_USER: qfdmo
-          POSTGRES_PASSWORD: qfdmo
+          POSTGRES_PASSWORD: qfdmo # pragma: allowlist secret
           POSTGRES_DB: qfdmo
         options: >-
           --health-cmd pg_isready
@@ -45,10 +47,10 @@ jobs:
       - name: Check for missing migrations
         run: python manage.py makemigrations --check --no-input --settings=core.test_settings
         env:
-          SECRET_KEY: votre_blanquette_est_bonne
-          DATABASE_URL: "postgres://fakeusername:fakepassword@postgres:5432/database"
+          SECRET_KEY: votre_blanquette_est_bonne # pragma: allowlist secret
+          DATABASE_URL: "postgres://fakeusername:fakepassword@postgres:5432/database" # pragma: allowlist secret
 
-  node:
+  frontend_tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -72,7 +74,7 @@ jobs:
         image: postgis/postgis:15-3.3-alpine
         env:
           POSTGRES_USER: qfdmo
-          POSTGRES_PASSWORD: qfdmo
+          POSTGRES_PASSWORD: qfdmo # pragma: allowlist secret
           POSTGRES_DB: qfdmo
         options: >-
           --health-cmd pg_isready