Merge pull request #2673 from PnX-SI/develop

Develop > Master / Clean git history 2.13

.github/workflows/cypress.yml

@@ -5,6 +5,7 @@ on:
       - master
       - hotfixes
       - develop
+      - fixtestfront
   pull_request:
     branches:
       - master
@@ -37,17 +38,14 @@ jobs:
           psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "postgis_raster";'
         env:
           PGPASSWORD: geonatpasswd
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Add database extensions
         run: |
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "hstore";'
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "uuid-ossp";'
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "pg_trgm";'
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "unaccent";'
+          psql -h localhost -U geonatadmin -d geonature2db -f install/assets/db/add_pg_extensions.sql
         env:
           PGPASSWORD: geonatpasswd
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
@@ -75,14 +73,7 @@ jobs:
         working-directory: ./backend
       - name: Install database
         run: |
-          geonature db upgrade geonature@head -x local-srid=2154
-          geonature db autoupgrade -x local-srid=2154
-          geonature db upgrade geonature-samples@head
-          geonature db upgrade taxhub@head
-          geonature taxref import-v15 --skip-bdc-statuts
-          geonature db upgrade nomenclatures_taxonomie_data@head
-          geonature db upgrade ref_geo_fr_departments@head
-
+          install/03b_populate_db_for_test.sh
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
         # FRONTEND
@@ -114,6 +105,7 @@ jobs:
           geonature install-gn-module contrib/gn_module_occhab OCCHAB --build=false
           geonature db upgrade occhab-samples@head
           geonature install-gn-module contrib/gn_module_validation VALIDATION --build=false
+          geonature permissions supergrant --group --nom "Grp_admin" --yes
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Run GeoNature backend

.github/workflows/lint.yml

@@ -18,8 +18,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - uses: actions/setup-node@v3
       - name: Frontend code formatting check (Prettier)
-        uses: creyD/[email protected]
-        with:
-          dry: True
-          prettier_options: --config frontend/.prettierrc --ignore-path frontend/.prettierignore --check frontend/**/*.ts contrib/*/frontend/**/*.ts
+        run: npm install prettier && npm run format:check
+        working-directory: ./frontend

.github/workflows/pytest.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        debian-version: [ '10', '11' ]
+        debian-version: [ '10', '11', '12' ]
         include:
           - debian-version: '10'
             python-version: '3.7'
@@ -29,6 +29,10 @@ jobs:
             python-version: '3.9'
             postgres-version: '13'
             postgis-version: '3.2'
+          - debian-version: '12'
+            python-version: '3.11'
+            postgres-version: '15'
+            postgis-version: '3.3'
 
     name: Debian ${{ matrix.debian-version }}
 
@@ -54,17 +58,14 @@ jobs:
           psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "postgis_raster";'
         env:
           PGPASSWORD: geonatpasswd
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Add database extensions
         run: |
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "hstore";'
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "uuid-ossp";'
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "pg_trgm";'
-          psql -h localhost -U geonatadmin -d geonature2db -tc 'CREATE EXTENSION "unaccent";'
+          psql -h localhost -U geonatadmin -d geonature2db -f install/assets/db/add_pg_extensions.sql
         env:
           PGPASSWORD: geonatpasswd
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -88,20 +89,7 @@ jobs:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Install database
         run: |
-          geonature db upgrade geonature@head -x local-srid=2154
-          geonature db autoupgrade -x local-srid=2154
-          geonature db upgrade ref_geo_fr_departments@head
-          geonature taxref import-v16
-          geonature db upgrade geonature-samples@head
-          geonature db upgrade nomenclatures_taxonomie_data@head
-          geonature db upgrade ref_geo_fr_municipalities@head
-          geonature db upgrade ref_geo_inpn_grids_5@head
-          geonature sensitivity add-referential \
-            --source-name "Référentiel sensibilité TAXREF v16 20230203" \
-            --url https://geonature.fr/data/inpn/sensitivity/RefSensibiliteV16_20230203.zip \
-            --zipfile RefSensibiliteV16_20230203.zip \
-            --csvfile RefSensibiliteV16_20230203/RefSensibilite_16.csv \
-            --encoding=iso-8859-15
+          install/03b_populate_db_for_test.sh
         env:
           GEONATURE_CONFIG_FILE: config/test_config.toml
       - name: Show database status

.gitignore

@@ -19,6 +19,7 @@ backend/static/node_modules
 backend/media
 
 backend/run.py
+cache/*
 
 .vscode
 

VERSION

@@ -1 +1 @@
-2.12.3
+2.13.0

backend/Dockerfile

@@ -139,19 +139,14 @@ COPY --from=build-utils /build/dist/*.whl .
 COPY --from=build-utils-geo /build/dist/*.whl .
 COPY --from=build-geonature /build/dist/*.whl .
 
+COPY --chmod=755 /install/03b_populate_db.sh /populate_db.sh
+COPY --chmod=755 /install/assets/docker_startup.sh /startup.sh
+
 ENV GEONATURE_CONFIG_FILE ""
 
 EXPOSE 8000
 
-CMD [ "gunicorn", "geonature:create_app()", \
-    "--worker-tmp-dir=/dev/shm", \
-    "--name=geonature", \
-    "--workers=2", \
-    "--threads=2", \
-    "--access-logfile=-", \
-    "--error-logfile=-", \
-    "--bind=0.0.0.0:8000" \
-]
+CMD ["/startup.sh"]
 
 
 FROM wheels-light AS wheels

backend/geonature/app.py

@@ -19,6 +19,8 @@
 from flask_sqlalchemy import before_models_committed
 from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.middleware.shared_data import SharedDataMiddleware
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from werkzeug.wrappers import Response
 from psycopg2.errors import UndefinedTable
 import sqlalchemy as sa
 from sqlalchemy.exc import OperationalError, ProgrammingError
@@ -51,14 +53,16 @@ def configure_alembic(alembic_config):
     'migrations' entry point value of the 'gn_module' group for all modules having such entry point.
     Thus, alembic will find migrations of all installed geonature modules.
     """
-    version_locations = alembic_config.get_main_option("version_locations", default="").split()
+    version_locations = set(
+        alembic_config.get_main_option("version_locations", default="").split()
+    )
     if "VERSION_LOCATIONS" in config["ALEMBIC"]:
-        version_locations.extend(config["ALEMBIC"]["VERSION_LOCATIONS"].split())
+        version_locations |= set(config["ALEMBIC"]["VERSION_LOCATIONS"].split())
     for entry_point in chain(
         entry_points(group="alembic", name="migrations"),
         entry_points(group="gn_module", name="migrations"),
     ):
-        version_locations += [entry_point.value]
+        version_locations.add(entry_point.value)
     alembic_config.set_main_option("version_locations", " ".join(version_locations))
     return alembic_config
 
@@ -95,9 +99,6 @@ def create_app(with_external_mods=True):
 
     app.config.update(config)
 
-    if "SCRIPT_NAME" not in os.environ:
-        os.environ["SCRIPT_NAME"] = app.config["APPLICATION_ROOT"].rstrip("/")
-
     # Enable deprecation warnings in debug mode
     if app.debug and not sys.warnoptions:
         warnings.filterwarnings(action="default", category=DeprecationWarning)
@@ -106,6 +107,11 @@ def create_app(with_external_mods=True):
     app.wsgi_app = SchemeFix(app.wsgi_app, scheme=config.get("PREFERRED_URL_SCHEME"))
     app.wsgi_app = ProxyFix(app.wsgi_app, x_host=1)
     app.wsgi_app = RequestID(app.wsgi_app)
+    if app.config["APPLICATION_ROOT"] != "/":
+        app.wsgi_app = DispatcherMiddleware(
+            Response("Not Found", status=404),
+            {app.config["APPLICATION_ROOT"].rstrip("/"): app.wsgi_app},
+        )
 
     if config.get("CUSTOM_STATIC_FOLDER"):
         app.wsgi_app = SharedDataMiddleware(
@@ -156,6 +162,8 @@ def load_current_user():
             g.current_user = user_from_token(request.cookies["token"]).role
         except (KeyError, UnreadableAccessRightsError, AccessRightsExpiredError):
             g.current_user = None
+        g._permissions_by_user = {}
+        g._permissions = {}
 
     if config.get("SENTRY_DSN"):
         from sentry_sdk import set_tag, set_user
@@ -190,7 +198,6 @@ def set_sentry_context():
         ("ref_geo.routes:routes", "/geo"),
         ("geonature.core.gn_commons.routes:routes", "/gn_commons"),
         ("geonature.core.gn_permissions.routes:routes", "/permissions"),
-        ("geonature.core.gn_permissions.backoffice.views:routes", "/permissions_backoffice"),
         ("geonature.core.users.routes:routes", "/users"),
         ("geonature.core.gn_synthese.routes:routes", "/synthese"),
         ("geonature.core.gn_meta.routes:routes", "/meta"),

backend/geonature/core/admin/__init__.py

@@ -0,0 +1 @@
+import geonature.core.gn_permissions.admin

backend/geonature/core/admin/utils.py

@@ -1,3 +1,5 @@
+from functools import partial
+
 from flask import g
 from werkzeug.exceptions import Unauthorized
 
@@ -33,3 +35,21 @@ def can_delete(self):
     @property
     def can_export(self):
         return self._can_action("E")
+
+
+# https://github.com/flask-admin/flask-admin/issues/1807
+# https://stackoverflow.com/questions/54638047/correct-way-to-register-flask-admin-views-with-application-factory
+class ReloadingIterator:
+    def __init__(self, iterator_factory):
+        self.iterator_factory = iterator_factory
+
+    def __iter__(self):
+        return self.iterator_factory()
+
+
+class DynamicOptionsMixin:
+    def get_dynamic_options(self, view):
+        raise NotImplementedError
+
+    def get_options(self, view):
+        return ReloadingIterator(partial(self.get_dynamic_options, view))

backend/geonature/core/gn_commons/admin.py

@@ -7,7 +7,7 @@
 
 from geonature.core.admin.utils import CruvedProtectedMixin
 from geonature.core.gn_commons.models import TModules
-from geonature.core.gn_permissions.models import TObjects
+from geonature.core.gn_permissions.models import PermObject
 from geonature.core.gn_commons.schemas import TAdditionalFieldsSchema
 from geonature.utils.env import DB
 
@@ -86,8 +86,8 @@ class BibFieldAdmin(CruvedProtectedMixin, ModelView):
             )
         },
         "objects": {
-            "query_factory": lambda: DB.session.query(TObjects).filter(
-                TObjects.code_object.in_(
+            "query_factory": lambda: DB.session.query(PermObject).filter(
+                PermObject.code_object.in_(
                     current_app.config["ADDITIONAL_FIELDS"]["IMPLEMENTED_OBJECTS"]
                 )
             )
@@ -97,9 +97,9 @@ class BibFieldAdmin(CruvedProtectedMixin, ModelView):
         "bib_nomenclature_type": "Si Type widget = Nomenclature",
         "field_label": "Label du champ en interface",
         "field_name": "Nom du champ en base de donnée",
-        "field_values": """Obligatoire si widget = select/multiselect/checkbox,radio (Format JSON : tableau de 'value/label'.Utilisez des doubles quotes pour les valeurs et les clés). 
+        "field_values": """Obligatoire si widget = select/multiselect/checkbox,radio (Format JSON : tableau de 'value/label'. Utilisez des doubles quotes pour les valeurs et les clés). 
             Exemple [{"label": "trois", "value": 3}, {"label": "quatre", "value": 4}]""",
-        "default_value": "La valeur par défaut doit être une des valeurs du champs 'Valeurs' ci dessus",
+        "default_value": "La valeur par défaut doit être une des valeurs du champs 'Valeurs' ci-dessus. Pour les valeurs textuelles, il n'est pas nécessaire de remettre la valeur entre guillement",
         "id_list": "Identifiant en BDD de la liste (pour Type widget = taxonomy/observers)",
         "field_order": "Numéro d'ordonnancement du champs (si plusieurs champs pour le même module/objet/JDD)",
         "modules": "Module(s) auquel le champs est rattaché. *Obligatoire",
@@ -110,7 +110,7 @@ class BibFieldAdmin(CruvedProtectedMixin, ModelView):
 
 class TMobileAppsAdmin(CruvedProtectedMixin, ModelView):
     module_code = "ADMIN"
-    object_code = "ALL"
+    object_code = "MOBILE_APPS"
 
     column_list = (
         "app_code",
@@ -132,7 +132,7 @@ class TMobileAppsAdmin(CruvedProtectedMixin, ModelView):
 
 class TModulesAdmin(CruvedProtectedMixin, ModelView):
     module_code = "ADMIN"
-    object_code = "ALL"
+    object_code = "MODULES"
 
     can_view_details = True
     action_disallowed_list = ["delete"]

backend/geonature/core/gn_commons/medias/routes.py

@@ -90,8 +90,6 @@ def insert_or_update_media(id_media=None):
 
     m = TMediaRepository(data=data, file=file, id_media=id_media).create_or_update_media()
 
-    TMediumRepository.sync_medias()
-
     return m.as_dict()
 
 
@@ -106,8 +104,6 @@ def delete_media(id_media):
 
     TMediaRepository(id_media=id_media).delete()
 
-    TMediumRepository.sync_medias()
-
     return {"resp": "media {} deleted".format(id_media)}
 
 

backend/geonature/core/gn_commons/models/additional_fields.py

@@ -10,7 +10,7 @@
 
 from .base import cor_field_module, cor_field_object, cor_field_dataset
 from geonature.core.gn_meta.models import TDatasets
-from geonature.core.gn_permissions.models import TObjects
+from geonature.core.gn_permissions.models import PermObject
 
 
 @serializable
@@ -49,8 +49,8 @@ class TAdditionalFields(DB.Model):
         "TModules",
         secondary=cor_field_module,
     )
-    objects = DB.relationship("TObjects", secondary=cor_field_object)
-    datasets = DB.relationship("TDatasets", secondary=cor_field_dataset)
+    objects = DB.relationship(PermObject, secondary=cor_field_object)
+    datasets = DB.relationship(TDatasets, secondary=cor_field_dataset)
 
     def __str__(self):
         return f"{self.field_label} ({self.description})"

backend/geonature/core/gn_commons/models/base.py

@@ -102,7 +102,7 @@ class TModules(DB.Model):
     meta_update_date = DB.Column(DB.DateTime)
 
     objects = DB.relationship(
-        "TObjects", secondary=lambda: _resolve_import_cor_object_module(), backref="modules"
+        "PermObject", secondary=lambda: _resolve_import_cor_object_module(), backref="modules"
     )
     # relationship datasets add via backref