Merge 2.7.3

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "backend/dependencies/UsersHub-authentification-module"]
+	path = backend/dependencies/UsersHub-authentification-module
+	url = https://github.com/PnX-SI/UsersHub-authentification-module
+[submodule "backend/dependencies/Nomenclature-api-module"]
+	path = backend/dependencies/Nomenclature-api-module
+	url = https://github.com/PnX-SI/Nomenclature-api-module
+[submodule "backend/dependencies/Habref-api-module"]
+	path = backend/dependencies/Habref-api-module
+	url = https://github.com/PnX-SI/Habref-api-module
+[submodule "backend/dependencies/Utils-Flask-SQLAlchemy"]
+	path = backend/dependencies/Utils-Flask-SQLAlchemy
+	url = https://github.com/PnX-SI/Utils-Flask-SQLAlchemy
+[submodule "backend/dependencies/TaxHub"]
+	path = backend/dependencies/TaxHub
+	url = https://github.com/PnX-SI/TaxHub
+[submodule "backend/dependencies/Utils-Flask-SQLAlchemy-Geo"]
+	path = backend/dependencies/Utils-Flask-SQLAlchemy-Geo
+	url = https://github.com/PnX-SI/Utils-Flask-SQLAlchemy-Geo

VERSION

@@ -1 +1 @@
-2.7.2
+2.7.3

backend/geonature/app.py

@@ -12,7 +12,7 @@
 from pkg_resources import iter_entry_points
 
 from geonature.utils.config import config
-from geonature.utils.env import MAIL, DB, MA, migrate
+from geonature.utils.env import MAIL, DB, MA, migrate, BACKEND_DIR
 from geonature.utils.logs import config_loggers
 from geonature.utils.module import import_backend_enabled_modules
 
@@ -50,7 +50,7 @@ def create_app(with_external_mods=True, with_flask_admin=True):
     # Bind app to DB
     DB.init_app(app)
 
-    migrate.init_app(app, DB)
+    migrate.init_app(app, DB, directory=BACKEND_DIR / 'geonature' / 'migrations')
 
     MAIL.init_app(app)
 
@@ -161,4 +161,4 @@ def on_before_models_committed(sender, changes):
                 app.config[module_config['MODULE_CODE']] = module_config
                 app.register_blueprint(module_blueprint, url_prefix=module_config['MODULE_URL'])
         _app = app
-    return app
+    return app

backend/geonature/core/command/create_gn_module.py

@@ -11,13 +11,17 @@
 import sys
 import logging
 import subprocess
-
+from pkg_resources import load_entry_point
 from pathlib import Path
 
 import click
+from click import ClickException
+from flask import current_app
+from flask_migrate import upgrade as db_upgrade
 from sqlalchemy.orm.exc import NoResultFound
 
-from geonature.utils.env import DB, DEFAULT_CONFIG_FILE
+from geonature.utils.env import DB, db, DEFAULT_CONFIG_FILE, GN_EXTERNAL_MODULE
+from geonature.utils.module import get_dist_from_code
 
 from geonature.utils.command import (
     build_geonature_front,
@@ -48,6 +52,69 @@
 
 log = logging.getLogger(__name__)
 
+@main.command()
+@click.argument("module_path")
+@click.argument("module_code")
+@click.option("--build", type=bool, required=False, default=True)
+def install_packaged_gn_module(module_path, module_code, build):
+    # install python package and dependencies
+    subprocess.run(f"pip install -e {module_path}", shell=True, check=True)
+
+    # load python package
+    module_dist = get_dist_from_code(module_code)
+    if not module_dist:
+        raise ClickException(f"Unable to load module with code {module_code}")
+
+    # add module to database
+    try:
+        module_picto = load_entry_point(module_dist, 'gn_module', 'picto')
+    except ImportError:
+        module_picto = "fa-puzzle-piece"
+    module_object = TModules.query.filter_by(module_code=module_code).one()
+    if not module_object:
+        module_object = TModules(
+                module_code=module_code,
+                module_label=module_code.lower(),
+                module_path=module_code.lower(),
+                module_target="_self",
+                module_picto=module_picto,
+                active_frontend=True,
+                active_backend=True,
+        )
+        db.session.add(module_object)
+    else:
+        module_object.module_picto=module_picto
+        db.session.merge(module_object)
+    db.session.commit()
+
+    db_upgrade(revision=module_code.lower())
+
+    # symlink module in exernal module directory
+    module_symlink = GN_EXTERNAL_MODULE / module_code.lower()
+    if os.path.exists(module_symlink):
+        target = os.readlink(module_symlink)
+        if os.path.abspath(module_path) != target:
+            raise ClickException(f"Module symlink has wrong target '{target}'")
+    else:
+        os.symlink(os.path.abspath(module_path), module_symlink)
+
+    ### Frontend
+    # creation du lien symbolique des assets externes
+    enable_frontend = create_external_assets_symlink(
+        module_path, module_code.lower()
+    )
+
+    install_frontend_dependencies(module_path)
+    # generation du fichier tsconfig.app.json
+    tsconfig_app_templating(app=current_app)
+    # generation du routing du frontend
+    frontend_routes_templating(app=current_app)
+    # generation du fichier de configuration du frontend
+    create_module_config(current_app, module_code, build=False)
+    if build:
+        # Rebuild the frontend
+        build_geonature_front(rebuild_sass=True)
+
 
 @main.command()
 @click.argument("module_path")

backend/geonature/core/command/main.py

@@ -6,6 +6,8 @@
 from os import environ
 
 import click
+from flask import current_app
+from flask.cli import run_command
 
 from geonature.utils.env import (
     DEFAULT_CONFIG_FILE,
@@ -23,16 +25,18 @@
     update_app_configuration,
 )
 from geonature import create_app
+from geonature.core.gn_meta.mtd.mtd_utils import import_all_dataset_af_and_actors
 
 # from rq import Queue, Connection, Worker
 # import redis
 from flask import Flask
+from flask.cli import FlaskGroup
 
 
 log = logging.getLogger()
 
 
-@click.group()
+@click.group(cls=FlaskGroup, create_app=create_app)
 @click.version_option(version=GEONATURE_VERSION)
 @click.pass_context
 def main(ctx):
@@ -78,7 +82,8 @@ def start_gunicorn(uri, worker):
 @main.command()
 @click.option("--host", default="0.0.0.0")
 @click.option("--port", default=8000)
-def dev_back(host, port):
+@click.pass_context
+def dev_back(ctx, host, port):
     """
         Lance l'api du backend avec flask
 
@@ -90,8 +95,7 @@ def dev_back(host, port):
     """
     if not environ.get('FLASK_ENV'):
         environ['FLASK_ENV'] = 'development'
-    app = create_app()
-    app.run(host=host, port=int(port))
+    ctx.invoke(run_command, host=host, port=port)
 
 
 @main.command()
@@ -171,7 +175,4 @@ def import_jdd_from_mtd(table_name):
     """
     Import les JDD et CA (et acters associé) à partir d'une table (ou vue) listant les UUID des JDD dans MTD
     """
-    app = create_app()
-    with app.app_context():
-        from geonature.core.gn_meta.mtd.mtd_utils import import_all_dataset_af_and_actors
-        import_all_dataset_af_and_actors(table_name)
+    import_all_dataset_af_and_actors(table_name)

backend/geonature/core/gn_meta/mtd/__init__.py

@@ -0,0 +1,162 @@
+from urllib.parse import urljoin
+
+import requests
+from lxml import etree
+
+from flask import current_app
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.sql import func
+
+from geonature.utils.config import config
+from geonature.utils.env import db
+from geonature.core.gn_meta.models import TAcquisitionFramework, TDatasets, CorAcquisitionFrameworkActor, CorDatasetActor
+from geonature.core.auth.routes import insert_user_and_org
+
+from pypnusershub.db.models import User, Organisme
+
+from .xml_parser import parse_acquisition_framework, parse_jdd_xml
+from .mtd_utils import create_cor_object_actors, NOMENCLATURE_MAPPING
+
+
+class MTDInstanceApi:
+    af_path = '/mtd/cadre/export/xml/GetRecordsByInstanceId?id={ID_INSTANCE}'
+    ds_path = '/mtd/cadre/jdd/export/xml/GetRecordsByInstanceId?id={ID_INSTANCE}'
+
+    def __init__(self, api_endpoint, instance_id):
+        self.api_endpoint = api_endpoint
+        self.instance_id = instance_id
+
+    def _get_xml(self, path):
+        url = urljoin(self.api_endpoint, path)
+        url = url.format(ID_INSTANCE=self.instance_id)
+        response = requests.get(url)
+        assert(response.status_code == 200)
+        return response.content
+
+    def _get_af_xml(self):
+        return self._get_xml(self.af_path)
+
+    def get_af_list(self):
+        xml = self._get_af_xml()
+        root = etree.fromstring(xml)
+        af_iter = root.iterfind(".//{http://inpn.mnhn.fr/mtd}CadreAcquisition")
+        af_list = []
+        for af in af_iter:
+            af_list.append(parse_acquisition_framework(af))
+        return af_list
+
+    def _get_ds_xml(self):
+        return self._get_xml(self.ds_path)
+
+    def get_ds_list(self):
+        xml = self._get_ds_xml()
+        return parse_jdd_xml(xml)
+
+
+class INPNCAS:
+    base_url = config['CAS']['CAS_USER_WS']['BASE_URL']
+    user = config['CAS']['CAS_USER_WS']['ID']
+    password = config['CAS']['CAS_USER_WS']['PASSWORD']
+    id_search_path = 'rechercheParId/{user_id}'
+
+    @classmethod
+    def _get_user_json(cls, user_id):
+        url = urljoin(cls.base_url, cls.id_search_path)
+        url = url.format(user_id=user_id)
+        response = requests.get(url, auth=(cls.user, cls.password))
+        return response.json()
+
+    @classmethod
+    def get_user(cls, user_id):
+        return cls._get_user_json(user_id)
+
+
+def add_unexisting_digitizer(id_digitizer):
+    if not db.session.query(User.query.filter_by(id_role=id_digitizer).exists()).scalar():
+        user = INPNCAS.get_user(id_digitizer)
+        insert_user_and_org(user)
+
+
+def associate_actors(actors, CorActor, pk_name, pk_value):
+    for actor in actors:
+        if not actor['uuid_organism']:
+            continue
+        statement = pg_insert(Organisme) \
+            .values(
+                uuid_organisme=actor['uuid_organism'],
+                nom_organisme=actor['organism'],
+                email_organisme=actor['email'],
+            ).on_conflict_do_update(
+                index_elements=['uuid_organisme'],
+                set_=dict(
+                    nom_organisme=actor['organism'],
+                    email_organisme=actor['email'],
+                ),
+            )
+        db.session.execute(statement)
+        # retrieve organism id
+        org = Organisme.query \
+                .filter_by(uuid_organisme=actor['uuid_organism']) \
+                .first()
+        statement = pg_insert(CorActor) \
+            .values(
+                #id_acquisition_framework=af.id_acquisition_framework,
+                id_organism=org.id_organisme,
+                id_nomenclature_actor_role=func.ref_nomenclatures.get_id_nomenclature(
+                    "ROLE_ACTEUR", actor["actor_role"]
+                ),
+                **{pk_name: pk_value},
+            ).on_conflict_do_nothing(
+                index_elements=[pk_name, 'id_organism', 'id_nomenclature_actor_role'],
+            )
+        db.session.execute(statement)
+
+
+def sync_af_and_ds():
+    cas_api = INPNCAS()
+    mtd_api = MTDInstanceApi(
+                config["MTD_API_ENDPOINT"],
+                config['MTD']['ID_INSTANCE_FILTER'])
+
+    af_list = mtd_api.get_af_list()
+    for af in af_list:
+        add_unexisting_digitizer(af['id_digitizer'])
+        actors = af.pop('actors')
+        statement = pg_insert(TAcquisitionFramework) \
+            .values(**af) \
+            .on_conflict_do_update(
+                index_elements=['unique_acquisition_framework_id'],
+                set_=af)
+        db.session.execute(statement)
+        af = TAcquisitionFramework.query \
+                .filter_by(unique_acquisition_framework_id=af['unique_acquisition_framework_id']) \
+                .first()
+        associate_actors(actors, CorAcquisitionFrameworkActor,
+                         'id_acquisition_framework', af.id_acquisition_framework)
+    db.session.commit()
+
+    ds_list = mtd_api.get_ds_list()
+    for ds in ds_list:
+        add_unexisting_digitizer(ds['id_digitizer'])
+        actors = ds.pop('actors')
+        af_uuid = ds.pop('uuid_acquisition_framework')
+        af = TAcquisitionFramework.query.filter_by(unique_acquisition_framework_id=af_uuid).first()
+        if af is None:
+            continue
+        ds['id_acquisition_framework'] = af.id_acquisition_framework
+        ds = { k: func.ref_nomenclatures.get_id_nomenclature(NOMENCLATURE_MAPPING[k], v)
+                  if k.startswith('id_nomenclature') else v
+               for k, v in ds.items()
+               if v is not None }
+        statement = pg_insert(TDatasets) \
+            .values(**ds) \
+            .on_conflict_do_update(
+                index_elements=['unique_dataset_id'],
+                set_=ds)
+        db.session.execute(statement)
+        ds = TDatasets.query \
+                .filter_by(unique_dataset_id=ds['unique_dataset_id']) \
+                .first()
+        associate_actors(actors, CorDatasetActor,
+                         'id_dataset', ds.id_dataset)
+    db.session.commit()

backend/geonature/core/gn_meta/mtd/mtd_webservice.py

@@ -1,9 +1,8 @@
 from geonature.utils import utilsrequests
 from geonature.utils.errors import GeonatureApiError
+from geonature.utils.config import config
 
-from flask import current_app
-
-api_endpoint = current_app.config["MTD_API_ENDPOINT"]
+api_endpoint = config["MTD_API_ENDPOINT"]
 
 
 def get_acquisition_framework(uuid_af):
@@ -49,4 +48,4 @@ def get_jdd_by_uuid(uuid):
         assert r.status_code == 200
     except AssertionError:
         print(f'NO JDD FOUND FOR UUID {uuid}')
-    return r.content
+    return r.content