From 3b9443d29d38b7963c01ebdfb85cca56c6e999c3 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Fri, 10 May 2024 15:29:35 +0200 Subject: [PATCH 1/2] [Fixes #240] Include the id of the new resource inside the executionrequest outputs --- importer/handlers/base.py | 34 ++++++++++++++++++++++++ importer/handlers/common/metadata.py | 12 +-------- importer/handlers/common/raster.py | 30 +-------------------- importer/handlers/common/tests_vector.py | 33 ++++++++++++++++++++++- importer/handlers/common/vector.py | 24 +---------------- runtest.sh | 2 +- 6 files changed, 70 insertions(+), 65 deletions(-) diff --git a/importer/handlers/base.py b/importer/handlers/base.py index 4b466030..ea02b786 100644 --- a/importer/handlers/base.py +++ b/importer/handlers/base.py @@ -5,6 +5,8 @@ from geonode.resource.enumerator import ExecutionRequestAction as exa from geonode.layers.models import Dataset from importer.utils import ImporterRequestAction as ira +from django_celery_results.models import TaskResult +from django.db.models import Q logger = logging.getLogger(__name__) @@ -117,6 +119,38 @@ def extract_params_from_data(_data): """ return [] + @staticmethod + def perform_last_step(execution_id): + """ + Override this method if there is some extra step to perform + before considering the execution as completed. + For example can be used to trigger an email-send to notify + that the execution is completed + """ + from importer.orchestrator import orchestrator + from importer.models import ResourceHandlerInfo + + # as last step, we delete the celery task to keep the number of rows under control + lower_exec_id = execution_id.replace("-", "_").lower() + TaskResult.objects.filter( + Q(task_args__icontains=lower_exec_id) + | Q(task_kwargs__icontains=lower_exec_id) + | Q(result__icontains=lower_exec_id) + | Q(task_args__icontains=execution_id) + | Q(task_kwargs__icontains=execution_id) + | Q(result__icontains=execution_id) + ).delete() + + _exec = orchestrator.get_execution_object(execution_id) + + resource_output_params = [ + {"detail_url": x.resource.detail_url, "id": x.resource.pk} + for x in ResourceHandlerInfo.objects.filter(execution_request=_exec) + ] + _exec.output_params.update({"resources": resource_output_params}) + _exec.save() + return _exec + def fixup_name(self, name): """ Emulate the LAUNDER option in ogr2ogr which will normalize the string. diff --git a/importer/handlers/common/metadata.py b/importer/handlers/common/metadata.py index 5b303577..14a80454 100644 --- a/importer/handlers/common/metadata.py +++ b/importer/handlers/common/metadata.py @@ -55,17 +55,7 @@ def extract_params_from_data(_data, action=None): @staticmethod def perform_last_step(execution_id): - _exec = orchestrator.get_execution_object(execution_id) - - _exec.output_params.update( - **{ - "detail_url": [ - x.resource.detail_url - for x in ResourceHandlerInfo.objects.filter(execution_request=_exec) - ] - } - ) - _exec.save() + BaseHandler.perform_last_step(execution_id=execution_id) def import_resource(self, files: dict, execution_id: str, **kwargs): _exec = orchestrator.get_execution_object(execution_id) diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index be8dcfb8..00870c57 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -9,7 +9,6 @@ from django.conf import settings from django.db.models import Q -from django_celery_results.models import TaskResult from geonode.base.models import ResourceBase from geonode.layers.models import Dataset from geonode.resource.enumerator import ExecutionRequestAction as exa @@ -186,34 +185,7 @@ def delete_resource(instance): @staticmethod def perform_last_step(execution_id): - """ - Override this method if there is some extra step to perform - before considering the execution as completed. - For example can be used to trigger an email-send to notify - that the execution is completed - """ - # as last step, we delete the celery task to keep the number of rows under control - lower_exec_id = execution_id.replace("-", "_").lower() - TaskResult.objects.filter( - Q(task_args__icontains=lower_exec_id) - | Q(task_kwargs__icontains=lower_exec_id) - | Q(result__icontains=lower_exec_id) - | Q(task_args__icontains=execution_id) - | Q(task_kwargs__icontains=execution_id) - | Q(result__icontains=execution_id) - ).delete() - - _exec = orchestrator.get_execution_object(execution_id) - - _exec.output_params.update( - **{ - "detail_url": [ - x.resource.detail_url - for x in ResourceHandlerInfo.objects.filter(execution_request=_exec) - ] - } - ) - _exec.save() + BaseHandler.perform_last_step(execution_id=execution_id) def extract_resource_to_publish( self, files, action, layer_name, alternate, **kwargs diff --git a/importer/handlers/common/tests_vector.py b/importer/handlers/common/tests_vector.py index a33c5b9b..2c306812 100644 --- a/importer/handlers/common/tests_vector.py +++ b/importer/handlers/common/tests_vector.py @@ -164,7 +164,9 @@ def test_import_resource_should_not_be_imported(self, celery_chord, ogr2ogr_driv files=self.valid_files, execution_id=str(exec_id) ) self.assertIn( - "No valid layers found", exception.exception.args[0], 'No valid layers found.' + "No valid layers found", + exception.exception.args[0], + "No valid layers found.", ) celery_chord.assert_not_called() @@ -319,3 +321,32 @@ def test_select_valid_layers(self): ) self.assertEqual(1, len(valid_layer)) self.assertEqual("mattia_test", valid_layer[0].GetName()) + + def test_perform_last_step(self): + """ + Output params in perform_last_step should return the detail_url and the ID + of the resource created + """ + # creating exec_id for the import + exec_id = orchestrator.create_execution_request( + user=get_user_model().objects.first(), + func_name="funct1", + step="step", + input_params={"files": self.valid_files, "store_spatial_file": True}, + ) + + # create_geonode_resource + resource = self.handler.create_geonode_resource( + "layer_name", + "layer_alternate", + str(exec_id), + ) + exec_obj = orchestrator.get_execution_object(str(exec_id)) + self.handler.create_resourcehandlerinfo(str(self.handler), resource, exec_obj) + # calling the last_step + self.handler.perform_last_step(str(exec_id)) + expected_output = { + "resources": [{"id": resource.pk, "detail_url": resource.detail_url}] + } + exec_obj.refresh_from_db() + self.assertDictEqual(expected_output, exec_obj.output_params) diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 96087b31..dda337fb 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -11,7 +11,6 @@ from celery import chord, group from django.conf import settings -from django_celery_results.models import TaskResult from dynamic_models.models import ModelSchema from dynamic_models.schema import ModelSchemaEditor from geonode.base.models import ResourceBase @@ -220,28 +219,7 @@ def perform_last_step(execution_id): For example can be used to trigger an email-send to notify that the execution is completed """ - # as last step, we delete the celery task to keep the number of rows under control - lower_exec_id = execution_id.replace("-", "_").lower() - TaskResult.objects.filter( - Q(task_args__icontains=lower_exec_id) - | Q(task_kwargs__icontains=lower_exec_id) - | Q(result__icontains=lower_exec_id) - | Q(task_args__icontains=execution_id) - | Q(task_kwargs__icontains=execution_id) - | Q(result__icontains=execution_id) - ).delete() - - _exec = orchestrator.get_execution_object(execution_id) - - _exec.output_params.update( - **{ - "detail_url": [ - x.resource.detail_url - for x in ResourceHandlerInfo.objects.filter(execution_request=_exec) - ] - } - ) - _exec.save() + _exec = BaseHandler.perform_last_step(execution_id=execution_id) if _exec and not _exec.input_params.get("store_spatial_file", False): resources = ResourceHandlerInfo.objects.filter(execution_request=_exec) # getting all files list diff --git a/runtest.sh b/runtest.sh index 0150efab..6280b2d8 100755 --- a/runtest.sh +++ b/runtest.sh @@ -2,4 +2,4 @@ set -a . ./.env_test set +a -coverage run --append --source='.' /usr/src/geonode/manage.py test importer -v2 --noinput +coverage run --append --source='.' /usr/src/geonode/manage.py test importer.handlers.common.tests_vector.TestBaseVectorFileHandler.test_perform_last_step -v2 --keepdb From 6514de8159da4596f71e5b0f0004646adbf20d7f Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Fri, 10 May 2024 15:30:26 +0200 Subject: [PATCH 2/2] [Fixes #240] Include the id of the new resource inside the executionrequest outputs --- runtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtest.sh b/runtest.sh index 6280b2d8..0150efab 100755 --- a/runtest.sh +++ b/runtest.sh @@ -2,4 +2,4 @@ set -a . ./.env_test set +a -coverage run --append --source='.' /usr/src/geonode/manage.py test importer.handlers.common.tests_vector.TestBaseVectorFileHandler.test_perform_last_step -v2 --keepdb +coverage run --append --source='.' /usr/src/geonode/manage.py test importer -v2 --noinput