From 6b5ca98acc37061806afb23d9e887ebb5d6a4907 Mon Sep 17 00:00:00 2001 From: Jayanth Kumar Date: Fri, 26 Jan 2024 19:00:42 +0530 Subject: [PATCH] Process deleting projects as background task 1. Fix Default Filtering in ProjectFilterSet: - Modified the `ProjectFilterSet` to include `is_marked_for_deletion` in the default filtering when `is_archived` is not provided. - This ensures that projects marked for deletion are excluded from default views. 2. Migration for `is_marked_for_deletion`: - Added a migration (`0052_project_is_marked_for_deletion.py`) to introduce the `is_marked_for_deletion` field in the `Project` model. 3. Updated `Project` Model: - Added `is_marked_for_deletion` field to the `Project` model with a default value of `False`. 4. Modified `ProjectListView` to Exclude Marked Projects: - Modified the `get_queryset` method in `ProjectListView` to only fetch projects with `is_marked_for_deletion=False`. 5. Introduced `mark_for_deletion` Method: - Added a `mark_for_deletion` method in the `Project` model to set the `is_marked_for_deletion` flag and save the model. 6. Changed `delete` Method in `Project` Model: - Renamed the `delete` method to `delete_action` in the `Project` model to avoid conflicts. - Introduced a new `delete` method that marks the project for deletion and enqueues a background deletion task. 7. Background Deletion Task: - Created a background deletion task using `django_rq` to handle project deletion asynchronously. - Checks if the project is still marked for deletion before proceeding. - If an error occurs during deletion, updates project status and logs an error. 8. Updated `ProjectActionView` Success Message: - Modified the success message in `ProjectActionView` for the "delete" action to provide a more informative message based on the count. Fixes: #1002 Signed-off-by: Jayanth Kumar --- scanpipe/filters.py | 4 ++-- .../0052_project_is_marked_for_deletion.py | 18 ++++++++++++++++++ scanpipe/models.py | 14 +++++++++++++- scanpipe/tasks.py | 16 ++++++++++++++++ scanpipe/views.py | 2 ++ 5 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 scanpipe/migrations/0052_project_is_marked_for_deletion.py diff --git a/scanpipe/filters.py b/scanpipe/filters.py index cdba64d8a..c7cf88122 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -349,9 +349,9 @@ def __init__(self, data=None, *args, **kwargs): # Default filtering by "Active" projects. if not data or data.get("is_archived", "") == "": - self.queryset = self.queryset.filter(is_archived=False) + self.queryset = self.queryset.filter(is_archived=False, is_marked_for_deletion=False) - active_count = Project.objects.filter(is_archived=False).count() + active_count = Project.objects.filter(is_archived=False, is_marked_for_deletion=False).count() archived_count = Project.objects.filter(is_archived=True).count() self.filters["is_archived"].extra["widget"] = BulmaLinkWidget( choices=[ diff --git a/scanpipe/migrations/0052_project_is_marked_for_deletion.py b/scanpipe/migrations/0052_project_is_marked_for_deletion.py new file mode 100644 index 000000000..a4e4f3b73 --- /dev/null +++ b/scanpipe/migrations/0052_project_is_marked_for_deletion.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.1 on 2024-01-26 12:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0051_rename_pipelines_data'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='is_marked_for_deletion', + field=models.BooleanField(default=False), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index e8461cdba..635275fbb 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -79,6 +79,7 @@ from rq.command import send_stop_job_command from rq.exceptions import NoSuchJobError from rq.job import Job +from rq import Queue from rq.job import JobStatus from taggit.managers import TaggableManager from taggit.models import GenericUUIDTaggedItemBase @@ -534,6 +535,7 @@ class Project(UUIDPKModel, ExtraDataFieldMixin, UpdateMixin, models.Model): labels = TaggableManager(through=UUIDTaggedItem) objects = ProjectQuerySet.as_manager() + is_marked_for_deletion = models.BooleanField(default=False) class Meta: ordering = ["-created_date"] @@ -621,7 +623,7 @@ def delete_related_objects(self): return deleted_counter - def delete(self, *args, **kwargs): + def delete_action(self, *args, **kwargs): """Delete the `work_directory` along project-related data in the database.""" self._raise_if_run_in_progress() @@ -633,6 +635,16 @@ def delete(self, *args, **kwargs): return super().delete(*args, **kwargs) + def mark_for_deletion(self): + self.is_marked_for_deletion = True + self.save() + + def delete(self): + # Mark the project for deletion and enqueue background deletion task + self.mark_for_deletion() + q = Queue("default", connection=redis.Redis()) + job = q.enqueue(tasks.background_delete_task, self) + def reset(self, keep_input=True): """ Reset the project by deleting all related database objects and all work diff --git a/scanpipe/tasks.py b/scanpipe/tasks.py index d6379c614..5da16b447 100644 --- a/scanpipe/tasks.py +++ b/scanpipe/tasks.py @@ -23,6 +23,7 @@ import logging from django.apps import apps +from django_rq import job logger = logging.getLogger(__name__) @@ -76,3 +77,18 @@ def execute_pipeline_task(run_pk): project.clear_tmp_directory() if next_run := project.get_next_run(): next_run.start() + +@job +def background_delete_task(project): + # Check if the project is still marked for deletion + if not project.is_marked_for_deletion: + return + + # Perform the deletion process + try: + project.delete_action() + except Exception as e: + # Handle errors and update project errors or display a banner + project.is_marked_for_deletion = False + project.save() + project.add_error(description=f"Deletion failed: {str(e)}") diff --git a/scanpipe/views.py b/scanpipe/views.py index e299052fb..e45bceb7e 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1086,6 +1086,8 @@ def perform_action(self, action, project_uuid, action_kwargs=None): raise Http404 def get_success_message(self, action, count): + if action == "delete": + return f"{count} project{'s' if count != 1 else ''} {'is' if count == 1 else 'are'} being deleted in the background." return f"{count} projects have been {action}."