Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TP2000-1387 Async Bulk Edit - Python 3.12 #1286

Merged
merged 39 commits into from
Sep 26, 2024
Merged
Show file tree
Hide file tree
Changes from 37 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
73232b9
Make MeasuresBulkEditor model
LaurenMullally Sep 2, 2024
4fe9c65
Export the MeasuresBulkEditor model
LaurenMullally Sep 2, 2024
4863177
Add flag for Measures async edit
LaurenMullally Sep 2, 2024
8d403bc
Split done function into sync and async done
LaurenMullally Sep 2, 2024
713d758
Add Mixin and kwarg functions to Start date form
LaurenMullally Sep 2, 2024
e77b66e
Add Mixin and kwarg functions to End date form
LaurenMullally Sep 2, 2024
dbab9ff
Add mixin and kwarg functions to Regulations form
LaurenMullally Sep 2, 2024
704fc63
Add Mixin and kwarg functions to Duties form
LaurenMullally Sep 2, 2024
21b8354
Add Mixin to Geographical Areas formset
LaurenMullally Sep 2, 2024
6472398
WIP - Move serializable functions into a wizard mixin
LaurenMullally Sep 2, 2024
4e9659a
Create SerializableWizardMixin for wizards that use async bulk functi…
LaurenMullally Sep 3, 2024
bcfb454
Add MeasureSerializableWizardMixin to Create Measures
LaurenMullally Sep 3, 2024
8db7aef
Add serializable wizard mixin and split the form lists
LaurenMullally Sep 3, 2024
ceadfee
Set up Celery route for Async bulk edit
LaurenMullally Sep 3, 2024
b8cd69f
WIP - Flesh out editing functionality - need to consider update funct…
LaurenMullally Sep 3, 2024
a3df250
WIP - Move Update functions into measures/util - will probs give circ…
LaurenMullally Sep 3, 2024
e687248
Move update functions into measures/util file
LaurenMullally Sep 4, 2024
bb3767a
WIP - Update functions in util
LaurenMullally Sep 5, 2024
7d37617
Fix circular import errors
LaurenMullally Sep 6, 2024
6b89412
Delete unnecessary if statement and comment
LaurenMullally Sep 6, 2024
1fca1c7
Clean up ready for testing
LaurenMullally Sep 6, 2024
d095671
Change back order of kwargs to fix test
LaurenMullally Sep 11, 2024
658484a
WIP - Mock won't work
LaurenMullally Sep 16, 2024
38404d6
Fix mock file path
LaurenMullally Sep 16, 2024
bc939fd
Fix measure/test_views.py tests
LaurenMullally Sep 16, 2024
4fb2f0b
Modify edit tests to be sync tests
LaurenMullally Sep 17, 2024
575f606
Refactor editing measures into a editor class so that both done route…
LaurenMullally Sep 17, 2024
92ca80f
Hook the async route up to the new editor class
LaurenMullally Sep 17, 2024
a316d9f
Change name of mock and fix circular imports
LaurenMullally Sep 17, 2024
dba7e6c
Test bulk processing
LaurenMullally Sep 18, 2024
12c1846
Take out log
LaurenMullally Sep 18, 2024
1054514
Test for Paul to look at
LaurenMullally Sep 19, 2024
46e3a1e
Add test to check form serialize and deserialize functionality
LaurenMullally Sep 20, 2024
54be1ce
Add Geo Areas serialize deserialise test
LaurenMullally Sep 20, 2024
3a05f91
Merge branch 'master' into TP2000-1387-py312-async-bulk-edit
LaurenMullally Sep 20, 2024
c9755e5
Run linter
LaurenMullally Sep 23, 2024
5725a65
Run linter
LaurenMullally Sep 23, 2024
9d00d1e
Add test to check errors are logged in a detailed manor
LaurenMullally Sep 25, 2024
038a41c
linter
LaurenMullally Sep 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 118 additions & 0 deletions measures/editors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
from typing import Dict
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the class that now handles editing the measures

from typing import List
from typing import Type

from django.db.transaction import atomic

from workbaskets import models as workbasket_models
from measures import models as measure_models
from common.util import TaricDateRange
from common.validators import UpdateType
from common.models.utils import override_current_transaction
from measures.util import update_measure_components
from measures.util import update_measure_condition_components
from measures.util import update_measure_excluded_geographical_areas
from measures.util import update_measure_footnote_associations


class MeasuresEditor:
"""Utility class used to edit measures from measures wizard accumulated
data."""

workbasket: Type["workbasket_models.WorkBasket"]
"""The workbasket with which created measures will be associated."""

selected_measures: List
""" The measures in which the edits will apply to."""

data: Dict
"""Validated, cleaned and accumulated data created by the Form instances of
`MeasureEditWizard`."""

def __init__(
self,
workbasket: Type["workbasket_models.WorkBasket"],
selected_measures: List,
data: Dict,
):
self.workbasket = workbasket
self.selected_measures = selected_measures
self.data = data

@atomic
def edit_measures(self) -> List["measure_models.Measure"]:
"""
Returns a list of the edited measures.

`data` must be a dictionary
of the accumulated cleaned / validated data created from the
`MeasureEditWizard`.
"""

with override_current_transaction(
transaction=self.workbasket.current_transaction,
):
new_start_date = self.data.get("start_date", None)
new_end_date = self.data.get("end_date", False)
new_quota_order_number = self.data.get("order_number", None)
new_generating_regulation = self.data.get("generating_regulation", None)
new_duties = self.data.get("duties", None)
new_exclusions = [
e["excluded_area"]
for e in self.data.get("formset-geographical_area_exclusions", [])
]

edited_measures = []

if self.selected_measures:
for measure in self.selected_measures:
new_measure = measure.new_version(
workbasket=self.workbasket,
update_type=UpdateType.UPDATE,
valid_between=TaricDateRange(
lower=(
new_start_date
if new_start_date
else measure.valid_between.lower
),
upper=(
new_end_date
if new_end_date
else measure.valid_between.upper
),
),
order_number=(
new_quota_order_number
if new_quota_order_number
else measure.order_number
),
generating_regulation=(
new_generating_regulation
if new_generating_regulation
else measure.generating_regulation
),
)
update_measure_components(
measure=new_measure,
duties=new_duties,
workbasket=self.workbasket,
)
update_measure_condition_components(
measure=new_measure,
workbasket=self.workbasket,
)
update_measure_excluded_geographical_areas(
edited="geographical_area_exclusions"
in self.data.get("fields_to_edit", []),
measure=new_measure,
exclusions=new_exclusions,
workbasket=self.workbasket,
)
update_measure_footnote_associations(
measure=new_measure,
workbasket=self.workbasket,
)

edited_measures.append(new_measure.id)

return edited_measures
114 changes: 109 additions & 5 deletions measures/forms/wizard.py
Original file line number Diff line number Diff line change
Expand Up @@ -781,7 +781,7 @@ def __init__(self, *args, **kwargs):
)


class MeasureStartDateForm(forms.Form):
class MeasureStartDateForm(forms.Form, SerializableFormMixin):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Throughout this file, I add some functions to each form, as well as the mixin, all of which aid the serializing and deserializing required for passing the form data to the celery task

start_date = DateInputFieldFixed(
label="Start date",
help_text="For example, 27 3 2008",
Expand Down Expand Up @@ -819,8 +819,34 @@ def clean(self):

return cleaned_data

@classmethod
def serializable_init_kwargs(cls, kwargs: Dict) -> Dict:
selected_measures = kwargs.get("selected_measures")
selected_measures_pks = []
for measure in selected_measures:
selected_measures_pks.append(measure.id)

serializable_kwargs = {
"selected_measures": selected_measures_pks,
}

return serializable_kwargs

@classmethod
def deserialize_init_kwargs(cls, form_kwargs: Dict) -> Dict:
serialized_selected_measures_pks = form_kwargs.get("selected_measures")
deserialized_selected_measures = models.Measure.objects.filter(
pk__in=serialized_selected_measures_pks
)

kwargs = {
"selected_measures": deserialized_selected_measures,
}

return kwargs


class MeasureEndDateForm(forms.Form):
class MeasureEndDateForm(forms.Form, SerializableFormMixin):
end_date = DateInputFieldFixed(
label="End date",
help_text="For example, 27 3 2008",
Expand Down Expand Up @@ -861,8 +887,34 @@ def clean(self):

return cleaned_data

@classmethod
def serializable_init_kwargs(cls, kwargs: Dict) -> Dict:
selected_measures = kwargs.get("selected_measures")
selected_measures_pks = []
for measure in selected_measures:
selected_measures_pks.append(measure.id)

serializable_kwargs = {
"selected_measures": selected_measures_pks,
}

return serializable_kwargs

@classmethod
def deserialize_init_kwargs(cls, form_kwargs: Dict) -> Dict:
serialized_selected_measures_pks = form_kwargs.get("selected_measures")
deserialized_selected_measures = models.Measure.objects.filter(
pk__in=serialized_selected_measures_pks
)

kwargs = {
"selected_measures": deserialized_selected_measures,
}

return kwargs


class MeasureRegulationForm(forms.Form):
class MeasureRegulationForm(forms.Form, SerializableFormMixin):
generating_regulation = AutoCompleteField(
label="Regulation ID",
help_text="Select the regulation which provides the legal basis for the measures.",
Expand All @@ -888,8 +940,34 @@ def __init__(self, *args, **kwargs):
),
)

@classmethod
def serializable_init_kwargs(cls, kwargs: Dict) -> Dict:
selected_measures = kwargs.get("selected_measures")
selected_measures_pks = []
for measure in selected_measures:
selected_measures_pks.append(measure.id)

serializable_kwargs = {
"selected_measures": selected_measures_pks,
}

return serializable_kwargs

@classmethod
def deserialize_init_kwargs(cls, form_kwargs: Dict) -> Dict:
serialized_selected_measures_pks = form_kwargs.get("selected_measures")
deserialized_selected_measures = models.Measure.objects.filter(
pk__in=serialized_selected_measures_pks
)

kwargs = {
"selected_measures": deserialized_selected_measures,
}

return kwargs


class MeasureDutiesForm(forms.Form):
class MeasureDutiesForm(forms.Form, SerializableFormMixin):
duties = forms.CharField(
label="Duties",
help_text="Enter the duty that applies to the measures.",
Expand Down Expand Up @@ -932,6 +1010,32 @@ def clean(self):

return cleaned_data

@classmethod
def serializable_init_kwargs(cls, kwargs: Dict) -> Dict:
selected_measures = kwargs.get("selected_measures")
selected_measures_pks = []
for measure in selected_measures:
selected_measures_pks.append(measure.id)

serializable_kwargs = {
"selected_measures": selected_measures_pks,
}

return serializable_kwargs

@classmethod
def deserialize_init_kwargs(cls, form_kwargs: Dict) -> Dict:
serialized_selected_measures_pks = form_kwargs.get("selected_measures")
deserialized_selected_measures = models.Measure.objects.filter(
pk__in=serialized_selected_measures_pks
)

kwargs = {
"selected_measures": deserialized_selected_measures,
}

return kwargs


class MeasureGeographicalAreaExclusionsForm(forms.Form):
excluded_area = forms.ModelChoiceField(
Expand Down Expand Up @@ -965,7 +1069,7 @@ def __init__(self, *args, **kwargs):
)


class MeasureGeographicalAreaExclusionsFormSet(FormSet):
class MeasureGeographicalAreaExclusionsFormSet(FormSet, SerializableFormMixin):
"""Allows editing the geographical area exclusions of multiple measures in
`MeasureEditWizard`."""

Expand Down
84 changes: 84 additions & 0 deletions measures/migrations/0017_measuresbulkeditor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# Generated by Django 4.2.15 on 2024-09-02 16:06

from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import django_fsm
import measures.models.bulk_processing


class Migration(migrations.Migration):

dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("workbaskets", "0008_datarow_dataupload"),
("measures", "0016_measuresbulkcreator"),
]

operations = [
migrations.CreateModel(
name="MeasuresBulkEditor",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"task_id",
models.CharField(blank=True, max_length=50, null=True, unique=True),
),
(
"processing_state",
django_fsm.FSMField(
choices=[
("AWAITING_PROCESSING", "Awaiting processing"),
("CURRENTLY_PROCESSING", "Currently processing"),
("SUCCESSFULLY_PROCESSED", "Successfully processed"),
("FAILED_PROCESSING", "Failed processing"),
("CANCELLED", "Cancelled"),
],
db_index=True,
default="AWAITING_PROCESSING",
editable=False,
max_length=50,
protected=True,
),
),
(
"successfully_processed_count",
models.PositiveIntegerField(default=0),
),
("form_data", models.JSONField()),
("form_kwargs", models.JSONField()),
("selected_measures", models.JSONField()),
(
"user",
models.ForeignKey(
editable=False,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to=settings.AUTH_USER_MODEL,
),
),
(
"workbasket",
models.ForeignKey(
editable=False,
null=True,
on_delete=measures.models.bulk_processing.REVOKE_TASKS_AND_SET_NULL,
to="workbaskets.workbasket",
),
),
],
options={
"abstract": False,
},
),
]
2 changes: 2 additions & 0 deletions measures/models/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from measures.models.bulk_processing import BulkProcessor
from measures.models.bulk_processing import MeasuresBulkCreator
from measures.models.bulk_processing import MeasuresBulkEditor
from measures.models.bulk_processing import ProcessingState
from measures.models.tracked_models import AdditionalCodeTypeMeasureType
from measures.models.tracked_models import DutyExpression
Expand All @@ -23,6 +24,7 @@
# - Classes exported from bulk_processing.py.
"BulkProcessor",
"MeasuresBulkCreator",
"MeasuresBulkEditor",
"ProcessingState",
# - Classes exported from tracked_model.py.
"AdditionalCodeTypeMeasureType",
Expand Down
Loading
Loading