From e585355017cca1e9aa1bb083b451a347f51bda31 Mon Sep 17 00:00:00 2001 From: Stuart Axon Date: Tue, 14 Jun 2022 15:36:59 +0100 Subject: [PATCH] Add --replay to dump_transactions: set transaction to the first one before the current workbasket to make the system act as it was when the workbasket was published. --- .../management/commands/dump_transactions.py | 70 +++++++++++++++++-- exporter/serializers.py | 1 - pii-ner-exclude.txt | 2 + .../management/commands/list_workbaskets.py | 19 +++++ workbaskets/management/util.py | 4 +- 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/exporter/management/commands/dump_transactions.py b/exporter/management/commands/dump_transactions.py index 9fffcae9ea..1bf6f43a3d 100644 --- a/exporter/management/commands/dump_transactions.py +++ b/exporter/management/commands/dump_transactions.py @@ -1,3 +1,4 @@ +import logging import os import sys @@ -6,6 +7,8 @@ from django.db.transaction import atomic from lxml import etree +from common.models import Transaction +from common.models.utils import override_current_transaction from common.serializers import validate_envelope from exporter.serializers import MultiFileEnvelopeTransactionSerializer from exporter.util import dit_file_generator @@ -17,6 +20,8 @@ # VARIATION_SELECTOR enables emoji presentation WARNING_SIGN_EMOJI = "\N{WARNING SIGN}\N{VARIATION SELECTOR-16}" +logger = logging.getLogger(__name__) + class Command(BaseCommand): """ @@ -48,7 +53,7 @@ def add_arguments(self, parser): parser.add_argument( "workbasket_ids", help=( - "Override the default selection of APPROVED workbaskets " + "Override the default selection of approved workbaskets " "with a comma-separated list of workbasket ids." ), nargs="*", @@ -72,19 +77,25 @@ def add_arguments(self, parser): action="store_true", ) + parser.add_argument( + "--replay", + help="Output transactions as at the time the workbasket was published, by setting current_transaction to " + "one before the specified workbasket.", + action="store_true", + ) + @atomic def handle(self, *args, **options): workbasket_ids = options.get("workbasket_ids") if workbasket_ids: query = dict(id__in=workbasket_ids) else: - query = dict(status=WorkflowStatus.APPROVED) + query = dict(status__in=WorkflowStatus.approved_statuses()) workbaskets = WorkBasket.objects.filter(**query) if not workbaskets: sys.exit("Nothing to upload: No workbaskets with status APPROVED.") - # transactions: will be serialized, then added to an envelope for uploaded. transactions = workbaskets.ordered_transactions() if not transactions: @@ -106,6 +117,54 @@ def handle(self, *args, **options): directory = options.get("directory", ".") + # 'replay' sets the current transaction to the one 'before' the specified workbasket + # to allow the system to export the data as it was when the workbasket was first published. + replay_from_tx = None + if options.get("replay"): + replay_from_tx = Transaction.objects.filter( + pk=transactions[0].pk - 1, + ).last() + + if replay_from_tx is None: + # No replay, just serialize the transactions. + if not self.do_serialize_envelopes( + directory, + envelope_id, + max_envelope_size, + transactions, + ): + sys.exit(1) + else: + # Simulate latest_transaction being set to replay_from_tx + if not set( + workbaskets.values_list("status", flat=True).distinct(), + ).issubset(WorkflowStatus.approved_statuses()): + sys.exit( + "Replay only applies to approved workbaskets.", + ) + + with override_current_transaction(replay_from_tx): + logging.debug("Replay from transaction: %s", replay_from_tx) + if not self.do_serialize_envelopes( + directory, + envelope_id, + max_envelope_size, + transactions, + ): + sys.exit(1) + + def do_serialize_envelopes( + self, + directory, + envelope_id, + max_envelope_size, + transactions, + ): + """ + Serialize transactions to a series of files. + + :return: True if no errors occurred. + """ output_file_constructor = dit_file_generator(directory, envelope_id) serializer = MultiFileEnvelopeTransactionSerializer( output_file_constructor, @@ -128,12 +187,11 @@ def handle(self, *args, **options): validate_envelope(envelope_file) except etree.DocumentInvalid: self.stdout.write( - f"{envelope_file.name} {WARNING_SIGN_EMOJI}️ Envelope invalid:", + f"{envelope_file.name} {WARNING_SIGN_EMOJI} ️ XML invalid.", ) else: total_transactions = len(rendered_envelope.transactions) self.stdout.write( f"{envelope_file.name} \N{WHITE HEAVY CHECK MARK} XML valid. {total_transactions} transactions, serialized in {time_to_render:.2f} seconds using {envelope_file.tell()} bytes.", ) - if errors: - sys.exit(1) + return not errors diff --git a/exporter/serializers.py b/exporter/serializers.py index bb3c9cdc1a..ceec2aded7 100644 --- a/exporter/serializers.py +++ b/exporter/serializers.py @@ -45,7 +45,6 @@ def __init__( self, output_constructor: callable, envelope_id=1, - benchmark=False, *args, **kwargs, ) -> None: diff --git a/pii-ner-exclude.txt b/pii-ner-exclude.txt index 298fd8d0fd..ccf96d8a49 100644 --- a/pii-ner-exclude.txt +++ b/pii-ner-exclude.txt @@ -1139,3 +1139,5 @@ self.linked_model "Attach BusinessRules WorkBasketOutputFormat Enum param kwargs: +Replay +Replay diff --git a/workbaskets/management/commands/list_workbaskets.py b/workbaskets/management/commands/list_workbaskets.py index 1975a47c1a..3ba6dbabdf 100644 --- a/workbaskets/management/commands/list_workbaskets.py +++ b/workbaskets/management/commands/list_workbaskets.py @@ -28,6 +28,17 @@ def add_arguments(self, parser: CommandParser) -> None: ), ) + approved_statuses = [ + status.name for status in WorkflowStatus.approved_statuses() + ] + parser.add_argument( + "-a", + "--approved-statuses", + dest="approved", + action="store_true", + help=f"List workbaskets with ANY of the approved statuses, equivile: [{', '.join(approved_statuses)}]", + ) + parser.add_argument( "-c", "--compact", @@ -44,7 +55,15 @@ def add_arguments(self, parser: CommandParser) -> None: def handle(self, *args: Any, **options: Any) -> Optional[str]: workbaskets = WorkBasket.objects.order_by("updated_at").all() + + workbasket_statuses = set() if options["status"]: + workbasket_statuses.update(options["status"]) + + if options.get("approved_statuses"): + workbasket_statuses.update(WorkflowStatus.approved_statuses()) + + if workbasket_statuses: workbaskets = workbaskets.filter(status__in=options["status"]) output_format = ( diff --git a/workbaskets/management/util.py b/workbaskets/management/util.py index 06280785a8..3c6be99c34 100644 --- a/workbaskets/management/util.py +++ b/workbaskets/management/util.py @@ -28,7 +28,7 @@ def _output_workbasket_readable( def _output_workbasket_compact(self, workbasket, show_transaction_info, **kwargs): self.stdout.write( - f"{workbasket.pk}, {first_line_of(workbasket.title)}, {first_line_of(workbasket.reason) or '-'}", + f"{workbasket.pk}, {first_line_of(workbasket.title)}, {first_line_of(workbasket.reason) or '-'}, {workbasket.status}", ending="" if show_transaction_info else "\n", ) if show_transaction_info: @@ -60,7 +60,7 @@ def output_workbaskets(self, workbaskets, show_transaction_info, output_format): """ if output_format == WorkBasketOutputFormat.COMPACT: self.stdout.write( - "pk, title, reason", + "pk, title, reason, status", ending="" if show_transaction_info else "\n", ) if show_transaction_info: