From e7d4fae454dbf00f88ab80d27d5e5d6477dc936a Mon Sep 17 00:00:00 2001 From: Elizabeth Esswein Date: Tue, 12 Mar 2024 16:57:12 -0400 Subject: [PATCH] update data object serialization --- .../bpmn/serializer/default/workflow.py | 2 - .../bpmn/serializer/migration/version_1_3.py | 32 +- .../serializer/migration/version_migration.py | 2 + .../data/serialization/v1.2-data-objects.json | 624 ++++++++++++++++++ .../bpmn/serializer/VersionMigrationTest.py | 25 + 5 files changed, 682 insertions(+), 3 deletions(-) create mode 100644 tests/SpiffWorkflow/bpmn/data/serialization/v1.2-data-objects.json diff --git a/SpiffWorkflow/bpmn/serializer/default/workflow.py b/SpiffWorkflow/bpmn/serializer/default/workflow.py index ae186324..c8d073bf 100644 --- a/SpiffWorkflow/bpmn/serializer/default/workflow.py +++ b/SpiffWorkflow/bpmn/serializer/default/workflow.py @@ -157,6 +157,4 @@ def subprocesses_from_dict(self, dct, workflow, top_workflow=None): sp = self.registry.restore(dct.pop(str(task.id)), task=task, top_workflow=top_workflow) top_workflow.subprocesses[task.id] = sp sp.completed_event.connect(task.task_spec._on_subworkflow_completed, task) - if len(sp.spec.data_objects) > 0: - sp.data = task.workflow.data self.subprocesses_from_dict(dct, sp, top_workflow) diff --git a/SpiffWorkflow/bpmn/serializer/migration/version_1_3.py b/SpiffWorkflow/bpmn/serializer/migration/version_1_3.py index 7bfd4781..74d3ca13 100644 --- a/SpiffWorkflow/bpmn/serializer/migration/version_1_3.py +++ b/SpiffWorkflow/bpmn/serializer/migration/version_1_3.py @@ -132,4 +132,34 @@ def add_new_typenames(dct): for sp in dct['subprocesses'].values(): sp['typename'] = 'BpmnSubWorkflow' for task in sp['tasks'].values(): - task['typename'] = 'Task' \ No newline at end of file + task['typename'] = 'Task' + +def update_data_objects(dct): + + def delete_duplicates(data_objects, spec, wf): + for name in data_objects: + spec['data_objects'].pop(name, None) + if name in wf['data']: + del wf['data'][name] + + def move_data_objects(data_objects, wf): + if len(data_objects) > 0: + wf['data']['data_objects'] = {} + for data_obj in data_objects: + if data_obj in wf['data']: + wf['data']['data_objects'][data_obj] = wf['data'].pop(data_obj) + + def update(wf, spec, parent_spec): + if len(spec.get('data_objects', [])) > 0 and 'data_objects' not in wf['data']: + for task in wf['tasks'].values(): + ts = spec['task_specs'].get(task['task_spec']) + if ts['typename'] in ['SubWorkflowTask', 'TransactionSubprocess'] and task['id'] in dct['subprocesses']: + sp_spec = dct['subprocess_specs'].get(ts['spec']) + sp = dct['subprocesses'][task['id']] + update(sp, sp_spec, spec) + if parent_spec is not None: + delete_duplicates(parent_spec['data_objects'], spec, wf) + move_data_objects(spec['data_objects'], wf) + + update(dct, dct['spec'], None) + diff --git a/SpiffWorkflow/bpmn/serializer/migration/version_migration.py b/SpiffWorkflow/bpmn/serializer/migration/version_migration.py index 7aaf98df..cf193283 100644 --- a/SpiffWorkflow/bpmn/serializer/migration/version_migration.py +++ b/SpiffWorkflow/bpmn/serializer/migration/version_migration.py @@ -33,6 +33,7 @@ remove_boundary_event_parent, remove_root_task, add_new_typenames, + update_data_objects, ) def from_version_1_2(dct): @@ -41,6 +42,7 @@ def from_version_1_2(dct): remove_boundary_event_parent(dct) remove_root_task(dct) add_new_typenames(dct) + update_data_objects(dct) def from_version_1_1(dct): diff --git a/tests/SpiffWorkflow/bpmn/data/serialization/v1.2-data-objects.json b/tests/SpiffWorkflow/bpmn/data/serialization/v1.2-data-objects.json new file mode 100644 index 00000000..f1795452 --- /dev/null +++ b/tests/SpiffWorkflow/bpmn/data/serialization/v1.2-data-objects.json @@ -0,0 +1,624 @@ +{ + "serializer_version": "1.2", + "data": { + "obj_1": "hello" + }, + "correlations": {}, + "last_task": "bd4ce289-cabd-406c-947c-91bd4af1aca2", + "success": true, + "tasks": { + "a829be95-412a-4100-884f-44062281dcd5": { + "id": "a829be95-412a-4100-884f-44062281dcd5", + "parent": null, + "children": [ + "db9f6ec1-94b7-43ba-b205-42c861036eb2" + ], + "last_state_change": 1710269712.3527431, + "state": 64, + "task_spec": "Start", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "db9f6ec1-94b7-43ba-b205-42c861036eb2": { + "id": "db9f6ec1-94b7-43ba-b205-42c861036eb2", + "parent": "a829be95-412a-4100-884f-44062281dcd5", + "children": [ + "92306cd9-bdee-46a2-af90-f3ebc5d2c5e2" + ], + "last_state_change": 1710269712.3546002, + "state": 64, + "task_spec": "Event_0kmwi7u", + "triggered": false, + "internal_data": { + "event_fired": true + }, + "data": {}, + "typename": "Task" + }, + "92306cd9-bdee-46a2-af90-f3ebc5d2c5e2": { + "id": "92306cd9-bdee-46a2-af90-f3ebc5d2c5e2", + "parent": "db9f6ec1-94b7-43ba-b205-42c861036eb2", + "children": [ + "3a25287d-e14e-4795-9568-77908cb4d0f1" + ], + "last_state_change": 1710269712.357311, + "state": 64, + "task_spec": "generate_data", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "3a25287d-e14e-4795-9568-77908cb4d0f1": { + "id": "3a25287d-e14e-4795-9568-77908cb4d0f1", + "parent": "92306cd9-bdee-46a2-af90-f3ebc5d2c5e2", + "children": [ + "bd4ce289-cabd-406c-947c-91bd4af1aca2" + ], + "last_state_change": 1710269712.3761137, + "state": 64, + "task_spec": "task_1", + "triggered": false, + "internal_data": {}, + "data": { + "obj_1": "hello again" + }, + "typename": "Task" + }, + "bd4ce289-cabd-406c-947c-91bd4af1aca2": { + "id": "bd4ce289-cabd-406c-947c-91bd4af1aca2", + "parent": "3a25287d-e14e-4795-9568-77908cb4d0f1", + "children": [ + "ff934c0a-014d-4320-a220-f4fdf8ae7f50" + ], + "last_state_change": 1710269712.3773048, + "state": 64, + "task_spec": "read_data", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "ff934c0a-014d-4320-a220-f4fdf8ae7f50": { + "id": "ff934c0a-014d-4320-a220-f4fdf8ae7f50", + "parent": "bd4ce289-cabd-406c-947c-91bd4af1aca2", + "children": [ + "4bd5af5c-fd10-4564-a2ce-1873349ec070" + ], + "last_state_change": 1710269712.3792934, + "state": 8, + "task_spec": "subprocess", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "4bd5af5c-fd10-4564-a2ce-1873349ec070": { + "id": "4bd5af5c-fd10-4564-a2ce-1873349ec070", + "parent": "ff934c0a-014d-4320-a220-f4fdf8ae7f50", + "children": [ + "fde13580-c26f-42ac-a44c-9275f0b17a15" + ], + "last_state_change": 1710269712.3500242, + "state": 4, + "task_spec": "Event_0qw1yr0", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "fde13580-c26f-42ac-a44c-9275f0b17a15": { + "id": "fde13580-c26f-42ac-a44c-9275f0b17a15", + "parent": "4bd5af5c-fd10-4564-a2ce-1873349ec070", + "children": [ + "9d30b468-b8c5-48a3-8801-3755d98a529c" + ], + "last_state_change": 1710269712.3501854, + "state": 4, + "task_spec": "Process.EndJoin", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "9d30b468-b8c5-48a3-8801-3755d98a529c": { + "id": "9d30b468-b8c5-48a3-8801-3755d98a529c", + "parent": "fde13580-c26f-42ac-a44c-9275f0b17a15", + "children": [], + "last_state_change": 1710269712.3504493, + "state": 4, + "task_spec": "End", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + } + }, + "root": "a829be95-412a-4100-884f-44062281dcd5", + "spec": { + "name": "Process", + "description": "Process", + "file": "/home/essweine/work/sartography/code/SpiffWorkflow/tests/SpiffWorkflow/bpmn/data/data_object.bpmn", + "task_specs": { + "Start": { + "name": "Start", + "description": "BPMN Task", + "manual": false, + "lookahead": 2, + "inputs": [], + "outputs": [ + "Event_0kmwi7u" + ], + "bpmn_id": null, + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "BpmnStartTask" + }, + "Process.EndJoin": { + "name": "Process.EndJoin", + "description": "BPMN Task", + "manual": false, + "lookahead": 2, + "inputs": [ + "Event_0qw1yr0" + ], + "outputs": [ + "End" + ], + "bpmn_id": null, + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "_EndJoin" + }, + "End": { + "name": "End", + "description": "BPMN Task", + "manual": false, + "lookahead": 2, + "inputs": [ + "Process.EndJoin" + ], + "outputs": [], + "bpmn_id": null, + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "SimpleBpmnTask" + }, + "Event_0kmwi7u": { + "name": "Event_0kmwi7u", + "description": "Default Start Event", + "manual": false, + "lookahead": 2, + "inputs": [ + "Start" + ], + "outputs": [ + "generate_data" + ], + "bpmn_id": "Event_0kmwi7u", + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "event_definition": { + "description": "Default", + "name": null, + "typename": "NoneEventDefinition" + }, + "typename": "StartEvent", + "extensions": {} + }, + "generate_data": { + "name": "generate_data", + "description": "User Task", + "manual": true, + "lookahead": 2, + "inputs": [ + "Event_0kmwi7u" + ], + "outputs": [ + "task_1" + ], + "bpmn_id": "generate_data", + "bpmn_name": "Generate Data", + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [ + { + "bpmn_id": "obj_1", + "bpmn_name": null, + "typename": "DataObject" + } + ], + "io_specification": null, + "typename": "UserTask", + "extensions": {} + }, + "task_1": { + "name": "task_1", + "description": "User Task", + "manual": true, + "lookahead": 2, + "inputs": [ + "generate_data" + ], + "outputs": [ + "read_data" + ], + "bpmn_id": "task_1", + "bpmn_name": "Task", + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "UserTask", + "extensions": {} + }, + "read_data": { + "name": "read_data", + "description": "User Task", + "manual": true, + "lookahead": 2, + "inputs": [ + "task_1" + ], + "outputs": [ + "subprocess" + ], + "bpmn_id": "read_data", + "bpmn_name": "Read Data", + "lane": null, + "documentation": null, + "data_input_associations": [ + { + "bpmn_id": "obj_1", + "bpmn_name": null, + "typename": "DataObject" + } + ], + "data_output_associations": [], + "io_specification": null, + "typename": "UserTask", + "extensions": {} + }, + "subprocess": { + "name": "subprocess", + "description": "Subprocess", + "manual": false, + "lookahead": 2, + "inputs": [ + "read_data" + ], + "outputs": [ + "Event_0qw1yr0" + ], + "bpmn_id": "subprocess", + "bpmn_name": "Subprocess", + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "spec": "subprocess", + "typename": "SubWorkflowTask", + "extensions": {} + }, + "Event_0qw1yr0": { + "name": "Event_0qw1yr0", + "description": "Default End Event", + "manual": false, + "lookahead": 2, + "inputs": [ + "subprocess" + ], + "outputs": [ + "Process.EndJoin" + ], + "bpmn_id": "Event_0qw1yr0", + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "event_definition": { + "description": "Default", + "name": null, + "typename": "NoneEventDefinition" + }, + "typename": "EndEvent", + "extensions": {} + } + }, + "io_specification": null, + "data_objects": { + "obj_1": { + "bpmn_id": "obj_1", + "bpmn_name": null, + "typename": "DataObject" + } + }, + "correlation_keys": {}, + "typename": "BpmnProcessSpec" + }, + "subprocess_specs": { + "subprocess": { + "name": "subprocess", + "description": "Subprocess", + "file": "/home/essweine/work/sartography/code/SpiffWorkflow/tests/SpiffWorkflow/bpmn/data/data_object.bpmn", + "task_specs": { + "Start": { + "name": "Start", + "description": "BPMN Task", + "manual": false, + "lookahead": 2, + "inputs": [], + "outputs": [ + "Event_1wuwx2f" + ], + "bpmn_id": null, + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "BpmnStartTask" + }, + "subprocess.EndJoin": { + "name": "subprocess.EndJoin", + "description": "BPMN Task", + "manual": false, + "lookahead": 2, + "inputs": [ + "Event_1qcnmnt" + ], + "outputs": [ + "End" + ], + "bpmn_id": null, + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "_EndJoin" + }, + "End": { + "name": "End", + "description": "BPMN Task", + "manual": false, + "lookahead": 2, + "inputs": [ + "subprocess.EndJoin" + ], + "outputs": [], + "bpmn_id": null, + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "typename": "SimpleBpmnTask" + }, + "Event_1wuwx2f": { + "name": "Event_1wuwx2f", + "description": "Default Start Event", + "manual": false, + "lookahead": 2, + "inputs": [ + "Start" + ], + "outputs": [ + "placeholder" + ], + "bpmn_id": "Event_1wuwx2f", + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "event_definition": { + "description": "Default", + "name": null, + "typename": "NoneEventDefinition" + }, + "typename": "StartEvent", + "extensions": {} + }, + "placeholder": { + "name": "placeholder", + "description": "Task", + "manual": true, + "lookahead": 2, + "inputs": [ + "Event_1wuwx2f" + ], + "outputs": [ + "Event_1qcnmnt" + ], + "bpmn_id": "placeholder", + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [ + { + "bpmn_id": "obj_1", + "bpmn_name": null, + "typename": "DataObject" + } + ], + "data_output_associations": [ + { + "bpmn_id": "obj_1", + "bpmn_name": null, + "typename": "DataObject" + } + ], + "io_specification": null, + "typename": "NoneTask", + "extensions": {} + }, + "Event_1qcnmnt": { + "name": "Event_1qcnmnt", + "description": "Default End Event", + "manual": false, + "lookahead": 2, + "inputs": [ + "placeholder" + ], + "outputs": [ + "subprocess.EndJoin" + ], + "bpmn_id": "Event_1qcnmnt", + "bpmn_name": null, + "lane": null, + "documentation": null, + "data_input_associations": [], + "data_output_associations": [], + "io_specification": null, + "event_definition": { + "description": "Default", + "name": null, + "typename": "NoneEventDefinition" + }, + "typename": "EndEvent", + "extensions": {} + } + }, + "io_specification": null, + "data_objects": { + "obj_1": { + "bpmn_id": "obj_1", + "bpmn_name": null, + "typename": "DataObject" + } + }, + "correlation_keys": {}, + "typename": "BpmnProcessSpec" + } + }, + "subprocesses": { + "ff934c0a-014d-4320-a220-f4fdf8ae7f50": { + "data": { + "obj_1": "hello" + }, + "correlations": {}, + "last_task": "98316300-1e10-4f0b-b833-209e0737cb85", + "success": true, + "tasks": { + "98316300-1e10-4f0b-b833-209e0737cb85": { + "id": "98316300-1e10-4f0b-b833-209e0737cb85", + "parent": null, + "children": [ + "5d16c0f5-5ff9-4a8b-b500-1c6247d88aef" + ], + "last_state_change": 1710269712.378701, + "state": 64, + "task_spec": "Start", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "5d16c0f5-5ff9-4a8b-b500-1c6247d88aef": { + "id": "5d16c0f5-5ff9-4a8b-b500-1c6247d88aef", + "parent": "98316300-1e10-4f0b-b833-209e0737cb85", + "children": [ + "72df5034-5f0f-43a5-995e-5ff77fb07863" + ], + "last_state_change": 1710269712.3788383, + "state": 16, + "task_spec": "Event_1wuwx2f", + "triggered": false, + "internal_data": { + "event_fired": true + }, + "data": {}, + "typename": "Task" + }, + "72df5034-5f0f-43a5-995e-5ff77fb07863": { + "id": "72df5034-5f0f-43a5-995e-5ff77fb07863", + "parent": "5d16c0f5-5ff9-4a8b-b500-1c6247d88aef", + "children": [ + "06a8459b-b041-41de-82f7-c37c8fee5182" + ], + "last_state_change": 1710269712.377747, + "state": 4, + "task_spec": "placeholder", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "06a8459b-b041-41de-82f7-c37c8fee5182": { + "id": "06a8459b-b041-41de-82f7-c37c8fee5182", + "parent": "72df5034-5f0f-43a5-995e-5ff77fb07863", + "children": [ + "2ada7c08-0e16-4055-8cf0-449ad10025b9" + ], + "last_state_change": 1710269712.377866, + "state": 4, + "task_spec": "Event_1qcnmnt", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "2ada7c08-0e16-4055-8cf0-449ad10025b9": { + "id": "2ada7c08-0e16-4055-8cf0-449ad10025b9", + "parent": "06a8459b-b041-41de-82f7-c37c8fee5182", + "children": [ + "d90e5d81-c3fc-4057-80fe-d47c86384afc" + ], + "last_state_change": 1710269712.3779793, + "state": 4, + "task_spec": "subprocess.EndJoin", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + }, + "d90e5d81-c3fc-4057-80fe-d47c86384afc": { + "id": "d90e5d81-c3fc-4057-80fe-d47c86384afc", + "parent": "2ada7c08-0e16-4055-8cf0-449ad10025b9", + "children": [], + "last_state_change": 1710269712.3780928, + "state": 4, + "task_spec": "End", + "triggered": false, + "internal_data": {}, + "data": {}, + "typename": "Task" + } + }, + "root": "98316300-1e10-4f0b-b833-209e0737cb85", + "parent_task_id": "ff934c0a-014d-4320-a220-f4fdf8ae7f50", + "spec": "subprocess", + "typename": "BpmnSubWorkflow" + } + }, + "bpmn_events": [], + "typename": "BpmnWorkflow" +} diff --git a/tests/SpiffWorkflow/bpmn/serializer/VersionMigrationTest.py b/tests/SpiffWorkflow/bpmn/serializer/VersionMigrationTest.py index 1ebed64f..ac245762 100644 --- a/tests/SpiffWorkflow/bpmn/serializer/VersionMigrationTest.py +++ b/tests/SpiffWorkflow/bpmn/serializer/VersionMigrationTest.py @@ -121,3 +121,28 @@ def test_remove_noninterrupting_boundary_events(self): wf.get_next_task(spec_name='sid-6FBBB56D-00CD-4C2B-9345-486986BB4992').run() wf.do_engine_steps() self.assertTrue(wf.is_completed()) + + def test_update_data_objects(self): + + # Workflow source: serialized from DataObjectTest after the subprocess has been created + wf = self.deserialize_workflow('v1.2-data-objects.json') + + # Check that the data objects were removed from the subprocess + sp_task = wf.get_next_task(spec_name='subprocess') + sp = wf.get_subprocess(sp_task) + self.assertNotIn('obj_1', sp.data) + self.assertNotIn('data_objects', sp.data) + sp_spec = wf.subprocess_specs.get('subprocess') + self.assertEqual(len(sp_spec.data_objects), 0) + + # Make sure we can complete the process as we did in the original test + wf.do_engine_steps() + ready_tasks = wf.get_tasks(state=TaskState.READY) + self.assertEqual(ready_tasks[0].data['obj_1'], 'hello') + ready_tasks[0].data['obj_1'] = 'hello again' + ready_tasks[0].run() + wf.do_engine_steps() + # The data object is not in the task data + self.assertNotIn('obj_1', sp_task.data) + # The update should persist in the main process + self.assertEqual(wf.data_objects['obj_1'], 'hello again')