Shifting to a different model, where the TaskEvents store ONLY the form data submitted for that task.
In order to allow proper deletion of tasks, we no longer merge data returned from the front end, we set it directly as the task_data. When returning data to the front end, we take any previous form submission and merge it into the current task data, allowing users to keep their previous submissions. There is now an "extract_form_data" method that does it's best job to calculate what form data might have changed from the front end.
This commit is contained in:
Dan Funk
parent
da048d358e
commit
6aec15cc7c
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "faaf0e1f31f4bf99df366e52df20bb148a05996a0e6467767660665c514af2d7"
|
||||
"sha256": "78a8da35dec2fb58b02a58afc8ffabe8b1c22bec8f054295e8b1ba3b4a6f4ec0"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -261,6 +261,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.1.2"
|
||||
},
|
||||
"flask-admin": {
|
||||
"hashes": [
|
||||
"sha256:68c761d8582d59b1f7702013e944a7ad11d7659a72f3006b89b68b0bd8df61b8"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.5.6"
|
||||
},
|
||||
"flask-bcrypt": {
|
||||
"hashes": [
|
||||
"sha256:d71c8585b2ee1c62024392ebdbc447438564e2c8c02b4e57b56a4cafd8d13c5f"
|
||||
|
|
@ -558,25 +565,25 @@
|
|||
},
|
||||
"pandas": {
|
||||
"hashes": [
|
||||
"sha256:034185bb615dc96d08fa13aacba8862949db19d5e7804d6ee242d086f07bcc46",
|
||||
"sha256:0c9b7f1933e3226cc16129cf2093338d63ace5c85db7c9588e3e1ac5c1937ad5",
|
||||
"sha256:1f6fcf0404626ca0475715da045a878c7062ed39bc859afc4ccf0ba0a586a0aa",
|
||||
"sha256:1fc963ba33c299973e92d45466e576d11f28611f3549469aec4a35658ef9f4cc",
|
||||
"sha256:29b4cfee5df2bc885607b8f016e901e63df7ffc8f00209000471778f46cc6678",
|
||||
"sha256:2a8b6c28607e3f3c344fe3e9b3cd76d2bf9f59bc8c0f2e582e3728b80e1786dc",
|
||||
"sha256:2bc2ff52091a6ac481cc75d514f06227dc1b10887df1eb72d535475e7b825e31",
|
||||
"sha256:415e4d52fcfd68c3d8f1851cef4d947399232741cc994c8f6aa5e6a9f2e4b1d8",
|
||||
"sha256:519678882fd0587410ece91e3ff7f73ad6ded60f6fcb8aa7bcc85c1dc20ecac6",
|
||||
"sha256:51e0abe6e9f5096d246232b461649b0aa627f46de8f6344597ca908f2240cbaa",
|
||||
"sha256:698e26372dba93f3aeb09cd7da2bb6dd6ade248338cfe423792c07116297f8f4",
|
||||
"sha256:83af85c8e539a7876d23b78433d90f6a0e8aa913e37320785cf3888c946ee874",
|
||||
"sha256:982cda36d1773076a415ec62766b3c0a21cdbae84525135bdb8f460c489bb5dd",
|
||||
"sha256:a647e44ba1b3344ebc5991c8aafeb7cca2b930010923657a273b41d86ae225c4",
|
||||
"sha256:b35d625282baa7b51e82e52622c300a1ca9f786711b2af7cbe64f1e6831f4126",
|
||||
"sha256:bab51855f8b318ef39c2af2c11095f45a10b74cbab4e3c8199efcc5af314c648"
|
||||
"sha256:02f1e8f71cd994ed7fcb9a35b6ddddeb4314822a0e09a9c5b2d278f8cb5d4096",
|
||||
"sha256:13f75fb18486759da3ff40f5345d9dd20e7d78f2a39c5884d013456cec9876f0",
|
||||
"sha256:35b670b0abcfed7cad76f2834041dcf7ae47fd9b22b63622d67cdc933d79f453",
|
||||
"sha256:4c73f373b0800eb3062ffd13d4a7a2a6d522792fa6eb204d67a4fad0a40f03dc",
|
||||
"sha256:5759edf0b686b6f25a5d4a447ea588983a33afc8a0081a0954184a4a87fd0dd7",
|
||||
"sha256:5a7cf6044467c1356b2b49ef69e50bf4d231e773c3ca0558807cdba56b76820b",
|
||||
"sha256:69c5d920a0b2a9838e677f78f4dde506b95ea8e4d30da25859db6469ded84fa8",
|
||||
"sha256:8778a5cc5a8437a561e3276b85367412e10ae9fff07db1eed986e427d9a674f8",
|
||||
"sha256:9871ef5ee17f388f1cb35f76dc6106d40cb8165c562d573470672f4cdefa59ef",
|
||||
"sha256:9c31d52f1a7dd2bb4681d9f62646c7aa554f19e8e9addc17e8b1b20011d7522d",
|
||||
"sha256:ab8173a8efe5418bbe50e43f321994ac6673afc5c7c4839014cf6401bbdd0705",
|
||||
"sha256:ae961f1f0e270f1e4e2273f6a539b2ea33248e0e3a11ffb479d757918a5e03a9",
|
||||
"sha256:b3c4f93fcb6e97d993bf87cdd917883b7dab7d20c627699f360a8fb49e9e0b91",
|
||||
"sha256:c9410ce8a3dee77653bc0684cfa1535a7f9c291663bd7ad79e39f5ab58f67ab3",
|
||||
"sha256:f69e0f7b7c09f1f612b1f8f59e2df72faa8a6b41c5a436dde5b615aaf948f107",
|
||||
"sha256:faa42a78d1350b02a7d2f0dbe3c80791cf785663d6997891549d0f86dc49125e"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.0.4"
|
||||
"version": "==1.0.5"
|
||||
},
|
||||
"psycopg2-binary": {
|
||||
"hashes": [
|
||||
|
|
@ -711,11 +718,11 @@
|
|||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee",
|
||||
"sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"
|
||||
"sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
|
||||
"sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.23.0"
|
||||
"version": "==2.24.0"
|
||||
},
|
||||
"sentry-sdk": {
|
||||
"extras": [
|
||||
|
|
@ -802,7 +809,7 @@
|
|||
"spiffworkflow": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/sartography/SpiffWorkflow.git",
|
||||
"ref": "b8a064a0bb76c705a1be04ee9bb8ac7beee56eb0"
|
||||
"ref": "5450dc0463a95811d386b7de063d950bf6179d2b"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
|
|
@ -890,6 +897,13 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"wtforms": {
|
||||
"hashes": [
|
||||
"sha256:6ff8635f4caeed9f38641d48cfe019d0d3896f41910ab04494143fc027866e1b",
|
||||
"sha256:861a13b3ae521d6700dac3b2771970bd354a63ba7043ecc3a82b5288596a1972"
|
||||
],
|
||||
"version": "==2.3.1"
|
||||
},
|
||||
"xlrd": {
|
||||
"hashes": [
|
||||
"sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2",
|
||||
|
|
|
|||
|
|
@ -57,9 +57,9 @@ def json_formatter(view, context, model, name):
|
|||
|
||||
class TaskEventView(AdminModelView):
|
||||
column_filters = ['workflow_id', 'action']
|
||||
column_list = ['study_id', 'user_id', 'workflow_id', 'action', 'task_title', 'task_data', 'date']
|
||||
column_list = ['study_id', 'user_id', 'workflow_id', 'action', 'task_title', 'form_data', 'date']
|
||||
column_formatters = {
|
||||
'task_data': json_formatter,
|
||||
'form_data': json_formatter,
|
||||
}
|
||||
|
||||
admin = Admin(app)
|
||||
|
|
|
|||
|
|
@ -145,14 +145,14 @@ def update_task(workflow_id, task_id, body):
|
|||
if spiff_task.state != spiff_task.READY:
|
||||
raise ApiError("invalid_state", "You may not update a task unless it is in the READY state. "
|
||||
"Consider calling a token reset to make this task Ready.")
|
||||
spiff_task.update_data(body)
|
||||
if body: # IF and only if we get the body back, update the task data with the content.
|
||||
spiff_task.data = body # Accept the data from the front end as complete. Do not merge it in, as then it is impossible to remove items.
|
||||
processor.complete_task(spiff_task)
|
||||
processor.do_engine_steps()
|
||||
processor.save()
|
||||
|
||||
WorkflowService.log_task_action(user_uid, workflow_model, spiff_task, WorkflowService.TASK_ACTION_COMPLETE,
|
||||
version=processor.get_version_string(),
|
||||
updated_data=spiff_task.data)
|
||||
version=processor.get_version_string())
|
||||
workflow_api_model = WorkflowService.processor_to_workflow_api(processor)
|
||||
return WorkflowApiSchema().dump(workflow_api_model)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class TaskEventModel(db.Model):
|
|||
task_title = db.Column(db.String)
|
||||
task_type = db.Column(db.String)
|
||||
task_state = db.Column(db.String)
|
||||
task_data = db.Column(db.JSON)
|
||||
form_data = db.Column(db.JSON) # And form data submitted when the task was completed.
|
||||
mi_type = db.Column(db.String)
|
||||
mi_count = db.Column(db.Integer)
|
||||
mi_index = db.Column(db.Integer)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import random
|
|||
import jinja2
|
||||
from SpiffWorkflow import Task as SpiffTask, WorkflowException
|
||||
from SpiffWorkflow.bpmn.specs.ManualTask import ManualTask
|
||||
from SpiffWorkflow.bpmn.specs.MultiInstanceTask import MultiInstanceTask
|
||||
from SpiffWorkflow.bpmn.specs.ScriptTask import ScriptTask
|
||||
from SpiffWorkflow.bpmn.specs.UserTask import UserTask
|
||||
from SpiffWorkflow.dmn.specs.BusinessRuleTask import BusinessRuleTask
|
||||
|
|
@ -232,23 +233,25 @@ class WorkflowService(object):
|
|||
# This may or may not work, sometimes there is no next task to complete.
|
||||
next_task = processor.next_task()
|
||||
if next_task:
|
||||
previous_form_data = WorkflowService.get_previously_submitted_data(processor.workflow_model.id, next_task)
|
||||
DeepMerge.merge(next_task.data, previous_form_data)
|
||||
workflow_api.next_task = WorkflowService.spiff_task_to_api_task(next_task, add_docs_and_forms=True)
|
||||
|
||||
return workflow_api
|
||||
|
||||
@staticmethod
|
||||
def get_previously_submitted_data(workflow_id, task):
|
||||
""" If the user has completed this task previously, find that data in the task events table, and return it."""
|
||||
""" If the user has completed this task previously, find the form data for the last submission."""
|
||||
latest_event = db.session.query(TaskEventModel) \
|
||||
.filter_by(workflow_id=workflow_id) \
|
||||
.filter_by(task_name=task.task_spec.name) \
|
||||
.filter_by(action=WorkflowService.TASK_ACTION_COMPLETE) \
|
||||
.order_by(TaskEventModel.date.desc()).first()
|
||||
if latest_event:
|
||||
if latest_event.task_data is not None:
|
||||
return latest_event.task_data
|
||||
if latest_event.form_data is not None:
|
||||
return latest_event.form_data
|
||||
else:
|
||||
app.logger.error("missing_task_data", "We have lost data for workflow %i, task %s, it is not "
|
||||
app.logger.error("missing_form_dat", "We have lost data for workflow %i, task %s, it is not "
|
||||
"in the task event model, "
|
||||
"and it should be." % (workflow_id, task.task_spec.name))
|
||||
return {}
|
||||
|
|
@ -387,9 +390,9 @@ class WorkflowService(object):
|
|||
field.options.append({"id": d.value, "name": d.label})
|
||||
|
||||
@staticmethod
|
||||
def log_task_action(user_uid, workflow_model, spiff_task, action,
|
||||
version, updated_data=None):
|
||||
def log_task_action(user_uid, workflow_model, spiff_task, action, version):
|
||||
task = WorkflowService.spiff_task_to_api_task(spiff_task)
|
||||
form_data = WorkflowService.extract_form_data(spiff_task.data, spiff_task)
|
||||
task_event = TaskEventModel(
|
||||
study_id=workflow_model.study_id,
|
||||
user_uid=user_uid,
|
||||
|
|
@ -402,7 +405,7 @@ class WorkflowService(object):
|
|||
task_title=task.title,
|
||||
task_type=str(task.type),
|
||||
task_state=task.state,
|
||||
task_data=updated_data,
|
||||
form_data=form_data,
|
||||
mi_type=task.multi_instance_type.value, # Some tasks have a repeat behavior.
|
||||
mi_count=task.multi_instance_count, # This is the number of times the task could repeat.
|
||||
mi_index=task.multi_instance_index, # And the index of the currently repeating task.
|
||||
|
|
@ -436,43 +439,40 @@ class WorkflowService(object):
|
|||
# added in subsequent tasks, just looking at form data, will not track the automated
|
||||
# task data additions, hopefully this doesn't hang us.
|
||||
for log in task_logs:
|
||||
if log.task_data is not None: # Only do this if the task event does not have data populated in it.
|
||||
continue
|
||||
# if log.task_data is not None: # Only do this if the task event does not have data populated in it.
|
||||
# continue
|
||||
data = copy.deepcopy(latest_data) # Or you end up with insane crazy issues.
|
||||
# In the simple case of RRT, there is exactly one task for the given task_spec
|
||||
task = processor.bpmn_workflow.get_tasks_from_spec_name(log.task_name)[0]
|
||||
data = WorkflowService.__remove_data_added_by_children(data, task.children[0])
|
||||
log.task_data = data
|
||||
data = WorkflowService.extract_form_data(data, task)
|
||||
log.form_data = data
|
||||
db.session.add(log)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def __remove_data_added_by_children(latest_data, child_task):
|
||||
def extract_form_data(latest_data, task):
|
||||
"""Removes data from latest_data that would be added by the child task or any of it's children."""
|
||||
if hasattr(child_task.task_spec, 'form'):
|
||||
for field in child_task.task_spec.form.fields:
|
||||
latest_data.pop(field.id, None)
|
||||
data = {}
|
||||
|
||||
if hasattr(task.task_spec, 'form'):
|
||||
for field in task.task_spec.form.fields:
|
||||
if field.has_property(Task.PROP_OPTIONS_READ_ONLY) and \
|
||||
field.get_property(Task.PROP_OPTIONS_READ_ONLY).lower().strip() == "true":
|
||||
continue # Don't pop off read only fields.
|
||||
if field.has_property(Task.PROP_OPTIONS_REPEAT):
|
||||
continue # Don't add read-only data
|
||||
elif field.has_property(Task.PROP_OPTIONS_REPEAT):
|
||||
group = field.get_property(Task.PROP_OPTIONS_REPEAT)
|
||||
group_data = []
|
||||
if group in latest_data:
|
||||
for item in latest_data[group]:
|
||||
item.pop(field.id, None)
|
||||
if item:
|
||||
group_data.append(item)
|
||||
latest_data[group] = group_data
|
||||
if not latest_data[group]:
|
||||
latest_data.pop(group, None)
|
||||
if isinstance(child_task.task_spec, BusinessRuleTask):
|
||||
for output in child_task.task_spec.dmnEngine.decisionTable.outputs:
|
||||
latest_data.pop(output.name, None)
|
||||
for child in child_task.children:
|
||||
latest_data = WorkflowService.__remove_data_added_by_children(latest_data, child)
|
||||
return latest_data
|
||||
data[group] = latest_data[group]
|
||||
elif isinstance(task.task_spec, MultiInstanceTask):
|
||||
group = task.task_spec.elementVar
|
||||
if group in latest_data:
|
||||
data[group] = latest_data[group]
|
||||
else:
|
||||
if field.id in latest_data:
|
||||
data[field.id] = latest_data[field.id]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
"""empty message
|
||||
|
||||
Revision ID: de30304ff5e6
|
||||
Revises: 1fdd1bdb600e
|
||||
Create Date: 2020-06-18 16:19:11.133665
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'de30304ff5e6'
|
||||
down_revision = '1fdd1bdb600e'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column('task_event', sa.Column('form_data', sa.JSON(), nullable=True))
|
||||
op.drop_column('task_event', 'task_data')
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column('task_event', sa.Column('task_data', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True))
|
||||
op.drop_column('task_event', 'form_data')
|
||||
# ### end Alembic commands ###
|
||||
|
|
@ -77,9 +77,8 @@ class TestTasksApi(BaseTest):
|
|||
self.assertEquals(task_in.process_name, event.process_name)
|
||||
self.assertIsNotNone(event.date)
|
||||
|
||||
# Assure that the data provided occurs in the task data log.
|
||||
for key in dict_data.keys():
|
||||
self.assertIn(key, event.task_data)
|
||||
# Assure that there is data in the form_data
|
||||
self.assertIsNotNone(event.form_data)
|
||||
|
||||
workflow = WorkflowApiSchema().load(json_data)
|
||||
return workflow
|
||||
|
|
@ -372,13 +371,13 @@ class TestTasksApi(BaseTest):
|
|||
self.assertEqual("UserTask", task.type)
|
||||
self.assertEqual("Activity_A", task.name)
|
||||
self.assertEqual("My Sub Process", task.process_name)
|
||||
workflow_api = self.complete_form(workflow, task, {"name": "Dan"})
|
||||
workflow_api = self.complete_form(workflow, task, {"FieldA": "Dan"})
|
||||
task = workflow_api.next_task
|
||||
self.assertIsNotNone(task)
|
||||
|
||||
self.assertEqual("Activity_B", task.name)
|
||||
self.assertEqual("Sub Workflow Example", task.process_name)
|
||||
workflow_api = self.complete_form(workflow, task, {"name": "Dan"})
|
||||
workflow_api = self.complete_form(workflow, task, {"FieldB": "Dan"})
|
||||
self.assertEqual(WorkflowStatus.complete, workflow_api.status)
|
||||
|
||||
def test_update_task_resets_token(self):
|
||||
|
|
@ -446,7 +445,9 @@ class TestTasksApi(BaseTest):
|
|||
|
||||
for i in random.sample(range(9), 9):
|
||||
task = TaskSchema().load(ready_items[i]['task'])
|
||||
self.complete_form(workflow, task, {"investigator":{"email": "dhf8r@virginia.edu"}})
|
||||
data = workflow_api.next_task.data
|
||||
data['investigator']['email'] = "dhf8r@virginia.edu"
|
||||
self.complete_form(workflow, task, data)
|
||||
#tasks = self.get_workflow_api(workflow).user_tasks
|
||||
|
||||
workflow = self.get_workflow_api(workflow)
|
||||
|
|
|
|||
|
|
@ -100,11 +100,10 @@ class TestWorkflowService(BaseTest):
|
|||
task_api = WorkflowService.spiff_task_to_api_task(task, add_docs_and_forms=True)
|
||||
WorkflowService.populate_form_with_random_data(task, task_api, False)
|
||||
task.complete()
|
||||
# create the task events with no task_data in them.
|
||||
# create the task events
|
||||
WorkflowService.log_task_action('dhf8r', workflow, task,
|
||||
WorkflowService.TASK_ACTION_COMPLETE,
|
||||
version=processor.get_version_string(),
|
||||
updated_data=None)
|
||||
version=processor.get_version_string())
|
||||
processor.save()
|
||||
db.session.commit()
|
||||
|
||||
|
|
@ -119,19 +118,17 @@ class TestWorkflowService(BaseTest):
|
|||
self.assertEqual(17, len(task_logs))
|
||||
for log in task_logs:
|
||||
task = processor.bpmn_workflow.get_tasks_from_spec_name(log.task_name)[0]
|
||||
self.assertIsNotNone(log.task_data)
|
||||
self.assertIsNotNone(log.form_data)
|
||||
# Each task should have the data in the form for that task in the task event.
|
||||
if hasattr(task.task_spec, 'form'):
|
||||
for field in task.task_spec.form.fields:
|
||||
if field.has_property(Task.PROP_OPTIONS_REPEAT):
|
||||
self.assertIn(field.get_property(Task.PROP_OPTIONS_REPEAT), log.task_data)
|
||||
self.assertIn(field.get_property(Task.PROP_OPTIONS_REPEAT), log.form_data)
|
||||
else:
|
||||
self.assertIn(field.id, log.task_data)
|
||||
self.assertIn(field.id, log.form_data)
|
||||
|
||||
# Some spot checks:
|
||||
# The first task should be empty, with all the data removed.
|
||||
self.assertEqual({}, task_logs[0].task_data)
|
||||
self.assertEqual({}, task_logs[0].form_data)
|
||||
|
||||
# The last task should have all the data.
|
||||
self.assertDictEqual(processor.bpmn_workflow.last_task.data, task_logs[16].task_data)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue