Squashed 'SpiffWorkflow/' changes from 5cdb881ed..a6392d190

a6392d190 SpiffWorkflow cold start improvements (#13)

git-subtree-dir: SpiffWorkflow
git-subtree-split: a6392d19061f623394f5705fb78af23673d3940d
This commit is contained in:
burnettk 2022-11-02 12:19:52 -04:00
parent 53bc093c53
commit c3cdca16b8
17 changed files with 84 additions and 84 deletions

View File

@ -3,23 +3,10 @@ import ast
import copy
import sys
import traceback
import datetime
import dateparser
import pytz
from SpiffWorkflow.bpmn.exceptions import WorkflowTaskExecException
from ..operators import Operator
# Would love to get rid of this altogether, as it rightly belongs in the
# backend, but leaving it here because that's the path of least resistance.
DEFAULT_GLOBALS = {
'timedelta': datetime.timedelta,
'datetime': datetime,
'dateparser': dateparser,
'pytz': pytz,
}
# Copyright (C) 2020 Kelly McDonald
#
@ -112,7 +99,7 @@ class PythonScriptEngine(object):
def __init__(self, default_globals=None, scripting_additions=None):
self.globals = default_globals or DEFAULT_GLOBALS
self.globals = default_globals or {}
self.globals.update(scripting_additions or {})
self.error_tasks = {}

View File

@ -25,13 +25,6 @@ from .base import TaskSpec
from ..operators import valueof, Attrib, PathAttrib
from ..util.deep_merge import DeepMerge
try:
from celery.app import default_app
except ImportError:
have_celery = False
else:
have_celery = True
logger = logging.getLogger('spiff')
@ -111,7 +104,10 @@ class Celery(TaskSpec):
:type kwargs: dict
:param kwargs: kwargs to pass to celery task.
"""
if not have_celery:
try:
from celery.app import default_app
except ImportError:
raise Exception("Unable to import python-celery imports.")
assert wf_spec is not None
assert name is not None

View File

@ -74,7 +74,7 @@ Serialization
.. warning::
Serialization Changed in Version 1.1.7.
Serialization Changed in Version 1.1.7.
Support for pre-1.1.7 serialization will be dropped in a future release.
The old serialization method still works but it is deprecated.
To migrate your system to the new version, see "Migrating between
@ -85,8 +85,8 @@ setting. This may not always be the case, we may be executing the workflow in th
may have a user request a web page where we open a specific workflow that we may be in the middle of, do one step of
that workflow and then the user may be back in a few minutes, or maybe a few hours depending on the application.
The :code:`BpmnWorkflowSerializer` class contains a serializer for a workflow containing only standard BPMN Tasks.
Since we are using custom task classes (the Camunda :code:`UserTask` and the DMN :code:`BusinessRuleTask`),
The :code:`BpmnWorkflowSerializer` class contains a serializer for a workflow containing only standard BPMN Tasks.
Since we are using custom task classes (the Camunda :code:`UserTask` and the DMN :code:`BusinessRuleTask`),
we'll need to supply serializers for those task specs as well.
Strictly speaking, these are not serializers per se: they actually convert the tasks into dictionaries of
@ -138,7 +138,7 @@ two components:
- a data converter (which handles workflow and task data).
The default workflow spec converter likely to meet your needs, either on its own, or with the inclusion of
:code:`UserTask` and :code:`BusinessRuleTask` in the :code:`camnuda` or :code:`spiff` and :code:`dmn` subpackages
:code:`UserTask` and :code:`BusinessRuleTask` in the :code:`camnuda` or :code:`spiff` and :code:`dmn` subpackages
of this library, and all you'll need to do is add them to the list of task converters, as we did above.
However, he default data converter is very simple, adding only JSON-serializable conversions of :code:`datetime`
@ -180,7 +180,7 @@ If you have written any custom task specs, you'll need to implement task spec co
Task Spec converters are also based on the :code:`DictionaryConverter`. You should be able to use the
`BpmnTaskSpecConverter <https://github.com/sartography/SpiffWorkflow/blob/main/SpiffWorkflow/bpmn/serializer/bpmn_converters.py>`_
as a basis for your custom specs. It provides some methods for extracting attributes from Spiff base classes as well as
as a basis for your custom specs. It provides some methods for extracting attributes from Spiff base classes as well as
standard BPNN attributes from tasks that inherit from :code:`BMPNSpecMixin`.
The `Camunda User Task Converter <https://github.com/sartography/SpiffWorkflow/blob/main/SpiffWorkflow/camunda/serializer/task_spec_converters.py>`_
@ -221,7 +221,7 @@ serialize the workflow in the new format:
new_json = serializer.serialize_json(workflow)
However, if you use custom tasks or data serialization, you'll also need to specify workflow spec or data
serializers, as in the examples in the previous section, before you'll be able to serialize with the new serializer.
serializers, as in the examples in the previous section, before you'll be able to serialize with the new serializer.
The code would then look more like this:
.. code:: python
@ -244,7 +244,7 @@ The code would then look more like this:
new_json = serializer.serialize_json(workflow)
Because the serializer is highly customizable, we've made it possible for you to manage your own versions of the
serialization. You can do this by passing a version number into the serializer, which will be embedded in the
serialization. You can do this by passing a version number into the serializer, which will be embedded in the
json of all workflows. This allow you to modify the serialization and customize it over time, and still manage
the different forms as you make adjustments without leaving people behind.
@ -253,11 +253,11 @@ Versioned Serializer
As we make changes to Spiff, we may change the serialization format. For example, in 1.1.8, we changed
how subprocesses were handled interally in BPMN workflows and updated how they are serialized. If you have
not overridden our version number with one of your own, the serializer will transform the 1.0 format to the
not overridden our version number with one of your own, the serializer will transform the 1.0 format to the
new 1.1 format.
If you've overridden the serializer version, you may need to incorporate our serialization changes with
your own. You can find our conversions in
your own. You can find our conversions in
`version_migrations.py <https://github.com/sartography/SpiffWorkflow/blob/main/SpiffWorkflow/bpmn/serializer/version_migration.py>`_
Custom Script Engines
@ -277,14 +277,9 @@ We'll cover a simple extension of custom script engine here. There is also an e
a similar engine based on `RestrictedPython <https://restrictedpython.readthedocs.io/en/latest/>`_
included alongside this example.
The default script engine imports the following objects:
The default script engine does not import any objects.
- :code:`timedelta`
- :code:`datetime`
- :code:`dateparser`
- :code:`pytz`
You could add other functions or classes from the standard python modules or any code you've
You could add functions or classes from the standard python modules or any code you've
implemented yourself. Your global environment can be passed in using the `default_globals`
argument when initializing the script engine. In our RestrictedPython example, we use their
`safe_globals` which prevents users from executing some potentially unsafe operations.

View File

@ -3,6 +3,4 @@
celery==5.2.3
coverage
lxml
dateparser
pytz
.

View File

@ -22,7 +22,7 @@ setup(name='SpiffWorkflow',
license='lGPLv2',
packages=find_packages(exclude=['tests', 'tests.*']),
package_data={'SpiffWorkflow.bpmn.parser.schema': ['*.xsd']},
install_requires=['configparser', 'lxml', 'celery', 'dateparser', 'pytz',
install_requires=['configparser', 'lxml', 'celery',
# required for python 3.7 - https://stackoverflow.com/a/73932581
'importlib-metadata<5.0; python_version <= "3.7"'],
keywords='spiff workflow bpmn engine',

View File

@ -5,6 +5,7 @@ import datetime
import time
from SpiffWorkflow.task import TaskState
from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
from tests.SpiffWorkflow.bpmn.BpmnWorkflowTestCase import BpmnWorkflowTestCase
__author__ = 'kellym'
@ -15,11 +16,12 @@ class NITimerDurationTest(BpmnWorkflowTestCase):
Non-Interrupting Timer boundary test
"""
def setUp(self):
self.script_engine = PythonScriptEngine(default_globals={"timedelta": datetime.timedelta})
spec, subprocesses = self.load_workflow_spec('timer-non-interrupt-boundary.bpmn', 'NonInterruptTimer')
self.workflow = BpmnWorkflow(spec, subprocesses)
self.workflow = BpmnWorkflow(spec, subprocesses, script_engine=self.script_engine)
def load_spec(self):
return
return
def testRunThroughHappy(self):
self.actual_test(save_restore=False)
@ -28,7 +30,7 @@ class NITimerDurationTest(BpmnWorkflowTestCase):
self.actual_test(save_restore=True)
def actual_test(self,save_restore = False):
ready_tasks = self.workflow.get_tasks(TaskState.READY)
self.assertEqual(1, len(ready_tasks))
self.workflow.complete_task_from_id(ready_tasks[0].id)
@ -47,7 +49,9 @@ class NITimerDurationTest(BpmnWorkflowTestCase):
ready_tasks = self.workflow.get_tasks(TaskState.READY)
if len(ready_tasks) > 1:
break
if save_restore: self.save_restore()
if save_restore:
self.save_restore()
self.workflow.script_engine = self.script_engine
#self.assertEqual(1, len(self.workflow.get_tasks(Task.WAITING)))
time.sleep(0.1)
self.workflow.complete_task_from_id(ready_tasks[0].id)

View File

@ -24,32 +24,11 @@ class PythonScriptEngineTest(BpmnWorkflowTestCase):
workflow.do_engine_steps()
self.task = workflow.last_task
def testDateTimeExpressions(self):
"""Basically, assure that we can use datime, dateutils, and pytz"""
script = """
# Create Current Date as UTC
now_utc = datetime.datetime.now(datetime.timezone.utc)
# Create Current Date at EST
now_est = now_utc.astimezone(pytz.timezone('US/Eastern'))
# Format a date from a date String in UTC
datestr = "2021-09-23 16:11:00 -0000" # 12 pm EST, 4pm UTC
dt = dateparser.parse(datestr)
localtime = dt.astimezone(pytz.timezone('US/Eastern'))
localtime_str = localtime.strftime("%Y-%m-%d %H:%M:%S")
"""
self.expressionEngine.execute(self.task, script)
self.assertEqual(self.task.data['now_utc'].utcoffset().days, 0)
self.assertEqual(self.task.data['now_est'].tzinfo.zone, "US/Eastern")
self.assertEqual(self.task.data['localtime_str'], "2021-09-23 12:11:00")
self.assertTrue(True)
def testFunctionsAndGlobalsAreRemoved(self):
self.assertIn('testvar', self.task.data)
self.assertIn('testvar2', self.task.data)
self.assertIn('sample', self.task.data)
self.assertNotIn('my_function', self.task.data)
self.assertNotIn('datetime', self.task.data)
def suite():
return unittest.TestLoader().loadTestsFromTestCase(PythonScriptEngineTest)

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import datetime
import unittest
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
@ -8,6 +9,15 @@ from tests.SpiffWorkflow.bpmn.BpmnWorkflowTestCase import BpmnWorkflowTestCase
__author__ = 'sartography'
class CustomScriptEngine(PythonScriptEngine):
"""This is a custom script processor that can be easily injected into Spiff Workflow.
It will execute python code read in from the bpmn. It will also make any scripts in the
scripts directory available for execution. """
def __init__(self):
augment_methods = {
'timedelta': datetime.timedelta,
}
super().__init__(scripting_additions=augment_methods)
class TooManyLoopsTest(BpmnWorkflowTestCase):
@ -23,7 +33,7 @@ class TooManyLoopsTest(BpmnWorkflowTestCase):
def actual_test(self,save_restore = False):
spec, subprocesses = self.load_workflow_spec('too_many_loops*.bpmn', 'loops')
self.workflow = BpmnWorkflow(spec, subprocesses, script_engine=PythonScriptEngine())
self.workflow = BpmnWorkflow(spec, subprocesses, script_engine=CustomScriptEngine())
counter = 0
data = {}
while not self.workflow.is_completed():
@ -34,6 +44,7 @@ class TooManyLoopsTest(BpmnWorkflowTestCase):
counter += 1 # There is a 10 millisecond wait task.
if save_restore:
self.save_restore()
self.workflow.script_engine = CustomScriptEngine()
self.assertEqual(20, self.workflow.last_task.data['counter'])
def test_with_sub_process(self):
@ -41,7 +52,7 @@ class TooManyLoopsTest(BpmnWorkflowTestCase):
# right after a sub-process. So assuring this is fixed.
counter = 0
spec, subprocesses = self.load_workflow_spec('too_many_loops_sub_process.bpmn', 'loops_sub')
self.workflow = BpmnWorkflow(spec, subprocesses, script_engine=PythonScriptEngine())
self.workflow = BpmnWorkflow(spec, subprocesses, script_engine=CustomScriptEngine())
data = {}
while not self.workflow.is_completed():
self.workflow.do_engine_steps()
@ -57,7 +68,7 @@ class TooManyLoopsTest(BpmnWorkflowTestCase):
def test_with_two_call_activities(self):
spec, subprocess = self.load_workflow_spec('sub_in_loop*.bpmn', 'main')
self.workflow = BpmnWorkflow(spec, subprocess)
self.workflow = BpmnWorkflow(spec, subprocess, script_engine=CustomScriptEngine())
self.workflow.do_engine_steps()
for loop in range(3):
ready = self.workflow.get_ready_user_tasks()
@ -66,6 +77,7 @@ class TooManyLoopsTest(BpmnWorkflowTestCase):
self.workflow.refresh_waiting_tasks()
self.workflow.do_engine_steps()
self.save_restore()
self.workflow.script_engine = CustomScriptEngine()
def suite():
return unittest.TestLoader().loadTestsFromTestCase(TooManyLoopsTest)

View File

@ -7,8 +7,8 @@
<bpmn:scriptTask id="Activity_1q1wged" name="Set Future Date">
<bpmn:incoming>Flow_1i73q45</bpmn:incoming>
<bpmn:outgoing>Flow_00e79cz</bpmn:outgoing>
<bpmn:script>futuredate = dateparser.parse('in 1 second') - timedelta(seconds=.95)
futuredate2 = dateparser.parse('September 1 2021 at 10am EDT')</bpmn:script>
<bpmn:script>futuredate = datetime.now() + timedelta(0, 1) - timedelta(seconds=.95)
futuredate2 = datetime.strptime('2021-09-01 10:00','%Y-%m-%d %H:%M')</bpmn:script>
</bpmn:scriptTask>
<bpmn:sequenceFlow id="Flow_1i73q45" sourceRef="Event_0u1rmur" targetRef="Activity_1q1wged" />
<bpmn:sequenceFlow id="Flow_00e79cz" sourceRef="Activity_1q1wged" targetRef="Event_0eb0w95" />

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import datetime
import unittest
import time
@ -21,7 +22,10 @@ class CustomScriptEngine(PythonScriptEngine):
It will execute python code read in from the bpmn. It will also make any scripts in the
scripts directory available for execution. """
def __init__(self):
augment_methods = {'custom_function': my_custom_function}
augment_methods = {
'custom_function': my_custom_function,
'timedelta': datetime.timedelta,
}
super().__init__(scripting_additions=augment_methods)

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import datetime
import unittest
import time
@ -21,7 +22,10 @@ class CustomScriptEngine(PythonScriptEngine):
It will execute python code read in from the bpmn. It will also make any scripts in the
scripts directory available for execution. """
def __init__(self):
augment_methods = {'custom_function': my_custom_function}
augment_methods = {
'custom_function': my_custom_function,
'timedelta': datetime.timedelta,
}
super().__init__(scripting_additions=augment_methods)

View File

@ -3,10 +3,10 @@
import unittest
import datetime
import time
import pytz
from SpiffWorkflow.task import TaskState
from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
from tests.SpiffWorkflow.bpmn.BpmnWorkflowTestCase import BpmnWorkflowTestCase
__author__ = 'kellym'
@ -15,8 +15,12 @@ __author__ = 'kellym'
class TimerDateTest(BpmnWorkflowTestCase):
def setUp(self):
self.script_engine = PythonScriptEngine(default_globals={
"datetime": datetime.datetime,
"timedelta": datetime.timedelta,
})
self.spec, self.subprocesses = self.load_workflow_spec('timer-date-start.bpmn', 'date_timer')
self.workflow = BpmnWorkflow(self.spec, self.subprocesses)
self.workflow = BpmnWorkflow(self.spec, self.subprocesses, script_engine=self.script_engine)
def testRunThroughHappy(self):
self.actual_test(save_restore=False)
@ -42,6 +46,7 @@ class TimerDateTest(BpmnWorkflowTestCase):
break
if save_restore:
self.save_restore()
self.workflow.script_engine = self.script_engine
waiting_tasks = self.workflow.get_tasks(TaskState.WAITING)
@ -50,8 +55,7 @@ class TimerDateTest(BpmnWorkflowTestCase):
loopcount = loopcount +1
endtime = datetime.datetime.now()
self.workflow.do_engine_steps()
tz = pytz.timezone('US/Eastern')
testdate = tz.localize(datetime.datetime.strptime('2021-09-01 10:00','%Y-%m-%d %H:%M'))
testdate = datetime.datetime.strptime('2021-09-01 10:00','%Y-%m-%d %H:%M')
self.assertEqual(self.workflow.last_task.data['futuredate2'],testdate)
self.assertTrue('completed' in self.workflow.last_task.data)
self.assertTrue(self.workflow.last_task.data['completed'])

View File

@ -3,10 +3,12 @@
import unittest
import datetime
import time
from datetime import timedelta
from SpiffWorkflow.bpmn.specs.events import EndEvent
from SpiffWorkflow.task import TaskState
from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
from tests.SpiffWorkflow.bpmn.BpmnWorkflowTestCase import BpmnWorkflowTestCase
__author__ = 'kellym'
@ -14,8 +16,9 @@ __author__ = 'kellym'
class TimerDurationTest(BpmnWorkflowTestCase):
def setUp(self):
self.script_engine = PythonScriptEngine(default_globals={"timedelta": timedelta})
self.spec, self.subprocesses = self.load_workflow_spec('boundary_timer_on_task.bpmn', 'test_timer')
self.workflow = BpmnWorkflow(self.spec, self.subprocesses)
self.workflow = BpmnWorkflow(self.spec, self.subprocesses, script_engine=self.script_engine)
def testRunThroughHappy(self):
self.actual_test(save_restore=False)
@ -43,9 +46,11 @@ class TimerDurationTest(BpmnWorkflowTestCase):
starttime = datetime.datetime.now()
self.workflow = BpmnWorkflow(self.spec)
self.workflow.script_engine = self.script_engine
self.workflow.do_engine_steps()
if save_restore:
self.save_restore()
self.workflow.script_engine = self.script_engine
time.sleep(0.1)
self.workflow.refresh_waiting_tasks()
self.workflow.do_engine_steps()

View File

@ -3,8 +3,10 @@
import unittest
import datetime
import time
from datetime import timedelta
from SpiffWorkflow.task import TaskState
from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
from tests.SpiffWorkflow.bpmn.BpmnWorkflowTestCase import BpmnWorkflowTestCase
__author__ = 'kellym'
@ -13,8 +15,9 @@ __author__ = 'kellym'
class TimerDurationTest(BpmnWorkflowTestCase):
def setUp(self):
self.script_engine = PythonScriptEngine(default_globals={"timedelta": timedelta})
self.spec, self.subprocesses = self.load_workflow_spec('timer.bpmn', 'timer')
self.workflow = BpmnWorkflow(self.spec, self.subprocesses)
self.workflow = BpmnWorkflow(self.spec, self.subprocesses, script_engine=self.script_engine)
def testRunThroughHappy(self):
self.actual_test(save_restore=False)
@ -40,7 +43,9 @@ class TimerDurationTest(BpmnWorkflowTestCase):
while loopcount < 10:
if len(self.workflow.get_tasks(TaskState.READY)) >= 1:
break
if save_restore: self.save_restore()
if save_restore:
self.save_restore()
self.workflow.script_engine = self.script_engine
self.assertEqual(1, len(self.workflow.get_tasks(TaskState.WAITING)))
time.sleep(0.1)
self.workflow.refresh_waiting_tasks()

View File

@ -24,7 +24,7 @@ class ExternalMessageBoundaryTest(BaseTestCase):
def actual_test(self,save_restore = False):
self.workflow.do_engine_steps()
ready_tasks = self.workflow.get_tasks(TaskState.READY)
self.assertEqual(1, len(ready_tasks),'Expected to have only one ready task')

View File

@ -3,8 +3,10 @@
import unittest
import time
from datetime import timedelta
from SpiffWorkflow.task import TaskState
from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
from .BaseTestCase import BaseTestCase
__author__ = 'kellym'
@ -13,8 +15,9 @@ __author__ = 'kellym'
class MessageBoundaryTest(BaseTestCase):
def setUp(self):
self.script_engine = PythonScriptEngine(default_globals={"timedelta": timedelta})
self.spec, self.subprocesses = self.load_workflow_spec('MessageBoundary.bpmn', 'Process_1kjyavs')
self.workflow = BpmnWorkflow(self.spec, self.subprocesses)
self.workflow = BpmnWorkflow(self.spec, self.subprocesses, script_engine=self.script_engine)
def testRunThroughHappy(self):
self.actual_test(save_restore=False)
@ -41,7 +44,9 @@ class MessageBoundaryTest(BaseTestCase):
self.workflow.do_engine_steps()
time.sleep(.01)
self.workflow.refresh_waiting_tasks()
if save_restore: self.save_restore()
if save_restore:
self.save_restore()
self.workflow.script_engine = self.script_engine
ready_tasks = self.workflow.get_tasks(TaskState.READY)
time.sleep(.01)
self.workflow.refresh_waiting_tasks()

View File

@ -1,3 +1,4 @@
import datetime
from decimal import Decimal
from SpiffWorkflow.bpmn.PythonScriptEngine import PythonScriptEngine
@ -7,4 +8,5 @@ from ..DecisionRunner import DecisionRunner
class PythonDecisionRunner(DecisionRunner):
def __init__(self, filename):
super().__init__(PythonScriptEngine(scripting_additions={'Decimal': Decimal}), filename, 'python_engine')
scripting_additions={'Decimal': Decimal, 'datetime': datetime}
super().__init__(PythonScriptEngine(scripting_additions=scripting_additions), filename, 'python_engine')