status-im
/
spiff-arena
mirror of https://github.com/status-im/spiff-arena.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into feature/fix_process_instance_rewind

This commit is contained in:
jasquat 2023-03-30 07:51:05 -04:00
parent d08ce856ec 2feaf68132
commit fa4d19916b
No known key found for this signature in database
19 changed files with 1214 additions and 3774 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.github/workflows/backend_tests.yml vendored
View File

@ -16,10 +16,7 @@ jobs:
fail-fast: false
matrix:
include:
# FIXME: https://github.com/mysql/mysql-connector-python/pull/86
# put back when poetry update protobuf mysql-connector-python updates protobuf
# right now mysql is forcing protobuf to version 3
# - { python: "3.11", os: "ubuntu-latest", session: "safety" }
- { python: "3.11", os: "ubuntu-latest", session: "safety" }
- { python: "3.11", os: "ubuntu-latest", session: "mypy" }
- { python: "3.10", os: "ubuntu-latest", session: "mypy" }
- { python: "3.9", os: "ubuntu-latest", session: "mypy" }
@ -176,6 +173,19 @@ jobs:
name: logs-${{matrix.python}}-${{matrix.os}}-${{matrix.database}}
path: "./log/*.log"
# burnettk created an account at https://app.snyk.io/org/kevin-jfx
# and added his SNYK_TOKEN secret under the spiff-arena repo.
snyk:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: Run Snyk to check for vulnerabilities
uses: snyk/actions/python@master
with:
args: spiffworkflow-backend
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
run_pre_commit_checks:
runs-on: ubuntu-latest
defaults:

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
pyrightconfig.json
.idea/
t
.dccache

2831
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

125
pyproject.toml
View File

@ -13,71 +13,8 @@ classifiers = [
[tool.poetry.dependencies]
python = ">=3.11,<3.12"
click = "^8.0.1"
flask = "2.2.2"
flask-admin = "*"
flask-bcrypt = "*"
flask-cors = "*"
flask-mail = "*"
flask-marshmallow = "*"
flask-migrate = "*"
flask-restful = "*"
werkzeug = "*"
# go back to main once https://github.com/sartography/SpiffWorkflow/pull/241 is merged
SpiffWorkflow = {git = "https://github.com/sartography/SpiffWorkflow", rev = "main"}
# SpiffWorkflow = {develop = true, path = "/Users/kevin/projects/github/sartography/SpiffWorkflow"}
# SpiffWorkflow = {develop = true, path = "/home/jason/projects/github/sartography/SpiffWorkflow"}
sentry-sdk = "^1.10"
sphinx-autoapi = "^2.0"
# flask-bpmn = {develop = true, path = "/home/jason/projects/github/sartography/flask-bpmn"}
# flask-bpmn = {develop = true, path = "/Users/kevin/projects/github/sartography/flask-bpmn"}
flask-bpmn = {git = "https://github.com/sartography/flask-bpmn", rev = "main"}
mysql-connector-python = "^8.0.29"
pytest-flask = "^1.2.0"
pytest-flask-sqlalchemy = "^1.1.0"
psycopg2 = "^2.9.3"
typing-extensions = "^4.4.0"
connexion = {extras = [ "swagger-ui",], version = "^2"}
lxml = "^4.9.1"
marshmallow-enum = "^1.5.1"
marshmallow-sqlalchemy = "^0.28.0"
PyJWT = "^2.6.0"
gunicorn = "^20.1.0"
python-keycloak = "^2.5.0"
APScheduler = "^3.9.1"
Jinja2 = "^3.1.2"
RestrictedPython = "^6.0"
Flask-SQLAlchemy = "^3"
# type hinting stuff
# these need to be in the normal (non dev-dependencies) section
# because if not then poetry export won't have them and nox -s mypy --pythons 3.10
# will fail
types-Werkzeug = "^1.0.9"
types-PyYAML = "^6.0.12"
types-Flask = "^1.1.6"
types-requests = "^2.28.6"
types-pytz = "^2022.1.1"
# https://github.com/dropbox/sqlalchemy-stubs/pull/251
# someday get off github
# sqlalchemy-stubs = "^0.4"
# sqlalchemy-stubs = { git = "https://github.com/dropbox/sqlalchemy-stubs.git", rev = "master" }
# sqlalchemy-stubs = {develop = true, path = "/Users/kevin/projects/github/sqlalchemy-stubs"}
# for now use my fork
sqlalchemy-stubs = { git = "https://github.com/burnettk/sqlalchemy-stubs.git", rev = "scoped-session-delete" }
simplejson = "^3.17.6"
[tool.poetry.dev-dependencies]
pytest = "^7.1.2"
coverage = {extras = ["toml"], version = "^6.1"}
safety = "^2.3.1"
mypy = ">=0.961"
typeguard = "^2.13.2"
xdoctest = {extras = ["colors"], version = "^1.0.1"}
sphinx = "^5.0.2"
sphinx-autobuild = ">=2021.3.14"
pre-commit = "^2.20.0"
flake8 = "^4.0.1"
black = ">=21.10b0"
@ -89,71 +26,9 @@ bandit = "1.7.2"
flake8-bugbear = "^22.10.25"
flake8-docstrings = "^1.6.0"
flake8-rst-docstrings = "^0.2.7"
# flask-sqlalchemy-stubs = "^0.2"
pep8-naming = "^0.13.2"
darglint = "^1.8.1"
reorder-python-imports = "^3.9.0"
pre-commit-hooks = "^4.0.1"
sphinx-click = "^4.3.0"
Pygments = "^2.10.0"
pyupgrade = "^3.1.0"
furo = ">=2021.11.12"
[tool.poetry.scripts]
spiffworkflow-backend = "spiffworkflow_backend.__main__:main"
[tool.poetry.group.dev.dependencies]
tomli = "^2.0.1"
[tool.pytest.ini_options]
# ignore deprecation warnings from various packages that we don't control
filterwarnings = [
# note the use of single quote below to denote "raw" strings in TOML
# kombu/utils/compat.py:82
'ignore:SelectableGroups dict interface is deprecated. Use select.',
# flask_marshmallow/__init__.py:34
# marshmallow_sqlalchemy/convert.py:17
'ignore:distutils Version classes are deprecated. Use packaging.version instead.',
# connexion/spec.py:50
'ignore:Passing a schema to Validator.iter_errors is deprecated and will be removed in a future release',
# connexion/decorators/validation.py:16
'ignore:Accessing jsonschema.draft4_format_checker is deprecated and will be removed in a future release.',
# connexion/apis/flask_api.py:236
"ignore:'_request_ctx_stack' is deprecated and will be removed in Flask 2.3",
"ignore:Setting 'json_encoder' on the app or a blueprint is deprecated and will be removed in Flask 2.3",
"ignore:'JSONEncoder' is deprecated and will be removed in Flask 2.3",
"ignore:'app.json_encoder' is deprecated and will be removed in Flask 2.3"
]
[tool.coverage.paths]
source = ["src", "*/site-packages"]
tests = ["tests", "*/tests"]
[tool.coverage.run]
branch = true
source = ["spiffworkflow_backend", "tests"]
[tool.coverage.report]
show_missing = true
fail_under = 50
[tool.mypy]
strict = true
disallow_any_generics = false
warn_unreachable = true
pretty = true
show_column_numbers = true
show_error_codes = true
show_error_context = true
plugins = "sqlmypy"
# We get 'error: Module has no attribute "set_context"' for sentry-sdk without this option
implicit_reexport = true
# allow for subdirs to NOT require __init__.py
namespace_packages = true
explicit_package_bases = false
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

11
spiffworkflow-backend/bin/recreate_db
View File

@ -44,6 +44,17 @@ if [[ "${1:-}" == "clean" ]]; then
# TODO: check to see if the db already exists and we can connect to it. also actually clean it up.
# start postgres in background with one db
if [[ "${SPIFFWORKFLOW_BACKEND_DATABASE_TYPE:-}" == "postgres" ]]; then
container_name="postgres-spiff"
container_regex="^postgres-spiff$"
if [[ -n "$(docker ps -qa -f name=$container_regex)" ]]; then
echo ":: Found postgres container - $container_name"
if [[ -n "$(docker ps -q -f name=$container_regex)" ]]; then
echo ":: Stopping running container - $container_name"
docker stop $container_name
fi
echo ":: Removing stopped container - $container_name"
docker rm $container_name
fi
if ! docker exec -it postgres-spiff psql -U spiffworkflow_backend spiffworkflow_backend_unit_testing -c "select 1"; then
docker run --name postgres-spiff -p 5432:5432 -e POSTGRES_PASSWORD=spiffworkflow_backend -e POSTGRES_USER=spiffworkflow_backend -e POSTGRES_DB=spiffworkflow_backend_unit_testing -d postgres
sleep 4 # classy

4
spiffworkflow-backend/conftest.py
View File

@ -45,8 +45,8 @@ def app() -> Flask:
def with_db_and_bpmn_file_cleanup() -> None:
"""Do it cleanly!"""
meta = db.metadata
db.session.execute(db.update(BpmnProcessModel, values={"top_level_process_id": None}))
db.session.execute(db.update(BpmnProcessModel, values={"direct_parent_process_id": None}))
db.session.execute(db.update(BpmnProcessModel).values(top_level_process_id=None))
db.session.execute(db.update(BpmnProcessModel).values(direct_parent_process_id=None))
for table in reversed(meta.sorted_tables):
db.session.execute(table.delete())

5
spiffworkflow-backend/keycloak/bin/start_keycloak
View File

@ -26,9 +26,10 @@ fi
# https://stackoverflow.com/a/60579344/6090676
container_name="keycloak"
if [[ -n "$(docker ps -qa -f name=$container_name)" ]]; then
container_regex="^keycloak$"
if [[ -n "$(docker ps -qa -f name=$container_regex)" ]]; then
echo ":: Found container - $container_name"
if [[ -n "$(docker ps -q -f name=$container_name)" ]]; then
if [[ -n "$(docker ps -q -f name=$container_regex)" ]]; then
echo ":: Stopping running container - $container_name"
docker stop $container_name
fi

1903
spiffworkflow-backend/poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

12
spiffworkflow-backend/pyproject.toml
View File

@ -39,10 +39,16 @@ pytest-flask = "^1.2.0"
pytest-flask-sqlalchemy = "^1.1.0"
psycopg2 = "^2.9.3"
typing-extensions = "^4.4.0"
# pinned to higher than 65.5.0 because of a vulnerability
# and to lower than 67 because i didn't feel like addressing
# new deprecation warnings. we don't need this library explicitly,
# but at one time it was pulled in by various libs we depend on.
setuptools = "^65.5.1"
connexion = {extras = [ "swagger-ui",], version = "^2"}
lxml = "^4.9.1"
marshmallow-enum = "^1.5.1"
marshmallow-sqlalchemy = "^0.28.0"
PyJWT = "^2.6.0"
gunicorn = "^20.1.0"
APScheduler = "*"
@ -75,7 +81,9 @@ flask-jwt-extended = "^4.4.4"
pylint = "^2.15.10"
flask-simple-crypt = "^0.3.3"
cryptography = "^39.0.2"
safety = "^2.3.5"
sqlalchemy = "^2.0.7"
marshmallow-sqlalchemy = "^0.29.0"
[tool.poetry.dev-dependencies]
pytest = "^7.1.2"

5
spiffworkflow-backend/src/spiffworkflow_backend/__init__.py
View File

@ -44,8 +44,9 @@ class MyJSONEncoder(DefaultJSONProvider):
return obj.serialized
elif isinstance(obj, sqlalchemy.engine.row.Row): # type: ignore
return_dict = {}
for row_key in obj.keys():
row_value = obj[row_key]
row_mapping = obj._mapping
for row_key in row_mapping.keys():
row_value = row_mapping[row_key]
if hasattr(row_value, "serialized"):
return_dict.update(row_value.serialized)
elif hasattr(row_value, "__dict__"):

3
spiffworkflow-backend/src/spiffworkflow_backend/helpers/db_helper.py
View File

@ -2,6 +2,7 @@
import time
import sqlalchemy
from sqlalchemy.sql import text
from spiffworkflow_backend.models.db import db
@ -9,7 +10,7 @@ from spiffworkflow_backend.models.db import db
def try_to_connect(start_time: float) -> None:
"""Try to connect."""
try:
db.first_or_404("select 1") # type: ignore
db.first_or_404(text("select 1")) # type: ignore
except sqlalchemy.exc.DatabaseError as exception:
if time.time() - start_time > 15:
raise exception

1
spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance.py
View File

@ -53,6 +53,7 @@ class ProcessInstanceModel(SpiffworkflowBaseDBModel):
"""ProcessInstanceModel."""
__tablename__ = "process_instance"
__allow_unmapped__ = True
id: int = db.Column(db.Integer, primary_key=True)
process_model_identifier: str = db.Column(db.String(255), nullable=False, index=True)
process_model_display_name: str = db.Column(db.String(255), nullable=False, index=True)

3
spiffworkflow-backend/src/spiffworkflow_backend/models/process_instance_report.py
View File

@ -8,7 +8,6 @@ from typing import Optional
from typing import TypedDict
from sqlalchemy import ForeignKey
from sqlalchemy.orm import deferred
from sqlalchemy.orm import relationship
from spiffworkflow_backend.exceptions.process_entity_not_found_error import (
@ -69,7 +68,7 @@ class ProcessInstanceReportModel(SpiffworkflowBaseDBModel):
id: int = db.Column(db.Integer, primary_key=True)
identifier: str = db.Column(db.String(50), nullable=False, index=True)
report_metadata: dict = deferred(db.Column(db.JSON)) # type: ignore
report_metadata: dict = db.Column(db.JSON)
created_by_id = db.Column(ForeignKey(UserModel.id), nullable=False, index=True) # type: ignore
created_by = relationship("UserModel")
created_at_in_seconds = db.Column(db.Integer)

1
spiffworkflow-backend/src/spiffworkflow_backend/models/task.py
View File

@ -47,6 +47,7 @@ class MultiInstanceType(enum.Enum):
@dataclass
class TaskModel(SpiffworkflowBaseDBModel):
__tablename__ = "task"
__allow_unmapped__ = True
id: int = db.Column(db.Integer, primary_key=True)
guid: str = db.Column(db.String(36), nullable=False, unique=True)
bpmn_process_id: int = db.Column(ForeignKey(BpmnProcessModel.id), nullable=False, index=True) # type: ignore

7
spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py
View File

@ -632,9 +632,10 @@ def process_instance_task_list(
status_code=400,
)
_parent_bpmn_processes, task_models_of_parent_bpmn_processes = (
TaskService.task_models_of_parent_bpmn_processes(to_task_model)
)
(
_parent_bpmn_processes,
task_models_of_parent_bpmn_processes,
) = TaskService.task_models_of_parent_bpmn_processes(to_task_model)
task_models_of_parent_bpmn_processes_guids = [p.guid for p in task_models_of_parent_bpmn_processes if p.guid]
task_model_query = task_model_query.filter(
or_(

24
spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py
View File

@ -43,11 +43,10 @@ from SpiffWorkflow.bpmn.specs.events.EndEvent import EndEvent # type: ignore
from SpiffWorkflow.bpmn.specs.events.StartEvent import StartEvent # type: ignore
from SpiffWorkflow.bpmn.specs.SubWorkflowTask import SubWorkflowTask # type: ignore
from SpiffWorkflow.bpmn.workflow import BpmnWorkflow # type: ignore
from SpiffWorkflow.dmn.parser.BpmnDmnParser import BpmnDmnParser # type: ignore
from SpiffWorkflow.dmn.serializer.task_spec import BusinessRuleTaskConverter # type: ignore
from SpiffWorkflow.exceptions import WorkflowException # type: ignore
from SpiffWorkflow.exceptions import WorkflowTaskException
from SpiffWorkflow.serializer.exceptions import MissingSpecError # type: ignore
from SpiffWorkflow.spiff.parser.process import SpiffBpmnParser # type: ignore
from SpiffWorkflow.spiff.serializer.config import SPIFF_SPEC_CONFIG # type: ignore
from SpiffWorkflow.task import Task as SpiffTask # type: ignore
from SpiffWorkflow.task import TaskState
@ -110,8 +109,6 @@ from spiffworkflow_backend.services.workflow_execution_service import (
WorkflowExecutionService,
)
SPIFF_SPEC_CONFIG["task_specs"].append(BusinessRuleTaskConverter)
# Sorry about all this crap. I wanted to move this thing to another file, but
# importing a bunch of types causes circular imports.
@ -1223,13 +1220,16 @@ class ProcessInstanceProcessor:
spiff_tasks_updated[task.id] = task
for updated_spiff_task in spiff_tasks_updated.values():
bpmn_process, task_model, new_task_models, new_json_data_dicts = (
TaskService.find_or_create_task_model_from_spiff_task(
updated_spiff_task,
self.process_instance_model,
self._serializer,
bpmn_definition_to_task_definitions_mappings=self.bpmn_definition_to_task_definitions_mappings,
)
(
bpmn_process,
task_model,
new_task_models,
new_json_data_dicts,
) = TaskService.find_or_create_task_model_from_spiff_task(
updated_spiff_task,
self.process_instance_model,
self._serializer,
bpmn_definition_to_task_definitions_mappings=self.bpmn_definition_to_task_definitions_mappings,
)
bpmn_process_to_use = bpmn_process or task_model.bpmn_process
bpmn_process_json_data = TaskService.update_task_data_on_bpmn_process(
@ -1439,7 +1439,7 @@ class ProcessInstanceProcessor:
@staticmethod
def update_spiff_parser_with_all_process_dependency_files(
parser: BpmnDmnParser,
parser: SpiffBpmnParser,
processed_identifiers: Optional[set[str]] = None,
) -> None:
"""Update_spiff_parser_with_all_process_dependency_files."""

9
spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_report_service.py
View File

@ -309,11 +309,14 @@ class ProcessInstanceReportService:
) -> list[dict]:
"""Add_metadata_columns_to_process_instance."""
results = []
for process_instance in process_instance_sqlalchemy_rows:
process_instance_dict = process_instance["ProcessInstanceModel"].serialized
for process_instance_row in process_instance_sqlalchemy_rows:
process_instance_mapping = process_instance_row._mapping
process_instance_dict = process_instance_row[0].serialized
for metadata_column in metadata_columns:
if metadata_column["accessor"] not in process_instance_dict:
process_instance_dict[metadata_column["accessor"]] = process_instance[metadata_column["accessor"]]
process_instance_dict[metadata_column["accessor"]] = process_instance_mapping[
metadata_column["accessor"]
]
results.append(process_instance_dict)
return results

3
spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_service.py
View File

@ -279,7 +279,8 @@ class ProcessInstanceService:
yield (identifier, list_value, list_index)
if isinstance(list_value, dict) and len(list_value) == 1:
for v in list_value.values():
yield (identifier, v, list_index)
if isinstance(v, str):
yield (identifier, v, list_index)
@classmethod
def file_data_models_for_data(

22
spiffworkflow-backend/src/spiffworkflow_backend/services/workflow_execution_service.py
View File

@ -1,4 +1,3 @@
import logging
import time
from typing import Callable
from typing import Optional
@ -149,13 +148,16 @@ class TaskModelSavingDelegate(EngineStepDelegate):
self.json_data_dicts[json_data_dict["hash"]] = json_data_dict
def _update_task_model_with_spiff_task(self, spiff_task: SpiffTask, task_failed: bool = False) -> TaskModel:
bpmn_process, task_model, new_task_models, new_json_data_dicts = (
TaskService.find_or_create_task_model_from_spiff_task(
spiff_task,
self.process_instance,
self.serializer,
bpmn_definition_to_task_definitions_mappings=self.bpmn_definition_to_task_definitions_mappings,
)
(
bpmn_process,
task_model,
new_task_models,
new_json_data_dicts,
) = TaskService.find_or_create_task_model_from_spiff_task(
spiff_task,
self.process_instance,
self.serializer,
bpmn_definition_to_task_definitions_mappings=self.bpmn_definition_to_task_definitions_mappings,
)
bpmn_process_json_data = TaskService.update_task_data_on_bpmn_process(
bpmn_process or task_model.bpmn_process, spiff_task.workflow.data
@ -317,10 +319,6 @@ class WorkflowExecutionService:
finally:
self.execution_strategy.save()
spiff_logger = logging.getLogger("spiff")
for handler in spiff_logger.handlers:
if hasattr(handler, "bulk_insert_logs"):
handler.bulk_insert_logs() # type: ignore
db.session.commit()
if save: