updating readme

This commit is contained in:
Dan 2022-02-11 09:47:49 -05:00
parent 3993297360
commit 1b993ea227
14 changed files with 183 additions and 206 deletions

View File

@ -95,9 +95,65 @@ and primary investigator to dhf8r - which is a user in the mock ldap service, an
fire up the interface.
### Configuration
1. `instance/config.py`: This will configure the application for your local instance, overriding the configuration
in config/default
This covers both local configurations for development, and production settings.
We will cover all the settings below, but perhaps the most important part of the configuration is setting
the location of your workflow specifications. If you start up without setting a path, then it will
use a few testing workflows as a default.
For CRConnect, there is a private repository that contains all the workflow specifications, so get this
checked out, then in a file called /instance/config.py add the following setting:
SYNC_FILE_ROOT = '/path/to/my/git/dir/crconnect-workflow-specs'
#### Local Configuration
`instance/config.py`: This will configure the application for your local instance, overriding the configuration
in config/default. Very handy for setting
#### Production Configuration
We use environment variables with identical names to the variables in the default configuration file to configure the
application for production deplyment in a docker container
#### Common Configuration Settings
While these can be set in instance/config.py or as environment settings in docker, I'll present this as how you would
define the Docker Container, as /config/default.py offers a good example of the former.
```yaml
cr-connect-backend-testing:
container_name: cr-connect-backend-testing
image: ghcr.io/sartography/cr-connect-workflow:master # We use GitHub's actions to publish images
volumes:
- /home/sartography/docker-volumes/testing:/var/lib/cr-connect/specs # where specs are located.
ports:
environment:
- PRODUCTION=true # Should be set to true if you aren't running locally for development.
- DEVELOPMENT=false # Should be the opposite of production.
- PB_ENABLED=true # Generally true, we should connect to Protocol Builder
- PREFERRED_URL_SCHEME=https # Generally you want to run on SSL, should be https
- SERVER_NAME=testing.crconnect.uvadcos.io # The url used to access this app.
- TOKEN_AUTH_SECRET_KEY=-0-0-0- TESTING SUPER SECURE -0-0-0- # Some random characters that seed our key gen.
- APPLICATION_ROOT=/api # Appended to SERVER_NAME, is the full path to this service
- ADMIN_UIDS=dhf8r,cah3us # A comma delimited list of people who can preform administrative tasks.
- CORS_ALLOW_ORIGINS=testing.crconnect.uvadcos.io,shibidp.its.virginia.edu,sp.uvadcos.io # CORS stuff
- FRONTEND=testing.crconnect.uvadcos.io # URL to reach the front end application.
- BPMN=testing.crconnect.uvadcos.io/bpmn # URL to reach the configuration interface.
- PB_BASE_URL=http://10.250.124.174:11022/pb/v2.0/ # URL for Protocol Builder
- UPGRADE_DB=true # Will run all migrations on startup if set to true. Generally a good idea for production.
- DB_USER=crc_user # Database user name
- DB_NAME=crc_testing # Database passwprd
- DB_HOST=10.250.124.186 # Domain/IP of database server.
- DB_PORT=15432 # Port of database server.
- DB_PASSWORD=XXXXX # Passwword for the database
- MAIL_PASSWORD=XXXX # Mail Password
- MAIL_USERNAME=XXXXX # Mail username
- LDAP_URL=privopenldap.its.virginia.edu # URL for the LDAP Server
- LDAP_PASS=XXXX # Password for the ldap server
- LDAP_USER=cn=crcconnect,ou=Special Users,o=University of Virginia,c=US #LDAP settings for search, likely these.
- SENTRY_ENVIRONMENT=testing.crconnect.uvadcos.io # Configuration for Sentry
- GITHUB_TOKEN=XXXX # A token for GitHub so we can push and pull changes.
- PORT0=11021 # The port on the server where this sytem should be avialable, could be 80, but we are behind a proxy.
- SYNC_FILE_ROOT=/var/lib/cr-connect/specs # This should be the same as the volumes above, a location where the specs are checked out in git.
```
### Project Initialization
1. Clone this repository.

View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="empty_workflow" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="EndEvent_0q4qzl9" />
<bpmn:endEvent id="EndEvent_0q4qzl9">
<bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
</bpmn:endEvent>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="empty_workflow">
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -1,12 +0,0 @@
{
"library": false,
"standalone": false,
"display_order": 2,
"id": "workflow_one",
"description": "Workflow One, is a workflow.",
"is_master_spec": false,
"libraries": [],
"display_name": "Workflow One",
"primary_file_name": "empty_workflow.bpmn",
"primary_process_id": "empty_workflow"
}

View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="empty_workflow" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="EndEvent_0q4qzl9" />
<bpmn:endEvent id="EndEvent_0q4qzl9">
<bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
</bpmn:endEvent>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="empty_workflow">
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -1,12 +0,0 @@
{
"library": false,
"standalone": false,
"display_order": 3,
"id": "workflow_three",
"description": "Workflow Three, is a workflow.",
"is_master_spec": false,
"libraries": [],
"display_name": "Workflow Three",
"primary_file_name": "empty_workflow.bpmn",
"primary_process_id": "empty_workflow"
}

View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="empty_workflow" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="EndEvent_0q4qzl9" />
<bpmn:endEvent id="EndEvent_0q4qzl9">
<bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
</bpmn:endEvent>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="empty_workflow">
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -1,12 +0,0 @@
{
"library": false,
"standalone": false,
"display_order": 3,
"id": "workflow_two",
"description": "Workflow Two, is a workflow.",
"is_master_spec": false,
"libraries": [],
"display_name": "Workflow Two",
"primary_file_name": "empty_workflow.bpmn",
"primary_process_id": "empty_workflow"
}

View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="empty_workflow_library" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="EndEvent_0q4qzl9" />
<bpmn:endEvent id="EndEvent_0q4qzl9">
<bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
</bpmn:endEvent>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="empty_workflow">
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="empty_workflow" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="EndEvent_0q4qzl9" />
<bpmn:endEvent id="EndEvent_0q4qzl9">
<bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
</bpmn:endEvent>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="empty_workflow">
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

View File

@ -1,12 +0,0 @@
{
"library": false,
"standalone": false,
"is_master_spec": true,
"display_name": "Top Level Workflow",
"description": "Determines the status of other workflows in a study",
"primary_file_name": "empty_workflow.bpmn",
"primary_process_id": "empty_workflow",
"libraries": [
{ "id": "library_one" }
]
}

View File

@ -1,23 +0,0 @@
{
"libraries": [
{
"id": "library_one",
"description": "The One Library",
"primary_file_name": "empty_workflow_library.bpmn",
"primary_process_id": "empty_workflow_library",
"display_name": "The One Library"
}
],
"categories": [
{
"admin": false,
"display_order": 0,
"display_name": "Category Number One",
},
{
"admin": false,
"display_order": 1,
"display_name": "Category Number Two",
}
]
}

View File

@ -97,4 +97,4 @@ MAIL_USERNAME = environ.get('MAIL_USERNAME', default='')
MAIL_PASSWORD = environ.get('MAIL_PASSWORD', default='')
# Local file path
SYNC_FILE_ROOT = './SPECS'
SYNC_FILE_ROOT = 'tests/data/IMPORT_TEST'

View File

@ -2,5 +2,121 @@
# loop over all the categories in the database
# assure we have a directory with the correct name
# assure it contains a valid json file called categories.json
import json
import os
import pathlib
import re
import shutil
categories = db.
from crc import db, app
from crc.models.file import FileModel
from crc.models.workflow import WorkflowSpecCategoryModel, WorkflowSpecCategoryModelSchema, WorkflowSpecModelSchema, \
WorkflowSpecModel
LIBRARY_SPECS = "Library Specs"
STAND_ALONE_SPECS = "Stand Alone"
MASTER_SPECIFICATION = "Master Specification"
REFERENCE_FILES = "Reference Files"
SPECIAL_FOLDERS = [LIBRARY_SPECS, MASTER_SPECIFICATION, REFERENCE_FILES]
CAT_JSON_FILE = "category.json"
WF_JSON_FILE = "workflow.json"
base_dir = '../SPECS'
app_root = app.root_path
path = os.path.join(app_root, '..', 'SPECS')
CAT_SCHEMA = WorkflowSpecCategoryModelSchema()
SPEC_SCHEMA = WorkflowSpecModelSchema()
def remove_all_json_files(path):
for json_file in pathlib.Path(path).glob('*.json'):
print("removing ", json_file)
os.remove(json_file)
def update_workflows_for_category(path, schemas, category_id):
for schema in schemas:
orig_path = os.path.join(path, schema.display_name)
new_path = os.path.join(path, schema.id)
if (os.path.exists(orig_path)):
os.rename(orig_path, new_path)
print(new_path)
update_spec(new_path, schema, category_id)
def update_spec(path, schema, category_id):
json_data = SPEC_SCHEMA.dump(schema)
remove_all_json_files(path)
# Fixup the libraries
lib_ids = list(map(lambda x: x['id'], json_data['libraries']))
# calculate the primary process id, and primary file name
file = db.session.query(FileModel).\
filter(FileModel.workflow_spec_id == schema.id).\
filter(FileModel.primary == True).first()
if(file):
json_data['primary_file_name'] = file.name
json_data['primary_process_id'] = file.primary_process_id
else:
print("This workflow doesn't have a primary process:", json_data)
json_data['category_id'] = category_id
json_data.pop('category', None)
json_data.pop('parents', None)
if json_data['library'] is None:
json_data['library'] = False
json_data['libraries'] = lib_ids
if not os.path.exists(path):
os.makedirs(path)
json_file_name = os.path.join(path, 'workflow.json')
with open(json_file_name, 'w') as wf_handle:
json.dump(json_data, wf_handle, indent=4)
# Clean up json files
remove_all_json_files(path)
# Clean up libraries
lib_path = os.path.join(path, LIBRARY_SPECS)
remove_all_json_files(lib_path)
workflows = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.library == True)
update_workflows_for_category(lib_path, workflows, "")
# Clean up reference Files
ref_path = os.path.join(path, REFERENCE_FILES)
old_ref_path = os.path.join(path,'Reference')
if os.path.exists(old_ref_path):
os.rename(old_ref_path, ref_path)
remove_all_json_files(ref_path)
# Clean up the master spec
tlw = os.path.join(path, MASTER_SPECIFICATION, "Top Level Workflow")
master_path = os.path.join(path, MASTER_SPECIFICATION)
if os.path.exists(tlw):
for src_file in pathlib.Path(tlw).glob('*.*'):
shutil.copy(src_file, master_path)
remove_all_json_files(master_path)
schema = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec == True).first()
update_spec(master_path, schema, "")
# Fix all the categories
categories = db.session.query(WorkflowSpecCategoryModel).all()
for category in categories:
json_data = CAT_SCHEMA.dump(category)
orig_path = os.path.join(path, category.display_name)
new_name = re.sub(r'[^A-Za-z0-9]', '_', category.display_name).lower()
new_path = os.path.join(path, new_name)
json_data['id'] = new_name
if (os.path.exists(orig_path)):
os.rename(orig_path, new_path)
remove_all_json_files(new_path)
json_file_name = os.path.join(new_path, 'category.json')
with open(json_file_name, 'w') as f_handle:
json.dump(json_data, f_handle, indent=4)
workflows = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == category.id)
update_workflows_for_category(new_path, workflows, new_name)

View File

@ -0,0 +1,6 @@
{
"admin": false,
"display_name": "Library Specs",
"id": "Library Specs",
"display_order": 10000
}