diff --git a/README.md b/README.md
index 945426c9..3ab6b22e 100644
--- a/README.md
+++ b/README.md
@@ -95,9 +95,65 @@ and primary investigator to dhf8r - which is a user in the mock ldap service, an
fire up the interface.
### Configuration
-1. `instance/config.py`: This will configure the application for your local instance, overriding the configuration
-in config/default
-
+This covers both local configurations for development, and production settings.
+We will cover all the settings below, but perhaps the most important part of the configuration is setting
+the location of your workflow specifications. If you start up without setting a path, then it will
+use a few testing workflows as a default.
+
+For CRConnect, there is a private repository that contains all the workflow specifications, so get this
+checked out, then in a file called /instance/config.py add the following setting:
+SYNC_FILE_ROOT = '/path/to/my/git/dir/crconnect-workflow-specs'
+
+#### Local Configuration
+`instance/config.py`: This will configure the application for your local instance, overriding the configuration
+in config/default. Very handy for setting
+
+#### Production Configuration
+We use environment variables with identical names to the variables in the default configuration file to configure the
+application for production deplyment in a docker container
+
+#### Common Configuration Settings
+While these can be set in instance/config.py or as environment settings in docker, I'll present this as how you would
+define the Docker Container, as /config/default.py offers a good example of the former.
+```yaml
+ cr-connect-backend-testing:
+ container_name: cr-connect-backend-testing
+ image: ghcr.io/sartography/cr-connect-workflow:master # We use GitHub's actions to publish images
+ volumes:
+ - /home/sartography/docker-volumes/testing:/var/lib/cr-connect/specs # where specs are located.
+ ports:
+ environment:
+ - PRODUCTION=true # Should be set to true if you aren't running locally for development.
+ - DEVELOPMENT=false # Should be the opposite of production.
+ - PB_ENABLED=true # Generally true, we should connect to Protocol Builder
+ - PREFERRED_URL_SCHEME=https # Generally you want to run on SSL, should be https
+ - SERVER_NAME=testing.crconnect.uvadcos.io # The url used to access this app.
+ - TOKEN_AUTH_SECRET_KEY=-0-0-0- TESTING SUPER SECURE -0-0-0- # Some random characters that seed our key gen.
+ - APPLICATION_ROOT=/api # Appended to SERVER_NAME, is the full path to this service
+ - ADMIN_UIDS=dhf8r,cah3us # A comma delimited list of people who can preform administrative tasks.
+ - CORS_ALLOW_ORIGINS=testing.crconnect.uvadcos.io,shibidp.its.virginia.edu,sp.uvadcos.io # CORS stuff
+ - FRONTEND=testing.crconnect.uvadcos.io # URL to reach the front end application.
+ - BPMN=testing.crconnect.uvadcos.io/bpmn # URL to reach the configuration interface.
+ - PB_BASE_URL=http://10.250.124.174:11022/pb/v2.0/ # URL for Protocol Builder
+ - UPGRADE_DB=true # Will run all migrations on startup if set to true. Generally a good idea for production.
+ - DB_USER=crc_user # Database user name
+ - DB_NAME=crc_testing # Database passwprd
+ - DB_HOST=10.250.124.186 # Domain/IP of database server.
+ - DB_PORT=15432 # Port of database server.
+ - DB_PASSWORD=XXXXX # Passwword for the database
+ - MAIL_PASSWORD=XXXX # Mail Password
+ - MAIL_USERNAME=XXXXX # Mail username
+ - LDAP_URL=privopenldap.its.virginia.edu # URL for the LDAP Server
+ - LDAP_PASS=XXXX # Password for the ldap server
+ - LDAP_USER=cn=crcconnect,ou=Special Users,o=University of Virginia,c=US #LDAP settings for search, likely these.
+ - SENTRY_ENVIRONMENT=testing.crconnect.uvadcos.io # Configuration for Sentry
+ - GITHUB_TOKEN=XXXX # A token for GitHub so we can push and pull changes.
+ - PORT0=11021 # The port on the server where this sytem should be avialable, could be 80, but we are behind a proxy.
+ - SYNC_FILE_ROOT=/var/lib/cr-connect/specs # This should be the same as the volumes above, a location where the specs are checked out in git.
+```
+
+
+
### Project Initialization
1. Clone this repository.
diff --git a/config/IMPORT_TEST/Category Number One/Workflow One/empty_workflow.bpmn b/config/IMPORT_TEST/Category Number One/Workflow One/empty_workflow.bpmn
deleted file mode 100644
index 17b8351c..00000000
--- a/config/IMPORT_TEST/Category Number One/Workflow One/empty_workflow.bpmn
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
- SequenceFlow_0lvudp8
-
-
-
- SequenceFlow_0lvudp8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/config/IMPORT_TEST/Category Number One/Workflow One/workflow.json b/config/IMPORT_TEST/Category Number One/Workflow One/workflow.json
deleted file mode 100644
index 19df2877..00000000
--- a/config/IMPORT_TEST/Category Number One/Workflow One/workflow.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "library": false,
- "standalone": false,
- "display_order": 2,
- "id": "workflow_one",
- "description": "Workflow One, is a workflow.",
- "is_master_spec": false,
- "libraries": [],
- "display_name": "Workflow One",
- "primary_file_name": "empty_workflow.bpmn",
- "primary_process_id": "empty_workflow"
-}
diff --git a/config/IMPORT_TEST/Category Number Two/Workflow Three/empty_workflow.bpmn b/config/IMPORT_TEST/Category Number Two/Workflow Three/empty_workflow.bpmn
deleted file mode 100644
index 17b8351c..00000000
--- a/config/IMPORT_TEST/Category Number Two/Workflow Three/empty_workflow.bpmn
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
- SequenceFlow_0lvudp8
-
-
-
- SequenceFlow_0lvudp8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/config/IMPORT_TEST/Category Number Two/Workflow Three/workflow.json b/config/IMPORT_TEST/Category Number Two/Workflow Three/workflow.json
deleted file mode 100644
index 8554038d..00000000
--- a/config/IMPORT_TEST/Category Number Two/Workflow Three/workflow.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "library": false,
- "standalone": false,
- "display_order": 3,
- "id": "workflow_three",
- "description": "Workflow Three, is a workflow.",
- "is_master_spec": false,
- "libraries": [],
- "display_name": "Workflow Three",
- "primary_file_name": "empty_workflow.bpmn",
- "primary_process_id": "empty_workflow"
-}
diff --git a/config/IMPORT_TEST/Category Number Two/Workflow Two/empty_workflow.bpmn b/config/IMPORT_TEST/Category Number Two/Workflow Two/empty_workflow.bpmn
deleted file mode 100644
index 17b8351c..00000000
--- a/config/IMPORT_TEST/Category Number Two/Workflow Two/empty_workflow.bpmn
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
- SequenceFlow_0lvudp8
-
-
-
- SequenceFlow_0lvudp8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/config/IMPORT_TEST/Category Number Two/Workflow Two/workflow.json b/config/IMPORT_TEST/Category Number Two/Workflow Two/workflow.json
deleted file mode 100644
index fa39ac80..00000000
--- a/config/IMPORT_TEST/Category Number Two/Workflow Two/workflow.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "library": false,
- "standalone": false,
- "display_order": 3,
- "id": "workflow_two",
- "description": "Workflow Two, is a workflow.",
- "is_master_spec": false,
- "libraries": [],
- "display_name": "Workflow Two",
- "primary_file_name": "empty_workflow.bpmn",
- "primary_process_id": "empty_workflow"
-}
diff --git a/config/IMPORT_TEST/Library Specs/Library One/empty_workflow_library.bpmn b/config/IMPORT_TEST/Library Specs/Library One/empty_workflow_library.bpmn
deleted file mode 100644
index cefff969..00000000
--- a/config/IMPORT_TEST/Library Specs/Library One/empty_workflow_library.bpmn
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
- SequenceFlow_0lvudp8
-
-
-
- SequenceFlow_0lvudp8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/config/IMPORT_TEST/Master Specification/Top Level Workflow/empty_workflow.bpmn b/config/IMPORT_TEST/Master Specification/Top Level Workflow/empty_workflow.bpmn
deleted file mode 100644
index 17b8351c..00000000
--- a/config/IMPORT_TEST/Master Specification/Top Level Workflow/empty_workflow.bpmn
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
- SequenceFlow_0lvudp8
-
-
-
- SequenceFlow_0lvudp8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/config/IMPORT_TEST/Master Specification/Top Level Workflow/workflow.json b/config/IMPORT_TEST/Master Specification/Top Level Workflow/workflow.json
deleted file mode 100644
index b0558f3a..00000000
--- a/config/IMPORT_TEST/Master Specification/Top Level Workflow/workflow.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "library": false,
- "standalone": false,
- "is_master_spec": true,
- "display_name": "Top Level Workflow",
- "description": "Determines the status of other workflows in a study",
- "primary_file_name": "empty_workflow.bpmn",
- "primary_process_id": "empty_workflow",
- "libraries": [
- { "id": "library_one" }
- ]
-}
diff --git a/config/IMPORT_TEST/categories.json b/config/IMPORT_TEST/categories.json
deleted file mode 100644
index b4ad0676..00000000
--- a/config/IMPORT_TEST/categories.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "libraries": [
- {
- "id": "library_one",
- "description": "The One Library",
- "primary_file_name": "empty_workflow_library.bpmn",
- "primary_process_id": "empty_workflow_library",
- "display_name": "The One Library"
- }
- ],
- "categories": [
- {
- "admin": false,
- "display_order": 0,
- "display_name": "Category Number One",
- },
- {
- "admin": false,
- "display_order": 1,
- "display_name": "Category Number Two",
- }
- ]
-}
\ No newline at end of file
diff --git a/config/default.py b/config/default.py
index 8b168a04..a0ccb25a 100644
--- a/config/default.py
+++ b/config/default.py
@@ -97,4 +97,4 @@ MAIL_USERNAME = environ.get('MAIL_USERNAME', default='')
MAIL_PASSWORD = environ.get('MAIL_PASSWORD', default='')
# Local file path
-SYNC_FILE_ROOT = './SPECS'
\ No newline at end of file
+SYNC_FILE_ROOT = 'tests/data/IMPORT_TEST'
\ No newline at end of file
diff --git a/crc/services/update_service.py b/crc/services/update_service.py
index e7424433..9a791ee7 100644
--- a/crc/services/update_service.py
+++ b/crc/services/update_service.py
@@ -2,5 +2,121 @@
# loop over all the categories in the database
# assure we have a directory with the correct name
# assure it contains a valid json file called categories.json
+import json
+import os
+import pathlib
+import re
+import shutil
-categories = db.
\ No newline at end of file
+from crc import db, app
+from crc.models.file import FileModel
+from crc.models.workflow import WorkflowSpecCategoryModel, WorkflowSpecCategoryModelSchema, WorkflowSpecModelSchema, \
+ WorkflowSpecModel
+
+LIBRARY_SPECS = "Library Specs"
+STAND_ALONE_SPECS = "Stand Alone"
+MASTER_SPECIFICATION = "Master Specification"
+REFERENCE_FILES = "Reference Files"
+SPECIAL_FOLDERS = [LIBRARY_SPECS, MASTER_SPECIFICATION, REFERENCE_FILES]
+CAT_JSON_FILE = "category.json"
+WF_JSON_FILE = "workflow.json"
+
+
+base_dir = '../SPECS'
+app_root = app.root_path
+path = os.path.join(app_root, '..', 'SPECS')
+CAT_SCHEMA = WorkflowSpecCategoryModelSchema()
+SPEC_SCHEMA = WorkflowSpecModelSchema()
+
+
+def remove_all_json_files(path):
+ for json_file in pathlib.Path(path).glob('*.json'):
+ print("removing ", json_file)
+ os.remove(json_file)
+
+
+def update_workflows_for_category(path, schemas, category_id):
+ for schema in schemas:
+ orig_path = os.path.join(path, schema.display_name)
+ new_path = os.path.join(path, schema.id)
+ if (os.path.exists(orig_path)):
+ os.rename(orig_path, new_path)
+ print(new_path)
+ update_spec(new_path, schema, category_id)
+
+
+def update_spec(path, schema, category_id):
+ json_data = SPEC_SCHEMA.dump(schema)
+ remove_all_json_files(path)
+
+ # Fixup the libraries
+ lib_ids = list(map(lambda x: x['id'], json_data['libraries']))
+
+ # calculate the primary process id, and primary file name
+ file = db.session.query(FileModel).\
+ filter(FileModel.workflow_spec_id == schema.id).\
+ filter(FileModel.primary == True).first()
+ if(file):
+ json_data['primary_file_name'] = file.name
+ json_data['primary_process_id'] = file.primary_process_id
+ else:
+ print("This workflow doesn't have a primary process:", json_data)
+
+ json_data['category_id'] = category_id
+ json_data.pop('category', None)
+ json_data.pop('parents', None)
+ if json_data['library'] is None:
+ json_data['library'] = False
+
+ json_data['libraries'] = lib_ids
+ if not os.path.exists(path):
+ os.makedirs(path)
+ json_file_name = os.path.join(path, 'workflow.json')
+ with open(json_file_name, 'w') as wf_handle:
+ json.dump(json_data, wf_handle, indent=4)
+
+# Clean up json files
+remove_all_json_files(path)
+
+# Clean up libraries
+lib_path = os.path.join(path, LIBRARY_SPECS)
+remove_all_json_files(lib_path)
+workflows = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.library == True)
+update_workflows_for_category(lib_path, workflows, "")
+
+# Clean up reference Files
+ref_path = os.path.join(path, REFERENCE_FILES)
+old_ref_path = os.path.join(path,'Reference')
+if os.path.exists(old_ref_path):
+ os.rename(old_ref_path, ref_path)
+remove_all_json_files(ref_path)
+
+# Clean up the master spec
+tlw = os.path.join(path, MASTER_SPECIFICATION, "Top Level Workflow")
+master_path = os.path.join(path, MASTER_SPECIFICATION)
+if os.path.exists(tlw):
+ for src_file in pathlib.Path(tlw).glob('*.*'):
+ shutil.copy(src_file, master_path)
+remove_all_json_files(master_path)
+schema = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.is_master_spec == True).first()
+update_spec(master_path, schema, "")
+
+
+# Fix all the categories
+categories = db.session.query(WorkflowSpecCategoryModel).all()
+for category in categories:
+ json_data = CAT_SCHEMA.dump(category)
+ orig_path = os.path.join(path, category.display_name)
+ new_name = re.sub(r'[^A-Za-z0-9]', '_', category.display_name).lower()
+ new_path = os.path.join(path, new_name)
+ json_data['id'] = new_name
+ if (os.path.exists(orig_path)):
+ os.rename(orig_path, new_path)
+
+ remove_all_json_files(new_path)
+ json_file_name = os.path.join(new_path, 'category.json')
+ with open(json_file_name, 'w') as f_handle:
+ json.dump(json_data, f_handle, indent=4)
+
+ workflows = db.session.query(WorkflowSpecModel).filter(WorkflowSpecModel.category_id == category.id)
+ update_workflows_for_category(new_path, workflows, new_name)
\ No newline at end of file
diff --git a/tests/data/IMPORT_TEST/Library Specs/category.json b/tests/data/IMPORT_TEST/Library Specs/category.json
new file mode 100644
index 00000000..7d09ddaf
--- /dev/null
+++ b/tests/data/IMPORT_TEST/Library Specs/category.json
@@ -0,0 +1,6 @@
+{
+ "admin": false,
+ "display_name": "Library Specs",
+ "id": "Library Specs",
+ "display_order": 10000
+}
\ No newline at end of file