sartography
/
cr-connect-workflow
mirror of https://github.com/sartography/cr-connect-workflow.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ldap lookup. 

Refactored calls into a new lookup_service to keep things tidy.

New keys for all enum/auto-complete fields:
    PROP_OPTIONS_FILE = "spreadsheet.name"
    PROP_OPTIONS_VALUE_COLUMN = "spreadsheet.value.column"
    PROP_OPTIONS_LABEL_COL = "spreadsheet.label.column"
    PROP_LDAP_LOOKUP = "ldap.lookup"
    FIELD_TYPE_AUTO_COMPLETE = "autocomplete"
This commit is contained in:
Dan Funk 2020-05-19 16:11:43 -04:00
parent dff92c6e4c
commit 951710d762
13 changed files with 360 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
crc/api/workflow.py
View File

@ -7,6 +7,7 @@ from crc.models.file import FileModel, LookupDataSchema
from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel, WorkflowSpecCategoryModel, \
WorkflowSpecCategoryModelSchema
from crc.services.file_service import FileService
from crc.services.lookup_service import LookupService
from crc.services.study_service import StudyService
from crc.services.workflow_processor import WorkflowProcessor
from crc.services.workflow_service import WorkflowService
@ -217,9 +218,9 @@ def delete_workflow_spec_category(cat_id):
def lookup(workflow_id, task_id, field_id, query, limit):
"""
given a field in a task, attempts to find the lookup table associated with that field
and runs a full-text query against it to locate the values and labels that would be
returned to a type-ahead box.
given a field in a task, attempts to find the lookup table or function associated
with that field and runs a full-text query against it to locate the values and
labels that would be returned to a type-ahead box.
"""
workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
if not workflow_model:
@ -236,6 +237,5 @@ def lookup(workflow_id, task_id, field_id, query, limit):
if not field:
raise ApiError("unknown_field", "No field named %s in task %s" % (task_id, spiff_task.task_spec.name))
lookup_table = WorkflowService.get_lookup_table(spiff_task, field)
lookup_data = WorkflowService.run_lookup_query(lookup_table, query, limit)
lookup_data = LookupService.lookup(spiff_task, field, query, limit)
return LookupDataSchema(many=True).dump(lookup_data)

10
crc/models/api_models.py
View File

@ -31,10 +31,12 @@ class NavigationItem(object):
class Task(object):
ENUM_OPTIONS_FILE_PROP = "enum.options.file"
EMUM_OPTIONS_VALUE_COL_PROP = "enum.options.value.column"
EMUM_OPTIONS_LABEL_COL_PROP = "enum.options.label.column"
EMUM_OPTIONS_AS_LOOKUP = "enum.options.lookup"
PROP_OPTIONS_FILE = "spreadsheet.name"
PROP_OPTIONS_VALUE_COLUMN = "spreadsheet.value.column"
PROP_OPTIONS_LABEL_COL = "spreadsheet.label.column"
PROP_LDAP_LOOKUP = "ldap.lookup"
FIELD_TYPE_AUTO_COMPLETE = "autocomplete"
def __init__(self, id, name, title, type, state, form, documentation, data,
multi_instance_type, multi_instance_count, multi_instance_index, process_name, properties):

27
crc/services/ldap_service.py
View File

@ -12,20 +12,19 @@ class LdapUserInfo(object):
self.display_name = entry.displayName.value
self.given_name = ", ".join(entry.givenName)
self.email = entry.mail.value
self.telephone_number= ", ".join(entry.telephoneNumber)
self.telephone_number = ", ".join(entry.telephoneNumber)
self.title = ", ".join(entry.title)
self.department = ", ".join(entry.uvaDisplayDepartment)
self.affiliation = ", ".join(entry.uvaPersonIAMAffiliation)
self.sponsor_type = ", ".join(entry.uvaPersonSponsoredType)
self.uid = entry.uid.value
class LdapService(object):
search_base = "ou=People,o=University of Virginia,c=US"
attributes = ['cn', 'displayName', 'givenName', 'mail', 'objectClass', 'UvaDisplayDepartment',
attributes = ['uid', 'cn', 'displayName', 'givenName', 'mail', 'objectClass', 'UvaDisplayDepartment',
'telephoneNumber', 'title', 'uvaPersonIAMAffiliation', 'uvaPersonSponsoredType']
search_string = "(&(objectclass=person)(uid=%s))"
uid_search_string = "(&(objectclass=person)(uid=%s))"
def __init__(self):
if app.config['TESTING']:
@ -46,9 +45,25 @@ class LdapService(object):
self.conn.unbind()
def user_info(self, uva_uid):
search_string = LdapService.search_string % uva_uid
search_string = LdapService.uid_search_string % uva_uid
self.conn.search(LdapService.search_base, search_string, attributes=LdapService.attributes)
if len(self.conn.entries) < 1:
raise ApiError("missing_ldap_record", "Unable to locate a user with id %s in LDAP" % uva_uid)
entry = self.conn.entries[0]
return(LdapUserInfo(entry))
def search_users(self, query, limit):
search_string = LdapService.uid_search_string % query
self.conn.search(LdapService.search_base, search_string, attributes=LdapService.attributes)
# Entries are returned as a generator, accessing entries
# can make subsequent calls to the ldap service, so limit
# those here.
count = 0
results = []
for entry in self.conn.entries:
if count > limit:
break
results.append(LdapUserInfo(entry))
count += 1
return results

143
crc/services/lookup_service.py Normal file
View File

@ -0,0 +1,143 @@
from pandas import ExcelFile
from crc import db
from crc.api.common import ApiError
from crc.models.api_models import Task
from crc.models.file import FileDataModel, LookupFileModel, LookupDataModel
from crc.services.file_service import FileService
from crc.services.ldap_service import LdapService
class LookupService(object):
"""Provides tools for doing lookups for auto-complete fields.
This can currently take two forms:
1) Lookup from spreadsheet data associated with a workflow specification.
in which case we store the spreadsheet data in a lookup table with full
text indexing enabled, and run searches against that table.
2) Lookup from LDAP records. In which case we call out to an external service
to pull back detailed records and return them.
I could imagine this growing to include other external services as tools to handle
lookup fields. I could also imagine using some sort of local cache so we don't
unnecessarily pound on external services for repeat searches for the same records.
"""
@staticmethod
def lookup(spiff_task, field, query, limit):
"""Executes the lookup for the given field."""
if field.type != Task.FIELD_TYPE_AUTO_COMPLETE:
raise ApiError.from_task("invalid_field_type",
"Field '%s' must be an autocomplete field to use lookups." % field.label,
task=spiff_task)
# If this field has an associated options file, then do the lookup against that field.
if field.has_property(Task.PROP_OPTIONS_FILE):
lookup_table = LookupService.get_lookup_table(spiff_task, field)
return LookupService._run_lookup_query(lookup_table, query, limit)
# If this is a ldap lookup, use the ldap service to provide the fields to return.
elif field.has_property(Task.PROP_LDAP_LOOKUP):
return LookupService._run_ldap_query(query, limit)
else:
raise ApiError.from_task("unknown_lookup_option",
"Lookup supports using spreadsheet options or ldap options, and neither was"
"provided.")
@staticmethod
def get_lookup_table(spiff_task, field):
""" Checks to see if the options are provided in a separate lookup table associated with the
workflow, and if so, assures that data exists in the database, and return a model than can be used
to locate that data.
Returns: an array of LookupData, suitable for returning to the api.
"""
if field.has_property(Task.PROP_OPTIONS_FILE):
if not field.has_property(Task.PROP_OPTIONS_VALUE_COLUMN) or \
not field.has_property(Task.PROP_OPTIONS_LABEL_COL):
raise ApiError.from_task("invalid_emum",
"For enumerations based on an xls file, you must include 3 properties: %s, "
"%s, and %s" % (Task.PROP_OPTIONS_FILE,
Task.PROP_OPTIONS_VALUE_COLUMN,
Task.PROP_OPTIONS_LABEL_COL),
task=spiff_task)
# Get the file data from the File Service
file_name = field.get_property(Task.PROP_OPTIONS_FILE)
value_column = field.get_property(Task.PROP_OPTIONS_VALUE_COLUMN)
label_column = field.get_property(Task.PROP_OPTIONS_LABEL_COL)
data_model = FileService.get_workflow_file_data(spiff_task.workflow, file_name)
lookup_model = LookupService.get_lookup_table_from_data_model(data_model, value_column, label_column)
return lookup_model
@staticmethod
def get_lookup_table_from_data_model(data_model: FileDataModel, value_column, label_column):
""" In some cases the lookup table can be very large. This method will add all values to the database
in a way that can be searched and returned via an api call - rather than sending the full set of
options along with the form. It will only open the file and process the options if something has
changed. """
lookup_model = db.session.query(LookupFileModel) \
.filter(LookupFileModel.file_data_model_id == data_model.id) \
.filter(LookupFileModel.value_column == value_column) \
.filter(LookupFileModel.label_column == label_column).first()
if not lookup_model:
xls = ExcelFile(data_model.data)
df = xls.parse(xls.sheet_names[0]) # Currently we only look at the fist sheet.
if value_column not in df:
raise ApiError("invalid_emum",
"The file %s does not contain a column named % s" % (data_model.file_model.name,
value_column))
if label_column not in df:
raise ApiError("invalid_emum",
"The file %s does not contain a column named % s" % (data_model.file_model.name,
label_column))
lookup_model = LookupFileModel(label_column=label_column, value_column=value_column,
file_data_model_id=data_model.id)
db.session.add(lookup_model)
for index, row in df.iterrows():
lookup_data = LookupDataModel(lookup_file_model=lookup_model,
value=row[value_column],
label=row[label_column],
data=row.to_json())
db.session.add(lookup_data)
db.session.commit()
return lookup_model
@staticmethod
def _run_lookup_query(lookup_file_model, query, limit):
db_query = LookupDataModel.query.filter(LookupDataModel.lookup_file_model == lookup_file_model)
query = query.strip()
if len(query) > 1:
if ' ' in query:
terms = query.split(' ')
new_terms = []
for t in terms:
new_terms.append(t + ":*")
query = '|'.join(new_terms)
else:
query = "%s:*" % query
db_query = db_query.filter(LookupDataModel.label.match(query))
# db_query = db_query.filter(text("lookup_data.label @@ to_tsquery('simple', '%s')" % query))
return db_query.limit(limit).all()
@staticmethod
def _run_ldap_query(query, limit):
users = LdapService().search_users(query, limit)
"""Converts the user models into something akin to the
LookupModel in models/file.py, so this can be returned in the same way
we return a lookup data model."""
user_list = []
for user in users:
user_list.append( {"value": user.uid,
"label": user.display_name + "(" + user.uid + ")",
"data": user.__dict__
})
return user_list

100
crc/services/workflow_service.py
View File

@ -1,5 +1,7 @@
from datetime import datetime
import jinja2
from SpiffWorkflow import Task as SpiffTask, WorkflowException
from SpiffWorkflow.bpmn.specs.ManualTask import ManualTask
from SpiffWorkflow.bpmn.specs.ScriptTask import ScriptTask
from SpiffWorkflow.bpmn.specs.UserTask import UserTask
@ -7,20 +9,16 @@ from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
from SpiffWorkflow.dmn.specs.BusinessRuleTask import BusinessRuleTask
from SpiffWorkflow.specs import CancelTask, StartTask
from flask import g
from pandas import ExcelFile
from sqlalchemy import func
from jinja2 import Template
from crc import db, app
from crc.api.common import ApiError
from crc.models.api_models import Task, MultiInstanceType
import jinja2
from jinja2 import Template
from crc.models.file import FileDataModel, LookupFileModel, LookupDataModel
from crc.models.file import LookupDataModel
from crc.models.stats import TaskEventModel
from crc.services.file_service import FileService
from crc.services.lookup_service import LookupService
from crc.services.workflow_processor import WorkflowProcessor, CustomBpmnScriptEngine
from SpiffWorkflow import Task as SpiffTask, WorkflowException
class WorkflowService(object):
@ -178,10 +176,10 @@ class WorkflowService(object):
@staticmethod
def process_options(spiff_task, field):
lookup_model = WorkflowService.get_lookup_table(spiff_task, field)
lookup_model = LookupService.get_lookup_table(spiff_task, field)
# If lookup is set to true, do not populate options, a lookup will happen later.
if field.has_property(Task.EMUM_OPTIONS_AS_LOOKUP) and field.get_property(Task.EMUM_OPTIONS_AS_LOOKUP):
# If this is an auto-complete field, do not populate options, a lookup will happen later.
if field.type == Task.FIELD_TYPE_AUTO_COMPLETE:
pass
else:
data = db.session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_model).all()
@ -190,88 +188,6 @@ class WorkflowService(object):
for d in data:
field.options.append({"id": d.value, "name": d.label})
@staticmethod
def get_lookup_table(spiff_task, field):
""" Checks to see if the options are provided in a separate lookup table associated with the
workflow, and if so, assures that data exists in the database, and return a model than can be used
to locate that data. """
if field.has_property(Task.ENUM_OPTIONS_FILE_PROP):
if not field.has_property(Task.EMUM_OPTIONS_VALUE_COL_PROP) or \
not field.has_property(Task.EMUM_OPTIONS_LABEL_COL_PROP):
raise ApiError.from_task("invalid_emum",
"For enumerations based on an xls file, you must include 3 properties: %s, "
"%s, and %s" % (Task.ENUM_OPTIONS_FILE_PROP,
Task.EMUM_OPTIONS_VALUE_COL_PROP,
Task.EMUM_OPTIONS_LABEL_COL_PROP),
task=spiff_task)
# Get the file data from the File Service
file_name = field.get_property(Task.ENUM_OPTIONS_FILE_PROP)
value_column = field.get_property(Task.EMUM_OPTIONS_VALUE_COL_PROP)
label_column = field.get_property(Task.EMUM_OPTIONS_LABEL_COL_PROP)
data_model = FileService.get_workflow_file_data(spiff_task.workflow, file_name)
lookup_model = WorkflowService._get_lookup_table_from_data_model(data_model, value_column, label_column)
return lookup_model
@staticmethod
def _get_lookup_table_from_data_model(data_model: FileDataModel, value_column, label_column):
""" In some cases the lookup table can be very large. This method will add all values to the database
in a way that can be searched and returned via an api call - rather than sending the full set of
options along with the form. It will only open the file and process the options if something has
changed. """
lookup_model = db.session.query(LookupFileModel) \
.filter(LookupFileModel.file_data_model_id == data_model.id) \
.filter(LookupFileModel.value_column == value_column) \
.filter(LookupFileModel.label_column == label_column).first()
if not lookup_model:
xls = ExcelFile(data_model.data)
df = xls.parse(xls.sheet_names[0]) # Currently we only look at the fist sheet.
if value_column not in df:
raise ApiError("invalid_emum",
"The file %s does not contain a column named % s" % (data_model.file_model.name,
value_column))
if label_column not in df:
raise ApiError("invalid_emum",
"The file %s does not contain a column named % s" % (data_model.file_model.name,
label_column))
lookup_model = LookupFileModel(label_column=label_column, value_column=value_column,
file_data_model_id=data_model.id)
db.session.add(lookup_model)
for index, row in df.iterrows():
lookup_data = LookupDataModel(lookup_file_model=lookup_model,
value=row[value_column],
label=row[label_column],
data=row.to_json())
db.session.add(lookup_data)
db.session.commit()
return lookup_model
@staticmethod
def run_lookup_query(lookupFileModel, query, limit):
db_query = LookupDataModel.query.filter(LookupDataModel.lookup_file_model == lookupFileModel)
query = query.strip()
if len(query) > 1:
if ' ' in query:
terms = query.split(' ')
query = ""
new_terms = []
for t in terms:
new_terms.append(t + ":*")
query = '|'.join(new_terms)
else:
query = "%s:*" % query
db_query = db_query.filter(LookupDataModel.label.match(query))
# db_query = db_query.filter(text("lookup_data.label @@ to_tsquery('simple', '%s')" % query))
return db_query.limit(limit).all()
@staticmethod
def log_task_action(processor, spiff_task, action):
task = WorkflowService.spiff_task_to_api_task(spiff_task)

9
tests/base_test.py
View File

@ -77,20 +77,21 @@ class BaseTest(unittest.TestCase):
app.config.from_object('config.testing')
cls.ctx = app.test_request_context()
cls.app = app.test_client()
cls.ctx.push()
db.create_all()
@classmethod
def tearDownClass(cls):
db.drop_all()
cls.ctx.pop()
session.remove()
pass
def setUp(self):
self.ctx.push()
pass
def tearDown(self):
ExampleDataLoader.clean_db() # This does not seem to work, some colision of sessions.
self.ctx.pop()
ExampleDataLoader.clean_db()
session.flush()
self.auths = {}
def logged_in_headers(self, user=None, redirect_url='http://some/frontend/url'):

28
tests/data/enum_options_from_file/enum_options_from_file.bpmn
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.4.1">
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="Process_1vu5nxl" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
@ -14,9 +14,9 @@
<camunda:formData>
<camunda:formField id="AllTheNames" label="Select a value" type="enum">
<camunda:properties>
<camunda:property id="enum.options.file" value="customer_list.xls" />
<camunda:property id="enum.options.value.column" value="CUSTOMER_NUMBER" />
<camunda:property id="enum.options.label.column" value="CUSTOMER_NAME" />
<camunda:property id="spreadsheet.name" value="customer_list.xls" />
<camunda:property id="spreadsheet.value.column" value="CUSTOMER_NUMBER" />
<camunda:property id="spreadsheet.label.column" value="CUSTOMER_NAME" />
</camunda:properties>
</camunda:formField>
</camunda:formData>
@ -27,20 +27,20 @@
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1vu5nxl">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
<di:waypoint x="370" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
<dc:Bounds x="270" y="77" width="100" height="80" />
</bpmndi:BPMNShape>

29
tests/data/enum_options_with_search/enum_options_with_search.bpmn
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.4.1">
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="Process_1vu5nxl" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
@ -14,10 +14,9 @@
<camunda:formData>
<camunda:formField id="sponsor" label="Select a value" type="autocomplete">
<camunda:properties>
<camunda:property id="enum.options.file" value="sponsors.xls" />
<camunda:property id="enum.options.value.column" value="CUSTOMER_NUMBER" />
<camunda:property id="enum.options.label.column" value="CUSTOMER_NAME" />
<camunda:property id="enum.options.lookup" value="True" />
<camunda:property id="spreadsheet.name" value="sponsors.xls" />
<camunda:property id="spreadsheet.value.column" value="CUSTOMER_NUMBER" />
<camunda:property id="spreadsheet.label.column" value="CUSTOMER_NAME" />
</camunda:properties>
</camunda:formField>
</camunda:formData>
@ -28,20 +27,20 @@
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1vu5nxl">
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
<di:waypoint x="370" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
<dc:Bounds x="270" y="77" width="100" height="80" />
</bpmndi:BPMNShape>

47
tests/data/ldap_lookup/ldap_lookup.bpmn Normal file
View File

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
<bpmn:process id="Process_1vu5nxl" isExecutable="true">
<bpmn:startEvent id="StartEvent_1">
<bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
</bpmn:startEvent>
<bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="Task_14svgcu" />
<bpmn:endEvent id="EndEvent_0q4qzl9">
<bpmn:incoming>SequenceFlow_02vev7n</bpmn:incoming>
</bpmn:endEvent>
<bpmn:sequenceFlow id="SequenceFlow_02vev7n" sourceRef="Task_14svgcu" targetRef="EndEvent_0q4qzl9" />
<bpmn:userTask id="Task_14svgcu" name="Enum Lookup Form in Ldap" camunda:formKey="EnumForm">
<bpmn:extensionElements>
<camunda:formData>
<camunda:formField id="Person" label="Select a value" type="autocomplete">
<camunda:properties>
<camunda:property id="ldap.lookup" value="true" />
</camunda:properties>
</camunda:formField>
</camunda:formData>
</bpmn:extensionElements>
<bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
<bpmn:outgoing>SequenceFlow_02vev7n</bpmn:outgoing>
</bpmn:userTask>
</bpmn:process>
<bpmndi:BPMNDiagram id="BPMNDiagram_1">
<bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1vu5nxl">
<bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
<di:waypoint x="370" y="117" />
<di:waypoint x="432" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
<di:waypoint x="215" y="117" />
<di:waypoint x="270" y="117" />
</bpmndi:BPMNEdge>
<bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
<dc:Bounds x="179" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
<dc:Bounds x="432" y="99" width="36" height="36" />
</bpmndi:BPMNShape>
<bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
<dc:Bounds x="270" y="77" width="100" height="80" />
</bpmndi:BPMNShape>
</bpmndi:BPMNPlane>
</bpmndi:BPMNDiagram>
</bpmn:definitions>

1
tests/test_ldap_service.py
View File

@ -18,6 +18,7 @@ class TestLdapService(BaseTest):
def test_get_single_user(self):
user_info = self.ldap_service.user_info("lb3dp")
self.assertIsNotNone(user_info)
self.assertEqual("lb3dp", user_info.uid)
self.assertEqual("Laura Barnes", user_info.display_name)
self.assertEqual("Laura", user_info.given_name)
self.assertEqual("lb3dp@virginia.edu", user_info.email)

79
tests/test_lookup_service.py Normal file
View File

@ -0,0 +1,79 @@
from crc import session
from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
from crc.services.file_service import FileService
from crc.services.lookup_service import LookupService
from crc.services.workflow_processor import WorkflowProcessor
from crc.services.workflow_service import WorkflowService
from tests.base_test import BaseTest
class TestLookupService(BaseTest):
def test_create_lookup_file_multiple_times_does_not_update_database(self):
spec = self.load_test_spec('enum_options_from_file')
file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(1, len(lookup_records))
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEquals(19, len(lookup_data))
# Using the same table with different lookup lable or value, does create additional records.
LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NAME", "CUSTOMER_NUMBER")
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(2, len(lookup_records))
FileService.delete_file(file_model.id) ## Assure we can delete the file.
def test_some_full_text_queries(self):
self.load_test_spec('enum_options_from_file')
file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
self.assertIsNotNone(file_model)
file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
lookup_table = LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
results = LookupService._run_lookup_query(lookup_table, "medicines", limit=10)
self.assertEquals(1, len(results), "words in the middle of label are detected.")
self.assertEquals("The Medicines Company", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "", limit=10)
self.assertEquals(10, len(results), "Blank queries return everything, to the limit")
results = LookupService._run_lookup_query(lookup_table, "UVA", limit=10)
self.assertEquals(1, len(results), "Beginning of label is found.")
self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "uva", limit=10)
self.assertEquals(1, len(results), "case does not matter.")
self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "medici", limit=10)
self.assertEquals(1, len(results), "partial words are picked up.")
self.assertEquals("The Medicines Company", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "Genetics Savings", limit=10)
self.assertEquals(1, len(results), "multiple terms are picked up..")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "Genetics Sav", limit=10)
self.assertEquals(1, len(results), "prefix queries still work with partial terms")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "Gen Sav", limit=10)
self.assertEquals(1, len(results), "prefix queries still work with ALL the partial terms")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService._run_lookup_query(lookup_table, "Inc", limit=10)
self.assertEquals(7, len(results), "short terms get multiple correct results.")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
# Fixme: Stop words are taken into account on the query side, and haven't found a fix yet.
#results = WorkflowService.run_lookup_query(lookup_table.id, "in", limit=10)
#self.assertEquals(7, len(results), "stop words are not removed.")
#self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)

15
tests/test_tasks_api.py
View File

@ -336,6 +336,21 @@ class TestTasksApi(BaseTest):
results = json.loads(rv.get_data(as_text=True))
self.assertEqual(5, len(results))
def test_lookup_endpoint_for_task_ldap_field_lookup(self):
self.load_example_data()
workflow = self.create_workflow('ldap_lookup')
# get the first form
workflow = self.get_workflow_api(workflow)
task = workflow.next_task
field_id = task.form['fields'][0]['id']
# lb3dp is a user record in the mock ldap responses for tests.
rv = self.app.get('/v1.0/workflow/%i/task/%s/lookup/%s?query=%s&limit=5' %
(workflow.id, task.id, field_id, 'lb3dp'),
headers=self.logged_in_headers(),
content_type="application/json")
self.assert_success(rv)
results = json.loads(rv.get_data(as_text=True))
self.assertEqual(1, len(results))
def test_sub_process(self):
self.load_example_data()

69
tests/test_workflow_service.py
View File

@ -1,6 +1,7 @@
from crc import session
from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
from crc.services.file_service import FileService
from crc.services.lookup_service import LookupService
from crc.services.workflow_processor import WorkflowProcessor
from crc.services.workflow_service import WorkflowService
from tests.base_test import BaseTest
@ -76,7 +77,7 @@ class TestWorkflowService(BaseTest):
spec = self.load_test_spec('enum_options_from_file')
file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(1, len(lookup_records))
@ -101,71 +102,5 @@ class TestWorkflowService(BaseTest):
search_results = LookupDataModel.query.filter(LookupDataModel.label.match("bio:*")).all()
self.assertEquals(2, len(search_results))
def test_create_lookup_file_multiple_times_does_not_update_database(self):
spec = self.load_test_spec('enum_options_from_file')
file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(1, len(lookup_records))
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEquals(19, len(lookup_data))
# Using the same table with different lookup lable or value, does create additional records.
WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NAME", "CUSTOMER_NUMBER")
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(2, len(lookup_records))
FileService.delete_file(file_model.id) ## Assure we can delete the file.
def test_some_full_text_queries(self):
self.load_test_spec('enum_options_from_file')
file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
lookup_table = WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_table).all()
results = WorkflowService.run_lookup_query(lookup_table, "medicines", limit=10)
self.assertEquals(1, len(results), "words in the middle of label are detected.")
self.assertEquals("The Medicines Company", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "", limit=10)
self.assertEquals(10, len(results), "Blank queries return everything, to the limit")
results = WorkflowService.run_lookup_query(lookup_table, "UVA", limit=10)
self.assertEquals(1, len(results), "Beginning of label is found.")
self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "uva", limit=10)
self.assertEquals(1, len(results), "case does not matter.")
self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "medici", limit=10)
self.assertEquals(1, len(results), "partial words are picked up.")
self.assertEquals("The Medicines Company", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "Genetics Savings", limit=10)
self.assertEquals(1, len(results), "multiple terms are picked up..")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "Genetics Sav", limit=10)
self.assertEquals(1, len(results), "prefix queries still work with partial terms")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "Gen Sav", limit=10)
self.assertEquals(1, len(results), "prefix queries still work with ALL the partial terms")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
results = WorkflowService.run_lookup_query(lookup_table, "Inc", limit=10)
self.assertEquals(7, len(results), "short terms get multiple correct results.")
self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
# Fixme: Stop words are taken into account on the query side, and haven't found a fix yet.
#results = WorkflowService.run_lookup_query(lookup_table.id, "in", limit=10)
#self.assertEquals(7, len(results), "stop words are not removed.")
#self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)