ldap lookup.

Refactored calls into a new lookup_service to keep things tidy.

New keys for all enum/auto-complete fields:
    PROP_OPTIONS_FILE = "spreadsheet.name"
    PROP_OPTIONS_VALUE_COLUMN = "spreadsheet.value.column"
    PROP_OPTIONS_LABEL_COL = "spreadsheet.label.column"
    PROP_LDAP_LOOKUP = "ldap.lookup"
    FIELD_TYPE_AUTO_COMPLETE = "autocomplete"

crc/api/workflow.py

@@ -7,6 +7,7 @@ from crc.models.file import FileModel, LookupDataSchema
 from crc.models.workflow import WorkflowModel, WorkflowSpecModelSchema, WorkflowSpecModel, WorkflowSpecCategoryModel, \
     WorkflowSpecCategoryModelSchema
 from crc.services.file_service import FileService
+from crc.services.lookup_service import LookupService
 from crc.services.study_service import StudyService
 from crc.services.workflow_processor import WorkflowProcessor
 from crc.services.workflow_service import WorkflowService
@@ -217,9 +218,9 @@ def delete_workflow_spec_category(cat_id):
 
 def lookup(workflow_id, task_id, field_id, query, limit):
     """
-    given a field in a task, attempts to find the lookup table associated with that field
-    and runs a full-text query against it to locate the values and labels that would be
-    returned to a type-ahead box.
+    given a field in a task, attempts to find the lookup table or function associated
+    with that field and runs a full-text query against it to locate the values and
+    labels that would be returned to a type-ahead box.
     """
     workflow_model = session.query(WorkflowModel).filter_by(id=workflow_id).first()
     if not workflow_model:
@@ -236,6 +237,5 @@ def lookup(workflow_id, task_id, field_id, query, limit):
     if not field:
         raise ApiError("unknown_field", "No field named %s in task %s" % (task_id, spiff_task.task_spec.name))
 
-    lookup_table = WorkflowService.get_lookup_table(spiff_task, field)
-    lookup_data = WorkflowService.run_lookup_query(lookup_table, query, limit)
+    lookup_data = LookupService.lookup(spiff_task, field, query, limit)
     return LookupDataSchema(many=True).dump(lookup_data)

crc/models/api_models.py

@@ -31,10 +31,12 @@ class NavigationItem(object):
 
 class Task(object):
 
-    ENUM_OPTIONS_FILE_PROP = "enum.options.file"
-    EMUM_OPTIONS_VALUE_COL_PROP = "enum.options.value.column"
-    EMUM_OPTIONS_LABEL_COL_PROP = "enum.options.label.column"
-    EMUM_OPTIONS_AS_LOOKUP = "enum.options.lookup"
+    PROP_OPTIONS_FILE = "spreadsheet.name"
+    PROP_OPTIONS_VALUE_COLUMN = "spreadsheet.value.column"
+    PROP_OPTIONS_LABEL_COL = "spreadsheet.label.column"
+    PROP_LDAP_LOOKUP = "ldap.lookup"
+    FIELD_TYPE_AUTO_COMPLETE = "autocomplete"
+
 
     def __init__(self, id, name, title, type, state, form, documentation, data,
                  multi_instance_type, multi_instance_count, multi_instance_index, process_name, properties):

crc/services/ldap_service.py

@@ -12,20 +12,19 @@ class LdapUserInfo(object):
         self.display_name = entry.displayName.value
         self.given_name = ", ".join(entry.givenName)
         self.email = entry.mail.value
-        self.telephone_number= ", ".join(entry.telephoneNumber)
+        self.telephone_number = ", ".join(entry.telephoneNumber)
         self.title = ", ".join(entry.title)
         self.department = ", ".join(entry.uvaDisplayDepartment)
         self.affiliation = ", ".join(entry.uvaPersonIAMAffiliation)
         self.sponsor_type = ", ".join(entry.uvaPersonSponsoredType)
-
-
+        self.uid = entry.uid.value
 
 
 class LdapService(object):
     search_base = "ou=People,o=University of Virginia,c=US"
-    attributes = ['cn', 'displayName', 'givenName', 'mail', 'objectClass', 'UvaDisplayDepartment',
+    attributes = ['uid', 'cn', 'displayName', 'givenName', 'mail', 'objectClass', 'UvaDisplayDepartment',
                   'telephoneNumber', 'title', 'uvaPersonIAMAffiliation', 'uvaPersonSponsoredType']
-    search_string = "(&(objectclass=person)(uid=%s))"
+    uid_search_string = "(&(objectclass=person)(uid=%s))"
 
     def __init__(self):
         if app.config['TESTING']:
@@ -46,9 +45,25 @@ class LdapService(object):
             self.conn.unbind()
 
     def user_info(self, uva_uid):
-        search_string = LdapService.search_string % uva_uid
+        search_string = LdapService.uid_search_string % uva_uid
         self.conn.search(LdapService.search_base, search_string, attributes=LdapService.attributes)
         if len(self.conn.entries) < 1:
             raise ApiError("missing_ldap_record", "Unable to locate a user with id %s in LDAP" % uva_uid)
         entry = self.conn.entries[0]
         return(LdapUserInfo(entry))
+
+    def search_users(self, query, limit):
+        search_string = LdapService.uid_search_string % query
+        self.conn.search(LdapService.search_base, search_string, attributes=LdapService.attributes)
+
+        # Entries are returned as a generator, accessing entries
+        # can make subsequent calls to the ldap service, so limit
+        # those here.
+        count = 0
+        results = []
+        for entry in self.conn.entries:
+            if count > limit:
+                break
+            results.append(LdapUserInfo(entry))
+            count += 1
+        return results

crc/services/lookup_service.py

@@ -0,0 +1,143 @@
+from pandas import ExcelFile
+
+from crc import db
+from crc.api.common import ApiError
+from crc.models.api_models import Task
+from crc.models.file import FileDataModel, LookupFileModel, LookupDataModel
+from crc.services.file_service import FileService
+from crc.services.ldap_service import LdapService
+
+
+class LookupService(object):
+
+    """Provides tools for doing lookups for auto-complete fields.
+    This can currently take two forms:
+    1) Lookup from spreadsheet data associated with a workflow specification.
+       in which case we store the spreadsheet data in a lookup table with full
+       text indexing enabled, and run searches against that table.
+    2) Lookup from LDAP records.  In which case we call out to an external service
+       to pull back detailed records and return them.
+
+    I could imagine this growing to include other external services as tools to handle
+    lookup fields.  I could also imagine using some sort of local cache so we don't
+    unnecessarily pound on external services for repeat searches for the same records.
+    """
+
+    @staticmethod
+    def lookup(spiff_task, field, query, limit):
+        """Executes the lookup for the given field."""
+        if field.type != Task.FIELD_TYPE_AUTO_COMPLETE:
+            raise ApiError.from_task("invalid_field_type",
+                                          "Field '%s' must be an autocomplete field to use lookups." % field.label,
+                                          task=spiff_task)
+
+        # If this field has an associated options file, then do the lookup against that field.
+        if field.has_property(Task.PROP_OPTIONS_FILE):
+            lookup_table = LookupService.get_lookup_table(spiff_task, field)
+            return LookupService._run_lookup_query(lookup_table, query, limit)
+        # If this is a ldap lookup, use the ldap service to provide the fields to return.
+        elif field.has_property(Task.PROP_LDAP_LOOKUP):
+            return LookupService._run_ldap_query(query, limit)
+        else:
+            raise ApiError.from_task("unknown_lookup_option",
+                                     "Lookup supports using spreadsheet options or ldap options, and neither was"
+                                     "provided.")
+
+    @staticmethod
+    def get_lookup_table(spiff_task, field):
+        """ Checks to see if the options are provided in a separate lookup table associated with the
+        workflow, and if so, assures that data exists in the database, and return a model than can be used
+        to locate that data.
+
+        Returns:  an array of LookupData, suitable for returning to the api.
+        """
+        if field.has_property(Task.PROP_OPTIONS_FILE):
+            if not field.has_property(Task.PROP_OPTIONS_VALUE_COLUMN) or \
+                    not field.has_property(Task.PROP_OPTIONS_LABEL_COL):
+                raise ApiError.from_task("invalid_emum",
+                                         "For enumerations based on an xls file, you must include 3 properties: %s, "
+                                         "%s, and %s" % (Task.PROP_OPTIONS_FILE,
+                                                         Task.PROP_OPTIONS_VALUE_COLUMN,
+                                                         Task.PROP_OPTIONS_LABEL_COL),
+                                         task=spiff_task)
+
+            # Get the file data from the File Service
+            file_name = field.get_property(Task.PROP_OPTIONS_FILE)
+            value_column = field.get_property(Task.PROP_OPTIONS_VALUE_COLUMN)
+            label_column = field.get_property(Task.PROP_OPTIONS_LABEL_COL)
+            data_model = FileService.get_workflow_file_data(spiff_task.workflow, file_name)
+            lookup_model = LookupService.get_lookup_table_from_data_model(data_model, value_column, label_column)
+            return lookup_model
+
+    @staticmethod
+    def get_lookup_table_from_data_model(data_model: FileDataModel, value_column, label_column):
+        """ In some cases the lookup table can be very large.  This method will add all values to the database
+         in a way that can be searched and returned via an api call - rather than sending the full set of
+          options along with the form.  It will only open the file and process the options if something has
+          changed.  """
+
+        lookup_model = db.session.query(LookupFileModel) \
+            .filter(LookupFileModel.file_data_model_id == data_model.id) \
+            .filter(LookupFileModel.value_column == value_column) \
+            .filter(LookupFileModel.label_column == label_column).first()
+
+        if not lookup_model:
+            xls = ExcelFile(data_model.data)
+            df = xls.parse(xls.sheet_names[0])  # Currently we only look at the fist sheet.
+            if value_column not in df:
+                raise ApiError("invalid_emum",
+                               "The file %s does not contain a column named % s" % (data_model.file_model.name,
+                                                                                    value_column))
+            if label_column not in df:
+                raise ApiError("invalid_emum",
+                               "The file %s does not contain a column named % s" % (data_model.file_model.name,
+                                                                                    label_column))
+
+            lookup_model = LookupFileModel(label_column=label_column, value_column=value_column,
+                                           file_data_model_id=data_model.id)
+
+            db.session.add(lookup_model)
+            for index, row in df.iterrows():
+                lookup_data = LookupDataModel(lookup_file_model=lookup_model,
+                                              value=row[value_column],
+                                              label=row[label_column],
+                                              data=row.to_json())
+                db.session.add(lookup_data)
+            db.session.commit()
+
+        return lookup_model
+
+    @staticmethod
+    def _run_lookup_query(lookup_file_model, query, limit):
+        db_query = LookupDataModel.query.filter(LookupDataModel.lookup_file_model == lookup_file_model)
+
+        query = query.strip()
+        if len(query) > 1:
+            if ' ' in query:
+                terms = query.split(' ')
+                new_terms = []
+                for t in terms:
+                    new_terms.append(t + ":*")
+                query = '|'.join(new_terms)
+            else:
+                query = "%s:*" % query
+            db_query = db_query.filter(LookupDataModel.label.match(query))
+
+        #            db_query = db_query.filter(text("lookup_data.label @@ to_tsquery('simple', '%s')" % query))
+
+        return db_query.limit(limit).all()
+
+    @staticmethod
+    def _run_ldap_query(query, limit):
+        users = LdapService().search_users(query, limit)
+
+        """Converts the user models into something akin to the
+        LookupModel in models/file.py, so this can be returned in the same way 
+         we return a lookup data model."""
+        user_list = []
+        for user in users:
+            user_list.append( {"value": user.uid,
+                                "label": user.display_name + "(" + user.uid + ")",
+                                "data": user.__dict__
+                               })
+        return user_list

crc/services/workflow_service.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 
+import jinja2
+from SpiffWorkflow import Task as SpiffTask, WorkflowException
 from SpiffWorkflow.bpmn.specs.ManualTask import ManualTask
 from SpiffWorkflow.bpmn.specs.ScriptTask import ScriptTask
 from SpiffWorkflow.bpmn.specs.UserTask import UserTask
@@ -7,20 +9,16 @@ from SpiffWorkflow.bpmn.workflow import BpmnWorkflow
 from SpiffWorkflow.dmn.specs.BusinessRuleTask import BusinessRuleTask
 from SpiffWorkflow.specs import CancelTask, StartTask
 from flask import g
-from pandas import ExcelFile
-from sqlalchemy import func
+from jinja2 import Template
 
 from crc import db, app
 from crc.api.common import ApiError
 from crc.models.api_models import Task, MultiInstanceType
-import jinja2
-from jinja2 import Template
-
-from crc.models.file import FileDataModel, LookupFileModel, LookupDataModel
+from crc.models.file import LookupDataModel
 from crc.models.stats import TaskEventModel
 from crc.services.file_service import FileService
+from crc.services.lookup_service import LookupService
 from crc.services.workflow_processor import WorkflowProcessor, CustomBpmnScriptEngine
-from SpiffWorkflow import Task as SpiffTask, WorkflowException
 
 
 class WorkflowService(object):
@@ -178,10 +176,10 @@ class WorkflowService(object):
 
     @staticmethod
     def process_options(spiff_task, field):
-        lookup_model = WorkflowService.get_lookup_table(spiff_task, field)
+        lookup_model = LookupService.get_lookup_table(spiff_task, field)
 
-        # If lookup is set to true, do not populate options, a lookup will happen later.
-        if field.has_property(Task.EMUM_OPTIONS_AS_LOOKUP) and field.get_property(Task.EMUM_OPTIONS_AS_LOOKUP):
+        # If this is an auto-complete field, do not populate options, a lookup will happen later.
+        if field.type == Task.FIELD_TYPE_AUTO_COMPLETE:
             pass
         else:
             data = db.session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_model).all()
@@ -190,88 +188,6 @@ class WorkflowService(object):
             for d in data:
                 field.options.append({"id": d.value, "name": d.label})
 
-    @staticmethod
-    def get_lookup_table(spiff_task, field):
-        """ Checks to see if the options are provided in a separate lookup table associated with the
-        workflow, and if so, assures that data exists in the database, and return a model than can be used
-        to locate that data. """
-        if field.has_property(Task.ENUM_OPTIONS_FILE_PROP):
-            if not field.has_property(Task.EMUM_OPTIONS_VALUE_COL_PROP) or \
-                    not field.has_property(Task.EMUM_OPTIONS_LABEL_COL_PROP):
-                raise ApiError.from_task("invalid_emum",
-                                         "For enumerations based on an xls file, you must include 3 properties: %s, "
-                                         "%s, and %s" % (Task.ENUM_OPTIONS_FILE_PROP,
-                                                         Task.EMUM_OPTIONS_VALUE_COL_PROP,
-                                                         Task.EMUM_OPTIONS_LABEL_COL_PROP),
-                                         task=spiff_task)
-
-            # Get the file data from the File Service
-            file_name = field.get_property(Task.ENUM_OPTIONS_FILE_PROP)
-            value_column = field.get_property(Task.EMUM_OPTIONS_VALUE_COL_PROP)
-            label_column = field.get_property(Task.EMUM_OPTIONS_LABEL_COL_PROP)
-            data_model = FileService.get_workflow_file_data(spiff_task.workflow, file_name)
-            lookup_model = WorkflowService._get_lookup_table_from_data_model(data_model, value_column, label_column)
-            return lookup_model
-
-    @staticmethod
-    def _get_lookup_table_from_data_model(data_model: FileDataModel, value_column, label_column):
-        """ In some cases the lookup table can be very large.  This method will add all values to the database
-         in a way that can be searched and returned via an api call - rather than sending the full set of
-          options along with the form.  It will only open the file and process the options if something has
-          changed.  """
-
-        lookup_model = db.session.query(LookupFileModel) \
-            .filter(LookupFileModel.file_data_model_id == data_model.id) \
-            .filter(LookupFileModel.value_column == value_column) \
-            .filter(LookupFileModel.label_column == label_column).first()
-
-        if not lookup_model:
-            xls = ExcelFile(data_model.data)
-            df = xls.parse(xls.sheet_names[0])  # Currently we only look at the fist sheet.
-            if value_column not in df:
-                raise ApiError("invalid_emum",
-                               "The file %s does not contain a column named % s" % (data_model.file_model.name,
-                                                                                    value_column))
-            if label_column not in df:
-                raise ApiError("invalid_emum",
-                               "The file %s does not contain a column named % s" % (data_model.file_model.name,
-                                                                                    label_column))
-
-            lookup_model = LookupFileModel(label_column=label_column, value_column=value_column,
-                                           file_data_model_id=data_model.id)
-
-            db.session.add(lookup_model)
-            for index, row in df.iterrows():
-                lookup_data = LookupDataModel(lookup_file_model=lookup_model,
-                                              value=row[value_column],
-                                              label=row[label_column],
-                                              data=row.to_json())
-                db.session.add(lookup_data)
-            db.session.commit()
-
-        return lookup_model
-
-    @staticmethod
-    def run_lookup_query(lookupFileModel, query, limit):
-        db_query = LookupDataModel.query.filter(LookupDataModel.lookup_file_model == lookupFileModel)
-
-        query = query.strip()
-        if len(query) > 1:
-            if ' ' in query:
-                terms = query.split(' ')
-                query = ""
-                new_terms = []
-                for t in terms:
-                    new_terms.append(t + ":*")
-                query = '|'.join(new_terms)
-            else:
-                query = "%s:*" % query
-            db_query = db_query.filter(LookupDataModel.label.match(query))
-
-        #            db_query = db_query.filter(text("lookup_data.label @@ to_tsquery('simple', '%s')" % query))
-
-        return db_query.limit(limit).all()
-
     @staticmethod
     def log_task_action(processor, spiff_task, action):
         task = WorkflowService.spiff_task_to_api_task(spiff_task)

tests/base_test.py

@@ -77,20 +77,21 @@ class BaseTest(unittest.TestCase):
         app.config.from_object('config.testing')
         cls.ctx = app.test_request_context()
         cls.app = app.test_client()
+        cls.ctx.push()
         db.create_all()
 
     @classmethod
     def tearDownClass(cls):
-        db.drop_all()
+        cls.ctx.pop()
         session.remove()
         pass
 
     def setUp(self):
-        self.ctx.push()
+        pass
 
     def tearDown(self):
-        ExampleDataLoader.clean_db()  # This does not seem to work, some colision of sessions.
-        self.ctx.pop()
+        ExampleDataLoader.clean_db()
+        session.flush()
         self.auths = {}
 
     def logged_in_headers(self, user=None, redirect_url='http://some/frontend/url'):

tests/data/enum_options_from_file/enum_options_from_file.bpmn

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.4.1">
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
   <bpmn:process id="Process_1vu5nxl" isExecutable="true">
     <bpmn:startEvent id="StartEvent_1">
       <bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
@@ -14,9 +14,9 @@
         <camunda:formData>
           <camunda:formField id="AllTheNames" label="Select a value" type="enum">
             <camunda:properties>
-              <camunda:property id="enum.options.file" value="customer_list.xls" />
-              <camunda:property id="enum.options.value.column" value="CUSTOMER_NUMBER" />
-              <camunda:property id="enum.options.label.column" value="CUSTOMER_NAME" />
+              <camunda:property id="spreadsheet.name" value="customer_list.xls" />
+              <camunda:property id="spreadsheet.value.column" value="CUSTOMER_NUMBER" />
+              <camunda:property id="spreadsheet.label.column" value="CUSTOMER_NAME" />
             </camunda:properties>
           </camunda:formField>
         </camunda:formData>
@@ -27,20 +27,20 @@
   </bpmn:process>
   <bpmndi:BPMNDiagram id="BPMNDiagram_1">
     <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1vu5nxl">
-      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
-        <dc:Bounds x="179" y="99" width="36" height="36" />
-      </bpmndi:BPMNShape>
-      <bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
-        <di:waypoint x="215" y="117" />
-        <di:waypoint x="270" y="117" />
-      </bpmndi:BPMNEdge>
-      <bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
-        <dc:Bounds x="432" y="99" width="36" height="36" />
-      </bpmndi:BPMNShape>
       <bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
         <di:waypoint x="370" y="117" />
         <di:waypoint x="432" y="117" />
       </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="270" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
+        <dc:Bounds x="432" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
         <dc:Bounds x="270" y="77" width="100" height="80" />
       </bpmndi:BPMNShape>

tests/data/enum_options_with_search/enum_options_with_search.bpmn

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.4.1">
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
   <bpmn:process id="Process_1vu5nxl" isExecutable="true">
     <bpmn:startEvent id="StartEvent_1">
       <bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
@@ -14,10 +14,9 @@
         <camunda:formData>
           <camunda:formField id="sponsor" label="Select a value" type="autocomplete">
             <camunda:properties>
-              <camunda:property id="enum.options.file" value="sponsors.xls" />
-              <camunda:property id="enum.options.value.column" value="CUSTOMER_NUMBER" />
-              <camunda:property id="enum.options.label.column" value="CUSTOMER_NAME" />
-              <camunda:property id="enum.options.lookup" value="True" />
+              <camunda:property id="spreadsheet.name" value="sponsors.xls" />
+              <camunda:property id="spreadsheet.value.column" value="CUSTOMER_NUMBER" />
+              <camunda:property id="spreadsheet.label.column" value="CUSTOMER_NAME" />
             </camunda:properties>
           </camunda:formField>
         </camunda:formData>
@@ -28,20 +27,20 @@
   </bpmn:process>
   <bpmndi:BPMNDiagram id="BPMNDiagram_1">
     <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1vu5nxl">
-      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
-        <dc:Bounds x="179" y="99" width="36" height="36" />
-      </bpmndi:BPMNShape>
-      <bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
-        <di:waypoint x="215" y="117" />
-        <di:waypoint x="270" y="117" />
-      </bpmndi:BPMNEdge>
-      <bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
-        <dc:Bounds x="432" y="99" width="36" height="36" />
-      </bpmndi:BPMNShape>
       <bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
         <di:waypoint x="370" y="117" />
         <di:waypoint x="432" y="117" />
       </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="270" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
+        <dc:Bounds x="432" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
       <bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
         <dc:Bounds x="270" y="77" width="100" height="80" />
       </bpmndi:BPMNShape>

tests/data/ldap_lookup/ldap_lookup.bpmn

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" id="Definitions_1v1rp1q" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="3.7.3">
+  <bpmn:process id="Process_1vu5nxl" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>SequenceFlow_0lvudp8</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="SequenceFlow_0lvudp8" sourceRef="StartEvent_1" targetRef="Task_14svgcu" />
+    <bpmn:endEvent id="EndEvent_0q4qzl9">
+      <bpmn:incoming>SequenceFlow_02vev7n</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:sequenceFlow id="SequenceFlow_02vev7n" sourceRef="Task_14svgcu" targetRef="EndEvent_0q4qzl9" />
+    <bpmn:userTask id="Task_14svgcu" name="Enum Lookup Form in Ldap" camunda:formKey="EnumForm">
+      <bpmn:extensionElements>
+        <camunda:formData>
+          <camunda:formField id="Person" label="Select a value" type="autocomplete">
+            <camunda:properties>
+              <camunda:property id="ldap.lookup" value="true" />
+            </camunda:properties>
+          </camunda:formField>
+        </camunda:formData>
+      </bpmn:extensionElements>
+      <bpmn:incoming>SequenceFlow_0lvudp8</bpmn:incoming>
+      <bpmn:outgoing>SequenceFlow_02vev7n</bpmn:outgoing>
+    </bpmn:userTask>
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Process_1vu5nxl">
+      <bpmndi:BPMNEdge id="SequenceFlow_02vev7n_di" bpmnElement="SequenceFlow_02vev7n">
+        <di:waypoint x="370" y="117" />
+        <di:waypoint x="432" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="SequenceFlow_0lvudp8_di" bpmnElement="SequenceFlow_0lvudp8">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="270" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="EndEvent_0q4qzl9_di" bpmnElement="EndEvent_0q4qzl9">
+        <dc:Bounds x="432" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="UserTask_18ly1yq_di" bpmnElement="Task_14svgcu">
+        <dc:Bounds x="270" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

tests/test_ldap_service.py

@@ -18,6 +18,7 @@ class TestLdapService(BaseTest):
     def test_get_single_user(self):
         user_info = self.ldap_service.user_info("lb3dp")
         self.assertIsNotNone(user_info)
+        self.assertEqual("lb3dp", user_info.uid)
         self.assertEqual("Laura Barnes", user_info.display_name)
         self.assertEqual("Laura", user_info.given_name)
         self.assertEqual("lb3dp@virginia.edu", user_info.email)

tests/test_lookup_service.py

@@ -0,0 +1,79 @@
+from crc import session
+from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
+from crc.services.file_service import FileService
+from crc.services.lookup_service import LookupService
+from crc.services.workflow_processor import WorkflowProcessor
+from crc.services.workflow_service import WorkflowService
+from tests.base_test import BaseTest
+
+
+class TestLookupService(BaseTest):
+
+    def test_create_lookup_file_multiple_times_does_not_update_database(self):
+        spec = self.load_test_spec('enum_options_from_file')
+        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
+        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
+        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+        lookup_records = session.query(LookupFileModel).all()
+        self.assertIsNotNone(lookup_records)
+        self.assertEqual(1, len(lookup_records))
+        lookup_record = lookup_records[0]
+        lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
+        self.assertEquals(19, len(lookup_data))
+        # Using the same table with different lookup lable or value, does create additional records.
+        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NAME", "CUSTOMER_NUMBER")
+        lookup_records = session.query(LookupFileModel).all()
+        self.assertIsNotNone(lookup_records)
+        self.assertEqual(2, len(lookup_records))
+        FileService.delete_file(file_model.id) ## Assure we can delete the file.
+
+    def test_some_full_text_queries(self):
+        self.load_test_spec('enum_options_from_file')
+        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
+        self.assertIsNotNone(file_model)
+        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
+        lookup_table = LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+
+        results = LookupService._run_lookup_query(lookup_table, "medicines", limit=10)
+        self.assertEquals(1, len(results), "words in the middle of label are detected.")
+        self.assertEquals("The Medicines Company", results[0].label)
+
+        results = LookupService._run_lookup_query(lookup_table, "", limit=10)
+        self.assertEquals(10, len(results), "Blank queries return everything, to the limit")
+
+        results = LookupService._run_lookup_query(lookup_table, "UVA", limit=10)
+        self.assertEquals(1, len(results), "Beginning of label is found.")
+        self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
+
+        results = LookupService._run_lookup_query(lookup_table, "uva", limit=10)
+        self.assertEquals(1, len(results), "case does not matter.")
+        self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
+
+
+
+        results = LookupService._run_lookup_query(lookup_table, "medici", limit=10)
+        self.assertEquals(1, len(results), "partial words are picked up.")
+        self.assertEquals("The Medicines Company", results[0].label)
+
+        results = LookupService._run_lookup_query(lookup_table, "Genetics Savings", limit=10)
+        self.assertEquals(1, len(results), "multiple terms are picked up..")
+        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
+
+        results = LookupService._run_lookup_query(lookup_table, "Genetics Sav", limit=10)
+        self.assertEquals(1, len(results), "prefix queries still work with partial terms")
+        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
+
+        results = LookupService._run_lookup_query(lookup_table, "Gen Sav", limit=10)
+        self.assertEquals(1, len(results), "prefix queries still work with ALL the partial terms")
+        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
+
+        results = LookupService._run_lookup_query(lookup_table, "Inc", limit=10)
+        self.assertEquals(7, len(results), "short terms get multiple correct results.")
+        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
+
+        # Fixme:  Stop words are taken into account on the query side, and haven't found a fix yet.
+        #results = WorkflowService.run_lookup_query(lookup_table.id, "in", limit=10)
+        #self.assertEquals(7, len(results), "stop words are not removed.")
+        #self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)

tests/test_tasks_api.py

@@ -336,6 +336,21 @@ class TestTasksApi(BaseTest):
         results = json.loads(rv.get_data(as_text=True))
         self.assertEqual(5, len(results))
 
+    def test_lookup_endpoint_for_task_ldap_field_lookup(self):
+        self.load_example_data()
+        workflow = self.create_workflow('ldap_lookup')
+        # get the first form
+        workflow = self.get_workflow_api(workflow)
+        task = workflow.next_task
+        field_id = task.form['fields'][0]['id']
+        # lb3dp is a user record in the mock ldap responses for tests.
+        rv = self.app.get('/v1.0/workflow/%i/task/%s/lookup/%s?query=%s&limit=5' %
+                          (workflow.id, task.id, field_id, 'lb3dp'),
+                          headers=self.logged_in_headers(),
+                          content_type="application/json")
+        self.assert_success(rv)
+        results = json.loads(rv.get_data(as_text=True))
+        self.assertEqual(1, len(results))
 
     def test_sub_process(self):
         self.load_example_data()

tests/test_workflow_service.py

@@ -1,6 +1,7 @@
 from crc import session
 from crc.models.file import FileDataModel, FileModel, LookupFileModel, LookupDataModel
 from crc.services.file_service import FileService
+from crc.services.lookup_service import LookupService
 from crc.services.workflow_processor import WorkflowProcessor
 from crc.services.workflow_service import WorkflowService
 from tests.base_test import BaseTest
@@ -76,7 +77,7 @@ class TestWorkflowService(BaseTest):
         spec = self.load_test_spec('enum_options_from_file')
         file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
         file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
-        WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
+        LookupService.get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
         lookup_records = session.query(LookupFileModel).all()
         self.assertIsNotNone(lookup_records)
         self.assertEqual(1, len(lookup_records))
@@ -101,71 +102,5 @@ class TestWorkflowService(BaseTest):
         search_results = LookupDataModel.query.filter(LookupDataModel.label.match("bio:*")).all()
         self.assertEquals(2, len(search_results))
 
-    def test_create_lookup_file_multiple_times_does_not_update_database(self):
-        spec = self.load_test_spec('enum_options_from_file')
-        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
-        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
-        WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        lookup_records = session.query(LookupFileModel).all()
-        self.assertIsNotNone(lookup_records)
-        self.assertEqual(1, len(lookup_records))
-        lookup_record = lookup_records[0]
-        lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_record).all()
-        self.assertEquals(19, len(lookup_data))
-        # Using the same table with different lookup lable or value, does create additional records.
-        WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NAME", "CUSTOMER_NUMBER")
-        lookup_records = session.query(LookupFileModel).all()
-        self.assertIsNotNone(lookup_records)
-        self.assertEqual(2, len(lookup_records))
-        FileService.delete_file(file_model.id) ## Assure we can delete the file.
-
-    def test_some_full_text_queries(self):
-        self.load_test_spec('enum_options_from_file')
-        file_model = session.query(FileModel).filter(FileModel.name == "customer_list.xls").first()
-        file_data_model = session.query(FileDataModel).filter(FileDataModel.file_model == file_model).first()
-        lookup_table = WorkflowService._get_lookup_table_from_data_model(file_data_model, "CUSTOMER_NUMBER", "CUSTOMER_NAME")
-        lookup_data = session.query(LookupDataModel).filter(LookupDataModel.lookup_file_model == lookup_table).all()
-
-        results = WorkflowService.run_lookup_query(lookup_table, "medicines", limit=10)
-        self.assertEquals(1, len(results), "words in the middle of label are detected.")
-        self.assertEquals("The Medicines Company", results[0].label)
-
-        results = WorkflowService.run_lookup_query(lookup_table, "", limit=10)
-        self.assertEquals(10, len(results), "Blank queries return everything, to the limit")
-
-        results = WorkflowService.run_lookup_query(lookup_table, "UVA", limit=10)
-        self.assertEquals(1, len(results), "Beginning of label is found.")
-        self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
-
-        results = WorkflowService.run_lookup_query(lookup_table, "uva", limit=10)
-        self.assertEquals(1, len(results), "case does not matter.")
-        self.assertEquals("UVA - INTERNAL - GM USE ONLY", results[0].label)
 
 
-
-        results = WorkflowService.run_lookup_query(lookup_table, "medici", limit=10)
-        self.assertEquals(1, len(results), "partial words are picked up.")
-        self.assertEquals("The Medicines Company", results[0].label)
-
-        results = WorkflowService.run_lookup_query(lookup_table, "Genetics Savings", limit=10)
-        self.assertEquals(1, len(results), "multiple terms are picked up..")
-        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
-
-        results = WorkflowService.run_lookup_query(lookup_table, "Genetics Sav", limit=10)
-        self.assertEquals(1, len(results), "prefix queries still work with partial terms")
-        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
-
-        results = WorkflowService.run_lookup_query(lookup_table, "Gen Sav", limit=10)
-        self.assertEquals(1, len(results), "prefix queries still work with ALL the partial terms")
-        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
-
-        results = WorkflowService.run_lookup_query(lookup_table, "Inc", limit=10)
-        self.assertEquals(7, len(results), "short terms get multiple correct results.")
-        self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)
-
-        # Fixme:  Stop words are taken into account on the query side, and haven't found a fix yet.
-        #results = WorkflowService.run_lookup_query(lookup_table.id, "in", limit=10)
-        #self.assertEquals(7, len(results), "stop words are not removed.")
-        #self.assertEquals("Genetics Savings & Clone, Inc.", results[0].label)