Feature/view data stores (#388)

* * Added /data-stores/ endpoint accessible by privileged users that will return a list of all data stores known to the system.
* Added a /data-stores/[type]/[name] endpoint that will return a list of all data stored within a data-store.
* Granted users with "elevated permissions" the right to access the data store.
* Added a "Data Store" link to the navigation bar beside messages.
* And a few useful tests.
* Still a little front end work to do to get it all looking pretty.

* Added a Data Store List component that, well, displays, you guessed it! A list of data stores and their contents.

Also, Carbon's paginator doesn't care how many pages you have, it's going to build the mother of all drop down lists - you got 60,000,000 records? Showing 5 at a time? It's going to be a dropdown list that contains all numbers between 1 and 12,000,000, because that makes sense! So, yea, not doing that, cutting it off at 1000 pages - you got more pages than that, the paginator can't take you there.  As you can show 100 items per page, that means you can access 100,000 items instantly.

* renaming data_store_items_list => data_store_item_list

spiffworkflow-backend/src/spiffworkflow_backend/api.yml

@@ -2322,6 +2322,51 @@ paths:
           #content:
           #  - application/json
 
+  /data-stores:
+    get:
+      operationId: spiffworkflow_backend.routes.data_store_controller.data_store_list
+      summary: Return a list of the data store objects.
+      tags:
+        - Data Stores
+      responses:
+        "200":
+          description: The list of currently defined data store objects
+  /data-stores/{data_store_type}/{name}:
+    parameters:
+      - name: data_store_type
+        in: path
+        required: true
+        description: The type of datastore, such as "typeahead"
+        schema:
+          type: string
+      - name: name
+        in: path
+        required: true
+        description: The name of the datastore, such as "cities"
+        schema:
+          type: string
+      - name: page
+        in: query
+        required: false
+        description: The page number to return. Defaults to page 1.
+        schema:
+          type: integer
+      - name: per_page
+        in: query
+        required: false
+        description: The page number to return. Defaults to page 1.
+        schema:
+          type: integer
+    get:
+      operationId: spiffworkflow_backend.routes.data_store_controller.data_store_item_list
+      summary: Returns a paginated list of the contents of a data store.
+      tags:
+        - Data Stores
+      responses:
+        "200":
+          description: A list of the data stored in the requested data store.
+
+
 components:
   securitySchemes:
     jwt:

spiffworkflow-backend/src/spiffworkflow_backend/routes/data_store_controller.py

@@ -0,0 +1,46 @@
+"""APIs for dealing with process groups, process models, and process instances."""
+
+import flask.wrappers
+from flask import jsonify
+from flask import make_response
+
+from spiffworkflow_backend import db
+from spiffworkflow_backend.exceptions.api_error import ApiError
+from spiffworkflow_backend.models.typeahead import TypeaheadModel
+
+
+def data_store_list() -> flask.wrappers.Response:
+    """Returns a list of the names of all the data stores."""
+    data_stores = []
+
+    # Right now the only data store we support is type ahead
+
+    for cat in db.session.query(TypeaheadModel.category).distinct():  # type: ignore
+        data_stores.append({"name": cat[0], "type": "typeahead"})
+
+    return make_response(jsonify(data_stores), 200)
+
+
+def data_store_item_list(
+    data_store_type: str, name: str, page: int = 1, per_page: int = 100
+) -> flask.wrappers.Response:
+    """Returns a list of the items in a data store."""
+    if data_store_type == "typeahead":
+        data_store_query = TypeaheadModel.query.filter_by(category=name)
+        data = data_store_query.paginate(page=page, per_page=per_page, error_out=False)
+        results = []
+        for typeahead in data.items:
+            result = typeahead.result
+            result["search_term"] = typeahead.search_term
+            results.append(result)
+        response_json = {
+            "results": results,
+            "pagination": {
+                "count": len(data.items),
+                "total": data.total,
+                "pages": data.pages,
+            },
+        }
+        return make_response(jsonify(response_json), 200)
+    else:
+        raise ApiError("unknown_data_store", f"Unknown data store type: {data_store_type}", status_code=400)

spiffworkflow-backend/src/spiffworkflow_backend/services/authorization_service.py

@@ -558,6 +558,8 @@ class AuthorizationService:
         for permission in ["create", "read", "update", "delete"]:
             permissions_to_assign.append(PermissionToAssign(permission=permission, target_uri="/process-instances/*"))
             permissions_to_assign.append(PermissionToAssign(permission=permission, target_uri="/secrets/*"))
+
+        permissions_to_assign.append(PermissionToAssign(permission="read", target_uri="/data-stores/*"))
         return permissions_to_assign
 
     @classmethod

spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_data_stores.py

@@ -0,0 +1,93 @@
+from flask import Flask
+from flask.testing import FlaskClient
+from spiffworkflow_backend.models.typeahead import TypeaheadModel
+from spiffworkflow_backend.models.user import UserModel
+
+from tests.spiffworkflow_backend.helpers.base_test import BaseTest
+
+
+class TestDataStores(BaseTest):
+    def load_data_store(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+        with_super_admin_user: UserModel,
+    ) -> None:
+        """
+        Populate a datastore with some mock data using a BPMN process that will load information
+        using the typeahead data store.  This should add 77 entries to the typeahead table.
+        """
+        process_group_id = "data_stores"
+        process_model_id = "cereals_data_store"
+        bpmn_file_location = "data_stores"
+        process_model = self.create_group_and_model_with_bpmn(
+            client,
+            with_super_admin_user,
+            process_group_id=process_group_id,
+            process_model_id=process_model_id,
+            bpmn_file_location=bpmn_file_location,
+        )
+
+        headers = self.logged_in_headers(with_super_admin_user)
+        response = self.create_process_instance_from_process_model_id_with_api(client, process_model.id, headers)
+        assert response.json is not None
+        process_instance_id = response.json["id"]
+
+        client.post(
+            f"/v1.0/process-instances/{self.modify_process_identifier_for_path_param(process_model.id)}/{process_instance_id}/run",
+            headers=self.logged_in_headers(with_super_admin_user),
+        )
+
+    def test_create_data_store_populates_db(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+        with_super_admin_user: UserModel,
+    ) -> None:
+        """Assure that when we run this workflow it will autofill the typeahead data store."""
+        self.load_data_store(app, client, with_db_and_bpmn_file_cleanup, with_super_admin_user)
+        typeaheads = TypeaheadModel.query.all()
+        assert len(typeaheads) == 153
+
+    def test_get_list_of_data_stores(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+        with_super_admin_user: UserModel,
+    ) -> None:
+        """
+        It should be possible to get a list of the data store categories that are available.
+        """
+        results = client.get("/v1.0/data-stores", headers=self.logged_in_headers(with_super_admin_user))
+        assert results.json == []
+
+        self.load_data_store(app, client, with_db_and_bpmn_file_cleanup, with_super_admin_user)
+        results = client.get("/v1.0/data-stores", headers=self.logged_in_headers(with_super_admin_user))
+        assert results.json == [{"name": "albums", "type": "typeahead"}, {"name": "cereals", "type": "typeahead"}]
+
+    def test_get_data_store_returns_paginated_results(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+        with_super_admin_user: UserModel,
+    ) -> None:
+        self.load_data_store(app, client, with_db_and_bpmn_file_cleanup, with_super_admin_user)
+        response = client.get(
+            "/v1.0/data-stores/typeahead/albums?per_page=10", headers=self.logged_in_headers(with_super_admin_user)
+        )
+
+        assert response.json is not None
+        assert len(response.json["results"]) == 10
+        assert response.json["pagination"]["count"] == 10
+        assert response.json["pagination"]["total"] == 76
+        assert response.json["pagination"]["pages"] == 8
+        assert response.json["results"][0] == {
+            "search_term": "Mama Said Knock You Out",
+            "year": 1990,
+            "album": "Mama Said Knock You Out",
+            "artist": "LL Cool J",
+        }

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_authorization_service.py

@@ -315,6 +315,7 @@ class TestAuthorizationService(BaseTest):
             [
                 ("/authentications", "read"),
                 ("/can-run-privileged-script/*", "create"),
+                ("/data-stores/*", "read"),
                 ("/debug/*", "create"),
                 ("/event-error-details/*", "read"),
                 ("/logs/*", "read"),

spiffworkflow-frontend/src/components/DataStoreList.tsx

@@ -0,0 +1,127 @@
+import { useEffect, useState } from 'react';
+import {
+  Dropdown,
+  Table,
+  TableHead,
+  TableHeader,
+  TableRow,
+} from '@carbon/react';
+import { TableBody, TableCell } from '@mui/material';
+import { useSearchParams } from 'react-router-dom';
+import HttpService from '../services/HttpService';
+import { DataStore, DataStoreRecords, PaginationObject } from '../interfaces';
+import PaginationForTable from './PaginationForTable';
+import { getPageInfoFromSearchParams } from '../helpers';
+
+export default function DataStoreList() {
+  const [dataStores, setDataStores] = useState<DataStore[]>([]);
+  const [dataStore, setDataStore] = useState<DataStore | null>(null);
+  const [pagination, setPagination] = useState<PaginationObject | null>(null);
+  const [results, setResults] = useState<any[]>([]);
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  useEffect(() => {
+    HttpService.makeCallToBackend({
+      path: `/data-stores`,
+      successCallback: (newStores: DataStore[]) => {
+        setDataStores(newStores);
+      },
+    });
+  }, []); // Do this once so we have a list of data stores to select from.
+
+  useEffect(() => {
+    const { page, perPage } = getPageInfoFromSearchParams(
+      searchParams,
+      10,
+      1,
+      'datastore'
+    );
+    console.log();
+    const dataStoreType = searchParams.get('type') || '';
+    const dataStoreName = searchParams.get('name') || '';
+
+    if (dataStoreType === '' || dataStoreName === '') {
+      return;
+    }
+    if (dataStores && dataStoreName && dataStoreType) {
+      dataStores.forEach((ds) => {
+        if (ds.name === dataStoreName && ds.type === dataStoreType) {
+          setDataStore(ds);
+        }
+      });
+    }
+    const queryParamString = `per_page=${perPage}&page=${page}`;
+    HttpService.makeCallToBackend({
+      path: `/data-stores/${dataStoreType}/${dataStoreName}?${queryParamString}`,
+      successCallback: (response: DataStoreRecords) => {
+        setResults(response.results);
+        setPagination(response.pagination);
+      },
+    });
+  }, [dataStores, searchParams]);
+
+  const getTable = () => {
+    if (results.length === 0) {
+      return null;
+    }
+    const firstResult = results[0];
+    console.log('Results', results);
+    const tableHeaders: any[] = [];
+    const keys = Object.keys(firstResult);
+    keys.forEach((key) => tableHeaders.push(<TableHeader>{key}</TableHeader>));
+
+    return (
+      <Table striped bordered>
+        <TableHead>
+          <TableRow>{tableHeaders}</TableRow>
+        </TableHead>
+        <TableBody>
+          {results.map((object) => {
+            return (
+              <TableRow>
+                {keys.map((key) => {
+                  return <TableCell>{object[key]}</TableCell>;
+                })}
+              </TableRow>
+            );
+          })}
+        </TableBody>
+      </Table>
+    );
+  };
+
+  const { page, perPage } = getPageInfoFromSearchParams(
+    searchParams,
+    10,
+    1,
+    'datastore'
+  );
+  return (
+    <>
+      <Dropdown
+        id="data-store-dropdown"
+        titleText="Select Data Store"
+        helperText="Select the data store you wish to view"
+        label="Please select a data store"
+        items={dataStores}
+        selectedItem={dataStore}
+        itemToString={(ds: DataStore) => (ds ? `${ds.name} (${ds.type})` : '')}
+        onChange={(event: any) => {
+          setDataStore(event.selectedItem);
+          searchParams.set('datastore_page', '1');
+          searchParams.set('datastore_per_page', '10');
+          searchParams.set('type', event.selectedItem.type);
+          searchParams.set('name', event.selectedItem.name);
+          setSearchParams(searchParams);
+        }}
+      />
+      <PaginationForTable
+        page={page}
+        perPage={perPage}
+        pagination={pagination}
+        tableToDisplay={getTable()}
+        paginationQueryParamPrefix="datastore"
+      />
+    </>
+  );
+}

spiffworkflow-frontend/src/components/NavigationBar.tsx

@@ -52,6 +52,7 @@ export default function NavigationBar() {
     [targetUris.authenticationListPath]: ['GET'],
     [targetUris.messageInstanceListPath]: ['GET'],
     [targetUris.secretListPath]: ['GET'],
+    [targetUris.dataStoreListPath]: ['GET'],
   };
   const { ability } = usePermissionFetcher(permissionRequestData);
 
@@ -76,6 +77,8 @@ export default function NavigationBar() {
       newActiveKey = '/admin/process-instances';
     } else if (location.pathname.match(/^\/admin\/configuration\b/)) {
       newActiveKey = '/admin/configuration';
+    } else if (location.pathname.match(/^\/admin\/datastore\b/)) {
+      newActiveKey = '/admin/datastore';
     } else if (location.pathname === '/') {
       newActiveKey = '/';
     } else if (location.pathname.match(/^\/tasks\b/)) {
@@ -228,6 +231,14 @@ export default function NavigationBar() {
             Messages
           </HeaderMenuItem>
         </Can>
+        <Can I="GET" a={targetUris.dataStoreListPath} ability={ability}>
+          <HeaderMenuItem
+            href="/admin/data-stores"
+            isCurrentPage={isActivePage('/admin/data-stores')}
+          >
+            Data Stores
+          </HeaderMenuItem>
+        </Can>
         {configurationElement()}
       </>
     );

spiffworkflow-frontend/src/components/PaginationForTable.tsx

@@ -44,6 +44,17 @@ export default function PaginationForTable({
   };
 
   if (pagination) {
+    const maxPages = 1000;
+    const pagesUnknown = pagination.pages > maxPages;
+    const totalItems =
+      pagination.pages < maxPages ? pagination.total : maxPages * perPage;
+    const itemText = () => {
+      const start = (page - 1) * perPage + 1;
+      return `Items ${start} to ${start + pagination.count} of ${
+        pagination.total
+      }`;
+    };
+
     return (
       <>
         {tableToDisplay}
@@ -55,10 +66,12 @@ export default function PaginationForTable({
           itemsPerPageText="Items per page:"
           page={page}
           pageNumberText="Page Number"
+          itemText={itemText}
           pageSize={perPage}
           pageSizes={perPageOptions || PER_PAGE_OPTIONS}
-          totalItems={pagination.total}
+          totalItems={totalItems}
           onChange={updateRows}
+          pagesUnknown={pagesUnknown}
         />
       </>
     );

spiffworkflow-frontend/src/hooks/UriListForPermissions.tsx

@@ -7,6 +7,7 @@ export const useUriListForPermissions = () => {
     return {
       authenticationListPath: `/v1.0/authentications`,
       messageInstanceListPath: '/v1.0/messages',
+      dataStoreListPath: '/v1.0/data-stores',
       processGroupListPath: '/v1.0/process-groups',
       processGroupShowPath: `/v1.0/process-groups/${params.process_group_id}`,
       processInstanceActionPath: `/v1.0/process-instances/${params.process_model_id}/${params.process_instance_id}`,

spiffworkflow-frontend/src/interfaces.ts

@@ -393,3 +393,13 @@ export interface TestCaseResults {
   failing: TestCaseResult[];
   passing: TestCaseResult[];
 }
+
+export interface DataStoreRecords {
+  results: any[];
+  pagination: PaginationObject;
+}
+
+export interface DataStore {
+  name: string;
+  type: string;
+}

spiffworkflow-frontend/src/routes/AdminRoutes.tsx

@@ -21,6 +21,7 @@ import ProcessModelNewExperimental from './ProcessModelNewExperimental';
 import ProcessInstanceFindById from './ProcessInstanceFindById';
 import ProcessInterstitialPage from './ProcessInterstitialPage';
 import MessageListPage from './MessageListPage';
+import DataStorePage from './DataStorePage';
 
 export default function AdminRoutes() {
   const location = useLocation();
@@ -125,6 +126,7 @@ export default function AdminRoutes() {
             element={<ProcessInstanceFindById />}
           />
           <Route path="messages" element={<MessageListPage />} />
+          <Route path="data-stores" element={<DataStorePage />} />
         </Routes>
       </div>
     );

spiffworkflow-frontend/src/routes/DataStorePage.tsx

@@ -0,0 +1,11 @@
+import React from 'react';
+import DataStoreList from '../components/DataStoreList';
+
+export default function DataStorePage() {
+  return (
+    <>
+      <h1>Data Stores</h1>
+      <DataStoreList />
+    </>
+  );
+}