Refactor finding process models to support finding other file types (#516)

spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py

@@ -1,7 +1,10 @@
+import os
+
 from flask import current_app
 from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
+from spiffworkflow_backend.services.file_system_service import FileSystemService
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
 from spiffworkflow_backend.services.spec_file_service import SpecFileService
 from sqlalchemy import insert
@@ -12,6 +15,15 @@ class DataSetupService:
     def run_setup(cls) -> list:
         return cls.save_all_process_models()
 
+    @classmethod
+    def add_unique_reference_cache_object(
+        cls, reference_objects: dict[str, ReferenceCacheModel], reference_cache: ReferenceCacheModel
+    ) -> None:
+        reference_cache_unique = (
+            f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
+        )
+        reference_objects[reference_cache_unique] = reference_cache
+
     @classmethod
     def save_all_process_models(cls) -> list:
         """Build a cache of all processes, messages, correlation keys, and start events.
@@ -20,38 +32,52 @@ class DataSetupService:
         from the database.
         """
         current_app.logger.debug("DataSetupService.save_all_process_models() start")
-        failing_process_models = []
-        process_models = ProcessModelService.get_process_models(recursive=True)
-        reference_objects = {}
-        for process_model in process_models:
-            current_app.logger.debug(f"Process Model: {process_model.display_name}")
-            try:
-                # FIXME: get_references_for_file_contents is erroring out for elements in the list
-                refs = SpecFileService.get_references_for_process(process_model)
 
-                for ref in refs:
-                    try:
-                        reference_cache = ReferenceCacheModel.from_spec_reference(ref)
-                        reference_cache_unique = (
-                            f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
-                        )
-                        reference_objects[reference_cache_unique] = reference_cache
-                        SpecFileService.update_caches_except_process(ref)
-                        db.session.commit()
-                    except Exception as ex:
-                        failing_process_models.append(
-                            (
-                                f"{ref.relative_location}/{ref.file_name}",
-                                str(ex),
+        failing_process_models = []
+        files = FileSystemService.walk_files_from_root_path(True, None)
+        reference_objects: dict[str, ReferenceCacheModel] = {}
+        for file in files:
+            if FileSystemService.is_process_model_json_file(file):
+                process_model = ProcessModelService.get_process_model_from_path(file)
+                current_app.logger.debug(f"Process Model: {process_model.display_name}")
+                try:
+                    # FIXME: get_references_for_file_contents is erroring out for elements in the list
+                    refs = SpecFileService.get_references_for_process(process_model)
+
+                    for ref in refs:
+                        try:
+                            reference_cache = ReferenceCacheModel.from_spec_reference(ref)
+                            cls.add_unique_reference_cache_object(reference_objects, reference_cache)
+                            SpecFileService.update_caches_except_process(ref)
+                            db.session.commit()
+                        except Exception as ex:
+                            failing_process_models.append(
+                                (
+                                    f"{ref.relative_location}/{ref.file_name}",
+                                    str(ex),
+                                )
                             )
+                except Exception as ex2:
+                    failing_process_models.append(
+                        (
+                            f"{process_model.id}",
+                            str(ex2),
                         )
-            except Exception as ex2:
-                failing_process_models.append(
-                    (
-                        f"{process_model.id}",
-                        str(ex2),
                     )
+            elif FileSystemService.is_data_store_json_file(file):
+                relative_location = FileSystemService.relative_location(file)
+                file_name = os.path.basename(file)
+                (identifier, _) = os.path.splitext(file_name)
+                reference_cache = ReferenceCacheModel.from_params(
+                    identifier,
+                    identifier,
+                    "data_store",
+                    file_name,
+                    relative_location,
+                    None,
+                    False,
                 )
+                cls.add_unique_reference_cache_object(reference_objects, reference_cache)
 
         current_app.logger.debug("DataSetupService.save_all_process_models() end")
 

spiffworkflow-backend/src/spiffworkflow_backend/services/file_system_service.py

@@ -1,5 +1,6 @@
 import json
 import os
+from collections.abc import Callable
 from collections.abc import Generator
 from contextlib import contextmanager
 from datetime import datetime
@@ -18,6 +19,11 @@ class ProcessModelFileNotFoundError(Exception):
     pass
 
 
+DirectoryPredicate = Callable[[str, int], bool] | None
+FilePredicate = Callable[[str], bool] | None
+FileGenerator = Generator[str, None, None]
+
+
 class FileSystemService:
 
     """Simple Service meant for extension that provides some useful
@@ -38,6 +44,45 @@ class FileSystemService:
         finally:
             os.chdir(prevdir)
 
+    @classmethod
+    def walk_files(
+        cls, start_dir: str, directory_predicate: DirectoryPredicate, file_predicate: FilePredicate
+    ) -> FileGenerator:
+        depth = 0
+        for root, subdirs, files in os.walk(start_dir):
+            if directory_predicate:
+                subdirs[:] = [dir for dir in subdirs if directory_predicate(dir, depth)]
+            for f in files:
+                file = os.path.join(root, f)
+                if file_predicate and not file_predicate(file):
+                    continue
+                yield file
+            depth += 1
+
+    @classmethod
+    def non_git_dir(cls, dirname: str, depth: int) -> bool:
+        return dirname != ".git"
+
+    @classmethod
+    def not_recursive(cls, dirname: str, depth: int) -> bool:
+        return depth == 0
+
+    @classmethod
+    def standard_directory_predicate(cls, recursive: bool) -> DirectoryPredicate:
+        return cls.non_git_dir if recursive else cls.not_recursive
+
+    @classmethod
+    def is_process_model_json_file(cls, file: str) -> bool:
+        return file.endswith(cls.PROCESS_MODEL_JSON_FILE)
+
+    @classmethod
+    def is_data_store_json_file(cls, file: str) -> bool:
+        return file.endswith("_datastore.json")
+
+    @classmethod
+    def walk_files_from_root_path(cls, recursive: bool, file_predicate: FilePredicate) -> FileGenerator:
+        yield from cls.walk_files(cls.root_path(), cls.standard_directory_predicate(recursive), file_predicate)
+
     @staticmethod
     def root_path() -> str:
         dir_name = current_app.config["SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR"]
@@ -137,6 +182,10 @@ class FileSystemService:
         workflow_path = FileSystemService.process_model_full_path(process_model)
         return os.path.relpath(workflow_path, start=FileSystemService.root_path())
 
+    @classmethod
+    def relative_location(cls, path: str) -> str:
+        return os.path.dirname(os.path.relpath(path, start=FileSystemService.root_path()))
+
     @staticmethod
     def process_group_path_for_spec(process_model: ProcessModelInfo) -> str:
         # os.path.split apparently returns 2 element tulple like: (first/path, last_item)

spiffworkflow-backend/src/spiffworkflow_backend/services/process_model_service.py

@@ -2,7 +2,6 @@ import json
 import os
 import shutil
 import uuid
-from glob import glob
 from json import JSONDecodeError
 from typing import TypeVar
 
@@ -153,6 +152,11 @@ class ProcessModelService(FileSystemService):
         path = os.path.join(FileSystemService.root_path(), relative_path)
         return cls.__scan_process_model(path)
 
+    @classmethod
+    def get_process_model_from_path(cls, path: str) -> ProcessModelInfo:
+        relative_path = os.path.relpath(path, start=FileSystemService.root_path())
+        return cls.get_process_model_from_relative_path(os.path.dirname(relative_path))
+
     @classmethod
     def get_process_model(cls, process_model_id: str) -> ProcessModelInfo:
         """Get a process model from a model and group id.
@@ -180,13 +184,18 @@ class ProcessModelService(FileSystemService):
             awesome_id = process_group_id.replace("/", os.sep)
             root_path = os.path.join(root_path, awesome_id)
 
-        process_model_glob = os.path.join(root_path, "*", "process_model.json")
-        if recursive:
-            process_model_glob = os.path.join(root_path, "**", "process_model.json")
+        if recursive is None:
+            recursive = False
+
+        process_model_files = FileSystemService.walk_files(
+            root_path,
+            FileSystemService.standard_directory_predicate(recursive),
+            FileSystemService.is_process_model_json_file,
+        )
+
+        for file in process_model_files:
+            process_model = cls.get_process_model_from_path(file)
 
-        for file in glob(process_model_glob, recursive=True):
-            process_model_relative_path = os.path.relpath(file, start=FileSystemService.root_path())
-            process_model = cls.get_process_model_from_relative_path(os.path.dirname(process_model_relative_path))
             if include_files:
                 files = FileSystemService.get_sorted_files(process_model)
                 for f in files: