Feature/new reference cache (#511)

* added basic model for new reference cache w/ burnettk

* switched out SpecReferenceCache for ReferenceCacheModel w/ burnettk jbirddog

* pyl w/ burnettk jbirddog

* save items to the db using the new cache with generation table w/ burnettk

* bulk save for performance

* tests are passing

* actually use the new generation table - we still need a test to ensure we are using it

* added test to ensure using new cache generation

* corrected reference interface on frontend w/ burnettk

* do not perform git pull in webhook if the revision is the same as the current w/ burnettk jbirddog

---------

Co-authored-by: jasquat <jasquat@users.noreply.github.com>
Co-authored-by: burnettk <burnettk@users.noreply.github.com>

spiffworkflow-backend/migrations/versions/4d438975ff4d_.py

@@ -0,0 +1,68 @@
+"""empty message
+
+Revision ID: 4d438975ff4d
+Revises: 9d5b6c5c31a5
+Create Date: 2023-09-21 16:25:44.574756
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4d438975ff4d'
+down_revision = '9d5b6c5c31a5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('cache_generation',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('cache_table', sa.String(length=255), nullable=False),
+    sa.Column('updated_at_in_seconds', sa.Integer(), nullable=True),
+    sa.Column('created_at_in_seconds', sa.Integer(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    with op.batch_alter_table('cache_generation', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_cache_generation_cache_table'), ['cache_table'], unique=False)
+
+    op.create_table('reference_cache',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('generation_id', sa.Integer(), nullable=False),
+    sa.Column('identifier', sa.String(length=255), nullable=False),
+    sa.Column('display_name', sa.String(length=255), nullable=False),
+    sa.Column('type', sa.String(length=255), nullable=False),
+    sa.Column('file_name', sa.String(length=255), nullable=False),
+    sa.Column('relative_location', sa.String(length=255), nullable=False),
+    sa.Column('properties', sa.JSON(), nullable=True),
+    sa.ForeignKeyConstraint(['generation_id'], ['cache_generation.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('generation_id', 'identifier', 'relative_location', 'type', name='reference_cache_uniq')
+    )
+    with op.batch_alter_table('reference_cache', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_reference_cache_display_name'), ['display_name'], unique=False)
+        batch_op.create_index(batch_op.f('ix_reference_cache_generation_id'), ['generation_id'], unique=False)
+        batch_op.create_index(batch_op.f('ix_reference_cache_identifier'), ['identifier'], unique=False)
+        batch_op.create_index(batch_op.f('ix_reference_cache_relative_location'), ['relative_location'], unique=False)
+        batch_op.create_index(batch_op.f('ix_reference_cache_type'), ['type'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('reference_cache', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_reference_cache_type'))
+        batch_op.drop_index(batch_op.f('ix_reference_cache_relative_location'))
+        batch_op.drop_index(batch_op.f('ix_reference_cache_identifier'))
+        batch_op.drop_index(batch_op.f('ix_reference_cache_generation_id'))
+        batch_op.drop_index(batch_op.f('ix_reference_cache_display_name'))
+
+    op.drop_table('reference_cache')
+    with op.batch_alter_table('cache_generation', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_cache_generation_cache_table'))
+
+    op.drop_table('cache_generation')
+    # ### end Alembic commands ###

spiffworkflow-backend/src/spiffworkflow_backend/load_database_models.py

@@ -26,6 +26,9 @@ from spiffworkflow_backend.models.human_task import HumanTaskModel  # noqa: F401
 from spiffworkflow_backend.models.spec_reference import (
     SpecReferenceCache,
 )  # noqa: F401
+from spiffworkflow_backend.models.reference_cache import (
+    ReferenceCacheModel,
+)  # noqa: F401
 from spiffworkflow_backend.models.process_caller import (
     ProcessCallerCacheModel,
 )  # noqa: F401

spiffworkflow-backend/src/spiffworkflow_backend/models/cache_generation.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy.orm import validates
+
+from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum
+from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel
+from spiffworkflow_backend.models.db import db
+
+
+class CacheGenerationTable(SpiffEnum):
+    reference_cache = "reference_cache"
+
+
+class CacheGenerationModel(SpiffworkflowBaseDBModel):
+    __tablename__ = "cache_generation"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    cache_table: str = db.Column(db.String(255), index=True, nullable=False)
+
+    updated_at_in_seconds: int = db.Column(db.Integer)
+    created_at_in_seconds: int = db.Column(db.Integer)
+
+    @classmethod
+    def newest_generation_for_table(cls, cache_table: str) -> CacheGenerationModel | None:
+        order_by_clause = CacheGenerationModel.id.desc()  # type: ignore
+        cache_generation: CacheGenerationModel | None = (
+            CacheGenerationModel.query.filter_by(cache_table=cache_table).order_by(order_by_clause).first()
+        )
+        return cache_generation
+
+    @validates("cache_table")
+    def validate_cache_table(self, key: str, value: Any) -> Any:
+        return self.validate_enum_field(key, value, CacheGenerationTable)

spiffworkflow-backend/src/spiffworkflow_backend/models/file.py

@@ -6,7 +6,7 @@ from datetime import datetime
 from typing import Any
 
 from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum
-from spiffworkflow_backend.models.spec_reference import SpecReference
+from spiffworkflow_backend.models.reference_cache import Reference
 
 
 class FileType(SpiffEnum):
@@ -67,7 +67,7 @@ class File:
     type: str
     last_modified: datetime
     size: int
-    references: list[SpecReference] | None = None
+    references: list[Reference] | None = None
     file_contents: bytes | None = None
     process_model_id: str | None = None
     bpmn_process_ids: list[str] | None = None

spiffworkflow-backend/src/spiffworkflow_backend/models/reference_cache.py

@@ -0,0 +1,160 @@
+import os
+from dataclasses import dataclass
+from typing import Any
+
+from flask_marshmallow import Schema  # type: ignore
+from marshmallow import INCLUDE
+from sqlalchemy import ForeignKey
+from sqlalchemy import UniqueConstraint
+from sqlalchemy.orm import relationship
+from sqlalchemy.orm import validates
+
+from spiffworkflow_backend.helpers.spiff_enum import SpiffEnum
+from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
+from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel
+from spiffworkflow_backend.models.db import db
+
+
+# SpecReferenceNotFoundError
+class ReferenceNotFoundError(Exception):
+    pass
+
+
+class ReferenceType(SpiffEnum):
+    decision = "decision"
+    process = "process"
+    data_store = "data_store"
+
+
+# SpecReference
+@dataclass()
+class Reference:
+    """File Reference Information.
+
+    Includes items such as the process id and name for a BPMN,
+    or the Decision id and Decision name for a DMN file.  There may be more than
+    one reference that points to a particular file - if for instance, there are
+    three executable processes in a collaboration within a BPMN Diagram.
+    """
+
+    identifier: str  # The id of the process or decision.  "Process_1234"
+    display_name: str  # The name of the process or decision. "Invoice Submission"
+    relative_location: str
+    type: str  # can be 'process' or 'decision'
+    file_name: str  # The name of the file where this process or decision is defined.
+    messages: dict  # Any messages defined in the same file where this process is defined.
+    correlations: dict  # Any correlations defined in the same file with this process.
+    start_messages: list  # The names of any messages that would start this process.
+    called_element_ids: list  # The element ids of any called elements
+
+    properties: dict
+
+    def prop_is_true(self, prop_name: str) -> bool:
+        return prop_name in self.properties and self.properties[prop_name] is True
+
+    def set_prop(self, prop_name: str, value: Any) -> None:
+        self.properties[prop_name] = value
+
+    def relative_path(self) -> str:
+        return os.path.join(self.relative_location, self.file_name).replace("/", os.sep)
+
+
+# SpecReferenceCache
+class ReferenceCacheModel(SpiffworkflowBaseDBModel):
+    """A cache of information about all the Processes and Decisions defined in all files."""
+
+    __tablename__ = "reference_cache"
+    __table_args__ = (
+        UniqueConstraint("generation_id", "identifier", "relative_location", "type", name="reference_cache_uniq"),
+    )
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    generation_id: int = db.Column(ForeignKey(CacheGenerationModel.id), nullable=False, index=True)  # type: ignore
+
+    identifier: str = db.Column(db.String(255), index=True, nullable=False)
+    display_name: str = db.Column(db.String(255), index=True, nullable=False)
+    type: str = db.Column(db.String(255), index=True, nullable=False)
+    file_name: str = db.Column(db.String(255), nullable=False)
+
+    # relative to SPIFFWORKFLOW_BACKEND_BPMN_SPEC_ABSOLUTE_DIR
+    relative_location: str = db.Column(db.String(255), index=True, nullable=False)
+
+    properties: dict | None = db.Column(db.JSON)
+    # has_lanes = db.Column(db.Boolean())
+    # is_executable = db.Column(db.Boolean())
+    # is_primary = db.Column(db.Boolean())
+
+    generation = relationship(CacheGenerationModel)
+
+    def relative_path(self) -> str:
+        return os.path.join(self.relative_location, self.file_name).replace("/", os.sep)
+
+    @classmethod
+    def basic_query(cls) -> Any:
+        cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
+        basic_query = cls.query
+        if cache_generation is not None:
+            basic_query = basic_query.filter_by(generation_id=cache_generation.id)
+        return basic_query
+
+    @classmethod
+    def from_params(
+        cls,
+        identifier: str,
+        display_name: str,
+        type: str,
+        file_name: str,
+        relative_location: str,
+        properties: dict | None = None,
+        use_current_cache_generation: bool = False,
+    ) -> "ReferenceCacheModel":
+        reference_cache = cls(
+            identifier=identifier,
+            display_name=display_name,
+            relative_location=relative_location,
+            type=type,
+            file_name=file_name,
+            properties=properties,
+        )
+        if use_current_cache_generation:
+            cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
+            if cache_generation is None:
+                # NOTE: we may want to raise here instead since this should never happen in real environments
+                # but it does happen in tests
+                cache_generation = CacheGenerationModel(cache_table="reference_cache")
+                db.session.add(cache_generation)
+            reference_cache.generation = cache_generation
+        return reference_cache
+
+    @classmethod
+    def from_spec_reference(cls, ref: Reference, use_current_cache_generation: bool = False) -> "ReferenceCacheModel":
+        reference_cache = cls.from_params(
+            identifier=ref.identifier,
+            display_name=ref.display_name,
+            relative_location=ref.relative_location,
+            type=ref.type,
+            file_name=ref.file_name,
+            properties=ref.properties,
+            use_current_cache_generation=use_current_cache_generation,
+        )
+        return reference_cache
+
+    @validates("type")
+    def validate_type(self, key: str, value: Any) -> Any:
+        return self.validate_enum_field(key, value, ReferenceType)
+
+
+# SpecReferenceSchema
+class ReferenceSchema(Schema):  # type: ignore
+    class Meta:
+        model = Reference
+        fields = [
+            "identifier",
+            "display_name",
+            "process_group_id",
+            "relative_location",
+            "type",
+            "file_name",
+            "properties",
+        ]
+        unknown = INCLUDE

spiffworkflow-backend/src/spiffworkflow_backend/routes/process_api_blueprint.py

@@ -16,8 +16,8 @@ from spiffworkflow_backend.models.principal import PrincipalModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
 from spiffworkflow_backend.models.process_instance_file_data import ProcessInstanceFileDataModel
 from spiffworkflow_backend.models.process_model import ProcessModelInfo
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
-from spiffworkflow_backend.models.spec_reference import SpecReferenceSchema
+from spiffworkflow_backend.models.reference_cache import ReferenceSchema
 from spiffworkflow_backend.models.task import TaskModel  # noqa: F401
 from spiffworkflow_backend.services.authorization_service import AuthorizationService
 from spiffworkflow_backend.services.git_service import GitService
@@ -63,8 +63,8 @@ def process_list() -> Any:
     This includes processes that are not the
     primary process - helpful for finding possible call activities.
     """
-    references = SpecReferenceCache.query.filter_by(type="process").all()
+    references = ReferenceCacheModel.basic_query().filter_by(type="process").all()
-    process_model_identifiers = [r.process_model_id for r in references]
+    process_model_identifiers = [r.relative_location for r in references]
     permitted_process_model_identifiers = ProcessModelService.process_model_identifiers_with_permission_for_user(
         user=g.user,
         permission_to_check="create",
@@ -73,17 +73,20 @@ def process_list() -> Any:
     )
     permitted_references = []
     for spec_reference in references:
-        if spec_reference.process_model_id in permitted_process_model_identifiers:
+        if spec_reference.relative_location in permitted_process_model_identifiers:
             permitted_references.append(spec_reference)
-    return SpecReferenceSchema(many=True).dump(permitted_references)
+    return ReferenceSchema(many=True).dump(permitted_references)
 
 
 def process_caller_list(bpmn_process_identifiers: list[str]) -> Any:
     callers = ProcessCallerService.callers(bpmn_process_identifiers)
     references = (
-        SpecReferenceCache.query.filter_by(type="process").filter(SpecReferenceCache.identifier.in_(callers)).all()
+        ReferenceCacheModel.basic_query()
+        .filter_by(type="process")
+        .filter(ReferenceCacheModel.identifier.in_(callers))  # type: ignore
+        .all()
     )
-    return SpecReferenceSchema(many=True).dump(references)
+    return ReferenceSchema(many=True).dump(references)
 
 
 def _process_data_fetcher(

spiffworkflow-backend/src/spiffworkflow_backend/routes/process_instances_controller.py

@@ -28,8 +28,8 @@ from spiffworkflow_backend.models.process_instance_queue import ProcessInstanceQ
 from spiffworkflow_backend.models.process_instance_report import ProcessInstanceReportModel
 from spiffworkflow_backend.models.process_instance_report import Report
 from spiffworkflow_backend.models.process_model import ProcessModelInfo
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
-from spiffworkflow_backend.models.spec_reference import SpecReferenceNotFoundError
+from spiffworkflow_backend.models.reference_cache import ReferenceNotFoundError
 from spiffworkflow_backend.models.task import TaskModel
 from spiffworkflow_backend.models.task_definition import TaskDefinitionModel
 from spiffworkflow_backend.routes.process_api_blueprint import _find_process_instance_by_id_or_raise
@@ -702,13 +702,13 @@ def _get_process_instance(
     process_model_with_diagram = None
     name_of_file_with_diagram = None
     if process_identifier:
-        spec_reference = SpecReferenceCache.query.filter_by(identifier=process_identifier, type="process").first()
+        spec_reference = (
+            ReferenceCacheModel.basic_query().filter_by(identifier=process_identifier, type="process").first()
+        )
         if spec_reference is None:
-            raise SpecReferenceNotFoundError(
+            raise ReferenceNotFoundError(f"Could not find given process identifier in the cache: {process_identifier}")
-                f"Could not find given process identifier in the cache: {process_identifier}"
-            )
 
-        process_model_with_diagram = ProcessModelService.get_process_model(spec_reference.process_model_id)
+        process_model_with_diagram = ProcessModelService.get_process_model(spec_reference.relative_location)
         name_of_file_with_diagram = spec_reference.file_name
         process_instance.process_model_with_diagram_identifier = process_model_with_diagram.id
     else:

spiffworkflow-backend/src/spiffworkflow_backend/services/data_setup_service.py

@@ -1,7 +1,10 @@
 from flask import current_app
+from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
 from spiffworkflow_backend.models.db import db
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
 from spiffworkflow_backend.services.spec_file_service import SpecFileService
+from sqlalchemy import insert
 
 
 class DataSetupService:
@@ -16,25 +19,29 @@ class DataSetupService:
         These all exist within processes located on the file system, so we can quickly reference them
         from the database.
         """
-        # Clear out all of the cached data.
-        SpecFileService.clear_caches()
-
         current_app.logger.debug("DataSetupService.save_all_process_models() start")
         failing_process_models = []
         process_models = ProcessModelService.get_process_models(recursive=True)
-        SpecFileService.clear_caches()
+        reference_objects = {}
         for process_model in process_models:
             current_app.logger.debug(f"Process Model: {process_model.display_name}")
             try:
+                # FIXME: get_references_for_file_contents is erroring out for elements in the list
                 refs = SpecFileService.get_references_for_process(process_model)
+
                 for ref in refs:
                     try:
-                        SpecFileService.update_caches(ref)
+                        reference_cache = ReferenceCacheModel.from_spec_reference(ref)
+                        reference_cache_unique = (
+                            f"{reference_cache.identifier}{reference_cache.relative_location}{reference_cache.type}"
+                        )
+                        reference_objects[reference_cache_unique] = reference_cache
+                        SpecFileService.update_caches_except_process(ref)
                         db.session.commit()
                     except Exception as ex:
                         failing_process_models.append(
                             (
-                                f"{ref.process_model_id}/{ref.file_name}",
+                                f"{ref.relative_location}/{ref.file_name}",
                                 str(ex),
                             )
                         )
@@ -47,5 +54,18 @@ class DataSetupService:
                 )
 
         current_app.logger.debug("DataSetupService.save_all_process_models() end")
+
+        # get inserted autoincrement primary key value back in a database agnostic way without committing the db session
+        ins = insert(CacheGenerationModel).values(cache_table="reference_cache")  # type: ignore
+        res = db.session.execute(ins)
+        cache_generation_id = res.inserted_primary_key[0]
+
+        # add primary key value to each element in reference objects list and store in new list
+        reference_object_list_with_cache_generation_id = []
+        for reference_object in reference_objects.values():
+            reference_object.generation_id = cache_generation_id
+            reference_object_list_with_cache_generation_id.append(reference_object)
+
+        db.session.bulk_save_objects(reference_object_list_with_cache_generation_id)
         db.session.commit()
         return failing_process_models

spiffworkflow-backend/src/spiffworkflow_backend/services/git_service.py

@@ -183,7 +183,15 @@ class GitService:
             return False
 
         if "ref" not in webhook:
-            raise InvalidGitWebhookBodyError(f"Could not find the 'ref' arg in the webhook boy: {webhook}")
+            raise InvalidGitWebhookBodyError(f"Could not find the 'ref' arg in the webhook body: {webhook}")
+        if "after" not in webhook:
+            raise InvalidGitWebhookBodyError(f"Could not find the 'after' arg in the webhook body: {webhook}")
+
+        git_revision_before_pull = cls.get_current_revision()
+        git_revision_after = webhook["after"]
+        if git_revision_before_pull == git_revision_after:
+            current_app.logger.info("Skipping git pull because we already have the current git revision, git boy!")
+            return True
 
         if current_app.config["SPIFFWORKFLOW_BACKEND_GIT_SOURCE_BRANCH"] is None:
             raise MissingGitConfigsError(

spiffworkflow-backend/src/spiffworkflow_backend/services/process_instance_processor.py

@@ -65,8 +65,8 @@ from spiffworkflow_backend.models.process_instance import ProcessInstanceStatus
 from spiffworkflow_backend.models.process_instance_event import ProcessInstanceEventType
 from spiffworkflow_backend.models.process_instance_metadata import ProcessInstanceMetadataModel
 from spiffworkflow_backend.models.process_model import ProcessModelInfo
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.models.script_attributes_context import ScriptAttributesContext
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
 from spiffworkflow_backend.models.task import TaskModel
 from spiffworkflow_backend.models.task import TaskNotFoundError
 from spiffworkflow_backend.models.task_definition import TaskDefinitionModel
@@ -1244,7 +1244,9 @@ class ProcessInstanceProcessor:
                 "bpmn_file_full_path_from_bpmn_process_identifier: bpmn_process_identifier is unexpectedly None"
             )
 
-        spec_reference = SpecReferenceCache.query.filter_by(identifier=bpmn_process_identifier, type="process").first()
+        spec_reference = (
+            ReferenceCacheModel.basic_query().filter_by(identifier=bpmn_process_identifier, type="process").first()
+        )
         bpmn_file_full_path = None
         if spec_reference is None:
             bpmn_file_full_path = ProcessInstanceProcessor.backfill_missing_spec_reference_records(
@@ -1253,7 +1255,7 @@ class ProcessInstanceProcessor:
         else:
             bpmn_file_full_path = os.path.join(
                 FileSystemService.root_path(),
-                spec_reference.relative_path,
+                spec_reference.relative_path(),
             )
         if bpmn_file_full_path is None:
             raise (

spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py

@@ -8,10 +8,10 @@ from spiffworkflow_backend.models.correlation_property_cache import CorrelationP
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.file import File
 from spiffworkflow_backend.models.file import FileType
-from spiffworkflow_backend.models.file import SpecReference
+from spiffworkflow_backend.models.file import Reference
 from spiffworkflow_backend.models.message_triggerable_process_model import MessageTriggerableProcessModel
 from spiffworkflow_backend.models.process_model import ProcessModelInfo
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.models.user import UserModel
 from spiffworkflow_backend.services.authentication_service import NotAuthorizedError
 from spiffworkflow_backend.services.custom_parser import MyCustomParser
@@ -32,7 +32,7 @@ class SpecFileService(FileSystemService):
     """
 
     @staticmethod
-    def reference_map(references: list[SpecReference]) -> dict[str, SpecReference]:
+    def reference_map(references: list[Reference]) -> dict[str, Reference]:
         """Creates a dict with provided references organized by id."""
         ref_map = {}
         for ref in references:
@@ -42,7 +42,7 @@ class SpecFileService(FileSystemService):
     @staticmethod
     def get_references_for_process(
         process_model_info: ProcessModelInfo,
-    ) -> list[SpecReference]:
+    ) -> list[Reference]:
         files = FileSystemService.get_files(process_model_info)
         references = []
         for file in files:
@@ -50,7 +50,7 @@ class SpecFileService(FileSystemService):
         return references
 
     @classmethod
-    def get_references_for_file(cls, file: File, process_model_info: ProcessModelInfo) -> list[SpecReference]:
+    def get_references_for_file(cls, file: File, process_model_info: ProcessModelInfo) -> list[Reference]:
         full_file_path = SpecFileService.full_file_path(process_model_info, file.name)
         file_contents: bytes = b""
         with open(full_file_path) as f:
@@ -71,7 +71,7 @@ class SpecFileService(FileSystemService):
     @classmethod
     def get_references_for_file_contents(
         cls, process_model_info: ProcessModelInfo, file_name: str, binary_data: bytes
-    ) -> list[SpecReference]:
+    ) -> list[Reference]:
         """Uses spiffworkflow to parse BPMN and DMN files to determine how they can be externally referenced.
 
         Returns a list of Reference objects that contain the type of reference, the id, the name.
@@ -80,8 +80,8 @@ class SpecFileService(FileSystemService):
         name = {str} 'Level 3'
         type = {str} 'process' / 'decision'
         """
-        references: list[SpecReference] = []
+        references: list[Reference] = []
-        file_path = os.path.join(process_model_info.id_for_file_path(), file_name)
+        os.path.join(process_model_info.id_for_file_path(), file_name)
         file_type = FileSystemService.file_type(file_name)
         parser = MyCustomParser()
         parser_type = None
@@ -116,20 +116,17 @@ class SpecFileService(FileSystemService):
                 called_element_ids = sub_parser.called_element_ids()
 
             references.append(
-                SpecReference(
+                Reference(
                     identifier=sub_parser.bpmn_id,
                     display_name=sub_parser.get_name(),
-                    process_model_id=process_model_info.id,
+                    relative_location=process_model_info.id,
                     type=parser_type,
                     file_name=file_name,
-                    relative_path=file_path,
-                    has_lanes=has_lanes,
-                    is_executable=is_executable,
                     messages=messages,
-                    is_primary=is_primary,
                     correlations=correlations,
                     start_messages=start_messages,
                     called_element_ids=called_element_ids,
+                    properties={"is_primary": is_primary, "has_lanes": has_lanes, "is_executable": is_executable},
                 )
             )
         return references
@@ -160,17 +157,19 @@ class SpecFileService(FileSystemService):
         cls.validate_bpmn_xml(file_name, binary_data)
 
         references = cls.get_references_for_file_contents(process_model_info, file_name, binary_data)
-        primary_process_ref = next((ref for ref in references if ref.is_primary and ref.is_executable), None)
+        primary_process_ref = next(
+            (ref for ref in references if ref.prop_is_true("is_primary") and ref.prop_is_true("is_executable")), None
+        )
 
         SpecFileService.clear_caches_for_file(file_name, process_model_info)
         all_called_element_ids: set[str] = set()
         for ref in references:
             # If no valid primary process is defined, default to the first process in the
             # updated file.
-            if not primary_process_ref and ref.type == "process" and ref.is_executable:
+            if not primary_process_ref and ref.type == "process" and ref.prop_is_true("is_executable"):
-                ref.is_primary = True
+                ref.set_prop("is_primary", True)
 
-            if ref.is_primary:
+            if ref.prop_is_true("is_primary"):
                 update_hash = {}
                 if not process_model_info.primary_file_name:
                     update_hash["primary_process_id"] = ref.identifier
@@ -185,14 +184,16 @@ class SpecFileService(FileSystemService):
                     )
 
             all_called_element_ids = all_called_element_ids | set(ref.called_element_ids)
-            SpecFileService.update_caches(ref)
+            SpecFileService.update_all_caches(ref)
 
         if user is not None:
-            called_element_refs = SpecReferenceCache.query.filter(
+            called_element_refs = (
-                SpecReferenceCache.identifier.in_(all_called_element_ids)
+                ReferenceCacheModel.basic_query()
-            ).all()
+                .filter(ReferenceCacheModel.identifier.in_(all_called_element_ids))  # type: ignore
+                .all()
+            )
             if len(called_element_refs) > 0:
-                process_model_identifiers: list[str] = [r.process_model_id for r in called_element_refs]
+                process_model_identifiers: list[str] = [r.relative_location for r in called_element_refs]
                 permitted_process_model_identifiers = (
                     ProcessModelService.process_model_identifiers_with_permission_for_user(
                         user=user,
@@ -246,8 +247,12 @@ class SpecFileService(FileSystemService):
     # fixme: Place all the caching stuff in a different service.
 
     @staticmethod
-    def update_caches(ref: SpecReference) -> None:
+    def update_all_caches(ref: Reference) -> None:
         SpecFileService.update_process_cache(ref)
+        SpecFileService.update_caches_except_process(ref)
+
+    @staticmethod
+    def update_caches_except_process(ref: Reference) -> None:
         SpecFileService.update_process_caller_cache(ref)
         SpecFileService.update_message_trigger_cache(ref)
         SpecFileService.update_correlation_cache(ref)
@@ -256,9 +261,9 @@ class SpecFileService(FileSystemService):
     def clear_caches_for_file(file_name: str, process_model_info: ProcessModelInfo) -> None:
         """Clear all caches related to a file."""
         records = (
-            db.session.query(SpecReferenceCache)
+            db.session.query(ReferenceCacheModel)
-            .filter(SpecReferenceCache.file_name == file_name)
+            .filter(ReferenceCacheModel.file_name == file_name)
-            .filter(SpecReferenceCache.process_model_id == process_model_info.id)
+            .filter(ReferenceCacheModel.relative_location == process_model_info.id)
             .all()
         )
 
@@ -273,38 +278,41 @@ class SpecFileService(FileSystemService):
 
     @staticmethod
     def clear_caches() -> None:
-        db.session.query(SpecReferenceCache).delete()
+        db.session.query(ReferenceCacheModel).delete()
         ProcessCallerService.clear_cache()
         # fixme:  likely the other caches should be cleared as well, but we don't have a clean way to do so yet.
 
     @staticmethod
-    def update_process_cache(ref: SpecReference) -> None:
+    def update_process_cache(ref: Reference) -> None:
         process_id_lookup = (
-            SpecReferenceCache.query.filter_by(identifier=ref.identifier).filter_by(type=ref.type).first()
+            ReferenceCacheModel.basic_query()
+            .filter_by(identifier=ref.identifier, relative_location=ref.relative_location, type=ref.type)
+            .first()
         )
         if process_id_lookup is None:
-            process_id_lookup = SpecReferenceCache.from_spec_reference(ref)
+            process_id_lookup = ReferenceCacheModel.from_spec_reference(ref, use_current_cache_generation=True)
             db.session.add(process_id_lookup)
         else:
-            if ref.relative_path != process_id_lookup.relative_path:
+            if ref.relative_path() != process_id_lookup.relative_path():
-                full_bpmn_file_path = SpecFileService.full_path_from_relative_path(process_id_lookup.relative_path)
+                full_bpmn_file_path = SpecFileService.full_path_from_relative_path(process_id_lookup.relative_path())
                 # if the old relative bpmn file no longer exists, then assume things were moved around
                 # on the file system. Otherwise, assume it is a duplicate process id and error.
                 if os.path.isfile(full_bpmn_file_path):
                     raise ProcessModelFileInvalidError(
                         f"Process id ({ref.identifier}) has already been used for "
-                        f"{process_id_lookup.relative_path}. It cannot be reused."
+                        f"{process_id_lookup.relative_path()}. It cannot be reused."
                     )
                 else:
-                    process_id_lookup.relative_path = ref.relative_path
+                    process_id_lookup.relative_location = ref.relative_location
+                    process_id_lookup.file_name = ref.file_name
                     db.session.add(process_id_lookup)
 
     @staticmethod
-    def update_process_caller_cache(ref: SpecReference) -> None:
+    def update_process_caller_cache(ref: Reference) -> None:
         ProcessCallerService.add_caller(ref.identifier, ref.called_element_ids)
 
     @staticmethod
-    def update_message_trigger_cache(ref: SpecReference) -> None:
+    def update_message_trigger_cache(ref: Reference) -> None:
         """Assure we know which messages can trigger the start of a process."""
         for message_name in ref.start_messages:
             message_triggerable_process_model = MessageTriggerableProcessModel.query.filter_by(
@@ -313,24 +321,24 @@ class SpecFileService(FileSystemService):
             if message_triggerable_process_model is None:
                 message_triggerable_process_model = MessageTriggerableProcessModel(
                     message_name=message_name,
-                    process_model_identifier=ref.process_model_id,
+                    process_model_identifier=ref.relative_location,
                 )
                 db.session.add(message_triggerable_process_model)
             else:
-                if message_triggerable_process_model.process_model_identifier != ref.process_model_id:
+                if message_triggerable_process_model.process_model_identifier != ref.relative_location:
                     raise ProcessModelFileInvalidError(
-                        f"Message model is already used to start process model {ref.process_model_id}"
+                        f"Message model is already used to start process model {ref.relative_location}"
                     )
 
     @staticmethod
-    def update_correlation_cache(ref: SpecReference) -> None:
+    def update_correlation_cache(ref: Reference) -> None:
         for name in ref.correlations.keys():
             correlation_property_retrieval_expressions = ref.correlations[name]["retrieval_expressions"]
 
             for cpre in correlation_property_retrieval_expressions:
                 message_name = ref.messages.get(cpre["messageRef"], None)
                 retrieval_expression = cpre["expression"]
-                process_model_id = ref.process_model_id
+                process_model_id = ref.relative_location
 
                 existing = CorrelationPropertyCache.query.filter_by(
                     name=name,

spiffworkflow-backend/src/spiffworkflow_backend/services/task_service.py

@@ -20,8 +20,8 @@ from spiffworkflow_backend.models.json_data import JsonDataModel
 from spiffworkflow_backend.models.process_instance import ProcessInstanceModel
 from spiffworkflow_backend.models.process_instance_event import ProcessInstanceEventModel
 from spiffworkflow_backend.models.process_instance_event import ProcessInstanceEventType
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
-from spiffworkflow_backend.models.spec_reference import SpecReferenceNotFoundError
+from spiffworkflow_backend.models.reference_cache import ReferenceNotFoundError
 from spiffworkflow_backend.models.task import TaskModel  # noqa: F401
 from spiffworkflow_backend.models.task import TaskNotFoundError
 from spiffworkflow_backend.models.task_definition import TaskDefinitionModel
@@ -683,21 +683,23 @@ class TaskService:
         try:
             filename: str | None = cls.get_spec_reference_from_bpmn_process(bpmn_process).file_name
             return filename
-        except SpecReferenceNotFoundError:
+        except ReferenceNotFoundError:
             return None
 
     @classmethod
-    def get_spec_reference_from_bpmn_process(cls, bpmn_process: BpmnProcessModel) -> SpecReferenceCache:
+    def get_spec_reference_from_bpmn_process(cls, bpmn_process: BpmnProcessModel) -> ReferenceCacheModel:
         """Get the bpmn file for a given task model.
 
         This involves several queries so avoid calling in a tight loop.
         """
         bpmn_process_definition = bpmn_process.bpmn_process_definition
-        spec_reference: SpecReferenceCache | None = SpecReferenceCache.query.filter_by(
+        spec_reference: ReferenceCacheModel | None = (
-            identifier=bpmn_process_definition.bpmn_identifier, type="process"
+            ReferenceCacheModel.basic_query()
-        ).first()
+            .filter_by(identifier=bpmn_process_definition.bpmn_identifier, type="process")
+            .first()
+        )
         if spec_reference is None:
-            raise SpecReferenceNotFoundError(
+            raise ReferenceNotFoundError(
                 f"Could not find given process identifier in the cache: {bpmn_process_definition.bpmn_identifier}"
             )
         return spec_reference

spiffworkflow-backend/tests/data/call_activity_duplicate/call_activity_nested.bpmn

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<bpmn:definitions xmlns:bpmn="http://www.omg.org/spec/BPMN/20100524/MODEL" xmlns:bpmndi="http://www.omg.org/spec/BPMN/20100524/DI" xmlns:dc="http://www.omg.org/spec/DD/20100524/DC" xmlns:di="http://www.omg.org/spec/DD/20100524/DI" xmlns:camunda="http://camunda.org/schema/1.0/bpmn" id="Definitions_f07329e" targetNamespace="http://bpmn.io/schema/bpmn" exporter="Camunda Modeler" exporterVersion="5.0.0">
+  <bpmn:process id="Level1" name="Level1" isExecutable="true">
+    <bpmn:startEvent id="StartEvent_1">
+      <bpmn:outgoing>Flow_1g3dpd7</bpmn:outgoing>
+    </bpmn:startEvent>
+    <bpmn:sequenceFlow id="Flow_1g3dpd7" sourceRef="StartEvent_1" targetRef="Activity_12zat0d" />
+    <bpmn:callActivity id="Activity_12zat0d" name="call level 2" calledElement="Level2">
+      <bpmn:incoming>Flow_1g3dpd7</bpmn:incoming>
+      <bpmn:outgoing>Flow_0qdgvah</bpmn:outgoing>
+    </bpmn:callActivity>
+    <bpmn:sequenceFlow id="Flow_0qdgvah" sourceRef="Activity_12zat0d" targetRef="Activity_0rkbhbz" />
+    <bpmn:endEvent id="Event_18dla68">
+      <bpmn:documentation># Main Workflow
+    Hello {{my_other_var}}
+
+  </bpmn:documentation>
+      <bpmn:incoming>Flow_0upce00</bpmn:incoming>
+    </bpmn:endEvent>
+    <bpmn:sequenceFlow id="Flow_0upce00" sourceRef="Activity_0rkbhbz" targetRef="Event_18dla68" />
+    <bpmn:callActivity id="Activity_0rkbhbz" name="call level 2B" calledElement="Level2b">
+      <bpmn:incoming>Flow_0qdgvah</bpmn:incoming>
+      <bpmn:outgoing>Flow_0upce00</bpmn:outgoing>
+    </bpmn:callActivity>
+  </bpmn:process>
+  <bpmndi:BPMNDiagram id="BPMNDiagram_1">
+    <bpmndi:BPMNPlane id="BPMNPlane_1" bpmnElement="Level1">
+      <bpmndi:BPMNShape id="_BPMNShape_StartEvent_2" bpmnElement="StartEvent_1">
+        <dc:Bounds x="179" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_0mcej1g_di" bpmnElement="Activity_12zat0d">
+        <dc:Bounds x="280" y="77" width="100" height="80" />
+        <bpmndi:BPMNLabel />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Event_18dla68_di" bpmnElement="Event_18dla68">
+        <dc:Bounds x="702" y="99" width="36" height="36" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNShape id="Activity_0jddvat_di" bpmnElement="Activity_0rkbhbz">
+        <dc:Bounds x="420" y="77" width="100" height="80" />
+      </bpmndi:BPMNShape>
+      <bpmndi:BPMNEdge id="Flow_1g3dpd7_di" bpmnElement="Flow_1g3dpd7">
+        <di:waypoint x="215" y="117" />
+        <di:waypoint x="280" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0qdgvah_di" bpmnElement="Flow_0qdgvah">
+        <di:waypoint x="380" y="117" />
+        <di:waypoint x="420" y="117" />
+      </bpmndi:BPMNEdge>
+      <bpmndi:BPMNEdge id="Flow_0upce00_di" bpmnElement="Flow_0upce00">
+        <di:waypoint x="520" y="117" />
+        <di:waypoint x="702" y="117" />
+      </bpmndi:BPMNEdge>
+    </bpmndi:BPMNPlane>
+  </bpmndi:BPMNDiagram>
+</bpmn:definitions>

spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py

@@ -22,7 +22,7 @@ from spiffworkflow_backend.models.process_instance_report import ProcessInstance
 from spiffworkflow_backend.models.process_instance_report import ReportMetadata
 from spiffworkflow_backend.models.process_model import NotificationType
 from spiffworkflow_backend.models.process_model import ProcessModelInfoSchema
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.models.task import TaskModel  # noqa: F401
 from spiffworkflow_backend.models.user import UserModel
 from spiffworkflow_backend.services.file_system_service import FileSystemService
@@ -505,7 +505,7 @@ class TestProcessApi(BaseTest):
             bpmn_file_name="simple_form",
         )
         # When adding a process model with one Process, no decisions, and some json files, only one process is recorded.
-        assert len(SpecReferenceCache.query.all()) == 1
+        assert len(ReferenceCacheModel.basic_query().all()) == 1
 
         self.create_group_and_model_with_bpmn(
             client=client,
@@ -515,7 +515,7 @@ class TestProcessApi(BaseTest):
             bpmn_file_location="call_activity_nested",
         )
         # When adding a process model with 4 processes and a decision, 5 new records will be in the Cache
-        assert len(SpecReferenceCache.query.all()) == 6
+        assert len(ReferenceCacheModel.basic_query().all()) == 6
 
         # get the results
         response = client.get(
@@ -529,10 +529,10 @@ class TestProcessApi(BaseTest):
         assert len(response.json) == 5
         simple_form = next(p for p in response.json if p["identifier"] == "Process_WithForm")
         assert simple_form["display_name"] == "Process With Form"
-        assert simple_form["process_model_id"] == "test_group_one/simple_form"
+        assert simple_form["relative_location"] == "test_group_one/simple_form"
-        assert simple_form["has_lanes"] is False
+        assert simple_form["properties"]["has_lanes"] is False
-        assert simple_form["is_executable"] is True
+        assert simple_form["properties"]["is_executable"] is True
-        assert simple_form["is_primary"] is True
+        assert simple_form["properties"]["is_primary"] is True
 
     def test_process_list_with_restricted_access(
         self,
@@ -547,7 +547,7 @@ class TestProcessApi(BaseTest):
             bpmn_file_name="simple_form",
         )
         # When adding a process model with one Process, no decisions, and some json files, only one process is recorded.
-        assert len(SpecReferenceCache.query.all()) == 1
+        assert len(ReferenceCacheModel.basic_query().all()) == 1
 
         self.create_group_and_model_with_bpmn(
             client=client,
@@ -557,7 +557,7 @@ class TestProcessApi(BaseTest):
             bpmn_file_location="call_activity_nested",
         )
         # When adding a process model with 4 processes and a decision, 5 new records will be in the Cache
-        assert len(SpecReferenceCache.query.all()) == 6
+        assert len(ReferenceCacheModel.basic_query().all()) == 6
 
         user_one = self.create_user_with_permission(
             username="user_one", target_uri="/v1.0/process-groups/test_group_one:*"
@@ -580,10 +580,10 @@ class TestProcessApi(BaseTest):
         assert len(response.json) == 1
         simple_form = next(p for p in response.json if p["identifier"] == "Process_WithForm")
         assert simple_form["display_name"] == "Process With Form"
-        assert simple_form["process_model_id"] == "test_group_one/simple_form"
+        assert simple_form["relative_location"] == "test_group_one/simple_form"
-        assert simple_form["has_lanes"] is False
+        assert simple_form["properties"]["has_lanes"] is False
-        assert simple_form["is_executable"] is True
+        assert simple_form["properties"]["is_executable"] is True
-        assert simple_form["is_primary"] is True
+        assert simple_form["properties"]["is_primary"] is True
 
     def test_process_callers(
         self,
@@ -599,7 +599,7 @@ class TestProcessApi(BaseTest):
             bpmn_file_name="simple_form",
         )
         # When adding a process model with one Process, no decisions, and some json files, only one process is recorded.
-        assert len(SpecReferenceCache.query.all()) == 1
+        assert len(ReferenceCacheModel.basic_query().all()) == 1
         # but no callers are recorded
         assert ProcessCallerService.count() == 0
 
@@ -611,7 +611,7 @@ class TestProcessApi(BaseTest):
             bpmn_file_location="call_activity_nested",
         )
         # When adding a process model with 4 processes and a decision, 5 new records will be in the Cache
-        assert len(SpecReferenceCache.query.all()) == 6
+        assert len(ReferenceCacheModel.basic_query().all()) == 6
         # and 4 callers recorded
         assert ProcessCallerService.count() == 4
 
@@ -1330,7 +1330,7 @@ class TestProcessApi(BaseTest):
             process_model_id=process_model_id,
             bpmn_file_location="call_activity_nested",
         )
-        spec_reference = SpecReferenceCache.query.filter_by(identifier="Level2b").first()
+        spec_reference = ReferenceCacheModel.basic_query().filter_by(identifier="Level2b").first()
         assert spec_reference
         modified_process_model_identifier = self.modify_process_identifier_for_path_param(process_model.id)
         headers = self.logged_in_headers(with_super_admin_user)
@@ -1356,7 +1356,7 @@ class TestProcessApi(BaseTest):
         with open(process_instance_file_path) as f_open:
             xml_file_contents = f_open.read()
             assert show_response.json["bpmn_xml_file_contents"] != xml_file_contents
-        spec_reference_file_path = os.path.join(file_system_root, spec_reference.relative_path)
+        spec_reference_file_path = os.path.join(file_system_root, spec_reference.relative_path())
         with open(spec_reference_file_path) as f_open:
             xml_file_contents = f_open.read()
             assert show_response.json["bpmn_xml_file_contents"] == xml_file_contents

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_model.py

@@ -5,7 +5,7 @@ from flask.app import Flask
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.process_instance_metadata import ProcessInstanceMetadataModel
 from spiffworkflow_backend.models.process_model import ProcessModelInfo
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.services.process_instance_processor import ProcessInstanceProcessor
 
 from tests.spiffworkflow_backend.helpers.base_test import BaseTest
@@ -86,7 +86,7 @@ class TestProcessModel(BaseTest):
 
         # delete all of the id lookup items to force to processor to find the correct
         # process model when running the process
-        db.session.query(SpecReferenceCache).delete()
+        db.session.query(ReferenceCacheModel).delete()
         db.session.commit()
         processor = ProcessInstanceProcessor(process_instance)
         processor.do_engine_steps(save=True, execution_strategy_name="greedy")

spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_spec_file_service.py

@@ -5,8 +5,9 @@ import pytest
 from flask import Flask
 from flask.testing import FlaskClient
 from lxml import etree  # type: ignore
+from spiffworkflow_backend.models.cache_generation import CacheGenerationModel
 from spiffworkflow_backend.models.db import db
-from spiffworkflow_backend.models.spec_reference import SpecReferenceCache
+from spiffworkflow_backend.models.reference_cache import ReferenceCacheModel
 from spiffworkflow_backend.services.process_model_service import ProcessModelService
 from spiffworkflow_backend.services.spec_file_service import ProcessModelFileInvalidError
 from spiffworkflow_backend.services.spec_file_service import SpecFileService
@@ -22,7 +23,7 @@ class TestSpecFileService(BaseTest):
     # process_model_id = "call_activity_nested"
     bpmn_file_name = "call_activity_nested.bpmn"
 
-    call_activity_nested_relative_file_path = os.path.join(process_group_id, process_model_id, bpmn_file_name)
+    call_activity_nested_relative_file_path = os.path.join(process_model_id, bpmn_file_name)
 
     def test_can_store_process_ids_for_lookup(
         self,
@@ -35,12 +36,12 @@ class TestSpecFileService(BaseTest):
             bpmn_file_name=self.bpmn_file_name,
             process_model_source_directory="call_activity_nested",
         )
-        bpmn_process_id_lookups = SpecReferenceCache.query.all()
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
         assert len(bpmn_process_id_lookups) == 1
         assert bpmn_process_id_lookups[0].identifier == "Level1"
-        assert bpmn_process_id_lookups[0].relative_path == self.call_activity_nested_relative_file_path
+        assert bpmn_process_id_lookups[0].relative_path() == self.call_activity_nested_relative_file_path
 
-    def test_fails_to_save_duplicate_process_id(
+    def test_fails_to_save_duplicate_process_id_in_same_process_model(
         self,
         app: Flask,
         client: FlaskClient,
@@ -48,23 +49,22 @@ class TestSpecFileService(BaseTest):
     ) -> None:
         bpmn_process_identifier = "Level1"
         load_test_spec(
-            process_model_id=self.process_model_id,
+            process_model_id="call_activity_duplicate",
             bpmn_file_name=self.bpmn_file_name,
-            process_model_source_directory="call_activity_nested",
+            process_model_source_directory="call_activity_duplicate",
         )
-        bpmn_process_id_lookups = SpecReferenceCache.query.all()
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
         assert len(bpmn_process_id_lookups) == 1
         assert bpmn_process_id_lookups[0].identifier == bpmn_process_identifier
-        assert bpmn_process_id_lookups[0].relative_path == self.call_activity_nested_relative_file_path
         with pytest.raises(ProcessModelFileInvalidError) as exception:
             load_test_spec(
-                "call_activity_nested_duplicate",
+                process_model_id="call_activity_duplicate",
                 process_model_source_directory="call_activity_duplicate",
                 bpmn_file_name="call_activity_nested_duplicate",
             )
-            assert f"Process id ({bpmn_process_identifier}) has already been used" in str(exception.value)
+        assert f"Process id ({bpmn_process_identifier}) has already been used" in str(exception.value)
 
-        process_model = ProcessModelService.get_process_model("call_activity_nested_duplicate")
+        process_model = ProcessModelService.get_process_model("call_activity_duplicate")
         full_file_path = SpecFileService.full_file_path(process_model, "call_activity_nested_duplicate.bpmn")
         assert not os.path.isfile(full_file_path)
 
@@ -75,10 +75,13 @@ class TestSpecFileService(BaseTest):
         with_db_and_bpmn_file_cleanup: None,
     ) -> None:
         bpmn_process_identifier = "Level1"
-        process_id_lookup = SpecReferenceCache(
+        process_id_lookup = ReferenceCacheModel.from_params(
             identifier=bpmn_process_identifier,
-            relative_path=self.call_activity_nested_relative_file_path,
+            display_name="WHO CARES",
+            relative_location=self.process_model_id,
+            file_name=self.bpmn_file_name,
             type="process",
+            use_current_cache_generation=True,
         )
         db.session.add(process_id_lookup)
         db.session.commit()
@@ -89,10 +92,10 @@ class TestSpecFileService(BaseTest):
             process_model_source_directory="call_activity_nested",
         )
 
-        bpmn_process_id_lookups = SpecReferenceCache.query.all()
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
         assert len(bpmn_process_id_lookups) == 1
         assert bpmn_process_id_lookups[0].identifier == bpmn_process_identifier
-        assert bpmn_process_id_lookups[0].relative_path == self.call_activity_nested_relative_file_path
+        assert bpmn_process_id_lookups[0].relative_path() == self.call_activity_nested_relative_file_path
 
     # this is really a test of your configuration.
     # sqlite and postgres are case sensitive by default,
@@ -102,15 +105,23 @@ class TestSpecFileService(BaseTest):
         app: Flask,
         with_db_and_bpmn_file_cleanup: None,
     ) -> None:
-        process_id_lookup = SpecReferenceCache(
+        process_id_lookup = ReferenceCacheModel.from_params(
             identifier="HOT",
+            display_name="WHO CARES",
+            relative_location=self.process_model_id,
+            file_name=self.bpmn_file_name,
             type="process",
+            use_current_cache_generation=True,
         )
         db.session.add(process_id_lookup)
         db.session.commit()
-        process_id_lookup = SpecReferenceCache(
+        process_id_lookup = ReferenceCacheModel.from_params(
             identifier="hot",
+            display_name="WHO CARES",
+            relative_location=self.process_model_id,
+            file_name=self.bpmn_file_name,
             type="process",
+            use_current_cache_generation=True,
         )
         db.session.add(process_id_lookup)
         db.session.commit()
@@ -123,12 +134,13 @@ class TestSpecFileService(BaseTest):
     ) -> None:
         """When a BPMN processes identifier is changed in a file, the old id is removed from the cache."""
         old_identifier = "ye_old_identifier"
-        process_id_lookup = SpecReferenceCache(
+        process_id_lookup = ReferenceCacheModel.from_params(
             identifier=old_identifier,
-            relative_path=self.call_activity_nested_relative_file_path,
+            display_name="WHO CARES",
+            relative_location=self.process_model_id,
             file_name=self.bpmn_file_name,
-            process_model_id=self.process_model_id,
             type="process",
+            use_current_cache_generation=True,
         )
         db.session.add(process_id_lookup)
         db.session.commit()
@@ -139,11 +151,11 @@ class TestSpecFileService(BaseTest):
             process_model_source_directory="call_activity_nested",
         )
 
-        bpmn_process_id_lookups = SpecReferenceCache.query.all()
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
         assert len(bpmn_process_id_lookups) == 1
         assert bpmn_process_id_lookups[0].identifier != old_identifier
         assert bpmn_process_id_lookups[0].identifier == "Level1"
-        assert bpmn_process_id_lookups[0].relative_path == self.call_activity_nested_relative_file_path
+        assert bpmn_process_id_lookups[0].relative_path() == self.call_activity_nested_relative_file_path
 
     def test_load_reference_information(
         self,
@@ -200,6 +212,63 @@ class TestSpecFileService(BaseTest):
         full_file_path = SpecFileService.full_file_path(process_model, "bad_xml.bpmn")
         assert not os.path.isfile(full_file_path)
 
+    def test_uses_correct_cache_generation(
+        self,
+        app: Flask,
+        client: FlaskClient,
+        with_db_and_bpmn_file_cleanup: None,
+    ) -> None:
+        current_cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
+        assert current_cache_generation is None
+
+        load_test_spec(
+            process_model_id=self.process_model_id,
+            bpmn_file_name=self.bpmn_file_name,
+            process_model_source_directory="call_activity_nested",
+        )
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
+        assert len(bpmn_process_id_lookups) == 1
+        assert bpmn_process_id_lookups[0].identifier == "Level1"
+        assert bpmn_process_id_lookups[0].relative_path() == self.call_activity_nested_relative_file_path
+
+        current_cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
+        assert current_cache_generation is not None
+        assert bpmn_process_id_lookups[0].generation_id == current_cache_generation.id
+
+        # make sure it doesn't add a new entry to the cache
+        load_test_spec(
+            process_model_id=self.process_model_id,
+            bpmn_file_name=self.bpmn_file_name,
+            process_model_source_directory="call_activity_nested",
+        )
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
+        assert len(bpmn_process_id_lookups) == 1
+        assert bpmn_process_id_lookups[0].identifier == "Level1"
+        assert bpmn_process_id_lookups[0].relative_path() == self.call_activity_nested_relative_file_path
+        assert bpmn_process_id_lookups[0].generation_id == current_cache_generation.id
+
+        cache_generations = CacheGenerationModel.query.all()
+        assert len(cache_generations) == 1
+
+        new_cache_generation = CacheGenerationModel(cache_table="reference_cache")
+        db.session.add(new_cache_generation)
+        db.session.commit()
+
+        cache_generations = CacheGenerationModel.query.all()
+        assert len(cache_generations) == 2
+        current_cache_generation = CacheGenerationModel.newest_generation_for_table("reference_cache")
+        assert current_cache_generation is not None
+
+        load_test_spec(
+            process_model_id=self.process_model_id,
+            bpmn_file_name=self.bpmn_file_name,
+            process_model_source_directory="call_activity_nested",
+        )
+        bpmn_process_id_lookups = ReferenceCacheModel.basic_query().all()
+        assert len(bpmn_process_id_lookups) == 1
+        assert bpmn_process_id_lookups[0].identifier == "Level1"
+        assert bpmn_process_id_lookups[0].generation_id == current_cache_generation.id
+
     @pytest.mark.skipif(
         sys.platform == "win32",
         reason="tmp file path is not valid xml for windows and it doesn't matter",

spiffworkflow-frontend/src/interfaces.ts

@@ -139,9 +139,7 @@ export interface ProcessReference {
   process_model_id: string;
   type: string; // either "decision" or "process"
   file_name: string;
-  has_lanes: boolean;
+  properties: any;
-  is_executable: boolean;
-  is_primary: boolean;
 }
 
 export type ObjectWithStringKeysAndValues = { [key: string]: string };