BPMN Parser was returning all retrieval expressions, rather than the ones specific to a correlation property, as was intended.

Adding a correlation cache - so we have a reference of all the messages and properties (though still lacking a description of keys)
Adding yet another migration, maybe should squash em.

spiffworkflow-backend/migrations/versions/4d47598d7181_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: 4d47598d7181
+Revises: 4ab08bf12666
+Create Date: 2023-02-23 16:12:11.603900
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4d47598d7181'
+down_revision = '4ab08bf12666'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('correlation_property_cache',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=50), nullable=False),
+    sa.Column('message_name', sa.String(length=50), nullable=False),
+    sa.Column('process_model_id', sa.String(length=255), nullable=False),
+    sa.Column('retrieval_expression', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('correlation_property_cache')
+    # ### end Alembic commands ###

spiffworkflow-backend/src/spiffworkflow_backend/models/correlation_property_cache.py

@@ -0,0 +1,28 @@
+"""Message_correlation."""
+from dataclasses import dataclass
+
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import relationship
+
+from spiffworkflow_backend.models.db import db
+from spiffworkflow_backend.models.db import SpiffworkflowBaseDBModel
+from spiffworkflow_backend.models.message_instance import MessageInstanceModel
+
+
+@dataclass
+class CorrelationPropertyCache(SpiffworkflowBaseDBModel):
+    """A list of known correlation properties as read from BPMN files.
+
+    This correlation properties are not directly linked to anything
+    but it provides a way to know what processes are talking about
+    what messages and correlation keys.  And could be useful as an
+    api endpoint if you wanted to know what another process model
+    is using.
+    """
+
+    __tablename__ = "correlation_property_cache"
+    id = db.Column(db.Integer, primary_key=True)
+    name: str = db.Column(db.String(50), nullable=False)
+    message_name: str = db.Column(db.String(50), nullable=False)
+    process_model_id: str = db.Column(db.String(255), nullable=False)
+    retrieval_expression: str = db.Column(db.String(255))

spiffworkflow-backend/src/spiffworkflow_backend/services/spec_file_service.py

@@ -8,6 +8,7 @@ from typing import Optional
 from lxml import etree  # type: ignore
 from SpiffWorkflow.bpmn.parser.BpmnParser import BpmnValidator  # type: ignore
 
+from spiffworkflow_backend.models.correlation_property_cache import CorrelationPropertyCache
 from spiffworkflow_backend.models.db import db
 from spiffworkflow_backend.models.file import File
 from spiffworkflow_backend.models.file import FileType
@@ -373,33 +374,30 @@ class SpecFileService(FileSystemService):
     @staticmethod
     def update_correlation_cache(ref: SpecReference) -> None:
         """Update_correlation_cache."""
-        # for correlation_identifier in ref.correlations.keys():
-        #     correlation_property_retrieval_expressions = ref.correlations[
-        #         correlation_identifier
-        #     ]["retrieval_expressions"]
-        #
-        #     for cpre in correlation_property_retrieval_expressions:
-        #         message_model_identifier = cpre["messageRef"]
-        #         message_model = MessageModel.query.filter_by(
-        #             identifier=message_model_identifier
-        #         ).first()
-        #         if message_model is None:
-        #             raise ProcessModelFileInvalidError(
-        #                 "Could not find message model with identifier"
-        #                 f" '{message_model_identifier}'specified by correlation"
-        #                 f" property: {cpre}"
-        #             )
-        #         message_correlation_property = (
-        #             MessageCorrelationPropertyModel.query.filter_by(
-        #                 identifier=correlation_identifier,
-        #                 message_model_id=message_model.id,
-        #             ).first()
-        #         )
-        #
-        #         if message_correlation_property is None:
-        #             message_correlation_property = MessageCorrelationPropertyModel(
-        #                 identifier=correlation_identifier,
-        #                 message_model_id=message_model.id,
-        #             )
-        #             db.session.add(message_correlation_property)
-        #             db.session.commit()
+        for name in ref.correlations.keys():
+            correlation_property_retrieval_expressions = ref.correlations[
+                name
+            ]["retrieval_expressions"]
+
+            for cpre in correlation_property_retrieval_expressions:
+                message_name = ref.messages.get(cpre["messageRef"], None)
+                retrieval_expression = cpre["expression"]
+                process_model_id = ref.process_model_id
+
+                existing = (
+                    CorrelationPropertyCache.query.filter_by(
+                        name=name,
+                        message_name=message_name,
+                        process_model_id=process_model_id,
+                        retrieval_expression=retrieval_expression
+                    ).first()
+                )
+                if existing is None:
+                    new_cache = CorrelationPropertyCache(
+                        name=name,
+                        message_name=message_name,
+                        process_model_id=process_model_id,
+                        retrieval_expression=retrieval_expression
+                    )
+                    db.session.add(new_cache)
+                    db.session.commit()