From eed946260cbe0b371f7de3e44df193c8f10c2143 Mon Sep 17 00:00:00 2001 From: jasquat <2487833+jasquat@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:38:59 -0400 Subject: [PATCH] Test migration change (#1784) * added tests to ensure migrations do not change w/ burnettk * fixed tests w/ burnettk * pyl w/ burnettk --------- Co-authored-by: jasquat --- .../src/spiffworkflow_backend/constants.py | 9 ++++ .../process_instance_migrator.py | 42 +++++++++++++++++++ .../unit/test_process_instance_migrator.py | 25 +++++++++++ 3 files changed, 76 insertions(+) diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/constants.py b/spiffworkflow-backend/src/spiffworkflow_backend/constants.py index b1811f01..dd182caa 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/constants.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/constants.py @@ -1 +1,10 @@ SPIFFWORKFLOW_BACKEND_SERIALIZER_VERSION = "5" + +# so we can tell if a migration has changed or if there is a new one +SPIFFWORKFLOW_BACKEND_DATA_MIGRATION_CHECKSUM = { + "version_1_3.py": "22636f5ffb8e6d56fa460d82f62a854c", + "version_2.py": "962b4fda4d466758bdbdc09d75099603", + "version_3.py": "0e7154d0575c54b59011e3acedebe8b5", + "version_4.py": "889399d1c37e230a669d099f3a485fd4", + "version_5.py": "a5a9e62798b51741d4dd9d8f69868ded", +} diff --git a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py index 35b80fab..4c8dbd3b 100644 --- a/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py +++ b/spiffworkflow-backend/src/spiffworkflow_backend/data_migrations/process_instance_migrator.py @@ -1,3 +1,6 @@ +import glob +import hashlib +import os import time from typing import Any @@ -11,6 +14,10 @@ from spiffworkflow_backend.models.db import db from spiffworkflow_backend.models.process_instance import ProcessInstanceModel +class DataMigrationFilesNotFoundError(Exception): + pass + + # simple decorator to time the func # https://stackoverflow.com/a/11151365/6090676, thank you def benchmark_log_func(func: Any) -> Any: @@ -70,3 +77,38 @@ class ProcessInstanceMigrator: process_instance.spiff_serializer_version = data_migration_version_class.version() db.session.add(process_instance) db.session.commit() + + @classmethod + def get_migration_files(cls) -> list[str]: + file_glob = os.path.join(current_app.instance_path, "..", "spiffworkflow_backend", "data_migrations", "version_*.py") + files = sorted(glob.glob(file_glob)) + if len(files) == 0: + raise DataMigrationFilesNotFoundError(f"Could not find data migration with expected glob: {file_glob}") + return files + + # modified from https://gist.github.com/vinovator/d864555d9e82d25e52fd + @classmethod + def generate_md5_for_file(cls, file: str, chunk_size: int = 4096) -> str: + """ + Function which takes a file name and returns md5 checksum of the file + """ + hash = hashlib.md5() # noqa: S324 + with open(file, "rb") as f: + # Read the 1st block of the file + chunk = f.read(chunk_size) + # Keep reading the file until the end and update hash + while chunk: + hash.update(chunk) + chunk = f.read(chunk_size) + + # Return the hex checksum + return hash.hexdigest() + + @classmethod + def generate_migration_checksum(cls) -> dict[str, str]: + files = cls.get_migration_files() + md5checksums = {} + for file in files: + md5checksum = cls.generate_md5_for_file(file) + md5checksums[os.path.basename(file)] = md5checksum + return md5checksums diff --git a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_migrator.py b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_migrator.py index d3bc33ad..5122f5dc 100644 --- a/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_migrator.py +++ b/spiffworkflow-backend/tests/spiffworkflow_backend/unit/test_process_instance_migrator.py @@ -5,6 +5,8 @@ import os from flask.app import Flask from flask.testing import FlaskClient from SpiffWorkflow.bpmn.serializer.migration.version_1_3 import update_data_objects # type: ignore +from spiffworkflow_backend.constants import SPIFFWORKFLOW_BACKEND_DATA_MIGRATION_CHECKSUM +from spiffworkflow_backend.constants import SPIFFWORKFLOW_BACKEND_SERIALIZER_VERSION from spiffworkflow_backend.data_migrations.process_instance_migrator import ProcessInstanceMigrator from spiffworkflow_backend.data_migrations.version_1_3 import VersionOneThree from spiffworkflow_backend.data_migrations.version_4 import Version4 @@ -25,6 +27,29 @@ from tests.spiffworkflow_backend.helpers.test_data import load_test_spec class TestProcessInstanceMigrator(BaseTest): + def test_data_migrations_directory_has_not_changed( + self, + app: Flask, + client: FlaskClient, + ) -> None: + md5checksums = ProcessInstanceMigrator.generate_migration_checksum() + assert md5checksums == SPIFFWORKFLOW_BACKEND_DATA_MIGRATION_CHECKSUM, ( + "Data migrations seem to have changed but checksum has not been updated. " + "Please update SPIFFWORKFLOW_BACKEND_DATA_MIGRATION_CHECKSUM" + ) + + highest_version = 0 + for file in ProcessInstanceMigrator.get_migration_files(): + current_version = os.path.basename(file).replace("version_", "").replace(".py", "").replace("_", ".") + if current_version == "1.3": + continue + if int(current_version) > highest_version: + highest_version = int(current_version) + assert highest_version == int(SPIFFWORKFLOW_BACKEND_SERIALIZER_VERSION), ( + f"Highest migration version file is '{highest_version}' however " + f"SPIFFWORKFLOW_BACKEND_SERIALIZER_VERSION is '{SPIFFWORKFLOW_BACKEND_SERIALIZER_VERSION}'" + ) + def test_can_run_all_migrations( self, app: Flask,