Merge pull request #70 from sartography/feature/git-integration

Feature/git integration
This commit is contained in:
jasquat 2022-12-08 17:14:39 -05:00 committed by GitHub
commit aa353c0351
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 414 additions and 60 deletions

View File

@ -27,3 +27,8 @@ per-file-ignores =
# this file overwrites methods from the logging library so we can't change them
# and ignore long comment line
spiffworkflow-backend/src/spiffworkflow_backend/services/logging_service.py:N802,B950
# TODO: fix the S issues:
# S607 Starting a process with a partial executable path
# S605 Starting a process with a shell: Seems safe, but may be changed in the future, consider rewriting without shell
spiffworkflow-backend/tests/spiffworkflow_backend/integration/test_process_api.py:S607,S101,D103,S605

View File

@ -27,3 +27,5 @@ per-file-ignores =
# this file overwrites methods from the logging library so we can't change them
# and ignore long comment line
src/spiffworkflow_backend/services/logging_service.py:N802,B950
tests/spiffworkflow_backend/integration/test_process_api.py:S607,S101,D103,S605

View File

@ -1,13 +1,13 @@
"""Grabs tickets from csv and makes process instances."""
import os
from spiffworkflow_backend import get_hacked_up_app_for_script
from spiffworkflow_backend import create_app
from spiffworkflow_backend.services.data_setup_service import DataSetupService
def main() -> None:
"""Main."""
app = get_hacked_up_app_for_script()
app = create_app()
with app.app_context():
failing_process_models = DataSetupService.save_all_process_models()
for bpmn_errors in failing_process_models:

View File

@ -445,6 +445,32 @@ paths:
schema:
$ref: "#/components/schemas/ProcessModel"
/process-models/{modified_process_model_identifier}/publish:
parameters:
- name: modified_process_model_identifier
in: path
required: true
description: the modified process model id
schema:
type: string
- name: branch_to_update
in: query
required: false
description: the name of the branch we want to merge into
schema:
type: string
post:
operationId: spiffworkflow_backend.routes.process_api_blueprint.process_model_publish
summary: Merge changes from this model to another branch.
tags:
- Process Models
responses:
"200":
description: The process model was published.
content:
application/json:
schema:
type: string
/processes:
get:

View File

@ -38,6 +38,17 @@ def setup_database_uri(app: Flask) -> None:
)
def load_config_file(app: Flask, env_config_module: str) -> None:
"""Load_config_file."""
try:
app.config.from_object(env_config_module)
except ImportStringError as exception:
if os.environ.get("TERRAFORM_DEPLOYED_ENVIRONMENT") != "true":
raise ModuleNotFoundError(
f"Cannot find config module: {env_config_module}"
) from exception
def setup_config(app: Flask) -> None:
"""Setup_config."""
# ensure the instance folder exists
@ -53,19 +64,14 @@ def setup_config(app: Flask) -> None:
app.config.from_object("spiffworkflow_backend.config.default")
env_config_prefix = "spiffworkflow_backend.config."
if (
os.environ.get("TERRAFORM_DEPLOYED_ENVIRONMENT") == "true"
and os.environ.get("SPIFFWORKFLOW_BACKEND_ENV") is not None
):
load_config_file(app, f"{env_config_prefix}terraform_deployed_environment")
env_config_module = env_config_prefix + app.config["ENV_IDENTIFIER"]
try:
app.config.from_object(env_config_module)
except ImportStringError as exception:
if (
os.environ.get("TERRAFORM_DEPLOYED_ENVIRONMENT") == "true"
and os.environ.get("SPIFFWORKFLOW_BACKEND_ENV") is not None
):
app.config.from_object(f"{env_config_prefix}terraform_deployed_environment")
else:
raise ModuleNotFoundError(
f"Cannot find config module: {env_config_module}"
) from exception
load_config_file(app, env_config_module)
# This allows config/testing.py or instance/config.py to override the default config
if "ENV_IDENTIFIER" in app.config and app.config["ENV_IDENTIFIER"] == "testing":

View File

@ -61,6 +61,10 @@ SPIFFWORKFLOW_BACKEND_LOG_LEVEL = environ.get(
"SPIFFWORKFLOW_BACKEND_LOG_LEVEL", default="info"
)
# When a user clicks on the `Publish` button, this is the default branch this server merges into.
# I.e., dev server could have `staging` here. Staging server might have `production` here.
GIT_MERGE_BRANCH = environ.get("GIT_MERGE_BRANCH", default="staging")
# Datbase Configuration
SPIFF_DATABASE_TYPE = environ.get(
"SPIFF_DATABASE_TYPE", default="mysql"

View File

@ -2,8 +2,8 @@
from os import environ
GIT_COMMIT_ON_SAVE = True
GIT_COMMIT_USERNAME = "demo"
GIT_COMMIT_EMAIL = "demo@example.com"
GIT_USERNAME = "demo"
GIT_USER_EMAIL = "demo@example.com"
SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get(
"SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME",
default="terraform_deployed_environment.yml",

View File

@ -0,0 +1,8 @@
"""Dev."""
from os import environ
GIT_MERGE_BRANCH = environ.get("GIT_MERGE_BRANCH", default="staging")
GIT_USERNAME = environ.get("GIT_USERNAME", default="sartography-automated-committer")
GIT_USER_EMAIL = environ.get(
"GIT_USER_EMAIL", default="sartography-automated-committer@users.noreply.github.com"
)

View File

@ -5,8 +5,8 @@ from os import environ
environment_identifier_for_this_config_file_only = environ["SPIFFWORKFLOW_BACKEND_ENV"]
GIT_COMMIT_ON_SAVE = True
GIT_COMMIT_USERNAME = environment_identifier_for_this_config_file_only
GIT_COMMIT_EMAIL = f"{environment_identifier_for_this_config_file_only}@example.com"
GIT_USERNAME = environment_identifier_for_this_config_file_only
GIT_USER_EMAIL = f"{environment_identifier_for_this_config_file_only}@example.com"
SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME = environ.get(
"SPIFFWORKFLOW_BACKEND_PERMISSIONS_FILE_NAME",
default="terraform_deployed_environment.yml",

View File

@ -370,6 +370,20 @@ def process_model_move(
return make_response(jsonify(new_process_model), 201)
def process_model_publish(
modified_process_model_identifier: str, branch_to_update: Optional[str] = None
) -> flask.wrappers.Response:
"""Process_model_publish."""
if branch_to_update is None:
branch_to_update = current_app.config["GIT_MERGE_BRANCH"]
process_model_identifier = un_modify_modified_process_model_id(
modified_process_model_identifier
)
pr_url = GitService().publish(process_model_identifier, branch_to_update)
data = {"ok": True, "pr_url": pr_url}
return Response(json.dumps(data), status=200, mimetype="application/json")
def process_model_list(
process_group_identifier: Optional[str] = None,
recursive: Optional[bool] = False,

View File

@ -1,7 +1,8 @@
"""File_system_service."""
import os
from contextlib import contextmanager
from datetime import datetime
from typing import List
from typing import Generator, List
from typing import Optional
import pytz
@ -23,13 +24,25 @@ class FileSystemService:
PROCESS_GROUP_JSON_FILE = "process_group.json"
PROCESS_MODEL_JSON_FILE = "process_model.json"
# https://stackoverflow.com/a/24176022/6090676
@staticmethod
@contextmanager
def cd(newdir: str) -> Generator:
"""Cd."""
prevdir = os.getcwd()
os.chdir(os.path.expanduser(newdir))
try:
yield
finally:
os.chdir(prevdir)
@staticmethod
def root_path() -> str:
"""Root_path."""
# fixme: allow absolute files
dir_name = current_app.config["BPMN_SPEC_ABSOLUTE_DIR"]
app_root = current_app.root_path
return os.path.join(app_root, "..", dir_name)
return os.path.abspath(os.path.join(app_root, "..", dir_name))
@staticmethod
def id_string_to_relative_path(id_string: str) -> str:

View File

@ -1,7 +1,11 @@
"""Git_service."""
import os
import shutil
import uuid
from typing import Optional
from flask import current_app
from flask import g
from spiffworkflow_backend.models.process_model import ProcessModelInfo
from spiffworkflow_backend.services.file_system_service import FileSystemService
@ -40,17 +44,74 @@ class GitService:
return file_contents.encode("utf-8")
@staticmethod
def commit(message: str) -> str:
def commit(message: str, repo_path: Optional[str] = None) -> str:
"""Commit."""
bpmn_spec_absolute_dir = current_app.config["BPMN_SPEC_ABSOLUTE_DIR"]
repo_path_to_use = repo_path
if repo_path is None:
repo_path_to_use = current_app.config["BPMN_SPEC_ABSOLUTE_DIR"]
git_username = ""
git_email = ""
if (
current_app.config["GIT_COMMIT_USERNAME"]
and current_app.config["GIT_COMMIT_EMAIL"]
):
git_username = current_app.config["GIT_COMMIT_USERNAME"]
git_email = current_app.config["GIT_COMMIT_EMAIL"]
shell_command = f"./bin/git_commit_bpmn_models_repo '{bpmn_spec_absolute_dir}' '{message}' '{git_username}' '{git_email}'"
if current_app.config["GIT_USERNAME"] and current_app.config["GIT_USER_EMAIL"]:
git_username = current_app.config["GIT_USERNAME"]
git_email = current_app.config["GIT_USER_EMAIL"]
shell_command_path = os.path.join(
current_app.root_path, "..", "..", "bin", "git_commit_bpmn_models_repo"
)
shell_command = f"{shell_command_path} '{repo_path_to_use}' '{message}' '{git_username}' '{git_email}'"
output = os.popen(shell_command).read() # noqa: S605
return output
@classmethod
def publish(cls, process_model_id: str, branch_to_update: str) -> str:
"""Publish."""
source_process_model_root = FileSystemService.root_path()
source_process_model_path = os.path.join(
source_process_model_root, process_model_id
)
unique_hex = uuid.uuid4().hex
clone_dir = f"sample-process-models.{unique_hex}"
# clone new instance of sample-process-models, checkout branch_to_update
# we are adding a guid to this so the flake8 issue has been mitigated
destination_process_root = f"/tmp/{clone_dir}" # noqa
cmd = (
f"git clone https://{current_app.config['GIT_USERNAME']}:{current_app.config['GIT_USER_PASSWORD']}"
f"@github.com/sartography/sample-process-models.git {destination_process_root}"
)
os.system(cmd) # noqa: S605
with FileSystemService.cd(destination_process_root):
# create publish branch from branch_to_update
os.system(f"git checkout {branch_to_update}") # noqa: S605
publish_branch = f"publish-{process_model_id}"
command = f"git show-ref --verify refs/remotes/origin/{publish_branch}"
output = os.popen(command).read() # noqa: S605
if output:
os.system(f"git checkout {publish_branch}") # noqa: S605
else:
os.system(f"git checkout -b {publish_branch}") # noqa: S605
# copy files from process model into the new publish branch
destination_process_model_path = os.path.join(
destination_process_root, process_model_id
)
if os.path.exists(destination_process_model_path):
shutil.rmtree(destination_process_model_path)
shutil.copytree(source_process_model_path, destination_process_model_path)
# add and commit files to publish_branch, then push
commit_message = f"Request to publish changes to {process_model_id}, from {g.user.username}"
cls.commit(commit_message, destination_process_root)
os.system("git push") # noqa
# build url for github page to open PR
output = os.popen("git config --get remote.origin.url").read() # noqa
remote_url = output.strip().replace(".git", "")
pr_url = f"{remote_url}/compare/{publish_branch}?expand=1"
# try to clean up
if os.path.exists(destination_process_root):
shutil.rmtree(destination_process_root)
return pr_url

View File

@ -2555,6 +2555,123 @@ class TestProcessApi(BaseTest):
new_process_group = ProcessModelService.get_process_group(new_sub_path)
assert new_process_group.id == new_sub_path
def test_process_model_publish(
self,
app: Flask,
client: FlaskClient,
with_db_and_bpmn_file_cleanup: None,
with_super_admin_user: UserModel,
) -> None:
"""Test_process_model_publish."""
bpmn_root = FileSystemService.root_path()
shell_command = f"git init {bpmn_root}"
output = os.popen(shell_command).read() # noqa: S605
assert output == f"Initialized empty Git repository in {bpmn_root}/.git/\n"
os.chdir(bpmn_root)
output = os.popen("git status").read() # noqa: S605
assert "On branch main" in output
assert "No commits yet" in output
assert (
'nothing to commit (create/copy files and use "git add" to track)' in output
)
process_group_id = "test_group"
self.create_process_group(
client, with_super_admin_user, process_group_id, process_group_id
)
sub_process_group_id = "test_group/test_sub_group"
process_model_id = "hello_world"
bpmn_file_name = "hello_world.bpmn"
bpmn_file_location = "hello_world"
process_model_identifier = self.create_group_and_model_with_bpmn(
client=client,
user=with_super_admin_user,
process_group_id=sub_process_group_id,
process_model_id=process_model_id,
bpmn_file_name=bpmn_file_name,
bpmn_file_location=bpmn_file_location,
)
process_model_absolute_dir = os.path.join(bpmn_root, process_model_identifier)
output = os.popen("git status").read() # noqa: S605
test_string = 'Untracked files:\n (use "git add <file>..." to include in what will be committed)\n\ttest_group'
assert test_string in output
os.system("git add .")
output = os.popen("git commit -m 'Initial Commit'").read()
assert "Initial Commit" in output
assert "4 files changed" in output
assert "test_group/process_group.json" in output
assert "test_group/test_sub_group/hello_world/hello_world.bpmn" in output
assert "test_group/test_sub_group/hello_world/process_model.json" in output
assert "test_group/test_sub_group/process_group.json" in output
output = os.popen("git status").read() # noqa: S605
assert "On branch main" in output
assert "nothing to commit" in output
assert "working tree clean" in output
output = os.popen("git branch --list").read() # noqa: S605
assert output == "* main\n"
os.system("git branch staging")
output = os.popen("git branch --list").read() # noqa: S605
assert output == "* main\n staging\n"
os.system("git checkout staging")
output = os.popen("git status").read() # noqa: S605
assert "On branch staging" in output
assert "nothing to commit" in output
assert "working tree clean" in output
# process_model = ProcessModelService.get_process_model(process_model_identifier)
listing = os.listdir(process_model_absolute_dir)
assert len(listing) == 2
assert "hello_world.bpmn" in listing
assert "process_model.json" in listing
os.system("git checkout main")
output = os.popen("git status").read() # noqa: S605
assert "On branch main" in output
assert "nothing to commit" in output
assert "working tree clean" in output
file_data = b"abc123"
new_file_path = os.path.join(process_model_absolute_dir, "new_file.txt")
with open(new_file_path, "wb") as f_open:
f_open.write(file_data)
output = os.popen("git status").read() # noqa: S605
assert "On branch main" in output
assert "Untracked files:" in output
assert "test_group/test_sub_group/hello_world/new_file.txt" in output
os.system(
"git add test_group/test_sub_group/hello_world/new_file.txt"
) # noqa: S605
output = os.popen("git commit -m 'add new_file.txt'").read() # noqa: S605
assert "add new_file.txt" in output
assert "1 file changed, 1 insertion(+)" in output
assert "test_group/test_sub_group/hello_world/new_file.txt" in output
listing = os.listdir(process_model_absolute_dir)
assert len(listing) == 3
assert "hello_world.bpmn" in listing
assert "process_model.json" in listing
assert "new_file.txt" in listing
# modified_process_model_id = process_model_identifier.replace("/", ":")
# response = client.post(
# f"/v1.0/process-models/{modified_process_model_id}/publish?branch_to_update=staging",
# headers=self.logged_in_headers(with_super_admin_user),
# )
print("test_process_model_publish")
def test_can_get_process_instance_list_with_report_metadata(
self,
app: Flask,

View File

@ -29,4 +29,4 @@ cypress/screenshots
/test*.json
# Editors
.idea
.idea

View File

@ -41,4 +41,3 @@
-->
</body>
</html>

View File

@ -0,0 +1,48 @@
import React from 'react';
// @ts-ignore
import { Close, CheckmarkFilled } from '@carbon/icons-react';
// @ts-ignore
import { Button } from '@carbon/react';
type OwnProps = {
title: string;
children: React.ReactNode;
onClose: (..._args: any[]) => any;
type?: string;
};
export function Notification({
title,
children,
onClose,
type = 'success',
}: OwnProps) {
let iconClassName = 'green-icon';
if (type === 'error') {
iconClassName = 'red-icon';
}
return (
<div
role="status"
className={`with-bottom-margin cds--inline-notification cds--inline-notification--low-contrast cds--inline-notification--${type}`}
>
<div className="cds--inline-notification__details">
<div className="cds--inline-notification__text-wrapper">
<CheckmarkFilled className={`${iconClassName} notification-icon`} />
<div className="cds--inline-notification__title">{title}</div>
<div className="cds--inline-notification__subtitle">{children}</div>
</div>
</div>
<Button
data-qa="close-publish-notification"
renderIcon={Close}
iconDescription="Close Notification"
className="cds--inline-notification__close-button"
hasIconOnly
size="sm"
kind=""
onClick={onClose}
/>
</div>
);
}

View File

@ -22,7 +22,6 @@ import {
TableRow,
TimePicker,
Tag,
InlineNotification,
Stack,
Modal,
ComboBox,
@ -65,6 +64,7 @@ import ProcessModelSearch from './ProcessModelSearch';
import ProcessInstanceReportSearch from './ProcessInstanceReportSearch';
import ProcessInstanceListSaveAsReport from './ProcessInstanceListSaveAsReport';
import { FormatProcessModelDisplayName } from './MiniComponents';
import { Notification } from './Notification';
const REFRESH_INTERVAL = 5;
const REFRESH_TIMEOUT = 600;
@ -372,18 +372,16 @@ export default function ProcessInstanceListTable({
titleOperation = 'Created';
}
return (
<>
<InlineNotification
title={`Perspective ${titleOperation}:`}
subtitle={`'${
processInstanceReportSelection
? processInstanceReportSelection.identifier
: ''
}'`}
kind="success"
/>
<br />
</>
<Notification
title={`Perspective: ${titleOperation}`}
onClose={() => setProcessInstanceReportJustSaved(null)}
>
<span>{`'${
processInstanceReportSelection
? processInstanceReportSelection.identifier
: ''
}'`}</span>
</Notification>
);
}
return null;

View File

@ -18,6 +18,7 @@ export const useUriListForPermissions = () => {
processModelCreatePath: `/v1.0/process-models/${params.process_group_id}`,
processModelFileCreatePath: `/v1.0/process-models/${params.process_model_id}/files`,
processModelFileShowPath: `/v1.0/process-models/${params.process_model_id}/files/${params.file_name}`,
processModelPublishPath: `/v1.0/process-models/${params.process_model_id}/publish`,
processModelShowPath: `/v1.0/process-models/${params.process_model_id}`,
secretListPath: `/v1.0/secrets`,
};

View File

@ -332,6 +332,14 @@ td.actions-cell {
fill: red;
}
svg.green-icon {
fill: #198038;
}
svg.notification-icon {
margin-right: 1rem;
}
.failure-string {
color: red;
}
@ -358,7 +366,7 @@ td.actions-cell {
}
/* lime green */
.tag-type-green:hover {
background-color: #00FF00;
background-color: #80ee90;
}

View File

@ -48,6 +48,7 @@ import ProcessInstanceListTable from '../components/ProcessInstanceListTable';
import { usePermissionFetcher } from '../hooks/PermissionService';
import { useUriListForPermissions } from '../hooks/UriListForPermissions';
import ProcessInstanceRun from '../components/ProcessInstanceRun';
import { Notification } from '../components/Notification';
export default function ProcessModelShow() {
const params = useParams();
@ -60,11 +61,14 @@ export default function ProcessModelShow() {
const [filesToUpload, setFilesToUpload] = useState<any>(null);
const [showFileUploadModal, setShowFileUploadModal] =
useState<boolean>(false);
const [processModelPublished, setProcessModelPublished] = useState<any>(null);
const [publishDisabled, setPublishDisabled] = useState<boolean>(false);
const navigate = useNavigate();
const { targetUris } = useUriListForPermissions();
const permissionRequestData: PermissionsToCheck = {
[targetUris.processModelShowPath]: ['PUT', 'DELETE'],
[targetUris.processModelPublishPath]: ['POST'],
[targetUris.processInstanceListPath]: ['GET'],
[targetUris.processInstanceCreatePath]: ['POST'],
[targetUris.processModelFileCreatePath]: ['POST', 'PUT', 'GET', 'DELETE'],
@ -91,19 +95,17 @@ export default function ProcessModelShow() {
const processInstanceRunResultTag = () => {
if (processInstance) {
return (
<div className="alert alert-success with-top-margin" role="alert">
<p>
Process Instance {processInstance.id} kicked off (
<Link
to={`/admin/process-instances/${modifiedProcessModelId}/${processInstance.id}`}
data-qa="process-instance-show-link"
>
view
</Link>
).
</p>
<br />
</div>
<Notification
title="Process Instance Kicked Off:"
onClose={() => setProcessInstance(null)}
>
<Link
to={`/admin/process-instances/${modifiedProcessModelId}/${processInstance.id}`}
data-qa="process-instance-show-link"
>
view
</Link>
</Notification>
);
}
return null;
@ -203,6 +205,21 @@ export default function ProcessModelShow() {
});
};
const postPublish = (value: any) => {
setPublishDisabled(false);
setProcessModelPublished(value);
};
const publishProcessModel = () => {
setPublishDisabled(true);
setProcessModelPublished(null);
HttpService.makeCallToBackend({
path: `/process-models/${modifiedProcessModelId}/publish`,
successCallback: postPublish,
httpMethod: 'POST',
});
};
const navigateToFileEdit = (processModelFile: ProcessFile) => {
const url = profileModelFileEditUrl(processModelFile);
if (url) {
@ -510,6 +527,23 @@ export default function ProcessModelShow() {
return null;
};
const processModelPublishMessage = () => {
if (processModelPublished) {
const prUrl: string = processModelPublished.pr_url;
return (
<Notification
title="Model Published:"
onClose={() => setProcessModelPublished(false)}
>
<a href={prUrl} target="_void()">
view the changes and create a Pull Request
</a>
</Notification>
);
}
return null;
};
if (processModel) {
return (
<>
@ -523,6 +557,8 @@ export default function ProcessModelShow() {
},
]}
/>
{processModelPublishMessage()}
{processInstanceRunResultTag()}
<Stack orientation="horizontal" gap={1}>
<h1 className="with-icons">
Process Model: {processModel.display_name}
@ -568,8 +604,16 @@ export default function ProcessModelShow() {
<br />
</>
</Can>
<Can
I="POST"
a={targetUris.processModelPublishPath}
ability={ability}
>
<Button disabled={publishDisabled} onClick={publishProcessModel}>
Publish Changes
</Button>
</Can>
</Stack>
{processInstanceRunResultTag()}
{processModelFilesSection()}
<Can I="GET" a={targetUris.processInstanceListPath} ability={ability}>
{processInstanceListTableButton()}