From ac0365da78034101b54877b9aa30df5bbba20e72 Mon Sep 17 00:00:00 2001 From: Dan Date: Thu, 29 Sep 2022 17:55:45 -0400 Subject: [PATCH] Basic ability to add and list items in DynamoDB, but there is a whole slew of additional things to consider. * Catch and return errors as json (otherwise the backend blows up) * Added Boto3 (AWS library) as a dependency * Not sure if it's right, but tried to divide the auth and commands. --- .gitignore | 3 + app.py | 18 ++- connectors/connector-aws/.gitignore | 129 ++++++++++++++++++ .../connector-aws/connector_aws/__init__.py | 0 .../connector_aws/auths/simpleAuth.py | 30 ++++ .../connector_aws/commands/__init__.py | 0 .../connector_aws/commands/addDynamoItem.py | 39 ++++++ .../commands/queryDynamoTable.py | 31 +++++ .../connector_aws/commands/scanDynamoTable.py | 28 ++++ .../connector_aws/commands/uploadFile.py | 45 ++++++ connectors/connector-aws/pyproject.toml | 15 ++ poetry.lock | 107 ++++++++++++++- pyproject.toml | 1 + 13 files changed, 441 insertions(+), 5 deletions(-) create mode 100644 connectors/connector-aws/.gitignore create mode 100644 connectors/connector-aws/connector_aws/__init__.py create mode 100644 connectors/connector-aws/connector_aws/auths/simpleAuth.py create mode 100644 connectors/connector-aws/connector_aws/commands/__init__.py create mode 100644 connectors/connector-aws/connector_aws/commands/addDynamoItem.py create mode 100644 connectors/connector-aws/connector_aws/commands/queryDynamoTable.py create mode 100644 connectors/connector-aws/connector_aws/commands/scanDynamoTable.py create mode 100644 connectors/connector-aws/connector_aws/commands/uploadFile.py create mode 100644 connectors/connector-aws/pyproject.toml diff --git a/.gitignore b/.gitignore index 5f0776c..5d6fcc9 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,6 @@ dmypy.json # Pyre type checker .pyre/ + +# IDEs +.idea \ No newline at end of file diff --git a/app.py b/app.py index 1df7b34..44123b6 100644 --- a/app.py +++ b/app.py @@ -93,12 +93,24 @@ def auth_callback(plugin_display_name, auth_name): def do_command(plugin_display_name, command_name): command = PluginService.command_named(plugin_display_name, command_name) if command is None: - return Response('Command not found', status=404) + return json_error_response(f'Command not found: {plugin_display_name}:{command_name}', status=404) params = request.args.to_dict() - result = command(**params).execute() + try: + result = command(**params).execute() + except Exception as e: + return json_error_response(f'Error encountered when executing {plugin_display_name}:{command_name} {str(e)}', + status=404) + + return Response(result['response'], mimetype=result['mimetype'], status=200) + +def json_error_response(message, status): + resp = { + 'error':message, + 'status':status + } + return Response(json.dumps(resp), status=status) - return Response(result['response'], status=result['status'], mimetype=result['mimetype']) # TODO move out to own home import importlib diff --git a/connectors/connector-aws/.gitignore b/connectors/connector-aws/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/connectors/connector-aws/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/connectors/connector-aws/connector_aws/__init__.py b/connectors/connector-aws/connector_aws/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/connectors/connector-aws/connector_aws/auths/simpleAuth.py b/connectors/connector-aws/connector_aws/auths/simpleAuth.py new file mode 100644 index 0000000..41c5a7d --- /dev/null +++ b/connectors/connector-aws/connector_aws/auths/simpleAuth.py @@ -0,0 +1,30 @@ +import boto3 +from botocore.config import Config + + +class SimpleAuth: + """Established a simple Boto 3 Client based on an access key and a secret key""" + + def __init__(self, resource_type: str, access_key: str, secret_key: str): + """ + :param access_key: AWS Access Key + :param secret_key: AWS Secret Key + """ + + my_config = Config( + region_name='us-east-1', + signature_version='v4', + retries={ + 'max_attempts': 10, + 'mode': 'standard' + } + ) + + # Get the service resource. + self.resource = boto3.resource(resource_type, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=my_config) + + def get_resource(self): + return self.resource diff --git a/connectors/connector-aws/connector_aws/commands/__init__.py b/connectors/connector-aws/connector_aws/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/connectors/connector-aws/connector_aws/commands/addDynamoItem.py b/connectors/connector-aws/connector_aws/commands/addDynamoItem.py new file mode 100644 index 0000000..4c670d7 --- /dev/null +++ b/connectors/connector-aws/connector_aws/commands/addDynamoItem.py @@ -0,0 +1,39 @@ +import json + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from connector_aws.auths.simpleAuth import SimpleAuth + + +class AddDynamoItem: + """Add a new record to a dynamo db table.""" + + def __init__(self, access_key: str, secret_key: str, table_name: str, item_data: str): + """ + :param access_key: AWS Access Key + :param secret_key: AWS Secret Key + :param table_name: The name of hte Dynamo DB table to add information to. + :param item_data: The data to add + :return: Json Data structure containing a http status code (hopefully '200' for success..) + and a response string. + """ + # Get the service resource. + self.dynamodb = SimpleAuth('dynamodb', access_key, secret_key).get_resource() + + + # Instantiate a table resource object without actually + # creating a DynamoDB table. Note that the attributes of this table + # are lazy-loaded: a request is not made nor are the attribute + # values populated until the attributes + # on the table resource are accessed or its load() method is called. + self.table = self.dynamodb.Table(table_name) + self.item_data = json.loads(item_data) + + def execute(self): + result = self.table.put_item(Item=self.item_data) + if 'ResponseMetadata' in result: + del result['ResponseMetadata'] + result_str = json.dumps(result) + return dict(response=result_str, mimetype='application/json') diff --git a/connectors/connector-aws/connector_aws/commands/queryDynamoTable.py b/connectors/connector-aws/connector_aws/commands/queryDynamoTable.py new file mode 100644 index 0000000..bd6b4a7 --- /dev/null +++ b/connectors/connector-aws/connector_aws/commands/queryDynamoTable.py @@ -0,0 +1,31 @@ +import json + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from connector_aws.auths.simpleAuth import SimpleAuth + + +class QueryDynamoTable: + """Return all records for a given partition key""" + + def __init__(self, access_key: str, secret_key: str, table_name: str, key: str): + """ + :param access_key: AWS Access Key + :param secret_key: AWS Secret Key + :param table_name: The name of hte Dynamo DB table to add information to. + :param key: The partition key for what to return. + :return: Json Data structure containing the requested data. + """ + + self.dynamodb = SimpleAuth('dynamodb', access_key, secret_key).get_resource() + self.table = self.dynamodb.Table(table_name) + self.key = key + + def execute(self): + result = self.table.get_item(Key={"primaryKeyName": self.key}) + if 'ResponseMetadata' in result: + del result['ResponseMetadata'] + result_str = json.dumps(result) + return dict(response=result_str, mimetype='application/json') diff --git a/connectors/connector-aws/connector_aws/commands/scanDynamoTable.py b/connectors/connector-aws/connector_aws/commands/scanDynamoTable.py new file mode 100644 index 0000000..5dc0652 --- /dev/null +++ b/connectors/connector-aws/connector_aws/commands/scanDynamoTable.py @@ -0,0 +1,28 @@ +import json + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from connector_aws.auths.simpleAuth import SimpleAuth + + +class ScanDynamoTable: + """Return all records in a given table. Potentially very expensive.""" + + def __init__(self, access_key: str, secret_key: str, table_name: str): + """ + :param access_key: AWS Access Key + :param secret_key: AWS Secret Key + :param table_name: The name of hte Dynamo DB table to scan + :return: Json Data structure containing the requested data. + """ + self.dynamodb = SimpleAuth('dynamodb', access_key, secret_key).get_resource() + self.table = self.dynamodb.Table(table_name) + + def execute(self): + result = self.table.scan() + if 'ResponseMetadata' in result: + del result['ResponseMetadata'] + result_str = json.dumps(result) + return dict(response=result_str, mimetype='application/json') diff --git a/connectors/connector-aws/connector_aws/commands/uploadFile.py b/connectors/connector-aws/connector_aws/commands/uploadFile.py new file mode 100644 index 0000000..d9b67bf --- /dev/null +++ b/connectors/connector-aws/connector_aws/commands/uploadFile.py @@ -0,0 +1,45 @@ +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from connector_aws.auths.simpleAuth import SimpleAuth + + +class UploadFile: + """ AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY need to be set in the environment for + BOTO3 to make the correct call. """ + + def __init__(self, access_key: str, secret_key: str, + file_name: str, file_data:bytes, bucket: str, object_name: str): + """ + :param access_key: AWS Access Key + :param secret_key: AWS Secret Key + :param file_name: File to upload + :param file_data: Contents of file to be uploaded + :param bucket: Bucket to upload to + :param object_name: S3 object name. If not specified then file_name is used + :return: Json Data structure containing a http status code (hopefully '200' for success..) + and a response string. + """ + self.client = SimpleAuth('kinesis', access_key, secret_key).get_resource() + self.file_name = file_name + self.file_data = file_data + self.bucket = bucket + self.object_name = object_name + + def execute(self): + + # If S3 object_name was not specified, use file_name + if self.object_name is None: + self.object_name = self.file_name + + # Upload the file + try: + response = self.client.upload_file(self.file_name, self.bucket, self.object_name) + except ClientError as e: + response = f'{ "error": "AWS Excetion {e}" }' + return { + 'response': 'success', + 'status': response.status_code, + 'mimetype': 'application/json' + } diff --git a/connectors/connector-aws/pyproject.toml b/connectors/connector-aws/pyproject.toml new file mode 100644 index 0000000..3568025 --- /dev/null +++ b/connectors/connector-aws/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "connector-aws" +version = "0.1.0" +description = "" +authors = ["Dan Funk "] + +[tool.poetry.dependencies] +python = "^3.10" +boto3 = "^1.27.81" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/poetry.lock b/poetry.lock index b1e2d9b..acd3c82 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,35 @@ +[[package]] +name = "boto3" +version = "1.24.81" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.27.81,<1.28.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.27.81" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.14.0)"] + [[package]] name = "cachelib" version = "0.9.0" @@ -44,6 +76,22 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "connector-aws" +version = "0.1.0" +description = "" +category = "main" +optional = false +python-versions = "^3.10" +develop = true + +[package.dependencies] +boto3 = "^1.24.80" + +[package.source] +type = "directory" +url = "connectors/connector-aws" + [[package]] name = "connector-bamboohr" version = "0.1.0" @@ -60,6 +108,22 @@ requests = "^2.28.1" type = "directory" url = "connectors/connector-bamboohr" +[[package]] +name = "connector-waku" +version = "0.1.0" +description = "" +category = "main" +optional = false +python-versions = "^3.10" +develop = true + +[package.dependencies] +requests = "^2.28.1" + +[package.source] +type = "directory" +url = "connectors/connector-waku" + [[package]] name = "connector-xero" version = "0.1.0" @@ -167,6 +231,14 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "markupsafe" version = "2.1.1" @@ -233,6 +305,20 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=2.1.0,<3.0.0)"] +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "setuptools" version = "65.3.0" @@ -297,9 +383,17 @@ urllib3 = "*" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "0eb091f1ca8474234eb6645715477ea69507e2a66773cd0669f53b85d5054136" +content-hash = "2f3ea439d480007afde2a86e758449d71e9ac3708b13573cf9d475d84f31c39e" [metadata.files] +boto3 = [ + {file = "boto3-1.24.81-py3-none-any.whl", hash = "sha256:a84f11ba5d369ee4bb3e6d4c9595fe05c8c804ab9b817e578f87d956b803dcfc"}, + {file = "boto3-1.24.81.tar.gz", hash = "sha256:75defbacdeb48b7fb321c2e283bc57b270595467e873c401b7914a79efd372c7"}, +] +botocore = [ + {file = "botocore-1.27.81-py3-none-any.whl", hash = "sha256:ad789bfc36ade270671c6846e314193c1968cc3523828aec1e12d012c900652f"}, + {file = "botocore-1.27.81.tar.gz", hash = "sha256:b6b54560b110666e6f0248c0d39e0588589410186c35f4cee44be847d83fec07"}, +] cachelib = [ {file = "cachelib-0.9.0-py3-none-any.whl", hash = "sha256:811ceeb1209d2fe51cd2b62810bd1eccf70feba5c52641532498be5c675493b3"}, {file = "cachelib-0.9.0.tar.gz", hash = "sha256:38222cc7c1b79a23606de5c2607f4925779e37cdcea1c2ad21b8bae94b5425a5"}, @@ -320,7 +414,9 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] +connector-aws = [] connector-bamboohr = [] +connector-waku = [] connector-xero = [] Flask = [ {file = "Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526"}, @@ -350,6 +446,10 @@ jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] markupsafe = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, @@ -407,7 +507,10 @@ requests = [ requests-oauthlib = [ {file = "requests-oauthlib-1.1.0.tar.gz", hash = "sha256:eabd8eb700ebed81ba080c6ead96d39d6bdc39996094bd23000204f6965786b0"}, {file = "requests_oauthlib-1.1.0-py2.py3-none-any.whl", hash = "sha256:be76f2bb72ca5525998e81d47913e09b1ca8b7957ae89b46f787a79e68ad5e61"}, - {file = "requests_oauthlib-1.1.0-py3.7.egg", hash = "sha256:490229d14a98e1b69612dcc1a22887ec14f5487dc1b8c6d7ba7f77a42ce7347b"}, +] +s3transfer = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, ] setuptools = [ {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, diff --git a/pyproject.toml b/pyproject.toml index e85e26d..93cb64d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ Flask = "^2.2.2" connector-xero = {develop=true, path="connectors/connector-xero"} connector-bamboohr = {develop=true, path="connectors/connector-bamboohr"} connector-waku = {develop=true, path="connectors/connector-waku"} +connector-aws = {develop=true, path="connectors/connector-aws"} gunicorn = "^20.1.0" Flask-OAuthlib = "^0.9.6" Flask-Session = "^0.4.0"