Protect against loading files without actually injesting them by reserving the commit to execute on all records, then calling a delete.
Add flask executor so that long running tasks do not get killed when called from the API endpoints. Allow passing a specific file when calling the notification endpoints so that we can send out notifications to only those individuals that are included in a specific import file from IVY.
This commit is contained in:
Dan
parent
5d9dbc12fe
commit
cf7607f2d3
1
Pipfile
1
Pipfile
|
|
@ -23,6 +23,7 @@ flask-migrate = "*"
|
|||
flask-restful = "*"
|
||||
flask-wtf = "*"
|
||||
flask-table = "*"
|
||||
flask-executor = "*"
|
||||
marshmallow = "*"
|
||||
marshmallow-enum = "*"
|
||||
marshmallow-sqlalchemy = "*"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "2c83271e4be449f1a2909054b8eb2c2e4851746b0d8cfa8283712bdadd3ad89e"
|
||||
"sha256": "8c3959650ad1414809faff06b0d2e9b842b633b97e8954ecfa403f68380510b7"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -74,11 +74,11 @@
|
|||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7",
|
||||
"sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8",
|
||||
"sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"
|
||||
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
|
||||
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
|
||||
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
|
||||
],
|
||||
"version": "==4.9.1"
|
||||
"version": "==4.9.3"
|
||||
},
|
||||
"blinker": {
|
||||
"hashes": [
|
||||
|
|
@ -157,10 +157,10 @@
|
|||
},
|
||||
"clickclick": {
|
||||
"hashes": [
|
||||
"sha256:4a890aaa9c3990cfabd446294eb34e3dc89701101ac7b41c1bff85fc210f6d23",
|
||||
"sha256:ab8f229fb9906a86634bdfc6fabfc2b665f44804170720db4f6e1d98f8a58f3d"
|
||||
"sha256:4efb13e62353e34c5eef7ed6582c4920b418d7dedc86d819e22ee089ba01802c",
|
||||
"sha256:c8f33e6d9ec83f68416dd2136a7950125bd256ec39ccc9a85c6e280a16be2bb5"
|
||||
],
|
||||
"version": "==1.2.2"
|
||||
"version": "==20.10.2"
|
||||
},
|
||||
"connexion": {
|
||||
"extras": [
|
||||
|
|
@ -280,11 +280,11 @@
|
|||
},
|
||||
"docxtpl": {
|
||||
"hashes": [
|
||||
"sha256:0e031ea5da63339f2bac0fd7eb7b3b137303571a9a92c950501148240ea22047",
|
||||
"sha256:45f04661b9ab1fd66b975a0a547b30c8811f457bef2f85249c2f3c5784a00052"
|
||||
"sha256:08f2b566092472ac41fdb0d83eb964dc50914df41305d07b068e388570ab10ea",
|
||||
"sha256:56e26f656cac565457c91e7fc094333c364f1b1b7aa435d9191eb4c6ee8b7cab"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.10.0"
|
||||
"version": "==0.10.5"
|
||||
},
|
||||
"flask": {
|
||||
"hashes": [
|
||||
|
|
@ -331,6 +331,14 @@
|
|||
"index": "pypi",
|
||||
"version": "==3.0.9"
|
||||
},
|
||||
"flask-executor": {
|
||||
"hashes": [
|
||||
"sha256:31a3a04bc2e4828ee3b6bd58aba6939c0aeec94ddab455ccc76993d20b281bb1",
|
||||
"sha256:a36ae8304f0f26c7b0b5341c9faf8e524c4035fb14b5100302400fed1245cec2"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.9.3"
|
||||
},
|
||||
"flask-mail": {
|
||||
"hashes": [
|
||||
"sha256:22e5eb9a940bf407bcf30410ecc3708f3c56cc44b29c34e1726fe85006935f41"
|
||||
|
|
@ -340,11 +348,11 @@
|
|||
},
|
||||
"flask-marshmallow": {
|
||||
"hashes": [
|
||||
"sha256:1da1e6454a56a3e15107b987121729f152325bdef23f3df2f9b52bbd074af38e",
|
||||
"sha256:aefc1f1d96256c430a409f08241bab75ffe97e5d14ac5d1f000764e39bf4873a"
|
||||
"sha256:2adcd782b5a4a6c5ae3c96701f320d8ca6997995a52b2661093c56cc3ed24754",
|
||||
"sha256:bd01a6372cbe50e36f205cfff0fc5dab0b7b662c4c8b2c4fc06a3151b2950950"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.13.0"
|
||||
"version": "==0.14.0"
|
||||
},
|
||||
"flask-migrate": {
|
||||
"hashes": [
|
||||
|
|
@ -356,6 +364,7 @@
|
|||
},
|
||||
"flask-paginate": {
|
||||
"hashes": [
|
||||
"sha256:949b93d0535d1223b91ac0048586bd878aaebf4044c54c1dc3068acc9bdf441f",
|
||||
"sha256:f69434e9a8d85e50bd248c52bd8b22b0ca0284edef45d3c51b3ab5e4f703b733"
|
||||
],
|
||||
"index": "pypi",
|
||||
|
|
@ -404,24 +413,24 @@
|
|||
"grpc"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:67e33a852dcca7cb7eff49abc35c8cc2c0bb8ab11397dc8306d911505cae2990",
|
||||
"sha256:779107f17e0fef8169c5239d56a8fbff03f9f72a3893c0c9e5842ec29dfedd54"
|
||||
"sha256:15e00ceb7e6dc44159e2a41a222830744e9ebcb3a553c580b61cb5a66572f2f0",
|
||||
"sha256:4a9d7ac2527a9e298eebb580a5e24e7e41d6afd97010848dd0f306cae198ec1a"
|
||||
],
|
||||
"version": "==1.22.2"
|
||||
"version": "==1.22.4"
|
||||
},
|
||||
"google-auth": {
|
||||
"hashes": [
|
||||
"sha256:31941bf019fb242c04d0de32845da10180788bfddb0de87d78c4bdf55555dda1",
|
||||
"sha256:873051a6317294b083795cffc467bcd05b6df483ef542bfe0069ddbfbac0a096"
|
||||
"sha256:712dd7d140a9a1ea218e5688c7fcb04af71b431a29ec9ce433e384c60e387b98",
|
||||
"sha256:9c0f71789438d703f77b94aad4ea545afaec9a65f10e6cc1bc8b89ce242244bb"
|
||||
],
|
||||
"version": "==1.21.3"
|
||||
"version": "==1.22.1"
|
||||
},
|
||||
"google-cloud-core": {
|
||||
"hashes": [
|
||||
"sha256:4c9e457fcfc026fdde2e492228f04417d4c717fb0f29f070122fb0ab89e34ebd",
|
||||
"sha256:613e56f164b6bee487dd34f606083a0130f66f42f7b10f99730afdf1630df507"
|
||||
"sha256:21afb70c1b0bce8eeb8abb5dca63c5fd37fc8aea18f4b6d60e803bd3d27e6b80",
|
||||
"sha256:75abff9056977809937127418323faa3917f32df68490704d39a4f0d492ebc2b"
|
||||
],
|
||||
"version": "==1.4.1"
|
||||
"version": "==1.4.3"
|
||||
},
|
||||
"google-cloud-firestore": {
|
||||
"hashes": [
|
||||
|
|
@ -648,11 +657,11 @@
|
|||
},
|
||||
"phonenumbers": {
|
||||
"hashes": [
|
||||
"sha256:b8644c1dccd45d4c0f54c5b10effcc8c3f733e6a3c2caf40c9175fabc5010ffe",
|
||||
"sha256:f887eceb3d9db17ec479a85245bd0ebe74c5f43489217784215ffb231f8c9e88"
|
||||
"sha256:17f39f06c1e0e20eabe69ff735b1c08e4547d12a12595da3d835fd3256a9ee0c",
|
||||
"sha256:f5277ce92ac8813cb1f4174c6d1ee1fd08d563da28f9dec1f4bed0031a780a80"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==8.12.9"
|
||||
"version": "==8.12.11"
|
||||
},
|
||||
"protobuf": {
|
||||
"hashes": [
|
||||
|
|
@ -689,6 +698,7 @@
|
|||
"sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
|
||||
"sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
|
||||
"sha256:833709a5c66ca52f1d21d41865a637223b368c0ee76ea54ca5bad6f2526c7679",
|
||||
"sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83",
|
||||
"sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77",
|
||||
"sha256:950bc22bb56ee6ff142a2cb9ee980b571dd0912b0334aa3fe0fe3788d860bea2",
|
||||
"sha256:a0c50db33c32594305b0ef9abc0cb7db13de7621d2cadf8392a1d9b3c437ef77",
|
||||
|
|
@ -864,44 +874,51 @@
|
|||
"sha256:1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55",
|
||||
"sha256:a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232"
|
||||
],
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==2.0.1"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:072766c3bd09294d716b2d114d46ffc5ccf8ea0b714a4e1c48253014b771c6bb",
|
||||
"sha256:107d4af989831d7b091e382d192955679ec07a9209996bf8090f1f539ffc5804",
|
||||
"sha256:15c0bcd3c14f4086701c33a9e87e2c7ceb3bcb4a246cd88ec54a49cf2a5bd1a6",
|
||||
"sha256:26c5ca9d09f0e21b8671a32f7d83caad5be1f6ff45eef5ec2f6fd0db85fc5dc0",
|
||||
"sha256:276936d41111a501cf4a1a0543e25449108d87e9f8c94714f7660eaea89ae5fe",
|
||||
"sha256:3292a28344922415f939ee7f4fc0c186f3d5a0bf02192ceabd4f1129d71b08de",
|
||||
"sha256:33d29ae8f1dc7c75b191bb6833f55a19c932514b9b5ce8c3ab9bc3047da5db36",
|
||||
"sha256:3bba2e9fbedb0511769780fe1d63007081008c5c2d7d715e91858c94dbaa260e",
|
||||
"sha256:465c999ef30b1c7525f81330184121521418a67189053bcf585824d833c05b66",
|
||||
"sha256:51064ee7938526bab92acd049d41a1dc797422256086b39c08bafeffb9d304c6",
|
||||
"sha256:5a49e8473b1ab1228302ed27365ea0fadd4bf44bc0f9e73fe38e10fdd3d6b4fc",
|
||||
"sha256:618db68745682f64cedc96ca93707805d1f3a031747b5a0d8e150cfd5055ae4d",
|
||||
"sha256:6547b27698b5b3bbfc5210233bd9523de849b2bb8a0329cd754c9308fc8a05ce",
|
||||
"sha256:6557af9e0d23f46b8cd56f8af08eaac72d2e3c632ac8d5cf4e20215a8dca7cea",
|
||||
"sha256:73a40d4fcd35fdedce07b5885905753d5d4edf413fbe53544dd871f27d48bd4f",
|
||||
"sha256:8280f9dae4adb5889ce0bb3ec6a541bf05434db5f9ab7673078c00713d148365",
|
||||
"sha256:83469ad15262402b0e0974e612546bc0b05f379b5aa9072ebf66d0f8fef16bea",
|
||||
"sha256:860d0fe234922fd5552b7f807fbb039e3e7ca58c18c8d38aa0d0a95ddf4f6c23",
|
||||
"sha256:883c9fb62cebd1e7126dd683222b3b919657590c3e2db33bdc50ebbad53e0338",
|
||||
"sha256:8afcb6f4064d234a43fea108859942d9795c4060ed0fbd9082b0f280181a15c1",
|
||||
"sha256:96f51489ac187f4bab588cf51f9ff2d40b6d170ac9a4270ffaed535c8404256b",
|
||||
"sha256:9e865835e36dfbb1873b65e722ea627c096c11b05f796831e3a9b542926e979e",
|
||||
"sha256:aa0554495fe06172b550098909be8db79b5accdf6ffb59611900bea345df5eba",
|
||||
"sha256:b595e71c51657f9ee3235db8b53d0b57c09eee74dfb5b77edff0e46d2218dc02",
|
||||
"sha256:b6ff91356354b7ff3bd208adcf875056d3d886ed7cef90c571aef2ab8a554b12",
|
||||
"sha256:b70bad2f1a5bd3460746c3fb3ab69e4e0eb5f59d977a23f9b66e5bdc74d97b86",
|
||||
"sha256:c7adb1f69a80573698c2def5ead584138ca00fff4ad9785a4b0b2bf927ba308d",
|
||||
"sha256:c898b3ebcc9eae7b36bd0b4bbbafce2d8076680f6868bcbacee2d39a7a9726a7",
|
||||
"sha256:e49947d583fe4d29af528677e4f0aa21f5e535ca2ae69c48270ebebd0d8843c0",
|
||||
"sha256:eb1d71643e4154398b02e88a42fc8b29db8c44ce4134cf0f4474bfc5cb5d4dac",
|
||||
"sha256:f2e8a9c0c8813a468aa659a01af6592f71cd30237ec27c4cc0683f089f90dcfc",
|
||||
"sha256:fe7fe11019fc3e6600819775a7d55abc5446dda07e9795f5954fdbf8a49e1c37"
|
||||
"sha256:009e8388d4d551a2107632921320886650b46332f61dc935e70c8bcf37d8e0d6",
|
||||
"sha256:0157c269701d88f5faf1fa0e4560e4d814f210c01a5b55df3cab95e9346a8bcc",
|
||||
"sha256:0a92745bb1ebbcb3985ed7bda379b94627f0edbc6c82e9e4bac4fb5647ae609a",
|
||||
"sha256:0cca1844ba870e81c03633a99aa3dc62256fb96323431a5dec7d4e503c26372d",
|
||||
"sha256:166917a729b9226decff29416f212c516227c2eb8a9c9f920d69ced24e30109f",
|
||||
"sha256:1f5f369202912be72fdf9a8f25067a5ece31a2b38507bb869306f173336348da",
|
||||
"sha256:2909dffe5c9a615b7e6c92d1ac2d31e3026dc436440a4f750f4749d114d88ceb",
|
||||
"sha256:2b5dafed97f778e9901b79cc01b88d39c605e0545b4541f2551a2fd785adc15b",
|
||||
"sha256:2e9bd5b23bba8ae8ce4219c9333974ff5e103c857d9ff0e4b73dc4cb244c7d86",
|
||||
"sha256:3aa6d45e149a16aa1f0c46816397e12313d5e37f22205c26e06975e150ffcf2a",
|
||||
"sha256:4bdbdb8ca577c6c366d15791747c1de6ab14529115a2eb52774240c412a7b403",
|
||||
"sha256:53fd857c6c8ffc0aa6a5a3a2619f6a74247e42ec9e46b836a8ffa4abe7aab327",
|
||||
"sha256:5cdfe54c1e37279dc70d92815464b77cd8ee30725adc9350f06074f91dbfeed2",
|
||||
"sha256:5d92c18458a4aa27497a986038d5d797b5279268a2de303cd00910658e8d149c",
|
||||
"sha256:632b32183c0cb0053194a4085c304bc2320e5299f77e3024556fa2aa395c2a8b",
|
||||
"sha256:7c735c7a6db8ee9554a3935e741cf288f7dcbe8706320251eb38c412e6a4281d",
|
||||
"sha256:7cd40cb4bc50d9e87b3540b23df6e6b24821ba7e1f305c1492b0806c33dbdbec",
|
||||
"sha256:84f0ac4a09971536b38cc5d515d6add7926a7e13baa25135a1dbb6afa351a376",
|
||||
"sha256:8dcbf377529a9af167cbfc5b8acec0fadd7c2357fc282a1494c222d3abfc9629",
|
||||
"sha256:950f0e17ffba7a7ceb0dd056567bc5ade22a11a75920b0e8298865dc28c0eff6",
|
||||
"sha256:9e379674728f43a0cd95c423ac0e95262500f9bfd81d33b999daa8ea1756d162",
|
||||
"sha256:b15002b9788ffe84e42baffc334739d3b68008a973d65fad0a410ca5d0531980",
|
||||
"sha256:b6f036ecc017ec2e2cc2a40615b41850dc7aaaea6a932628c0afc73ab98ba3fb",
|
||||
"sha256:bad73f9888d30f9e1d57ac8829f8a12091bdee4949b91db279569774a866a18e",
|
||||
"sha256:bbc58fca72ce45a64bb02b87f73df58e29848b693869e58bd890b2ddbb42d83b",
|
||||
"sha256:bca4d367a725694dae3dfdc86cf1d1622b9f414e70bd19651f5ac4fb3aa96d61",
|
||||
"sha256:be41d5de7a8e241864189b7530ca4aaf56a5204332caa70555c2d96379e18079",
|
||||
"sha256:bf53d8dddfc3e53a5bda65f7f4aa40fae306843641e3e8e701c18a5609471edf",
|
||||
"sha256:c092fe282de83d48e64d306b4bce03114859cdbfe19bf8a978a78a0d44ddadb1",
|
||||
"sha256:c3ab23ee9674336654bf9cac30eb75ac6acb9150dc4b1391bec533a7a4126471",
|
||||
"sha256:ce64a44c867d128ab8e675f587aae7f61bd2db836a3c4ba522d884cd7c298a77",
|
||||
"sha256:d05cef4a164b44ffda58200efcb22355350979e000828479971ebca49b82ddb1",
|
||||
"sha256:d2f25c7f410338d31666d7ddedfa67570900e248b940d186b48461bd4e5569a1",
|
||||
"sha256:d3b709d64b5cf064972b3763b47139e4a0dc4ae28a36437757f7663f67b99710",
|
||||
"sha256:e32e3455db14602b6117f0f422f46bc297a3853ae2c322ecd1e2c4c04daf6ed5",
|
||||
"sha256:ed53209b5f0f383acb49a927179fa51a6e2259878e164273ebc6815f3a752465",
|
||||
"sha256:f605f348f4e6a2ba00acb3399c71d213b92f27f2383fc4abebf7a37368c12142",
|
||||
"sha256:fcdb3755a7c355bc29df1b5e6fb8226d5c8b90551d202d69d0076a8a5649d68b"
|
||||
],
|
||||
"version": "==1.3.19"
|
||||
"version": "==1.3.20"
|
||||
},
|
||||
"swagger-ui-bundle": {
|
||||
"hashes": [
|
||||
|
|
@ -919,10 +936,10 @@
|
|||
},
|
||||
"twilio": {
|
||||
"hashes": [
|
||||
"sha256:df1cf8f7e62fbe4d412e66204ee7f948cd31ff18173ff4690475834174eedaf0"
|
||||
"sha256:9d591617b22e75b26cda11a10d353e2001d990a7ca1696d92e50abfc6ecdcb73"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==6.45.3"
|
||||
"version": "==6.46.0"
|
||||
},
|
||||
"tzlocal": {
|
||||
"hashes": [
|
||||
|
|
@ -983,10 +1000,10 @@
|
|||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:43f4fa8d8bb313e65d8323a3952ef8756bf40f9a5c3ea7334be23ee4ec8278b6",
|
||||
"sha256:b52f22895f4cfce194bc8172f3819ee8de7540aa6d873535a8668b730b8b411f"
|
||||
"sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b",
|
||||
"sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066"
|
||||
],
|
||||
"version": "==3.2.0"
|
||||
"version": "==3.3.0"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
|
|
@ -1047,17 +1064,9 @@
|
|||
},
|
||||
"iniconfig": {
|
||||
"hashes": [
|
||||
"sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437",
|
||||
"sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"
|
||||
"sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
|
||||
],
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20",
|
||||
"sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"
|
||||
],
|
||||
"version": "==8.5.0"
|
||||
"version": "==1.1.1"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
|
|
@ -1097,11 +1106,11 @@
|
|||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:0e37f61339c4578776e090c3b8f6b16ce4db333889d65d0efb305243ec544b40",
|
||||
"sha256:c8f57c2a30983f469bf03e68cdfa74dc474ce56b8f280ddcb080dfd91df01043"
|
||||
"sha256:7a8190790c17d79a11f847fba0b004ee9a8122582ebff4729a082c109e81a4c9",
|
||||
"sha256:8f593023c1a0f916110285b6efd7f99db07d59546e3d8c36fc60e2ab05d3be92"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==6.0.2"
|
||||
"version": "==6.1.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
|
|
@ -1119,10 +1128,10 @@
|
|||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:43f4fa8d8bb313e65d8323a3952ef8756bf40f9a5c3ea7334be23ee4ec8278b6",
|
||||
"sha256:b52f22895f4cfce194bc8172f3819ee8de7540aa6d873535a8668b730b8b411f"
|
||||
"sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b",
|
||||
"sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066"
|
||||
],
|
||||
"version": "==3.2.0"
|
||||
"version": "==3.3.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from flask_paginate import Pagination, get_page_parameter
|
|||
from flask_sqlalchemy import SQLAlchemy
|
||||
from sentry_sdk.integrations.flask import FlaskIntegration
|
||||
from webassets import Bundle
|
||||
from flask_executor import Executor
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
|
@ -22,6 +23,9 @@ logging.basicConfig(level=logging.INFO)
|
|||
connexion_app = connexion.FlaskApp(__name__)
|
||||
app = connexion_app.app
|
||||
|
||||
# Executor for long running tasks
|
||||
executor = Executor(app)
|
||||
|
||||
# Configuration
|
||||
app.config.from_object('config.default')
|
||||
if "TESTING" in os.environ and os.environ["TESTING"] == "true":
|
||||
|
|
|
|||
|
|
@ -57,6 +57,12 @@ paths:
|
|||
type: string
|
||||
/notify_by_email:
|
||||
get:
|
||||
parameters:
|
||||
- in: query
|
||||
name: file_name
|
||||
schema:
|
||||
type: string
|
||||
description: An optional file name, only records from this ivy file will be notified.
|
||||
operationId: communicator.api.admin.notify_by_email
|
||||
summary: when called, reviews all samples, and sends out any pending email notifications.
|
||||
security: [] # Disable security for this endpoint only.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from communicator import db, app
|
||||
from communicator import db, app, executor
|
||||
from communicator.models import Sample
|
||||
from communicator.models.invitation import Invitation
|
||||
from communicator.models.notification import Notification, EMAIL_TYPE, TEXT_TYPE
|
||||
|
|
@ -9,7 +9,8 @@ from time import sleep
|
|||
|
||||
|
||||
def status():
|
||||
return {"status":"good"}
|
||||
return {"status": "good"}
|
||||
|
||||
|
||||
def add_sample(body):
|
||||
sample = Sample(barcode=body['barcode'],
|
||||
|
|
@ -18,6 +19,7 @@ def add_sample(body):
|
|||
location=body['location'])
|
||||
SampleService().add_or_update_records([sample])
|
||||
|
||||
|
||||
def clear_samples():
|
||||
db.session.query(Notification).delete()
|
||||
db.session.query(Sample).delete()
|
||||
|
|
@ -26,29 +28,51 @@ def clear_samples():
|
|||
|
||||
|
||||
def update_and_notify():
|
||||
update_data()
|
||||
notify_by_email()
|
||||
notify_by_text()
|
||||
# These can take a very long time to execute.
|
||||
executor.submit(_update_data)
|
||||
executor.submit(_notify_by_email)
|
||||
executor.submit(_notify_by_text)
|
||||
return "Task scheduled and running the background"
|
||||
|
||||
|
||||
def update_data():
|
||||
executor.submit(_update_data)
|
||||
return "Task scheduled and running the background"
|
||||
|
||||
|
||||
def _update_data():
|
||||
"""Updates the database based on local files placed by IVY. No longer attempts
|
||||
to pull files from the Firebase service."""
|
||||
ivy_service = IvyService()
|
||||
ivy_service.request_transfer()
|
||||
samples = ivy_service.load_directory()
|
||||
files, samples = ivy_service.load_directory()
|
||||
SampleService().add_or_update_records(samples)
|
||||
for file in files:
|
||||
db.session.add(file)
|
||||
db.session.commit()
|
||||
ivy_service.delete_file(file.file_name)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def merge_similar_records():
|
||||
sample_service = SampleService()
|
||||
sample_service.merge_similar_records()
|
||||
|
||||
|
||||
def notify_by_email():
|
||||
def notify_by_email(file_name=None):
|
||||
executor.submit(_notify_by_email, file_name)
|
||||
return "Task scheduled and running the background"
|
||||
|
||||
|
||||
def _notify_by_email(file_name=None):
|
||||
"""Sends out notifications via email"""
|
||||
samples = db.session.query(Sample)\
|
||||
.filter(Sample.result_code != None)\
|
||||
.filter(Sample.email_notified == False).all()
|
||||
sample_query = db.session.query(Sample) \
|
||||
.filter(Sample.result_code != None) \
|
||||
.filter(Sample.email_notified == False)
|
||||
if file_name:
|
||||
sample_query = sample_query.filter(Sample.ivy_file == file_name)
|
||||
|
||||
samples = sample_query.all()
|
||||
with NotificationService(app) as notifier:
|
||||
for sample in samples:
|
||||
last_failure = sample.last_failure_by_type(EMAIL_TYPE)
|
||||
|
|
@ -64,15 +88,23 @@ def notify_by_email():
|
|||
sleep(0.5)
|
||||
|
||||
|
||||
def notify_by_text():
|
||||
def notify_by_text(file_name=None):
|
||||
executor.submit(_notify_by_text, file_name)
|
||||
return "Task scheduled and running the background"
|
||||
|
||||
|
||||
def _notify_by_text(file_name):
|
||||
"""Sends out notifications via SMS Message, but only at reasonable times of day"""
|
||||
with NotificationService(app) as notifier:
|
||||
if not notifier.is_reasonable_hour_for_text_messages:
|
||||
print("Skipping text messages, it's not a good time to get one.")
|
||||
return
|
||||
samples = db.session.query(Sample)\
|
||||
.filter(Sample.result_code != None)\
|
||||
.filter(Sample.text_notified == False).all()
|
||||
sample_query = db.session.query(Sample) \
|
||||
.filter(Sample.result_code != None) \
|
||||
.filter(Sample.text_notified == False)
|
||||
if file_name:
|
||||
sample_query = sample_query.filter(Sample.ivy_file == file_name)
|
||||
samples = sample_query.all()
|
||||
for sample in samples:
|
||||
last_failure = sample.last_failure_by_type(TEXT_TYPE)
|
||||
if last_failure: continue
|
||||
|
|
@ -85,6 +117,3 @@ def notify_by_text():
|
|||
error_message=str(e)))
|
||||
db.session.commit()
|
||||
sleep(0.5)
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ class Sample(db.Model):
|
|||
phone = db.Column(db.String)
|
||||
email = db.Column(db.String)
|
||||
result_code = db.Column(db.String)
|
||||
ivy_file = db.Column(db.String)
|
||||
in_firebase = db.Column(db.Boolean, default=False) # Does this record exist in Firebase?
|
||||
in_ivy = db.Column(db.Boolean, default=False) # Has this record come in from the IVY?
|
||||
email_notified = db.Column(db.Boolean, default=False)
|
||||
|
|
|
|||
|
|
@ -10,19 +10,22 @@ from communicator.api import admin
|
|||
def update_and_notify():
|
||||
with app.app_context():
|
||||
if app.config['RUN_SCHEDULED_TASKS']:
|
||||
admin.update_and_notify()
|
||||
else:
|
||||
app.logger.info("Currently not running scheduled tasks RUN_SCHEDULED_TASKS"
|
||||
admin._update_data()
|
||||
admin._notify_by_email()
|
||||
admin._notify_by_text()
|
||||
|
||||
if app.config['RUN_SCHEDULED_TASKS']:
|
||||
scheduler = BackgroundScheduler()
|
||||
scheduler.add_jobstore('sqlalchemy', url=db.engine.url)
|
||||
scheduler.add_job(
|
||||
update_and_notify, 'interval', minutes=app.config['SCHEDULED_TASK_MINUTES'],
|
||||
id='update_data', replace_existing=True
|
||||
)
|
||||
|
||||
scheduler.start()
|
||||
|
||||
# Shut down the scheduler when exiting the app
|
||||
atexit.register(lambda: scheduler.shutdown())
|
||||
else:
|
||||
app.logger.info("Currently not running scheduled tasks RUN_SCHEDULED_TASKS"
|
||||
" is set to false in configuration.")
|
||||
|
||||
scheduler = BackgroundScheduler()
|
||||
scheduler.add_jobstore('sqlalchemy', url=db.engine.url)
|
||||
scheduler.add_job(
|
||||
update_and_notify, 'interval', minutes=app.config['SCHEDULED_TASK_MINUTES'],
|
||||
id='update_data', replace_existing=True
|
||||
)
|
||||
|
||||
scheduler.start()
|
||||
|
||||
# Shut down the scheduler when exiting the app
|
||||
atexit.register(lambda: scheduler.shutdown())
|
||||
|
|
@ -28,14 +28,15 @@ class IvyService(object):
|
|||
self.transfer_client = None
|
||||
self.transfer_client_date = datetime.now()
|
||||
|
||||
def load_directory(self, delete_from_globus=True):
|
||||
"""Loads files from a local directory, and optionally issues a delete command
|
||||
to the Globus file manager to remove the file once it is ingested. """
|
||||
def load_directory(self):
|
||||
"""Loads files from a local directory, returning a tuple containing the list
|
||||
of files, and the list of samples respectively"""
|
||||
|
||||
onlyfiles = [f for f in listdir(self.path) if isfile(join(self.path, f))]
|
||||
app.logger.info(f'Loading directory {self.path}')
|
||||
|
||||
samples = []
|
||||
files = []
|
||||
for file_name in onlyfiles:
|
||||
samples = IvyService.samples_from_ivy_file(join(self.path, file_name))
|
||||
ivy_file = db.session.query(IvyFile).filter(IvyFile.file_name == file_name).first()
|
||||
|
|
@ -44,12 +45,9 @@ class IvyService(object):
|
|||
else:
|
||||
ivy_file.date_added = datetime.now()
|
||||
ivy_file.sample_count = len(samples)
|
||||
db.session.add(ivy_file)
|
||||
db.session.commit()
|
||||
app.logger.info(f'Loading file {file_name}')
|
||||
if(delete_from_globus):
|
||||
self.delete_file(file_name)
|
||||
return samples
|
||||
files.append(ivy_file)
|
||||
app.logger.info(f'Loaded {len(samples)} samples from file {file_name}')
|
||||
return files, samples
|
||||
|
||||
@staticmethod
|
||||
def samples_from_ivy_file(file_name):
|
||||
|
|
@ -57,12 +55,12 @@ class IvyService(object):
|
|||
with open(file_name, 'r') as csv_file:
|
||||
reader = csv.DictReader(csv_file, delimiter='|')
|
||||
for row in reader:
|
||||
sample = IvyService.record_to_sample(row)
|
||||
sample = IvyService.record_to_sample(row, file_name)
|
||||
rows.append(sample)
|
||||
return rows
|
||||
|
||||
@staticmethod
|
||||
def record_to_sample(dictionary):
|
||||
def record_to_sample(dictionary, file_name):
|
||||
"""Creates a Test Result from a record read in from the IVY CSV File"""
|
||||
sample = Sample()
|
||||
try:
|
||||
|
|
@ -73,6 +71,7 @@ class IvyService(object):
|
|||
sample.date = parser.parse(dictionary["Test Date Time"])
|
||||
sample.location = dictionary["Test Kiosk Loc"]
|
||||
sample.result_code = dictionary["Test Result Code"]
|
||||
sample.ivy_file = file_name
|
||||
sample.in_ivy = True
|
||||
return sample
|
||||
except KeyError as e:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ class SampleService(object):
|
|||
first()
|
||||
if sample2:
|
||||
sample.merge(sample2)
|
||||
# Move notifications over as well.
|
||||
notifications = sample2.notifications
|
||||
sample.notifications = notifications
|
||||
sample2.notifications = []
|
||||
db.session.add(sample)
|
||||
db.session.delete(sample2)
|
||||
db.session.commit()
|
||||
|
|
|
|||
|
|
@ -1,14 +1,7 @@
|
|||
Student ID|Student Cellphone|Student Email|Test Date Time|Test Kiosk Loc|Test Result Code
|
||||
987654321|555/555-5555|rkc7h@virginia.edu|202009030809|4321|8726520277
|
||||
987654322|555/555-5556|testpositive@virginia.edu|202009060919|4321|8269722523
|
||||
987655321|555/555-5558|testnegetive@virginia.edu|202009070719|4321|1142270225
|
||||
000000111|555/555-5555|rkc7h@virginia.edu|202009091449|4321|8726520277
|
||||
000000222|555/555-5556|testpositive@virginia.edu|202009091449|4321|8269722523
|
||||
000000333|555/555-5558|testnegetive@virginia.edu|202009091449|4321|1142270225
|
||||
987654555|555/555-5555|rkc7h@virginia.edu|202009030809|4321|8726520277
|
||||
987654666|555/555-5556|testpositive@virginia.edu|202009060919|4321|8269722523
|
||||
987655777|555/555-5558|testnegetive@virginia.edu|202009070719|4321|1142270225
|
||||
000000888|555/555-5555|rkc7h@virginia.edu|202009091449|4321|8726520277
|
||||
000000999|555/555-5556|testpositive@virginia.edu|202009091449|4321|8269722523
|
||||
000000121|555/555-5558|testnegetive@virginia.edu|202009091449|4321|1142270225
|
||||
111000000|540-457-0024|daniel.h.funk@gmail.com|202009091449|4321|1142270225
|
||||
Student ID|Student Cellphone|Student Email|Test Date Time|Test Kiosk Loc|Test Result Code|Test Bar Code
|
||||
987654321|555/555-5555|rkc7h@virginia.edu|202009030809|4321|8726520277|987654321-RKC-202009030809-4321
|
||||
987654322|555/555-5556|testpositive@virginia.edu|202009060919|4321|8269722523|987654322-TP-202009060919-4321
|
||||
987655321|555/555-5558|testnegetive@virginia.edu|202009070719|4321|1142270225|987655321-TN-202009070719-4321
|
||||
000000111|555/555-5555|rkc7h@virginia.edu|202009091449|4321|8726520277|000000111-RKC-202009091449-4321
|
||||
000000222|555/555-5556|testpositive@virginia.edu|202009091449|4321|8269722523|000000222-TP-202009091449-4321
|
||||
000000333|555/555-5558|testnegetive@virginia.edu|202009091449|4321|1142270225|000000333-TN-202009091449-4321
|
||||
|
|
|
|||
|
|
|
@ -1,16 +1,20 @@
|
|||
# Set environment variable to testing before loading.
|
||||
# IMPORTANT - Environment must be loaded before app, models, etc....
|
||||
import base64
|
||||
import ctypes
|
||||
import os
|
||||
import quopri
|
||||
import re
|
||||
import unittest
|
||||
|
||||
|
||||
os.environ["TESTING"] = "true"
|
||||
|
||||
from communicator.models import Sample
|
||||
from communicator.models.notification import Notification
|
||||
|
||||
|
||||
from communicator import app, db
|
||||
from communicator import app, db, executor
|
||||
|
||||
import logging
|
||||
logging.basicConfig()
|
||||
|
|
@ -42,13 +46,16 @@ class BaseTest(unittest.TestCase):
|
|||
def tearDownClass(cls):
|
||||
cls.ctx.pop()
|
||||
db.drop_all()
|
||||
executor.shutdown(wait=False)
|
||||
pass
|
||||
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def tearDown(self):
|
||||
db.session.query(Notification).delete()
|
||||
db.session.query(Sample).delete()
|
||||
executor.shutdown(wait=False)
|
||||
db.session.commit()
|
||||
|
||||
def decode(self, encoded_words):
|
||||
|
|
|
|||
|
|
@ -27,8 +27,7 @@ class IvyServiceTest(BaseTest):
|
|||
def test_load_directory(self):
|
||||
self.assertEquals(0, db.session.query(IvyFile).count())
|
||||
app.config['IVY_IMPORT_DIR'] = os.path.join(app.root_path, '..', 'tests', 'data', 'import_directory')
|
||||
records = IvyService().load_directory(delete_from_globus=False)
|
||||
files = db.session.query(IvyFile).all()
|
||||
files, records = IvyService().load_directory()
|
||||
self.assertEquals(4, len(files))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,20 @@
|
|||
from time import sleep
|
||||
|
||||
from tests.base_test import BaseTest
|
||||
|
||||
import json
|
||||
from communicator.models import Sample
|
||||
from communicator import db
|
||||
from communicator.api import admin
|
||||
|
||||
|
||||
class TestSampleEndpoint(BaseTest):
|
||||
|
||||
sample_json = {"barcode": "000000111-202009091449-4321",
|
||||
"location": "4321",
|
||||
"date": "2020-09-09T14:49:00+0000",
|
||||
"student_id": "000000111"}
|
||||
|
||||
def test_create_sample(self):
|
||||
|
||||
# Test add sample
|
||||
samples = db.session.query(Sample).all()
|
||||
self.assertEquals(0, len(samples))
|
||||
|
|
@ -26,15 +27,39 @@ class TestSampleEndpoint(BaseTest):
|
|||
self.assertEquals(1, len(samples))
|
||||
|
||||
def test_create_duplicate_sample_does_not_raise_error(self):
|
||||
|
||||
# Test add sample
|
||||
samples = db.session.query(Sample).all()
|
||||
self.assertEquals(0, len(samples))
|
||||
|
||||
rv = self.app.post('/v1.0/sample',content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample',content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample',content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample',content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample', content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample', content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample', content_type="application/json", data=json.dumps(self.sample_json))
|
||||
rv = self.app.post('/v1.0/sample', content_type="application/json", data=json.dumps(self.sample_json))
|
||||
|
||||
samples = db.session.query(Sample).all()
|
||||
self.assertEquals(1, len(samples))
|
||||
|
||||
def test_notify_by_email_by_file_name(self):
|
||||
db.session.add(Sample(barcode="000000111-202009091449-4321",
|
||||
location="4321",
|
||||
date="2020-09-09T14:49:00+0000",
|
||||
student_id="000000111",
|
||||
email="daniel.h.funk@gmail.com",
|
||||
result_code="12345",
|
||||
ivy_file="xxx"))
|
||||
db.session.add(Sample(barcode="000000112-202009091449-4321",
|
||||
location="4321",
|
||||
date="2020-09-09T14:49:00+0000",
|
||||
student_id="000000112",
|
||||
email="dan@gmail.com",
|
||||
result_code="12345",
|
||||
ivy_file="yyy"))
|
||||
db.session.commit()
|
||||
admin._notify_by_email('xxx')
|
||||
samples = db.session.query(Sample).filter(Sample.email_notified == True).all()
|
||||
self.assertEquals(1, len(samples))
|
||||
samples = db.session.query(Sample).filter(Sample.email_notified != True).all()
|
||||
self.assertEquals(1, len(samples))
|
||||
admin._notify_by_email()
|
||||
samples = db.session.query(Sample).filter(Sample.email_notified == True).all()
|
||||
self.assertEquals(2, len(samples))
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from tests.base_test import BaseTest
|
||||
import json
|
||||
|
||||
from dateutil import parser
|
||||
|
||||
from tests.base_test import BaseTest
|
||||
from communicator.models.notification import Notification
|
||||
from communicator import db
|
||||
from communicator.models.sample import Sample
|
||||
from communicator.services.ivy_service import IvyService
|
||||
|
|
@ -70,16 +71,20 @@ class IvyServiceTest(BaseTest):
|
|||
def test_merge_similar_records(self):
|
||||
service = SampleService()
|
||||
# 511908685 - 202010051136 - 0202
|
||||
db.session.add(Sample(barcode="111111111-AAA-202010050000-0000",
|
||||
s1 = Sample(barcode="111111111-AAA-202010050000-0000",
|
||||
student_id=111111111,
|
||||
date = parser.parse("202010050000"),
|
||||
location=0))
|
||||
db.session.add(Sample(barcode="111111111-202010050000-0000",
|
||||
location=0)
|
||||
s2 = Sample(barcode="111111111-202010050000-0000",
|
||||
student_id=111111111,
|
||||
date = parser.parse("202010050000"),
|
||||
location=0,
|
||||
email="dan@sartography.com",
|
||||
phone="555-555-5555"))
|
||||
phone="555-555-5555")
|
||||
s2n = Notification(date=parser.parse("202010050000"), type="email", successful=True)
|
||||
s2.notifications = [s2n]
|
||||
db.session.add(s1)
|
||||
db.session.add(s2)
|
||||
db.session.commit()
|
||||
self.assertEquals(2, len(db.session.query(Sample).all()))
|
||||
service.merge_similar_records()
|
||||
|
|
@ -87,6 +92,7 @@ class IvyServiceTest(BaseTest):
|
|||
sample = db.session.query(Sample).first()
|
||||
self.assertEquals("dan@sartography.com", sample.email)
|
||||
self.assertEquals("111111111-AAA-202010050000-0000", sample.barcode)
|
||||
self.assertEquals(1, len(sample.notifications))
|
||||
|
||||
def test_merge_non_similar_records(self):
|
||||
service = SampleService()
|
||||
|
|
|
|||
Loading…
Reference in New Issue