diff --git a/.github/workflows/cleanup.yaml b/.github/workflows/cleanup.yaml index 1151f5f..5f49620 100644 --- a/.github/workflows/cleanup.yaml +++ b/.github/workflows/cleanup.yaml @@ -89,4 +89,9 @@ jobs: run: pipenv run sceptre --debug --var namespace=${{ github.event.ref }} delete develop/namespaced --yes - name: Remove artifacts - run: pipenv run python src/scripts/manage_artifacts/artifacts.py --remove --namespace ${{ github.event.ref }} --cfn_bucket ${{ vars.CFN_BUCKET }} + run: | + pipenv run python src/scripts/manage_artifacts/artifacts.py + --remove + --namespace ${{ github.event.ref }} + --cfn_bucket ${{ vars.CFN_BUCKET }} + --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }} diff --git a/.github/workflows/upload-and-deploy-to-prod-main.yaml b/.github/workflows/upload-and-deploy-to-prod-main.yaml index 929940b..e8cfdf3 100644 --- a/.github/workflows/upload-and-deploy-to-prod-main.yaml +++ b/.github/workflows/upload-and-deploy-to-prod-main.yaml @@ -41,7 +41,12 @@ jobs: python_version: ${{ env.PYTHON_VERSION }} - name: Copy files to templates bucket - run: python src/scripts/manage_artifacts/artifacts.py --upload --namespace $NAMESPACE --cfn_bucket ${{ vars.CFN_BUCKET }} + run: > + python src/scripts/manage_artifacts/artifacts.py + --upload + --namespace $NAMESPACE + --cfn_bucket ${{ vars.CFN_BUCKET }} + --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }} sceptre-deploy-main: diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index 5420909..ea8b877 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -115,6 +115,7 @@ jobs: --upload --namespace $NAMESPACE --cfn_bucket ${{ vars.CFN_BUCKET }} + --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }} nonglue-unit-tests: name: Runs unit tests that are not dependent on aws-glue package resources @@ -437,6 +438,7 @@ jobs: --upload --namespace staging --cfn_bucket ${{ vars.CFN_BUCKET }} + --shareable-artifacts-bucket ${{ vars.SHAREABLE_ARTIFACTS_BUCKET }} - name: Create directory for remote sceptre templates run: mkdir -p templates/remote/ diff --git a/Pipfile b/Pipfile index 1afc004..4a3913f 100644 --- a/Pipfile +++ b/Pipfile @@ -9,10 +9,11 @@ python_version = "3.9" [dev-packages] pytest = "*" pyarrow = "~=11.0" -pre-commit = "*" +pre-commit = "~=4.0" sceptre = ">=3.2.0" sceptre-sam-handler = "*" synapseclient = "~=2.7" +numpy = "<2.0" # See issue "A module that was compiled using NumPy 1.x cannot be run..." pandas = "<1.5" moto = "~=4.1" datacompy = "~=0.8" diff --git a/Pipfile.lock b/Pipfile.lock index 9464b52..ed615af 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f099bcf5b668dc8f3196d9738600c180c1aecc6f137ac9ea50c506e6fdd7b303" + "sha256": "76e9fb55a63bc554b8f9bbadd48253b27beac95c9180b5ad142c6f49bdda4ccc" }, "pipfile-spec": 6, "requires": { @@ -19,11 +19,11 @@ "develop": { "adagio": { "hashes": [ - "sha256:c6c4d812f629fc3141284a0b3cfe483731b28da3a1b18f3d5498695ff87dcc12", - "sha256:e58abc4539184a65faf9956957d3787616bedeb1303ac5c9b1a201d8af6b87d7" + "sha256:0c32768f3aba0e05273b36f9420a482034f2510f059171040d7e98ba34128d7a", + "sha256:1bb8317d41bfff8b11373bc03c9859ff166c498214bb2b7ce1e21638c0babb2c" ], - "markers": "python_version >= '3.6'", - "version": "==0.2.4" + "markers": "python_version >= '3.8'", + "version": "==0.2.6" }, "appdirs": { "hashes": [ @@ -34,11 +34,11 @@ }, "attrs": { "hashes": [ - "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", - "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" + "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", + "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" ], "markers": "python_version >= '3.7'", - "version": "==23.2.0" + "version": "==24.2.0" }, "blinker": { "hashes": [ @@ -50,86 +50,100 @@ }, "boto3": { "hashes": [ - "sha256:81518aa95fad71279411fb5c94da4b4a554a5d53fc876faca62b7b5c8737f1cb", - "sha256:f79c15e33eb7706f197d98d828b193cf0891966682ad3ec5e900f6f9e7362e35" + "sha256:2d5e160b614db55fbee7981001c54476cb827c441cef65b2fcb2c52a62019909", + "sha256:7d9c359bbbc858a60b51c86328db813353c8bd1940212cdbd0a7da835291c2e1" ], "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==1.34.138" + "version": "==1.35.54" }, "botocore": { "hashes": [ - "sha256:84e96a954c39a6f09cae4ea95b2ae582b5ae01b5040c92507b60509c9be5377a", - "sha256:f558bbea96c4a4abbaeeedc477dabb00902311ba1ca6327974a6819b9f384920" + "sha256:131bb59ce59c8a939b31e8e647242d70cf11d32d4529fa4dca01feea1e891a76", + "sha256:9cca1811094b6cdc144c2c063a3ec2db6d7c88194b04d4277cd34fc8e3473aff" ], "markers": "python_version >= '3.8'", - "version": "==1.34.138" + "version": "==1.35.54" }, "certifi": { "hashes": [ - "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516", - "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56" + "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", + "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" ], "markers": "python_version >= '3.6'", - "version": "==2024.6.2" + "version": "==2024.8.30" }, "cffi": { "hashes": [ - "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", - "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", - "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", - "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", - "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", - "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", - "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", - "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", - "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", - "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", - "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", - "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", - "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", - "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", - "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", - "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", - "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", - "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", - "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", - "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", - "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", - "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", - "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", - "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", - "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", - "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", - "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", - "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", - "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", - "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", - "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", - "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", - "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", - "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", - "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", - "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", - "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", - "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", - "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", - "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", - "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", - "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", - "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", - "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", - "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", - "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", - "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", - "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", - "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", - "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", - "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", - "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", + "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", + "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1", + "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", + "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", + "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", + "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8", + "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36", + "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", + "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", + "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc", + "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", + "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", + "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", + "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", + "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", + "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", + "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", + "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", + "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b", + "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", + "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", + "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c", + "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", + "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", + "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", + "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8", + "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1", + "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", + "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", + "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", + "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595", + "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0", + "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", + "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", + "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", + "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", + "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", + "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", + "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16", + "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", + "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e", + "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", + "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964", + "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", + "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576", + "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", + "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3", + "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662", + "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", + "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", + "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", + "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", + "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", + "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", + "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", + "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", + "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9", + "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7", + "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", + "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a", + "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", + "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", + "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", + "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", + "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87", + "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b" ], "markers": "platform_python_implementation != 'PyPy'", - "version": "==1.16.0" + "version": "==1.17.1" }, "cfgv": { "hashes": [ @@ -148,99 +162,114 @@ }, "charset-normalizer": { "hashes": [ - "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", - "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", - "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", - "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", - "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", - "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", - "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", - "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", - "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", - "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", - "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", - "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", - "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", - "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", - "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", - "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", - "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", - "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", - "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", - "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", - "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", - "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", - "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", - "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", - "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", - "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", - "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", - "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", - "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", - "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", - "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", - "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", - "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", - "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", - "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", - "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", - "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", - "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", - "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", - "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", - "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", - "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", - "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", - "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", - "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", - "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", - "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", - "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", - "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", - "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", - "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", - "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", - "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", - "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", - "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", - "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", - "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", - "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", - "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", - "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", - "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", - "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", - "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", - "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", - "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", - "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", - "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", - "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", - "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", - "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", - "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", - "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", - "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", - "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", - "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", - "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", - "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", - "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", - "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", - "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", - "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", - "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", - "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", - "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", - "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", - "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", - "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", - "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", - "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", - "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621", + "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", + "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", + "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", + "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", + "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", + "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", + "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d", + "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", + "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", + "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", + "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", + "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab", + "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be", + "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", + "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", + "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0", + "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2", + "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62", + "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62", + "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", + "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", + "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", + "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", + "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455", + "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858", + "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", + "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", + "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", + "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", + "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", + "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea", + "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", + "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", + "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", + "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", + "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd", + "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", + "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242", + "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee", + "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", + "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", + "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51", + "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", + "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8", + "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", + "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613", + "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742", + "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", + "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", + "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", + "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", + "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", + "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", + "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", + "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", + "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417", + "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", + "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", + "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca", + "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa", + "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", + "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149", + "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41", + "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574", + "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0", + "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f", + "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", + "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654", + "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3", + "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19", + "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", + "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578", + "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", + "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", + "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51", + "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", + "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", + "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a", + "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", + "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade", + "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", + "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", + "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6", + "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", + "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", + "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6", + "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2", + "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12", + "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf", + "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", + "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7", + "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", + "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", + "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b", + "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", + "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", + "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4", + "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", + "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", + "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a", + "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748", + "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", + "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", + "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.2" + "version": "==3.4.0" }, "click": { "hashes": [ @@ -260,50 +289,44 @@ }, "cryptography": { "hashes": [ - "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad", - "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583", - "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b", - "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c", - "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1", - "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648", - "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949", - "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba", - "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c", - "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9", - "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d", - "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c", - "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e", - "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2", - "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d", - "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7", - "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70", - "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2", - "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7", - "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14", - "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe", - "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e", - "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71", - "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961", - "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7", - "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c", - "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28", - "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842", - "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902", - "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801", - "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a", - "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e" + "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362", + "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4", + "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa", + "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83", + "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff", + "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805", + "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6", + "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664", + "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08", + "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e", + "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18", + "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f", + "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73", + "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5", + "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984", + "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd", + "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3", + "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e", + "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405", + "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2", + "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c", + "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995", + "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73", + "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16", + "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7", + "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd", + "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7" ], "markers": "python_version >= '3.7'", - "version": "==42.0.8" + "version": "==43.0.3" }, "datacompy": { "hashes": [ - "sha256:388090e39081041a851b715f0fdffa49de8ecd23864e0f63185ed96ebe509f92", - "sha256:b3eeb4ff15eec7fe645862f704256edaa301c3b872077ae92d2965064ed6af93" + "sha256:f64b50b5a49e0dbd82a14beae7b940255fadc2190bd72bedc60a6a961f40e10f", + "sha256:f823ea5d3574edac229f62c001fadb37e412fb17a8b49a020197eb7d1f9b0e71" ], "index": "pypi", - "markers": "python_full_version >= '3.9.0'", - "version": "==0.13.1" + "version": "==0.14.3" }, "deepdiff": { "hashes": [ @@ -330,10 +353,10 @@ }, "distlib": { "hashes": [ - "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784", - "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64" + "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", + "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403" ], - "version": "==0.3.8" + "version": "==0.3.9" }, "docker": { "hashes": [ @@ -341,7 +364,6 @@ "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9" ], "index": "pypi", - "markers": "python_version >= '3.7'", "version": "==6.1.3" }, "ecs-logging": { @@ -350,24 +372,23 @@ "sha256:f6e22d267770b06f797076f49b5fcc9d97108b22f452f5f9ed4b5367b1e61b5b" ], "index": "pypi", - "markers": "python_version >= '3.6'", "version": "==2.2.0" }, "exceptiongroup": { "hashes": [ - "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad", - "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16" + "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", + "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc" ], "markers": "python_version < '3.11'", - "version": "==1.2.1" + "version": "==1.2.2" }, "filelock": { "hashes": [ - "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb", - "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7" + "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", + "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435" ], "markers": "python_version >= '3.8'", - "version": "==3.15.4" + "version": "==3.16.1" }, "flask": { "hashes": [ @@ -375,7 +396,6 @@ "sha256:f69fcd559dc907ed196ab9df0e48471709175e696d6e698dd4dbe940f96ce66b" ], "index": "pypi", - "markers": "python_version >= '3.8'", "version": "==2.3.3" }, "flask-cors": { @@ -395,11 +415,11 @@ }, "fsspec": { "hashes": [ - "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e", - "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49" + "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871", + "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493" ], "markers": "python_version >= '3.8'", - "version": "==2024.6.1" + "version": "==2024.10.0" }, "fugue": { "hashes": [ @@ -411,26 +431,26 @@ }, "identify": { "hashes": [ - "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa", - "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d" + "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0", + "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98" ], "markers": "python_version >= '3.8'", - "version": "==2.5.36" + "version": "==2.6.1" }, "idna": { "hashes": [ - "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", - "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" ], - "markers": "python_version >= '3.5'", - "version": "==3.7" + "markers": "python_version >= '3.6'", + "version": "==3.10" }, "importlib-metadata": { "hashes": [ "sha256:8a8a81bcf996e74fee46f0d16bd3eaa382a7eb20fd82445c3ad11f4090334116", "sha256:dd0173e8f150d6815e098fd354f6414b0f079af4644ddfe90c71e2fc6174346d" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.10'", "version": "==4.13.0" }, "iniconfig": { @@ -449,14 +469,6 @@ "markers": "python_version >= '3.8'", "version": "==2.2.0" }, - "jeepney": { - "hashes": [ - "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806", - "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755" - ], - "markers": "sys_platform == 'linux'", - "version": "==0.8.0" - }, "jinja2": { "hashes": [ "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369", @@ -488,79 +500,72 @@ "markers": "python_version >= '3.6'", "version": "==23.4.1" }, - "keyrings.alt": { - "hashes": [ - "sha256:6a00fa799baf1385cf9620bd01bcc815aa56e6970342a567bcfea0c4d21abe5f", - "sha256:b59c86b67b9027a86e841a49efc41025bcc3b1b0308629617b66b7011e52db5a" - ], - "markers": "sys_platform == 'linux'", - "version": "==3.1" - }, "markupsafe": { "hashes": [ - "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", - "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", - "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", - "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", - "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", - "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", - "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", - "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", - "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", - "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", - "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", - "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", - "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", - "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", - "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", - "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", - "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", - "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", - "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", - "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", - "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", - "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", - "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", - "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", - "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", - "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", - "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", - "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", - "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", - "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", - "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", - "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", - "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", - "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", - "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", - "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", - "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", - "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", - "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", - "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", - "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", - "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", - "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", - "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", - "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", - "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", - "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", - "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", - "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", - "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", - "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", - "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", - "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", - "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", - "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", - "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", - "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", - "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", - "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", - "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" + "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", + "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", + "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0", + "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", + "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", + "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13", + "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", + "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", + "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", + "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", + "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0", + "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", + "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", + "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", + "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", + "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff", + "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", + "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", + "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", + "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", + "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", + "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", + "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", + "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", + "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a", + "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", + "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", + "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", + "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", + "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144", + "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f", + "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", + "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", + "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", + "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", + "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", + "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", + "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", + "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", + "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", + "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", + "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", + "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", + "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", + "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", + "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", + "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", + "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", + "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29", + "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", + "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", + "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", + "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", + "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", + "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", + "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a", + "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178", + "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", + "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", + "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", + "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50" ], - "markers": "python_version >= '3.7'", - "version": "==2.1.5" + "markers": "python_version >= '3.9'", + "version": "==3.0.2" }, "moto": { "hashes": [ @@ -568,7 +573,6 @@ "sha256:8f9263ca70b646f091edcc93e97cda864a542e6d16ed04066b1370ed217bd190" ], "index": "pypi", - "markers": "python_version >= '3.7'", "version": "==4.2.14" }, "networkx": { @@ -626,7 +630,7 @@ "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3", "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f" ], - "markers": "python_version >= '3.9'", + "index": "pypi", "version": "==1.26.4" }, "ordered-set": { @@ -639,11 +643,11 @@ }, "packaging": { "hashes": [ - "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", - "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", + "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" ], - "markers": "python_version >= '3.6'", - "version": "==21.3" + "markers": "python_version >= '3.8'", + "version": "==24.1" }, "pandas": { "hashes": [ @@ -670,16 +674,15 @@ "sha256:ee6f1848148ed3204235967613b0a32be2d77f214e9623f554511047705c1e04" ], "index": "pypi", - "markers": "python_version >= '3.8'", "version": "==1.4.4" }, "platformdirs": { "hashes": [ - "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", - "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" + "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907", + "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb" ], "markers": "python_version >= '3.8'", - "version": "==4.2.2" + "version": "==4.3.6" }, "pluggy": { "hashes": [ @@ -691,24 +694,23 @@ }, "polars": { "hashes": [ - "sha256:00f62dec6bf43a4e2a5db58b99bf0e79699fe761c80ae665868eaea5168f3bbb", - "sha256:24b82441f93409e0e8abd6f427b029db102f02b8de328cee9a680f84b84e3736", - "sha256:2d7567c9fd9d3b9aa93387ca9880d9e8f7acea3c0a0555c03d8c0c2f0715d43c", - "sha256:67f2fe842262b7e1b9371edad21b760f6734d28b74c78dda88dff1bf031b9499", - "sha256:86454ade5ed302bbf87f145cfcb1b14f7a5765a9440e448659e1f3dba6ac4e79", - "sha256:87f43bce4d41abf8c8c5658d881e4b8378e5c61010a696bfea8b4106b908e916" + "sha256:1c811b772c9476f7f0bb4445a8387d2ab6d86f5e79140b1bfba914a32788d261", + "sha256:6d1665c23e3574ebd47a26a5d7b619e6e73e53718c3b0bfd7d08b6a0a4ae7daa", + "sha256:a166adb429f8ee099c9d803e7470a80c76368437a8b272c67cef9eef6d5e9da1", + "sha256:d7e8d5e577883a9755bc3be92ecbf6f20bced68267bdb8bdb440120e905cc19c", + "sha256:d7f3abf085adf034720b358119c4c8e144bcc2d96010b7e7d0afa11b80da383c", + "sha256:ffae15ffa80fda5cc3af44a340b565bcf7f2ab6d7854d3f967baf505710c78e2" ], "markers": "python_version >= '3.8'", - "version": "==0.20.31" + "version": "==1.6.0" }, "pre-commit": { "hashes": [ - "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a", - "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5" + "sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2", + "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878" ], "index": "pypi", - "markers": "python_version >= '3.9'", - "version": "==3.7.1" + "version": "==4.0.1" }, "pyarrow": { "hashes": [ @@ -739,7 +741,6 @@ "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34" ], "index": "pypi", - "markers": "python_version >= '3.7'", "version": "==11.0.0" }, "pycparser": { @@ -750,14 +751,6 @@ "markers": "python_version >= '3.8'", "version": "==2.22" }, - "pyparsing": { - "hashes": [ - "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad", - "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742" - ], - "markers": "python_full_version >= '3.6.8'", - "version": "==3.1.2" - }, "pyrsistent": { "hashes": [ "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f", @@ -798,84 +791,85 @@ }, "pytest": { "hashes": [ - "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343", - "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977" + "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181", + "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2" ], "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==8.2.2" + "version": "==8.3.3" }, "python-dateutil": { "hashes": [ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "pytz": { "hashes": [ - "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812", - "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319" + "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a", + "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725" ], - "version": "==2024.1" + "version": "==2024.2" }, "pyyaml": { "hashes": [ - "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", - "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", - "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", - "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", - "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", - "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", - "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", - "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", - "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", - "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", - "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", - "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", - "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", - "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", - "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", - "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", - "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", - "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", - "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", - "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", - "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", - "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", - "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", - "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", - "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", - "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", - "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", - "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", - "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", - "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", - "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", - "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", - "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", - "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", - "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", - "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", - "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", - "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", - "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", - "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", - "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", - "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", - "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", - "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", - "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", - "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", - "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", - "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", - "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", - "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", - "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff", + "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", + "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", + "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", + "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", + "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", + "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", + "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", + "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", + "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", + "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a", + "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", + "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", + "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", + "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", + "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", + "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", + "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a", + "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", + "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", + "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", + "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", + "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", + "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", + "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", + "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", + "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", + "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", + "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", + "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706", + "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", + "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", + "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", + "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083", + "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", + "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", + "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", + "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", + "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", + "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", + "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", + "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", + "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", + "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", + "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5", + "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d", + "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", + "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", + "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", + "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", + "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", + "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", + "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" ], - "markers": "python_version >= '3.6'", - "version": "==6.0.1" + "markers": "python_version >= '3.8'", + "version": "==6.0.2" }, "requests": { "hashes": [ @@ -895,20 +889,19 @@ }, "s3transfer": { "hashes": [ - "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6", - "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69" + "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d", + "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c" ], "markers": "python_version >= '3.8'", - "version": "==0.10.2" + "version": "==0.10.3" }, "sceptre": { "hashes": [ - "sha256:5bb5683233346dc9bb7584d817851403f7b1c483d4d3ace805c76ebd09b9a49a", - "sha256:b5072ca390640f9966bd0277fa3ef26285f741557d8814ef51a81fa715925c6f" + "sha256:94cbafb90a1048a18893788d441f3f8ec5a009a656c53013806e006ab0f49e94", + "sha256:f88bd63b053d66f8f8493453d073c0488f84c7078e10ff65bc3d2421d6b72f36" ], "index": "pypi", - "markers": "python_version >= '3.8' and python_version < '4.0'", - "version": "==4.4.2" + "version": "==4.5.2" }, "sceptre-cmd-resolver": { "hashes": [ @@ -930,31 +923,22 @@ "sha256:f7ed9f80fe7ed4b6b12ba215152d98661e634429958a40419250e085ae51bcad" ], "index": "pypi", - "markers": "python_version >= '3.6'", "version": "==1.0.0" }, - "secretstorage": { - "hashes": [ - "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", - "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99" - ], - "markers": "sys_platform == 'linux'", - "version": "==3.3.3" - }, "setuptools": { "hashes": [ - "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05", - "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1" + "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd", + "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686" ], "markers": "python_version >= '3.8'", - "version": "==70.2.0" + "version": "==75.3.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "synapseclient": { @@ -963,16 +947,15 @@ "sha256:dd8b1a1b4667d08311bb651469431f43fe2eeab83c0ef1fe5a03c2929aeb26cd" ], "index": "pypi", - "markers": "python_version >= '3.7'", "version": "==2.7.2" }, "tomli": { "hashes": [ - "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38", + "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed" ], "markers": "python_version < '3.11'", - "version": "==2.0.1" + "version": "==2.0.2" }, "triad": { "hashes": [ @@ -984,19 +967,19 @@ }, "urllib3": { "hashes": [ - "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", - "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" + "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", + "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.19" + "version": "==1.26.20" }, "virtualenv": { "hashes": [ - "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a", - "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589" + "sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba", + "sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4" ], - "markers": "python_version >= '3.7'", - "version": "==20.26.3" + "markers": "python_version >= '3.8'", + "version": "==20.27.1" }, "websocket-client": { "hashes": [ @@ -1008,11 +991,11 @@ }, "werkzeug": { "hashes": [ - "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18", - "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8" + "sha256:4f7d1a5de312c810a8a2c6f0b47e9f6a7cffb7c8322def35e4d4d9841ff85597", + "sha256:f471a4cd167233077e9d2a8190c3471c5bc520c636a9e3c1e9300c33bced03bc" ], - "markers": "python_version >= '3.8'", - "version": "==3.0.3" + "markers": "python_version >= '3.9'", + "version": "==3.1.2" }, "wrapt": { "hashes": [ @@ -1092,19 +1075,19 @@ }, "xmltodict": { "hashes": [ - "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", - "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553", + "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac" ], - "markers": "python_version >= '3.4'", - "version": "==0.13.0" + "markers": "python_version >= '3.6'", + "version": "==0.14.2" }, "zipp": { "hashes": [ - "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19", - "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c" + "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350", + "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29" ], "markers": "python_version >= '3.8'", - "version": "==3.19.2" + "version": "==3.20.2" } } } diff --git a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml index 1e6d461..d37da00 100644 --- a/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml +++ b/config/develop/namespaced/glue-job-run-great-expectations-on-parquet.yaml @@ -7,12 +7,16 @@ parameters: Namespace: {{ stack_group_config.namespace }} JobDescription: Runs great expectations on a set of data JobRole: !stack_output_external glue-job-role::RoleArn - TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }} + ParquetBucket: {{ stack_group_config.processed_data_bucket_name }} + ShareableArtifactsBucket: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }} S3ScriptBucket: {{ stack_group_config.template_bucket_name }} S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py' + ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json" + GXConfigKey: "{{ stack_group_config.namespace }}/src/glue/resources/great_expectations.yml" GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}" AdditionalPythonModules: "great_expectations~=0.18,urllib3<2" stack_tags: {{ stack_group_config.default_stack_tags }} sceptre_user_data: dataset_schemas: !file src/glue/resources/table_columns.yaml + data_values_expectations: !file src/glue/resources/data_values_expectations.json diff --git a/config/develop/namespaced/glue-workflow.yaml b/config/develop/namespaced/glue-workflow.yaml index 6861a72..1f51a5b 100644 --- a/config/develop/namespaced/glue-workflow.yaml +++ b/config/develop/namespaced/glue-workflow.yaml @@ -20,8 +20,6 @@ parameters: CompareParquetMainNamespace: "main" S3SourceBucketName: {{ stack_group_config.input_bucket_name }} CloudformationBucketName: {{ stack_group_config.template_bucket_name }} - ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }} - ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json" stack_tags: {{ stack_group_config.default_stack_tags }} sceptre_user_data: diff --git a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml index f0e8dd2..93ea716 100644 --- a/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml +++ b/config/prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml @@ -7,12 +7,16 @@ parameters: Namespace: {{ stack_group_config.namespace }} JobDescription: Runs great expectations on a set of data JobRole: !stack_output_external glue-job-role::RoleArn - TempS3Bucket: {{ stack_group_config.processed_data_bucket_name }} + ParquetBucket: {{ stack_group_config.processed_data_bucket_name }} + ShareableArtifactsBucket: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }} S3ScriptBucket: {{ stack_group_config.template_bucket_name }} S3ScriptKey: '{{ stack_group_config.namespace }}/src/glue/jobs/run_great_expectations_on_parquet.py' + ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json" + GXConfigKey: "{{ stack_group_config.namespace }}/src/glue/resources/great_expectations.yml" GlueVersion: "{{ stack_group_config.great_expectations_job_glue_version }}" AdditionalPythonModules: "great_expectations~=0.18,urllib3<2" stack_tags: {{ stack_group_config.default_stack_tags }} sceptre_user_data: dataset_schemas: !file src/glue/resources/table_columns.yaml + data_values_expectations: !file src/glue/resources/data_values_expectations.json diff --git a/config/prod/namespaced/glue-workflow.yaml b/config/prod/namespaced/glue-workflow.yaml index 3223adb..ca20b3f 100644 --- a/config/prod/namespaced/glue-workflow.yaml +++ b/config/prod/namespaced/glue-workflow.yaml @@ -20,8 +20,6 @@ parameters: CompareParquetMainNamespace: "main" S3SourceBucketName: {{ stack_group_config.input_bucket_name }} CloudformationBucketName: {{ stack_group_config.template_bucket_name }} - ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }} - ExpectationSuiteKey: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json" stack_tags: {{ stack_group_config.default_stack_tags }} sceptre_user_data: diff --git a/src/glue/jobs/run_great_expectations_on_parquet.py b/src/glue/jobs/run_great_expectations_on_parquet.py index 68c0e99..c2799b4 100644 --- a/src/glue/jobs/run_great_expectations_on_parquet.py +++ b/src/glue/jobs/run_great_expectations_on_parquet.py @@ -1,23 +1,16 @@ import json import logging +import os import sys from datetime import datetime from typing import Dict import boto3 import great_expectations as gx +import pyspark +import yaml from awsglue.context import GlueContext from awsglue.utils import getResolvedOptions -from great_expectations.core.batch import RuntimeBatchRequest -from great_expectations.core.expectation_configuration import ExpectationConfiguration -from great_expectations.core.run_identifier import RunIdentifier -from great_expectations.core.yaml_handler import YAMLHandler -from great_expectations.data_context.types.base import DataContextConfig -from great_expectations.data_context.types.resource_identifiers import ( - ExpectationSuiteIdentifier, - ValidationResultIdentifier, -) -from pyspark.context import SparkContext logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -39,6 +32,7 @@ def read_args() -> dict: "namespace", "data-type", "expectation-suite-key", + "gx-config-key", ], ) for arg in args: @@ -57,149 +51,75 @@ def validate_args(value: str) -> None: """ if value == "": raise ValueError("Argument value cannot be an empty string") - else: - return None + return None -def create_context( - s3_bucket: str, namespace: str, key_prefix: str -) -> "EphemeralDataContext": - """Creates the data context and adds stores, - datasource and data docs configurations - - Args: - s3_bucket (str): name of s3 bucket to store to - namespace (str): namespace - key_prefix (str): s3 key prefix - - Returns: - EphemeralDataContext: context object with all - configurations - """ - context = gx.get_context() - add_datasource(context) - add_validation_stores(context, s3_bucket, namespace, key_prefix) - add_data_docs_sites(context, s3_bucket, namespace, key_prefix) - return context - +def configure_gx_config( + gx_config_bucket: str, + gx_config_key: str, + shareable_artifacts_bucket: str, + namespace: str, +) -> dict: + """Download and configure a `great_expectations.yml` file locally -def add_datasource(context: "EphemeralDataContext") -> "EphemeralDataContext": - """Adds the spark datasource + This function will download a `great_expectations.yml` file from S3 to a `gx` directory. + This file will be automatically be used to configue the GX data context when calling + `gx.get_context()`. Args: - context (EphemeralDataContext): data context to add to - - Returns: - EphemeralDataContext: data context object with datasource configuration - added + gx_config_bucket (str): S3 bucket containing the `great_expectations.yml` file. + gx_config_key (str): S3 key where this file is located. + shareable_artifacts_bucket (str): S3 bucket where shareable artifacts are written. + namespace (str): The current namespace """ - yaml = YAMLHandler() - context.add_datasource( - **yaml.load( - """ - name: spark_datasource - class_name: Datasource - execution_engine: - class_name: SparkDFExecutionEngine - force_reuse_spark_context: true - data_connectors: - runtime_data_connector: - class_name: RuntimeDataConnector - batch_identifiers: - - batch_identifier - """ + gx_config_path = "gx/great_expectations.yml" + os.makedirs("gx", exist_ok=True) + s3_client = boto3.client("s3") + logger.info( + f"Downloading s3://{gx_config_bucket}/{gx_config_key} to {gx_config_path}" + ) + s3_client.download_file( + Bucket=gx_config_bucket, Key=gx_config_key, Filename=gx_config_path + ) + with open(gx_config_path, "rb") as gx_config_obj: + gx_config = yaml.safe_load(gx_config_obj) + # fmt: off + gx_config["stores"]["validations_store"]["store_backend"]["bucket"] = ( + shareable_artifacts_bucket + ) + gx_config["stores"]["validations_store"]["store_backend"]["prefix"] = ( + gx_config["stores"]["validations_store"]["store_backend"]["prefix"].format( + namespace=namespace ) ) - return context - - -def add_validation_stores( - context: "EphemeralDataContext", - s3_bucket: str, - namespace: str, - key_prefix: str, -) -> "EphemeralDataContext": - """Adds the validation store configurations to the context object - - Args: - context (EphemeralDataContext): data context to add to - s3_bucket (str): name of the s3 bucket to save validation results to - namespace (str): name of the namespace - key_prefix (str): s3 key prefix to save the - validation results to - - Returns: - EphemeralDataContext: data context object with validation stores' - configuration added - """ - # Programmatically configure the validation result store and - # DataDocs to use S3 - context.add_store( - "validation_result_store", - { - "class_name": "ValidationsStore", - "store_backend": { - "class_name": "TupleS3StoreBackend", - "bucket": s3_bucket, - "prefix": f"{namespace}/{key_prefix}", - }, - }, + gx_config["data_docs_sites"]["s3_site"]["store_backend"]["bucket"] = ( + shareable_artifacts_bucket ) - return context - - -def add_data_docs_sites( - context: "EphemeralDataContext", - s3_bucket: str, - namespace: str, - key_prefix: str, -) -> "EphemeralDataContext": - """Adds the data docs sites configuration to the context object - so data docs can be saved to a s3 location. This is a special - workaround to add the data docs because we're using EphemeralDataContext - context objects and they don't store to memory. - - Args: - context (EphemeralDataContext): data context to add to - s3_bucket (str): name of the s3 bucket to save gx docs to - namespace (str): name of the namespace - key_prefix (str): s3 key prefix to save the - gx docs to - - Returns: - EphemeralDataContext: data context object with data docs sites' - configuration added - """ - data_context_config = DataContextConfig() - data_context_config["data_docs_sites"] = { - "s3_site": { - "class_name": "SiteBuilder", - "store_backend": { - "class_name": "TupleS3StoreBackend", - "bucket": s3_bucket, - "prefix": f"{namespace}/{key_prefix}", - }, - "site_index_builder": {"class_name": "DefaultSiteIndexBuilder"}, - } - } - context._project_config["data_docs_sites"] = data_context_config["data_docs_sites"] - return context + gx_config["data_docs_sites"]["s3_site"]["store_backend"]["prefix"] = ( + gx_config["data_docs_sites"]["s3_site"]["store_backend"]["prefix"].format( + namespace=namespace + ) + ) + # fmt: on + with open(gx_config_path, "w", encoding="utf-8") as gx_config_obj: + yaml.dump(gx_config, gx_config_obj) + return gx_config def get_spark_df( glue_context: GlueContext, parquet_bucket: str, namespace: str, data_type: str ) -> "pyspark.sql.dataframe.DataFrame": - """Reads in the parquet dataset as a Dynamic Frame and converts it - to a spark dataframe + """ + Read a data-type-specific Parquet dataset Args: - glue_context (GlueContext): the aws glue context object - parquet_bucket (str): the name of the bucket holding parquet files - namespace (str): the namespace - data_type (str): the data type name + glue_context (GlueContext): The AWS Glue context object + parquet_bucket (str): The S3 bucket containing the data-type-specific Parquet dataset + namespace (str): The associated namespace + data_type (str): The associated data type Returns: - pyspark.sql.dataframe.DataFrame: spark dataframe of the read in parquet dataset + pyspark.sql.dataframe.DataFrame: A Spark dataframe over our data-type-specific Parquet dataset """ s3_parquet_path = f"s3://{parquet_bucket}/{namespace}/parquet/dataset_{data_type}/" dynamic_frame = glue_context.create_dynamic_frame_from_options( @@ -212,29 +132,24 @@ def get_spark_df( def get_batch_request( + gx_context: gx.data_context.AbstractDataContext, spark_dataset: "pyspark.sql.dataframe.DataFrame", data_type: str, - run_id: RunIdentifier, -) -> RuntimeBatchRequest: - """Retrieves the unique metadata for this batch request +) -> gx.datasource.fluent.batch_request.BatchRequest: + """ + Get a GX batch request over a Spark dataframe Args: - spark_dataset (pyspark.sql.dataframe.DataFrame): parquet dataset as spark df - data_type (str): data type name - run_id (RunIdentifier): contains the run name and - run time metadata of this batch run + spark_dataset (pyspark.sql.dataframe.DataFrame): A Spark dataframe + data_type (str): The data type Returns: - RuntimeBatchRequest: contains metadata for the batch run request - to identify this great expectations run + BatchRequest: A batch request which can be used in conjunction + with an expectation suite to validate our data. """ - batch_request = RuntimeBatchRequest( - datasource_name="spark_datasource", - data_connector_name="runtime_data_connector", - data_asset_name=f"{data_type}-parquet-data-asset", - runtime_parameters={"batch_data": spark_dataset}, - batch_identifiers={"batch_identifier": f"{data_type}_{run_id.run_name}_batch"}, - ) + data_source = gx_context.sources.add_or_update_spark(name="parquet") + data_asset = data_source.add_dataframe_asset(name=f"{data_type}_spark_dataframe") + batch_request = data_asset.build_batch_request(dataframe=spark_dataset) return batch_request @@ -243,13 +158,13 @@ def read_json( s3_bucket: str, key: str, ) -> Dict[str, str]: - """Reads in a json object + """ + Read a JSON file from an S3 bucket Args: - s3 (boto3.client): s3 client connection - s3_bucket (str): name of the s3 bucket to read from - key (str): s3 key prefix of the - location of the json to read from + s3 (boto3.client): An S3 client + s3_bucket (str): The S3 bucket containing the JSON file + key (str): The S3 key of the JSON file Returns: Dict[str, str]: the data read in from json @@ -263,112 +178,65 @@ def read_json( def add_expectations_from_json( expectations_data: Dict[str, str], - context: "EphemeralDataContext", - data_type: str, -) -> "EphemeralDataContext": - """Adds in the read in expectations to the context object - - Args: - expectations_data (Dict[str, str]): expectations - context (EphemeralDataContext): context object - data_type (str): name of the data type - - Raises: - ValueError: thrown when no expectations exist for this data type - - Returns: - EphemeralDataContext: context object with expectations added + context: gx.data_context.AbstractDataContext, +) -> gx.data_context.AbstractDataContext: """ - # Ensure the data type exists in the JSON file - if data_type not in expectations_data: - raise ValueError(f"No expectations found for data type '{data_type}'") - - # Extract the expectation suite and expectations for the dataset - suite_data = expectations_data[data_type] - expectation_suite_name = suite_data["expectation_suite_name"] - new_expectations = suite_data["expectations"] - - # Convert new expectations from JSON format to ExpectationConfiguration objects - new_expectations_configs = [ - ExpectationConfiguration( - expectation_type=exp["expectation_type"], kwargs=exp["kwargs"] - ) - for exp in new_expectations - ] - - # Update the expectation suite in the data context - context.add_or_update_expectation_suite( - expectation_suite_name=expectation_suite_name, - expectations=new_expectations_configs, - ) - return context - - -def add_validation_results_to_store( - context: "EphemeralDataContext", - expectation_suite_name: str, - validation_result: Dict[str, str], - batch_identifier: RuntimeBatchRequest, - run_identifier: RunIdentifier, -) -> "EphemeralDataContext": - """Adds the validation results manually to the validation store. - This is a workaround for a EphemeralDataContext context object, - and for us to avoid complicating our folder structure to include - checkpoints/other more persistent data context object types - until we need that feature + Add an expectation suite with expectations to our GX data context for each data type. Args: - context (EphemeralDataContext): context object to add results to - expectation_suite_name (str): name of expectation suite - validation_result (Dict[str, str]): results outputted by gx - validator to be stored - batch_identifier (RuntimeBatchRequest): metadata containing details of - the batch request - run_identifier (RunIdentifier): metadata containing details of the gx run + expectations_data (Dict[str, str]): A mapping of data types to their expectations. + The expectations should be formatted like so: + + { + "expectation_suite_name": "string", + "expectations": { + "expectation_type": "str", + "kwargs": "readable by `ExpectationConfiguration`" + } + } + context (gx.data_context.AbstractDataContext): context object Returns: - EphemeralDataContext: context object with validation results added to + gx.data_context.AbstractDataContext: A GX data context object with expectation suites added """ - expectation_suite = context.get_expectation_suite(expectation_suite_name) - # Create an ExpectationSuiteIdentifier - expectation_suite_identifier = ExpectationSuiteIdentifier( - expectation_suite_name=expectation_suite.expectation_suite_name - ) - - # Create a ValidationResultIdentifier using the run_id, expectation suite, and batch identifier - validation_result_identifier = ValidationResultIdentifier( - expectation_suite_identifier=expectation_suite_identifier, - batch_identifier=batch_identifier, - run_id=run_identifier, - ) - - context.validations_store.set(validation_result_identifier, validation_result) + for data_type in expectations_data: + suite_data = expectations_data[data_type] + expectation_suite_name = suite_data["expectation_suite_name"] + new_expectations = suite_data["expectations"] + + # Convert new expectations from dict to ExpectationConfiguration objects + new_expectations_configs = [ + gx.core.expectation_configuration.ExpectationConfiguration( + expectation_type=exp["expectation_type"], kwargs=exp["kwargs"] + ) + for exp in new_expectations + ] + + # Update the expectation suite in the data context + context.add_or_update_expectation_suite( + expectation_suite_name=expectation_suite_name, + expectations=new_expectations_configs, + ) return context def main(): args = read_args() - run_id = RunIdentifier(run_name=f"run_{datetime.now().strftime('%Y%m%d_%H%M%S')}") - expectation_suite_name = f"{args['data_type']}_expectations" s3 = boto3.client("s3") - context = create_context( - s3_bucket=args["shareable_artifacts_bucket"], - namespace=args["namespace"], - key_prefix=f"great_expectation_reports/{args['data_type']}/parquet/", + run_id = gx.core.run_identifier.RunIdentifier( + run_name=f"run_{datetime.now().strftime('%Y%m%d_%H%M%S')}" ) - glue_context = GlueContext(SparkContext.getOrCreate()) - logger.info("get_spark_df") - spark_df = get_spark_df( - glue_context=glue_context, - parquet_bucket=args["parquet_bucket"], + expectation_suite_name = f"{args['data_type']}_expectations" + + # Set up Great Expectations + logger.info("configure_gx_config") + configure_gx_config( + gx_config_bucket=args["cfn_bucket"], + gx_config_key=args["gx_config_key"], + shareable_artifacts_bucket=args["shareable_artifacts_bucket"], namespace=args["namespace"], - data_type=args["data_type"], ) - logger.info("get_batch_request") - batch_request = get_batch_request(spark_df, args["data_type"], run_id) - logger.info("add_expectations") - - # Load the JSON file with the expectations + gx_context = gx.get_context() logger.info("reads_expectations_from_json") expectations_data = read_json( s3=s3, @@ -376,32 +244,37 @@ def main(): key=args["expectation_suite_key"], ) logger.info("adds_expectations_from_json") - add_expectations_from_json( + gx_context = add_expectations_from_json( expectations_data=expectations_data, - context=context, - data_type=args["data_type"], - ) - logger.info("get_validator") - validator = context.get_validator( - batch_request=batch_request, - expectation_suite_name=expectation_suite_name, + context=gx_context, ) - logger.info("validator.validate") - validation_result = validator.validate() - logger.info("validation_result: %s", validation_result) + # Set up Spark + glue_context = GlueContext(pyspark.context.SparkContext.getOrCreate()) + logger.info("get_spark_df") + spark_df = get_spark_df( + glue_context=glue_context, + parquet_bucket=args["parquet_bucket"], + namespace=args["namespace"], + data_type=args["data_type"], + ) - add_validation_results_to_store( - context, - expectation_suite_name, - validation_result, - batch_identifier=batch_request["batch_identifiers"]["batch_identifier"], - run_identifier=run_id, + # Put the two together and validate the GX expectations + logger.info("get_batch_request") + batch_request = get_batch_request( + gx_context=gx_context, spark_dataset=spark_df, data_type=args["data_type"] ) - context.build_data_docs( - site_names=["s3_site"], + logger.info("add_or_update_checkpoint") + # The default checkpoint action list is: + # StoreValidationResultAction, StoreEvaluationParametersAction, UpdateDataDocsAction + checkpoint = gx_context.add_or_update_checkpoint( + name=f"{args['data_type']}-checkpoint", + expectation_suite_name=expectation_suite_name, + batch_request=batch_request, ) - logger.info("data docs saved!") + logger.info("run checkpoint") + checkpoint_result = checkpoint.run(run_id=run_id) + logger.info("data docs updated!") if __name__ == "__main__": diff --git a/src/glue/resources/great_expectations.yml b/src/glue/resources/great_expectations.yml new file mode 100644 index 0000000..d210b4c --- /dev/null +++ b/src/glue/resources/great_expectations.yml @@ -0,0 +1,50 @@ +config_version: 3.0 +stores: + expectations_store: + class_name: ExpectationsStore + store_backend: + class_name: TupleFilesystemStoreBackend + base_directory: expectations/ + validations_store: + class_name: ValidationsStore + store_backend: + class_name: TupleS3StoreBackend + suppress_store_backend_id: true + bucket: "{shareable_artifacts_bucket}" + prefix: "{namespace}/great_expectation_reports/parquet/validations/" + evaluation_parameter_store: + class_name: EvaluationParameterStore + checkpoint_store: + class_name: CheckpointStore + store_backend: + class_name: TupleFilesystemStoreBackend + base_directory: checkpoints/ + profiler_store: + class_name: ProfilerStore + store_backend: + class_name: TupleFilesystemStoreBackend + base_directory: profilers/ +expectations_store_name: expectations_store +validations_store_name: validations_store +evaluation_parameter_store_name: evaluation_parameter_store +checkpoint_store_name: checkpoint_store +data_docs_sites: + s3_site: + class_name: SiteBuilder + store_backend: + class_name: TupleS3StoreBackend + bucket: "{shareable_artifacts_bucket}" + prefix: "{namespace}/great_expectation_reports/parquet/" + site_index_builder: + class_name: DefaultSiteIndexBuilder +include_rendered_content: + globally: false + expectation_suite: false + expectation_validation_result: false +fluent_datasources: + my_parquet_datasource: + type: spark + assets: + my_dataframe_asset: + type: dataframe + batch_metadata: {} diff --git a/src/scripts/manage_artifacts/artifacts.py b/src/scripts/manage_artifacts/artifacts.py index cef9399..0e11602 100755 --- a/src/scripts/manage_artifacts/artifacts.py +++ b/src/scripts/manage_artifacts/artifacts.py @@ -10,6 +10,7 @@ def read_args(): parser = argparse.ArgumentParser(description="") parser.add_argument("--namespace") parser.add_argument("--cfn_bucket", required=True) + parser.add_argument("--shareable-artifacts-bucket", required=True) group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--upload", action="store_true") group.add_argument("--remove", action="store_true") @@ -18,9 +19,9 @@ def read_args(): return args -def execute_command(cmd: str): +def execute_command(cmd: list[str]): print(f'Invoking command: {" ".join(cmd)}') - subprocess.run(cmd) + subprocess.run(cmd, check=True) def upload(namespace: str, cfn_bucket: str): @@ -65,6 +66,7 @@ def main(args): upload(args.namespace, args.cfn_bucket) elif args.remove: delete(args.namespace, args.cfn_bucket) + delete(args.namespace, args.shareable_artifacts_bucket) else: list_namespaces(args.cfn_bucket) diff --git a/templates/glue-job-run-great-expectations-on-parquet.j2 b/templates/glue-job-run-great-expectations-on-parquet.j2 index 455b5e8..a125857 100644 --- a/templates/glue-job-run-great-expectations-on-parquet.j2 +++ b/templates/glue-job-run-great-expectations-on-parquet.j2 @@ -19,9 +19,14 @@ Parameters: Type: String Description: The name or ARN of the IAM role that will run this job. - TempS3Bucket: + ParquetBucket: Type: String - Description: The name of the S3 bucket where temporary files and logs are written. + Description: The name of the S3 bucket where Parquet data is written. This is + also where we will write temporary files and logs. + + ShareableArtifactsBucket: + Type: String + Description: The name of the bucket where shareable artifacts are stored. S3ScriptBucket: Type: String @@ -48,6 +53,14 @@ Parameters: FitbitIntradayCombined and HealthKitV2Samples. Default: 1 + ExpectationSuiteKey: + Type: String + Description: The S3 key of the GX expectation file. + + GXConfigKey: + Type: String + Description: The S3 key of the GX configuration file/template. + MaxRetries: Type: Number Description: How many times to retry the job if it fails (integer). @@ -71,12 +84,11 @@ Resources: {% for v in sceptre_user_data.dataset_schemas.tables.keys() if not "Deleted" in v %} {% set dataset = {} %} {% do dataset.update({"type": v}) %} - {% do dataset.update({"table_name": "dataset_" + v.lower()})%} {% do dataset.update({"stackname_prefix": "{}".format(v.replace("_",""))}) %} {% do datasets.append(dataset) %} {% endfor %} - {% for dataset in datasets %} + {% for dataset in datasets if dataset["type"].lower() in sceptre_user_data.data_values_expectations %} {{ dataset["stackname_prefix"] }}GreatExpectationsParquetJob: Type: AWS::Glue::Job Properties: @@ -84,15 +96,21 @@ Resources: Name: glueetl ScriptLocation: !Sub s3://${S3ScriptBucket}/${S3ScriptKey} DefaultArguments: - --TempDir: !Sub s3://${TempS3Bucket}/tmp + --TempDir: !Sub s3://${ParquetBucket}/tmp --enable-continuous-cloudwatch-log: true --enable-metrics: true --enable-spark-ui: true - --spark-event-logs-path: !Sub s3://${TempS3Bucket}/spark-logs/${AWS::StackName}/ + --spark-event-logs-path: !Sub s3://${ParquetBucket}/spark-logs/${AWS::StackName}/ --job-bookmark-option: job-bookmark-disable --job-language: python --additional-python-modules: !Ref AdditionalPythonModules - # --conf spark.sql.adaptive.enabled + --data-type: {{ dataset["type"].lower() }} + --namespace: !Ref Namespace + --cfn-bucket: !Ref S3ScriptBucket + --parquet-bucket: !Ref ParquetBucket + --shareable-artifacts-bucket: !Ref ShareableArtifactsBucket + --expectation-suite-key: !Ref ExpectationSuiteKey + --gx-config-key: !Ref GXConfigKey Description: !Sub "${JobDescription} for data type {{ dataset['type'] }}" GlueVersion: !Ref GlueVersion MaxRetries: !Ref MaxRetries diff --git a/templates/glue-workflow.j2 b/templates/glue-workflow.j2 index 16d8950..5483368 100644 --- a/templates/glue-workflow.j2 +++ b/templates/glue-workflow.j2 @@ -40,7 +40,7 @@ Parameters: ParquetKeyPrefix: Type: String - Description: S3 key prefix where JSON datasets are stored. + Description: S3 key prefix where Parquet datasets are stored. Default: parquet GlueDatabase: @@ -82,13 +82,6 @@ Parameters: Description: >- The name of the bucket where the cloudformation and artifacts are stored. - ShareableArtifactsBucketName: - Type: String - Description: The name of the bucket where shareable artifacts are stored. - - ExpectationSuiteKey: - Type: String - Description: The s3 key prefix of the expectation suite. Conditions: IsMainNamespace: !Equals [!Ref Namespace, "main"] @@ -316,14 +309,6 @@ Resources: Name: !Sub "${Namespace}-{{ dataset['stackname_prefix'] }}GreatExpectationsParquetTrigger" Actions: - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-GreatExpectationsParquetJob - Arguments: - "--data-type": {{ dataset["data_type"].lower() }} - "--namespace": !Ref Namespace - "--cfn-bucket": !Ref CloudformationBucketName - "--parquet-bucket": !Ref ParquetBucketName - "--shareable-artifacts-bucket": !Ref ShareableArtifactsBucketName - "--expectation-suite-key": !Ref ExpectationSuiteKey - "--additional-python-modules": "great_expectations~=0.18,urllib3<2" Description: This trigger runs the great expectation parquet job for this data type after completion of the JSON to Parquet job for this data type Type: CONDITIONAL Predicate: diff --git a/tests/test_run_great_expectations_on_parquet.py b/tests/test_run_great_expectations_on_parquet.py index 074e140..2168725 100644 --- a/tests/test_run_great_expectations_on_parquet.py +++ b/tests/test_run_great_expectations_on_parquet.py @@ -1,152 +1,169 @@ -from unittest.mock import MagicMock, patch +import os +import shutil +import unittest +import boto3 import great_expectations as gx +import pyspark import pytest -from great_expectations.core.batch import RuntimeBatchRequest -from great_expectations.core.run_identifier import RunIdentifier -from great_expectations.core.yaml_handler import YAMLHandler -from great_expectations.data_context.types.resource_identifiers import ( - ExpectationSuiteIdentifier, - ValidationResultIdentifier, -) -from pyspark.sql import SparkSession +import yaml +from moto import mock_s3 from src.glue.jobs import run_great_expectations_on_parquet as run_gx_on_pq @pytest.fixture -def test_context(scope="function"): +def gx_context(scope="function"): context = gx.get_context() yield context @pytest.fixture(scope="function") -def test_spark(): - yield SparkSession.builder.appName("BatchRequestTest").getOrCreate() - - -def test_create_context(): - with ( - patch.object(gx, "get_context") as mock_get_context, - patch.object(run_gx_on_pq, "add_datasource") as mock_add_datasource, - patch.object( - run_gx_on_pq, "add_validation_stores" - ) as mock_add_validation_stores, - patch.object(run_gx_on_pq, "add_data_docs_sites") as mock_add_data_docs_sites, - ): - mock_context = MagicMock() - mock_get_context.return_value = mock_context - - s3_bucket = "test-bucket" - namespace = "test-namespace" - key_prefix = "test-prefix" - - # Call the function - result_context = run_gx_on_pq.create_context(s3_bucket, namespace, key_prefix) - - # Assert that the context returned is the mock context - assert result_context == mock_context - - # Assert that the other functions were called - mock_add_datasource.assert_called_once_with(mock_context) - mock_add_validation_stores.assert_called_once_with( - mock_context, s3_bucket, namespace, key_prefix - ) - mock_add_data_docs_sites.assert_called_once_with( - mock_context, s3_bucket, namespace, key_prefix +def spark_session(): + yield pyspark.sql.SparkSession.builder.appName("BatchRequestTest").getOrCreate() + + +@pytest.fixture() +def cloudformation_bucket(): + with mock_s3(): + # Create a mock S3 client + s3 = boto3.client("s3") + + # Define the bucket name + bucket_name = "test-great-expectations-bucket" + + # Create the mock bucket + s3.create_bucket(Bucket=bucket_name) + + # Create a sample great_expectations.yml with just the components we modify + great_expectations_content = """ + config_version: 3.0 + stores: + validations_store: + class_name: ValidationsStore + store_backend: + class_name: TupleS3StoreBackend + suppress_store_backend_id: true + bucket: "{shareable_artifacts_bucket}" + prefix: "{namespace}/great_expectation_reports/parquet/validations/" + data_docs_sites: + s3_site: + class_name: SiteBuilder + store_backend: + class_name: TupleS3StoreBackend + bucket: "{shareable_artifacts_bucket}" + prefix: "{namespace}/great_expectation_reports/parquet/" + site_index_builder: + class_name: DefaultSiteIndexBuilder + """ + + # Upload the great_expectations.yml file to the mocked bucket + s3.put_object( + Bucket=bucket_name, + Key="great_expectations.yml", + Body=great_expectations_content, ) + # Yield the bucket name for use in tests + yield { + "bucket": bucket_name, + "great_expectations_configuration_key": "great_expectations.yml", + "great_expectations_content": great_expectations_content, + } -def test_that_add_datasource_calls_correctly(): - mock_context = MagicMock() - result_context = run_gx_on_pq.add_datasource(mock_context) - # Verify that the datasource was added - mock_context.add_datasource.assert_called_once() - assert result_context == mock_context +@pytest.fixture() +def clean_up_after_configure_gx_config(): + """Remove artifacts of `configure_gx_config` function""" + yield + if os.path.isdir("gx"): + shutil.rmtree("gx") -@pytest.mark.integration -def test_that_add_datasource_adds_correctly(test_context): - # Assuming you've already added a datasource, you can list it - run_gx_on_pq.add_datasource(test_context) - datasources = test_context.list_datasources() - - # Define the expected datasource name - expected_datasource_name = "spark_datasource" - - # Check that the expected datasource is present and other details are correct - assert any( - ds["name"] == expected_datasource_name for ds in datasources - ), f"Datasource '{expected_datasource_name}' was not added correctly." - datasource = next( - ds for ds in datasources if ds["name"] == expected_datasource_name +def test_configure_gx_config_validations_store_bucket( + cloudformation_bucket, clean_up_after_configure_gx_config +): + gx_config = run_gx_on_pq.configure_gx_config( + gx_config_bucket=cloudformation_bucket["bucket"], + gx_config_key=cloudformation_bucket["great_expectations_configuration_key"], + shareable_artifacts_bucket="shareable_artifacts_bucket", + namespace="namespace", + ) + assert ( + gx_config["stores"]["validations_store"]["store_backend"]["bucket"] + == "shareable_artifacts_bucket" ) - assert datasource["class_name"] == "Datasource" - assert "SparkDFExecutionEngine" in datasource["execution_engine"]["class_name"] - - -def test_add_validation_stores_has_expected_calls(): - mock_context = MagicMock() - s3_bucket = "test-bucket" - namespace = "test-namespace" - key_prefix = "test-prefix" - with patch.object(mock_context, "add_store") as mock_add_store: - # Call the function - result_context = run_gx_on_pq.add_validation_stores( - mock_context, s3_bucket, namespace, key_prefix - ) - # Verify that the validation store is added - mock_add_store.assert_called_once_with( - "validation_result_store", - { - "class_name": "ValidationsStore", - "store_backend": { - "class_name": "TupleS3StoreBackend", - "bucket": s3_bucket, - "prefix": f"{namespace}/{key_prefix}", - }, - }, +def test_configure_gx_config_validations_store_prefix( + cloudformation_bucket, clean_up_after_configure_gx_config +): + gx_config = run_gx_on_pq.configure_gx_config( + gx_config_bucket=cloudformation_bucket["bucket"], + gx_config_key=cloudformation_bucket["great_expectations_configuration_key"], + shareable_artifacts_bucket="shareable_artifacts_bucket", + namespace="namespace", + ) + original_gx_config = yaml.safe_load( + cloudformation_bucket["great_expectations_content"] + ) + # fmt: off + assert ( + gx_config["stores"]["validations_store"]["store_backend"]["prefix"] + == original_gx_config["stores"]["validations_store"]["store_backend"]["prefix"].format( + namespace="namespace" ) - - assert result_context == mock_context + ) + # fmt: on -@pytest.mark.integration -def test_validation_store_details(test_context): - # Mock context and stores - run_gx_on_pq.add_validation_stores( - test_context, - s3_bucket="test-bucket", - namespace="test", - key_prefix="test_folder/", +def test_configure_gx_config_data_docs_sites_bucket( + cloudformation_bucket, clean_up_after_configure_gx_config +): + gx_config = run_gx_on_pq.configure_gx_config( + gx_config_bucket=cloudformation_bucket["bucket"], + gx_config_key=cloudformation_bucket["great_expectations_configuration_key"], + shareable_artifacts_bucket="shareable_artifacts_bucket", + namespace="namespace", + ) + original_gx_config = yaml.safe_load( + cloudformation_bucket["great_expectations_content"] + ) + assert ( + gx_config["data_docs_sites"]["s3_site"]["store_backend"]["bucket"] + == "shareable_artifacts_bucket" ) - # Run the test logic - stores = test_context.list_stores() - expected_store_name = "validation_result_store" - - assert any(store["name"] == expected_store_name for store in stores) - # pulls the store we want - store_config = [store for store in stores if store["name"] == expected_store_name][ - 0 - ] - assert store_config["class_name"] == "ValidationsStore" - assert store_config["store_backend"]["class_name"] == "TupleS3StoreBackend" - assert store_config["store_backend"]["bucket"] == "test-bucket" - assert store_config["store_backend"]["prefix"] == "test/test_folder/" +def test_configure_gx_config_data_docs_sites_prefix( + cloudformation_bucket, clean_up_after_configure_gx_config +): + gx_config = run_gx_on_pq.configure_gx_config( + gx_config_bucket=cloudformation_bucket["bucket"], + gx_config_key=cloudformation_bucket["great_expectations_configuration_key"], + shareable_artifacts_bucket="shareable_artifacts_bucket", + namespace="namespace", + ) + original_gx_config = yaml.safe_load( + cloudformation_bucket["great_expectations_content"] + ) + # fmt: off + assert ( + gx_config["data_docs_sites"]["s3_site"]["store_backend"]["prefix"] + == original_gx_config["data_docs_sites"]["s3_site"]["store_backend"]["prefix"].format( + namespace="namespace" + ) + ) + # fmt: on def test_get_spark_df_has_expected_calls(): - glue_context = MagicMock() - mock_dynamic_frame = MagicMock() - mock_spark_df = MagicMock() + glue_context = unittest.mock.MagicMock() + mock_dynamic_frame = unittest.mock.MagicMock() + mock_spark_df = unittest.mock.MagicMock() mock_dynamic_frame.toDF.return_value = mock_spark_df - with patch.object( + with unittest.mock.patch.object( glue_context, "create_dynamic_frame_from_options" ) as mock_create_dynamic_frame: mock_create_dynamic_frame.return_value = mock_dynamic_frame @@ -171,46 +188,14 @@ def test_get_spark_df_has_expected_calls(): assert result_df == mock_spark_df -def test_get_batch_request(): - spark_dataset = MagicMock() +def test_get_batch_request(gx_context): + spark_dataset = unittest.mock.MagicMock() data_type = "test-data" - run_id = RunIdentifier(run_name="2023_09_04") - - batch_request = run_gx_on_pq.get_batch_request(spark_dataset, data_type, run_id) - - # Verify the RuntimeBatchRequest is correctly set up - assert isinstance(batch_request, RuntimeBatchRequest) - assert batch_request.data_asset_name == f"{data_type}-parquet-data-asset" - assert batch_request.batch_identifiers == { - "batch_identifier": f"{data_type}_{run_id.run_name}_batch" - } - assert batch_request.runtime_parameters == {"batch_data": spark_dataset} - - -@pytest.mark.integration -def test_that_get_batch_request_details_are_correct(test_spark): - # Create a simple PySpark DataFrame to simulate the dataset - data = [("Alice", 34), ("Bob", 45), ("Charlie", 29)] - columns = ["name", "age"] - spark_dataset = test_spark.createDataFrame(data, columns) - - # Create a RunIdentifier - run_id = RunIdentifier(run_name="test_run_2023") - - # Call the function and get the RuntimeBatchRequest - data_type = "user_data" - batch_request = run_gx_on_pq.get_batch_request(spark_dataset, data_type, run_id) - - # Assertions to check that the batch request is properly populated - assert isinstance(batch_request, RuntimeBatchRequest) - assert batch_request.datasource_name == "spark_datasource" - assert batch_request.data_connector_name == "runtime_data_connector" - assert batch_request.data_asset_name == "user_data-parquet-data-asset" - assert ( - batch_request.batch_identifiers["batch_identifier"] - == "user_data_test_run_2023_batch" + batch_request = run_gx_on_pq.get_batch_request( + gx_context=gx_context, spark_dataset=spark_dataset, data_type=data_type ) - assert batch_request.runtime_parameters["batch_data"] == spark_dataset + assert isinstance(batch_request, gx.datasource.fluent.batch_request.BatchRequest) + assert batch_request.data_asset_name == "test-data_spark_dataframe" def test_read_json_correctly_returns_expected_values(): @@ -218,13 +203,13 @@ def test_read_json_correctly_returns_expected_values(): key = "test-key" # Mock the S3 response - mock_s3_response = MagicMock() + mock_s3_response = unittest.mock.MagicMock() mock_s3_response["Body"].read.return_value = '{"test_key": "test_value"}'.encode( "utf-8" ) # Use patch to mock the boto3 s3 client - with patch("boto3.client") as mock_s3_client: + with unittest.mock.patch("boto3.client") as mock_s3_client: # Mock get_object method mock_s3_client.return_value.get_object.return_value = mock_s3_response @@ -240,62 +225,25 @@ def test_read_json_correctly_returns_expected_values(): assert result == {"test_key": "test_value"} -def test_that_add_expectations_from_json_has_expected_call(): - mock_context = MagicMock() - - # Sample expectations data +@pytest.mark.integration +def test_add_expectations_from_json_adds_details_correctly(gx_context): + # Mock expectations data expectations_data = { - "test-data": { - "expectation_suite_name": "test_suite", + "user_data_one": { + "expectation_suite_name": "user_data_one_suite", "expectations": [ { "expectation_type": "expect_column_to_exist", - "kwargs": {"column": "test_column"}, + "kwargs": {"column": "user_id"}, }, - ], - } - } - - data_type = "test-data" - - # Call the function - run_gx_on_pq.add_expectations_from_json(expectations_data, mock_context, data_type) - - # Verify expectations were added to the context - mock_context.add_or_update_expectation_suite.assert_called_once() - - -def test_that_add_expectations_from_json_throws_value_error(): - mock_context = MagicMock() - - # Sample expectations data - expectations_data = { - "not-test-data": { - "expectation_suite_name": "test_suite", - "expectations": [ { - "expectation_type": "expect_column_to_exist", - "kwargs": {"column": "test_column"}, + "expectation_type": "expect_column_values_to_be_between", + "kwargs": {"column": "age", "min_value": 18, "max_value": 65}, }, ], - } - } - - data_type = "test-data" - with pytest.raises( - ValueError, match="No expectations found for data type 'test-data'" - ): - run_gx_on_pq.add_expectations_from_json( - expectations_data, mock_context, data_type - ) - - -@pytest.mark.integration -def test_add_expectations_from_json_adds_details_correctly(test_context): - # Mock expectations data - expectations_data = { - "user_data": { - "expectation_suite_name": "user_data_suite", + }, + "user_data_two": { + "expectation_suite_name": "user_data_two_suite", "expectations": [ { "expectation_type": "expect_column_to_exist", @@ -306,20 +254,25 @@ def test_add_expectations_from_json_adds_details_correctly(test_context): "kwargs": {"column": "age", "min_value": 18, "max_value": 65}, }, ], - } + }, } - data_type = "user_data" - # Call the function to add expectations - test_context = run_gx_on_pq.add_expectations_from_json( - expectations_data, test_context, data_type + run_gx_on_pq.add_expectations_from_json( + expectations_data=expectations_data, context=gx_context ) + expectation_suites_in_store = [ + suite.expectation_suite_name + for suite in gx_context.expectations_store.list_keys() + ] + assert "user_data_one_suite" in expectation_suites_in_store + assert "user_data_two_suite" in expectation_suites_in_store + # Retrieve the expectation suite to verify that expectations were added - expectation_suite = test_context.get_expectation_suite("user_data_suite") + expectation_suite = gx_context.get_expectation_suite("user_data_one_suite") - assert expectation_suite.expectation_suite_name == "user_data_suite" + assert expectation_suite.expectation_suite_name == "user_data_one_suite" assert len(expectation_suite.expectations) == 2 # Verify the details of the first expectation @@ -335,47 +288,3 @@ def test_add_expectations_from_json_adds_details_correctly(test_context): "min_value": 18, "max_value": 65, } - - -def test_that_add_validation_results_to_store_has_expected_calls(): - # Mock the EphemeralDataContext and the necessary components - mock_context = MagicMock() - mock_expectation_suite = MagicMock() - mock_context.get_expectation_suite.return_value = mock_expectation_suite - mock_expectation_suite.expectation_suite_name = "test_suite" - - # Mock the validation result data - validation_result = {"result": "test_result"} - - # Create a mock batch identifier and run identifier - mock_batch_identifier = MagicMock(spec=RuntimeBatchRequest) - mock_run_identifier = MagicMock(spec=RunIdentifier) - - # Call the function with mocked inputs - result_context = run_gx_on_pq.add_validation_results_to_store( - context=mock_context, - expectation_suite_name="test_suite", - validation_result=validation_result, - batch_identifier=mock_batch_identifier, - run_identifier=mock_run_identifier, - ) - - # Assert that the expectation suite was retrieved correctly - mock_context.get_expectation_suite.assert_called_once_with("test_suite") - - expected_expectation_suite_identifier = ExpectationSuiteIdentifier( - expectation_suite_name="test_suite" - ) - expected_validation_result_identifier = ValidationResultIdentifier( - expectation_suite_identifier=expected_expectation_suite_identifier, - batch_identifier=mock_batch_identifier, - run_id=mock_run_identifier, - ) - - # Verify that the validation result was added to the validations store - mock_context.validations_store.set.assert_called_once_with( - expected_validation_result_identifier, validation_result - ) - - # Check that the context is returned - assert result_context == mock_context