From f4c9c40ad4d99534c784b732a67c3213617e51a3 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Tue, 20 Jun 2023 14:05:35 -0400 Subject: [PATCH 01/56] create ci poetry environment with some possible tools --- ci/poetry.lock | 234 ++++++++++++++++++++++++++++++++++++++++++++++ ci/pyproject.toml | 18 ++++ 2 files changed, 252 insertions(+) create mode 100644 ci/poetry.lock create mode 100644 ci/pyproject.toml diff --git a/ci/poetry.lock b/ci/poetry.lock new file mode 100644 index 0000000000..a49c07a92f --- /dev/null +++ b/ci/poetry.lock @@ -0,0 +1,234 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "cloudpickle" +version = "2.2.1" +description = "Extended pickling support for Python objects" +optional = false +python-versions = ">=3.6" +files = [ + {file = "cloudpickle-2.2.1-py3-none-any.whl", hash = "sha256:61f594d1f4c295fa5cd9014ceb3a1fc4a70b0de1164b94fbc2d854ccba056f9f"}, + {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, +] + +[[package]] +name = "confz" +version = "1.8.1" +description = "ConfZ is a configuration management library for Python based on pydantic." +optional = false +python-versions = ">=3.7.2,<4.0.0" +files = [ + {file = "confz-1.8.1-py3-none-any.whl", hash = "sha256:64c39218366902a7053c9f10baf368f34576e61121f465286c04d14327cde685"}, + {file = "confz-1.8.1.tar.gz", hash = "sha256:61a8465930e91a2324dc505aedf4b5658e2a5516042df183ae2b15f3e90ab751"}, +] + +[package.dependencies] +pydantic = ">=1.9.0,<2.0.0" +python-dotenv = ">=0.19.2,<0.22.0" +PyYAML = ">=5.4.1,<7.0.0" +toml = ">=0.10.2,<0.11.0" + +[[package]] +name = "doit" +version = "0.36.0" +description = "doit - Automation Tool" +optional = false +python-versions = ">=3.8" +files = [ + {file = "doit-0.36.0-py3-none-any.whl", hash = "sha256:ebc285f6666871b5300091c26eafdff3de968a6bd60ea35dd1e3fc6f2e32479a"}, + {file = "doit-0.36.0.tar.gz", hash = "sha256:71d07ccc9514cb22fe59d98999577665eaab57e16f644d04336ae0b4bae234bc"}, +] + +[package.dependencies] +cloudpickle = "*" +importlib-metadata = ">=4.4" + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "importlib-metadata" +version = "6.7.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, + {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] + +[[package]] +name = "invoke" +version = "2.1.3" +description = "Pythonic task execution" +optional = false +python-versions = ">=3.6" +files = [ + {file = "invoke-2.1.3-py3-none-any.whl", hash = "sha256:51e86a08d964160e01c44eccd22f50b25842bd96a9c63c11177032594cb86740"}, + {file = "invoke-2.1.3.tar.gz", hash = "sha256:a3b15d52d50bbabd851b8a39582c772180b614000fa1612b4d92484d54d38c6b"}, +] + +[[package]] +name = "pydantic" +version = "1.10.9" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e692dec4a40bfb40ca530e07805b1208c1de071a18d26af4a2a0d79015b352ca"}, + {file = "pydantic-1.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c52eb595db83e189419bf337b59154bdcca642ee4b2a09e5d7797e41ace783f"}, + {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:939328fd539b8d0edf244327398a667b6b140afd3bf7e347cf9813c736211896"}, + {file = "pydantic-1.10.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b48d3d634bca23b172f47f2335c617d3fcb4b3ba18481c96b7943a4c634f5c8d"}, + {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f0b7628fb8efe60fe66fd4adadd7ad2304014770cdc1f4934db41fe46cc8825f"}, + {file = "pydantic-1.10.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1aa5c2410769ca28aa9a7841b80d9d9a1c5f223928ca8bec7e7c9a34d26b1d4"}, + {file = "pydantic-1.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:eec39224b2b2e861259d6f3c8b6290d4e0fbdce147adb797484a42278a1a486f"}, + {file = "pydantic-1.10.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d111a21bbbfd85c17248130deac02bbd9b5e20b303338e0dbe0faa78330e37e0"}, + {file = "pydantic-1.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e9aec8627a1a6823fc62fb96480abe3eb10168fd0d859ee3d3b395105ae19a7"}, + {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07293ab08e7b4d3c9d7de4949a0ea571f11e4557d19ea24dd3ae0c524c0c334d"}, + {file = "pydantic-1.10.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee829b86ce984261d99ff2fd6e88f2230068d96c2a582f29583ed602ef3fc2c"}, + {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4b466a23009ff5cdd7076eb56aca537c745ca491293cc38e72bf1e0e00de5b91"}, + {file = "pydantic-1.10.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7847ca62e581e6088d9000f3c497267868ca2fa89432714e21a4fb33a04d52e8"}, + {file = "pydantic-1.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:7845b31959468bc5b78d7b95ec52fe5be32b55d0d09983a877cca6aedc51068f"}, + {file = "pydantic-1.10.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:517a681919bf880ce1dac7e5bc0c3af1e58ba118fd774da2ffcd93c5f96eaece"}, + {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67195274fd27780f15c4c372f4ba9a5c02dad6d50647b917b6a92bf00b3d301a"}, + {file = "pydantic-1.10.9-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2196c06484da2b3fded1ab6dbe182bdabeb09f6318b7fdc412609ee2b564c49a"}, + {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6257bb45ad78abacda13f15bde5886efd6bf549dd71085e64b8dcf9919c38b60"}, + {file = "pydantic-1.10.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3283b574b01e8dbc982080d8287c968489d25329a463b29a90d4157de4f2baaf"}, + {file = "pydantic-1.10.9-cp37-cp37m-win_amd64.whl", hash = "sha256:5f8bbaf4013b9a50e8100333cc4e3fa2f81214033e05ac5aa44fa24a98670a29"}, + {file = "pydantic-1.10.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9cd67fb763248cbe38f0593cd8611bfe4b8ad82acb3bdf2b0898c23415a1f82"}, + {file = "pydantic-1.10.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f50e1764ce9353be67267e7fd0da08349397c7db17a562ad036aa7c8f4adfdb6"}, + {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73ef93e5e1d3c8e83f1ff2e7fdd026d9e063c7e089394869a6e2985696693766"}, + {file = "pydantic-1.10.9-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128d9453d92e6e81e881dd7e2484e08d8b164da5507f62d06ceecf84bf2e21d3"}, + {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ad428e92ab68798d9326bb3e5515bc927444a3d71a93b4a2ca02a8a5d795c572"}, + {file = "pydantic-1.10.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fab81a92f42d6d525dd47ced310b0c3e10c416bbfae5d59523e63ea22f82b31e"}, + {file = "pydantic-1.10.9-cp38-cp38-win_amd64.whl", hash = "sha256:963671eda0b6ba6926d8fc759e3e10335e1dc1b71ff2a43ed2efd6996634dafb"}, + {file = "pydantic-1.10.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:970b1bdc6243ef663ba5c7e36ac9ab1f2bfecb8ad297c9824b542d41a750b298"}, + {file = "pydantic-1.10.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7e1d5290044f620f80cf1c969c542a5468f3656de47b41aa78100c5baa2b8276"}, + {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83fcff3c7df7adff880622a98022626f4f6dbce6639a88a15a3ce0f96466cb60"}, + {file = "pydantic-1.10.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0da48717dc9495d3a8f215e0d012599db6b8092db02acac5e0d58a65248ec5bc"}, + {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0a2aabdc73c2a5960e87c3ffebca6ccde88665616d1fd6d3db3178ef427b267a"}, + {file = "pydantic-1.10.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9863b9420d99dfa9c064042304868e8ba08e89081428a1c471858aa2af6f57c4"}, + {file = "pydantic-1.10.9-cp39-cp39-win_amd64.whl", hash = "sha256:e7c9900b43ac14110efa977be3da28931ffc74c27e96ee89fbcaaf0b0fe338e1"}, + {file = "pydantic-1.10.9-py3-none-any.whl", hash = "sha256:6cafde02f6699ce4ff643417d1a9223716ec25e228ddc3b436fe7e2d25a1f305"}, + {file = "pydantic-1.10.9.tar.gz", hash = "sha256:95c70da2cd3b6ddf3b9645ecaa8d98f3d80c606624b6d245558d202cd23ea3be"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "python-dotenv" +version = "0.21.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-dotenv-0.21.1.tar.gz", hash = "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49"}, + {file = "python_dotenv-0.21.1-py3-none-any.whl", hash = "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.6.3" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, + {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, +] + +[[package]] +name = "zipp" +version = "3.15.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "e179a328b7069f7813917e5d49b46a16b8f7191298ef253badc7961d6a79d17e" diff --git a/ci/pyproject.toml b/ci/pyproject.toml new file mode 100644 index 0000000000..97dc31acae --- /dev/null +++ b/ci/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "ci" +version = "0.1.0" +description = "" +authors = ["Andrew Vaccaro "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.9" +invoke = "^2.1.3" +pydantic = "^1.10.9" +confz = "^1.8.1" +doit = "^0.36.0" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" From 19d50a142cb2e241773c93381fbaf3c8e80210af Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Tue, 20 Jun 2023 15:22:35 -0400 Subject: [PATCH 02/56] add gitpython and decouple --- ci/poetry.lock | 52 ++++++++++++++++++++++++++++++++++++++++++++++- ci/pyproject.toml | 2 ++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/ci/poetry.lock b/ci/poetry.lock index a49c07a92f..271a583458 100644 --- a/ci/poetry.lock +++ b/ci/poetry.lock @@ -46,6 +46,34 @@ importlib-metadata = ">=4.4" [package.extras] toml = ["tomli"] +[[package]] +name = "gitdb" +version = "4.0.10" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, + {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.31" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.31-py3-none-any.whl", hash = "sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d"}, + {file = "GitPython-3.1.31.tar.gz", hash = "sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + [[package]] name = "importlib-metadata" version = "6.7.0" @@ -128,6 +156,17 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "python-decouple" +version = "3.8" +description = "Strict separation of settings from code." +optional = false +python-versions = "*" +files = [ + {file = "python-decouple-3.8.tar.gz", hash = "sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f"}, + {file = "python_decouple-3.8-py3-none-any.whl", hash = "sha256:d0d45340815b25f4de59c974b855bb38d03151d81b037d9e3f463b0c9f8cbd66"}, +] + [[package]] name = "python-dotenv" version = "0.21.1" @@ -191,6 +230,17 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.6" +files = [ + {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, + {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, +] + [[package]] name = "toml" version = "0.10.2" @@ -231,4 +281,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "e179a328b7069f7813917e5d49b46a16b8f7191298ef253badc7961d6a79d17e" +content-hash = "b1d70a78175d918f107b55eb5f56d2c574648afb0c90ed7a62d113c13d6bf910" diff --git a/ci/pyproject.toml b/ci/pyproject.toml index 97dc31acae..c660931633 100644 --- a/ci/pyproject.toml +++ b/ci/pyproject.toml @@ -11,6 +11,8 @@ invoke = "^2.1.3" pydantic = "^1.10.9" confz = "^1.8.1" doit = "^0.36.0" +python-decouple = "^3.8" +gitpython = "^3.1.31" [build-system] From c36cbf1b642822aac53a81eb9e44dc1136a552c1 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Tue, 20 Jun 2023 16:44:10 -0400 Subject: [PATCH 03/56] start on diffing in CI --- .github/workflows/service-release-diff.yml | 57 ++++++++++++++++++++++ ci/tasks.py | 45 +++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 .github/workflows/service-release-diff.yml create mode 100644 ci/tasks.py diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml new file mode 100644 index 0000000000..80d8006bfe --- /dev/null +++ b/.github/workflows/service-release-diff.yml @@ -0,0 +1,57 @@ +name: Show diff for release channel + +on: + pull_request: + branches: + - 'releases/*' + +jobs: + release: + runs-on: ubuntu-latest + env: + CLOUDSDK_CORE_PROJECT: cal-itp-data-infra + GKE_NAME: data-infra-apps + GKE_REGION: us-west1 + USE_GKE_GCLOUD_AUTH_PLUGIN: True + steps: + # Setup + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - run: gcloud components install gke-gcloud-auth-plugin + - uses: google-github-actions/get-gke-credentials@v1 + with: + cluster_name: ${{ env.GKE_NAME }} + location: ${{ env.GKE_REGION }} + - run: curl -sSL https://install.python-poetry.org | python - + + # Diff and write back to PR + - id: diff + name: Run ci/workflows/service-release.sh + shell: bash + working-directory: $GITHUB_WORKSPACE/ci + run: | + # GITHUB_JOB is not populated until the job is running + git config user.name "Github Action $GITHUB_JOB" + git config user.email "$(whoami)@$(uname -n)" + export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} + printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" + set -- "$GITHUB_WORKSPACE/ci/vars/project.env" + poetry run invoke kdiff $RELEASE_CHANNEL + + - uses: peter-evans/find-comment@v2 + id: fc + with: + issue-number: ${{ github.event.number }} + comment-author: 'github-actions[bot]' + body-includes: --cal-itp-previews.netlify.app + - uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.number }} + body: ${{ steps.diff.outputs.DIFF }} + edit-mode: replace diff --git a/ci/tasks.py b/ci/tasks.py new file mode 100644 index 0000000000..c9ecdd2425 --- /dev/null +++ b/ci/tasks.py @@ -0,0 +1,45 @@ +import os +from pathlib import Path + +import git +from decouple import Config, RepositoryEnv +from invoke import Result, task + + +@task +def fail(c): + c.run("this does not exist") + + +@task +def kdiff( + c, + channel, + app=None, + releases_dir="./vars/releases", + outfile=os.getenv("GITHUB_OUTPUT"), +): + repo = git.Repo(".", search_parent_directories=True) + + full_diff = "" + + for release in Path(releases_dir).glob(f"{channel}-*"): + env = Config(RepositoryEnv(release)) + if env.get("RELEASE_DRIVER") == "kustomize" and ( + not app or app == release.name + ): + kustomize_dir = Path(repo.working_tree_dir) / Path( + env.get("RELEASE_KUSTOMIZE_DIR") + ) + result: Result = c.run( + f"kubectl diff -k {kustomize_dir}", + echo=True, + warn=True, + ) + if result.exited != 0: + full_diff += result.stdout + + if full_diff and outfile: + print(f"writing to {outfile}") + with open(outfile, "a") as f: + f.write(f"DIFF={full_diff}") From e5fbb67ef08dd1167b66ccb495bc60ce8dd294ed Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 13:08:14 -0400 Subject: [PATCH 04/56] fix diff working dir --- .github/workflows/service-release-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 80d8006bfe..06205bae4a 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -33,7 +33,7 @@ jobs: - id: diff name: Run ci/workflows/service-release.sh shell: bash - working-directory: $GITHUB_WORKSPACE/ci + working-directory: ci run: | # GITHUB_JOB is not populated until the job is running git config user.name "Github Action $GITHUB_JOB" From 168688d0164d7f50162af590a24ea7dde09db9cc Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 13:40:13 -0400 Subject: [PATCH 05/56] run diff when commits are pushed on PR --- .github/workflows/service-release-diff.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 06205bae4a..49e421c767 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -4,6 +4,10 @@ on: pull_request: branches: - 'releases/*' + types: + - opened + - synchronize + - reopened jobs: release: From ec6b2dae026ec3426a483670d2e780acd4b85978 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 13:46:57 -0400 Subject: [PATCH 06/56] install deps --- .github/workflows/service-release-diff.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 49e421c767..1980f7b142 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -45,6 +45,7 @@ jobs: export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" + poetry install poetry run invoke kdiff $RELEASE_CHANNEL - uses: peter-evans/find-comment@v2 From 9109b47310bd4851105bb0d385056ab2f4e0c099 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 13:51:48 -0400 Subject: [PATCH 07/56] add missing token --- .github/workflows/service-release-diff.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 1980f7b142..5e0551dc85 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -9,6 +9,9 @@ on: - synchronize - reopened +env: + GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + jobs: release: runs-on: ubuntu-latest From 4b52ee437193fea65e9e2bdcd36d76c6b3634e12 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 14:05:13 -0400 Subject: [PATCH 08/56] try specifying token --- .github/workflows/service-release-diff.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 5e0551dc85..c1e8fb7a13 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -25,6 +25,8 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + token: ${{ env.GITHUB_TOKEN }} + - uses: google-github-actions/setup-gcloud@v0 with: service_account_key: ${{ secrets.GCP_SA_KEY }} From bf97e1ef1c67b1de73735afdb2be1fd321b57a42 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 14:07:41 -0400 Subject: [PATCH 09/56] try not persisting credentials in candidate action --- .github/workflows/service-release-candidate.yml | 1 + .github/workflows/service-release-diff.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service-release-candidate.yml b/.github/workflows/service-release-candidate.yml index 262e107b7d..b0e96b5247 100644 --- a/.github/workflows/service-release-candidate.yml +++ b/.github/workflows/service-release-candidate.yml @@ -28,6 +28,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + persist-credentials: false - name: 'Build release candidate branch' run: | diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index c1e8fb7a13..41dc3144d2 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -25,7 +25,6 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - token: ${{ env.GITHUB_TOKEN }} - uses: google-github-actions/setup-gcloud@v0 with: From ea0b444ea0c65a2a2edfc2444e68ccea7255b91b Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 14:11:00 -0400 Subject: [PATCH 10/56] try specifying token --- .github/workflows/service-release-candidate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service-release-candidate.yml b/.github/workflows/service-release-candidate.yml index b0e96b5247..fd95cd8bbd 100644 --- a/.github/workflows/service-release-candidate.yml +++ b/.github/workflows/service-release-candidate.yml @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - persist-credentials: false + token: ${{ secrets.BOT_GITHUB_TOKEN }} - name: 'Build release candidate branch' run: | From 8dd127dba361cb47314b7a6111d59f285cd8f438 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 15:58:58 -0400 Subject: [PATCH 11/56] specify dotenv, remove unnecessary deps --- ci/poetry.lock | 91 ++--------------------------------------------- ci/pyproject.toml | 6 ++-- 2 files changed, 4 insertions(+), 93 deletions(-) diff --git a/ci/poetry.lock b/ci/poetry.lock index 271a583458..c85a1eaf43 100644 --- a/ci/poetry.lock +++ b/ci/poetry.lock @@ -11,23 +11,6 @@ files = [ {file = "cloudpickle-2.2.1.tar.gz", hash = "sha256:d89684b8de9e34a2a43b3460fbca07d09d6e25ce858df4d5a44240403b6178f5"}, ] -[[package]] -name = "confz" -version = "1.8.1" -description = "ConfZ is a configuration management library for Python based on pydantic." -optional = false -python-versions = ">=3.7.2,<4.0.0" -files = [ - {file = "confz-1.8.1-py3-none-any.whl", hash = "sha256:64c39218366902a7053c9f10baf368f34576e61121f465286c04d14327cde685"}, - {file = "confz-1.8.1.tar.gz", hash = "sha256:61a8465930e91a2324dc505aedf4b5658e2a5516042df183ae2b15f3e90ab751"}, -] - -[package.dependencies] -pydantic = ">=1.9.0,<2.0.0" -python-dotenv = ">=0.19.2,<0.22.0" -PyYAML = ">=5.4.1,<7.0.0" -toml = ">=0.10.2,<0.11.0" - [[package]] name = "doit" version = "0.36.0" @@ -150,23 +133,13 @@ files = [ ] [package.dependencies] +python-dotenv = {version = ">=0.10.4", optional = true, markers = "extra == \"dotenv\""} typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] -[[package]] -name = "python-decouple" -version = "3.8" -description = "Strict separation of settings from code." -optional = false -python-versions = "*" -files = [ - {file = "python-decouple-3.8.tar.gz", hash = "sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f"}, - {file = "python_decouple-3.8-py3-none-any.whl", hash = "sha256:d0d45340815b25f4de59c974b855bb38d03151d81b037d9e3f463b0c9f8cbd66"}, -] - [[package]] name = "python-dotenv" version = "0.21.1" @@ -181,55 +154,6 @@ files = [ [package.extras] cli = ["click (>=5.0)"] -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] - [[package]] name = "smmap" version = "5.0.0" @@ -241,17 +165,6 @@ files = [ {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, ] -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - [[package]] name = "typing-extensions" version = "4.6.3" @@ -281,4 +194,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "b1d70a78175d918f107b55eb5f56d2c574648afb0c90ed7a62d113c13d6bf910" +content-hash = "3ae5c3948bb986f1dfca430fde5a8f242259cb04884fd7f2111a460f6fa641f2" diff --git a/ci/pyproject.toml b/ci/pyproject.toml index c660931633..c3cd337c14 100644 --- a/ci/pyproject.toml +++ b/ci/pyproject.toml @@ -7,12 +7,10 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.9" +pydantic = {extras = ["dotenv"], version = "^1.10.9"} +gitpython = "^3.1.31" invoke = "^2.1.3" -pydantic = "^1.10.9" -confz = "^1.8.1" doit = "^0.36.0" -python-decouple = "^3.8" -gitpython = "^3.1.31" [build-system] From ed7b28c4a5599be482cb1126ac3bbfbac119c6fc Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:00:50 -0400 Subject: [PATCH 12/56] use new bot token for actions --- .github/workflows/service-release-candidate.yml | 4 ++-- .github/workflows/service-release-diff.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/service-release-candidate.yml b/.github/workflows/service-release-candidate.yml index fd95cd8bbd..af8b885acb 100644 --- a/.github/workflows/service-release-candidate.yml +++ b/.github/workflows/service-release-candidate.yml @@ -15,7 +15,7 @@ on: env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} BIO_RELEASE: 1.6.372 @@ -28,7 +28,7 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - token: ${{ secrets.BOT_GITHUB_TOKEN }} + token: ${{ secrets.GH_ACTIONS_TOKEN }} - name: 'Build release candidate branch' run: | diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 41dc3144d2..c5be795294 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -10,7 +10,7 @@ on: - reopened env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} jobs: release: From 6bcde57a925843a012c1e77193173d490b85fe28 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:06:18 -0400 Subject: [PATCH 13/56] start on pydantic with invoke, and change test consumer to test --- ci/tasks.py | 57 +++++++++++++++---- .../consumer.patch.yaml | 2 +- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index c9ecdd2425..4618295c67 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -1,14 +1,52 @@ import os +from enum import Enum from pathlib import Path +from typing import Any, Generator, List, Optional import git -from decouple import Config, RepositoryEnv from invoke import Result, task +from pydantic import BaseSettings -@task -def fail(c): - c.run("this does not exist") +class ReleaseDriver(str, Enum): + helm = "helm" + kustomize = "kustomize" + + +class Release(BaseSettings): + name: str + release_driver: ReleaseDriver + + # for helm + release_namespace: Optional[str] + release_helm_name: Optional[str] + release_helm_chart: Optional[Path] + release_helm_values: Optional[List[Path]] + + # for kustomize + release_kustomize_dir: Optional[Path] + + class Config: + @classmethod + def parse_env_var(cls, field_name: str, raw_val: str) -> Any: + if field_name == "release_helm_values": + return raw_val.split(":") + return cls.json_loads(raw_val) + + +def get_releases( + channel: str, + releases_dir="./vars/releases", + driver: Optional[ReleaseDriver] = None, +) -> Generator[Release, None, None]: + for release in Path(releases_dir).glob(f"{channel}-*"): + r = Release( + name=str(release), + _env_file=release, + _env_file_encoding="utf-8", + ) + if not driver or r.release_driver == driver: + yield r @task @@ -16,20 +54,16 @@ def kdiff( c, channel, app=None, - releases_dir="./vars/releases", outfile=os.getenv("GITHUB_OUTPUT"), ): repo = git.Repo(".", search_parent_directories=True) full_diff = "" - for release in Path(releases_dir).glob(f"{channel}-*"): - env = Config(RepositoryEnv(release)) - if env.get("RELEASE_DRIVER") == "kustomize" and ( - not app or app == release.name - ): + for release in get_releases(channel, driver=ReleaseDriver.kustomize): + if not app or app == release.name: kustomize_dir = Path(repo.working_tree_dir) / Path( - env.get("RELEASE_KUSTOMIZE_DIR") + release.release_kustomize_dir ) result: Result = c.run( f"kubectl diff -k {kustomize_dir}", @@ -43,3 +77,4 @@ def kdiff( print(f"writing to {outfile}") with open(outfile, "a") as f: f.write(f"DIFF={full_diff}") + print(flush=True) diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml index 93a3eaa99e..a11ad89357 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml @@ -4,7 +4,7 @@ kind: Deployment metadata: name: gtfs-rt-archiver-consumer spec: - replicas: 2 + replicas: 1 template: spec: containers: From e2e658fdfb5ea05e8c709f633b86d5db99052db0 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:15:35 -0400 Subject: [PATCH 14/56] switch back, try to fix body-file comment --- .github/workflows/service-release-diff.yml | 5 ++--- ci/tasks.py | 8 +++----- .../overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index c5be795294..d41dbe3f8e 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -39,7 +39,6 @@ jobs: # Diff and write back to PR - id: diff - name: Run ci/workflows/service-release.sh shell: bash working-directory: ci run: | @@ -50,7 +49,7 @@ jobs: printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" poetry install - poetry run invoke kdiff $RELEASE_CHANNEL + poetry run invoke kdiff $RELEASE_CHANNEL --outfile=kdiff.txt - uses: peter-evans/find-comment@v2 id: fc @@ -62,5 +61,5 @@ jobs: with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} - body: ${{ steps.diff.outputs.DIFF }} + body-file: "${{ env.GITHUB_WORKSPACE }}/ci/kdiff.txt" edit-mode: replace diff --git a/ci/tasks.py b/ci/tasks.py index 4618295c67..68220aaba7 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -1,4 +1,3 @@ -import os from enum import Enum from pathlib import Path from typing import Any, Generator, List, Optional @@ -54,7 +53,7 @@ def kdiff( c, channel, app=None, - outfile=os.getenv("GITHUB_OUTPUT"), + outfile=None, ): repo = git.Repo(".", search_parent_directories=True) @@ -75,6 +74,5 @@ def kdiff( if full_diff and outfile: print(f"writing to {outfile}") - with open(outfile, "a") as f: - f.write(f"DIFF={full_diff}") - print(flush=True) + with open(outfile, "w") as f: + f.write(full_diff) diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml index a11ad89357..93a3eaa99e 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml @@ -4,7 +4,7 @@ kind: Deployment metadata: name: gtfs-rt-archiver-consumer spec: - replicas: 1 + replicas: 2 template: spec: containers: From 8faff755a90fa17277c88c72f55b4006dea5a085 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:20:33 -0400 Subject: [PATCH 15/56] not in env? --- .github/workflows/service-release-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index d41dbe3f8e..2f5ca99fb8 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -61,5 +61,5 @@ jobs: with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} - body-file: "${{ env.GITHUB_WORKSPACE }}/ci/kdiff.txt" + body-file: "${{ GITHUB_WORKSPACE }}/ci/kdiff.txt" edit-mode: replace From 2964cf54bb10b59986c4d846476bac47c67857f6 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:24:24 -0400 Subject: [PATCH 16/56] just set work dir --- .github/workflows/service-release-diff.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 2f5ca99fb8..d95cf44368 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -58,8 +58,9 @@ jobs: comment-author: 'github-actions[bot]' body-includes: --cal-itp-previews.netlify.app - uses: peter-evans/create-or-update-comment@v2 + working-directory: ci with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} - body-file: "${{ GITHUB_WORKSPACE }}/ci/kdiff.txt" + body-file: "kdiff.txt" edit-mode: replace From 4d190bd8b3bfe9681a0c30ea03d1224117d60022 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:35:06 -0400 Subject: [PATCH 17/56] one more --- .github/workflows/service-release-diff.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index d95cf44368..0bf21f9165 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -58,9 +58,8 @@ jobs: comment-author: 'github-actions[bot]' body-includes: --cal-itp-previews.netlify.app - uses: peter-evans/create-or-update-comment@v2 - working-directory: ci with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} - body-file: "kdiff.txt" + body-file: "ci/kdiff.txt" edit-mode: replace From e9254f587bf83e4b93e491103144dd5c3608262d Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:46:38 -0400 Subject: [PATCH 18/56] always write to kdiff file --- ci/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index 68220aaba7..ec7709846a 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -72,7 +72,7 @@ def kdiff( if result.exited != 0: full_diff += result.stdout - if full_diff and outfile: - print(f"writing to {outfile}") + if outfile: + print(f"writing {len(full_diff)=} to {outfile}", flush=True) with open(outfile, "w") as f: f.write(full_diff) From 390b460541ac8940271c214df5a81831fc0c8e0f Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 16:53:39 -0400 Subject: [PATCH 19/56] logging --- ci/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/tasks.py b/ci/tasks.py index ec7709846a..d27c774bdf 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -64,8 +64,10 @@ def kdiff( kustomize_dir = Path(repo.working_tree_dir) / Path( release.release_kustomize_dir ) + cmd = f"kubectl diff -k {kustomize_dir}" + print(cmd, flush=True) result: Result = c.run( - f"kubectl diff -k {kustomize_dir}", + cmd, echo=True, warn=True, ) From 5f2e922b172b625675fb3aaa48602a34ff98654a Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 17:05:08 -0400 Subject: [PATCH 20/56] change github ref handling --- .github/workflows/service-release-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 0bf21f9165..ad611408ec 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -45,7 +45,7 @@ jobs: # GITHUB_JOB is not populated until the job is running git config user.name "Github Action $GITHUB_JOB" git config user.email "$(whoami)@$(uname -n)" - export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} + export RELEASE_CHANNEL=${GITHUB_BASE_REF#releases/} printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" poetry install From b3c506ba93cdad48a111e738c3d3d3225853d692 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 17:16:55 -0400 Subject: [PATCH 21/56] markdown formatting and handle empty diffs --- ci/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index d27c774bdf..7a20294a50 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -75,6 +75,7 @@ def kdiff( full_diff += result.stdout if outfile: - print(f"writing {len(full_diff)=} to {outfile}", flush=True) + msg = f"```{full_diff}```" if full_diff else "No kustomize changes found." + print(f"writing {len(msg)=} to {outfile}", flush=True) with open(outfile, "w") as f: - f.write(full_diff) + f.write(msg) From 0fbc1435fe13456af33b5b30e06085d95edf024d Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 21 Jun 2023 17:36:21 -0400 Subject: [PATCH 22/56] fix getting diff comment to edit --- .github/workflows/service-release-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index ad611408ec..99d54d8dee 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -56,7 +56,7 @@ jobs: with: issue-number: ${{ github.event.number }} comment-author: 'github-actions[bot]' - body-includes: --cal-itp-previews.netlify.app + direction: last - uses: peter-evans/create-or-update-comment@v2 with: comment-id: ${{ steps.fc.outputs.comment-id }} From 4163abffb200f3549da02c764854b5bb836b3384 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 13:34:04 -0400 Subject: [PATCH 23/56] start representing channels as yaml --- ci/invoke.yaml | 6 +++ ci/poetry.lock | 51 ++++++++++++++++++++++- ci/pyproject.toml | 1 + ci/tasks.py | 83 ++++++++++++++++++++++++-------------- ci/vars/channels/test.yaml | 20 +++++++++ 5 files changed, 130 insertions(+), 31 deletions(-) create mode 100644 ci/invoke.yaml create mode 100644 ci/vars/channels/test.yaml diff --git a/ci/invoke.yaml b/ci/invoke.yaml new file mode 100644 index 0000000000..bac6b8bcb8 --- /dev/null +++ b/ci/invoke.yaml @@ -0,0 +1,6 @@ +run: + echo: true +# would be cool if we can define a pydantic type for this +calitp: + git_repo_path: . + kdiff_outfile: kdiff.txt diff --git a/ci/poetry.lock b/ci/poetry.lock index c85a1eaf43..4699459618 100644 --- a/ci/poetry.lock +++ b/ci/poetry.lock @@ -154,6 +154,55 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] + [[package]] name = "smmap" version = "5.0.0" @@ -194,4 +243,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "3ae5c3948bb986f1dfca430fde5a8f242259cb04884fd7f2111a460f6fa641f2" +content-hash = "4e02af70483866dead32999c3bd96ce72196eacf2558a232fa25a16239500e70" diff --git a/ci/pyproject.toml b/ci/pyproject.toml index c3cd337c14..fdd83afd62 100644 --- a/ci/pyproject.toml +++ b/ci/pyproject.toml @@ -11,6 +11,7 @@ pydantic = {extras = ["dotenv"], version = "^1.10.9"} gitpython = "^3.1.31" invoke = "^2.1.3" doit = "^0.36.0" +pyyaml = "^6.0" [build-system] diff --git a/ci/tasks.py b/ci/tasks.py index 7a20294a50..e604b137ed 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -1,10 +1,18 @@ from enum import Enum from pathlib import Path -from typing import Any, Generator, List, Optional +from typing import List, Optional import git from invoke import Result, task -from pydantic import BaseSettings +from pydantic import BaseModel, validator + +KUSTOMIZE_HELM_TEMPLATE = """ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: {release_namespace} +resources: +- manifest.yaml +""" class ReleaseDriver(str, Enum): @@ -12,7 +20,7 @@ class ReleaseDriver(str, Enum): kustomize = "kustomize" -class Release(BaseSettings): +class Release(BaseModel): name: str release_driver: ReleaseDriver @@ -25,42 +33,37 @@ class Release(BaseSettings): # for kustomize release_kustomize_dir: Optional[Path] - class Config: - @classmethod - def parse_env_var(cls, field_name: str, raw_val: str) -> Any: - if field_name == "release_helm_values": - return raw_val.split(":") - return cls.json_loads(raw_val) - - -def get_releases( - channel: str, - releases_dir="./vars/releases", - driver: Optional[ReleaseDriver] = None, -) -> Generator[Release, None, None]: - for release in Path(releases_dir).glob(f"{channel}-*"): - r = Release( - name=str(release), - _env_file=release, - _env_file_encoding="utf-8", - ) - if not driver or r.release_driver == driver: - yield r + @validator("release_helm_values", pre=True) + def split_release_helm_values(cls, v): + return v.split(":") @task +def load_release(c): + c.update( + { + "releases": [ + Release(**release) for release in c.config._config["calitp"]["releases"] + ] + } + ) + + +@task(load_release) def kdiff( c, - channel, app=None, outfile=None, ): - repo = git.Repo(".", search_parent_directories=True) + repo = git.Repo(c.config.calitp.git_repo_path, search_parent_directories=True) full_diff = "" - for release in get_releases(channel, driver=ReleaseDriver.kustomize): - if not app or app == release.name: + release: Release + for release in c.releases: + if release.release_driver == ReleaseDriver.kustomize and ( + not app or app == release.name + ): kustomize_dir = Path(repo.working_tree_dir) / Path( release.release_kustomize_dir ) @@ -68,14 +71,34 @@ def kdiff( print(cmd, flush=True) result: Result = c.run( cmd, - echo=True, warn=True, ) if result.exited != 0: full_diff += result.stdout - + c.update({"kdiff": full_diff}) if outfile: msg = f"```{full_diff}```" if full_diff else "No kustomize changes found." print(f"writing {len(msg)=} to {outfile}", flush=True) with open(outfile, "w") as f: f.write(msg) + + +@task(kdiff) +def krelease(c, app=None): + try: + c.kdiff + + repo = git.Repo(c.config.calitp.git_repo_path, search_parent_directories=True) + release: Release + for release in c.releases: + if release.release_driver == ReleaseDriver.kustomize and ( + not app or app == release.name + ): + kustomize_dir = Path(repo.working_tree_dir) / Path( + release.release_kustomize_dir + ) + cmd = f"kubectl apply -k {kustomize_dir}" + print(cmd, flush=True) + c.run(cmd) + except AttributeError: + pass diff --git a/ci/vars/channels/test.yaml b/ci/vars/channels/test.yaml new file mode 100644 index 0000000000..4bdb33e991 --- /dev/null +++ b/ci/vars/channels/test.yaml @@ -0,0 +1,20 @@ +calitp: + name: test + releases: + - name: archiver + release_driver: kustomize + release_kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test + + - name: metabase + release_driver: helm + release_namespace: metabase-test + release_helm_name: metabase-test + release_helm_chart: kubernetes/apps/charts/metabase + release_helm_values: kubernetes/apps/values/metabase-test.yaml + + - name: postgresql-backup-metabase + release_driver: helm + release_namespace: metabase-test + release_helm_name: postgresql-backup + release_helm_chart: kubernetes/apps/charts/postgresql-backup + release_helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml From da6cca6f314f8d92fcd2059ea5b0685e03ed5731 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 13:34:51 -0400 Subject: [PATCH 24/56] remove release prefix --- ci/tasks.py | 28 +++++++++++++--------------- ci/vars/channels/test.yaml | 24 ++++++++++++------------ 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index e604b137ed..c78eb04ae4 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -9,7 +9,7 @@ KUSTOMIZE_HELM_TEMPLATE = """ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: {release_namespace} +namespace: {namespace} resources: - manifest.yaml """ @@ -22,19 +22,19 @@ class ReleaseDriver(str, Enum): class Release(BaseModel): name: str - release_driver: ReleaseDriver + driver: ReleaseDriver # for helm - release_namespace: Optional[str] - release_helm_name: Optional[str] - release_helm_chart: Optional[Path] - release_helm_values: Optional[List[Path]] + namespace: Optional[str] + helm_name: Optional[str] + helm_chart: Optional[Path] + helm_values: Optional[List[Path]] # for kustomize - release_kustomize_dir: Optional[Path] + kustomize_dir: Optional[Path] - @validator("release_helm_values", pre=True) - def split_release_helm_values(cls, v): + @validator("helm_values", pre=True) + def split_helm_values(cls, v): return v.split(":") @@ -61,12 +61,10 @@ def kdiff( release: Release for release in c.releases: - if release.release_driver == ReleaseDriver.kustomize and ( + if release.driver == ReleaseDriver.kustomize and ( not app or app == release.name ): - kustomize_dir = Path(repo.working_tree_dir) / Path( - release.release_kustomize_dir - ) + kustomize_dir = Path(repo.working_tree_dir) / Path(release.kustomize_dir) cmd = f"kubectl diff -k {kustomize_dir}" print(cmd, flush=True) result: Result = c.run( @@ -91,11 +89,11 @@ def krelease(c, app=None): repo = git.Repo(c.config.calitp.git_repo_path, search_parent_directories=True) release: Release for release in c.releases: - if release.release_driver == ReleaseDriver.kustomize and ( + if release.driver == ReleaseDriver.kustomize and ( not app or app == release.name ): kustomize_dir = Path(repo.working_tree_dir) / Path( - release.release_kustomize_dir + release.kustomize_dir ) cmd = f"kubectl apply -k {kustomize_dir}" print(cmd, flush=True) diff --git a/ci/vars/channels/test.yaml b/ci/vars/channels/test.yaml index 4bdb33e991..6002a53fdb 100644 --- a/ci/vars/channels/test.yaml +++ b/ci/vars/channels/test.yaml @@ -2,19 +2,19 @@ calitp: name: test releases: - name: archiver - release_driver: kustomize - release_kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test + driver: kustomize + kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test - name: metabase - release_driver: helm - release_namespace: metabase-test - release_helm_name: metabase-test - release_helm_chart: kubernetes/apps/charts/metabase - release_helm_values: kubernetes/apps/values/metabase-test.yaml + driver: helm + namespace: metabase-test + helm_name: metabase-test + helm_chart: kubernetes/apps/charts/metabase + helm_values: kubernetes/apps/values/metabase-test.yaml - name: postgresql-backup-metabase - release_driver: helm - release_namespace: metabase-test - release_helm_name: postgresql-backup - release_helm_chart: kubernetes/apps/charts/postgresql-backup - release_helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml + driver: helm + namespace: metabase-test + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml From 13b02755e1e6defd33a374c6de783461d8a724a0 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 14:03:34 -0400 Subject: [PATCH 25/56] move channels into invoke.yaml --- ci/invoke.yaml | 26 +++++++++++-- ci/tasks.py | 79 ++++++++++++++++++++++---------------- ci/vars/channels/test.yaml | 20 ---------- 3 files changed, 69 insertions(+), 56 deletions(-) delete mode 100644 ci/vars/channels/test.yaml diff --git a/ci/invoke.yaml b/ci/invoke.yaml index bac6b8bcb8..ffcadf480e 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -1,6 +1,26 @@ run: - echo: true + echo: true # would be cool if we can define a pydantic type for this calitp: - git_repo_path: . - kdiff_outfile: kdiff.txt + git_repo: . + kdiff_outfile: kdiff.txt + channels: + test: + releases: + - name: archiver + driver: kustomize + kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test + + - name: metabase + driver: helm + namespace: metabase-test + helm_name: metabase-test + helm_chart: kubernetes/apps/charts/metabase + helm_values: kubernetes/apps/values/metabase-test.yaml + + - name: postgresql-backup-metabase + driver: helm + namespace: metabase-test + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml diff --git a/ci/tasks.py b/ci/tasks.py index c78eb04ae4..0ee4d99344 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -1,6 +1,6 @@ from enum import Enum from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional import git from invoke import Result, task @@ -38,33 +38,46 @@ def split_helm_values(cls, v): return v.split(":") +class Channel(BaseModel): + releases: List[Release] + + +# TODO: rename this +class Config(BaseModel): + git_repo: git.Repo + kdiff_outfile: str + channels: Dict[str, Channel] + + class Config: + arbitrary_types_allowed = True + + @validator("git_repo", pre=True) + def parse_git_repo(cls, v): + return git.Repo(v, search_parent_directories=True) + + @task -def load_release(c): - c.update( - { - "releases": [ - Release(**release) for release in c.config._config["calitp"]["releases"] - ] - } - ) +def parse_calitp_config(c): + c.update({"calitp_config": Config(**c.config._config["calitp"])}) -@task(load_release) +@task(parse_calitp_config) def kdiff( c, + channel, app=None, outfile=None, ): - repo = git.Repo(c.config.calitp.git_repo_path, search_parent_directories=True) - full_diff = "" release: Release - for release in c.releases: + for release in c.calitp_config.channels[channel].releases: if release.driver == ReleaseDriver.kustomize and ( not app or app == release.name ): - kustomize_dir = Path(repo.working_tree_dir) / Path(release.kustomize_dir) + kustomize_dir = Path(c.calitp_config.git_repo.working_tree_dir) / Path( + release.kustomize_dir + ) cmd = f"kubectl diff -k {kustomize_dir}" print(cmd, flush=True) result: Result = c.run( @@ -81,22 +94,22 @@ def kdiff( f.write(msg) -@task(kdiff) -def krelease(c, app=None): - try: - c.kdiff - - repo = git.Repo(c.config.calitp.git_repo_path, search_parent_directories=True) - release: Release - for release in c.releases: - if release.driver == ReleaseDriver.kustomize and ( - not app or app == release.name - ): - kustomize_dir = Path(repo.working_tree_dir) / Path( - release.kustomize_dir - ) - cmd = f"kubectl apply -k {kustomize_dir}" - print(cmd, flush=True) - c.run(cmd) - except AttributeError: - pass +@task(parse_calitp_config) +def hdiff(c, channel, app=None): + pass + + +# TODO: we may want to split up channels into separate files so channel is not an argument but a config file +@task(parse_calitp_config) +def krelease(c, channel, app=None): + release: Release + for release in c.calitp_config.channels[channel].releases: + if release.driver == ReleaseDriver.kustomize and ( + not app or app == release.name + ): + kustomize_dir = Path(c.calitp_config.git_repo.working_tree_dir) / Path( + release.kustomize_dir + ) + cmd = f"kubectl apply -k {kustomize_dir}" + print(cmd, flush=True) + c.run(cmd) diff --git a/ci/vars/channels/test.yaml b/ci/vars/channels/test.yaml deleted file mode 100644 index 6002a53fdb..0000000000 --- a/ci/vars/channels/test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -calitp: - name: test - releases: - - name: archiver - driver: kustomize - kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test - - - name: metabase - driver: helm - namespace: metabase-test - helm_name: metabase-test - helm_chart: kubernetes/apps/charts/metabase - helm_values: kubernetes/apps/values/metabase-test.yaml - - - name: postgresql-backup-metabase - driver: helm - namespace: metabase-test - helm_name: postgresql-backup - helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml From 1901e82c3fb1f73ec30052000415302a61741cba Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 14:34:40 -0400 Subject: [PATCH 26/56] helm diff in invoke --- ci/tasks.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index 0ee4d99344..432d57c570 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -1,3 +1,4 @@ +import tempfile from enum import Enum from pathlib import Path from typing import Dict, List, Optional @@ -55,6 +56,10 @@ class Config: def parse_git_repo(cls, v): return git.Repo(v, search_parent_directories=True) + @property + def git_root(self) -> Path: + return Path(self.git_repo.working_tree_dir) + @task def parse_calitp_config(c): @@ -75,9 +80,7 @@ def kdiff( if release.driver == ReleaseDriver.kustomize and ( not app or app == release.name ): - kustomize_dir = Path(c.calitp_config.git_repo.working_tree_dir) / Path( - release.kustomize_dir - ) + kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) cmd = f"kubectl diff -k {kustomize_dir}" print(cmd, flush=True) result: Result = c.run( @@ -95,8 +98,55 @@ def kdiff( @task(parse_calitp_config) -def hdiff(c, channel, app=None): - pass +def hdiff( + c, + channel, + app=None, + outfile=None, +): + full_diff = "" + + release: Release + for release in c.calitp_config.channels[channel].releases: + if release.driver == ReleaseDriver.helm and (not app or app == release.name): + chart_path = c.calitp_config.git_root / Path(release.helm_chart) + cmd = f"helm dependency update {chart_path}" + print(cmd, flush=True) + c.run( + cmd, + warn=True, + ) + with tempfile.TemporaryDirectory() as tmpdir: + manifest_path = Path(tmpdir) / Path("manifest.yaml") + kustomization_path = Path(tmpdir) / Path("kustomization.yaml") + values_str = " ".join( + [ + f"--values {c.calitp_config.git_root / Path(values_file)}" + for values_file in release.helm_values + ] + ) + cmd = f"helm template {release.helm_name} {chart_path} --namespace {release.namespace} {values_str} > {manifest_path}" + print(cmd, flush=True) + c.run( + cmd, + warn=True, + ) + with open(kustomization_path, "w") as f: + f.write(KUSTOMIZE_HELM_TEMPLATE.format(namespace=release.namespace)) + cmd = f"kubectl diff -k {tmpdir}" + print(cmd, flush=True) + result: Result = c.run( + cmd, + warn=True, + ) + if result.exited != 0: + full_diff += result.stdout + c.update({"hdiff": full_diff}) + if outfile: + msg = f"```{full_diff}```" if full_diff else "No kustomize changes found." + print(f"writing {len(msg)=} to {outfile}", flush=True) + with open(outfile, "w") as f: + f.write(msg) # TODO: we may want to split up channels into separate files so channel is not an argument but a config file @@ -107,9 +157,7 @@ def krelease(c, channel, app=None): if release.driver == ReleaseDriver.kustomize and ( not app or app == release.name ): - kustomize_dir = Path(c.calitp_config.git_repo.working_tree_dir) / Path( - release.kustomize_dir - ) + kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) cmd = f"kubectl apply -k {kustomize_dir}" print(cmd, flush=True) c.run(cmd) From b1a0443f1b8137cfdc5bdd6657ef6118f7a030f5 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 14:45:23 -0400 Subject: [PATCH 27/56] handle empty helm values, put prod releases in yaml --- ci/invoke.yaml | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-- ci/tasks.py | 2 +- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/ci/invoke.yaml b/ci/invoke.yaml index ffcadf480e..e77a325b8c 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -10,17 +10,75 @@ calitp: - name: archiver driver: kustomize kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test - - name: metabase driver: helm namespace: metabase-test helm_name: metabase-test helm_chart: kubernetes/apps/charts/metabase helm_values: kubernetes/apps/values/metabase-test.yaml - - name: postgresql-backup-metabase driver: helm namespace: metabase-test helm_name: postgresql-backup helm_chart: kubernetes/apps/charts/postgresql-backup helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml + prod: + releases: + - name: airflow-jobs + driver: kustomize + kustomize_dir: kubernetes/apps/manifests/airflow-jobs + - name: grafana + driver: helm + namespace: monitoring-grafana + helm_name: grafana + helm_chart: kubernetes/apps/charts/grafana + helm_values: kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml + - name: jupyterhub + driver: helm + namespace: jupyterhub + helm_name: jupyterhub + helm_chart: kubernetes/apps/charts/jupyterhub + - name: loki + driver: helm + namespace: monitoring-loki + helm_name: loki + helm_chart: kubernetes/apps/charts/loki + - name: metabase + driver: helm + namespace: metabase + helm_name: metabase + helm_chart: kubernetes/apps/charts/metabase + helm_values: kubernetes/apps/values/metabase.yaml + - name: postgresql-backup-grafana + driver: helm + namespace: grafana + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: kubernetes/apps/values/postgresql-backup-grafana.yaml + - name: postgresql-backup-metabase + driver: helm + namespace: metabase + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml + - name: postgresql-backup-sentry + driver: helm + namespace: sentry + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: kubernetes/apps/values/postgresql-backup-sentry.yaml + - name: prometheus + driver: helm + namespace: monitoring-prometheus + helm_name: prometheus + helm_chart: kubernetes/apps/charts/prometheus + helm_values: kubernetes/apps/values/prometheus.yaml + - name: promtail + driver: helm + namespace: monitoring-loki + helm_name: promtail + helm_chart: kubernetes/apps/charts/promtail + helm_values: kubernetes/apps/values/promtail.yaml + - name: sftp-ingest-elavon + driver: kustomize + kustomize_dir: kubernetes/apps/overlays/prod-sftp-ingest-elavon diff --git a/ci/tasks.py b/ci/tasks.py index 432d57c570..725c2b8de5 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -29,7 +29,7 @@ class Release(BaseModel): namespace: Optional[str] helm_name: Optional[str] helm_chart: Optional[Path] - helm_values: Optional[List[Path]] + helm_values: List[Path] = [] # for kustomize kustomize_dir: Optional[Path] From 09607486dd84a558c6a2b19ae623b90ab94f6c8c Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 14:47:44 -0400 Subject: [PATCH 28/56] call helm diff and cat together with kustomize diff --- .github/workflows/service-release-diff.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 99d54d8dee..483b15a547 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -50,6 +50,8 @@ jobs: set -- "$GITHUB_WORKSPACE/ci/vars/project.env" poetry install poetry run invoke kdiff $RELEASE_CHANNEL --outfile=kdiff.txt + poetry run invoke hdiff $RELEASE_CHANNEL --outfile=hdiff.txt + cat kdiff.txt hdiff.txt > diff.txt - uses: peter-evans/find-comment@v2 id: fc @@ -61,5 +63,5 @@ jobs: with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} - body-file: "ci/kdiff.txt" + body-file: "ci/diff.txt" edit-mode: replace From 4eed516ee494e9bb45d8e310ba263697635c4fb0 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 14:53:05 -0400 Subject: [PATCH 29/56] fix printing, and use release candidate in metabase-test to test --- ci/tasks.py | 4 ++-- kubernetes/apps/values/metabase-test.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index 725c2b8de5..efd945d144 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -91,7 +91,7 @@ def kdiff( full_diff += result.stdout c.update({"kdiff": full_diff}) if outfile: - msg = f"```{full_diff}```" if full_diff else "No kustomize changes found." + msg = f"```{full_diff}```" if full_diff else "No kustomize changes found.\n" print(f"writing {len(msg)=} to {outfile}", flush=True) with open(outfile, "w") as f: f.write(msg) @@ -143,7 +143,7 @@ def hdiff( full_diff += result.stdout c.update({"hdiff": full_diff}) if outfile: - msg = f"```{full_diff}```" if full_diff else "No kustomize changes found." + msg = f"```{full_diff}```" if full_diff else "No helm changes found.\n" print(f"writing {len(msg)=} to {outfile}", flush=True) with open(outfile, "w") as f: f.write(msg) diff --git a/kubernetes/apps/values/metabase-test.yaml b/kubernetes/apps/values/metabase-test.yaml index 73f7cbc926..b76a47faf2 100644 --- a/kubernetes/apps/values/metabase-test.yaml +++ b/kubernetes/apps/values/metabase-test.yaml @@ -1,6 +1,6 @@ images: metabase: - src: metabase/metabase:v0.46.5 + src: metabase/metabase:v0.47.0-RC1 workloads: metabase: From b7812570f89a4ac8f734e89ea3d984a7f0fae81e Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 16:22:07 -0400 Subject: [PATCH 30/56] implement helm releases, remove legacy test envs --- ci/tasks.py | 33 +++++++++++++++++++ ci/vars/releases/test-archiver.env | 2 -- ci/vars/releases/test-metabase.env | 5 --- .../test-postgresql-backup-metabase.env | 5 --- 4 files changed, 33 insertions(+), 12 deletions(-) delete mode 100644 ci/vars/releases/test-archiver.env delete mode 100644 ci/vars/releases/test-metabase.env delete mode 100644 ci/vars/releases/test-postgresql-backup-metabase.env diff --git a/ci/tasks.py b/ci/tasks.py index efd945d144..5d0d5a0b56 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -161,3 +161,36 @@ def krelease(c, channel, app=None): cmd = f"kubectl apply -k {kustomize_dir}" print(cmd, flush=True) c.run(cmd) + + +@task(parse_calitp_config) +def hrelease( + c, + channel, + app=None, +): + release: Release + for release in c.calitp_config.channels[channel].releases: + if release.driver == ReleaseDriver.helm and (not app or app == release.name): + chart_path = c.calitp_config.git_root / Path(release.helm_chart) + cmd = f"helm dependency update {chart_path}" + print(cmd, flush=True) + c.run(cmd, warn=True) + values_str = " ".join( + [ + f"--values {c.calitp_config.git_root / Path(values_file)}" + for values_file in release.helm_values + ] + ) + cmd = f"kubectl get ns {release.namespace}" + result: Result = c.run(cmd) + verb = "upgrade" + + if result.exited != 0: + # namespace does not exist yet + c.run(f"kubectl create ns {release.namespace}") + verb = "install" + + c.run( + f"helm {verb} {release.helm_name} {chart_path} --namespace {release.namespace} {values_str}" + ) diff --git a/ci/vars/releases/test-archiver.env b/ci/vars/releases/test-archiver.env deleted file mode 100644 index a0d95846a0..0000000000 --- a/ci/vars/releases/test-archiver.env +++ /dev/null @@ -1,2 +0,0 @@ -RELEASE_DRIVER=kustomize -RELEASE_KUSTOMIZE_DIR=kubernetes/apps/overlays/gtfs-rt-archiver-v3-test diff --git a/ci/vars/releases/test-metabase.env b/ci/vars/releases/test-metabase.env deleted file mode 100644 index e19a97e837..0000000000 --- a/ci/vars/releases/test-metabase.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=metabase-test -RELEASE_HELM_NAME=metabase-test -RELEASE_HELM_CHART=kubernetes/apps/charts/metabase -RELEASE_HELM_VALUES=kubernetes/apps/values/metabase-test.yaml diff --git a/ci/vars/releases/test-postgresql-backup-metabase.env b/ci/vars/releases/test-postgresql-backup-metabase.env deleted file mode 100644 index 0cb9992cc6..0000000000 --- a/ci/vars/releases/test-postgresql-backup-metabase.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=metabase-test -RELEASE_HELM_NAME=postgresql-backup -RELEASE_HELM_CHART=kubernetes/apps/charts/postgresql-backup -RELEASE_HELM_VALUES=kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml From abce0583f3b764d15f95163ccd4129272d3cf535 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 16:24:15 -0400 Subject: [PATCH 31/56] switch service release to invoke --- .github/workflows/service-release-channel.yml | 16 +++++++++------- .github/workflows/service-release-diff.yml | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml index 4cae128e6c..6ff990c21c 100644 --- a/.github/workflows/service-release-channel.yml +++ b/.github/workflows/service-release-channel.yml @@ -14,28 +14,28 @@ jobs: GKE_REGION: us-west1 USE_GKE_GCLOUD_AUTH_PLUGIN: True steps: - + # Setup - name: Check out repo uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: google-github-actions/setup-gcloud@v0 with: service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - - name: install auth plugin run: gcloud components install gke-gcloud-auth-plugin - - uses: google-github-actions/get-gke-credentials@v1 with: cluster_name: ${{ env.GKE_NAME }} location: ${{ env.GKE_REGION }} + - run: curl -sSL https://install.python-poetry.org | python - - - id: service-release - name: Run ci/workflows/service-release.sh + # Release to channel + - id: poetry-invoke + name: Run poetry invoke shell: bash + working-directory: ci run: | # GITHUB_JOB is not populated until the job is running git config user.name "Github Action $GITHUB_JOB" @@ -43,4 +43,6 @@ jobs: export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" - source "$GITHUB_WORKSPACE/ci/workflows/service-release.sh" + poetry install + poetry run invoke kdiff $RELEASE_CHANNEL --outfile=kdiff.txt + poetry run invoke hdiff $RELEASE_CHANNEL --outfile=hdiff.txt diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 483b15a547..7faa26009a 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -39,6 +39,7 @@ jobs: # Diff and write back to PR - id: diff + name: Run poetry invoke shell: bash working-directory: ci run: | From 7ca124269d24e59647e11f2544bdaa3ee58a25ca Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 16:26:06 -0400 Subject: [PATCH 32/56] fix invoke call --- .github/workflows/service-release-channel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml index 6ff990c21c..695187defc 100644 --- a/.github/workflows/service-release-channel.yml +++ b/.github/workflows/service-release-channel.yml @@ -44,5 +44,5 @@ jobs: printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" poetry install - poetry run invoke kdiff $RELEASE_CHANNEL --outfile=kdiff.txt - poetry run invoke hdiff $RELEASE_CHANNEL --outfile=hdiff.txt + poetry run invoke krelease $RELEASE_CHANNEL + poetry run invoke hrelease $RELEASE_CHANNEL From 64e730844e91a38b374a16186a2c50fa69c7d11b Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 17:27:43 -0400 Subject: [PATCH 33/56] delete legacy prod ci --- ci/steps/configure-git-remote.sh | 22 ----- ci/steps/release-git-notes.sh | 38 --------- ci/steps/release-helm-changes.sh | 84 ------------------- ci/steps/release-kustomize-changes.sh | 26 ------ ci/vars/releases/prod-airflow-jobs.env | 2 - ci/vars/releases/prod-grafana.env | 5 -- ci/vars/releases/prod-jupyterhub.env | 5 -- ci/vars/releases/prod-loki.env | 5 -- ci/vars/releases/prod-metabase.env | 5 -- .../prod-postgresql-backup-grafana.env | 5 -- .../prod-postgresql-backup-metabase.env | 5 -- .../prod-postgresql-backup-sentry.env | 5 -- ci/vars/releases/prod-prometheus.env | 5 -- ci/vars/releases/prod-promtail.env | 5 -- ci/vars/releases/prod-sftp-ingest-elavon.env | 2 - ci/workflows/service-release.sh | 72 ---------------- 16 files changed, 291 deletions(-) delete mode 100755 ci/steps/configure-git-remote.sh delete mode 100755 ci/steps/release-git-notes.sh delete mode 100755 ci/steps/release-helm-changes.sh delete mode 100755 ci/steps/release-kustomize-changes.sh delete mode 100644 ci/vars/releases/prod-airflow-jobs.env delete mode 100644 ci/vars/releases/prod-grafana.env delete mode 100644 ci/vars/releases/prod-jupyterhub.env delete mode 100644 ci/vars/releases/prod-loki.env delete mode 100644 ci/vars/releases/prod-metabase.env delete mode 100644 ci/vars/releases/prod-postgresql-backup-grafana.env delete mode 100644 ci/vars/releases/prod-postgresql-backup-metabase.env delete mode 100644 ci/vars/releases/prod-postgresql-backup-sentry.env delete mode 100644 ci/vars/releases/prod-prometheus.env delete mode 100644 ci/vars/releases/prod-promtail.env delete mode 100644 ci/vars/releases/prod-sftp-ingest-elavon.env delete mode 100755 ci/workflows/service-release.sh diff --git a/ci/steps/configure-git-remote.sh b/ci/steps/configure-git-remote.sh deleted file mode 100755 index af22359440..0000000000 --- a/ci/steps/configure-git-remote.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e - -required_missing=() - -test "$CONFIGURE_GIT_REMOTE_NAME" || CONFIGURE_GIT_REMOTE_NAME= -test "$CONFIGURE_GIT_REMOTE_URL" || CONFIGURE_GIT_REMOTE_URL= - -if [[ ${#required_missing[*]} -gt 0 ]]; then - printf 'error: missing required variables: %s\n' "${required_missing[*]}" >&2 - exit 1 -fi - -if [[ $CONFIGURE_GIT_REMOTE_NAME ]]; then - - if ! git remote | grep "^$CONFIGURE_GIT_REMOTE_NAME$"; then - git remote add "$CONFIGURE_GIT_REMOTE_NAME" "$CONFIGURE_GIT_REMOTE_URL" - elif [[ $CONFIGURE_GIT_REMOTE_URL && $CONFIGURE_GIT_REMOTE_URL != $(git remote get-url "$CONFIGURE_GIT_REMOTE_NAME") ]]; then - git remote set-url "$CONFIGURE_GIT_REMOTE_NAME" "$CONFIGURE_GIT_REMOTE_URL" - fi - -fi diff --git a/ci/steps/release-git-notes.sh b/ci/steps/release-git-notes.sh deleted file mode 100755 index 3bb881df29..0000000000 --- a/ci/steps/release-git-notes.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e - -required_missing=() - -test "$RELEASE_NOTES" || RELEASE_NOTES= -test "$CONFIGURE_GIT_REMOTE_NAME" || CONFIGURE_GIT_REMOTE_NAME= -test "$GIT_NOTES_REF" || GIT_NOTES_REF=refs/notes/releases/$RELEASE_CHANNEL - -export GIT_NOTES_REF - -if [[ ${#required_missing[*]} -gt 0 ]]; then - printf 'error: missing required variables: %s\n' "${required_missing[*]}" >&2 - exit 1 -fi - -git_notes_remote_ref=refs/notes/remotes/$CONFIGURE_GIT_REMOTE_NAME/${GIT_NOTES_REF#refs/notes/} - -if [[ $RELEASE_NOTES ]]; then - - # Capture release notes - git notes add -f --file - <<-EOF - $RELEASE_NOTES - EOF - - # Push release notes - if [[ $CONFIGURE_GIT_REMOTE_NAME ]]; then - - if [[ $(git ls-remote "$CONFIGURE_GIT_REMOTE_NAME" "$GIT_NOTES_REF") ]]; then - git fetch "$CONFIGURE_GIT_REMOTE_NAME" "$GIT_NOTES_REF":"$git_notes_remote_ref" - git notes merge -s ours "$git_notes_remote_ref" - fi - - git push "$CONFIGURE_GIT_REMOTE_NAME" "$GIT_NOTES_REF" - - fi - -fi diff --git a/ci/steps/release-helm-changes.sh b/ci/steps/release-helm-changes.sh deleted file mode 100755 index e78b6313d0..0000000000 --- a/ci/steps/release-helm-changes.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -set -e - -required_missing=() - -test "$RELEASE_CHANNEL" || required_missing+=('RELEASE_CHANNEL') -test "$RELEASE_HELM_CHART" || required_missing+=('RELEASE_HELM_CHART') -test "$RELEASE_HELM_NAME" || RELEASE_HELM_NAME=$(basename "$RELEASE_HELM_CHART") -test "$RELEASE_HELM_VALUES" || RELEASE_HELM_VALUES= -test "$RELEASE_NAMESPACE" || RELEASE_NAMESPACE= - -export KUBECONFIG - -if [[ ${#required_missing[*]} -gt 0 ]]; then - printf 'error: missing required variables: %s\n' "${required_missing[*]}" >&2 - exit 1 -fi - -helm_verb=upgrade -helm_opts=() -chart_path=$(git rev-parse --show-toplevel)/$RELEASE_HELM_CHART - -IFS=: values_relpaths=( $RELEASE_HELM_VALUES ) -IFS=$' \t\n' - -if [[ ! -e $chart_path ]]; then - printf 'error: chart not found: %s\n' "$RELASE_HELM_CHART" >&2 - exit 1 -fi - -if [[ $RELEASE_NAMESPACE ]]; then - helm_opts+=('--namespace' "$RELEASE_NAMESPACE") -fi - -if [[ ! $(helm status "$RELEASE_HELM_NAME" "${helm_opts[@]}" 2>/dev/null) ]]; then - helm_verb=install -fi - -for relpath in "${values_relpaths[@]}"; do - abspath=$(git rev-parse --show-toplevel)/$relpath - if [[ -e $abspath ]]; then - helm_opts+=('--values' "$abspath") - fi -done - -while read dep_name dep_version dep_repo dep_status; do - test "$dep_status" || continue - if [[ $dep_status != ok ]]; then - printf 'chart %s: dependency update\n' "$RELEASE_HELM_CHART" - helm dependency update "$chart_path" - break - fi -done <<< "$(helm dependency list "$chart_path" | tail -n +2)" - -if [[ $RELEASE_NAMESPACE ]] && [[ ! $(kubectl get ns "$RELEASE_NAMESPACE" 2>/dev/null) ]]; then - printf '%s: create namespace: %s\n' "$RELEASE_HELM_NAME" "$RELEASE_NAMESPACE" - kubectl create ns "$RELEASE_NAMESPACE" -fi - -# helm does not have a native diff command and the output of helm template does -# not attach the namespace passed using -n to resulting resources. this means -# helm template output cannot be passed directly to kubectl diff. -# this workaround wraps the output of helm template in a kustomize directory to allow -# native diffing. -diff_tmpdir=$(mktemp -d -t "$RELEASE_HELM_NAME.XXXXXX") -trap "rm -rf $diff_tmpdir" EXIT -helm template "$RELEASE_HELM_NAME" "$chart_path" "${helm_opts[@]}" > "$diff_tmpdir/manifest.yaml" -cat < "$diff_tmpdir/kustomization.yaml" -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: $RELEASE_NAMESPACE -resources: -- manifest.yaml -EOF -diff_contents=$(kubectl diff -k "$diff_tmpdir" || true) - -if [[ $diff_contents ]]; then - printf 'release: %s: helm %s\n' "$RELEASE_HELM_NAME" "$helm_verb" - helm "$helm_verb" "$RELEASE_HELM_NAME" "$chart_path" "${helm_opts[@]}" - test -z "$RELEASE_NOTES" || RELEASE_NOTES+=$'\n' - RELEASE_NOTES+=$(printf '[helm:%s=%s]\n\n%s\n' "$RELEASE_HELM_NAME" "$RELEASE_HELM_CHART" "$diff_contents") -else - printf 'skip: %s\n' "$RELEASE_HELM_NAME" -fi diff --git a/ci/steps/release-kustomize-changes.sh b/ci/steps/release-kustomize-changes.sh deleted file mode 100755 index fc4d714537..0000000000 --- a/ci/steps/release-kustomize-changes.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e - -required_missing=() - -test "$RELEASE_CHANNEL" || required_missing+=('RELEASE_CHANNEL') -test "$RELEASE_KUSTOMIZE_DIR" || required_missing+=('RELEASE_KUSTOMIZE_DIR') - -export KUBECONFIG - -if [[ ${#required_missing[*]} -gt 0 ]]; then - printf 'error: missing required variables: %s\n' "${required_missing[*]}" >&2 - exit 1 -fi - -kustomize_path=$(git rev-parse --show-toplevel)/$RELEASE_KUSTOMIZE_DIR -diff_contents=$(kubectl diff -k "$kustomize_path" || true) - -if [[ $diff_contents ]]; then - printf 'release: %s\n' "$RELEASE_KUSTOMIZE_DIR" - kubectl apply -k "$kustomize_path" - test -z "$RELEASE_NOTES" || RELEASE_NOTES+=$'\n' - RELEASE_NOTES+=$(printf '[kustomize:%s]\n\n%s\n' "$RELEASE_KUSTOMIZE_DIR" "$diff_contents") -else - printf 'skip: %s\n' "$RELEASE_KUSTOMIZE_DIR" -fi diff --git a/ci/vars/releases/prod-airflow-jobs.env b/ci/vars/releases/prod-airflow-jobs.env deleted file mode 100644 index b11059f1b6..0000000000 --- a/ci/vars/releases/prod-airflow-jobs.env +++ /dev/null @@ -1,2 +0,0 @@ -RELEASE_DRIVER=kustomize -RELEASE_KUSTOMIZE_DIR=kubernetes/apps/manifests/airflow-jobs diff --git a/ci/vars/releases/prod-grafana.env b/ci/vars/releases/prod-grafana.env deleted file mode 100644 index 367bd57273..0000000000 --- a/ci/vars/releases/prod-grafana.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=monitoring-grafana -RELEASE_HELM_NAME=grafana -RELEASE_HELM_CHART=kubernetes/apps/charts/grafana -RELEASE_HELM_VALUES=kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml diff --git a/ci/vars/releases/prod-jupyterhub.env b/ci/vars/releases/prod-jupyterhub.env deleted file mode 100644 index 8a0b3369ce..0000000000 --- a/ci/vars/releases/prod-jupyterhub.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=jupyterhub -RELEASE_HELM_NAME=jupyterhub -RELEASE_HELM_CHART=kubernetes/apps/charts/jupyterhub -RELEASE_HELM_VALUES= diff --git a/ci/vars/releases/prod-loki.env b/ci/vars/releases/prod-loki.env deleted file mode 100644 index 19e2d0f0d1..0000000000 --- a/ci/vars/releases/prod-loki.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=monitoring-loki -RELEASE_HELM_NAME=loki -RELEASE_HELM_CHART=kubernetes/apps/charts/loki -RELEASE_HELM_VALUES= diff --git a/ci/vars/releases/prod-metabase.env b/ci/vars/releases/prod-metabase.env deleted file mode 100644 index 7577003780..0000000000 --- a/ci/vars/releases/prod-metabase.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=metabase -RELEASE_HELM_NAME=metabase -RELEASE_HELM_CHART=kubernetes/apps/charts/metabase -RELEASE_HELM_VALUES=kubernetes/apps/values/metabase.yaml diff --git a/ci/vars/releases/prod-postgresql-backup-grafana.env b/ci/vars/releases/prod-postgresql-backup-grafana.env deleted file mode 100644 index 61d67fe130..0000000000 --- a/ci/vars/releases/prod-postgresql-backup-grafana.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=monitoring-grafana -RELEASE_HELM_NAME=postgresql-backup -RELEASE_HELM_CHART=kubernetes/apps/charts/postgresql-backup -RELEASE_HELM_VALUES=kubernetes/apps/values/postgresql-backup-grafana.yaml diff --git a/ci/vars/releases/prod-postgresql-backup-metabase.env b/ci/vars/releases/prod-postgresql-backup-metabase.env deleted file mode 100644 index e4ed5d8ffd..0000000000 --- a/ci/vars/releases/prod-postgresql-backup-metabase.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=metabase -RELEASE_HELM_NAME=postgresql-backup -RELEASE_HELM_CHART=kubernetes/apps/charts/postgresql-backup -RELEASE_HELM_VALUES=kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml diff --git a/ci/vars/releases/prod-postgresql-backup-sentry.env b/ci/vars/releases/prod-postgresql-backup-sentry.env deleted file mode 100644 index 4e67118092..0000000000 --- a/ci/vars/releases/prod-postgresql-backup-sentry.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=sentry -RELEASE_HELM_NAME=postgresql-backup -RELEASE_HELM_CHART=kubernetes/apps/charts/postgresql-backup -RELEASE_HELM_VALUES=kubernetes/apps/values/postgresql-backup-sentry.yaml diff --git a/ci/vars/releases/prod-prometheus.env b/ci/vars/releases/prod-prometheus.env deleted file mode 100644 index ec04525d50..0000000000 --- a/ci/vars/releases/prod-prometheus.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=monitoring-prometheus -RELEASE_HELM_NAME=prometheus -RELEASE_HELM_CHART=kubernetes/apps/charts/prometheus -RELEASE_HELM_VALUES=kubernetes/apps/values/prometheus.yaml diff --git a/ci/vars/releases/prod-promtail.env b/ci/vars/releases/prod-promtail.env deleted file mode 100644 index aa33a8319a..0000000000 --- a/ci/vars/releases/prod-promtail.env +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_DRIVER=helm -RELEASE_NAMESPACE=monitoring-loki -RELEASE_HELM_NAME=promtail -RELEASE_HELM_CHART=kubernetes/apps/charts/promtail -RELEASE_HELM_VALUES=kubernetes/apps/values/promtail.yaml diff --git a/ci/vars/releases/prod-sftp-ingest-elavon.env b/ci/vars/releases/prod-sftp-ingest-elavon.env deleted file mode 100644 index d69b713dda..0000000000 --- a/ci/vars/releases/prod-sftp-ingest-elavon.env +++ /dev/null @@ -1,2 +0,0 @@ -RELEASE_DRIVER=kustomize -RELEASE_KUSTOMIZE_DIR=kubernetes/apps/overlays/prod-sftp-ingest-elavon diff --git a/ci/workflows/service-release.sh b/ci/workflows/service-release.sh deleted file mode 100755 index 72bc640ca2..0000000000 --- a/ci/workflows/service-release.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -set -e - -# -# Env files -# - -for env_file in "$@"; do - - if ! [[ -e "$env_file" ]]; then - printf 'error: all cli arguments must be paths to environment files\n' >&2 - exit 1 - fi - - while read line; do - if [[ $line =~ ^([[:alnum:]_]+)=(.*)$ ]] && ! [[ $(declare -p "${BASH_REMATCH[1]}" 2>/dev/null) ]]; then - declare -x "$line" - else - continue - fi - done < "$env_file" - -done - -test "$CI_STEPS_DIR" || CI_STEPS_DIR=$(git rev-parse --show-toplevel)/ci/steps - -# -# Defaults -# - -test "$RELEASE_CHANNEL" || RELEASE_CHANNEL=$(basename "$(git symbolic-ref HEAD)") - -# -# Optional -# - -test "$CONFIGURE_GIT_REMOTE_NAME" || CONFIGURE_GIT_REMOTE_NAME= -test "$CONFIGURE_GIT_REMOTE_URL" || CONFIGURE_GIT_REMOTE_URL= - -export KUBECONFIG - -# -# Steps -# - -release_vars_root=$(git rev-parse --show-toplevel)/ci/vars/releases - -printf 'BEGIN STEP: configure-git-remote\n' -source "$CI_STEPS_DIR/configure-git-remote.sh" - -for env_file in "$release_vars_root"/"$RELEASE_CHANNEL"-*.env; do - # - # Per-app variable overrides - # - # CAUTION: these vars are not being loaded within a subshell, which means each - # app's variables are loaded into the global pipeline scope. The reason a - # subshell is avoided here is to allow each step to cumulatively modify the - # RELEASE_NOTES variable. - while read line; do - if [[ $line =~ ^([[:alnum:]_]+)=(.*)$ ]]; then - declare -x "$line" - else - continue - fi - done < "$env_file" - app_name=$(basename "$env_file" | sed 's/\.env$//') - printf 'BEGIN STEP: release-%s-changes: %s\n' "$RELEASE_DRIVER" "$app_name" - source "$CI_STEPS_DIR/release-$RELEASE_DRIVER-changes.sh" -done - -printf 'BEGIN STEP: release-git-notes\n' -source "$CI_STEPS_DIR/release-git-notes.sh" From 9a44d9350fa17dbb44e2977c489109cec2eb937b Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 17:27:49 -0400 Subject: [PATCH 34/56] remove outdated comment --- ci/invoke.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/invoke.yaml b/ci/invoke.yaml index e77a325b8c..134ecdb000 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -1,6 +1,5 @@ run: echo: true -# would be cool if we can define a pydantic type for this calitp: git_repo: . kdiff_outfile: kdiff.txt From 3b95b749fed0199fd9c0403da47f38acea8c0220 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Thu, 22 Jun 2023 17:28:30 -0400 Subject: [PATCH 35/56] one more --- ci/vars/project.env | 1 - 1 file changed, 1 deletion(-) delete mode 100644 ci/vars/project.env diff --git a/ci/vars/project.env b/ci/vars/project.env deleted file mode 100644 index d570e8aba7..0000000000 --- a/ci/vars/project.env +++ /dev/null @@ -1 +0,0 @@ -CONFIGURE_GIT_REMOTE_NAME=origin From 5733cf654cb0a1e7fbb15aa0b3b79e87e2479838 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 12:06:57 -0400 Subject: [PATCH 36/56] wip on updating infra/ci docs --- docs/_toc.yml | 14 +- docs/ci/README.md | 197 ------------------ docs/ci/steps/build-docker-image.md | 23 -- docs/ci/steps/build-git-notes.md | 20 -- docs/ci/steps/build-git-tag.md | 13 -- docs/ci/steps/configure-build-git-notes.md | 9 - docs/ci/steps/configure-docker-login.md | 26 --- docs/ci/steps/configure-git-remote.md | 11 - docs/ci/steps/release-kube-overlay.md | 7 - docs/ci/steps/validate-build-git-tag.md | 9 - docs/ci/steps/validate-clean-worktree.md | 3 - .../Git-Flow-Services.md | 0 .../Pipeline-Variables.md | 0 docs/infrastructure/README.md | 34 +++ 14 files changed, 35 insertions(+), 331 deletions(-) delete mode 100644 docs/ci/README.md delete mode 100644 docs/ci/steps/build-docker-image.md delete mode 100644 docs/ci/steps/build-git-notes.md delete mode 100644 docs/ci/steps/build-git-tag.md delete mode 100644 docs/ci/steps/configure-build-git-notes.md delete mode 100644 docs/ci/steps/configure-docker-login.md delete mode 100644 docs/ci/steps/configure-git-remote.md delete mode 100644 docs/ci/steps/release-kube-overlay.md delete mode 100644 docs/ci/steps/validate-build-git-tag.md delete mode 100644 docs/ci/steps/validate-clean-worktree.md rename docs/{ci => infrastructure}/Git-Flow-Services.md (100%) rename docs/{ci => infrastructure}/Pipeline-Variables.md (100%) create mode 100644 docs/infrastructure/README.md diff --git a/docs/_toc.yml b/docs/_toc.yml index a920d01bc7..8bbc557383 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -49,19 +49,7 @@ parts: - file: kubernetes/JupyterHub - file: kubernetes/architecture - file: kubernetes/deployment - - file: ci/README - sections: - - file: ci/Git-Flow-Services - - file: ci/Pipeline-Variables - - file: ci/steps/build-docker-image - - file: ci/steps/build-git-notes - - file: ci/steps/build-git-tag - - file: ci/steps/configure-build-git-notes - - file: ci/steps/configure-docker-login - - file: ci/steps/configure-git-remote - - file: ci/steps/release-kube-overlay - - file: ci/steps/validate-build-git-tag - - file: ci/steps/validate-clean-worktree + - file: infrastructure/README - file: github/contribute_to_repos.md - file: services/overview sections: diff --git a/docs/ci/README.md b/docs/ci/README.md deleted file mode 100644 index a5e737205a..0000000000 --- a/docs/ci/README.md +++ /dev/null @@ -1,197 +0,0 @@ -# CI - -The `ci` tree contains bash scripts which provide a framework for automation -pipelines. The pipelines framework is delivered as embedded bash scripts to -simplify the process of providing remote/local parity and platform independence -for pipeline runs. - -This document covers high level concepts and certain concrete descriptions of -the ci pipeline framework. For a higher level guide on utilizing git flows -powered by this framework, see -[the "Git Flow for Services" documentation](./Git-Flow-Services.md) - -## 1.0 Concepts - -This section breaks down the conceptual design of pipelines built with the -embedded ci framework. - -### 1.1 steps - -A step is a discrete, atomic task within a pipeline. Each step can be loosely -thought of as being part of a _phase_ and will consume zero or more _variables_. - -### 1.2 phases - -Each pipeline can be thought of as being conceptually divided into a series of -phases, though not every pipeline will necessarily need to run steps in every -phase. These phases are loose organizational concepts and are in no way depended -on at any sort of technical implementation level. - -- _validate_: steps which check the contents of variables for the existence of - a value, properly formed values, or any other acceptability criteria. -- _configure_: steps which apply configurations to the execution environment of - the pipeline based on input variables OR steps which populate variables from - external sources, e.g., commands that query remote APIs or git commands -- _build_: steps which produce or contribute to the production of an artifact. -- _release_: steps which perform or contribute to the performance of deploying a - build artifact. - -These phases will generally follow the above sequence, but the sequencing is by -no means strict. E.g., there may be plenty of instances where a _validate_ step -is executed after a _configure_ step, and perhaps even additional _configure_ -steps are run after that. - -### 1.3 variables - -Each pipeline is driven by a set of variables which may be provided from the -environment (i.e., values are interpretd without shell features such as array -syntax, interpolation, etc). Variables conceptually exist within a global -pipeline namespace; one variable may be consumed by multiple steps, but the same -variable will always be used to describe the same object or property across all -steps which consume it. By convention, every pipeline variable is named with one -of the following prefixes: - -- `CI_`: Control variables for configuring the pipeline run itself. -- `CONFIGURE_`: Variables which describe desired configuration within the - pipeline execution environment, such as git config parameters, `ssh_config` - parameters, kubeconfig data, etc. -- `BUILD_`: Variables which describe a build artifact or contain required - information to produce, package, or publish such an artifact. -- `RELEASE_`: Variables which describe a target environment, deployment - configuration, or other required information to deploy an artifact or - application (which is not already described by an existing `BUILD_` variable). - -### 1.4 workflows - -A workflow is a sequence of steps. There is no reason a workflow could not -include other workflows in place of or addition to individual steps. A complete -pipeline will generally consist of one or more workflows. - -## 2.0 Subtree Contents - -### 2.1 githooks - -This subtree contains hooks which can be symlinked into a `$GIT_DIR/hooks` -directory tp execute local automation workflows. These symlinks may be installed -by hand, or by executing a `make` command from within the subtree. Though these -hooks will generally attempt to replicate the git flow of a canonical remote, -the limitations of git hooks available in local environments typically make -perfect parity unreasonable to achieve. - -### 2.2 steps - -Each script in the `steps` subtree is an implementation of a single pipeline -step. Each step can be executed directly on a command line or included within a -workflow. It is not recommended to source these steps into a user shell as any -errors will force exit the shell. - -### 2.3 workflows - -Each script within the `workflows` subtree executes a sequence of one or more -steps. Workflows can be executed directly as commands or can be sourced into -the shell. The latter mode of operation is only recommended within CI systems, -as any errors will cause a force exit of the shell. When executed as a command, -each argument passed to the workflow is processed as an environment file which -provides variables to the workflow. The files are processed in argument order -with the first found definition of a variable taking precedence over any later -definition. Variable values are not evaluated using shell syntax; all values are -interpreted literally. - -## 3.0.0 Script Conventions - -The following conventions are relevant to writing new workflow or step scripts -or to parsing existing ones. - -### 3.1.0 steps - -#### 3.1.1 naming and header - -The name of each step script should be prefixed with the name of the phase which -it belongs to. E.g., `configure-*` for _configure_ phase steps, `build-*` for -_build_ phase steps, etc. - -Every step script should be made executable and MUST use `#!/bin/bash`, NOT -`#!/bin/sh`. - -Every step script MUST apply `set -e` in the script header. - -#### 3.1.2 variable declaration - -Pipeline variables (as described in section 1.3) consumed by a step MUST be -declared at the top of the script using the `test "$VAR_NAME"` convention in -order to test whether the variable is defined. If the variable is optional, the -test statement should be followed by an empty assignment using the `|| -VAR_NAME=` convention in order to clearly denote its optional nature. A sensible -default may also be set instead where relevant. If the variable is required, it -should be added to the conventional `required_missing` array. After the variable -declaration block, the step MUST evaluate the `required_missing` array and exit -with code 1 and the conventional error message if the array contains any -elements. See existing steps for examples of this convention. - -Variables which are localized to the step itself should be named in all lower -case and should always be initialized to some starting value, as previous steps -within the same workflow may have used localized variables of the same name. - -#### 3.1.3 idempotency - -Step scripts should be implemented idempotently-- that is, if the desired end -state of any work being performed by the step has already been achieved, that -work should be skipped. Step scripts should not ever, without exceptionally good -reason, cause errors in cases where the desired end state has already been -achieved. Step scripts are encouraged to provide helpful info messages whenever -work is skipped. - -### 3.2.0 workflows - -#### 3.2.1 naming and header - -Workflows names may generally follow the form `{object}-{action}` where -`{object}` is any domain-relevant entity and `{action}` describes the high level -task the workflow performs on that entity. This should be considered a loose -convention-- workflow names should truly take whatever form they need to in -order to plainly and usefully describe their purpose. - -Every workflow script should be made executable and MUST use `#!/bin/bash`, NOT -`#!/bin/sh`. - -Every workflow script MUST apply `set -e` in the script header. - -#### 3.2.3 environment files - -The first thing every workflow script MUST do is evaluate all environment files -passed as command line arguments. Refer to existing workflow scripts to see how -to conventionally process and evaluate these environment file arguments. - -Values assigned from declarations in these files MUST NOT be evaluated by the -shell, as this violates the tenets of an environment file as well as the user -contract. - -#### 3.2.3 variable declaration - -Workflows should include a default declaration for every pipeline variable -declared within every included step. This allows operators to easily determine -the variables available to drive a workflow without having to open and read each -individually included step, and further allows them to easily determine which -variables they are required to provide and the default values for variables -which they are not. - -Any pipeline variable which is required by the workflow and not provided at -runtime should be prompted for. See existing workflow scripts for how to -conventionally prompt for required variables. - -All pipeline variables should be declared with default or prompted values prior -to the execution of the first step. - -#### 3.2.4 step execution - -Each step should be executed using a `source` statement, and the `source` -statement should be preceded by an info message which prints the name of the -step which will execute next. - -Pipeline variables may be changed/redeclared in between steps, but evaluation of -logic between steps is generally discouraged. E.g., conditional execution should -be avoidable in most cases where steps are properly idempotent and explicitly -listing multiple instances of a step should be preferred in cases where a step -will be re-executed a deterministic number of times. Where there is good reason -to do so however, the full power of bash logic and evaluation may be employed to -wrap step execution. diff --git a/docs/ci/steps/build-docker-image.md b/docs/ci/steps/build-docker-image.md deleted file mode 100644 index f423256231..0000000000 --- a/docs/ci/steps/build-docker-image.md +++ /dev/null @@ -1,23 +0,0 @@ -# build-docker-image - -This step performs a docker image build and optioinally pushes the built image -to a remote registry. - -## Variables - -- `BUILD_DIR`: Path to the docker image context directory. Passed through to - `docker build`. -- `BUILD_REPO`: The name of the image. This is joined with `BUILD_ID` by a ":" - character to compose a fully qualified image name as passed to - `docker build -t`. When this value points to a remote registry, the step will - execute a `docker push` after the image is built. -- `BUILD_ID`: The image tag. This is joined with `BUILD_REPO` by a ":" character - to compose a fully qualified image name as passed to `docker build -t`. Do not - specify `latest` here; the step will automatically apply the `latest` tag to the - built image in addition to the tag specified by this value. Furthermore, doing - so will probably cause unexpected build skips on account of the idempotency - check. -- `BUILD_FORCE`: By default, when `BUILD_REPO` refers to a remote registry and - that registry already contains an image with a tag matching `BUILD_ID`, the - build step will skip work. Setting this flag will force the step to rebuild - and repush the image regardless of its presence in the remote registry. diff --git a/docs/ci/steps/build-git-notes.md b/docs/ci/steps/build-git-notes.md deleted file mode 100644 index 3b9bfb4b05..0000000000 --- a/docs/ci/steps/build-git-notes.md +++ /dev/null @@ -1,20 +0,0 @@ -# build-git-notes - -Store build variables in a git note for later referencing/loading. - -## Control Variables - -The following variables impact the behavior of the step itself: - -- `BUILD_GIT_TAG`: The git tag which the resulting note will be attached to -- `GIT_NOTES_REF`: Exported for consumption by `git-notes(1)`. Defaults to - `refs/notes/builds`. - -## Stored Variables - -The following variables and their set values will be saved to the resulting git -note. These are all required. - -- `BUILD_APP` -- `BUILD_ID` -- `BUILD_REPO` diff --git a/docs/ci/steps/build-git-tag.md b/docs/ci/steps/build-git-tag.md deleted file mode 100644 index e657f5c5ee..0000000000 --- a/docs/ci/steps/build-git-tag.md +++ /dev/null @@ -1,13 +0,0 @@ -# build-git-tag - -Create an annotated tag that follows the strict naming convention -`{BUILD_APP}/{BUILD_ID}`, populate the tag message with a changelog for -`{BUILD_APP}` since the last `{BUILD_ID}` it was tagged with, and set the new -tag name as the value of `BUILD_GIT_TAG`. - -## Variables - -- `BUILD_APP`: The service the tag is being created for. -- `BUILD_ID`: The new version `BUILD_APP` is to be tagged with. -- `CONFIGURE_GIT_REMOTE_NAME`: If configured, the newly created tag will be - pushed to this remote. diff --git a/docs/ci/steps/configure-build-git-notes.md b/docs/ci/steps/configure-build-git-notes.md deleted file mode 100644 index 559bc45906..0000000000 --- a/docs/ci/steps/configure-build-git-notes.md +++ /dev/null @@ -1,9 +0,0 @@ -# configure-build-git-notes - -Load build variables from a git note as prepared by the `build-git-notes` step. - -## Variables - -- `BUILD_GIT_TAG`: The git tag which the build note was attached to -- `GIT_NOTES_REF`: Exported for consumption by `git-notes(1)`. Defaults to - `refs/notes/build`. diff --git a/docs/ci/steps/configure-docker-login.md b/docs/ci/steps/configure-docker-login.md deleted file mode 100644 index 9cbc687f26..0000000000 --- a/docs/ci/steps/configure-docker-login.md +++ /dev/null @@ -1,26 +0,0 @@ -# configure-docker-login - -Setup authentication to a remote docker registry using `docker login` with -automatic recognition of and special handling for GCR and ECR registries. - -## Variables - -- `BUILD_REPO`: An image repository name as used by `build-docker-image`. The - remote registry domain and port will be extracted from this value. The step - will also use the value of this variable to detect an ECR or GCR registry and - activate special logic branches as needed. -- `BUILD_REPO_USER`: The registry username as passed to `docker login -u`. This - value is ignored for GCR registries and hard coded to "AWS" for ECR registries. -- `BUILD_REPO_SECRET`: The registry password as passed to `docker login - --password-stdin`. This value is ignored for GCR registries and will be - auto-populated from `aws ecr get-login-password` for ECR registries. - -## GCR Additional Notes - -Ensure a GCP login has been performed with `gcloud auth login` and the correct -project is selected with `gcloud config set project` prior to running this step. - -## ECR Additional notes - -Ensure `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are set appropriately -prior to running this step. diff --git a/docs/ci/steps/configure-git-remote.md b/docs/ci/steps/configure-git-remote.md deleted file mode 100644 index cc69b6409c..0000000000 --- a/docs/ci/steps/configure-git-remote.md +++ /dev/null @@ -1,11 +0,0 @@ -# configure-git-remote - -Ensure the presence of a git remote in the local repo. - -## Variables - -- `CONFIGURE_GIT_REMOTE_NAME`: The name of the git remote to configure -- `CONFIGURE_GIT_REMOTE_URL`: The url to be pointed to by - `CONFIGURE_GIT_REMOTE_NAME`. If `CONFIGURE_GIT_REMOTE_NAME` already exists and - points to a different url, it will be updated to point to - `CONFIGURE_GIT_REMOTE_URL`. diff --git a/docs/ci/steps/release-kube-overlay.md b/docs/ci/steps/release-kube-overlay.md deleted file mode 100644 index 2a09f6e774..0000000000 --- a/docs/ci/steps/release-kube-overlay.md +++ /dev/null @@ -1,7 +0,0 @@ -# release-kube-overlay - -Execute `kubectl apply -k` on the specified path. - -## Variables - -- `RELEASE_KUBE_OVERLAY`: Path to pass to `kubectl apply -k`. diff --git a/docs/ci/steps/validate-build-git-tag.md b/docs/ci/steps/validate-build-git-tag.md deleted file mode 100644 index dc3e41e9e9..0000000000 --- a/docs/ci/steps/validate-build-git-tag.md +++ /dev/null @@ -1,9 +0,0 @@ -# validate-build-git-tag - -Force exit with an error if `BUILD_GIT_TAG` is not defined or if it is not an -annotated tag of which the name adheres to the strict form -`{BUILD_APP}/{BUILD_ID}`. - -## Variables - -- `BUILD_GIT_TAG`: The name of the tag to be validated. diff --git a/docs/ci/steps/validate-clean-worktree.md b/docs/ci/steps/validate-clean-worktree.md deleted file mode 100644 index ef891bc450..0000000000 --- a/docs/ci/steps/validate-clean-worktree.md +++ /dev/null @@ -1,3 +0,0 @@ -# validate-clean-worktree - -Force exit with an error if the repository working tree is dirty. diff --git a/docs/ci/Git-Flow-Services.md b/docs/infrastructure/Git-Flow-Services.md similarity index 100% rename from docs/ci/Git-Flow-Services.md rename to docs/infrastructure/Git-Flow-Services.md diff --git a/docs/ci/Pipeline-Variables.md b/docs/infrastructure/Pipeline-Variables.md similarity index 100% rename from docs/ci/Pipeline-Variables.md rename to docs/infrastructure/Pipeline-Variables.md diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md new file mode 100644 index 0000000000..7e132b784b --- /dev/null +++ b/docs/infrastructure/README.md @@ -0,0 +1,34 @@ +# Infrastructure code + +# pyinvoke + +[invoke](https://docs.pyinvoke.org/en/stable/) is a Python framework for executing subprocess and building a CLI application. +The tasks are defined in `tasks.py` and configuration in `invoke.yaml`; config values under the top-level `calitp` +are specific to our defined tasks. + +Run `poetry run invoke -l` to list the available commands, and `poetry run invoke -h ` to get more detailed help for each individual command. + +## CI/CD + +All CI/CD automation in this project is executed via GitHub Actions, whose workflow files . + +## deploy-airflow.yml + +While we're using GCP Composer, "deployment" of Airflow consists of two parts: + +1. Calling `gcloud composer environments update ...` to update the Composer environment with new (or specific versions of) packages +2. Copying the [dags](../../airflow/dags) and [plugins](../../airflow/plugins) folders to a GCS bucket that Composer reads + +## build-*.yml workflows + +Workflows prefixed with `build-` generally lint, test, and (usually) publish either a Python package or a Docker image. + +## service-*.yml workflows + +Workflows prefixed with `service-` deal with Kubernetes deployments. + +* `service-release-candidate.yml` creates candidate branches, using hologit to bring in external Helm charts and remove irrelevant (i.e. non-infra) code +* `service-release-diff.yml` renders kubectl diffs on PRs targeting release branches +* `service-release-channel.yml` deploys to a given channel (i.e. environment) on updates to a release branch + +Some of these workflows use the same `invoke` framework described earlier. From 7b71f1f31a0c125998a6c48416c3a6290ea708a5 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 12:30:00 -0400 Subject: [PATCH 37/56] start streamlining invoke tasks --- .github/workflows/service-release-channel.yml | 3 +- .github/workflows/service-release-diff.yml | 4 +- ci/invoke.yaml | 1 + ci/tasks.py | 136 ++++++++---------- 4 files changed, 64 insertions(+), 80 deletions(-) diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml index 695187defc..21c77fb364 100644 --- a/.github/workflows/service-release-channel.yml +++ b/.github/workflows/service-release-channel.yml @@ -44,5 +44,4 @@ jobs: printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" poetry install - poetry run invoke krelease $RELEASE_CHANNEL - poetry run invoke hrelease $RELEASE_CHANNEL + poetry run invoke release $RELEASE_CHANNEL diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 7faa26009a..e01c9e8521 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -50,9 +50,7 @@ jobs: printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" set -- "$GITHUB_WORKSPACE/ci/vars/project.env" poetry install - poetry run invoke kdiff $RELEASE_CHANNEL --outfile=kdiff.txt - poetry run invoke hdiff $RELEASE_CHANNEL --outfile=hdiff.txt - cat kdiff.txt hdiff.txt > diff.txt + poetry run invoke diff $RELEASE_CHANNEL --outfile=diff.txt - uses: peter-evans/find-comment@v2 id: fc diff --git a/ci/invoke.yaml b/ci/invoke.yaml index 134ecdb000..8e66398238 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -1,5 +1,6 @@ run: echo: true + echo_format: "Executing: {command}" # without this, the echoed text is white which does not show on light themes calitp: git_repo: . kdiff_outfile: kdiff.txt diff --git a/ci/tasks.py b/ci/tasks.py index 5d0d5a0b56..8ebecd40c7 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -63,59 +63,51 @@ def git_root(self) -> Path: @task def parse_calitp_config(c): + """ + Parses the top-level calitp configuration key via Pydantic + """ c.update({"calitp_config": Config(**c.config._config["calitp"])}) -@task(parse_calitp_config) -def kdiff( - c, - channel, - app=None, - outfile=None, -): - full_diff = "" - - release: Release +def get_releases(c, channel, driver: ReleaseDriver, app=None) -> List[Release]: + ret = [] for release in c.calitp_config.channels[channel].releases: - if release.driver == ReleaseDriver.kustomize and ( + if (not driver or release.driver == driver) and ( not app or app == release.name ): - kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) - cmd = f"kubectl diff -k {kustomize_dir}" - print(cmd, flush=True) - result: Result = c.run( - cmd, - warn=True, - ) - if result.exited != 0: - full_diff += result.stdout - c.update({"kdiff": full_diff}) - if outfile: - msg = f"```{full_diff}```" if full_diff else "No kustomize changes found.\n" - print(f"writing {len(msg)=} to {outfile}", flush=True) - with open(outfile, "w") as f: - f.write(msg) - - -@task(parse_calitp_config) -def hdiff( + ret.append(release) + return ret + + +@task( + parse_calitp_config, + help={ + "channel": "The release channel/environment, e.g. test or prod", + "driver": "The k8s driver (kustomize or helm)", + }, +) +def diff( c, - channel, + channel: str, + driver=None, app=None, outfile=None, ): + """ + Applies kubectl diff to manifests for a given channel. + """ + actual_driver = ReleaseDriver[driver] if driver else None full_diff = "" release: Release - for release in c.calitp_config.channels[channel].releases: - if release.driver == ReleaseDriver.helm and (not app or app == release.name): + for release in get_releases(c, channel, driver=actual_driver, app=app): + if release.driver == ReleaseDriver.kustomize: + kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) + result: Result = c.run(f"kubectl diff -k {kustomize_dir}", warn=True) + elif release.driver == ReleaseDriver.helm: chart_path = c.calitp_config.git_root / Path(release.helm_chart) - cmd = f"helm dependency update {chart_path}" - print(cmd, flush=True) - c.run( - cmd, - warn=True, - ) + c.run(f"helm dependency update {chart_path}") + with tempfile.TemporaryDirectory() as tmpdir: manifest_path = Path(tmpdir) / Path("manifest.yaml") kustomization_path = Path(tmpdir) / Path("kustomization.yaml") @@ -125,65 +117,56 @@ def hdiff( for values_file in release.helm_values ] ) - cmd = f"helm template {release.helm_name} {chart_path} --namespace {release.namespace} {values_str} > {manifest_path}" - print(cmd, flush=True) c.run( - cmd, - warn=True, + f"helm template {release.helm_name} {chart_path} --namespace {release.namespace} {values_str} > {manifest_path}" ) with open(kustomization_path, "w") as f: f.write(KUSTOMIZE_HELM_TEMPLATE.format(namespace=release.namespace)) - cmd = f"kubectl diff -k {tmpdir}" - print(cmd, flush=True) - result: Result = c.run( - cmd, - warn=True, - ) - if result.exited != 0: - full_diff += result.stdout - c.update({"hdiff": full_diff}) + result: Result = c.run(f"kubectl diff -k {tmpdir}", warn=True) + else: + print(f"Encountered unknown driver: {release.driver}", flush=True) + raise RuntimeError + + if result.exited != 0: + full_diff += result.stdout + + msg = ( + f"```{full_diff}```" + if full_diff + else f"No {driver if driver else 'manifest'} changes found for {channel}.\n" + ) if outfile: - msg = f"```{full_diff}```" if full_diff else "No helm changes found.\n" print(f"writing {len(msg)=} to {outfile}", flush=True) with open(outfile, "w") as f: f.write(msg) + else: + print(msg, flush=True) # TODO: we may want to split up channels into separate files so channel is not an argument but a config file @task(parse_calitp_config) -def krelease(c, channel, app=None): - release: Release - for release in c.calitp_config.channels[channel].releases: - if release.driver == ReleaseDriver.kustomize and ( - not app or app == release.name - ): - kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) - cmd = f"kubectl apply -k {kustomize_dir}" - print(cmd, flush=True) - c.run(cmd) - - -@task(parse_calitp_config) -def hrelease( +def release( c, - channel, + channel: str, + driver=None, app=None, ): release: Release - for release in c.calitp_config.channels[channel].releases: - if release.driver == ReleaseDriver.helm and (not app or app == release.name): + actual_driver = ReleaseDriver[driver] if driver else None + for release in get_releases(c, channel, driver=actual_driver, app=app): + if release.driver == ReleaseDriver.kustomize: + kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) + c.run(f"kubectl apply -k {kustomize_dir}") + elif release.driver == ReleaseDriver.helm: chart_path = c.calitp_config.git_root / Path(release.helm_chart) - cmd = f"helm dependency update {chart_path}" - print(cmd, flush=True) - c.run(cmd, warn=True) + c.run(f"helm dependency update {chart_path}", warn=True) values_str = " ".join( [ f"--values {c.calitp_config.git_root / Path(values_file)}" for values_file in release.helm_values ] ) - cmd = f"kubectl get ns {release.namespace}" - result: Result = c.run(cmd) + result: Result = c.run(f"kubectl get ns {release.namespace}") verb = "upgrade" if result.exited != 0: @@ -194,3 +177,6 @@ def hrelease( c.run( f"helm {verb} {release.helm_name} {chart_path} --namespace {release.namespace} {values_str}" ) + else: + print(f"Encountered unknown driver: {release.driver}", flush=True) + raise RuntimeError From 25e62f0399bee95e6ceaa01910ba81a4af0c5307 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 12:32:33 -0400 Subject: [PATCH 38/56] remove unused config var --- ci/invoke.yaml | 1 - ci/tasks.py | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/invoke.yaml b/ci/invoke.yaml index 8e66398238..060bd3e931 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -3,7 +3,6 @@ run: echo_format: "Executing: {command}" # without this, the echoed text is white which does not show on light themes calitp: git_repo: . - kdiff_outfile: kdiff.txt channels: test: releases: diff --git a/ci/tasks.py b/ci/tasks.py index 8ebecd40c7..8158b9fdcb 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -46,7 +46,6 @@ class Channel(BaseModel): # TODO: rename this class Config(BaseModel): git_repo: git.Repo - kdiff_outfile: str channels: Dict[str, Channel] class Config: From ebd07ced02957a1a273ba7335be5ab58a29c5d76 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 12:34:56 -0400 Subject: [PATCH 39/56] help text --- ci/tasks.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index 8158b9fdcb..8f3af70919 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -78,11 +78,18 @@ def get_releases(c, channel, driver: ReleaseDriver, app=None) -> List[Release]: return ret +GENERIC_HELP = { + "channel": "The release channel/environment, e.g. test or prod", + "driver": "The k8s driver (kustomize or helm)", + "app": "The specific app/release (e.g. metabase or archiver)", +} + + @task( parse_calitp_config, help={ - "channel": "The release channel/environment, e.g. test or prod", - "driver": "The k8s driver (kustomize or helm)", + **GENERIC_HELP, + "outfile": "File in which to save the combined kubectl diff output", }, ) def diff( @@ -143,13 +150,16 @@ def diff( # TODO: we may want to split up channels into separate files so channel is not an argument but a config file -@task(parse_calitp_config) +@task(parse_calitp_config, help=GENERIC_HELP) def release( c, channel: str, driver=None, app=None, ): + """ + Releases (i.e. deploys) apps into a specific channel. + """ release: Release actual_driver = ReleaseDriver[driver] if driver else None for release in get_releases(c, channel, driver=actual_driver, app=app): From 2edb932a890009ea203797a0a3889732c3ae6cb4 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 12:40:09 -0400 Subject: [PATCH 40/56] add mypy, make pass --- ci/poetry.lock | 70 ++++++++++++++++++++++++++++++++++++++++++++++- ci/pyproject.toml | 3 ++ ci/tasks.py | 22 +++++++++++---- 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/ci/poetry.lock b/ci/poetry.lock index 4699459618..fee4d5989c 100644 --- a/ci/poetry.lock +++ b/ci/poetry.lock @@ -87,6 +87,63 @@ files = [ {file = "invoke-2.1.3.tar.gz", hash = "sha256:a3b15d52d50bbabd851b8a39582c772180b614000fa1612b4d92484d54d38c6b"}, ] +[[package]] +name = "mypy" +version = "1.4.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mypy-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3af348e0925a59213244f28c7c0c3a2c2088b4ba2fe9d6c8d4fbb0aba0b7d05"}, + {file = "mypy-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0b2e0da7ff9dd8d2066d093d35a169305fc4e38db378281fce096768a3dbdbf"}, + {file = "mypy-1.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210fe0f39ec5be45dd9d0de253cb79245f0a6f27631d62e0c9c7988be7152965"}, + {file = "mypy-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f7a5971490fd4a5a436e143105a1f78fa8b3fe95b30fff2a77542b4f3227a01f"}, + {file = "mypy-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:50f65f0e9985f1e50040e603baebab83efed9eb37e15a22a4246fa7cd660f981"}, + {file = "mypy-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1b5c875fcf3e7217a3de7f708166f641ca154b589664c44a6fd6d9f17d9e7e"}, + {file = "mypy-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4c734d947e761c7ceb1f09a98359dd5666460acbc39f7d0a6b6beec373c5840"}, + {file = "mypy-1.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5984a8d13d35624e3b235a793c814433d810acba9eeefe665cdfed3d08bc3af"}, + {file = "mypy-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0f98973e39e4a98709546a9afd82e1ffcc50c6ec9ce6f7870f33ebbf0bd4f26d"}, + {file = "mypy-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:19d42b08c7532d736a7e0fb29525855e355fa51fd6aef4f9bbc80749ff64b1a2"}, + {file = "mypy-1.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ba9a69172abaa73910643744d3848877d6aac4a20c41742027dcfd8d78f05d9"}, + {file = "mypy-1.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a34eed094c16cad0f6b0d889811592c7a9b7acf10d10a7356349e325d8704b4f"}, + {file = "mypy-1.4.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:53c2a1fed81e05ded10a4557fe12bae05b9ecf9153f162c662a71d924d504135"}, + {file = "mypy-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bba57b4d2328740749f676807fcf3036e9de723530781405cc5a5e41fc6e20de"}, + {file = "mypy-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:653863c75f0dbb687d92eb0d4bd9fe7047d096987ecac93bb7b1bc336de48ebd"}, + {file = "mypy-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7461469e163f87a087a5e7aa224102a30f037c11a096a0ceeb721cb0dce274c8"}, + {file = "mypy-1.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cf0ca95e4b8adeaf07815a78b4096b65adf64ea7871b39a2116c19497fcd0dd"}, + {file = "mypy-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:94a81b9354545123feb1a99b960faeff9e1fa204fce47e0042335b473d71530d"}, + {file = "mypy-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:67242d5b28ed0fa88edd8f880aed24da481929467fdbca6487167cb5e3fd31ff"}, + {file = "mypy-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f2b353eebef669529d9bd5ae3566905a685ae98b3af3aad7476d0d519714758"}, + {file = "mypy-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:62bf18d97c6b089f77f0067b4e321db089d8520cdeefc6ae3ec0f873621c22e5"}, + {file = "mypy-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca33ab70a4aaa75bb01086a0b04f0ba8441e51e06fc57e28585176b08cad533b"}, + {file = "mypy-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5a0ee54c2cb0f957f8a6f41794d68f1a7e32b9968675ade5846f538504856d42"}, + {file = "mypy-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6c34d43e3d54ad05024576aef28081d9d0580f6fa7f131255f54020eb12f5352"}, + {file = "mypy-1.4.0-py3-none-any.whl", hash = "sha256:f051ca656be0c179c735a4c3193f307d34c92fdc4908d44fd4516fbe8b10567d"}, + {file = "mypy-1.4.0.tar.gz", hash = "sha256:de1e7e68148a213036276d1f5303b3836ad9a774188961eb2684eddff593b042"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "pydantic" version = "1.10.9" @@ -214,6 +271,17 @@ files = [ {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, ] +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + [[package]] name = "typing-extensions" version = "4.6.3" @@ -243,4 +311,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "4e02af70483866dead32999c3bd96ce72196eacf2558a232fa25a16239500e70" +content-hash = "a862514a7275573803be6574106d0034ba52c0b638e38f0dce335e0998f3bb50" diff --git a/ci/pyproject.toml b/ci/pyproject.toml index fdd83afd62..39ea9b4d36 100644 --- a/ci/pyproject.toml +++ b/ci/pyproject.toml @@ -14,6 +14,9 @@ doit = "^0.36.0" pyyaml = "^6.0" +[tool.poetry.group.dev.dependencies] +mypy = "^1.4.0" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/ci/tasks.py b/ci/tasks.py index 8f3af70919..9cd33bf308 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -56,7 +56,9 @@ def parse_git_repo(cls, v): return git.Repo(v, search_parent_directories=True) @property - def git_root(self) -> Path: + def git_root(self) -> Optional[Path]: + if not self.git_repo.working_tree_dir: + return None return Path(self.git_repo.working_tree_dir) @@ -68,7 +70,9 @@ def parse_calitp_config(c): c.update({"calitp_config": Config(**c.config._config["calitp"])}) -def get_releases(c, channel, driver: ReleaseDriver, app=None) -> List[Release]: +def get_releases( + c, channel, driver: Optional[ReleaseDriver] = None, app=None +) -> List[Release]: ret = [] for release in c.calitp_config.channels[channel].releases: if (not driver or release.driver == driver) and ( @@ -102,15 +106,18 @@ def diff( """ Applies kubectl diff to manifests for a given channel. """ + assert c.calitp_config.git_root is not None actual_driver = ReleaseDriver[driver] if driver else None full_diff = "" + result: Result - release: Release for release in get_releases(c, channel, driver=actual_driver, app=app): if release.driver == ReleaseDriver.kustomize: + assert release.kustomize_dir is not None kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) - result: Result = c.run(f"kubectl diff -k {kustomize_dir}", warn=True) + result = c.run(f"kubectl diff -k {kustomize_dir}", warn=True) elif release.driver == ReleaseDriver.helm: + assert release.helm_chart is not None chart_path = c.calitp_config.git_root / Path(release.helm_chart) c.run(f"helm dependency update {chart_path}") @@ -128,7 +135,7 @@ def diff( ) with open(kustomization_path, "w") as f: f.write(KUSTOMIZE_HELM_TEMPLATE.format(namespace=release.namespace)) - result: Result = c.run(f"kubectl diff -k {tmpdir}", warn=True) + result = c.run(f"kubectl diff -k {tmpdir}", warn=True) else: print(f"Encountered unknown driver: {release.driver}", flush=True) raise RuntimeError @@ -160,13 +167,16 @@ def release( """ Releases (i.e. deploys) apps into a specific channel. """ - release: Release + assert c.calitp_config.git_root is not None actual_driver = ReleaseDriver[driver] if driver else None + for release in get_releases(c, channel, driver=actual_driver, app=app): if release.driver == ReleaseDriver.kustomize: + assert release.kustomize_dir is not None kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) c.run(f"kubectl apply -k {kustomize_dir}") elif release.driver == ReleaseDriver.helm: + assert release.helm_chart is not None chart_path = c.calitp_config.git_root / Path(release.helm_chart) c.run(f"helm dependency update {chart_path}", warn=True) values_str = " ".join( From 063367ddab4f84eb766a0a201b1261aa5691984b Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 12:53:09 -0400 Subject: [PATCH 41/56] clean up gitops docs --- docs/infrastructure/Git-Flow-Services.md | 203 ---------------------- docs/infrastructure/Pipeline-Variables.md | 34 ---- docs/infrastructure/README.md | 16 ++ 3 files changed, 16 insertions(+), 237 deletions(-) delete mode 100644 docs/infrastructure/Git-Flow-Services.md delete mode 100644 docs/infrastructure/Pipeline-Variables.md diff --git a/docs/infrastructure/Git-Flow-Services.md b/docs/infrastructure/Git-Flow-Services.md deleted file mode 100644 index 30252a961c..0000000000 --- a/docs/infrastructure/Git-Flow-Services.md +++ /dev/null @@ -1,203 +0,0 @@ -# Git Flow for Services - -This document describes how to utilize git flows powered by the CI framework -embedded in this repository in order to automate the build of docker images -defined in the `services` tree and automate the deployment of applications -defined in the `kubernetes/apps` tree. - -## 1.0 Build - -This flow is powered by: -[`.github/workflows/service-build.yml`](https://github.com/cal-itp/data-infra/blob/main/.github/workflows/service-build.yml) - -### 1.1 quick start - -```bash -# this is a path under services/$app_name -app_name=gtfs-rt-archive -# this would normally be something like 1.0, 1.1, etc -app_version=$(whoami).$(date +%Y-%M-%d@%H%M) -# choose your appropriate remote name if different -git_remote=origin - -# an annotated tag is a requirement; lightweight tags are not accepted -# see below for details on using the more convenient build-git-tag interface -git tag -a "${app_name}/${app_version}" - -# push the annotated tag; this triggers the image build which is then pushed to -# the gcr repository -git push "$git_remote" "${app_name}/${app_version}" -``` - -### 1.2 details - -The tag name MUST adhere to the `${app_name}/${app_version}` convention. This -name will be parsed in order to help name the resulting docker image. - -The `$app_name` in the git tag must match a path at `services/$app_name` which -contains a Dockerfile. This name also corresponds to the repository portion of -the image name name which is pushed to the GCR registry. - -The `$app_version` in the git tag will be used as the tag portion of the docker -image name which is pushed to the registry. - -Consequently, the resulting image build is pushed to -`{gcr_registry}/{app_name}:{app_version}`. - -It is recommended to use the `build-git-tag` step to create the tag, like: - -```bash -export BUILD_APP=$app_name -export BUILD_ID=$app_version -export CONFIGURE_GIT_REMOTE_NAME=$git_remote -./ci/steps/build-git-tag.sh -``` - -Using this build step to create a service tag offers a few conveniences: - -- Inputs are validated, ensuring the git tag is created with a properly formed - name. -- The tag message is auto-generated, consisting of a changelog of all changes to - `services/$app_name` since the previous release of the service. -- The tag is auto-pushed to the git remote after creation, triggering the build - process. - -## 2.0.0 Deploy - -This flow is powered by: - -- [`.github/workflows/service-release-candidate.yml`](https://github.com/cal-itp/data-infra/blob/main/.github/workflows/service-release-candidate.yml) -- [`.github/workflows/service-release-channel.yml`](https://github.com/cal-itp/data-infra/blob/main/.github/workflows/service-release-channel.yml) - -### 2.1.0 quick start - -#### 2.1.1 deploy your topic branch to test - -```bash -# this can be determined from a release config under ci/vars/releases -app_name=gtfs-rt-archive -# choose your appropriate remote name if different -git_remote=origin -topic_branch=$(git branch --show-current) -# load the helm or kustomize specific parameters -source ci/vars/releases/test-$app_name.env - -if [[ $RELEASE_DRIVER == 'kustomize' ]]; then - # bump the newTag version to match the newly pushed app version - $EDITOR kubernetes/apps/overlays/$app_name-release/kustomization.yaml -elif [[ $RELEASE_DRIVER == 'helm' ]]; then - # update the tag part of the image parameter; how to do this may vary from app to app - # the RELEASE_HELM_VALUES parameter in the release config points to the values file which should be changed - $EDITOR $RELEASE_HELM_VALUES -fi - -# commit the change -git commit -am "ops($app_name): release new version" - -# trigger a release candidate build -git push $git_remote - -# -# wait for the "Build release candidate" action to complete in GitHub -# - -# fetch the release candidate -git fetch $git_remote candidates/$topic_branch - -# deploy to test -git push -f $git_remote $git_remote/candidates/$topic_branch:releases/test - -# after it looks good in test, merge your changes into main. -``` - -#### 2.1.2 Deploy your changes to prod - -In GitHub: - -1. Go through the typical PR process to merge your topic branch into `main` -2. Open a PR from `candidates/main` into `releases/prod` -3. Receive an approval -4. Merge the change - -### 2.2.0 details - -The automated deploy process essentially consists of: - -1. Modifying yaml files associated with an app as desired -2. Push those files to the GitHub repository -3. Wait for a release candidate to build -4. Merge or push the release candidate into a release branch - -#### 2.2.1 apps - -App configurations for kubernetes deployed apps are organized under -`kubernetes/apps`. There are two deployment patterns which are supported for -kubertnetes: - -1. kustomize -2. helm - - -##### kustomize - -There are two locations where kustomize manifests are stored: - -- `kubernetes/apps/manifests`: holds base kubernetes manifests. These should be - suiltable to be directly applied using `kubectl apply -f`, but will generally - be applied using `kubectl apply -k`. -- `kubernetes/apps/overlays`: holds overlays which are based on manifests in - `kubernetes/apps/manifests`. These are only ever expected be suitable for - application using `kubectl apply -k`. - -When deploying changed kustomize manifests, the ci pipeline will essentially -execute the command: - -```bash -kubectl apply -k kubernetes/apps/overlays/$app_name-$release_channel -``` - -##### helm - -There are separate locations for storing embedded helm charts and helm values -files. Not all values files will necessarily have a corresponding chart; some -charts are pulled from external sources. - -- charts location: `kubernetes/apps/charts` -- values location: `kubernetes/apps/values` - -The pipeline will run a `helm install` or `helm upgrade` command as appropriate -when deploying these applications. - -#### 2.2.2 release channels - -Each release branch is tied to a specific "release channel", which is simply a -less ambiguous term used to describe an "environment" (e.g., "production", "staging", etc). -There are currently two release branches: - -1. `releases/test` -2. `releases/prod` - -The basename of the branch (i.e., the name after the last "/" character) is used -to determine the name of the release channel being deployed into. - -#### 2.2.3 release candidates - -Whenever changes to deployment-relevant files (mostly kubernetes yaml files or -ci scripts) are pushed up to a topic branch, it triggers the build of a "release -candidate" branch. This is a special branch which is pruned down to include only -the files which are relevant to performing deployments into release channels. A -deployment should be triggered by pushing or merging a release candidate branch -into a release channel branch. - -The release candidate for each topic branch is pushed to -`candidates/$topic_branch_name`. - -The release candidate `candidates/main` is the only candidate which should ever -be merged into `releases/prod` and this must be done via PRs. - -#### 2.2.4 release configs - -Each app has a channel-specific release configuration stored at -`ci/vars/releases/$release_channel-$app_name.env`. This config file informs the -ci pipeline which deployment pattern the app is using and provides helm or -kustomize specific parameters for the deployment. diff --git a/docs/infrastructure/Pipeline-Variables.md b/docs/infrastructure/Pipeline-Variables.md deleted file mode 100644 index ff3c4fbfe0..0000000000 --- a/docs/infrastructure/Pipeline-Variables.md +++ /dev/null @@ -1,34 +0,0 @@ -# Pipeline Variables - -This document describes the general meaning of each available pipeline variable. -For more context specific meanings, refer to individual step documentation. Step -scripts should always use existing pipeline variables where one can adequately -describe the desired input. - -## Configure Phase - -- `CONFIGURE_GIT_REMOTE_NAME`: The name of a git remote to configure. -- `CONFIGURE_GIT_REMOTE_URL`: The url to associate with the remote - `CONFIGURE_GIT_REMOTE_NAME`. - -## Build Phase - -- `BUILD_APP`: The name of the application the build artifact is associated - with. -- `BUILD_ID`: A unique version number for the build artifact. -- `BUILD_DIR`: A path pointing to a source directory for the build. -- `BUILD_REPO`: A location (may be local or remote; step dependent) to which the - build artifact should be published. -- `BUILD_REPO_USER`: A username to authenticate against `BUILD_REPO` with. -- `BUILD_REPO_SECRET`: A secret to authenticate against `BUILD_REPO` with. -- `BUILD_FORCE`: Disable build idempotency; perform the build step even if there - is already an existing build for `BUILD_ID` of `BUILD_APP`. -- `BUILD_GIT_TAG`: A git tag identifying the source revision the build artifact - is built from. - -## Release Phase - -- `RELEASE_CHANNEL`: Identifies the environment a build should be deployed to. - May be something like "production", "staging", etc. -- `RELEASE_KUBE_OVERLAY`: Path to a directory containing a `kustomization.yaml` - to be deployed. Passed directly as an argument to `kubectl apply -k`. diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index 7e132b784b..1b324f7a10 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -32,3 +32,19 @@ Workflows prefixed with `service-` deal with Kubernetes deployments. * `service-release-channel.yml` deploys to a given channel (i.e. environment) on updates to a release branch Some of these workflows use the same `invoke` framework described earlier. + +## GitOps +The workflows described above also define their triggers. In general, developer workflows should follow these steps. + +1. Check out a feature branch +2. Put up a PR for that feature branch, targeting `main` + * `service-release-candidate` will run and create a remote branch named `candidate/ Date: Fri, 23 Jun 2023 15:12:26 -0400 Subject: [PATCH 42/56] wip on secrets, and just use yaml lists where appropriate --- ci/invoke.yaml | 30 +++- ci/poetry.lock | 427 +++++++++++++++++++++++++++++++++++++++++++++- ci/pyproject.toml | 1 + ci/tasks.py | 49 +++++- 4 files changed, 493 insertions(+), 14 deletions(-) diff --git a/ci/invoke.yaml b/ci/invoke.yaml index 060bd3e931..8ac0d3e92a 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -9,18 +9,23 @@ calitp: - name: archiver driver: kustomize kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test + secrets: + - gtfs-rt-v3-test_gtfs-feed-secrets - name: metabase driver: helm namespace: metabase-test helm_name: metabase-test helm_chart: kubernetes/apps/charts/metabase - helm_values: kubernetes/apps/values/metabase-test.yaml + helm_values: + - kubernetes/apps/values/metabase-test.yaml - name: postgresql-backup-metabase driver: helm namespace: metabase-test helm_name: postgresql-backup helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-test.yaml + helm_values: + - kubernetes/apps/values/postgresql-backup-metabase.yaml + - kubernetes/apps/values/postgresql-backup-metabase-test.yaml prod: releases: - name: airflow-jobs @@ -31,7 +36,8 @@ calitp: namespace: monitoring-grafana helm_name: grafana helm_chart: kubernetes/apps/charts/grafana - helm_values: kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml + helm_values: + - kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml - name: jupyterhub driver: helm namespace: jupyterhub @@ -47,37 +53,43 @@ calitp: namespace: metabase helm_name: metabase helm_chart: kubernetes/apps/charts/metabase - helm_values: kubernetes/apps/values/metabase.yaml + helm_values: + - kubernetes/apps/values/metabase.yaml - name: postgresql-backup-grafana driver: helm namespace: grafana helm_name: postgresql-backup helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: kubernetes/apps/values/postgresql-backup-grafana.yaml + helm_values: + - kubernetes/apps/values/postgresql-backup-grafana.yaml - name: postgresql-backup-metabase driver: helm namespace: metabase helm_name: postgresql-backup helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml + helm_values: + - kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml - name: postgresql-backup-sentry driver: helm namespace: sentry helm_name: postgresql-backup helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: kubernetes/apps/values/postgresql-backup-sentry.yaml + helm_values: + - kubernetes/apps/values/postgresql-backup-sentry.yaml - name: prometheus driver: helm namespace: monitoring-prometheus helm_name: prometheus helm_chart: kubernetes/apps/charts/prometheus - helm_values: kubernetes/apps/values/prometheus.yaml + helm_values: + - kubernetes/apps/values/prometheus.yaml - name: promtail driver: helm namespace: monitoring-loki helm_name: promtail helm_chart: kubernetes/apps/charts/promtail - helm_values: kubernetes/apps/values/promtail.yaml + helm_values: + - kubernetes/apps/values/promtail.yaml - name: sftp-ingest-elavon driver: kustomize kustomize_dir: kubernetes/apps/overlays/prod-sftp-ingest-elavon diff --git a/ci/poetry.lock b/ci/poetry.lock index fee4d5989c..437e3dcbba 100644 --- a/ci/poetry.lock +++ b/ci/poetry.lock @@ -1,5 +1,111 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +[[package]] +name = "cachetools" +version = "5.3.1" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, +] + +[[package]] +name = "certifi" +version = "2023.5.7" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] + [[package]] name = "cloudpickle" version = "2.2.1" @@ -57,6 +163,199 @@ files = [ [package.dependencies] gitdb = ">=4.0.1,<5" +[[package]] +name = "google-api-core" +version = "2.11.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, + {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +grpcio = [ + {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, +] +grpcio-status = [ + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.20.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-2.20.0.tar.gz", hash = "sha256:030af34138909ccde0fbce611afc178f1d65d32fbff281f25738b1fe1c6f3eaa"}, + {file = "google_auth-2.20.0-py2.py3-none-any.whl", hash = "sha256:23b7b0950fcda519bfb6692bf0d5289d2ea49fc143717cc7188458ec620e63fa"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" +six = ">=1.9.0" +urllib3 = "<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-secret-manager" +version = "2.16.1" +description = "Google Cloud Secret Manager API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-secret-manager-2.16.1.tar.gz", hash = "sha256:149d11ce9be7ea81d4ac3544d3fcd4c716a9edb2cb775d9c075231570b079fbb"}, + {file = "google_cloud_secret_manager-2.16.1-py2.py3-none-any.whl", hash = "sha256:dad28c24921fb62961aafe808be0e7935a99096f03ac29eeeefa04b85534c1f3"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" +proto-plus = [ + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, + {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "googleapis-common-protos" +version = "1.59.1" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, + {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, +] + +[package.dependencies] +grpcio = {version = ">=1.44.0,<2.0.0.dev0", optional = true, markers = "extra == \"grpc\""} +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "grpc-google-iam-v1" +version = "0.12.6" +description = "IAM API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpc-google-iam-v1-0.12.6.tar.gz", hash = "sha256:2bc4b8fdf22115a65d751c9317329322602c39b7c86a289c9b72d228d960ef5f"}, + {file = "grpc_google_iam_v1-0.12.6-py2.py3-none-any.whl", hash = "sha256:5c10f3d8dc2d88678ab1a9b0cb5482735c5efee71e6c0cd59f872eef22913f5c"}, +] + +[package.dependencies] +googleapis-common-protos = {version = ">=1.56.0,<2.0.0dev", extras = ["grpc"]} +grpcio = ">=1.44.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "grpcio" +version = "1.56.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fb34ace11419f1ae321c36ccaa18d81cd3f20728cd191250be42949d6845bb2d"}, + {file = "grpcio-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:008767c0aed4899e657b50f2e0beacbabccab51359eba547f860e7c55f2be6ba"}, + {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:17f47aeb9be0da5337f9ff33ebb8795899021e6c0741ee68bd69774a7804ca86"}, + {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43c50d810cc26349b093bf2cfe86756ab3e9aba3e7e681d360930c1268e1399a"}, + {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:187b8f71bad7d41eea15e0c9812aaa2b87adfb343895fffb704fb040ca731863"}, + {file = "grpcio-1.56.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:881575f240eb5db72ddca4dc5602898c29bc082e0d94599bf20588fb7d1ee6a0"}, + {file = "grpcio-1.56.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c243b158dd7585021d16c50498c4b2ec0a64a6119967440c5ff2d8c89e72330e"}, + {file = "grpcio-1.56.0-cp310-cp310-win32.whl", hash = "sha256:8b3b2c7b5feef90bc9a5fa1c7f97637e55ec3e76460c6d16c3013952ee479cd9"}, + {file = "grpcio-1.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:03a80451530fd3b8b155e0c4480434f6be669daf7ecba56f73ef98f94222ee01"}, + {file = "grpcio-1.56.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:64bd3abcf9fb4a9fa4ede8d0d34686314a7075f62a1502217b227991d9ca4245"}, + {file = "grpcio-1.56.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:fdc3a895791af4addbb826808d4c9c35917c59bb5c430d729f44224e51c92d61"}, + {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:4f84a6fd4482e5fe73b297d4874b62a535bc75dc6aec8e9fe0dc88106cd40397"}, + {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14e70b4dda3183abea94c72d41d5930c333b21f8561c1904a372d80370592ef3"}, + {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b5ce42a5ebe3e04796246ba50357f1813c44a6efe17a37f8dc7a5c470377312"}, + {file = "grpcio-1.56.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8219f17baf069fe8e42bd8ca0b312b875595e43a70cabf397be4fda488e2f27d"}, + {file = "grpcio-1.56.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:defdd14b518e6e468466f799aaa69db0355bca8d3a5ea75fb912d28ba6f8af31"}, + {file = "grpcio-1.56.0-cp311-cp311-win32.whl", hash = "sha256:50f4daa698835accbbcc60e61e0bc29636c0156ddcafb3891c987e533a0031ba"}, + {file = "grpcio-1.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:59c4e606993a47146fbeaf304b9e78c447f5b9ee5641cae013028c4cca784617"}, + {file = "grpcio-1.56.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b1f4b6f25a87d80b28dd6d02e87d63fe1577fe6d04a60a17454e3f8077a38279"}, + {file = "grpcio-1.56.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:c2148170e01d464d41011a878088444c13413264418b557f0bdcd1bf1b674a0e"}, + {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:0409de787ebbf08c9d2bca2bcc7762c1efe72eada164af78b50567a8dfc7253c"}, + {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66f0369d27f4c105cd21059d635860bb2ea81bd593061c45fb64875103f40e4a"}, + {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38fdf5bd0a1c754ce6bf9311a3c2c7ebe56e88b8763593316b69e0e9a56af1de"}, + {file = "grpcio-1.56.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:79d4c5911d12a7aa671e5eb40cbb50a830396525014d2d6f254ea2ba180ce637"}, + {file = "grpcio-1.56.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5d2fc471668a7222e213f86ef76933b18cdda6a51ea1322034478df8c6519959"}, + {file = "grpcio-1.56.0-cp37-cp37m-win_amd64.whl", hash = "sha256:991224fd485e088d3cb5e34366053691a4848a6b7112b8f5625a411305c26691"}, + {file = "grpcio-1.56.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c6f36621aabecbaff3e70c4d1d924c76c8e6a7ffec60c331893640a4af0a8037"}, + {file = "grpcio-1.56.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:1eadd6de258901929223f422ffed7f8b310c0323324caf59227f9899ea1b1674"}, + {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:72836b5a1d4f508ffbcfe35033d027859cc737972f9dddbe33fb75d687421e2e"}, + {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f92a99ab0c7772fb6859bf2e4f44ad30088d18f7c67b83205297bfb229e0d2cf"}, + {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa08affbf672d051cd3da62303901aeb7042a2c188c03b2c2a2d346fc5e81c14"}, + {file = "grpcio-1.56.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2db108b4c8e29c145e95b0226973a66d73ae3e3e7fae00329294af4e27f1c42"}, + {file = "grpcio-1.56.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8674fdbd28266d8efbcddacf4ec3643f76fe6376f73283fd63a8374c14b0ef7c"}, + {file = "grpcio-1.56.0-cp38-cp38-win32.whl", hash = "sha256:bd55f743e654fb050c665968d7ec2c33f03578a4bbb163cfce38024775ff54cc"}, + {file = "grpcio-1.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:c63bc5ac6c7e646c296fed9139097ae0f0e63f36f0864d7ce431cce61fe0118a"}, + {file = "grpcio-1.56.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c0bc9dda550785d23f4f025be614b7faa8d0293e10811f0f8536cf50435b7a30"}, + {file = "grpcio-1.56.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:d596408bab632ec7b947761e83ce6b3e7632e26b76d64c239ba66b554b7ee286"}, + {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76b6e6e1ee9bda32e6e933efd61c512e9a9f377d7c580977f090d1a9c78cca44"}, + {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7beb84ebd0a3f732625124b73969d12b7350c5d9d64ddf81ae739bbc63d5b1ed"}, + {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83ec714bbbe9b9502177c842417fde39f7a267031e01fa3cd83f1ca49688f537"}, + {file = "grpcio-1.56.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4feee75565d1b5ab09cb3a5da672b84ca7f6dd80ee07a50f5537207a9af543a4"}, + {file = "grpcio-1.56.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b4638a796778329cc8e142e4f57c705adb286b3ba64e00b0fa91eeb919611be8"}, + {file = "grpcio-1.56.0-cp39-cp39-win32.whl", hash = "sha256:437af5a7673bca89c4bc0a993382200592d104dd7bf55eddcd141cef91f40bab"}, + {file = "grpcio-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:4241a1c2c76e748023c834995cd916570e7180ee478969c2d79a60ce007bc837"}, + {file = "grpcio-1.56.0.tar.gz", hash = "sha256:4c08ee21b3d10315b8dc26f6c13917b20ed574cdbed2d2d80c53d5508fdcc0f2"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.56.0)"] + +[[package]] +name = "grpcio-status" +version = "1.56.0" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.6" +files = [ + {file = "grpcio-status-1.56.0.tar.gz", hash = "sha256:9eca0b2dcda0782d3702df225918efd6d820f75f93cd5c51c7fb6a4ffbfea12c"}, + {file = "grpcio_status-1.56.0-py3-none-any.whl", hash = "sha256:e5f101c96686e9d4e94a114567960fdb00052aa3c818b029745e3db37dc9c613"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.56.0" +protobuf = ">=4.21.6" + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + [[package]] name = "importlib-metadata" version = "6.7.0" @@ -144,6 +443,70 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "proto-plus" +version = "1.22.3" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "proto-plus-1.22.3.tar.gz", hash = "sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b"}, + {file = "proto_plus-1.22.3-py3-none-any.whl", hash = "sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "4.23.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "protobuf-4.23.3-cp310-abi3-win32.whl", hash = "sha256:514b6bbd54a41ca50c86dd5ad6488afe9505901b3557c5e0f7823a0cf67106fb"}, + {file = "protobuf-4.23.3-cp310-abi3-win_amd64.whl", hash = "sha256:cc14358a8742c4e06b1bfe4be1afbdf5c9f6bd094dff3e14edb78a1513893ff5"}, + {file = "protobuf-4.23.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2991f5e7690dab569f8f81702e6700e7364cc3b5e572725098215d3da5ccc6ac"}, + {file = "protobuf-4.23.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:08fe19d267608d438aa37019236db02b306e33f6b9902c3163838b8e75970223"}, + {file = "protobuf-4.23.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3b01a5274ac920feb75d0b372d901524f7e3ad39c63b1a2d55043f3887afe0c1"}, + {file = "protobuf-4.23.3-cp37-cp37m-win32.whl", hash = "sha256:aca6e86a08c5c5962f55eac9b5bd6fce6ed98645d77e8bfc2b952ecd4a8e4f6a"}, + {file = "protobuf-4.23.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0149053336a466e3e0b040e54d0b615fc71de86da66791c592cc3c8d18150bf8"}, + {file = "protobuf-4.23.3-cp38-cp38-win32.whl", hash = "sha256:84ea0bd90c2fdd70ddd9f3d3fc0197cc24ecec1345856c2b5ba70e4d99815359"}, + {file = "protobuf-4.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:3bcbeb2bf4bb61fe960dd6e005801a23a43578200ea8ceb726d1f6bd0e562ba1"}, + {file = "protobuf-4.23.3-cp39-cp39-win32.whl", hash = "sha256:5cb9e41188737f321f4fce9a4337bf40a5414b8d03227e1d9fbc59bc3a216e35"}, + {file = "protobuf-4.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:29660574cd769f2324a57fb78127cda59327eb6664381ecfe1c69731b83e8288"}, + {file = "protobuf-4.23.3-py3-none-any.whl", hash = "sha256:447b9786ac8e50ae72cae7a2eec5c5df6a9dbf9aa6f908f1b8bda6032644ea62"}, + {file = "protobuf-4.23.3.tar.gz", hash = "sha256:7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + [[package]] name = "pydantic" version = "1.10.9" @@ -260,6 +623,52 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "smmap" version = "5.0.0" @@ -293,6 +702,22 @@ files = [ {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, ] +[[package]] +name = "urllib3" +version = "1.26.16" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "zipp" version = "3.15.0" @@ -311,4 +736,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "a862514a7275573803be6574106d0034ba52c0b638e38f0dce335e0998f3bb50" +content-hash = "9efa10092523133041f627d77b0293ae04109563f4e86a998e8e712ce4103d29" diff --git a/ci/pyproject.toml b/ci/pyproject.toml index 39ea9b4d36..a7f3199aeb 100644 --- a/ci/pyproject.toml +++ b/ci/pyproject.toml @@ -12,6 +12,7 @@ gitpython = "^3.1.31" invoke = "^2.1.3" doit = "^0.36.0" pyyaml = "^6.0" +google-cloud-secret-manager = "^2.16.1" [tool.poetry.group.dev.dependencies] diff --git a/ci/tasks.py b/ci/tasks.py index 9cd33bf308..3e0e322fea 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -4,6 +4,7 @@ from typing import Dict, List, Optional import git +from google.cloud import secretmanager from invoke import Result, task from pydantic import BaseModel, validator @@ -25,6 +26,10 @@ class Release(BaseModel): name: str driver: ReleaseDriver + # we could consider labeling secrets and pulling all secrets by label + # but that moves some logic into Secret Manager itself + secrets: Optional[List[str]] + # for helm namespace: Optional[str] helm_name: Optional[str] @@ -34,10 +39,6 @@ class Release(BaseModel): # for kustomize kustomize_dir: Optional[Path] - @validator("helm_values", pre=True) - def split_helm_values(cls, v): - return v.split(":") - class Channel(BaseModel): releases: List[Release] @@ -89,6 +90,46 @@ def get_releases( } +@task( + parse_calitp_config, + help={ + "channel": GENERIC_HELP["channel"], + "app": GENERIC_HELP["app"], + "secret": "Optionally, specify a single secret to deploy", + }, +) +def deploy_secret( + c, + channel: str, + app=None, + secret=None, +): + """ + Deploy secret(s) by channel, and optionally app or secret name. + """ + client = secretmanager.SecretManagerServiceClient() + found_secret = False + for release in get_releases(c, channel, app=app): + for release_secret in release.secrets: + if not secret or secret == release_secret: + with tempfile.TemporaryDirectory() as tmpdir: + secret_path = Path(tmpdir) / Path(f"{release_secret}.yml") + # this ID maps to cal-itp-data-infra; there's probably a better way to do this + name = f"projects/1005246706141/secrets/{release_secret}/versions/latest" + secret_contents = client.access_secret_version( + request={"name": name} + ).payload.data.decode("UTF-8") + with open(secret_path, "w") as f: + f.write(secret_contents) + c.run( + f"kubectl apply --namespace {release.namespace} -f {secret_path}" + ) + found_secret = True + + if not found_secret: + print("Failed to deploy any secrets.") + + @task( parse_calitp_config, help={ From 23d94858504ace352d161e5ded7ea432887ad17a Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 15:19:38 -0400 Subject: [PATCH 43/56] couple cleanups from review --- .github/workflows/service-release-channel.yml | 6 +----- .github/workflows/service-release-diff.yml | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml index 21c77fb364..a074191b4b 100644 --- a/.github/workflows/service-release-channel.yml +++ b/.github/workflows/service-release-channel.yml @@ -37,11 +37,7 @@ jobs: shell: bash working-directory: ci run: | - # GITHUB_JOB is not populated until the job is running - git config user.name "Github Action $GITHUB_JOB" - git config user.email "$(whoami)@$(uname -n)" export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} - printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" - set -- "$GITHUB_WORKSPACE/ci/vars/project.env" + printf 'WORKFLOW: service-release-channel; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install poetry run invoke release $RELEASE_CHANNEL diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index e01c9e8521..07213991c1 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -43,12 +43,8 @@ jobs: shell: bash working-directory: ci run: | - # GITHUB_JOB is not populated until the job is running - git config user.name "Github Action $GITHUB_JOB" - git config user.email "$(whoami)@$(uname -n)" export RELEASE_CHANNEL=${GITHUB_BASE_REF#releases/} - printf 'WORKFLOW: service-release; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" - set -- "$GITHUB_WORKSPACE/ci/vars/project.env" + printf 'WORKFLOW: service-release-diff; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install poetry run invoke diff $RELEASE_CHANNEL --outfile=diff.txt From a25381412cd60184851629bd6bed60af80eb14b1 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 15:41:14 -0400 Subject: [PATCH 44/56] finish handling secrets, do in test --- .github/workflows/service-release-channel.yml | 1 + ci/invoke.yaml | 3 +++ ci/tasks.py | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml index a074191b4b..cbf519eca6 100644 --- a/.github/workflows/service-release-channel.yml +++ b/.github/workflows/service-release-channel.yml @@ -40,4 +40,5 @@ jobs: export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} printf 'WORKFLOW: service-release-channel; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install + poetry run invoke secrets $RELEASE_CHANNEL poetry run invoke release $RELEASE_CHANNEL diff --git a/ci/invoke.yaml b/ci/invoke.yaml index 8ac0d3e92a..4331b24229 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -26,6 +26,9 @@ calitp: helm_values: - kubernetes/apps/values/postgresql-backup-metabase.yaml - kubernetes/apps/values/postgresql-backup-metabase-test.yaml + secrets: + - metabase-test_database-backup + - metabase-test_gcs-upload-svcacct prod: releases: - name: airflow-jobs diff --git a/ci/tasks.py b/ci/tasks.py index 3e0e322fea..55cfc18a07 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -4,6 +4,7 @@ from typing import Dict, List, Optional import git +import yaml from google.cloud import secretmanager from invoke import Result, task from pydantic import BaseModel, validator @@ -28,7 +29,7 @@ class Release(BaseModel): # we could consider labeling secrets and pulling all secrets by label # but that moves some logic into Secret Manager itself - secrets: Optional[List[str]] + secrets: List[str] = [] # for helm namespace: Optional[str] @@ -98,7 +99,7 @@ def get_releases( "secret": "Optionally, specify a single secret to deploy", }, ) -def deploy_secret( +def secrets( c, channel: str, app=None, @@ -119,11 +120,18 @@ def deploy_secret( secret_contents = client.access_secret_version( request={"name": name} ).payload.data.decode("UTF-8") + + if release.namespace: + ns_str = f"--namespace {release.namespace}" + else: + ns_str = "" + assert ( + "namespace" in yaml.safe_load(secret_contents)["metadata"] + ) + with open(secret_path, "w") as f: f.write(secret_contents) - c.run( - f"kubectl apply --namespace {release.namespace} -f {secret_path}" - ) + c.run(f"kubectl apply {ns_str} -f {secret_path}") found_secret = True if not found_secret: From 14d2fb3150e66ebee3f6872e44534786f200fd7d Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:00:25 -0400 Subject: [PATCH 45/56] split channels into separate config files --- ci/channels/prod.yaml | 68 +++++++++++++++++++++++++++++++ ci/channels/test.yaml | 26 ++++++++++++ ci/invoke.yaml | 93 ------------------------------------------- ci/tasks.py | 40 ++++++++----------- 4 files changed, 111 insertions(+), 116 deletions(-) create mode 100644 ci/channels/prod.yaml create mode 100644 ci/channels/test.yaml diff --git a/ci/channels/prod.yaml b/ci/channels/prod.yaml new file mode 100644 index 0000000000..af29f5d9f8 --- /dev/null +++ b/ci/channels/prod.yaml @@ -0,0 +1,68 @@ +calitp: + channel: prod + releases: + - name: airflow-jobs + driver: kustomize + kustomize_dir: kubernetes/apps/manifests/airflow-jobs + - name: grafana + driver: helm + namespace: monitoring-grafana + helm_name: grafana + helm_chart: kubernetes/apps/charts/grafana + helm_values: + - kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml + - name: jupyterhub + driver: helm + namespace: jupyterhub + helm_name: jupyterhub + helm_chart: kubernetes/apps/charts/jupyterhub + - name: loki + driver: helm + namespace: monitoring-loki + helm_name: loki + helm_chart: kubernetes/apps/charts/loki + - name: metabase + driver: helm + namespace: metabase + helm_name: metabase + helm_chart: kubernetes/apps/charts/metabase + helm_values: + - kubernetes/apps/values/metabase.yaml + - name: postgresql-backup-grafana + driver: helm + namespace: grafana + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: + - kubernetes/apps/values/postgresql-backup-grafana.yaml + - name: postgresql-backup-metabase + driver: helm + namespace: metabase + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: + - kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml + - name: postgresql-backup-sentry + driver: helm + namespace: sentry + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: + - kubernetes/apps/values/postgresql-backup-sentry.yaml + - name: prometheus + driver: helm + namespace: monitoring-prometheus + helm_name: prometheus + helm_chart: kubernetes/apps/charts/prometheus + helm_values: + - kubernetes/apps/values/prometheus.yaml + - name: promtail + driver: helm + namespace: monitoring-loki + helm_name: promtail + helm_chart: kubernetes/apps/charts/promtail + helm_values: + - kubernetes/apps/values/promtail.yaml + - name: sftp-ingest-elavon + driver: kustomize + kustomize_dir: kubernetes/apps/overlays/prod-sftp-ingest-elavon diff --git a/ci/channels/test.yaml b/ci/channels/test.yaml new file mode 100644 index 0000000000..0170499e62 --- /dev/null +++ b/ci/channels/test.yaml @@ -0,0 +1,26 @@ +calitp: + channel: test + releases: + - name: archiver + driver: kustomize + kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test + secrets: + - gtfs-rt-v3-test_gtfs-feed-secrets + - name: metabase + driver: helm + namespace: metabase-test + helm_name: metabase-test + helm_chart: kubernetes/apps/charts/metabase + helm_values: + - kubernetes/apps/values/metabase-test.yaml + - name: postgresql-backup-metabase + driver: helm + namespace: metabase-test + helm_name: postgresql-backup + helm_chart: kubernetes/apps/charts/postgresql-backup + helm_values: + - kubernetes/apps/values/postgresql-backup-metabase.yaml + - kubernetes/apps/values/postgresql-backup-metabase-test.yaml + secrets: + - metabase-test_database-backup + - metabase-test_gcs-upload-svcacct diff --git a/ci/invoke.yaml b/ci/invoke.yaml index 4331b24229..86357b3e7c 100644 --- a/ci/invoke.yaml +++ b/ci/invoke.yaml @@ -3,96 +3,3 @@ run: echo_format: "Executing: {command}" # without this, the echoed text is white which does not show on light themes calitp: git_repo: . - channels: - test: - releases: - - name: archiver - driver: kustomize - kustomize_dir: kubernetes/apps/overlays/gtfs-rt-archiver-v3-test - secrets: - - gtfs-rt-v3-test_gtfs-feed-secrets - - name: metabase - driver: helm - namespace: metabase-test - helm_name: metabase-test - helm_chart: kubernetes/apps/charts/metabase - helm_values: - - kubernetes/apps/values/metabase-test.yaml - - name: postgresql-backup-metabase - driver: helm - namespace: metabase-test - helm_name: postgresql-backup - helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: - - kubernetes/apps/values/postgresql-backup-metabase.yaml - - kubernetes/apps/values/postgresql-backup-metabase-test.yaml - secrets: - - metabase-test_database-backup - - metabase-test_gcs-upload-svcacct - prod: - releases: - - name: airflow-jobs - driver: kustomize - kustomize_dir: kubernetes/apps/manifests/airflow-jobs - - name: grafana - driver: helm - namespace: monitoring-grafana - helm_name: grafana - helm_chart: kubernetes/apps/charts/grafana - helm_values: - - kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml - - name: jupyterhub - driver: helm - namespace: jupyterhub - helm_name: jupyterhub - helm_chart: kubernetes/apps/charts/jupyterhub - - name: loki - driver: helm - namespace: monitoring-loki - helm_name: loki - helm_chart: kubernetes/apps/charts/loki - - name: metabase - driver: helm - namespace: metabase - helm_name: metabase - helm_chart: kubernetes/apps/charts/metabase - helm_values: - - kubernetes/apps/values/metabase.yaml - - name: postgresql-backup-grafana - driver: helm - namespace: grafana - helm_name: postgresql-backup - helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: - - kubernetes/apps/values/postgresql-backup-grafana.yaml - - name: postgresql-backup-metabase - driver: helm - namespace: metabase - helm_name: postgresql-backup - helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: - - kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml - - name: postgresql-backup-sentry - driver: helm - namespace: sentry - helm_name: postgresql-backup - helm_chart: kubernetes/apps/charts/postgresql-backup - helm_values: - - kubernetes/apps/values/postgresql-backup-sentry.yaml - - name: prometheus - driver: helm - namespace: monitoring-prometheus - helm_name: prometheus - helm_chart: kubernetes/apps/charts/prometheus - helm_values: - - kubernetes/apps/values/prometheus.yaml - - name: promtail - driver: helm - namespace: monitoring-loki - helm_name: promtail - helm_chart: kubernetes/apps/charts/promtail - helm_values: - - kubernetes/apps/values/promtail.yaml - - name: sftp-ingest-elavon - driver: kustomize - kustomize_dir: kubernetes/apps/overlays/prod-sftp-ingest-elavon diff --git a/ci/tasks.py b/ci/tasks.py index 55cfc18a07..1a05ebf4ec 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -1,7 +1,7 @@ import tempfile from enum import Enum from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Optional import git import yaml @@ -18,6 +18,12 @@ """ +GENERIC_HELP = { + "driver": "The k8s driver (kustomize or helm)", + "app": "The specific app/release (e.g. metabase or archiver)", +} + + class ReleaseDriver(str, Enum): helm = "helm" kustomize = "kustomize" @@ -41,14 +47,11 @@ class Release(BaseModel): kustomize_dir: Optional[Path] -class Channel(BaseModel): - releases: List[Release] - - # TODO: rename this class Config(BaseModel): git_repo: git.Repo - channels: Dict[str, Channel] + channel: str # this is a bit weird, but I want to be able to log this value + releases: List[Release] class Config: arbitrary_types_allowed = True @@ -73,10 +76,12 @@ def parse_calitp_config(c): def get_releases( - c, channel, driver: Optional[ReleaseDriver] = None, app=None + c, + driver: Optional[ReleaseDriver] = None, + app=None, ) -> List[Release]: ret = [] - for release in c.calitp_config.channels[channel].releases: + for release in c.calitp_config.releases: if (not driver or release.driver == driver) and ( not app or app == release.name ): @@ -84,24 +89,15 @@ def get_releases( return ret -GENERIC_HELP = { - "channel": "The release channel/environment, e.g. test or prod", - "driver": "The k8s driver (kustomize or helm)", - "app": "The specific app/release (e.g. metabase or archiver)", -} - - @task( parse_calitp_config, help={ - "channel": GENERIC_HELP["channel"], "app": GENERIC_HELP["app"], "secret": "Optionally, specify a single secret to deploy", }, ) def secrets( c, - channel: str, app=None, secret=None, ): @@ -110,7 +106,7 @@ def secrets( """ client = secretmanager.SecretManagerServiceClient() found_secret = False - for release in get_releases(c, channel, app=app): + for release in get_releases(c, app=app): for release_secret in release.secrets: if not secret or secret == release_secret: with tempfile.TemporaryDirectory() as tmpdir: @@ -147,7 +143,6 @@ def secrets( ) def diff( c, - channel: str, driver=None, app=None, outfile=None, @@ -160,7 +155,7 @@ def diff( full_diff = "" result: Result - for release in get_releases(c, channel, driver=actual_driver, app=app): + for release in get_releases(c, driver=actual_driver, app=app): if release.driver == ReleaseDriver.kustomize: assert release.kustomize_dir is not None kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) @@ -195,7 +190,7 @@ def diff( msg = ( f"```{full_diff}```" if full_diff - else f"No {driver if driver else 'manifest'} changes found for {channel}.\n" + else f"No {driver if driver else 'manifest'} changes found for {c.calitp_config.channel}.\n" ) if outfile: print(f"writing {len(msg)=} to {outfile}", flush=True) @@ -209,7 +204,6 @@ def diff( @task(parse_calitp_config, help=GENERIC_HELP) def release( c, - channel: str, driver=None, app=None, ): @@ -219,7 +213,7 @@ def release( assert c.calitp_config.git_root is not None actual_driver = ReleaseDriver[driver] if driver else None - for release in get_releases(c, channel, driver=actual_driver, app=app): + for release in get_releases(c, driver=actual_driver, app=app): if release.driver == ReleaseDriver.kustomize: assert release.kustomize_dir is not None kustomize_dir = c.calitp_config.git_root / Path(release.kustomize_dir) From 7b8027c43cd620810f8bf2b55be9e7854165f814 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:01:11 -0400 Subject: [PATCH 46/56] add pyyaml types --- ci/poetry.lock | 13 ++++++++++++- ci/pyproject.toml | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/poetry.lock b/ci/poetry.lock index 437e3dcbba..4de51a1488 100644 --- a/ci/poetry.lock +++ b/ci/poetry.lock @@ -691,6 +691,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.10" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.10.tar.gz", hash = "sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97"}, + {file = "types_PyYAML-6.0.12.10-py3-none-any.whl", hash = "sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f"}, +] + [[package]] name = "typing-extensions" version = "4.6.3" @@ -736,4 +747,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "9efa10092523133041f627d77b0293ae04109563f4e86a998e8e712ce4103d29" +content-hash = "fffc6395c0ede1cbc19cd6eabacb7d4bc99056ef584e0be5630749fd03b6670a" diff --git a/ci/pyproject.toml b/ci/pyproject.toml index a7f3199aeb..ff3e0602ec 100644 --- a/ci/pyproject.toml +++ b/ci/pyproject.toml @@ -17,6 +17,7 @@ google-cloud-secret-manager = "^2.16.1" [tool.poetry.group.dev.dependencies] mypy = "^1.4.0" +types-pyyaml = "^6.0.12.10" [build-system] requires = ["poetry-core"] From e07374c10fd5690b505b1de58cedf0885e70b5b4 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:12:05 -0400 Subject: [PATCH 47/56] define prod secrets --- ci/channels/prod.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ci/channels/prod.yaml b/ci/channels/prod.yaml index af29f5d9f8..c643e130d1 100644 --- a/ci/channels/prod.yaml +++ b/ci/channels/prod.yaml @@ -4,6 +4,8 @@ calitp: - name: airflow-jobs driver: kustomize kustomize_dir: kubernetes/apps/manifests/airflow-jobs + secrets: + - airflow-jobs_jobs-data - name: grafana driver: helm namespace: monitoring-grafana @@ -11,11 +13,16 @@ calitp: helm_chart: kubernetes/apps/charts/grafana helm_values: - kubernetes/apps/values/grafana.yaml:kubernetes/apps/values/grafana-prod.yaml + secrets: + - monitoring-grafana__grafana-initial-admin - name: jupyterhub driver: helm namespace: jupyterhub helm_name: jupyterhub helm_chart: kubernetes/apps/charts/jupyterhub + secrets: + - jupyterhub_jupyterhub-gcloud-service-key + - jupyterhub_jupyterhub-github-config - name: loki driver: helm namespace: monitoring-loki @@ -28,6 +35,8 @@ calitp: helm_chart: kubernetes/apps/charts/metabase helm_values: - kubernetes/apps/values/metabase.yaml + secrets: + - metabase_service_account_key - name: postgresql-backup-grafana driver: helm namespace: grafana @@ -35,6 +44,9 @@ calitp: helm_chart: kubernetes/apps/charts/postgresql-backup helm_values: - kubernetes/apps/values/postgresql-backup-grafana.yaml + secrets: + - monitoring-grafana_database-backup + - monitoring-grafana_grafana-postgresql - name: postgresql-backup-metabase driver: helm namespace: metabase @@ -42,6 +54,9 @@ calitp: helm_chart: kubernetes/apps/charts/postgresql-backup helm_values: - kubernetes/apps/values/postgresql-backup-metabase.yaml:kubernetes/apps/values/postgresql-backup-metabase-prod.yaml + secrets: + - metabase_database-backup + - metabase_gcs-upload-svcacct - name: postgresql-backup-sentry driver: helm namespace: sentry @@ -49,6 +64,8 @@ calitp: helm_chart: kubernetes/apps/charts/postgresql-backup helm_values: - kubernetes/apps/values/postgresql-backup-sentry.yaml + secrets: + - sentry_database-backup - name: prometheus driver: helm namespace: monitoring-prometheus @@ -66,3 +83,9 @@ calitp: - name: sftp-ingest-elavon driver: kustomize kustomize_dir: kubernetes/apps/overlays/prod-sftp-ingest-elavon + # TODO: sftp secrets + # TODO: Sentry +# - name: sentry +# secrets: +# - sentry_sentry-secret +# - sentry_sentry-sentry-postgresql From a6e93daf2180bc1cce940cb26ca9e732a2b35348 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:13:57 -0400 Subject: [PATCH 48/56] make docs build --- docs/infrastructure/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index 1b324f7a10..b736c80855 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -17,7 +17,7 @@ All CI/CD automation in this project is executed via GitHub Actions, whose workf While we're using GCP Composer, "deployment" of Airflow consists of two parts: 1. Calling `gcloud composer environments update ...` to update the Composer environment with new (or specific versions of) packages -2. Copying the [dags](../../airflow/dags) and [plugins](../../airflow/plugins) folders to a GCS bucket that Composer reads +2. Copying the `dags` and `plugins` folders to a GCS bucket that Composer reads ## build-*.yml workflows From bc7c44cd82c9ec3a99bb3887de35bb1081c63398 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:14:57 -0400 Subject: [PATCH 49/56] only run 1 test consumer to test diff/deploy --- .../apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml index 93a3eaa99e..a11ad89357 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-test/consumer.patch.yaml @@ -4,7 +4,7 @@ kind: Deployment metadata: name: gtfs-rt-archiver-consumer spec: - replicas: 2 + replicas: 1 template: spec: containers: From 6936bd8a18315aed25ffb64c3a8c5241ff424113 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:25:35 -0400 Subject: [PATCH 50/56] add quotes --- .github/workflows/service-release-channel.yml | 4 ++-- .github/workflows/service-release-diff.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml index cbf519eca6..4d50390de7 100644 --- a/.github/workflows/service-release-channel.yml +++ b/.github/workflows/service-release-channel.yml @@ -40,5 +40,5 @@ jobs: export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} printf 'WORKFLOW: service-release-channel; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install - poetry run invoke secrets $RELEASE_CHANNEL - poetry run invoke release $RELEASE_CHANNEL + poetry run invoke secrets -f "./channels/$RELEASE_CHANNEL.yaml" + poetry run invoke release -f "./channels/$RELEASE_CHANNEL.yaml" diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/service-release-diff.yml index 07213991c1..b9dcdd7f1a 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/service-release-diff.yml @@ -46,7 +46,7 @@ jobs: export RELEASE_CHANNEL=${GITHUB_BASE_REF#releases/} printf 'WORKFLOW: service-release-diff; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install - poetry run invoke diff $RELEASE_CHANNEL --outfile=diff.txt + poetry run invoke diff -f "./channels/$RELEASE_CHANNEL.yaml" --outfile=diff.txt - uses: peter-evans/find-comment@v2 id: fc From 020fbb7af5933520c3b2fa510f1a4b62ba1b46b8 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:31:52 -0400 Subject: [PATCH 51/56] add a todo --- ci/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/tasks.py b/ci/tasks.py index 1a05ebf4ec..5a4b048add 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -174,6 +174,7 @@ def diff( for values_file in release.helm_values ] ) + # TODO: consider looking into https://github.com/databus23/helm-diff c.run( f"helm template {release.helm_name} {chart_path} --namespace {release.namespace} {values_str} > {manifest_path}" ) From 0de07eb83e3c908c9e71aaa81ae2710ce7f09ade Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 23 Jun 2023 16:39:32 -0400 Subject: [PATCH 52/56] another todo --- ci/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/tasks.py b/ci/tasks.py index 5a4b048add..84d5209e10 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -89,6 +89,8 @@ def get_releases( return ret +# TODO: kubectl diff now supports masking secrets, so we shoulds be able to render secret diffs in PRs +# see https://github.com/kubernetes/kubernetes/pull/96084 @task( parse_calitp_config, help={ From c9e2019d785a40c793983855bef5a583ba3e59b2 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Mon, 26 Jun 2023 10:08:19 -0400 Subject: [PATCH 53/56] add todo --- ci/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/tasks.py b/ci/tasks.py index 84d5209e10..c64c48d774 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -114,6 +114,7 @@ def secrets( with tempfile.TemporaryDirectory() as tmpdir: secret_path = Path(tmpdir) / Path(f"{release_secret}.yml") # this ID maps to cal-itp-data-infra; there's probably a better way to do this + # TODO: we could probably just use gcloud CLI for this name = f"projects/1005246706141/secrets/{release_secret}/versions/latest" secret_contents = client.access_secret_version( request={"name": name} From 525a13925a4b58d183a8aca5b6f467f40794da63 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Mon, 26 Jun 2023 10:38:23 -0400 Subject: [PATCH 54/56] ensure secrets/exceptions are not echoed --- ci/tasks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/tasks.py b/ci/tasks.py index c64c48d774..9d27ef5a3a 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -130,7 +130,12 @@ def secrets( with open(secret_path, "w") as f: f.write(secret_contents) - c.run(f"kubectl apply {ns_str} -f {secret_path}") + result = c.run( + f"kubectl apply {ns_str} -f {secret_path}", hide=True, warn=True + ) + if result.exited: + print("FAILURE: Failed to apply secret.", flush=True) + raise RuntimeError found_secret = True if not found_secret: From 5ec2ae30e0a661d9fb9058821f44fb8002b25432 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Mon, 26 Jun 2023 10:58:24 -0400 Subject: [PATCH 55/56] fix secret handling --- ci/tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index 9d27ef5a3a..7b730aa7b8 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -6,7 +6,7 @@ import git import yaml from google.cloud import secretmanager -from invoke import Result, task +from invoke import Exit, Result, task from pydantic import BaseModel, validator KUSTOMIZE_HELM_TEMPLATE = """ @@ -102,6 +102,7 @@ def secrets( c, app=None, secret=None, + hide=True, ): """ Deploy secret(s) by channel, and optionally app or secret name. @@ -130,16 +131,21 @@ def secrets( with open(secret_path, "w") as f: f.write(secret_contents) + print(f"Applying {release_secret}...", flush=True) result = c.run( - f"kubectl apply {ns_str} -f {secret_path}", hide=True, warn=True + f"kubectl apply {ns_str} -f {secret_path}", hide=hide, warn=True ) if result.exited: - print("FAILURE: Failed to apply secret.", flush=True) - raise RuntimeError + print( + f"FAILURE: Failed to apply secret {release_secret}.", + flush=True, + ) + raise Exit + print(f"Successfully applied {release_secret}.", flush=True) found_secret = True if not found_secret: - print("Failed to deploy any secrets.") + print("WARNING: Failed to deploy any secrets.", flush=True) @task( From 1ab46133155521e82bd0f6c4a0f5f2947b56bba9 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Mon, 26 Jun 2023 13:01:08 -0400 Subject: [PATCH 56/56] address PR comments --- ci/tasks.py | 8 +++----- docs/infrastructure/README.md | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index 7b730aa7b8..b29bab37dd 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -47,8 +47,7 @@ class Release(BaseModel): kustomize_dir: Optional[Path] -# TODO: rename this -class Config(BaseModel): +class CalitpConfig(BaseModel): git_repo: git.Repo channel: str # this is a bit weird, but I want to be able to log this value releases: List[Release] @@ -72,7 +71,7 @@ def parse_calitp_config(c): """ Parses the top-level calitp configuration key via Pydantic """ - c.update({"calitp_config": Config(**c.config._config["calitp"])}) + c.update({"calitp_config": CalitpConfig(**c.config._config["calitp"])}) def get_releases( @@ -89,7 +88,7 @@ def get_releases( return ret -# TODO: kubectl diff now supports masking secrets, so we shoulds be able to render secret diffs in PRs +# TODO: kubectl diff now supports masking secrets, so we should be able to render secret diffs in PRs # see https://github.com/kubernetes/kubernetes/pull/96084 @task( parse_calitp_config, @@ -215,7 +214,6 @@ def diff( print(msg, flush=True) -# TODO: we may want to split up channels into separate files so channel is not an argument but a config file @task(parse_calitp_config, help=GENERIC_HELP) def release( c, diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index b736c80855..f28d6d0668 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -10,14 +10,14 @@ Run `poetry run invoke -l` to list the available commands, and `poetry run invok ## CI/CD -All CI/CD automation in this project is executed via GitHub Actions, whose workflow files . +All CI/CD automation in this project is executed via GitHub Actions, whose workflow files live in the `.github` directory. ## deploy-airflow.yml While we're using GCP Composer, "deployment" of Airflow consists of two parts: 1. Calling `gcloud composer environments update ...` to update the Composer environment with new (or specific versions of) packages -2. Copying the `dags` and `plugins` folders to a GCS bucket that Composer reads +2. Copying the `dags` and `plugins` folders to a GCS bucket that Composer reads (this is specified in the Composer Environment) ## build-*.yml workflows @@ -27,7 +27,7 @@ Workflows prefixed with `build-` generally lint, test, and (usually) publish eit Workflows prefixed with `service-` deal with Kubernetes deployments. -* `service-release-candidate.yml` creates candidate branches, using hologit to bring in external Helm charts and remove irrelevant (i.e. non-infra) code +* `service-release-candidate.yml` creates candidate branches, using [hologit](https://github.com/JarvusInnovations/hologit) to bring in external Helm charts and remove irrelevant (i.e. non-infra) code * `service-release-diff.yml` renders kubectl diffs on PRs targeting release branches * `service-release-channel.yml` deploys to a given channel (i.e. environment) on updates to a release branch