From a410f191f85fe914c253083915e7edf413391a9f Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:27:40 -0700 Subject: [PATCH 1/8] Initial commit --- .github/workflows/aws-deploy.yaml | 70 ++++++ .gitignore | 129 +++++++++++ .pre-commit-config.yaml | 19 ++ .yamllint | 29 +++ LICENSE | 21 ++ Pipfile | 13 ++ Pipfile.lock | 356 ++++++++++++++++++++++++++++++ README.md | 49 ++++ config/config.yaml | 2 + config/develop/config.yaml | 0 config/develop/example-stack.yaml | 4 + config/prod/config.yaml | 0 config/prod/example-stack.yaml | 4 + templates/example-template.yaml | 21 ++ 14 files changed, 717 insertions(+) create mode 100644 .github/workflows/aws-deploy.yaml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint create mode 100644 LICENSE create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 README.md create mode 100644 config/config.yaml create mode 100644 config/develop/config.yaml create mode 100644 config/develop/example-stack.yaml create mode 100644 config/prod/config.yaml create mode 100644 config/prod/example-stack.yaml create mode 100644 templates/example-template.yaml diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml new file mode 100644 index 0000000000..b09cb55af6 --- /dev/null +++ b/.github/workflows/aws-deploy.yaml @@ -0,0 +1,70 @@ +name: aws-deploy + +on: push + +jobs: + pre-commit: + name: Run pre-commit hooks against all files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.2 + + sceptre-deploy: + name: Deploy CloudFormation templates using sceptre + runs-on: ubuntu-latest + needs: pre-commit + if: github.ref == 'refs/heads/main' + strategy: + matrix: + job-environment: + - develop + - prod + include: + - job-environment: develop + sceptre-environment: develop + - job-environment: prod + sceptre-environment: prod + environment: ${{ matrix.job-environment }} + steps: + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install pipenv + run: python -m pip install pipenv + + - name: Set up dependency caching + uses: actions/cache@v2 + with: + path: ~/.local/share/virtualenvs + key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }} + restore-keys: | + ${{ runner.os }}-pipenv-v2- + + - name: Install dependencies + if: steps.pipenv-cache.outputs.cache-hit != 'true' + env: + PIPENV_NOSPIN: 'true' + WORKON_HOME: ~/.local/share/virtualenvs + PIPENV_CACHE_DIR: ~/.local/share/pipcache + run: | + pipenv install --dev + + - name: Assume AWS role in dev account + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.CI_USER_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.CI_USER_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + role-to-assume: ${{ secrets.CI_ROLE_TO_ASSUME }} + role-duration-seconds: 1200 + + - name: Deploy sceptre stacks + run: pipenv run sceptre launch ${{ matrix.sceptre-environment }} --yes diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b6e47617de --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..51fe7fe4ba --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/adrienverge/yamllint + rev: v1.26.3 + hooks: + - id: yamllint + - repo: https://github.com/awslabs/cfn-python-lint + rev: v0.54.0 + hooks: + - id: cfn-python-lint + files: templates/.*\.(json|yml|yaml)$ + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.1.10 + hooks: + - id: remove-tabs diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000000..496d5615b5 --- /dev/null +++ b/.yamllint @@ -0,0 +1,29 @@ +--- + +extends: default + +rules: + braces: + level: warning + max-spaces-inside: 1 + brackets: + level: warning + max-spaces-inside: 1 + colons: + level: warning + commas: + level: warning + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: + level: warning + hyphens: + level: warning + indentation: + level: warning + indent-sequences: consistent + line-length: disable + truthy: disable + new-line-at-end-of-file: + level: warning diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..abda5ccb94 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 T. Thyer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000000..8ea2b0c886 --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[requires] +python_version = "3.9" + +[dev-packages] +pre-commit = "==2.*" +sceptre = "==2.*" + +[packages] diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000000..b56eb735ef --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,356 @@ +{ + "_meta": { + "hash": { + "sha256": "50898c784ec4d62ec4f1ea14af21a1b1157ed8c0a78642b202ade647bc6d5f66" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": { + "backports.entry-points-selectable": { + "hashes": [ + "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a", + "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc" + ], + "markers": "python_version >= '2.7'", + "version": "==1.1.0" + }, + "boto3": { + "hashes": [ + "sha256:63b9846c26e0905f4e9e39d6b59f152330c53a926d693439161c43dcf9779365", + "sha256:a9232185d8e7e2fd2b166c0ebee5d7b1f787fdb3093f33bbf5aa932c08f0ccac" + ], + "markers": "python_version >= '3.6'", + "version": "==1.18.42" + }, + "botocore": { + "hashes": [ + "sha256:0952d1200968365b440045efe8e45bbae38cf603fee12bcfc3d7b5f963cbfa18", + "sha256:6de4fec4ee10987e4dea96f289553c2f45109fcaafcb74a5baee1221926e1306" + ], + "markers": "python_version >= '3.6'", + "version": "==1.21.42" + }, + "certifi": { + "hashes": [ + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + ], + "version": "==2021.5.30" + }, + "cfgv": { + "hashes": [ + "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426", + "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==3.3.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:7098e7e862f6370a2a8d1a6398cd359815c45d12626267652c3f13dec58e2367", + "sha256:fa471a601dfea0f492e4f4fca035cd82155e65dc45c9b83bf4322dfab63755dd" + ], + "markers": "python_version >= '3'", + "version": "==2.0.5" + }, + "click": { + "hashes": [ + "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", + "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" + ], + "markers": "python_version >= '3.6'", + "version": "==8.0.1" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.4" + }, + "decorator": { + "hashes": [ + "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", + "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" + ], + "version": "==4.4.2" + }, + "distlib": { + "hashes": [ + "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736", + "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c" + ], + "version": "==0.3.2" + }, + "filelock": { + "hashes": [ + "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", + "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" + ], + "version": "==3.0.12" + }, + "identify": { + "hashes": [ + "sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d", + "sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf" + ], + "markers": "python_full_version >= '3.6.1'", + "version": "==2.2.14" + }, + "idna": { + "hashes": [ + "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", + "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" + ], + "markers": "python_version >= '3'", + "version": "==3.2" + }, + "jinja2": { + "hashes": [ + "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", + "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.11.3" + }, + "jmespath": { + "hashes": [ + "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", + "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==0.10.0" + }, + "markupsafe": { + "hashes": [ + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", + "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", + "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", + "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", + "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", + "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", + "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", + "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "networkx": { + "hashes": [ + "sha256:0635858ed7e989f4c574c2328380b452df892ae85084144c73d8cd819f0c4e06", + "sha256:109cd585cac41297f71103c3c42ac6ef7379f29788eb54cb751be5a663bb235a" + ], + "markers": "python_version >= '3.6'", + "version": "==2.5.1" + }, + "nodeenv": { + "hashes": [ + "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b", + "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7" + ], + "version": "==1.6.0" + }, + "packaging": { + "hashes": [ + "sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e", + "sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388" + ], + "version": "==16.8" + }, + "platformdirs": { + "hashes": [ + "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f", + "sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648" + ], + "markers": "python_version >= '3.6'", + "version": "==2.3.0" + }, + "pre-commit": { + "hashes": [ + "sha256:3c25add78dbdfb6a28a651780d5c311ac40dd17f160eb3954a0c59da40a505a7", + "sha256:a4ed01000afcb484d9eb8d504272e642c4c4099bbad3a6b27e519bd6a3e928a6" + ], + "index": "pypi", + "version": "==2.15.0" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.4.7" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==5.4.1" + }, + "requests": { + "hashes": [ + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.26.0" + }, + "s3transfer": { + "hashes": [ + "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", + "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" + ], + "markers": "python_version >= '3.6'", + "version": "==0.5.0" + }, + "sceptre": { + "hashes": [ + "sha256:1e23a0b087ec9cd1ced25b929e04a09509011bc5c228866f8fa0004e0b6b0db5", + "sha256:550473ae9dd3d5b2c54395bdc01f230bea1fb57311754dcbed7f787845b84c92" + ], + "index": "pypi", + "version": "==2.6.3" + }, + "sceptre-cmd-resolver": { + "hashes": [ + "sha256:4490387b7689f0d29ff58c79ca9232c091ba1885c6089f2300329bca038a08c1", + "sha256:8f23f6e8a42baf7271cbece95cc8f365ce00717ea2c255db2f61b5eff87bff2f" + ], + "version": "==1.1.3" + }, + "sceptre-file-resolver": { + "hashes": [ + "sha256:797344aa234559d5af834eee220ab6959bbb011cf078d212dc16df2bbd9ff59c", + "sha256:8c148653e723fb71d7a442ee2ff9ecb499f44b17fd1d7f7af3a2130f56b399f5" + ], + "version": "==1.0.4" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==0.10.2" + }, + "urllib3": { + "hashes": [ + "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", + "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.6" + }, + "virtualenv": { + "hashes": [ + "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0", + "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==20.7.2" + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000000..1c1d294586 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# infra-template + +This is an opinionated template for the creation of AWS infrastructure. + +## Setup +1. Install [awscli v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +1. This relies on python dependencies. We recommmend installing one of the latest versions of python3. +1. Install [pipenv](git@github.com:tthyer/infra-template.git) for python environment management. +1. Run `pipenv install --dev` to install [sceptre](https://sceptre.cloudreach.com/2.6.3/) and [pre-commit](https://pre-commit.com/). +1. Run `pipenv run pre-commit install` to install git hooks. +1. [Github actions](https://docs.github.com/en/actions) for CICD +1. Install [pre-commit](https://pre-commit.com/), then run `pre-commit install`. + +The Pipfile installs the following dependencies in a virtual environment: +* [sceptre](https://sceptre.cloudreach.com/2.6.3/) for better AWS CloudFormation deployment +* [pre-commit](https://pre-commit.com/), to ensure + +## Testing sceptre deployment + +If your text editor (_e.g._ Visual Studio Code) or shell (_e.g._ using [`direnv`](https://direnv.net/)) can automatically activate the `pipenv` virtual environment, you can omit the `pipenv shell` command. + +``` +# Activate the pipenv virtual environment to use sceptre +pipenv shell + +# Test the deployment of a single stack in the 'develop' stack group +sceptre launch develop/my-template.yaml + +# Delete the test deployment of a single stack the 'develop' stack group +sceptre delete develop/my-template.yaml + +# Test deploying the entire 'develop' stack group +sceptre launch develop + +# Remove the entire 'develop' stacck group +sceptre delete develop +``` + +## Environments +This repository template requires the creation of Github Environments that +pair with the defined sceptre stack groups: `develop` and `prod`. If you are +not doing development of new Cloudformation templates in this repository, but +using externally developed ones (such as from aws-infra), you can remove +the `develop` folder. + +The Github action to deploy AWS stacks relies on setting up the secrets used by +the workflow in [Github Environments](https://docs.github.com/en/actions/reference/environments). +Set up environments for each AWS account you're deploying to. This is where +you'll put secrets such as the ones for your CI user credentials. diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000000..d17bb80c6f --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,2 @@ +project_code: infra-template +region: us-east-1 diff --git a/config/develop/config.yaml b/config/develop/config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/develop/example-stack.yaml b/config/develop/example-stack.yaml new file mode 100644 index 0000000000..b5a30aac70 --- /dev/null +++ b/config/develop/example-stack.yaml @@ -0,0 +1,4 @@ +template_path: example-template.yaml +stack_name: example-stack +parameters: + LogGroupName: example-stack-log-group diff --git a/config/prod/config.yaml b/config/prod/config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/prod/example-stack.yaml b/config/prod/example-stack.yaml new file mode 100644 index 0000000000..b5a30aac70 --- /dev/null +++ b/config/prod/example-stack.yaml @@ -0,0 +1,4 @@ +template_path: example-template.yaml +stack_name: example-stack +parameters: + LogGroupName: example-stack-log-group diff --git a/templates/example-template.yaml b/templates/example-template.yaml new file mode 100644 index 0000000000..9600e88c6b --- /dev/null +++ b/templates/example-template.yaml @@ -0,0 +1,21 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Serverless Aurora MySQL DB for NextFlow Tower + +Parameters: + + LogGroupName: + Description: Name of your log group. + Type: String + + DaysToRetainLogs: + Description: Number of days to keep logs before deletion. + Type: Number + Default: 7 + +Resources: + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Ref LogGroupName + RetentionInDays: !Ref DaysToRetainLogs From 57927c7a7c3eeb72e8b2b3ccbc5945ee4effab29 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:32:41 -0700 Subject: [PATCH 2/8] Add in readme and dockerfile for creating our own otel collector image --- .github/workflows/aws-deploy.yaml | 70 ------ .pre-commit-config.yaml | 19 -- Dockerfile | 5 + Pipfile | 13 -- Pipfile.lock | 356 ------------------------------ README.md | 68 +++--- config/config.yaml | 2 - config/develop/config.yaml | 0 config/develop/example-stack.yaml | 4 - config/prod/config.yaml | 0 config/prod/example-stack.yaml | 4 - templates/example-template.yaml | 21 -- 12 files changed, 32 insertions(+), 530 deletions(-) delete mode 100644 .github/workflows/aws-deploy.yaml delete mode 100644 .pre-commit-config.yaml create mode 100644 Dockerfile delete mode 100644 Pipfile delete mode 100644 Pipfile.lock delete mode 100644 config/config.yaml delete mode 100644 config/develop/config.yaml delete mode 100644 config/develop/example-stack.yaml delete mode 100644 config/prod/config.yaml delete mode 100644 config/prod/example-stack.yaml delete mode 100644 templates/example-template.yaml diff --git a/.github/workflows/aws-deploy.yaml b/.github/workflows/aws-deploy.yaml deleted file mode 100644 index b09cb55af6..0000000000 --- a/.github/workflows/aws-deploy.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: aws-deploy - -on: push - -jobs: - pre-commit: - name: Run pre-commit hooks against all files - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.2 - - sceptre-deploy: - name: Deploy CloudFormation templates using sceptre - runs-on: ubuntu-latest - needs: pre-commit - if: github.ref == 'refs/heads/main' - strategy: - matrix: - job-environment: - - develop - - prod - include: - - job-environment: develop - sceptre-environment: develop - - job-environment: prod - sceptre-environment: prod - environment: ${{ matrix.job-environment }} - steps: - - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Set up python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - - name: Install pipenv - run: python -m pip install pipenv - - - name: Set up dependency caching - uses: actions/cache@v2 - with: - path: ~/.local/share/virtualenvs - key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }} - restore-keys: | - ${{ runner.os }}-pipenv-v2- - - - name: Install dependencies - if: steps.pipenv-cache.outputs.cache-hit != 'true' - env: - PIPENV_NOSPIN: 'true' - WORKON_HOME: ~/.local/share/virtualenvs - PIPENV_CACHE_DIR: ~/.local/share/pipcache - run: | - pipenv install --dev - - - name: Assume AWS role in dev account - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.CI_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.CI_USER_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - role-to-assume: ${{ secrets.CI_ROLE_TO_ASSUME }} - role-duration-seconds: 1200 - - - name: Deploy sceptre stacks - run: pipenv run sceptre launch ${{ matrix.sceptre-environment }} --yes diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 51fe7fe4ba..0000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - repo: https://github.com/adrienverge/yamllint - rev: v1.26.3 - hooks: - - id: yamllint - - repo: https://github.com/awslabs/cfn-python-lint - rev: v0.54.0 - hooks: - - id: cfn-python-lint - files: templates/.*\.(json|yml|yaml)$ - - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.10 - hooks: - - id: remove-tabs diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..698fc11bbf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM public.ecr.aws/aws-observability/aws-otel-collector:latest as aws-otel + +FROM otel/opentelemetry-collector-contrib:0.113.0 + +COPY --from=aws-otel /healthcheck /healthcheck diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 8ea2b0c886..0000000000 --- a/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[requires] -python_version = "3.9" - -[dev-packages] -pre-commit = "==2.*" -sceptre = "==2.*" - -[packages] diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index b56eb735ef..0000000000 --- a/Pipfile.lock +++ /dev/null @@ -1,356 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "50898c784ec4d62ec4f1ea14af21a1b1157ed8c0a78642b202ade647bc6d5f66" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.9" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": {}, - "develop": { - "backports.entry-points-selectable": { - "hashes": [ - "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a", - "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc" - ], - "markers": "python_version >= '2.7'", - "version": "==1.1.0" - }, - "boto3": { - "hashes": [ - "sha256:63b9846c26e0905f4e9e39d6b59f152330c53a926d693439161c43dcf9779365", - "sha256:a9232185d8e7e2fd2b166c0ebee5d7b1f787fdb3093f33bbf5aa932c08f0ccac" - ], - "markers": "python_version >= '3.6'", - "version": "==1.18.42" - }, - "botocore": { - "hashes": [ - "sha256:0952d1200968365b440045efe8e45bbae38cf603fee12bcfc3d7b5f963cbfa18", - "sha256:6de4fec4ee10987e4dea96f289553c2f45109fcaafcb74a5baee1221926e1306" - ], - "markers": "python_version >= '3.6'", - "version": "==1.21.42" - }, - "certifi": { - "hashes": [ - "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", - "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" - ], - "version": "==2021.5.30" - }, - "cfgv": { - "hashes": [ - "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426", - "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736" - ], - "markers": "python_full_version >= '3.6.1'", - "version": "==3.3.1" - }, - "charset-normalizer": { - "hashes": [ - "sha256:7098e7e862f6370a2a8d1a6398cd359815c45d12626267652c3f13dec58e2367", - "sha256:fa471a601dfea0f492e4f4fca035cd82155e65dc45c9b83bf4322dfab63755dd" - ], - "markers": "python_version >= '3'", - "version": "==2.0.5" - }, - "click": { - "hashes": [ - "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a", - "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6" - ], - "markers": "python_version >= '3.6'", - "version": "==8.0.1" - }, - "colorama": { - "hashes": [ - "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", - "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==0.4.4" - }, - "decorator": { - "hashes": [ - "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760", - "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7" - ], - "version": "==4.4.2" - }, - "distlib": { - "hashes": [ - "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736", - "sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c" - ], - "version": "==0.3.2" - }, - "filelock": { - "hashes": [ - "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", - "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" - ], - "version": "==3.0.12" - }, - "identify": { - "hashes": [ - "sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d", - "sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf" - ], - "markers": "python_full_version >= '3.6.1'", - "version": "==2.2.14" - }, - "idna": { - "hashes": [ - "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", - "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" - ], - "markers": "python_version >= '3'", - "version": "==3.2" - }, - "jinja2": { - "hashes": [ - "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", - "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.11.3" - }, - "jmespath": { - "hashes": [ - "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", - "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", - "version": "==0.10.0" - }, - "markupsafe": { - "hashes": [ - "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", - "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", - "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", - "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", - "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", - "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", - "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", - "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", - "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", - "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", - "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", - "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", - "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", - "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", - "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", - "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", - "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", - "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", - "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", - "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", - "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", - "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", - "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", - "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", - "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", - "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", - "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", - "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", - "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", - "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", - "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", - "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", - "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", - "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", - "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", - "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", - "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", - "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", - "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", - "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", - "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", - "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", - "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", - "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", - "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", - "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", - "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", - "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", - "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", - "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", - "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", - "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", - "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", - "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" - ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" - }, - "networkx": { - "hashes": [ - "sha256:0635858ed7e989f4c574c2328380b452df892ae85084144c73d8cd819f0c4e06", - "sha256:109cd585cac41297f71103c3c42ac6ef7379f29788eb54cb751be5a663bb235a" - ], - "markers": "python_version >= '3.6'", - "version": "==2.5.1" - }, - "nodeenv": { - "hashes": [ - "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b", - "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7" - ], - "version": "==1.6.0" - }, - "packaging": { - "hashes": [ - "sha256:5d50835fdf0a7edf0b55e311b7c887786504efea1177abd7e69329a8e5ea619e", - "sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388" - ], - "version": "==16.8" - }, - "platformdirs": { - "hashes": [ - "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f", - "sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648" - ], - "markers": "python_version >= '3.6'", - "version": "==2.3.0" - }, - "pre-commit": { - "hashes": [ - "sha256:3c25add78dbdfb6a28a651780d5c311ac40dd17f160eb3954a0c59da40a505a7", - "sha256:a4ed01000afcb484d9eb8d504272e642c4c4099bbad3a6b27e519bd6a3e928a6" - ], - "index": "pypi", - "version": "==2.15.0" - }, - "pyparsing": { - "hashes": [ - "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", - "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.4.7" - }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.8.2" - }, - "pyyaml": { - "hashes": [ - "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", - "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", - "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", - "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", - "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", - "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", - "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", - "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", - "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", - "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", - "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", - "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", - "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", - "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", - "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", - "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", - "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", - "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", - "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", - "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", - "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", - "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", - "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", - "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", - "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", - "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", - "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", - "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", - "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==5.4.1" - }, - "requests": { - "hashes": [ - "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", - "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==2.26.0" - }, - "s3transfer": { - "hashes": [ - "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", - "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" - ], - "markers": "python_version >= '3.6'", - "version": "==0.5.0" - }, - "sceptre": { - "hashes": [ - "sha256:1e23a0b087ec9cd1ced25b929e04a09509011bc5c228866f8fa0004e0b6b0db5", - "sha256:550473ae9dd3d5b2c54395bdc01f230bea1fb57311754dcbed7f787845b84c92" - ], - "index": "pypi", - "version": "==2.6.3" - }, - "sceptre-cmd-resolver": { - "hashes": [ - "sha256:4490387b7689f0d29ff58c79ca9232c091ba1885c6089f2300329bca038a08c1", - "sha256:8f23f6e8a42baf7271cbece95cc8f365ce00717ea2c255db2f61b5eff87bff2f" - ], - "version": "==1.1.3" - }, - "sceptre-file-resolver": { - "hashes": [ - "sha256:797344aa234559d5af834eee220ab6959bbb011cf078d212dc16df2bbd9ff59c", - "sha256:8c148653e723fb71d7a442ee2ff9ecb499f44b17fd1d7f7af3a2130f56b399f5" - ], - "version": "==1.0.4" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==1.16.0" - }, - "toml": { - "hashes": [ - "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", - "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", - "version": "==0.10.2" - }, - "urllib3": { - "hashes": [ - "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", - "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.6" - }, - "virtualenv": { - "hashes": [ - "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0", - "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.7.2" - } - } -} diff --git a/README.md b/README.md index 1c1d294586..f2cdea465e 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,35 @@ -# infra-template +# Purpose +As discussed in this Github Issue: +The official opentelemetry (OTEL) collector image does not contain cURL or related shell +commands required to do container level health checks. It is reliant on external +services such as the application load balancer in AWS to perform these checks. This is +problematic with our deployment of the OTEL collector as we are using AWS +service connect with AWS ECS to allow other containers within the namespace to connect +to the collector. As such, there is no load balancer in-front of the container to handle +its lifecycle. Within ECS, the recommended way from AWS to handle container level health +checks is to let ECS perform commands in the container. +Source: -This is an opinionated template for the creation of AWS infrastructure. -## Setup -1. Install [awscli v2](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -1. This relies on python dependencies. We recommmend installing one of the latest versions of python3. -1. Install [pipenv](git@github.com:tthyer/infra-template.git) for python environment management. -1. Run `pipenv install --dev` to install [sceptre](https://sceptre.cloudreach.com/2.6.3/) and [pre-commit](https://pre-commit.com/). -1. Run `pipenv run pre-commit install` to install git hooks. -1. [Github actions](https://docs.github.com/en/actions) for CICD -1. Install [pre-commit](https://pre-commit.com/), then run `pre-commit install`. +Since the OTEL collector does not have a shell, nor cURL available we need to accomplish +this another way. In the official AWS OTEL collector distro they accomplish this by +compiling a golang script down into a binary that can be run within the container. +Unfortunately we cannot use the AWS OTEL collector because they are not supporting the +`oauth2clientauthextension`: . -The Pipfile installs the following dependencies in a virtual environment: -* [sceptre](https://sceptre.cloudreach.com/2.6.3/) for better AWS CloudFormation deployment -* [pre-commit](https://pre-commit.com/), to ensure -## Testing sceptre deployment +For our purposes we are creating a new image based off the `otel/opentelemetry-collector-contrib` image, +but with the addition of the healthcheck binary from the AWS OTEL distro. This +combination lets us use the oauth2 extension, and have container level health checks. -If your text editor (_e.g._ Visual Studio Code) or shell (_e.g._ using [`direnv`](https://direnv.net/)) can automatically activate the `pipenv` virtual environment, you can omit the `pipenv shell` command. -``` -# Activate the pipenv virtual environment to use sceptre -pipenv shell +## Creating a new image (To automate later on) +As new base images are updated we will need to in-turn create a new otel collector +image that we deploy to ECS. -# Test the deployment of a single stack in the 'develop' stack group -sceptre launch develop/my-template.yaml +1) Update values in the `Dockerfile` +2) Run `docker build -t ghcr.io/sage-bionetworks/sage-otel-collector:vX.X.X .` (Replace the version) +3) Run `docker push ghcr.io/sage-bionetworks/sage-otel-collector:vX.X.X` (Replace the version) -# Delete the test deployment of a single stack the 'develop' stack group -sceptre delete develop/my-template.yaml - -# Test deploying the entire 'develop' stack group -sceptre launch develop - -# Remove the entire 'develop' stacck group -sceptre delete develop -``` - -## Environments -This repository template requires the creation of Github Environments that -pair with the defined sceptre stack groups: `develop` and `prod`. If you are -not doing development of new Cloudformation templates in this repository, but -using externally developed ones (such as from aws-infra), you can remove -the `develop` folder. - -The Github action to deploy AWS stacks relies on setting up the secrets used by -the workflow in [Github Environments](https://docs.github.com/en/actions/reference/environments). -Set up environments for each AWS account you're deploying to. This is where -you'll put secrets such as the ones for your CI user credentials. +Once a new image is built and pushed, then you'll want to update the values in the CDK +scripts to use the new image version. diff --git a/config/config.yaml b/config/config.yaml deleted file mode 100644 index d17bb80c6f..0000000000 --- a/config/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -project_code: infra-template -region: us-east-1 diff --git a/config/develop/config.yaml b/config/develop/config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/config/develop/example-stack.yaml b/config/develop/example-stack.yaml deleted file mode 100644 index b5a30aac70..0000000000 --- a/config/develop/example-stack.yaml +++ /dev/null @@ -1,4 +0,0 @@ -template_path: example-template.yaml -stack_name: example-stack -parameters: - LogGroupName: example-stack-log-group diff --git a/config/prod/config.yaml b/config/prod/config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/config/prod/example-stack.yaml b/config/prod/example-stack.yaml deleted file mode 100644 index b5a30aac70..0000000000 --- a/config/prod/example-stack.yaml +++ /dev/null @@ -1,4 +0,0 @@ -template_path: example-template.yaml -stack_name: example-stack -parameters: - LogGroupName: example-stack-log-group diff --git a/templates/example-template.yaml b/templates/example-template.yaml deleted file mode 100644 index 9600e88c6b..0000000000 --- a/templates/example-template.yaml +++ /dev/null @@ -1,21 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: Serverless Aurora MySQL DB for NextFlow Tower - -Parameters: - - LogGroupName: - Description: Name of your log group. - Type: String - - DaysToRetainLogs: - Description: Number of days to keep logs before deletion. - Type: Number - Default: 7 - -Resources: - - LogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Ref LogGroupName - RetentionInDays: !Ref DaysToRetainLogs From 97b1adab72f86c6f66d2166e2bd3179ece567b9a Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 17:13:08 +0000 Subject: [PATCH 3/8] move original files to apps/sage/otel-collector --- apps/sage/otel-collector/.gitignore | 129 ++++++++++++++++++++++++++++ apps/sage/otel-collector/Dockerfile | 5 ++ apps/sage/otel-collector/LICENSE | 21 +++++ apps/sage/otel-collector/README.md | 35 ++++++++ 4 files changed, 190 insertions(+) create mode 100644 apps/sage/otel-collector/.gitignore create mode 100644 apps/sage/otel-collector/Dockerfile create mode 100644 apps/sage/otel-collector/LICENSE create mode 100644 apps/sage/otel-collector/README.md diff --git a/apps/sage/otel-collector/.gitignore b/apps/sage/otel-collector/.gitignore new file mode 100644 index 0000000000..b6e47617de --- /dev/null +++ b/apps/sage/otel-collector/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/apps/sage/otel-collector/Dockerfile b/apps/sage/otel-collector/Dockerfile new file mode 100644 index 0000000000..698fc11bbf --- /dev/null +++ b/apps/sage/otel-collector/Dockerfile @@ -0,0 +1,5 @@ +FROM public.ecr.aws/aws-observability/aws-otel-collector:latest as aws-otel + +FROM otel/opentelemetry-collector-contrib:0.113.0 + +COPY --from=aws-otel /healthcheck /healthcheck diff --git a/apps/sage/otel-collector/LICENSE b/apps/sage/otel-collector/LICENSE new file mode 100644 index 0000000000..abda5ccb94 --- /dev/null +++ b/apps/sage/otel-collector/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 T. Thyer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apps/sage/otel-collector/README.md b/apps/sage/otel-collector/README.md new file mode 100644 index 0000000000..f2cdea465e --- /dev/null +++ b/apps/sage/otel-collector/README.md @@ -0,0 +1,35 @@ +# Purpose +As discussed in this Github Issue: +The official opentelemetry (OTEL) collector image does not contain cURL or related shell +commands required to do container level health checks. It is reliant on external +services such as the application load balancer in AWS to perform these checks. This is +problematic with our deployment of the OTEL collector as we are using AWS +service connect with AWS ECS to allow other containers within the namespace to connect +to the collector. As such, there is no load balancer in-front of the container to handle +its lifecycle. Within ECS, the recommended way from AWS to handle container level health +checks is to let ECS perform commands in the container. +Source: + + +Since the OTEL collector does not have a shell, nor cURL available we need to accomplish +this another way. In the official AWS OTEL collector distro they accomplish this by +compiling a golang script down into a binary that can be run within the container. +Unfortunately we cannot use the AWS OTEL collector because they are not supporting the +`oauth2clientauthextension`: . + + +For our purposes we are creating a new image based off the `otel/opentelemetry-collector-contrib` image, +but with the addition of the healthcheck binary from the AWS OTEL distro. This +combination lets us use the oauth2 extension, and have container level health checks. + + +## Creating a new image (To automate later on) +As new base images are updated we will need to in-turn create a new otel collector +image that we deploy to ECS. + +1) Update values in the `Dockerfile` +2) Run `docker build -t ghcr.io/sage-bionetworks/sage-otel-collector:vX.X.X .` (Replace the version) +3) Run `docker push ghcr.io/sage-bionetworks/sage-otel-collector:vX.X.X` (Replace the version) + +Once a new image is built and pushed, then you'll want to update the values in the CDK +scripts to use the new image version. From 5421a353fa6e8070ba85d40b751029fb26e96508 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 17:25:18 +0000 Subject: [PATCH 4/8] remove the MIT license in favor of the monorepo default Apache license --- apps/sage/otel-collector/LICENSE | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 apps/sage/otel-collector/LICENSE diff --git a/apps/sage/otel-collector/LICENSE b/apps/sage/otel-collector/LICENSE deleted file mode 100644 index abda5ccb94..0000000000 --- a/apps/sage/otel-collector/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 T. Thyer - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From 913f64d761a61d38420cc80f50811ac022cce513 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 17:28:25 +0000 Subject: [PATCH 5/8] add project.json --- apps/sage/otel-collector/project.json | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 apps/sage/otel-collector/project.json diff --git a/apps/sage/otel-collector/project.json b/apps/sage/otel-collector/project.json new file mode 100644 index 0000000000..0a4ee8c250 --- /dev/null +++ b/apps/sage/otel-collector/project.json @@ -0,0 +1,32 @@ +{ + "name": "sage-otel-collector", + "$schema": "../../../node_modules/nx/schemas/project-schema.json", + "projectType": "application", + "targets": { + "serve-detach": { + "executor": "nx:run-commands", + "options": { + "command": "docker/sage/serve-detach.sh sage-otel-collector" + } + }, + "publish-image": { + "executor": "@nx-tools/nx-container:build", + "options": { + "context": "apps/sage/otel-collector", + "metadata": { + "images": ["ghcr.io/sage-bionetworks/{projectName}"], + "tags": ["type=edge,branch=main", "type=sha"] + }, + "push": true + }, + "dependsOn": ["build-image"] + }, + "scan-image": { + "executor": "nx:run-commands", + "options": { + "command": "trivy image ghcr.io/sage-bionetworks/{projectName}:local --quiet", + "color": true + } + } + } +} From 72232ac5da29e3b772dee101904c0661eb4973a1 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 17:33:53 +0000 Subject: [PATCH 6/8] add minimal docker compose files for sage-* containers --- docker/sage/networks.yml | 4 ++++ docker/sage/serve-detach.sh | 12 ++++++++++++ docker/sage/services/otel-collector.yml | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 docker/sage/networks.yml create mode 100755 docker/sage/serve-detach.sh create mode 100644 docker/sage/services/otel-collector.yml diff --git a/docker/sage/networks.yml b/docker/sage/networks.yml new file mode 100644 index 0000000000..3872357442 --- /dev/null +++ b/docker/sage/networks.yml @@ -0,0 +1,4 @@ +networks: + sage: + name: sage + driver: bridge diff --git a/docker/sage/serve-detach.sh b/docker/sage/serve-detach.sh new file mode 100755 index 0000000000..28dc29a315 --- /dev/null +++ b/docker/sage/serve-detach.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +args=( + # List of services in alphanumeric order + --file docker/sage/services/otel-collector.yml + + --file docker/sage/networks.yml + + up $1 --detach --remove-orphans +) + +docker compose "${args[@]}" \ No newline at end of file diff --git a/docker/sage/services/otel-collector.yml b/docker/sage/services/otel-collector.yml new file mode 100644 index 0000000000..bc65a7c756 --- /dev/null +++ b/docker/sage/services/otel-collector.yml @@ -0,0 +1,18 @@ +services: + sage-otel-collector: + image: ghcr.io/sage-bionetworks/sage-otel-collector:${SAGE_VERSION:-local} + container_name: sage-otel-collector + restart: always + networks: + - sage + ports: + - '1888:1888' # pprof extension + - '8888:8888' # Prometheus metrics exposed by the collector + - '8889:8889' # Prometheus exporter metrics + - '13133:13133' # health_check extension + - '4317:4317' # OTLP gRPC receiver + - '55679:55679' # zpages extension + deploy: + resources: + limits: + memory: 200M From 7b6c9c3d1fc62fec00f982a1701d6ea80e55c52d Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 17:35:30 +0000 Subject: [PATCH 7/8] enable task inference for sage projects --- libs/sage-monorepo/nx-plugin/src/plugins/plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/sage-monorepo/nx-plugin/src/plugins/plugin.ts b/libs/sage-monorepo/nx-plugin/src/plugins/plugin.ts index 1f31e43d8f..54cabc564a 100644 --- a/libs/sage-monorepo/nx-plugin/src/plugins/plugin.ts +++ b/libs/sage-monorepo/nx-plugin/src/plugins/plugin.ts @@ -34,7 +34,7 @@ function writeProjectConfigurationsToCache( writeJsonFile(cachePath, results); } -const projectFilePattern = '{apps,libs}/{openchallenges,agora,sandbox}/**/project.json'; +const projectFilePattern = '{apps,libs}/{openchallenges,agora,sage,sandbox}/**/project.json'; export const createNodesV2: CreateNodesV2 = [ projectFilePattern, From d1e1d6c34190b4ce93a892a29dc1ba93d6dd9f88 Mon Sep 17 00:00:00 2001 From: Thomas Schaffter Date: Wed, 13 Nov 2024 17:44:26 +0000 Subject: [PATCH 8/8] fix hadolint issue by pinning the version of aws-observability/aws-otel-collector --- apps/sage/otel-collector/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sage/otel-collector/Dockerfile b/apps/sage/otel-collector/Dockerfile index 698fc11bbf..2abe929971 100644 --- a/apps/sage/otel-collector/Dockerfile +++ b/apps/sage/otel-collector/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/aws-observability/aws-otel-collector:latest as aws-otel +FROM public.ecr.aws/aws-observability/aws-otel-collector:v0.41.1 as aws-otel FROM otel/opentelemetry-collector-contrib:0.113.0