From 0f0941e4353fbfa13f4c24ffe924e0e589966303 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 22 Apr 2024 11:32:06 -0400 Subject: [PATCH 1/5] PRO-499 REFACTOR Make a proxy bin that is used to install missing packages --- Dockerfile | 1 + entrypoint.sh | 2 ++ {bin => proxy/bin}/gcloud | 0 3 files changed, 3 insertions(+) rename {bin => proxy/bin}/gcloud (100%) diff --git a/Dockerfile b/Dockerfile index b0ab8989..7fc04824 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM ghcr.io/terrateamio/action-base:latest +COPY proxy /usr/local/ COPY conftest-wrapper /usr/local/bin/conftest-wrapper COPY checkov-wrapper /usr/local/bin/checkov-wrapper COPY cdktf-setup.sh /cdktf-setup.sh diff --git a/entrypoint.sh b/entrypoint.sh index f7108729..985a506b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,5 +1,7 @@ #! /bin/sh +export PATH="$PATH":/usr/local/proxy/bin + WORK_TOKEN="$1" API_BASE_URL="$2" diff --git a/bin/gcloud b/proxy/bin/gcloud similarity index 100% rename from bin/gcloud rename to proxy/bin/gcloud From 392fa8192abbb6066b767082fbd900a5ebc9c2dd Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 22 Apr 2024 11:38:15 -0400 Subject: [PATCH 2/5] PRO-499 ADD Resourcely proxy --- Dockerfile | 2 ++ proxy/bin/resourcely-cli | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100755 proxy/bin/resourcely-cli diff --git a/Dockerfile b/Dockerfile index 7fc04824..5a378039 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM ghcr.io/terrateamio/action-base:latest +ENV RESOURCELY_VERSION=1.0.12 + COPY proxy /usr/local/ COPY conftest-wrapper /usr/local/bin/conftest-wrapper COPY checkov-wrapper /usr/local/bin/checkov-wrapper diff --git a/proxy/bin/resourcely-cli b/proxy/bin/resourcely-cli new file mode 100755 index 00000000..73053d2f --- /dev/null +++ b/proxy/bin/resourcely-cli @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +set -e +set -u + +flock /tmp/resourcely-install curl -s -L -o /tmp/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz https://github.com/Resourcely-Inc/resourcely-container-registry/releases/download/v1.0.12/resourcely-cli-v1.0.12-linux-amd64.tar.gz + +flock /tmp/resourcely-install tar -xzf /tmp/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz + +exec resourcely-cli "$@" From 1c17619b2c9b73b87f750f4b9471191aceba93a7 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 22 Apr 2024 12:25:18 -0400 Subject: [PATCH 3/5] PRO-499 REFACTOR Allow run-time workflow steps --- terrat_runner/github_actions/run_time.py | 7 +++++++ terrat_runner/workflow_step.py | 10 ++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/terrat_runner/github_actions/run_time.py b/terrat_runner/github_actions/run_time.py index 3f19f71a..f5e7cbde 100644 --- a/terrat_runner/github_actions/run_time.py +++ b/terrat_runner/github_actions/run_time.py @@ -9,6 +9,8 @@ from . import core +from . import workflow_step_drift_create_issue + class Run_time(object): def initialize(self, state): @@ -82,3 +84,8 @@ def work_index(self, state): requests_retry.put(state.api_base_url + '/v1/work-manifests/' + state.work_token, json=output) + + def steps(self): + return { + 'drift_create_issue': workflow_step_drift_create_issue.run, + } diff --git a/terrat_runner/workflow_step.py b/terrat_runner/workflow_step.py index d8f9823c..e4a2c34a 100644 --- a/terrat_runner/workflow_step.py +++ b/terrat_runner/workflow_step.py @@ -2,8 +2,8 @@ # which each take their own configuration parameters. import logging -import github_actions.workflow_step_drift_create_issue import workflow + import workflow_step_apply import workflow_step_env import workflow_step_infracost_setup @@ -18,7 +18,6 @@ STEPS = { 'apply': workflow_step_apply.run, - 'drift_create_issue': github_actions.workflow_step_drift_create_issue.run, 'env': workflow_step_env.run, 'infracost_setup': workflow_step_infracost_setup.run, 'init': workflow_step_init.run, @@ -32,19 +31,22 @@ def run_steps(state, steps, restrict_types=None): + valid_steps = STEPS.copy() + valid_steps.update(state.run_time.steps()) + results = [] for step in steps: if 'type' not in step: raise Exception('Step must contain a type') - elif step['type'] not in STEPS: + elif step['type'] not in valid_steps: raise Exception('Step type {} is unknown'.format(step['type'])) elif restrict_types and step['type'] not in restrict_types: raise Exception('Step type {} not allowed in this mode'.format(step['type'])) else: try: logging.info('STEP : RUN : %s : %r', state.working_dir, step) - result = STEPS[step['type']](state, step) + result = valid_steps[step['type']](state, step) state = result.state except Exception as exn: logging.exception(exn) From 7823fb4d689aa59b9c1d84dba450bfdcaf4c13d3 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 22 Apr 2024 12:59:03 -0400 Subject: [PATCH 4/5] PRO-499 REFACTOR Include the executed command in the workflow run step --- terrat_runner/workflow_step_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terrat_runner/workflow_step_run.py b/terrat_runner/workflow_step_run.py index f9546288..4ccff8a5 100644 --- a/terrat_runner/workflow_step_run.py +++ b/terrat_runner/workflow_step_run.py @@ -45,7 +45,7 @@ def run(state, config): state=state, workflow_step={ 'type': 'run', - 'cmd': config['cmd'], + 'cmd': proc.args, 'exit_code': proc.returncode }, outputs=outputs) From 0b9d8d8fbd42e997d3520e9e40a45a0c9c901c5d Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 22 Apr 2024 15:33:20 -0400 Subject: [PATCH 5/5] PRO-499 ADD Resourcely support --- Dockerfile | 2 +- proxy/bin/resourcely-cli | 15 +++++-- terrat_runner/github_actions/run_time.py | 2 + .../workflow_step_resourcely.py | 41 +++++++++++++++++++ terrat_runner/repo_config.py | 16 +++++++- terrat_runner/work_plan.py | 11 ++++- 6 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 terrat_runner/github_actions/workflow_step_resourcely.py diff --git a/Dockerfile b/Dockerfile index 5a378039..8c282bb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ghcr.io/terrateamio/action-base:latest ENV RESOURCELY_VERSION=1.0.12 -COPY proxy /usr/local/ +COPY proxy/bin /usr/local/proxy/bin COPY conftest-wrapper /usr/local/bin/conftest-wrapper COPY checkov-wrapper /usr/local/bin/checkov-wrapper COPY cdktf-setup.sh /cdktf-setup.sh diff --git a/proxy/bin/resourcely-cli b/proxy/bin/resourcely-cli index 73053d2f..20a2d96b 100755 --- a/proxy/bin/resourcely-cli +++ b/proxy/bin/resourcely-cli @@ -3,8 +3,17 @@ set -e set -u -flock /tmp/resourcely-install curl -s -L -o /tmp/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz https://github.com/Resourcely-Inc/resourcely-container-registry/releases/download/v1.0.12/resourcely-cli-v1.0.12-linux-amd64.tar.gz +if [[ ! -f /usr/local/bin/resourcely-cli ]]; then + flock /tmp/resourcely-install \ + curl \ + -s \ + -L \ + -o \ + /tmp/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz \ + https://github.com/Resourcely-Inc/resourcely-container-registry/releases/download/v"$RESOURCELY_VERSION"/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz -flock /tmp/resourcely-install tar -xzf /tmp/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz + flock /tmp/resourcely-install tar -xzf /tmp/resourcely-cli-v"$RESOURCELY_VERSION"-linux-amd64.tar.gz + flock /tmp/resourcely-install mv resourcely-cli /usr/local/bin/ || true +fi -exec resourcely-cli "$@" +exec /usr/local/bin/resourcely-cli "$@" diff --git a/terrat_runner/github_actions/run_time.py b/terrat_runner/github_actions/run_time.py index f5e7cbde..50fc29e0 100644 --- a/terrat_runner/github_actions/run_time.py +++ b/terrat_runner/github_actions/run_time.py @@ -10,6 +10,7 @@ from . import core from . import workflow_step_drift_create_issue +from . import workflow_step_resourcely class Run_time(object): @@ -88,4 +89,5 @@ def work_index(self, state): def steps(self): return { 'drift_create_issue': workflow_step_drift_create_issue.run, + 'resourcely': workflow_step_resourcely.run } diff --git a/terrat_runner/github_actions/workflow_step_resourcely.py b/terrat_runner/github_actions/workflow_step_resourcely.py new file mode 100644 index 00000000..1a9b0edf --- /dev/null +++ b/terrat_runner/github_actions/workflow_step_resourcely.py @@ -0,0 +1,41 @@ +import os +import tempfile + +import workflow_step_run +import workflow_step_terraform + + +def run(state, config): + result = workflow_step_terraform.run(state, + { + 'args': ['show', '-json', '$TERRATEAM_PLAN_FILE'], + 'output_key': 'plan_json' + }) + + if result.failed: + return result._replace(workflow_step={'type': 'resourcely'}) + + plan_json = result.outputs['text'] + + with tempfile.TemporaryDirectory() as tmpdir: + json_file = os.path.join(tmpdir, 'json') + with open(json_file, 'w') as f: + f.write(plan_json) + + run_config = { + 'cmd': ['resourcely-cli', + '--no_color', + 'evaluate', + '--change_request_url', + 'https://github.com/{}/pull/{}'.format(state.env['GITHUB_REPOSITORY'], + state.work_manifest['run_kind_data']['id']), + '--change_request_sha', + '${GITHUB_SHA}', + '--plan', + json_file], + 'capture_output': True + } + + result = workflow_step_run.run(state, run_config) + + return result diff --git a/terrat_runner/repo_config.py b/terrat_runner/repo_config.py index d40a29fe..0993ea65 100644 --- a/terrat_runner/repo_config.py +++ b/terrat_runner/repo_config.py @@ -44,6 +44,18 @@ def _get_hooks(hooks): } +def _get_integrations(repo_config, workflow_integrations): + global_integrations = _get(repo_config, 'integrations', {}) + resourcely = _get(workflow_integrations, + 'resourcely', + _get(global_integrations, 'resourcely', {})) + return { + 'resourcely': { + 'enabled': _get(resourcely, 'enabled', False) + } + } + + def get_all_hooks(repo_config): return _get_hooks(_get(_get(repo_config, 'hooks', {}), 'all', {})) @@ -87,6 +99,7 @@ def get_workflow(repo_config, idx): cfg = { 'apply': workflow.get('apply', _default_apply_workflow()), 'plan': workflow.get('plan', _default_plan_workflow()), + 'integrations': _get_integrations(repo_config, _get(workflow, 'integrations', {})) } default_engine = get_engine(repo_config) @@ -144,7 +157,8 @@ def get_default_workflow(repo_config): return { 'apply': _default_apply_workflow(), 'plan': _default_plan_workflow(), - 'engine': get_engine(repo_config) + 'engine': get_engine(repo_config), + 'integrations': _get_integrations(repo_config, {}) } diff --git a/terrat_runner/work_plan.py b/terrat_runner/work_plan.py index 90dc7eb6..3a39d445 100644 --- a/terrat_runner/work_plan.py +++ b/terrat_runner/work_plan.py @@ -8,6 +8,13 @@ import workflow_step_terrateam_ssh_key_setup +def _merge_integrations(state, steps, integrations): + if state.work_manifest.get('run_kind') == 'pr' and integrations['resourcely']['enabled']: + steps = steps + [{'type': 'resourcely'}] + + return steps + + class Exec(work_exec.ExecInterface): def pre_hooks(self, state): pre_hooks = rc.get_all_hooks(state.repo_config)['pre'] @@ -80,12 +87,14 @@ def exec(self, state, d): state = state._replace(env=env) + plan_steps = _merge_integrations(state, workflow['plan'], workflow['integrations']) + state = workflow_step.run_steps( state._replace(working_dir=os.path.join(state.working_dir, path), path=path, workspace=workspace, workflow=workflow), - workflow['plan']) + plan_steps) result = { 'path': path,