From 64bd9db6b15b998d023f3629752d2fa9ae9c374b Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 15 Jan 2024 18:42:18 +0100 Subject: [PATCH] PRO-411 ADD OpenTofu support and refactor how engine config works --- Dockerfile.base | 18 ++++++ bin/tofu | 12 ++++ terrat_runner/main.py | 5 +- terrat_runner/repo_config.py | 78 +++++++++++++++++++++--- terrat_runner/work_apply.py | 12 ++-- terrat_runner/work_exec.py | 47 +++++++++++++- terrat_runner/work_plan.py | 12 ++-- terrat_runner/work_unsafe_apply.py | 12 ++-- terrat_runner/workflow_step_init.py | 13 ++-- terrat_runner/workflow_step_terraform.py | 15 +++-- 10 files changed, 176 insertions(+), 48 deletions(-) create mode 100755 bin/tofu diff --git a/Dockerfile.base b/Dockerfile.base index 3febb194..2a6e9d4e 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -49,6 +49,24 @@ RUN mkdir /tmp/awscli \ ENV CHECKOV_VERSION=2.5.10 RUN pip3 install checkov==${CHECKOV_VERSION} +# Temporarily pull from our branch until changes are merged back into a release +# +# ENV TOFUENV_VERSION v1.0.3 +# RUN curl -fsSL -o /tmp/tofuenv.zip \ +# "https://github.com/terrateamio/tofuenv/archive/refs/tags/v${TOFUENV_VERSION}.zip" \ +# && cd /tmp/ \ +# && unzip /tmp/tofuenv.zip \ +# && mv /tmp/tofuenv-${TOFUENV_VERSION} /usr/local/lib/tofuenv \ +# && echo "latest" > /usr/local/lib/tofuenv/version + + +ENV TOFUENV_DEFAULT_VERSION latest +RUN curl -fsSL -o /tmp/tofuenv.zip \ + "https://github.com/terrateamio/tofuenv/archive/refs/heads/pro-411-add-opentofu-support.zip" \ + && cd /tmp/ \ + && unzip /tmp/tofuenv.zip \ + && mv /tmp/tofuenv-pro-411-add-opentofu-support /usr/local/lib/tofuenv + COPY ./bin/ /usr/local/bin ENV DEFAULT_TERRAFORM_VERSION 1.5.7 COPY ./install-terraform-version /install-terraform-version diff --git a/bin/tofu b/bin/tofu new file mode 100755 index 00000000..2e9314c1 --- /dev/null +++ b/bin/tofu @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +set -x + +# Perform the install with an flock, this ensure that if there are multipel +# parallel runs, then only one can happen at a time. We don't have a guarantee +# that tofuenv is parallel-safe. +flock /tmp/tofuenv.install /usr/local/lib/tofuenv/bin/tofuenv install \ + || flock /tmp/tofuenv.install /usr/local/lib/tofuenv/bin/tofuenv install \ + || flock /tmp/tofuenv.install /usr/local/lib/tofuenv/bin/tofuenv install \ + +exec /usr/local/lib/tofuenv/bin/tofu "$@" diff --git a/terrat_runner/main.py b/terrat_runner/main.py index 2bf3ad41..570d6e78 100644 --- a/terrat_runner/main.py +++ b/terrat_runner/main.py @@ -45,13 +45,12 @@ def perform_merge(working_dir, base_ref): def maybe_setup_cdktf(rc, work_manifest, env): - # Determine if any workflows use cdktf and only install it if it is - # required. + # Determine if any engine uses cdktf and only install it if it is required. cdktf_used = False for d in work_manifest['changed_dirspaces']: if 'workflow' in d: workflow = repo_config.get_workflow(rc, d['workflow']) - cdktf_used = cdktf_used or workflow['cdktf'] + cdktf_used = cdktf_used or workflow['engine']['name'] == 'cdktf' if cdktf_used: subprocess.check_call(['/cdktf-setup.sh']) diff --git a/terrat_runner/repo_config.py b/terrat_runner/repo_config.py index fd47666a..d40a29fe 100644 --- a/terrat_runner/repo_config.py +++ b/terrat_runner/repo_config.py @@ -3,6 +3,7 @@ def _get(d, k, default): + """Return [default] if it is not set or set to None.""" v = d.get(k, default) if v is None: return default @@ -63,24 +64,87 @@ def get_apply_workflow(repo_config, idx): return repo_config['workflows'][idx].get('apply', _default_apply_workflow()) +def get_engine(repo_config): + # Get the engine config. If one is already there, we will take it verbatim, + # however if we need to construct a default one, we specify terraform and we + # also want to use the [default_tf_version] if present. This is to maintain + # compatibility with existing configurations. + if 'engine' in repo_config: + engine = repo_config['engine'].copy() + if engine['name'] in ['cdktf', 'terragrunt'] and 'tf_cmd' not in engine: + engine['tf_cmd'] = 'terraform' + + return engine + else: + return { + 'name': 'terraform', + 'version': repo_config.get('default_tf_version') + } + + def get_workflow(repo_config, idx): workflow = repo_config['workflows'][idx] - return { + cfg = { 'apply': workflow.get('apply', _default_apply_workflow()), - 'cdktf': workflow.get('cdktf', False), 'plan': workflow.get('plan', _default_plan_workflow()), - 'terraform_version': workflow.get('terraform_version', get_default_tf_version(repo_config)), - 'terragrunt': workflow.get('terragrunt', False), } + default_engine = get_engine(repo_config) + engine = _get(workflow, 'engine', {}).copy() + + # In order to maintain backwards compatibility, we need to do some work to + # transform an existing workflow configuration to one with an engine. + # Additionally, we want to make future lookups easy so we fill in the + # configurations that would be inferred. + if 'engine' not in workflow: + # If no engine is specified, convert any legacy configuration to the + # engine config. Fill in the minimal configuration and the rest will be + # done next. + if workflow.get('terragrunt'): + engine = { + 'name': 'terragrunt', + } + elif workflow.get('cdktf'): + engine = { + 'name': 'cdktf', + } + elif workflow.get('terraform_version'): + engine = { + 'name': 'terraform', + 'version': workflow['terraform_version'] + } + else: + engine = default_engine.copy() + + if default_engine['name'] == 'tofu': + default_tf_cmd = 'tofu' + default_tf_version = default_engine.get('version') + else: + default_tf_cmd = 'terraform' + default_tf_version = None + + if engine['name'] in ['terragrunt', 'cdktf']: + engine['tf_cmd'] = _get(engine, 'tf_cmd', default_tf_cmd) + if engine['tf_cmd'] == 'terraform': + engine['tf_version'] = _get(engine, 'tf_version', get_default_tf_version(repo_config)) + else: + engine['tf_version'] = _get(engine, 'tf_version', default_tf_version) + elif engine['name'] == 'terraform': + engine['version'] = _get(engine, 'version', get_default_tf_version(repo_config)) + elif engine['name'] == 'tofu': + engine['version'] = _get(engine, 'version', default_tf_version) + else: + raise Exception('Unknown engine') + + cfg['engine'] = engine + return cfg + def get_default_workflow(repo_config): return { 'apply': _default_apply_workflow(), - 'cdktf': False, 'plan': _default_plan_workflow(), - 'terraform_version': get_default_tf_version(repo_config), - 'terragrunt': False, + 'engine': get_engine(repo_config) } diff --git a/terrat_runner/work_apply.py b/terrat_runner/work_apply.py index c5242a42..166d8b1f 100644 --- a/terrat_runner/work_apply.py +++ b/terrat_runner/work_apply.py @@ -57,14 +57,12 @@ def exec(self, state, d): path, create_and_select_workspace) - logging.info('APPLY : CDKTF : %s : %r', - path, - workflow['cdktf']) - - env['TERRATEAM_TERRAFORM_VERSION'] = work_exec.determine_tf_version( + work_exec.set_tf_version_env( + env, + state.repo_config, + workflow['engine'], state.working_dir, - os.path.join(state.working_dir, path), - workflow['terraform_version']) + os.path.join(state.working_dir, path)) state = state._replace(env=env) diff --git a/terrat_runner/work_exec.py b/terrat_runner/work_exec.py index 6a1a050a..097a0ba7 100644 --- a/terrat_runner/work_exec.py +++ b/terrat_runner/work_exec.py @@ -41,6 +41,45 @@ def _read(fname): return workflow_version +def set_tf_version_env(env, repo_config, engine, repo_root, working_dir): + TF_CMD_ENV_NAME = 'TERRATEAM_TF_CMD' + TOFU_ENV_NAME = 'TOFUENV_TOFU_DEFAULT_VERSION' + TERRAFORM_ENV_NAME = 'TERRATEAM_TERRAFORM_VERSION' + + if engine['name'] == 'tofu': + env[TF_CMD_ENV_NAME] = 'tofu' + version = engine.get('version') + if version: + env[TOFU_ENV_NAME] = version + elif engine['name'] in ['cdktf', 'terragrunt']: + # If cdktf or terragrunt, set the appropriate terraform/tofu version if + # it exists. + if engine['tf_cmd'] == 'tofu': + env[TF_CMD_ENV_NAME] = 'tofu' + version_env_name = TOFU_ENV_NAME + version = engine.get('tf_version') + if version: + env[version_env_name] = version + else: + env[TF_CMD_ENV_NAME] = 'terraform' + env[TERRAFORM_ENV_NAME] = engine.get('tf_version', + rc.get_default_tf_version(repo_config)) + else: + env[TF_CMD_ENV_NAME] = 'terraform' + version = engine.get('version') + + if version: + env[TERRAFORM_ENV_NAME] = determine_tf_version( + repo_root, + working_dir, + version) + else: + env[TERRAFORM_ENV_NAME] = determine_tf_version( + repo_root, + working_dir, + rc.get_default_tf_version(repo_config)) + + def _store_results(work_token, api_base_url, results): res = requests_retry.put(api_base_url + '/v1/work-manifests/' + work_token, json=results) @@ -54,10 +93,12 @@ def _run(state, exec_cb): # Using state.working_dir twice as a bit of a hack because # determine_tf_version expects the directory that we are running the command # in as an option as well, but at this point there is none. - env['TERRATEAM_TERRAFORM_VERSION'] = determine_tf_version( - state.working_dir, + set_tf_version_env( + env, + state.repo_config, + rc.get_engine(state.repo_config), state.working_dir, - rc.get_default_tf_version(state.repo_config)) + state.working_dir) env['TERRATEAM_TMPDIR'] = state.tmpdir state = state._replace(env=env) diff --git a/terrat_runner/work_plan.py b/terrat_runner/work_plan.py index fefcaa94..a782de86 100644 --- a/terrat_runner/work_plan.py +++ b/terrat_runner/work_plan.py @@ -69,14 +69,12 @@ def exec(self, state, d): path, create_and_select_workspace) - logging.info('PLAN : CDKTF : %s : %r', - path, - workflow['cdktf']) - - env['TERRATEAM_TERRAFORM_VERSION'] = work_exec.determine_tf_version( + work_exec.set_tf_version_env( + env, + state.repo_config, + workflow['engine'], state.working_dir, - os.path.join(state.working_dir, path), - workflow['terraform_version']) + os.path.join(state.working_dir, path)) state = state._replace(env=env) diff --git a/terrat_runner/work_unsafe_apply.py b/terrat_runner/work_unsafe_apply.py index fdc0d41f..df0a0269 100644 --- a/terrat_runner/work_unsafe_apply.py +++ b/terrat_runner/work_unsafe_apply.py @@ -68,19 +68,17 @@ def exec(self, state, d): path, create_and_select_workspace) - logging.info('UNSAFE_APPLY : CDKTF : %s : %r', - path, - workflow['cdktf']) - if workflow_idx is None: workflow = rc.get_default_workflow(state.repo_config) else: workflow = rc.get_workflow(state.repo_config, workflow_idx) - env['TERRATEAM_TERRAFORM_VERSION'] = work_exec.determine_tf_version( + work_exec.set_tf_version_env( + env, + state.repo_config, + workflow['engine'], state.working_dir, - os.path.join(state.working_dir, path), - workflow['terraform_version']) + os.path.join(state.working_dir, path)) state = state._replace(env=env) diff --git a/terrat_runner/workflow_step_init.py b/terrat_runner/workflow_step_init.py index 709a5d8b..498d46a2 100644 --- a/terrat_runner/workflow_step_init.py +++ b/terrat_runner/workflow_step_init.py @@ -33,28 +33,25 @@ def run(state, config): if result.failed: return result - cdktf = state.workflow['cdktf'] - terragrunt = state.workflow['terragrunt'] create_and_select_workspace = repo_config.get_create_and_select_workspace(state.repo_config, state.path) logging.info( ('WORKFLOW_STEP_INIT : ' 'CREATE_AND_SELECT_WORKSPACE : %s : ' - 'terragrunt=%r : cdktf=%r : create_and_select_workspace=%r'), + 'engine=%s : create_and_select_workspace=%r'), result.state.path, - terragrunt, - cdktf, + state.workflow['engine']['name'], create_and_select_workspace) - if not terragrunt and not cdktf and create_and_select_workspace: + if state.workflow['engine']['name'] in ['terraform', 'tofu'] and create_and_select_workspace: config = original_config.copy() - config['cmd'] = ['terraform', 'workspace', 'select', state.workspace] + config['cmd'] = ['${TERRATEAM_TF_CMD}', 'workspace', 'select', state.workspace] proc = cmd.run(state, config) if proc.returncode != 0: # TODO: Is this safe?! - config['cmd'] = ['terraform', 'workspace', 'new', state.workspace] + config['cmd'] = ['${TERRATEAM_TF_CMD}', 'workspace', 'new', state.workspace] proc = cmd.run(state, config) return result._replace(failed=proc.returncode != 0) diff --git a/terrat_runner/workflow_step_terraform.py b/terrat_runner/workflow_step_terraform.py index e5e70b00..2e9f9b15 100644 --- a/terrat_runner/workflow_step_terraform.py +++ b/terrat_runner/workflow_step_terraform.py @@ -6,6 +6,10 @@ import workflow +TERRAFORM_BIN = 'terraform' +TOFU_BIN = 'tofu' + + class SynthError(Exception): def __init__(self, msg): self.msg = msg @@ -38,16 +42,15 @@ def get_cdktf_working_dir(state): def run_terraform(state, config): args = config['args'] - terraform_bin_path = os.path.join('/usr', 'local', 'bin', 'terraform') env = config.get('env', {}) - if state.workflow['terragrunt']: + if state.workflow['engine']['name'] == 'terragrunt': cmd = ['terragrunt'] env = env.copy() - env['TERRAGRUNT_TFPATH'] = terraform_bin_path + env['TERRAGRUNT_TFPATH'] = state.env['TERRATEAM_TF_CMD'] else: - cmd = [terraform_bin_path] + cmd = ['${TERRATEAM_TF_CMD}'] extra_args = config.get('extra_args', []) config = { @@ -72,12 +75,12 @@ def run(state, config): # directory and run Terraform and then switch back the directory to the # directory with the code, so the experience is seamless to the user. try: - if state.workflow['cdktf'] and args[0] == 'init': + if state.workflow['engine']['name'] == 'cdktf' and args[0] == 'init': synth_cdktf(state, config) cdktf_working_dir = get_cdktf_working_dir(state) state = state._replace(working_dir=cdktf_working_dir) return update_result_working_dir(run_terraform(state, config), working_dir) - elif state.workflow['cdktf']: + elif state.workflow['engine']['name'] == 'cdktf': cdktf_working_dir = get_cdktf_working_dir(state) state = state._replace(working_dir=cdktf_working_dir) return update_result_working_dir(run_terraform(state, config), working_dir)