Merge pull request #243 from terrateamio/main

Release v1

Dockerfile.base

@@ -49,6 +49,24 @@ RUN mkdir /tmp/awscli \
 ENV CHECKOV_VERSION=2.5.10
 RUN pip3 install checkov==${CHECKOV_VERSION}
 
+# Temporarily pull from our branch until changes are merged back into a release
+#
+# ENV TOFUENV_VERSION v1.0.3
+# RUN curl -fsSL -o /tmp/tofuenv.zip \
+#    "https://github.com/terrateamio/tofuenv/archive/refs/tags/v${TOFUENV_VERSION}.zip" \
+#    && cd /tmp/ \
+#    && unzip /tmp/tofuenv.zip \
+#    && mv /tmp/tofuenv-${TOFUENV_VERSION} /usr/local/lib/tofuenv \
+#    && echo "latest" > /usr/local/lib/tofuenv/version
+
+
+ENV TOFUENV_DEFAULT_VERSION latest
+RUN curl -fsSL -o /tmp/tofuenv.zip \
+    "https://github.com/terrateamio/tofuenv/archive/refs/heads/pro-411-add-opentofu-support.zip" \
+    && cd /tmp/ \
+    && unzip /tmp/tofuenv.zip \
+    && mv /tmp/tofuenv-pro-411-add-opentofu-support /usr/local/lib/tofuenv
+
 COPY ./bin/ /usr/local/bin
 ENV DEFAULT_TERRAFORM_VERSION 1.5.7
 COPY ./install-terraform-version /install-terraform-version

bin/tofu

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+set -x
+
+# Perform the install with an flock, this ensure that if there are multipel
+# parallel runs, then only one can happen at a time.  We don't have a guarantee
+# that tofuenv is parallel-safe.
+flock /tmp/tofuenv.install /usr/local/lib/tofuenv/bin/tofuenv install \
+    || flock /tmp/tofuenv.install /usr/local/lib/tofuenv/bin/tofuenv install \
+    || flock /tmp/tofuenv.install /usr/local/lib/tofuenv/bin/tofuenv install \
+
+exec /usr/local/lib/tofuenv/bin/tofu "$@"

terrat_runner/main.py

@@ -45,13 +45,12 @@ def perform_merge(working_dir, base_ref):
 
 
 def maybe_setup_cdktf(rc, work_manifest, env):
-    # Determine if any workflows use cdktf and only install it if it is
-    # required.
+    # Determine if any engine uses cdktf and only install it if it is required.
     cdktf_used = False
     for d in work_manifest['changed_dirspaces']:
         if 'workflow' in d:
             workflow = repo_config.get_workflow(rc, d['workflow'])
-            cdktf_used = cdktf_used or workflow['cdktf']
+            cdktf_used = cdktf_used or workflow['engine']['name'] == 'cdktf'
 
     if cdktf_used:
         subprocess.check_call(['/cdktf-setup.sh'])

terrat_runner/repo_config.py

@@ -3,6 +3,7 @@
 
 
 def _get(d, k, default):
+    """Return [default] if it is not set or set to None."""
     v = d.get(k, default)
     if v is None:
         return default
@@ -63,24 +64,87 @@ def get_apply_workflow(repo_config, idx):
     return repo_config['workflows'][idx].get('apply', _default_apply_workflow())
 
 
+def get_engine(repo_config):
+    # Get the engine config.  If one is already there, we will take it verbatim,
+    # however if we need to construct a default one, we specify terraform and we
+    # also want to use the [default_tf_version] if present.  This is to maintain
+    # compatibility with existing configurations.
+    if 'engine' in repo_config:
+        engine = repo_config['engine'].copy()
+        if engine['name'] in ['cdktf', 'terragrunt'] and 'tf_cmd' not in engine:
+            engine['tf_cmd'] = 'terraform'
+
+        return engine
+    else:
+        return {
+            'name': 'terraform',
+            'version': repo_config.get('default_tf_version')
+        }
+
+
 def get_workflow(repo_config, idx):
     workflow = repo_config['workflows'][idx]
-    return {
+    cfg = {
         'apply': workflow.get('apply', _default_apply_workflow()),
-        'cdktf': workflow.get('cdktf', False),
         'plan': workflow.get('plan', _default_plan_workflow()),
-        'terraform_version': workflow.get('terraform_version', get_default_tf_version(repo_config)),
-        'terragrunt': workflow.get('terragrunt', False),
     }
 
+    default_engine = get_engine(repo_config)
+    engine = _get(workflow, 'engine', {}).copy()
+
+    # In order to maintain backwards compatibility, we need to do some work to
+    # transform an existing workflow configuration to one with an engine.
+    # Additionally, we want to make future lookups easy so we fill in the
+    # configurations that would be inferred.
+    if 'engine' not in workflow:
+        # If no engine is specified, convert any legacy configuration to the
+        # engine config.  Fill in the minimal configuration and the rest will be
+        # done next.
+        if workflow.get('terragrunt'):
+            engine = {
+                'name': 'terragrunt',
+            }
+        elif workflow.get('cdktf'):
+            engine = {
+                'name': 'cdktf',
+            }
+        elif workflow.get('terraform_version'):
+            engine = {
+                'name': 'terraform',
+                'version': workflow['terraform_version']
+            }
+        else:
+            engine = default_engine.copy()
+
+    if default_engine['name'] == 'tofu':
+        default_tf_cmd = 'tofu'
+        default_tf_version = default_engine.get('version')
+    else:
+        default_tf_cmd = 'terraform'
+        default_tf_version = None
+
+    if engine['name'] in ['terragrunt', 'cdktf']:
+        engine['tf_cmd'] = _get(engine, 'tf_cmd',  default_tf_cmd)
+        if engine['tf_cmd'] == 'terraform':
+            engine['tf_version'] = _get(engine, 'tf_version', get_default_tf_version(repo_config))
+        else:
+            engine['tf_version'] = _get(engine, 'tf_version', default_tf_version)
+    elif engine['name'] == 'terraform':
+        engine['version'] = _get(engine, 'version', get_default_tf_version(repo_config))
+    elif engine['name'] == 'tofu':
+        engine['version'] = _get(engine, 'version', default_tf_version)
+    else:
+        raise Exception('Unknown engine')
+
+    cfg['engine'] = engine
+    return cfg
+
 
 def get_default_workflow(repo_config):
     return {
         'apply': _default_apply_workflow(),
-        'cdktf': False,
         'plan': _default_plan_workflow(),
-        'terraform_version': get_default_tf_version(repo_config),
-        'terragrunt': False,
+        'engine': get_engine(repo_config)
     }
 
 

terrat_runner/work_apply.py

@@ -57,14 +57,12 @@ def exec(self, state, d):
                          path,
                          create_and_select_workspace)
 
-            logging.info('APPLY : CDKTF : %s : %r',
-                         path,
-                         workflow['cdktf'])
-
-            env['TERRATEAM_TERRAFORM_VERSION'] = work_exec.determine_tf_version(
+            work_exec.set_tf_version_env(
+                env,
+                state.repo_config,
+                workflow['engine'],
                 state.working_dir,
-                os.path.join(state.working_dir, path),
-                workflow['terraform_version'])
+                os.path.join(state.working_dir, path))
 
             state = state._replace(env=env)
 

terrat_runner/work_exec.py

@@ -41,6 +41,45 @@ def _read(fname):
         return workflow_version
 
 
+def set_tf_version_env(env, repo_config, engine, repo_root, working_dir):
+    TF_CMD_ENV_NAME = 'TERRATEAM_TF_CMD'
+    TOFU_ENV_NAME = 'TOFUENV_TOFU_DEFAULT_VERSION'
+    TERRAFORM_ENV_NAME = 'TERRATEAM_TERRAFORM_VERSION'
+
+    if engine['name'] == 'tofu':
+        env[TF_CMD_ENV_NAME] = 'tofu'
+        version = engine.get('version')
+        if version:
+            env[TOFU_ENV_NAME] = version
+    elif engine['name'] in ['cdktf', 'terragrunt']:
+        # If cdktf or terragrunt, set the appropriate terraform/tofu version if
+        # it exists.
+        if engine['tf_cmd'] == 'tofu':
+            env[TF_CMD_ENV_NAME] = 'tofu'
+            version_env_name = TOFU_ENV_NAME
+            version = engine.get('tf_version')
+            if version:
+                env[version_env_name] = version
+        else:
+            env[TF_CMD_ENV_NAME] = 'terraform'
+            env[TERRAFORM_ENV_NAME] = engine.get('tf_version',
+                                                 rc.get_default_tf_version(repo_config))
+    else:
+        env[TF_CMD_ENV_NAME] = 'terraform'
+        version = engine.get('version')
+
+        if version:
+            env[TERRAFORM_ENV_NAME] = determine_tf_version(
+                repo_root,
+                working_dir,
+                version)
+        else:
+            env[TERRAFORM_ENV_NAME] = determine_tf_version(
+                repo_root,
+                working_dir,
+                rc.get_default_tf_version(repo_config))
+
+
 def _store_results(work_token, api_base_url, results):
     res = requests_retry.put(api_base_url + '/v1/work-manifests/' + work_token,
                              json=results)
@@ -54,10 +93,12 @@ def _run(state, exec_cb):
     # Using state.working_dir twice as a bit of a hack because
     # determine_tf_version expects the directory that we are running the command
     # in as an option as well, but at this point there is none.
-    env['TERRATEAM_TERRAFORM_VERSION'] = determine_tf_version(
-        state.working_dir,
+    set_tf_version_env(
+        env,
+        state.repo_config,
+        rc.get_engine(state.repo_config),
         state.working_dir,
-        rc.get_default_tf_version(state.repo_config))
+        state.working_dir)
 
     env['TERRATEAM_TMPDIR'] = state.tmpdir
     state = state._replace(env=env)

terrat_runner/work_plan.py

@@ -69,14 +69,12 @@ def exec(self, state, d):
                          path,
                          create_and_select_workspace)
 
-            logging.info('PLAN : CDKTF : %s : %r',
-                         path,
-                         workflow['cdktf'])
-
-            env['TERRATEAM_TERRAFORM_VERSION'] = work_exec.determine_tf_version(
+            work_exec.set_tf_version_env(
+                env,
+                state.repo_config,
+                workflow['engine'],
                 state.working_dir,
-                os.path.join(state.working_dir, path),
-                workflow['terraform_version'])
+                os.path.join(state.working_dir, path))
 
             state = state._replace(env=env)
 

terrat_runner/work_unsafe_apply.py

@@ -68,19 +68,17 @@ def exec(self, state, d):
                          path,
                          create_and_select_workspace)
 
-            logging.info('UNSAFE_APPLY : CDKTF : %s : %r',
-                         path,
-                         workflow['cdktf'])
-
             if workflow_idx is None:
                 workflow = rc.get_default_workflow(state.repo_config)
             else:
                 workflow = rc.get_workflow(state.repo_config, workflow_idx)
 
-            env['TERRATEAM_TERRAFORM_VERSION'] = work_exec.determine_tf_version(
+            work_exec.set_tf_version_env(
+                env,
+                state.repo_config,
+                workflow['engine'],
                 state.working_dir,
-                os.path.join(state.working_dir, path),
-                workflow['terraform_version'])
+                os.path.join(state.working_dir, path))
 
             state = state._replace(env=env)
 

terrat_runner/workflow_step_init.py

@@ -33,28 +33,25 @@ def run(state, config):
     if result.failed:
         return result
 
-    cdktf = state.workflow['cdktf']
-    terragrunt = state.workflow['terragrunt']
     create_and_select_workspace = repo_config.get_create_and_select_workspace(state.repo_config,
                                                                               state.path)
 
     logging.info(
         ('WORKFLOW_STEP_INIT : '
          'CREATE_AND_SELECT_WORKSPACE : %s : '
-         'terragrunt=%r : cdktf=%r : create_and_select_workspace=%r'),
+         'engine=%s : create_and_select_workspace=%r'),
         result.state.path,
-        terragrunt,
-        cdktf,
+        state.workflow['engine']['name'],
         create_and_select_workspace)
 
-    if not terragrunt and not cdktf and create_and_select_workspace:
+    if state.workflow['engine']['name'] in ['terraform', 'tofu'] and create_and_select_workspace:
         config = original_config.copy()
-        config['cmd'] = ['terraform', 'workspace', 'select', state.workspace]
+        config['cmd'] = ['${TERRATEAM_TF_CMD}', 'workspace', 'select', state.workspace]
         proc = cmd.run(state, config)
 
         if proc.returncode != 0:
             # TODO: Is this safe?!
-            config['cmd'] = ['terraform', 'workspace', 'new', state.workspace]
+            config['cmd'] = ['${TERRATEAM_TF_CMD}', 'workspace', 'new', state.workspace]
             proc = cmd.run(state, config)
 
             return result._replace(failed=proc.returncode != 0)

terrat_runner/workflow_step_terraform.py

@@ -6,6 +6,10 @@
 import workflow
 
 
+TERRAFORM_BIN = 'terraform'
+TOFU_BIN = 'tofu'
+
+
 class SynthError(Exception):
     def __init__(self, msg):
         self.msg = msg
@@ -38,16 +42,15 @@ def get_cdktf_working_dir(state):
 
 def run_terraform(state, config):
     args = config['args']
-    terraform_bin_path = os.path.join('/usr', 'local', 'bin', 'terraform')
 
     env = config.get('env', {})
 
-    if state.workflow['terragrunt']:
+    if state.workflow['engine']['name'] == 'terragrunt':
         cmd = ['terragrunt']
         env = env.copy()
-        env['TERRAGRUNT_TFPATH'] = terraform_bin_path
+        env['TERRAGRUNT_TFPATH'] = state.env['TERRATEAM_TF_CMD']
     else:
-        cmd = [terraform_bin_path]
+        cmd = ['${TERRATEAM_TF_CMD}']
 
     extra_args = config.get('extra_args', [])
     config = {
@@ -72,12 +75,12 @@ def run(state, config):
     # directory and run Terraform and then switch back the directory to the
     # directory with the code, so the experience is seamless to the user.
     try:
-        if state.workflow['cdktf'] and args[0] == 'init':
+        if state.workflow['engine']['name'] == 'cdktf' and args[0] == 'init':
             synth_cdktf(state, config)
             cdktf_working_dir = get_cdktf_working_dir(state)
             state = state._replace(working_dir=cdktf_working_dir)
             return update_result_working_dir(run_terraform(state, config), working_dir)
-        elif state.workflow['cdktf']:
+        elif state.workflow['engine']['name'] == 'cdktf':
             cdktf_working_dir = get_cdktf_working_dir(state)
             state = state._replace(working_dir=cdktf_working_dir)
             return update_result_working_dir(run_terraform(state, config), working_dir)