Merge pull request #244 from kbalk/add-pytest-targets

Add targets for testing Terraform modules using pytest, tftest

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.14.1
+current_version = 0.15.0
 commit = True
 message = Bumps version to {new_version}
 tag = False

.gitignore

@@ -13,3 +13,6 @@
 
 # ec (editorconfig-checker)
 .git/
+
+# python caching when pytest is used
+__pycache__

CHANGELOG.md

@@ -4,6 +4,26 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/).
 
+### 0.15.0
+
+**Released**: 2021.07.15
+
+**Commit Delta**: [Change from 0.14.1 release](https://github.com/plus3it/tardigrade-ci/compare/0.14.1..0.15.0)
+
+**Summary**:
+
+*   Adds new Makefile targets for the purpose of testing Terraform modules 
+    using Terraform and the mock AWS stacks LocalStack and/or moto.  The new
+    readme file `INTEGRATION_TESTING.md` describes the expected testing
+    environment and the associated Makefile targets.
+
+*   Updates tool versions:
+    * docker-compose 1.29.2
+
+*   Adds install targets for the tools:
+    * pytest-compose 6.2.4
+    * tftest 1.6.0
+
 ### 0.14.1
 
 **Released**: 2021.07.9

INTEGRATION_TESTING.md

@@ -0,0 +1,117 @@
+# Tardigrade-ci Integration Testing
+
+The tardigrade-ci `Makefile` provides targets to facilitate automated 
+integration testing of terraform modules through the use of a mock AWS stack.
+
+This document describes the integration-specific `Makefile` targets and
+the environment variables to customize those targets.  In addition,
+the steps to try out a test are described as are the potential CI/CD
+changes to incorporate automated integration testing.
+
+## Integration test-specific targets and environment variables
+
+| Target name      | Description |
+| ---------------- | ------------------------------------------ |
+| mockstack/pytest | From within a Docker container, invoke `pytest` to execute the integration tests. |
+| mockstack/up     | Start up a Docker container running the mock AWS stack. |
+| mockstack/down   | Bring down the Docker container running the mock AWS stack. |
+| mockstack/clean  | Bring down the Docker container running the mock AWS stack, then remove the docker image. |
+| terraform/pytest | Invoke `pytest` to execute the integration tests. The mock AWS stack must be started before using this `Makefile` target. |
+
+Defaults:
+
+* `LocalStack` is used for the mock AWS stack, with moto serving ports for 
+services not yet suported by `LocalStack`.
+* The Terraform modules used for the integration tests are expected to
+be located in the directory `tests` off the repo\'s root directory.
+
+### Environment variables
+
+| Environment variable             | Default value |
+| -------------------------------- | --------------------------------------- |
+| INTEGRATION_TEST_BASE_IMAGE_NAME | $(basename $PWD)-integration-test |
+| TERRAFORM_PYTEST_ARGS            | |
+| TERRAFORM_PYTEST_DIR             | $PWD/tests/terraform/pytest |
+
+### Arguments to the automation script
+
+These are values that can be specified through the environment variable
+TERRAFORM_PYTEST_ARGS.
+
+| Command line option | Description |
+| ------------------- | ----------------------------------------------- |
+| --nomock            | Use AWS, not mocked AWS services |
+| --alternate-profile | Configure an alternate profile in addition to default profile |
+| --tf-dir=TF_DIR     | Directory of Terraform files under test; default: './tests' |
+
+## Executing a Terraform test
+
+The tardigrade-ci `Makefile` expects the test Terraform modules to be under
+the repo\s `tests` directory.  There can be multiple sets of tests, each
+under their own `tests` subdirectory.
+
+If a test requires an initial test setup, then those Terraform "test setup"
+files should be placed in the directory `prereq` under that test\'s
+subdirectory.  For example:
+
+```
+.
+├── tests
+│   ├── create_all
+│   │   ├── main.tf
+│   │   └── prereq
+│   │       └── main.tf
+│   ├── create_groups
+│   │   ├── main.tf
+
+...
+```
+
+To verify that a Terraform test will work, bring up the default AWS mock
+stack (`LocalStack`) first, then execute the test:
+
+```bash
+make mockstack/up
+make terraform/pytest
+
+# To execute a specific set of tests, use the "-k" option and a string
+# that will pattern match on the desired subdirectory name.  The "-k"
+# option also allows booleans, e.g., "not" or "or".
+#
+# The following will match on the subdirectory "create_groups".
+make terraform/pytest TERRAFORM_PYTEST_ARGS="-k groups"
+
+# When testing is complete:
+make mockstack/clean
+```
+
+## Potential CI/CD changes 
+
+These are suggested changes and may not apply to all repos:
+
+1.  If the repo currently does not contain or use Python scripts,
+    update `.gitignore` to add:
+
+```bash
+# Caching directory created by pytest.
+tests/__pycache__/
+```
+
+2.  For a Travis workflow, logic similar to the following would
+    need to be added to the `.travis.yml` file:
+
+```bash
+stage: test
+
+name: Terraform Integration Tests
+
+language: python
+
+python:
+  - "3.8"
+
+install: make mockstack/up
+
+script: make mockstack/pytest
+after_script: make mockstack/clean
+```

Makefile

@@ -166,10 +166,22 @@ install/pip_pkg_with_no_cli/%: | guard/env/PYPI_PKG_NAME
 	@ echo "[$@]: Installing $*..."
 	$(PIP) install $(PYPI_PKG_NAME)
 
+docker-compose/install: DOCKER_COMPOSE_VERSION ?= $(call match_pattern_in_file,$(TARDIGRADE_CI_PYTHON_TOOLS),'docker-compose==','$(SEMVER_PATTERN)')
+docker-compose/install:
+	@ $(MAKE) install/pip/$(@D) PYPI_PKG_NAME='$(@D)==$(DOCKER_COMPOSE_VERSION)'
+
 black/install: BLACK_VERSION ?= $(call match_pattern_in_file,$(TARDIGRADE_CI_PYTHON_TOOLS),'black==','[0-9]+\.[0-9]+(b[0-9]+)?')
 black/install:
 	@ $(MAKE) install/pip/$(@D) PYPI_PKG_NAME='$(@D)==$(BLACK_VERSION)'
 
+tftest/install: TFTEST_VERSION ?= $(call match_pattern_in_file,$(TARDIGRADE_CI_PYTHON_TOOLS),'tftest==','$(SEMVER_PATTERN)')
+tftest/install:
+	@ $(MAKE) install/pip_pkg_with_no_cli/$(@D) PYPI_PKG_NAME='$(@D)==$(TFTEST_VERSION)'
+
+pytest/install: PYTEST_VERSION ?= $(call match_pattern_in_file,$(TARDIGRADE_CI_PYTHON_TOOLS),'^pytest==','$(SEMVER_PATTERN)')
+pytest/install:
+	@ $(MAKE) install/pip/$(@D) PYPI_PKG_NAME='$(@D)==$(PYTEST_VERSION)'
+
 pylint/install: PYLINT_VERSION ?= $(call match_pattern_in_file,$(TARDIGRADE_CI_PYTHON_TOOLS),'pylint==','$(SEMVER_PATTERN)')
 pylint/install:
 	@ $(MAKE) install/pip/$(@D) PYPI_PKG_NAME='$(@D)==$(PYLINT_VERSION)'
@@ -408,8 +420,38 @@ terratest/test:
 	cd $(TERRAFORM_TEST_DIR) && go test -count=1 -timeout $(TIMEOUT)
 	@ echo "[$@]: Completed successfully!"
 
+TERRAFORM_PYTEST_ARGS ?=
+TERRAFORM_PYTEST_DIR ?= $(TARDIGRADE_CI_PATH)/tests/terraform_pytest
+terraform/pytest: | guard/program/terraform guard/program/pytest guard/python_pkg/tftest
+	@ echo "[$@] Starting Terraform integration test"
+	pytest -v $(TERRAFORM_PYTEST_DIR) $(TERRAFORM_PYTEST_ARGS)
+	@ echo "[$@]: Completed successfully!"
+
+.PHONY: mockstack/pytest mockstack/up mockstack/down mockstack/clean
+INTEGRATION_TEST_BASE_IMAGE_NAME ?= $(shell basename $(PWD))-integration-test
+mockstack/%: MOCKSTACK ?= localstack
+mockstack/pytest:
+	@ echo "[$@] Running Terraform tests against LocalStack"
+	DOCKER_RUN_FLAGS="--network terraform_pytest_default --rm -e MOCKSTACK_HOST=$(MOCKSTACK) -e TERRAFORM_PYTEST_ARGS=$(TERRAFORM_PYTEST_ARGS)" \
+		IMAGE_NAME=$(INTEGRATION_TEST_BASE_IMAGE_NAME):latest \
+		$(MAKE) docker/run target=terraform/pytest
+	@ echo "[$@]: Completed successfully!"
+
+mockstack/up:
+	@ echo "[$@] Starting LocalStack container"
+	TERRAFORM_PYTEST_DIR=$(TERRAFORM_PYTEST_DIR) docker-compose -f $(TERRAFORM_PYTEST_DIR)/docker-compose-localstack.yml up --detach
+
+mockstack/down:
+	@ echo "[$@] Stopping LocalStack container"
+	TERRAFORM_PYTEST_DIR=$(TERRAFORM_PYTEST_DIR) docker-compose -f $(TERRAFORM_PYTEST_DIR)/docker-compose-localstack.yml down
+
+mockstack/clean: | mockstack/down
+	@ echo "[$@] Stopping and removing LocalStack image"
+	set +o pipefail; docker images | grep $(INTEGRATION_TEST_BASE_IMAGE_NAME) | \
+		awk '{print $$1 ":" $$2}' | xargs -r docker rmi
+
 ## Runs terraform tests in the tests directory
-test: terratest/test
+test: terraform/pytest
 
 bats/install: BATS_VERSION ?= tags/v$(call match_pattern_in_file,$(TARDIGRADE_CI_DOCKERFILE_TOOLS),'bats/bats','$(SEMVER_PATTERN)')
 bats/install:
@@ -431,7 +473,8 @@ project/validate:
 	@ echo "[$@]: Target test folder validation successful"
 
 install: terragrunt/install terraform/install shellcheck/install terraform-docs/install
-install: bats/install black/install pylint/install pylint-pytest/install pydocstyle/install
+install: bats/install black/install pylint/install pylint-pytest/install pydocstyle/install pytest/install tftest/install
 install: ec/install yamllint/install cfn-lint/install yq/install bumpversion/install jq/install
+install: docker-compose/install
 
 lint: project/validate terraform/lint sh/lint json/lint docs/lint python/lint ec/lint cfn/lint hcl/lint

README.md

@@ -35,7 +35,7 @@ Available targets:
   sh/lint                             Lints bash script files
   terraform/format                    Formats terraform files
   terraform/lint                      Lints terraform files
-  test                                Runs terraform tests in the tests directory
+  test                                Runs terraform tests under the tests directory
   yaml/lint                           Lints YAML files
 ```
 

requirements.txt

@@ -9,10 +9,16 @@ bump2version==1.0.1
 
 cfn-lint==0.52.0
 
+docker-compose==1.29.2
+
 pydocstyle==6.1.1
 
 pylint==2.9.3
 
 pylint-pytest==1.1.2
 
+pytest==6.2.4
+
+tftest==1.6.0
+
 yamllint==1.26.1

tests/make/terraform_pytest_failure.bats

@@ -0,0 +1,43 @@
+#!/usr/bin/env bats
+
+TEST_DIR="$(pwd)/terraform_pytest_failure"
+
+function setup() {
+rm -rf "$TEST_DIR"
+working_dirs=("$TEST_DIR/test" "$TEST_DIR/test/prereq")
+
+# Bad prereq Terraform file
+mkdir -p "$TEST_DIR/test/prereq"
+cat > "$TEST_DIR/test/prereq/main.tf" <<-EOF
+variable "foo" {
+  default = "bar"
+}
+
+output "baz" {
+  # this variable does not exist
+  value = var.bar
+}
+EOF
+
+# Good Terraform file
+mkdir -p "$TEST_DIR/test"
+cat > "$TEST_DIR/test/main.tf" <<-EOF
+variable "foo" {
+  default = "bar"
+}
+
+output "baz" {
+  value = foo.bar
+}
+EOF
+}
+
+@test "test: terraform pytest failure" {
+  run make terraform/pytest TERRAFORM_PYTEST_DIR=../terraform_pytest TERRAFORM_PYTEST_ARGS="--tf-dir $TEST_DIR"
+  [ "$status" -eq 2 ]
+  [[ "$output" == *"FAILED ../terraform_pytest/test_terraform_install.py::test_modules[test]"* ]]
+}
+
+function teardown() {
+  rm -rf "$TEST_DIR"
+}

tests/make/terraform_pytest_success.bats

@@ -0,0 +1,32 @@
+#!/usr/bin/env bats
+
+TEST_DIR="$(pwd)/terraform_pytest_success"
+
+function setup() {
+rm -rf "$TEST_DIR"
+working_dirs=("$TEST_DIR/test" "$TEST_DIR/test/prereq")
+for working_dir in "${working_dirs[@]}"
+do
+
+  mkdir -p "$working_dir"
+  cat > "$working_dir/main.tf" <<-EOF
+variable "foo" {
+  default = "bar"
+}
+
+output "baz" {
+  value = var.foo
+}
+EOF
+done
+}
+
+@test "test: terraform pytest success" {
+  run make terraform/pytest TERRAFORM_PYTEST_DIR=../terraform_pytest TERRAFORM_PYTEST_ARGS="--tf-dir $TEST_DIR"
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"test_modules[test] PASSED [100%]"* ]]
+}
+
+function teardown() {
+  rm -rf "$TEST_DIR"
+}

tests/make/terraform_test_failure.bats

@@ -16,7 +16,7 @@ EOF
 }
 
 @test "test: terraform test failure" {
-  run make TERRAFORM_TEST_DIR="../terraform" test
+  run make TERRAFORM_TEST_DIR="../terraform" terratest/test
   [ "$status" -eq 2 ]
 }
 

tests/make/terraform_test_success.bats

@@ -16,7 +16,7 @@ EOF
 
 
 @test "test: terraform test success" {
-  run make TERRAFORM_TEST_DIR="../terraform" test
+  run make TERRAFORM_TEST_DIR="../terraform" terratest/test
   [ "$status" -eq 0 ]
 }
 

tests/terraform_pytest/aws.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = "us-east-1"
+}