From de08460ef6778ebd1363e490aa70e9e845654892 Mon Sep 17 00:00:00 2001
From: Jeff McCune <jeff@openinfrastructure.co>
Date: Wed, 14 Jun 2017 16:37:20 -0400
Subject: [PATCH] Add Gitlab CI jobs

Without this patch the control repository does not contain any scripts
to execute CI jobs for Puppet code deployment.  This patch addresses the
problem by adding a Gitlab CI job configuration with three jobs defined
across two stages.  The test stage executes first with a Syntax and a
Lint job executing in parallel.  If an environment branch has been
pushed, one matching `developemnt`, `testing`, `production` or starting
with the prefix `playground`, the deploy stage is executed.

The deploy stage requires a Gitlab secret environment variable,
PUPPET_TOKEN.  This environment variable is the puppet access token
configured with Code Manager.

The test stage runs the following syntax checks:

 * yaml files (*.yml, *.yaml)
 * json files (*.json)
 * bash scripts (*.sh)
 * ruby scripts (*.rb)
 * puppet manifests (*.pp)
 * erb files (*.erb)

The behavior of the checks are to check only files modified relative to
a base branch, defaulting to `production`.  This avoids running syntax
checks against files which have not been modified by the merge request.
The check uses `git diff --name-status` to identify changed files across
multiple commits in a topic branch.

The lint checks rely on bundler and the Gemfile to install numerous lint
checks.

Gem libraries are installed in a per-job location in $HOME.  This is an
intentional compromise between installing into a system GEM_PATH
element, which would create library conflicts with other jobs, and the
job workspace, which would cause gem libraries to be installed from the
network on each job invocation.
---
 .gitlab-ci.yml           | 39 +++++++++++++++++++++
 Gemfile                  | 19 ++++++++++
 scripts/lint_check.sh    | 72 +++++++++++++++++++++++++++++++++++++
 scripts/puppet_deploy.sh | 60 +++++++++++++++++++++++++++++++
 scripts/syntax_check.sh  | 76 ++++++++++++++++++++++++++++++++++++++++
 5 files changed, 266 insertions(+)
 create mode 100644 .gitlab-ci.yml
 create mode 100644 Gemfile
 create mode 100755 scripts/lint_check.sh
 create mode 100755 scripts/puppet_deploy.sh
 create mode 100755 scripts/syntax_check.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..fc91a5609
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,39 @@
+---
+stages:
+  - test
+  - deploy
+
+# Jobs are defined below
+Syntax Check:
+  tags:
+    - puppet agent
+  stage: test
+  script:
+    - scripts/syntax_check.sh
+
+Lint Check:
+  tags:
+    - puppet agent
+  stage: test
+  script:
+    - scripts/lint_check.sh
+
+# Unless TCP port 8170 is open to the PE master, this job must execute on the
+# PE Monolithic Master itself.
+Deploy to Puppet:
+  tags:
+    - puppet monolithic master
+  stage: deploy
+  variables:
+    URL: https://puppet:8170/code-manager
+  only:
+    - development
+    - testing
+    - production
+    - /^playground/
+  script:
+    - scripts/puppet_deploy.sh
+
+# vim:tabstop=2
+# vim:shiftwidth=2
+# vim:expandtab
diff --git a/Gemfile b/Gemfile
new file mode 100644
index 000000000..eb4309354
--- /dev/null
+++ b/Gemfile
@@ -0,0 +1,19 @@
+source ENV['GEM_SOURCE'] || 'https://rubygems.org'
+
+if puppetversion = ENV['PUPPET_GEM_VERSION']
+  gem 'puppet', puppetversion, require: false
+else
+  gem 'puppet', require: false
+end
+
+gem 'puppet-lint', '~> 2.0'
+gem 'puppet-lint-absolute_classname-check'
+gem 'puppet-lint-alias-check'
+gem 'puppet-lint-empty_string-check'
+gem 'puppet-lint-file_ensure-check'
+gem 'puppet-lint-file_source_rights-check'
+gem 'puppet-lint-leading_zero-check'
+gem 'puppet-lint-trailing_comma-check'
+gem 'puppet-lint-undef_in_function-check'
+gem 'puppet-lint-unquoted_string-check'
+gem 'puppet-lint-variable_contains_upcase'
diff --git a/scripts/lint_check.sh b/scripts/lint_check.sh
new file mode 100755
index 000000000..da6687410
--- /dev/null
+++ b/scripts/lint_check.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Error out if there are any failures
+set -e
+set -o pipefail
+set -u
+
+# Notes
+# xargs -P2 is used to run 2 parallel processes at once.  This speeds up
+# performance on multi-core systems.
+
+if [ -e /proc/cpuinfo ]; then
+  cores=$(awk 'BEGIN { c = 0 }; $1 == "processor" { c++ }; END { print c }' /proc/cpuinfo)
+else
+  cores=2
+fi
+
+# Use Puppet Enterprise Ruby to check ruby and yaml files
+export PATH="/opt/puppetlabs/puppet/bin:/opt/puppetlabs/bin:$PATH"
+
+# Localize Gems on a per-job basis to prevent conflicts
+gem_home="$(gem env gempath | cut -d: -f1)"
+# Trim off the leading part of $HOME
+gem_suffix=${gem_home##*/.gem/}
+# Set GEM_HOME to a job specific location
+export GEM_HOME="${HOME}/jobs/${CI_JOB_NAME:-lint}/gem/${gem_suffix}"
+
+# If we need to install a gem, do so into HOME
+# e.g. /home/gitlab-runner/.gem/ruby/2.1.0
+export PATH="${GEM_HOME}/bin:$PATH"
+
+echo '######## BEGIN DEPENDENCY SETUP #########'
+
+# Display the gem environment
+gem env
+
+if ! (which bundle 2>&1 >/dev/null); then
+  gem install bundler --no-ri --no-rdoc
+fi
+
+# List the files changes from $BASEBRANCH on stdout
+files_changed() {
+  # File status flags:
+  # M modified - File has been modified
+  # C copy-edit - File has been copied and modified
+  # R rename-edit - File has been renamed and modified
+  # A added - File has been added
+  # D deleted - File has been deleted
+  # U unmerged - File has conflicts after a merge
+  git diff --name-status "${BASEBRANCH:=production}" \
+    | awk '$1 ~ /^[MCRA]$/' \
+    | cut -f2-
+}
+
+# Install dependencies
+bundle install
+
+echo '######## END DEPENDENCY SETUP #########'
+echo
+echo
+echo '######## BEGIN LINT CHECKS #########'
+# Lint only the manifest files changed
+files_changed \
+  | awk '/manifests\/.*\.(pp)$/' \
+  | xargs --no-run-if-empty -t -P$cores -n1 \
+  bundle exec puppet-lint
+
+echo '######## END LINT CHECKS #########'
+
+# vim:tabstop=2
+# vim:shiftwidth=2
+# vim:expandtab
diff --git a/scripts/puppet_deploy.sh b/scripts/puppet_deploy.sh
new file mode 100755
index 000000000..90c4d3a28
--- /dev/null
+++ b/scripts/puppet_deploy.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+set -u
+
+export PATH="/opt/puppetlabs/bin:$PATH"
+# Allow these environment variable to be overriden
+: ${URL:='https://puppet:8170/code-manager'}
+# CI_BUILD_REF_NAME is a variable set by gitlab
+: ${ENVIRONMENT:="$CI_BUILD_REF_NAME"}
+
+err() {
+  echo "$1" >&2
+}
+
+if [ -z "${PUPPET_TOKEN:-}" ]; then
+  err "ERROR: PUPPET_TOKEN environment variable must be set!"
+  err "SUGGESTION: Did you push to origin instead of upstream?"
+  err "PUPPET_TOKEN must be set as an environment variable in CI"
+  exit 1
+fi
+
+if ! [ -x /opt/puppetlabs/bin/puppet-code ]; then
+    err "ERROR: /opt/puppetlabs/bin/puppet-code does not exist"
+    err "SUGGESTION: Install the puppet client tools"
+    err "https://docs.puppet.com/pe/2016.4/install_pe_client_tools.html#install-on-a-linux-workstation"
+    exit 2
+fi
+
+# Save the token to a temporary file so we can use it with puppet code deploy
+scratch="$(mktemp -d)"
+remove_scratch() {
+  [ -e "${scratch:-}" ] && rm -rf "$scratch"
+}
+trap remove_scratch EXIT
+# Subsequent calls to mktemp should be inside our scratch dir
+export TMPDIR="$scratch"
+
+tokenfile="$(mktemp)"
+echo -n "$PUPPET_TOKEN" > "$tokenfile"
+
+# Turn on debug logging after the token has been written to the file system
+set -x
+# Deploy the code
+puppet-code deploy \
+  --service-url "$URL" \
+  --token-file "$tokenfile" \
+  --wait "${ENVIRONMENT}"
+rval=$?
+set +x
+
+if [ $rval -ne 0 ]; then
+  echo "ERROR: puppet-code deploy failed with exit code $rval" >&2
+  exit $rval
+fi
+
+echo "Exiting with exit value $rval"
+exit $rval
+
+# vim:tabstop=2
+# vim:shiftwidth=2
+# vim:expandtab
diff --git a/scripts/syntax_check.sh b/scripts/syntax_check.sh
new file mode 100755
index 000000000..2584018a6
--- /dev/null
+++ b/scripts/syntax_check.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# Error out if there are any failures
+set -e
+set -o pipefail
+set -u
+
+# Notes
+# xargs -P2 is used to run 2 parallel processes at once.  This speeds up
+# performance on multi-core systems.
+
+if [ -e /proc/cpuinfo ]; then
+  cores=$(awk 'BEGIN { c = 0 }; $1 == "processor" { c++ }; END { print c }' /proc/cpuinfo)
+else
+  cores=2
+fi
+
+# Use Puppet Enterprise Ruby to check ruby and yaml files
+export PATH="/opt/puppetlabs/puppet/bin:$PATH"
+
+# List the files changes from $BASEBRANCH on stdout
+files_changed() {
+  # File status flags:
+  # M modified - File has been modified
+  # C copy-edit - File has been copied and modified
+  # R rename-edit - File has been renamed and modified
+  # A added - File has been added
+  # D deleted - File has been deleted
+  # U unmerged - File has conflicts after a merge
+  git diff --name-status "${BASEBRANCH:=production}" \
+    | awk '$1 ~ /^[MCRA]$/' \
+    | cut -f2-
+}
+
+# Check the Puppetfile
+echo -n "Checking Puppetfile ... "
+ruby -c Puppetfile
+
+files_changed \
+  | awk '/\.(sh)$/' \
+  | xargs --no-run-if-empty -t -P$cores -n1 \
+  bash -n
+
+# Check all YAML files
+# See: http://stackoverflow.com/questions/3971822/yaml-syntax-validator
+files_changed \
+  | awk '/\.(yml|yaml)$/' \
+  | xargs --no-run-if-empty -t -P$cores -n1 \
+  ruby -r yaml -e 'YAML.load_file(ARGV[0])'
+
+# Check all JSON files
+files_changed \
+  | awk '/\.(json)$/' \
+  | xargs --no-run-if-empty -t -P$cores -n1 \
+  ruby -r json -e 'JSON.load(File.read(ARGV[0]))'
+
+files_changed \
+  | awk '/\.(rb)$/' \
+  | xargs --no-run-if-empty -t -P$cores -n1 \
+  ruby -c
+
+# Check all erb files
+files_changed \
+  | awk '/\.(erb)$/' \
+  | xargs -l --no-run-if-empty -t -P$cores -n1 \
+  bash -c 'erb -P -x -T- $0 | ruby -c'
+
+# Check all Puppet manifest files
+files_changed \
+  | awk '/manifests\/.*\.(pp)$/' \
+  | xargs --no-run-if-empty -t -P$cores -n1 \
+  puppet parser validate
+
+# vim:tabstop=2
+# vim:shiftwidth=2
+# vim:expandtab