Merge pull request #2 from Zipcar/namespaces

Namespaces

Makefile

@@ -1,4 +1,4 @@
-# Vault Playground V2.0.0 Makefile
+# Vault Playground V.2.1.0 Makefile
 
 # Help Helper matches comments at the start of the task block so make help gives users information about each task
 .PHONY: help
@@ -7,28 +7,32 @@ help: ## Displays information about available make tasks
 
 .PHONY: init
 init: ## Spin up local Docker instances for all dependencies on a dedicated Docker network
-	cd tasks && ./init
+	@cd tasks && ./init
 
 .PHONY: destroy
 destroy: ## Destroy local Docker instances and their Docker network
-	cd tasks && ./destroy
+	@cd tasks && ./destroy
 
 .PHONY: snapshot
 snapshot: ## Backup the state of Vault by taking a snapshot of Consul and storing it in the local cache
-	cd tasks && ./snapshot
+	@cd tasks && ./snapshot
 
 .PHONY: purge
 purge: ## Delete the local cache of snapshots and initialization keys
-	cd tasks && ./purge
+	@cd tasks && ./purge
 
 .PHONY: restore
 restore: ## Restore previous Vault state by restoring a Consul snapshot
-	cd tasks && ./restore
+	@cd tasks && ./restore
 
 .PHONY: creds
 creds: ## Shows the root token and unseal keys for the currently running Vault instance cached
-	cd tasks && ./creds
+	@cd tasks && ./creds
 
 .PHONY: status
 status: ## Displays the current state of the Vault Playground network in Docker.
-	cd tasks && ./status
+	@cd tasks && ./status
+
+.PHONY: vault-leader
+vault-leader: ## Displays the address of the current Vault leader
+	@cd tasks && ./vault-leader

README.md

@@ -1,4 +1,4 @@
-# Vault Playground V2.0.0
+# Vault Playground V.2.1.0
 
 This repo is meant to make it easier for developers, operators, and CI servers to work locally with a production-like Vault environment.
 The Makefile contained in this repository will allow users to spin up (by default) a Consul cluster with 3 nodes, and 2 Vault servers running
@@ -43,39 +43,36 @@ for each batch of tests.
 1. Clone this repo to the directory of your choice
 1. Run `make init` to setup your environment
 1. Run `make help` to see other available commands:
-```bash
+
+```text
 help                           Displays information about available make tasks
 init                           Spin up local Docker instances for all dependencies on a dedicated Docker network
 destroy                        Destroy local Docker instances and their Docker network
 snapshot                       Backup the state of Vault by taking a snapshot of Consul and storing it in the local cache
 purge                          Delete the local cache of snapshots and initialization keys
 restore                        Restore previous Vault state by restoring a Consul snapshot
-creds                          Shows the root token and unseal keys for the currently running Vault instance if available in the local cache
-status                         Displays the current state of the vault-playground network in Docker.
+creds                          Shows the root token and unseal keys for the currently running Vault instance cached
+status                         Displays the current state of the Vault Playground network in Docker.
+vault-leader                   Displays the address of the current Vault leader
 ```
 
 ## Talking To Vault
 
 This tool is meant to be run locally and make it easier to test and debug Vault workflows, so it _does not_ enable TLS. As a result
 you'll have to explicitly tell the Vault CLI to connect over HTTP. Fortunately, Vault supports the `VAULT_ADDR` environment variable. 
-Just set it to `http://127.0.0.1:8200` and you should be all set. If you're using `docker exec` this environment variable has already
-been set inside the Vault instances.
+If you're using `docker exec` this environment variable has already been set inside the Vault instances.
 
 ```bash
 docker exec vp-vault1 vault status
 ```
-or
-
-```bash
-VAULT_ADDR=http://127.0.0.1:8200 vault status
-```
 
-or better still, export the variable in your session or persist it through your `.bash_rc` or `.bash_profile`:
+Locally, you can export it:
 
 ```bash
-export VAULT_ADDR=http://127.0.0.1:8200
+export VAULT_ADDR=$(make vault-leader)
 ```
-and use Vault normally
+
+and use Vault as you normally would:
 
 ```bash
 vault status
@@ -91,24 +88,29 @@ behavior altered by environment variables.
 ### init
 
 **Environment**
+  - `VP_NAMESPACE` (vp) This is the name of the Docker network that will be created. This string will also prefix all container names. Essentially this is a namespace for Vault Playground.
   - `VP_AUTO_INIT` (true) If true, after launching Vault the script will also run init, cache the resulting keys, and automatically unseal Vault
   - `VP_VAULT_CLUSTER_SIZE` (2) The script will launch this many Vault nodes clustered in HA mode.
   - `VP_CONSUL_CLUSTER_SIZE` (3) How many Consul servers do you need?
-  - `VP_CONSUL_PORT` (8500) The port on your host machine where you can access Consul
-  - `VP_VAULT_PORT` (8200) The port on your host machine where you can access Vault
 
 This script creates a dedicated docker network (called `vp`) and spins up the configured number of Vault and Consul servers. 
-By default this also initializes and unseals Vault automatically so you can use it immediately. You can connect to Consul
-from [http://localhost:8500](http://localhost:8500) and communicate with Vault through Docker exec or the Vault CLI.
+By default this also initializes and unseals Vault automatically so you can use it immediately. When the script completes
+it will output the addresses where Consul and Vault can be reached locally:
+
+```text
+Consul is now running at: http://127.0.0.1:32845
+Vault is now running at: http://127.0.0.1:32849 set your host's VAULT_ADDR environment variable to communicate directly with it using the Vault CLI.
+```
 
 If `VP_AUTO_INIT` is true, the script will cache the output of the initialize API call locally (`$HOME/.vault-playground/init_dumps`) in a file 
 named after the Docker ID of the main vault server. 
 
 ### snapshot
 
 **Environment**
+  - `VP_NAMESPACE` (vp) This is the name of the Docker network any local containers are running on.
   - `VP_SNAPSHOT_NAME` (timestamp of the form: `%Y-%m-%d-%H-%M-%S`) Snapshots are named using the ID of the active Vault instance concatenated with this value. 
-  - `VP_CONSUL_TARGET` - (`http://127.0.0.1:8500` The Docker node) The Consul server that should be snapshotted
+  - `VP_CONSUL_TARGET` - (The Vault Playground Consul node) The Consul server that should be snapshotted
   - `VP_CONSUL_DATACENTER` - (dc1) The Consul data center that should be snapshotted, locally this will almost always be the default. 
 
 This script creates a snapshot in the local cache (`$HOME/.vault-playground/snapshots`) that by default is named with a timestamp.
@@ -124,9 +126,10 @@ VP_CONSUL_TARGET=https://myconsul.biz:8500 VP_SNAPSHOT_NAME=myconsul-biz-12-31-2
 ### restore
 
 **Environment**
+  - `VP_NAMESPACE` (vp) This is the name of the Docker network any local containers are running on.
   - `VP_SNAPSHOT` (empty string) Path to the Consul snapshot to restore. If this is blank, restore will list all the snapshots in its cache (`$HOME/.vault-playground/snapshots`).
   - `VP_INIT_DUMP` (empty string) Path to a file containing the output of the Vault initialization command. If this file doesn't exist, restore will check it's cache (`$HOME/.vault-playground/init_dumps`) if it finds nothing it will still restore the snapshot, but leave Vault sealed.
-  - `VP_CONSUL_TARGET` - (`http://127.0.0.1:8500` The Docker node) The Consul server that the snapshot should be restored to
+  - `VP_CONSUL_TARGET` - (The Vault Playground Consul node) The Consul server that the snapshot should be restored to
   - `VP_VAULT_TARGETS` - (all running Vault Playground Vault containers) A space delimited list of Vault servers that should be contacted for unsealing if an init dump file was provided or existed in the cache.
 
 **Dependencies**
@@ -140,13 +143,35 @@ snapshot will still be restored.
 
 ### creds
 
+**Environment**
+
+  - `VP_NAMESPACE` (vp) This is the name of the Docker network any local containers are running on.
+
 This is a helper task that looks in the cache for any initialization dumps from the currently running Vault instance and
 outputs them to the screen, allowing the user to see both the root and unseal keys for the currently running Vault.
 
 ### destroy
 
+**Environment**
+
+  - `VP_NAMESPACE` (vp) This is the name of the Docker network any local containers are running on.
+
 This script terminates and removes all containers deployed in the Vault Playground docker network (`vp`).
 
+### vault-leader
+
+**Environment**
+
+  - `VP_NAMESPACE` (vp) This is the name of the Docker network any local containers are running on.
+
+
+This script outputs the current leader and the port it's exporting on the host. Useful for setting the VAULT_ADDR 
+environment variable:
+
+```bash
+export VAULT_ADDR=$(make vault-leader)
+```
+
 ### purge
 
 This script deletes all cached credentials and snapshots by removing the `$HOME/.vault-playground` directory.

tasks/creds

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 
-# Vault Playground V2.0.0
+# Vault Playground V.2.1.0 creds
 #
 # This is a helper task that looks in the cache for any initialization dumps from the currently running Vault instance
 # and outputs them to the screen.
 #
 
+: "${VP_NAMESPACE:=vp}" # Namespace for the local Vault playground, used for network and container names
+
 vp_init_cache=$HOME/.vault-playground/init_dumps
 
 if [ ! $(command -v docker) ]; then
@@ -18,7 +20,7 @@ if [ ! $(command -v jq) ]; then
   exit 1
 fi
 
-vault_short_id=$(docker ps -q -f name=vp-vault1)
+vault_short_id=$(docker ps -q -f name=${VP_NAMESPACE}-vault1)
 vault_init_dump_path=${vp_init_cache}/${vault_short_id}.json
 if [ -e "${vault_init_dump_path}" ]; then
   echo "Found cached creds file: $vault_init_dump_path"

tasks/destroy

@@ -1,24 +1,24 @@
 #!/usr/bin/env bash
 
-# Vault Playground V2.0.0 destroy
+# Vault Playground V.2.1.0 destroy
 #
 # This script terminates and removes all containers deployed in the Vault Playground docker network (vp).
 #
 
-vp_network_name=vp
+: "${VP_NAMESPACE:=vp}" # Namespace for the local Vault playground, used for network and container names
 
 if [ ! $(command -v docker) ]; then
   echo -e "\ndocker not found! It must be installed before proceeding: https://www.docker.com/get-docker\n"
   exit 1
 fi
 
-containers_to_destroy=$(docker ps -qa --no-trunc -f network=${vp_network_name})
+containers_to_destroy=$(docker ps -qa --no-trunc -f network=${VP_NAMESPACE})
 
 if [ ${#containers_to_destroy} != 0 ]; then
   docker rm -f ${containers_to_destroy}
-  docker network rm ${vp_network_name}
+  docker network rm ${VP_NAMESPACE}
 else
-  echo "No containers found in network or network does not exist. Maybe everything was already destroyed?"
+  echo "No containers found in the $VP_NAMESPACE network or network does not exist. Maybe everything was already destroyed or you need to pass in VP_NAMESPACE?"
 fi
 
 echo "To purge all cached data run the purge script"

tasks/init

@@ -1,29 +1,27 @@
 #!/usr/bin/env bash
-# Vault Playground V2.0.0 init
+# Vault Playground V.2.1.0 init
 #
 # This script creates a dedicated docker network (called vp) and spins up the configured number of Vault and Consul servers.
-# By default this also initializes and unseals Vault automatically so you can use it immediately. You can connect to Consul
-# from http://localhost:8500 and communicate with Vault through Docker exec or the Vault CLI.
+# By default this also initializes and unseals Vault automatically so you can use it immediately. When the script completes
+# it will output the addresses where Consul and Vault can be reached locally
 #
 # This file is meant to be used for local testing and development. The flow and architecture of this setup is meant to
 # mirror production, but certain behaviors have been added for developer convenience that would be inappropriate in a
 # production environment; for example: writing out the unseal keys to a file and automating the unsealing of vault.
 #
 
 vp_init_cache=$HOME/.vault-playground/init_dumps
-vp_network_name=vp
 vp_vault_shares=5
 vp_vault_threshold=3
 
 # Environment variables needed by this script defaulted for local use
+: "${VP_NAMESPACE:=vp}" # Namespace for the local Vault playground, used for network and container names
 : "${VP_AUTO_INIT:=true}" # If true, after launching Vault also runs init, caches the resulting keys, and automatically unseals Vault.
 : "${VP_VAULT_CLUSTER_SIZE:=2}" # The script will launch this many Vault nodes clustered in HA mode.
 : "${VP_CONSUL_CLUSTER_SIZE:=3}" # How many Consul servers do you need?
-: "${VP_CONSUL_PORT:=8500}" # The port on your host machine where you can access Consul's web-ui.
-: "${VP_VAULT_PORT:=8200}" # The port on your host machine where you can talk to the first vault container. Any HA Vault nodes will forward a random port number.
 
-if [ $(docker network ls -f name=${vp_network_name} -q) ]; then
-  echo "Looks like there's currently some vault playground infrastructure running in Docker's $vp_network_name network, try running destroy first."
+if [ $(docker network ls -f name=${VP_NAMESPACE} -q) ]; then
+  echo "Looks like there's currently some vault playground infrastructure running in Docker's $VP_NAMESPACE network, try running destroy first."
   exit 0
 fi
 
@@ -45,33 +43,35 @@ fi
 mkdir -p ${vp_init_cache}
 
 # Create Docker network for testing
-docker network create --driver bridge ${vp_network_name} 2>/dev/null || echo ""
+docker network create --driver bridge ${VP_NAMESPACE} 2>/dev/null || echo ""
 
 # Spin up consul to back vault and then spin up vault.
-docker run -d -p ${VP_CONSUL_PORT}:8500 --network ${vp_network_name} -e CONSUL_BIND_INTERFACE=eth0 --name vp-consul1 consul && docker run -d -p ${VP_VAULT_PORT}:8200 --cap-add=IPC_LOCK --network ${vp_network_name} --name vp-vault1 -e VAULT_ADDR=http://127.0.0.1:8200 -e 'VAULT_LOCAL_CONFIG={"backend": {"consul": {"path": "/vault/", "check_timeout": "5s", "max_parallel": "128", "address": "http://vp-consul1:8500", "cluster_addr": "http://vp-vault1:8200", "api_addr": "http://vp-vault1:8200"}}, "listener": { "tcp": { "address":"0.0.0.0:8200", "tls_disable":1}}, "default_lease_ttl": "168h", "max_lease_ttl": "720h"}' vault server
+docker run -d -P --network ${VP_NAMESPACE} -e CONSUL_BIND_INTERFACE=eth0 --name ${VP_NAMESPACE}-consul1 consul && docker run -d -P --cap-add=IPC_LOCK --network ${VP_NAMESPACE} --name ${VP_NAMESPACE}-vault1 -e VAULT_ADDR=http://127.0.0.1:8200 -e 'VAULT_LOCAL_CONFIG={"backend": {"consul": {"path": "/vault/", "check_timeout": "5s", "max_parallel": "128", "address": "'"http://$VP_NAMESPACE-consul1:8500"'", "cluster_addr": "'"http://$VP_NAMESPACE-vault1:8200"'", "api_addr": "'"http://$VP_NAMESPACE-vault1:8200"'"}}, "listener": { "tcp": { "address":"0.0.0.0:8200", "tls_disable":1}}, "default_lease_ttl": "168h", "max_lease_ttl": "720h"}' vault server
 
 # Launch the requested number of Vault servers
 for (( i=2; i<=$VP_VAULT_CLUSTER_SIZE; i++ ))
 do
-  if [ $(docker ps -q -f name=vp-vault${i}) ]; then
+  if [ $(docker ps -q -f name=${VP_NAMESPACE}-vault${i}) ]; then
     echo "Scaling Vault: Server #$i exists"
   else
     echo "Scaling Vault: Launching server #$i"
-    docker run -d -P --cap-add=IPC_LOCK --network ${vp_network_name} --name vp-vault2 -e VAULT_ADDR=http://127.0.0.1:8200 -e 'VAULT_LOCAL_CONFIG={"backend": {"consul": {"path": "/vault/", "check_timeout": "5s", "max_parallel": "128", "address": "'"http://vp-consul1:8500"'", "cluster_addr": "'"http://vp-vault$i:8200"'", "api_addr": "'"http://vp-vault$i:8200"'"}}, "listener": { "tcp": { "address":"0.0.0.0:8200", "tls_disable":1}}, "default_lease_ttl": "168h", "max_lease_ttl": "720h"}' vault server
+    docker run -d -P --cap-add=IPC_LOCK --network ${VP_NAMESPACE} --name ${VP_NAMESPACE}-vault2 -e VAULT_ADDR=http://127.0.0.1:8200 -e 'VAULT_LOCAL_CONFIG={"backend": {"consul": {"path": "/vault/", "check_timeout": "5s", "max_parallel": "128", "address": "'"http://$VP_NAMESPACE-consul1:8500"'", "cluster_addr": "'"http://$VP_NAMESPACE-vault$i:8200"'", "api_addr": "'"http://$VP_NAMESPACE-vault$i:8200"'"}}, "listener": { "tcp": { "address":"0.0.0.0:8200", "tls_disable":1}}, "default_lease_ttl": "168h", "max_lease_ttl": "720h"}' vault server
   fi
 done
 
 # Give vault a couple of seconds to start before initializing and unsealing
 # It's a bad idea to persist the unseal info locally and automate the unsealing of vault in production, in dev it's fine.
 if [ ${VP_AUTO_INIT} == "true" ]; then
   sleep 3s
-  short_vault_id=$(docker ps -q -f name=vp-vault1)
+  short_vault_id=$(docker ps -q -f name=${VP_NAMESPACE}-vault1)
   vault_init_dump_path=${vp_init_cache}/${short_vault_id}.json
 
-  curl --request PUT  -H "Content-Type: application/json" -d "{\"secret_shares\":$vp_vault_shares, \"secret_threshold\":$vp_vault_threshold}" http://127.0.0.1:8200/v1/sys/init > ${vault_init_dump_path}
+  # Fetch the local port
+  vp_vault1_local_port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "8200/tcp") 0).HostPort}}' ${VP_NAMESPACE}-vault1)
+  curl --request PUT  -H "Content-Type: application/json" -d "{\"secret_shares\":$vp_vault_shares, \"secret_threshold\":$vp_vault_threshold}" http://127.0.0.1:${vp_vault1_local_port}/v1/sys/init > ${vault_init_dump_path}
 
   # Unseal all Vault instances
-  containers=$(docker ps --filter name=vp-vault | awk '{if(NR>1) print $NF}')
+  containers=$(docker ps --filter name=${VP_NAMESPACE}-vault | awk '{if(NR>1) print $NF}')
   for container in ${containers}
   do
     jq -r '.keys[]' ${vault_init_dump_path} | xargs -I % docker exec ${container} vault unseal %
@@ -82,27 +82,30 @@ fi
 # Launch the requested number of consul servers
 for (( i=2; i<=$VP_CONSUL_CLUSTER_SIZE; i++ ))
 do
-  if [ $(docker ps -q -f name=vp-consul${i}) ]; then
+  if [ $(docker ps -q -f name=${VP_NAMESPACE}-consul${i}) ]; then
     echo "Scaling Consul: Server #$i exists"
   else
     echo "Scaling Consul: Launching server #$i"
-    docker run -d --name vp-consul${i} --network ${vp_network_name} -e CONSUL_BIND_INTERFACE=eth0 consul agent -dev -join=vp-consul1
+    docker run -d --name ${VP_NAMESPACE}-consul${i} --network ${VP_NAMESPACE} -e CONSUL_BIND_INTERFACE=eth0 consul agent -dev -join=${VP_NAMESPACE}-consul1
   fi
 done
 
 echo "Vault Playground Test Environment Deployed"
 echo "Consul Status:"
-docker exec vp-consul1 consul members
+docker exec ${VP_NAMESPACE}-consul1 consul members
 
 echo "Vault Status:"
 
 if [ ${VP_AUTO_INIT} == "true" ]; then
-  docker exec vp-vault1 vault status
+  docker exec ${VP_NAMESPACE}-vault1 vault status
   echo "Vault Initialization information dumped to: ${vault_init_dump_path}"
   echo "Root Token:"
   jq -r '.root_token' ${vault_init_dump_path}
 else
   echo "Auto initialization of Vault was disabled... no status to report."
 fi
 
-echo "Vault is now running at: http://127.0.0.1:$VP_VAULT_PORT set your host's VAULT_ADDR environment variable to communicate directly with it using the Vault CLI."
+echo "Vault is now running at:"
+docker ps -f name=${VP_NAMESPACE}-vault --format "{{.Names}}: http://{{printf \"%.13s\" .Ports}}"
+
+echo "Set your host's VAULT_ADDR environment variable to communicate directly with it using the Vault CLI."

tasks/purge

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Vault Playground V2.0.0 purge
+# Vault Playground V.2.1.0 purge
 #
 # This script deletes all cached credentials and snapshots by removing the $HOME/.vault-playground directory
 #