Encrypt secrets with sops and age (#373)

Co-authored-by: David Gauldie <[email protected]>

README.md

@@ -3,25 +3,78 @@
 # Orchestration
 Scripts and deployment information needed to setup and run TERArium
 
+## Common Tasks
+
+### Deploy and Environment
+In the `/kubernetes` folder, run the script `deploy.sh`
+
+To test the Staging environment's kustomize script eg:
+```shell
+/deploy.sh test staging
+```
+
+To get help about `deploy.sh`
+```shell
+/deploy.sh help
+```
+
+### Decrypt a Secret
+Secrets are encrypted using SPOS and AGE (see Requirements below for details on those).  To view a secret, in the `/kubernetes` directory, use:
+```shell
+./get_secret_ui.sh
+```
+
+NB: the first time running `get_secret_ui.sh` it will need to install a number of packages.  This can take a few moments.
+
+Caveat: `get_secret_ui.sh` is currently unable to encode a secret - this is a TODO
+
+Alternatively, to view secrets, or modify secrets one can:
+```shell
+/deploy.sh decrypt [staging|production]
+```
+but one will have to base64 decode/encode any given value by hand
+
+
 ## Authorization Gateway
 
 See [Uncharted-Auth](https://github.com/unchartedsoftware/uncharted-auth) for details about building the Gateway containers.
 
 ## Requirements for Building locally
 
-### Install Ansible
+### Install Sops
 Required to encrypt/decrypt secrets.
 
-x86/64 processors:
+[Mozilla's Secret OPerationS : sops](https://github.com/mozilla/sops)
+
+```shell
+brew install sops
+```
+
+### Install Nix-Shell
+Required to encrypt/decrypt via `get_secrets_ui.sh` (a helpful script to aid in finding the value of a secret).
+
+MacOS
 ```shell
-brew install ansible
+curl -L https://nixos.org/nix/install | sh
 ```
 
-ARM64 processors:
+Linux
+```shell
+curl -L https://nixos.org/nix/install | sh -s -- --daemon
+```
+
+For more details please visit: https://nix.dev/install-nix
+
+
+### Obtain AGE key
+
+Fetch `https://drive.google.com/file/d/1DiCAxgjAgXOt72nVSktcmXDmWrcfJwSg/view?usp=drive_link` and store in your home directory.
+
 ```shell
-arch -arm64 brew install ansible
+cp kubernetes/env.template kubernetes/.env
 ```
 
+edit `kubernetes/.env` changing <user> to your user directory (or if not on a Mac edit as appropriate)
 
 ### Enabling Kubernetes
 

kubernetes/.gitignore

@@ -0,0 +1,18 @@
+#Secrets
+overlays/prod/base/keycloak/certificates/*.pem
+!overlays/prod/base/keycloak/certificates/*.enc.pem
+
+overlays/prod/overlays/askem-staging/check-latest/check-latest-rsa
+overlays/prod/overlays/askem-staging/check-latest/secrets.yaml
+overlays/prod/overlays/askem-production/check-latest/check-latest-rsa
+overlays/prod/overlays/askem-production/check-latest/secrets.yaml
+
+overlays/prod/overlays/askem-staging/keycloak/realm/*.json
+!overlays/prod/overlays/askem-staging/keycloak/realm/*.enc.json
+overlays/prod/overlays/askem-production/keycloak/realm/*.json
+!overlays/prod/overlays/askem-production/keycloak/realm/*.enc.json
+
+overlays/prod/overlays/askem-staging/secrets/*.yaml
+!overlays/prod/overlays/askem-staging/secrets/*.enc.yaml
+overlays/prod/overlays/askem-production/secrets/*.yaml
+!overlays/prod/overlays/askem-production/secrets/*.enc.yaml

kubernetes/base/hmi/server/spicedb-service.yaml

@@ -9,7 +9,7 @@ metadata:
     software.uncharted.terarium/service: spicedb
     software.uncharted.terarium/part-of: hmi
   annotations:
-    alb.ingress.kubernetes.io/backend-protocol-version: GRPC #This tells AWS to send traffic from the ALB using HTTP2. Can use GRPC as well if you want to leverage GRPC specific features
+    alb.ingress.kubernetes.io/backend-protocol-version: GRPC  # This tells AWS to send traffic from the ALB using HTTP2. Can use GRPC as well if you want to leverage GRPC specific features
     alb.ingress.kubernetes.io/healthcheck-path: /
     alb.ingress.kubernetes.io/success-codes: 0-99
 spec:

kubernetes/deploy.sh

@@ -1,15 +1,49 @@
 #!/bin/bash
 
-## import enviroment variables (.env file)
-#unamestr=$(uname)
-#if [ "$unamestr" = 'Linux' ]; then
-#  export $(grep -v '^#' .env | xargs -d '\n')
-#elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'Darwin' ]; then
-#  export $(grep -v '^#' .env | xargs -0)
-#fi
+help() {
+	echo "
+NAME
+    deploy.sh - deploy TERArium
+
+SYNOPSIS
+    deploy.sh [up | down | status | test | decrypt | encrypt] ENVIRONMENT
+
+DESCRIPTION
+  Environment:
+    ENVIRONMENT        Must be supplied to indicate which environment should be processed
+      staging
+      production
+
+  Launch commands:
+    up                Launches the entire TERArium stack
+    down              Tears down the entire TERArium stack
+
+  Other commands:
+    status            Displays the status of the TERArium cluster
+    encrypt           Encrypt secrets for adding to git repo
+    decrypt           Decrypt secrets for editing
+
+  Environment Variables will be read from a '.env' file, the following can be set
+    AGE_PUBLIC_KEY    the 'askem.agekey' file's public key
+    SOPS_AGE_KEY_FILE location of the file 'askem.agekey'
+  "
+}
+
+if [ ! -f .env ]; then
+  echo "Missing .env file"
+  help
+  exit 1
+fi
 
-SECRET_FILES=()
+# import enviroment variables (.env file)
+unamestr=$(uname)
+if [ "$unamestr" = 'Linux' ]; then
+ export $(grep -v '^#' .env | xargs -d '\n')
+elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'Darwin' ]; then
+ export $(grep -v '^#' .env | xargs -0)
+fi
 
+source secret_files.sh
 source functions.sh
 
 while [[ $# -gt 0 ]]; do
@@ -68,18 +102,12 @@ fi
 
 case ${ENVIRONMENT} in
 staging)
-	SECRET_FILES+=("overlays/prod/base/keycloak/certificates/cert.pem" "overlays/prod/base/keycloak/certificates/key.pem")
-	SECRET_FILES+=("overlays/prod/overlays/askem-staging/secrets/*.yaml")
-	SECRET_FILES+=("overlays/prod/overlays/askem-staging/keycloak/realm/*.json")
-	SECRET_FILES+=("overlays/prod/overlays/askem-staging/check-latest/check-latest-rsa" "overlays/prod/overlays/askem-staging/check-latest/secrets.yaml")
+  SECRET_FILES=${STAGING_SECRET_FILES[@]}
 	KUSTOMIZATION=overlays/prod/overlays/askem-staging
 	KUBECTL_CMD="ssh uncharted-askem-prod-askem-staging-kube-manager-1 sudo kubectl"
 	;;
 production)
-	SECRET_FILES+=("overlays/prod/base/keycloak/certificates/cert.pem" "overlays/prod/base/keycloak/certificates/key.pem")
-	SECRET_FILES+=("overlays/prod/overlays/askem-production/secrets/*.yaml")
-	SECRET_FILES+=("overlays/prod/overlays/askem-production/keycloak/realm/*.json")
-	SECRET_FILES+=("overlays/prod/overlays/askem-production/check-latest/check-latest-rsa" "overlays/prod/overlays/askem-production/check-latest/secrets.yaml")
+  SECRET_FILES=${PRODUCTION_SECRET_FILES[@]}
 	KUSTOMIZATION=overlays/prod/overlays/askem-production
 	KUBECTL_CMD="ssh uncharted-askem-prod-askem-prod-kube-manager-1 sudo kubectl"
 	;;
@@ -128,31 +156,6 @@ encrypt)
 	encrypt
 	;;
 help)
-	echo "
-NAME
-    deploy.sh - deploy TERArium
-
-SYNOPSIS
-    deploy.sh [up | down | status | test | decrypt | encrypt] ENVIRONMENT
-
-DESCRIPTION
-  Environment:
-    ENVIRONMENT        Must be supplied to indicate which environment should be processed
-      staging
-      production
-
-  Launch commands:
-    up                Launches the entire TERArium stack
-    down              Tears down the entire TERArium stack
-
-  Other commands:
-    status            Displays the status of the TERArium cluster
-    encrypt           Encrypt secrets for adding to git repo
-    decrypt           Decrypt secrets for editing
-    "
-	;;
-
-	#  Environment Variables will be read from a '.env' file, the following can be set
-	#    AGE_PUBLIC_KEY    the 'askem.agekey' file's public key
-	#    SOPS_AGE_KEY_FILE location of the file 'askem.agekey'
+  help
+  ;;
 esac

kubernetes/env.template

@@ -0,0 +1,2 @@
+AGE_PUBLIC_KEY=age1q5q7lt6twttye7h5dx4zu0eek04pu6szqfx28w00vq3z6nmlc9dq7md4xd
+SOPS_AGE_KEY_FILE=/Users/<user>/askem.agekey

kubernetes/functions.sh

@@ -15,33 +15,70 @@ checkPrograms() {
 	fi
 }
 
+get_enc_filename() {
+  SECRET_FILE=${1}
+
+  EXTENSION="${SECRET_FILE##*.}"
+  FILENAME="${SECRET_FILE%.*}"
+  ENC_FILENAME="${FILENAME}.enc.${EXTENSION}"
+  if [[ ${EXTENSION} == ${FILENAME} ]]; then
+    ENC_FILENAME="${FILENAME}.enc"
+  fi
+
+  echo ${ENC_FILENAME}
+}
+
 decrypt() {
-	DECRYPTED_FILES=()
-	for SECRET_FILE in "${SECRET_FILES[@]}"; do
-		echo "decrypting file ${SECRET_FILE}"
-		#unpack wildcard - now failing
-		for FILE in $(ls ${SECRET_FILE}); do
-			ansible-vault decrypt --vault-id ~/askem-vault-id.txt "${FILE}"
-		done
-		STATUS=$?
-		if [[ ${STATUS} -eq 0 ]]; then
-			DECRYPTED_FILES+=("${SECRET_FILE}")
-		fi
-	done
+  DECRYPTED_FILES=()
+  for SECRET_FILE in ${SECRET_FILES[@]}; do
+    decrypt_file ${SECRET_FILE}
+  done
+}
+
+decrypt_file() {
+  SECRET_FILE=${1}
+  ENC_FILENAME=$(get_enc_filename ${SECRET_FILE})
+
+  # echo "decrypting file ${SECRET_FILE}"
+  sops --decrypt ${ENC_FILENAME} > ${SECRET_FILE}
+  STATUS=$?
+  if [[ ${STATUS} -eq 0 ]]; then
+    DECRYPTED_FILES+=( ${SECRET_FILE} )
+  fi
 }
 
 encrypt() {
-	for SECRET_FILE in "${SECRET_FILES[@]}"; do
-		#unpack wildcard - now failing
-		for FILE in $(ls ${SECRET_FILE}); do
-			ansible-vault encrypt --vault-id ~/askem-vault-id.txt "${FILE}"
-		done
-	done
+  if [[ -z ${AGE_PUBLIC_KEY} ]]; then
+    echo "Encryption key not set correctly in .env"
+  else
+    for SECRET_FILE in ${SECRET_FILES[@]}; do
+      encrypt_file ${SECRET_FILE}
+    done
+  fi
+}
+
+encrypt_file() {
+  SECRET_FILE=${1}
+  ENC_FILENAME=$(get_enc_filename ${SECRET_FILE})
+
+  # echo "encrypting file ${SECRET_FILE}"
+  if [[ ${EXTENSION} == yaml ]]; then
+    # YAML
+    sops --age=${AGE_PUBLIC_KEY} --encrypt --encrypted-regex '^(data|stringData)$' ${SECRET_FILE} > ${ENC_FILENAME}
+  else
+    # JSON and other
+    sops --age=${AGE_PUBLIC_KEY} --encrypt ${SECRET_FILE} > ${ENC_FILENAME}
+  fi
+  STATUS=$?
+  if [[ ${STATUS} -eq 0 ]]; then
+    DECRYPTED_FILES+=( ${SECRET_FILE} )
+  fi
 }
 
 restore() {
 	for SECRET_FILE in "${DECRYPTED_FILES[@]}"; do
-		git restore "${SECRET_FILE}"
+		rm ${SECRET_FILE}
+		# git restore "${SECRET_FILE}"
 	done
 }