Encrypt secrets with sops and age

README.md

@@ -9,19 +9,25 @@ See [Uncharted-Auth](https://github.com/unchartedsoftware/uncharted-auth) for de
 
 ## Requirements for Building locally
 
-### Install Ansible
+### Install Sops
 Required to encrypt/decrypt secrets.
 
-x86/64 processors:
+[Mozilla's Secret OPerationS : sops](https://github.com/mozilla/sops)
+
 ```shell
-brew install ansible
+brew install sops
 ```
 
-ARM64 processors:
+### Obtain AGE key
+
+Fetch `https://drive.google.com/file/d/1DiCAxgjAgXOt72nVSktcmXDmWrcfJwSg/view?usp=drive_link` and store in your home directory.
+
 ```shell
-arch -arm64 brew install ansible
+cp kubernetes/env.template kubernetes/.env
 ```
 
+edit `kubernetes/.env` changing <user> to your user directory (or if not on a Mac edit as appropriate)
+
 
 ### Enabling Kubernetes
 

kubernetes/.gitignore

@@ -0,0 +1,18 @@
+#Secrets
+overlays/prod/base/keycloak/certificates/*.pem
+!overlays/prod/base/keycloak/certificates/*.enc.pem
+
+overlays/prod/overlays/askem-staging/check-latest/check-latest-rsa
+overlays/prod/overlays/askem-staging/check-latest/secrets.yaml
+overlays/prod/overlays/askem-production/check-latest/check-latest-rsa
+overlays/prod/overlays/askem-production/check-latest/secrets.yaml
+
+overlays/prod/overlays/askem-staging/keycloak/realm/*.json
+!overlays/prod/overlays/askem-staging/keycloak/realm/*.enc.json
+overlays/prod/overlays/askem-production/keycloak/realm/*.json
+!overlays/prod/overlays/askem-production/keycloak/realm/*.enc.json
+
+overlays/prod/overlays/askem-staging/secrets/*.yaml
+!overlays/prod/overlays/askem-staging/secrets/*.enc.yaml
+overlays/prod/overlays/askem-production/secrets/*.yaml
+!overlays/prod/overlays/askem-production/secrets/*.enc.yaml

kubernetes/base/hmi/server/spicedb-service.yaml

@@ -9,7 +9,7 @@ metadata:
     software.uncharted.terarium/service: spicedb
     software.uncharted.terarium/part-of: hmi
   annotations:
-    alb.ingress.kubernetes.io/backend-protocol-version: GRPC #This tells AWS to send traffic from the ALB using HTTP2. Can use GRPC as well if you want to leverage GRPC specific features
+    alb.ingress.kubernetes.io/backend-protocol-version: GRPC  # This tells AWS to send traffic from the ALB using HTTP2. Can use GRPC as well if you want to leverage GRPC specific features
     alb.ingress.kubernetes.io/healthcheck-path: /
     alb.ingress.kubernetes.io/success-codes: 0-99
 spec:

kubernetes/deploy.sh

@@ -1,15 +1,49 @@
 #!/bin/bash
 
-## import enviroment variables (.env file)
-#unamestr=$(uname)
-#if [ "$unamestr" = 'Linux' ]; then
-#  export $(grep -v '^#' .env | xargs -d '\n')
-#elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'Darwin' ]; then
-#  export $(grep -v '^#' .env | xargs -0)
-#fi
+help() {
+	echo "
+NAME
+    deploy.sh - deploy TERArium
+
+SYNOPSIS
+    deploy.sh [up | down | status | test | decrypt | encrypt] ENVIRONMENT
+
+DESCRIPTION
+  Environment:
+    ENVIRONMENT        Must be supplied to indicate which environment should be processed
+      staging
+      production
+
+  Launch commands:
+    up                Launches the entire TERArium stack
+    down              Tears down the entire TERArium stack
+
+  Other commands:
+    status            Displays the status of the TERArium cluster
+    encrypt           Encrypt secrets for adding to git repo
+    decrypt           Decrypt secrets for editing
+
+  Environment Variables will be read from a '.env' file, the following can be set
+    AGE_PUBLIC_KEY    the 'askem.agekey' file's public key
+    SOPS_AGE_KEY_FILE location of the file 'askem.agekey'
+  "
+}
+
+if [ ! -f .env ]; then
+  echo "Missing .env file"
+  help
+  exit 1
+fi
 
-SECRET_FILES=()
+# import enviroment variables (.env file)
+unamestr=$(uname)
+if [ "$unamestr" = 'Linux' ]; then
+ export $(grep -v '^#' .env | xargs -d '\n')
+elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'Darwin' ]; then
+ export $(grep -v '^#' .env | xargs -0)
+fi
 
+source secret_files.sh
 source functions.sh
 
 while [[ $# -gt 0 ]]; do
@@ -68,18 +102,12 @@ fi
 
 case ${ENVIRONMENT} in
 staging)
-	SECRET_FILES+=("overlays/prod/base/keycloak/certificates/cert.pem" "overlays/prod/base/keycloak/certificates/key.pem")
-	SECRET_FILES+=("overlays/prod/overlays/askem-staging/secrets/*.yaml")
-	SECRET_FILES+=("overlays/prod/overlays/askem-staging/keycloak/realm/*.json")
-	SECRET_FILES+=("overlays/prod/overlays/askem-staging/check-latest/check-latest-rsa" "overlays/prod/overlays/askem-staging/check-latest/secrets.yaml")
+  SECRET_FILES=${STAGING_SECRET_FILES[@]}
 	KUSTOMIZATION=overlays/prod/overlays/askem-staging
 	KUBECTL_CMD="ssh uncharted-askem-prod-askem-staging-kube-manager-1 sudo kubectl"
 	;;
 production)
-	SECRET_FILES+=("overlays/prod/base/keycloak/certificates/cert.pem" "overlays/prod/base/keycloak/certificates/key.pem")
-	SECRET_FILES+=("overlays/prod/overlays/askem-production/secrets/*.yaml")
-	SECRET_FILES+=("overlays/prod/overlays/askem-production/keycloak/realm/*.json")
-	SECRET_FILES+=("overlays/prod/overlays/askem-production/check-latest/check-latest-rsa" "overlays/prod/overlays/askem-production/check-latest/secrets.yaml")
+  SECRET_FILES=${PRODUCTION_SECRET_FILES[@]}
 	KUSTOMIZATION=overlays/prod/overlays/askem-production
 	KUBECTL_CMD="ssh uncharted-askem-prod-askem-prod-kube-manager-1 sudo kubectl"
 	;;
@@ -128,31 +156,6 @@ encrypt)
 	encrypt
 	;;
 help)
-	echo "
-NAME
-    deploy.sh - deploy TERArium
-
-SYNOPSIS
-    deploy.sh [up | down | status | test | decrypt | encrypt] ENVIRONMENT
-
-DESCRIPTION
-  Environment:
-    ENVIRONMENT        Must be supplied to indicate which environment should be processed
-      staging
-      production
-
-  Launch commands:
-    up                Launches the entire TERArium stack
-    down              Tears down the entire TERArium stack
-
-  Other commands:
-    status            Displays the status of the TERArium cluster
-    encrypt           Encrypt secrets for adding to git repo
-    decrypt           Decrypt secrets for editing
-    "
-	;;
-
-	#  Environment Variables will be read from a '.env' file, the following can be set
-	#    AGE_PUBLIC_KEY    the 'askem.agekey' file's public key
-	#    SOPS_AGE_KEY_FILE location of the file 'askem.agekey'
+  help
+  ;;
 esac

kubernetes/env.template

@@ -0,0 +1,2 @@
+AGE_PUBLIC_KEY=age1q5q7lt6twttye7h5dx4zu0eek04pu6szqfx28w00vq3z6nmlc9dq7md4xd
+SOPS_AGE_KEY_FILE=/Users/<user>/askem.agekey

kubernetes/functions.sh

@@ -15,33 +15,70 @@ checkPrograms() {
 	fi
 }
 
+get_enc_filename() {
+  SECRET_FILE=${1}
+
+  EXTENSION="${SECRET_FILE##*.}"
+  FILENAME="${SECRET_FILE%.*}"
+  ENC_FILENAME="${FILENAME}.enc.${EXTENSION}"
+  if [[ ${EXTENSION} == ${FILENAME} ]]; then
+    ENC_FILENAME="${FILENAME}.enc"
+  fi
+
+  echo ${ENC_FILENAME}
+}
+
 decrypt() {
-	DECRYPTED_FILES=()
-	for SECRET_FILE in "${SECRET_FILES[@]}"; do
-		echo "decrypting file ${SECRET_FILE}"
-		#unpack wildcard - now failing
-		for FILE in $(ls ${SECRET_FILE}); do
-			ansible-vault decrypt --vault-id ~/askem-vault-id.txt "${FILE}"
-		done
-		STATUS=$?
-		if [[ ${STATUS} -eq 0 ]]; then
-			DECRYPTED_FILES+=("${SECRET_FILE}")
-		fi
-	done
+  DECRYPTED_FILES=()
+  for SECRET_FILE in ${SECRET_FILES[@]}; do
+    decrypt_file ${SECRET_FILE}
+  done
+}
+
+decrypt_file() {
+  SECRET_FILE=${1}
+  ENC_FILENAME=$(get_enc_filename ${SECRET_FILE})
+
+  # echo "decrypting file ${SECRET_FILE}"
+  sops --decrypt ${ENC_FILENAME} > ${SECRET_FILE}
+  STATUS=$?
+  if [[ ${STATUS} -eq 0 ]]; then
+    DECRYPTED_FILES+=( ${SECRET_FILE} )
+  fi
 }
 
 encrypt() {
-	for SECRET_FILE in "${SECRET_FILES[@]}"; do
-		#unpack wildcard - now failing
-		for FILE in $(ls ${SECRET_FILE}); do
-			ansible-vault encrypt --vault-id ~/askem-vault-id.txt "${FILE}"
-		done
-	done
+  if [[ -z ${AGE_PUBLIC_KEY} ]]; then
+    echo "Encryption key not set correctly in .env"
+  else
+    for SECRET_FILE in ${SECRET_FILES[@]}; do
+      encrypt_file ${SECRET_FILE}
+    done
+  fi
+}
+
+encrypt_file() {
+  SECRET_FILE=${1}
+  ENC_FILENAME=$(get_enc_filename ${SECRET_FILE})
+
+  # echo "encrypting file ${SECRET_FILE}"
+  if [[ ${EXTENSION} == yaml ]]; then
+    # YAML
+    sops --age=${AGE_PUBLIC_KEY} --encrypt --encrypted-regex '^(data|stringData)$' ${SECRET_FILE} > ${ENC_FILENAME}
+  else
+    # JSON and other
+    sops --age=${AGE_PUBLIC_KEY} --encrypt ${SECRET_FILE} > ${ENC_FILENAME}
+  fi
+  STATUS=$?
+  if [[ ${STATUS} -eq 0 ]]; then
+    DECRYPTED_FILES+=( ${SECRET_FILE} )
+  fi
 }
 
 restore() {
 	for SECRET_FILE in "${DECRYPTED_FILES[@]}"; do
-		git restore "${SECRET_FILE}"
+		rm ${SECRET_FILE}
+		# git restore "${SECRET_FILE}"
 	done
 }
 

kubernetes/get_secret_ui.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash
+#! nix-shell -p gum yq-go jq
+
+if [[ -z $IN_NIX_SHELL ]]; then
+  echo "Please install `nix-shell` - for Mac use:"
+  echo ""
+  echo "curl -L https://nixos.org/nix/install | sh"
+  echo ""
+  echo "For more details see: https://nix.dev/install-nix"
+  echo ""
+  exit 1
+fi
+
+if [ ! -f .env ]; then
+  echo "Missing .env file"
+  help
+  exit 1
+fi
+
+# import enviroment variables (.env file)
+unamestr=$(uname)
+if [ "$unamestr" = 'Linux' ]; then
+ export $(grep -v '^#' .env | xargs -d '\n')
+elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'Darwin' ]; then
+ export $(grep -v '^#' .env | xargs -0)
+fi
+
+source secret_files.sh
+source functions.sh
+
+set -e
+
+gum style \
+	--foreground 212 --border-foreground 212 --border double \
+	--margin "1 2" --padding "2 4" \
+	'Extract Secret.
+
+This script will find all the secrets for a given environment,
+decrypting them as requested.'
+
+OPERATION=$(gum choose "decrypt" "encrypt")
+
+ENVIRONMENT=$(gum choose "staging" "production")
+
+echo "Using $(gum style --foreground 212 "${ENVIRONMENT}") environment"
+
+case ${ENVIRONMENT} in
+staging)
+  SECRET_FILES=(${STAGING_YAML[@]})
+	;;
+production)
+  SECRET_FILES=(${PRODUCTION_YAML[@]})
+	;;
+esac
+
+select_file() {
+  # delcare array/map called FILE_LOCATIONS
+  declare -A FILE_LOCATIONS
+
+  local FILES=()
+  for SECRET_FILE in ${SECRET_FILES[@]}; do
+    local FILE=${SECRET_FILE##*/}
+    FILE_LOCATIONS+=([${FILE}]=${SECRET_FILE})
+    FILES+=(${FILE})
+  done
+
+  local FILE=$(gum choose ${FILES[@]})
+  echo ${FILE_LOCATIONS[${FILE}]}
+}
+
+FILE=$(select_file)
+
+echo "Using secrets from $(gum style --foreground 212 "${FILE}")"
+
+ENC_FILENAME=$(get_enc_filename ${FILE})
+
+echo ${ENC_FILENAME}
+
+KEYS=$(yq -o json '.data | keys' ${ENC_FILENAME} | jq -r '@sh' | tr -d \')
+
+KEY=$(gum choose ${KEYS[@]})
+
+decrypt_file ${FILE}
+
+if [ ${OPERATION} == decrypt ]; then
+  BASE64_VALUE=$(argkey="${KEY}" yq -o json '.data[env(argkey)]' ${FILE} | tr -d \")
+
+  echo "For key $(gum style --foreground 212 "${KEY}")"
+  echo "  Base64 Value is $(gum style --foreground 212 "${BASE64_VALUE}")"
+
+  VALUE=$(echo ${BASE64_VALUE} | base64 -d)
+
+  echo "  Value is $(gum style --foreground 212 "${VALUE}")"
+else
+  echo "TODO encrypt"
+fi
+
+restore

kubernetes/overlays/prod/base/keycloak/certificates/cert.enc.pem

@@ -0,0 +1,20 @@
+{
+	"data": "ENC[AES256_GCM,data:R7tb5kiarcedYHMO8gfcv1GDZsMf+6aTf6As5hW6jydI1cuLjqcCHOLLryc3t3iy9QmdSm4mWg8dr4AtZIadYg889FuOKisxiTBG6lefXF4dpbVUpSpnZDW/ejxrkpbvzMOzacO+QT6hs3fT8Xl+8eTdFa+6YGQWufF9Y/ayMzHJYL5NHSwo8MgF7eCTsdM0gPR6/YdY/eWe+OHUigCEZjQmBPEasinWhuTx1E+/wGYVPOmoNi0nFMiWZKo/dYIjn+MOVu0rTJIlEAuNJeargELYXnVMflidmu2GIAe6HxDmrIOLYD4hHgFSOw0+RQ4e95qZTh4esHP2tdYSaDFGZ5xZRJr/ViprFKMTQMo++ruQYVDaZNIX8fxXaYxjTswPGn0gxXgWfMGlV7pGRDMACtqS2ppHkqn1pV+I7BUqFDVPN+6iqXLaQ3dPGyk0+Lbuqd2pvNubBPDFt3UDTMRFKOpw+6O/SbWqYVc2U1U2zqZN+3micZKNRu0yyQZz3tPG/2xtIC50O3mPTHC3rw/Nk8AM1EL3tOlKEJl0HOaXCy7VS0x+5PCX6r6zB5s7JpNuwz2rZbev0tUuU9E+DTV9CR1V4vsogdtUtugky3i6xCMnJSKOivsft9wjNZOaeC4m2Z3LqJn1heWLzjq51OGVRlVScNDMvZMHEInao0gZZGQ8WF25UbKqHV74HWIenlCl1Ak6VdjCAY0464/c0Ndy3YS8IuCfjV1rNpMOuNoqz+8Ofpd+FGiPIrL0m/pvueYlEdPMAne0QIIh1myqVFErZiuHcY/yrQ7uk9CQVQsS7pj2themypOK67OZWtYpoaDISwYs0qHI2PkJGULk+VlM6XjHxCX1qxhUyRH82RsiBXeEw5fQ4ab0NWdgS5VI0BBVvNaZ0olBh7a95d0s7d15fIgv95WkPJ4C7IzfdGjrbbwmb970OfuEiLVb0jpyns2Ec2tWapetmGQMO/9bM4Xei0Lfyny+F1CURMwymTEtGUZ5CE7lmmDK24W9LrQkU8Z3eYbFjAqa5S8oJdaNLiIVgGwGLMSjo+W4Nxi0B3HSzGpm4tyB2vZv3nTVDruIurZoPtG7ugeM+YssJq1Hq/XbAfPLAzBFXtdnz6MmoIN7bVmANvYs92+eRWNx2VaU+bqNW52OkMuljSgFFi/mIlcDmbyNsOwzav19j2OJFA7lwXWqeqhHFM/fzba8SC38wLpF3N8EAC2/jnqnH33X/EfO0dyo6BWqDbb16PNwtMtdH/WdT86oIzHMc4TfeAUBUEG1Mp5MHf3rCe/LP2oY4xk3qhWaITmwm6gQp6FYoFWYFKZZBucSPTdIf70FWb5eo36k5i+5Wvypy9brCgdeYKLkWKBkxUG+x1Zfk/Oi+nz3HV3hZKfe/06LU0wd68hs6E6EEHwl61mL0sfqnjJSmN6pGt07XuOR8Ot4tLxWZnAtDa/hKMXfn/ZfHivYewJDxfNkggb11qbfvLt13xLxJF6t2o3JLFTHHVsckGALgmNvsj4AbGTwSgGMo+osCFps8hQmCcuzY0bnYrZBsRt55wgYOlYZ7UgZV/jndxR1g3kHF0XErez7b4t99L/oM9B/168G158KeOrFv49ysTO6x+L0mQ==,iv:VvZ/Fcu8jvss4RIuiohnmufjFVn7cZHqdKX7vDLvWqQ=,tag:n1gf019U9hJqn4wlftYA6A==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": [
+			{
+				"recipient": "age1q5q7lt6twttye7h5dx4zu0eek04pu6szqfx28w00vq3z6nmlc9dq7md4xd",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHam1wM0xwYUY2SzVzK0hG\nd1lmMExlTTJYbmQrdGRkTHlpd3RXRGljQ21vCjBmSk1iQXZvMVcvbDloSEpQMzVZ\nMTJ5by93OTFqWVRwSkNhOWtaSC9FaWsKLS0tIFo4bXN1UUZWRTJqMFE2dFVMbFBh\nVWxIUFNoZFRMVDYvMThNeUw4RHEvWU0KEBve8cxO6xtlGXEMcN7dKyDni1y6aCOE\ncFQGO8q3Ynqazo+xxtdpFO6PKtcAHOx42bHp4/zYBsMj6K6ewkMO5A==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2024-02-21T14:31:30Z",
+		"mac": "ENC[AES256_GCM,data:Gv65Wn3lDYUV3nNUZcuOm4v1jAaef3kK/M3EyOrRRixyw96DLgB2jWXmxLLJBjiAa1T+isCtlzzb7U24CmTEGgNnQY4rNMWdHPjjZxb4caCPB17rC/MMOCyvMuCP65XHK63OsGG4kFXYN2bod9plf3jzBtZRKUrA8wDN1a67gu8=,iv:dEl8gqYEcawBgvnVHqx9aTwH/NspG6PGV9uGubIYlTg=,tag:+VCKyy3zVAolU2bWwyMyNQ==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.8.1"
+	}
+}