Merge pull request #387 from systemaccounting/386-add-dev-k8s

386 add dev k8s

.gitignore

@@ -80,4 +80,7 @@ migrations/dumps/*.sql
 **/*.profraw
 
 # vscode
-tests/thunder-tests/thunderActivity.json
+tests/thunder-tests/thunderActivity.json
+
+# ssh
+k8s/*.pem

crates/pg/src/model.rs

@@ -1578,7 +1578,7 @@ mod integration_tests {
 
     #[cfg_attr(not(feature = "db_tests"), ignore)]
     #[tokio::test]
-    async fn it_crates_a_selects_approvers_query() {
+    async fn it_creates_a_selects_approvers_query() {
         _before_each();
 
         let test_conn = _get_conn().await;

infrastructure/terraform/aws/environments/dev/main.tf

@@ -57,7 +57,7 @@ module "dev" {
   rds_allow_major_version_upgrade = true
   rds_instance_class              = "db.t3.micro"
   rds_parameter_group             = "default.postgres14"
-  rds_engine_version              = "14.10"
+  rds_engine_version              = "14.12"
   rds_instance_name               = "${local.RDS_PREFIX}-${local.ID_ENV}"
   db_snapshot_id                  = null
 
@@ -73,6 +73,11 @@ module "dev" {
   // apigw v2
   enable_api_auto_deploy = true
 
+  ############### k8s ###############
+
+  microk8s_instance_type = "t2.medium"
+  enable_microk8s = false
+
   ############### client ###############
 
   client_origin_bucket_name = "${local.ORIGIN_PREFIX}-${local.ID_ENV}"

infrastructure/terraform/aws/environments/prod/main.tf

@@ -86,6 +86,11 @@ module "prod" {
   // apigw v2
   enable_api_auto_deploy = true
 
+  ############### k8s ###############
+
+  microk8s_instance_type = "t2.medium"
+  enable_microk8s = false
+
   ############### client ###############
 
   client_origin_bucket_name = "${local.ORIGIN_PREFIX}-${local.ID_ENV}"

infrastructure/terraform/aws/modules/environment/v001/k8s.tf

@@ -0,0 +1,8 @@
+module "microk8s" {
+  count         = var.enable_microk8s ? 1 : 0
+  source        = "../../microk8s/v001"
+  env           = var.env
+  env_id        = var.env_id
+  ssm_prefix    = var.ssm_prefix
+  instance_type = var.microk8s_instance_type
+}

infrastructure/terraform/aws/modules/environment/v001/variables.tf

@@ -20,3 +20,5 @@ variable "env_id" {}
 variable "build_db" { type = bool }
 variable "build_cache" { type = bool }
 variable "readiness_check_path" { default = "/healthz" } // todo: assign from root project.yaml
+variable "microk8s_instance_type" {}
+variable "enable_microk8s" { type = bool }

infrastructure/terraform/aws/modules/microk8s/v001/ec2.tf

@@ -0,0 +1,84 @@
+resource "aws_instance" "default" {
+  ami                    = data.aws_ami.ubuntu.id
+  instance_type          = var.instance_type
+  key_name               = aws_key_pair.default.key_name
+  vpc_security_group_ids = [aws_security_group.default.id]
+  # provision in the same subnet as rds postgres
+  subnet_id = tolist(data.aws_db_subnet_group.default.subnet_ids)[0]
+  user_data = file("${path.module}/user-data.sh")
+  tags = {
+    Name = "${local.NAME}-${local.ID_ENV}"
+  }
+}
+
+# passed around in the state file, demo only
+resource "tls_private_key" "default" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "aws_key_pair" "default" {
+  key_name   = "${local.MICROK8S_SSH_KEY_NAME_PREFIX}-${local.ID_ENV}"
+  public_key = sensitive(tls_private_key.default.public_key_openssh)
+}
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]
+  }
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+}
+
+# permissive for demo
+resource "aws_security_group" "default" {
+  name        = "${local.NAME}-sec-grp-${local.ID_ENV}"
+  description = "${local.NAME} access in ${local.SPACED_ID_ENV}"
+  vpc_id      = data.aws_vpc.default.id
+
+  ingress {
+    description = "ssh"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "microk8s"
+    from_port   = 16443
+    to_port     = 16443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "measure node port"
+    from_port   = 30010
+    to_port     = 30010
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+data "aws_db_subnet_group" "default" {
+  # todo: replace hardcoded "db-subnet-group" prefix with variable here and
+  # in infrastructure/terraform/aws/modules/environment/v001/rds.tf
+  name = "db-subnet-group-${local.ID_ENV}"
+}
+
+data "aws_vpc" "default" {
+  default = true
+}

infrastructure/terraform/aws/modules/microk8s/v001/locals.tf

@@ -0,0 +1,10 @@
+locals {
+  NAME                         = "microk8s"
+  ID_ENV                       = "${var.env_id}-${var.env}"
+  TITLED_ID_ENV                = replace(title(local.ID_ENV), "-", "")
+  SPACED_ID_ENV                = replace(local.ID_ENV, "-", " ")
+  PROJECT_CONF                 = yamldecode(file("../../../../../project.yaml"))
+  MICROK8S_CONF                = local.PROJECT_CONF.infrastructure.terraform.aws.modules.microk8s.env_var.set
+  MICROK8S_SSH_KEY_NAME_PREFIX = local.MICROK8S_CONF.MICROK8S_SSH_KEY_NAME_PREFIX.default
+  MICROK8S_SSH_PORT            = local.MICROK8S_CONF.MICROK8S_SSH_PORT.default
+}

infrastructure/terraform/aws/modules/microk8s/v001/ssm.tf

@@ -0,0 +1,20 @@
+resource "aws_ssm_parameter" "ssh_pub_key" {
+  name        = "/${var.ssm_prefix}/${local.MICROK8S_CONF.MICROK8S_SSH_PUB_KEY.ssm}"
+  description = "microk8s ssh public key in ${local.SPACED_ID_ENV}"
+  type        = "SecureString"
+  value       = aws_key_pair.default.public_key
+}
+
+resource "aws_ssm_parameter" "ssh_priv_key" {
+  name        = "/${var.ssm_prefix}/${local.MICROK8S_CONF.MICROK8S_SSH_PRIV_KEY.ssm}"
+  description = "microk8s ssh private key in ${local.SPACED_ID_ENV}"
+  type        = "SecureString"
+  value       = tls_private_key.default.private_key_pem
+}
+
+resource "aws_ssm_parameter" "ssh_host" {
+  name        = "/${var.ssm_prefix}/${local.MICROK8S_CONF.MICROK8S_SSH_HOST.ssm}"
+  description = "microk8s ssh host in ${local.SPACED_ID_ENV}"
+  type        = "SecureString"
+  value       = aws_instance.default.public_ip
+}

infrastructure/terraform/aws/modules/microk8s/v001/user-data.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+sudo apt update
+sudo snap install microk8s --classic --channel=1.31
+sudo microk8s enable dns ingress
+sudo usermod -a -G microk8s ubuntu
+mkdir -p /home/ubuntu/.kube
+sudo chown -f -R ubuntu /home/ubuntu/.kube
+chmod 0700 /home/ubuntu/.kube
+wget -O- https://carvel.dev/install.sh >install.sh
+sudo bash install.sh
+rm install.sh

infrastructure/terraform/aws/modules/microk8s/v001/variables.tf

@@ -0,0 +1,4 @@
+variable "env" {}
+variable "env_id" {}
+variable "ssm_prefix" {}
+variable "instance_type" {}

k8s/balance-by-account.yml

@@ -32,7 +32,11 @@ spec:
             - name: PGPASSWORD
               value: #@ data.values.PGPASSWORD
             - name: PGHOST
+              #@ if data.values.PGHOST != "localhost":
+              value: #@ data.values.PGHOST
+              #@ else:
               value: postgres
+              #@ end
             - name: PGPORT
               value: #@ "{}".format(data.values.PGPORT)
             - name: PG_MAX_CONNECTIONS

k8s/dev/makefile

@@ -0,0 +1,2 @@
+RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH")
+include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk

k8s/dev/node-port.yml

@@ -0,0 +1,19 @@
+#@ load("@ytt:data", "data")
+#@ def convert_to_node_port(port):
+#@   port_str = str(port)
+#@   new_port_str = "3" + port_str[1:]
+#@   return int(new_port_str)
+#@ end
+apiVersion: v1
+kind: Service
+metadata:
+  name: measure-node-port
+spec:
+  selector:
+    app: measure
+  ports:
+    - protocol: TCP
+      port: #@ data.values.MEASURE_PORT
+      targetPort: #@ data.values.MEASURE_PORT
+      nodePort: #@ convert_to_node_port(data.values.MEASURE_PORT)
+  type: NodePort

k8s/event.yml

@@ -29,14 +29,21 @@ spec:
             - name: PGPASSWORD
               value: #@ data.values.PGPASSWORD
             - name: PGHOST
+              #@ if data.values.PGHOST != "localhost":
+              value: #@ data.values.PGHOST
+              #@ else:
               value: postgres
+              #@ end
             - name: PGPORT
               value: #@ "{}".format(data.values.PGPORT)
             - name: REDIS_DB
               value: #@ "{}".format(data.values.REDIS_DB)
             - name: REDIS_HOST
-              #! todo: remove hardcode
+              #@ if data.values.REDIS_HOST != "localhost":
+              value: #@ data.values.REDIS_HOST
+              #@ else:
               value: redis
+              #@ end
             - name: REDIS_PORT
               value: #@ "{}".format(data.values.REDIS_PORT)
             - name: REDIS_USERNAME

k8s/local/node-ports.yml

@@ -163,4 +163,18 @@ spec:
       port: #@ data.values.PGPORT
       targetPort: #@ data.values.PGPORT
       nodePort: #@ convert_to_node_port(data.values.PGPORT)
+  type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: measure-node-port
+spec:
+  selector:
+    app: measure
+  ports:
+    - protocol: TCP
+      port: #@ data.values.MEASURE_PORT
+      targetPort: #@ data.values.MEASURE_PORT
+      nodePort: #@ convert_to_node_port(data.values.MEASURE_PORT)
   type: NodePort

k8s/makefile

@@ -1,18 +1,65 @@
 RELATIVE_PROJECT_ROOT_PATH=..
 include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk
+SSH_KEY_SSM_SUFFIX=$(shell yq '.infrastructure.terraform.aws.modules.microk8s.env_var.set.MICROK8S_SSH_PRIV_KEY.ssm' $(PROJECT_CONF))
+MICROK8S_SSH_KEY_NAME_PREFIX=$(shell yq '.infrastructure.terraform.aws.modules.microk8s.env_var.set.MICROK8S_SSH_KEY_NAME_PREFIX.default' $(PROJECT_CONF))
+MICROK8S_SSH_USER=$(shell yq '.infrastructure.terraform.aws.modules.microk8s.env_var.set.MICROK8S_SSH_USER.default' $(PROJECT_CONF))
+MICROK8S_MANIFESTS_DIR=$(shell yq '.infrastructure.terraform.aws.modules.microk8s.env_var.set.MICROK8S_MANIFESTS_DIR.default' $(PROJECT_CONF))
+
+# avoid assigning ENV_ID on local commands or when ENV is not assigned
+ifneq ($(origin ENV), undefined)
+ifneq ($(ENV), local)
+    ENV_ID=$(shell cd $(RELATIVE_PROJECT_ROOT_PATH); ENV=$(ENV) bash scripts/print-env-id.sh)
+    ID_ENV=$(ENV_ID)-$(ENV)
+    SSH_KEY=$(ID_ENV).pem
+	MICROK8S_NAME=$(MICROK8S_SSH_KEY_NAME_PREFIX)-$(ID_ENV)
+	MANIFESTS_DIR=/home/$(MICROK8S_SSH_USER)/$(MICROK8S_MANIFESTS_DIR)
+endif
+endif
 
 deploy-local:
 	@$(MAKE) --no-print-directory -C local env ENV=local
-	@$(MAKE) --no-print-directory env ENV=local
-	ytt --data-values-file ./local/$(ENV_FILE_NAME) --data-values-file $(ENV_FILE_NAME) -f . | kubectl apply -f -
+	ytt --data-values-file ./local/$(ENV_FILE_NAME) -f . | kubectl apply -f -
 
 delete-local:
 	@$(MAKE) --no-print-directory -C local env ENV=local
-	@$(MAKE) --no-print-directory env ENV=local
-	ytt --data-values-file ./local/$(ENV_FILE_NAME) --data-values-file $(ENV_FILE_NAME) -f . | kubectl delete -f -
+	ytt --data-values-file ./local/$(ENV_FILE_NAME) -f . | kubectl delete -f -
 
 delete-all-local:
 	kubectl delete all --all -n default
 
 list-all:
-	kubectl get all -n default
+	kubectl get all -n default
+
+### cloud
+
+get-ssh-key:
+	@$(MAKE) -s test-env-arg
+	(cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/get-ssh-key.sh --env $(ENV) --ssm-suffix $(SSH_KEY_SSM_SUFFIX) --dir $(CURDIR))
+
+connect:
+	@$(MAKE) -s test-env-arg
+	@$(MAKE) -s get-ssh-key
+	@K8S_IP=$$(aws ec2 describe-instances --filters "Name=instance-state-name,Values=running" "Name=tag:Name,Values=$(MICROK8S_NAME)" --query "Reservations[].Instances[0].PublicIpAddress" --output text); \
+	ssh -q -o StrictHostKeyChecking=accept-new -o IdentitiesOnly=yes -i $(SSH_KEY) $(MICROK8S_SSH_USER)@$$K8S_IP
+
+deploy:
+	@$(MAKE) -s test-env-arg
+	@$(MAKE) --no-print-directory -C dev env ENV=dev
+	@$(MAKE) -s get-ssh-key
+	@echo "*** wating for ok instance status before deploying..."
+	@INSTANCE_ID=$$(aws ec2 describe-instances --filters "Name=instance-state-name,Values=running" "Name=tag:Name,Values=$(MICROK8S_NAME)" --query "Reservations[].Instances[0].InstanceId" --output text); \
+	aws ec2 wait instance-status-ok --instance-ids $$INSTANCE_ID
+	@$(MAKE) -s deploy-now
+
+deploy-now:
+	@$(MAKE) -s test-env-arg
+	@echo "*** deploying..."
+	@K8S_IP=$$(aws ec2 describe-instances --filters "Name=instance-state-name,Values=running" "Name=tag:Name,Values=$(MICROK8S_NAME)" --query "Reservations[].Instances[0].PublicIpAddress" --output text); \
+	scp -q -o StrictHostKeyChecking=accept-new -o IdentitiesOnly=yes -i $(SSH_KEY) event.yml measure.yml redis.yml $(MICROK8S_SSH_USER)@$$K8S_IP:$(MANIFESTS_DIR); \
+	scp -q -o StrictHostKeyChecking=accept-new -o IdentitiesOnly=yes -i $(SSH_KEY) -r ./$(ENV) $(MICROK8S_SSH_USER)@$$K8S_IP:$(MANIFESTS_DIR)/; \
+	ssh -q -o StrictHostKeyChecking=accept-new -o IdentitiesOnly=yes -i $(SSH_KEY) $(MICROK8S_SSH_USER)@$$K8S_IP 'ytt --data-values-file $(MANIFESTS_DIR)/$(ENV)/$(ENV_FILE_NAME) -f $(MANIFESTS_DIR) | microk8s kubectl apply -f -'
+
+delete-all:
+	@$(MAKE) -s test-env-arg
+	@K8S_IP=$$(aws ec2 describe-instances --filters "Name=instance-state-name,Values=running" "Name=tag:Name,Values=$(MICROK8S_NAME)" --query "Reservations[].Instances[0].PublicIpAddress" --output text); \
+	ssh -q -o StrictHostKeyChecking=accept-new -o IdentitiesOnly=yes -i $(SSH_KEY) $(MICROK8S_SSH_USER)@$$K8S_IP 'microk8s kubectl delete all --all -n default'