From ba88cf7b0127720390dc2e8f60c2a0d3b013dba6 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 19 Sep 2023 21:57:06 -0500 Subject: [PATCH 01/17] Add helm charts to deploy decoder and encoder --- sam-helm-charts/.DS_Store | Bin 0 -> 6148 bytes sam-helm-charts/.helmignore | 23 ++++++++ sam-helm-charts/Chart.yaml | 24 +++++++++ sam-helm-charts/templates/.DS_Store | Bin 0 -> 6148 bytes .../templates/decoder/deployment.yml | 46 ++++++++++++++++ sam-helm-charts/templates/decoder/pvc.yml | 47 +++++++++++++++++ sam-helm-charts/templates/decoder/service.yml | 34 ++++++++++++ .../templates/encoder/deployment.yml | 44 ++++++++++++++++ sam-helm-charts/templates/encoder/pvc.yml | 47 +++++++++++++++++ sam-helm-charts/templates/encoder/service.yml | 26 ++++++++++ sam-helm-charts/values.yaml | 49 ++++++++++++++++++ 11 files changed, 340 insertions(+) create mode 100644 sam-helm-charts/.DS_Store create mode 100644 sam-helm-charts/.helmignore create mode 100644 sam-helm-charts/Chart.yaml create mode 100644 sam-helm-charts/templates/.DS_Store create mode 100644 sam-helm-charts/templates/decoder/deployment.yml create mode 100644 sam-helm-charts/templates/decoder/pvc.yml create mode 100644 sam-helm-charts/templates/decoder/service.yml create mode 100644 sam-helm-charts/templates/encoder/deployment.yml create mode 100644 sam-helm-charts/templates/encoder/pvc.yml create mode 100644 sam-helm-charts/templates/encoder/service.yml create mode 100644 sam-helm-charts/values.yaml diff --git a/sam-helm-charts/.DS_Store b/sam-helm-charts/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..fbe64eae72bb9aeee07bc45e95181c216f302abd GIT binary patch literal 6148 zcmeHKPfNov6i>G4R))}n!j1v21G^1n;-%F21+3^nWwxwnv1`WK*~1w0tY64a;^*7kGdumV?pHV8wm?o&LWnD z#ac8Oej@|8cb3|-F`KYY`6+oFHmTP&=`-u}VK`S39rr}A0T$>GmavSV-wFVNYrvL}C}@`uLVW6b4;` zrAF|8aGeUMQ@MF!aGefzVd7karAD33xSAQhV`k>&g~HYBU>7Q!aaSX?!~ij{%0N{Q zZLI%iU%&rXlW0T?5CeaT0bU#U0}qyDYU|qKu+|FDJ5UsiD>Z(kKu48gh{aOe0#yQb XfgM2CV5t!-AaoH>G*Ckf{3rwOy~0g~ literal 0 HcmV?d00001 diff --git a/sam-helm-charts/.helmignore b/sam-helm-charts/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/sam-helm-charts/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/sam-helm-charts/Chart.yaml b/sam-helm-charts/Chart.yaml new file mode 100644 index 0000000..ece92e3 --- /dev/null +++ b/sam-helm-charts/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: sam +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.27.0" diff --git a/sam-helm-charts/templates/.DS_Store b/sam-helm-charts/templates/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Mon, 25 Sep 2023 14:41:38 -0500 Subject: [PATCH 02/17] Add number of replicas --- sam-helm-charts/templates/decoder/deployment.yml | 2 +- sam-helm-charts/templates/encoder/deployment.yml | 2 +- sam-helm-charts/values.yaml | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sam-helm-charts/templates/decoder/deployment.yml b/sam-helm-charts/templates/decoder/deployment.yml index d16ae5f..0936ff2 100644 --- a/sam-helm-charts/templates/decoder/deployment.yml +++ b/sam-helm-charts/templates/decoder/deployment.yml @@ -9,7 +9,7 @@ metadata: environment: {{ .Values.environment }} release: {{ .Release.Name }} spec: - replicas: 1 + replicas: {{ .Values.decoder.replicas }} selector: matchLabels: app: {{ .Release.Name }}-decoder-app diff --git a/sam-helm-charts/templates/encoder/deployment.yml b/sam-helm-charts/templates/encoder/deployment.yml index 2c8c313..88ab635 100644 --- a/sam-helm-charts/templates/encoder/deployment.yml +++ b/sam-helm-charts/templates/encoder/deployment.yml @@ -9,7 +9,7 @@ metadata: environment: {{ .Values.environment }} release: {{ .Release.Name }} spec: - replicas: 1 + replicas: {{ .Values.encoder.replicas }} selector: matchLabels: app: {{ .Release.Name }}-encoder-app diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index 13d6146..c47a160 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -19,6 +19,7 @@ adminEmail: admin@sam.org # ==================================================== encoder: enabled: true + replicas: 0 image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service tag: gpu-a198226c2ddb2271896f1e950e3b43a3d3bbac48 @@ -32,10 +33,11 @@ encoder: nodeSelector: enabled: true label_key: nodegroup_type - label_value: g5-xlarge-ondemand + label_value: g4dn-xlarge-spot decoder: enabled: true + replicas: 0 image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service tag: cpu-e9b3d1c112f3f26838318e03f52d9f777e548054 @@ -46,4 +48,4 @@ decoder: nodeSelector: enabled: true label_key: nodegroup_type - label_value: admin + label_value: cpu-worker-spot From 9fcc4c683b89368816b0de12f9965eef8ae71f37 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 26 Sep 2023 10:54:22 -0500 Subject: [PATCH 03/17] Update templates for https --- .../templates/decoder/deployment.yml | 3 +- sam-helm-charts/templates/decoder/service.yml | 31 ++++++++++++------- .../templates/encoder/deployment.yml | 3 +- sam-helm-charts/templates/encoder/service.yml | 20 ++++++++++-- sam-helm-charts/values.yaml | 21 +++++-------- 5 files changed, 47 insertions(+), 31 deletions(-) diff --git a/sam-helm-charts/templates/decoder/deployment.yml b/sam-helm-charts/templates/decoder/deployment.yml index 0936ff2..d4c0d90 100644 --- a/sam-helm-charts/templates/decoder/deployment.yml +++ b/sam-helm-charts/templates/decoder/deployment.yml @@ -27,8 +27,7 @@ spec: - "./start" ports: - containerPort: 7080 - # - containerPort: 7081 - # - containerPort: 7082 + name: backend-http env: - name: GitSha value: {{ .Values.decoder.env.GitSha }} diff --git a/sam-helm-charts/templates/decoder/service.yml b/sam-helm-charts/templates/decoder/service.yml index afdf1f4..18fdfe8 100644 --- a/sam-helm-charts/templates/decoder/service.yml +++ b/sam-helm-charts/templates/decoder/service.yml @@ -3,7 +3,21 @@ apiVersion: v1 kind: Service metadata: annotations: + {{- if eq .Values.decoder.serviceType "LoadBalancer" }} service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" + {{- end }} + {{- if and (eq .Values.decoder.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + {{- end }} + {{- if eq .Values.decoder.serviceType "ClusterIP" }} + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod-issuer + {{- end }} + {{- with .Values.decoder.serviceAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} name: {{ .Release.Name }}-decoder-service namespace: {{ .Values.namespace }} labels: @@ -11,24 +25,17 @@ metadata: environment: {{ .Values.environment }} release: {{ .Release.Name }} spec: + type: {{.Values.decoder.serviceType}} ports: - port: 80 - targetPort: 7080 + targetPort: backend-http protocol: TCP - name: "http7080" - # - port: 7081 - # targetPort: 7081 - # protocol: TCP - # name: "http7081" - # - port: 7082 - # targetPort: 7082 - # protocol: TCP - # name: "http7082" + name: http - port: 443 - targetPort: http + targetPort: backend-http protocol: TCP name: https - type: LoadBalancer + selector: app: {{ .Release.Name }}-decoder-app {{- end }} diff --git a/sam-helm-charts/templates/encoder/deployment.yml b/sam-helm-charts/templates/encoder/deployment.yml index 88ab635..d21a1fe 100644 --- a/sam-helm-charts/templates/encoder/deployment.yml +++ b/sam-helm-charts/templates/encoder/deployment.yml @@ -26,7 +26,8 @@ spec: - -c - "./start" ports: - - containerPort: 8080 + - containerPort: 7080 + name: backend-http env: - name: GitSha value: {{ .Values.encoder.env.GitSha }} diff --git a/sam-helm-charts/templates/encoder/service.yml b/sam-helm-charts/templates/encoder/service.yml index f3c2236..f8db3fa 100644 --- a/sam-helm-charts/templates/encoder/service.yml +++ b/sam-helm-charts/templates/encoder/service.yml @@ -3,7 +3,21 @@ apiVersion: v1 kind: Service metadata: annotations: + {{- if eq .Values.encoder.serviceType "LoadBalancer" }} service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" + {{- end }} + {{- if and (eq .Values.encoder.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + {{- end }} + {{- if eq .Values.encoder.serviceType "ClusterIP" }} + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod-issuer + {{- end }} + {{- with .Values.encoder.serviceAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} name: {{ .Release.Name }}-encoder-service namespace: {{ .Values.namespace }} labels: @@ -11,16 +25,16 @@ metadata: environment: {{ .Values.environment }} release: {{ .Release.Name }} spec: + type: {{.Values.encoder.serviceType}} ports: - port: 80 - targetPort: 8080 + targetPort: backend-http protocol: TCP name: http - port: 443 - targetPort: http + targetPort: backend-http protocol: TCP name: https - type: LoadBalancer selector: app: {{ .Release.Name }}-encoder-app {{- end }} diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index c47a160..a2e6eb8 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -1,25 +1,17 @@ ## Namespace namespace: default - -## Enviroment, value for NODE_ENV -environment: staging - -# Cloud provider cloudProvider: aws -AWS_SSL_ARN: false -serviceType: LoadBalancer - -# Domain -domain: staging.sam.org -adminEmail: admin@sam.org +## for ingress +AWS_SSL_ARN: arn:aws:acm:us-west-1:685980093638:certificate/a4ea3620-d410-456b-ad13-a78fe0de9b9b +domain: segmentanythingservice.com # ==================================================== # encoder # ==================================================== encoder: enabled: true - replicas: 0 + replicas: 1 image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service tag: gpu-a198226c2ddb2271896f1e950e3b43a3d3bbac48 @@ -37,7 +29,10 @@ encoder: decoder: enabled: true - replicas: 0 + replicas: 1 + serviceType: LoadBalancer + serviceAnnotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service tag: cpu-e9b3d1c112f3f26838318e03f52d9f777e548054 From ca9e4f89889fe605bfdcf71fd17f3877f222633e Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 26 Sep 2023 11:02:00 -0500 Subject: [PATCH 04/17] Persistent disk for decoder to start faster --- sam-helm-charts/templates/encoder/deployment.yml | 14 +++++++------- sam-helm-charts/values.yaml | 13 ++++++------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/sam-helm-charts/templates/encoder/deployment.yml b/sam-helm-charts/templates/encoder/deployment.yml index d21a1fe..9bcd0db 100644 --- a/sam-helm-charts/templates/encoder/deployment.yml +++ b/sam-helm-charts/templates/encoder/deployment.yml @@ -31,13 +31,13 @@ spec: env: - name: GitSha value: {{ .Values.encoder.env.GitSha }} - # volumeMounts: - # - name: sam-ebs-volume - # mountPath: {{ .Values.encoder.persistenceDisk.MODEL_DIR }} - # volumes: - # - name: sam-ebs-volume - # persistentVolumeClaim: - # claimName: {{ .Release.Name }}-encoder-ebs-volume-clain + volumeMounts: + - name: sam-ebs-volume + mountPath: {{ .Values.encoder.persistenceDisk.MODEL_DIR }} + volumes: + - name: sam-ebs-volume + persistentVolumeClaim: + claimName: {{ .Release.Name }}-encoder-ebs-volume-clain {{- if .Values.encoder.nodeSelector.enabled }} nodeSelector: {{ .Values.encoder.nodeSelector.label_key }} : {{ .Values.encoder.nodeSelector.label_value }} diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index a2e6eb8..c2af06c 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -1,11 +1,8 @@ ## Namespace namespace: default cloudProvider: aws - -## for ingress AWS_SSL_ARN: arn:aws:acm:us-west-1:685980093638:certificate/a4ea3620-d410-456b-ad13-a78fe0de9b9b domain: segmentanythingservice.com - # ==================================================== # encoder # ==================================================== @@ -18,15 +15,17 @@ encoder: env: GitSha: 47553b0190b63932d832454c044f5362be2967ab persistenceDisk: - enabled: false - AWS_ElasticBlockStore_volumeID: vol-05d59eb2671da15ee - AWS_ElasticBlockStore_size: 100Gi + enabled: true + AWS_ElasticBlockStore_volumeID: vol-0888a8922504d118a + AWS_ElasticBlockStore_size: 50Gi MODEL_DIR: /home/model-server/volume nodeSelector: enabled: true label_key: nodegroup_type label_value: g4dn-xlarge-spot - +# ==================================================== +# decoder +# ==================================================== decoder: enabled: true replicas: 1 From 52d92d12167a12654fdf49da6b51c8f73672953c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 26 Sep 2023 11:47:30 -0500 Subject: [PATCH 05/17] Update for decoder disk --- sam-helm-charts/templates/decoder/deployment.yml | 14 +++++++------- sam-helm-charts/values.yaml | 5 ++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sam-helm-charts/templates/decoder/deployment.yml b/sam-helm-charts/templates/decoder/deployment.yml index d4c0d90..d0febe3 100644 --- a/sam-helm-charts/templates/decoder/deployment.yml +++ b/sam-helm-charts/templates/decoder/deployment.yml @@ -31,13 +31,13 @@ spec: env: - name: GitSha value: {{ .Values.decoder.env.GitSha }} - # volumeMounts: - # - name: sam-ebs-volume - # mountPath: {{ .Values.decoder.persistenceDisk.MODEL_DIR }} - # volumes: - # - name: sam-ebs-volume - # persistentVolumeClaim: - # claimName: {{ .Release.Name }}-decoder-ebs-volume-clain + volumeMounts: + - name: sam-ebs-volume + mountPath: {{ .Values.decoder.persistenceDisk.MODEL_DIR }} + volumes: + - name: sam-ebs-volume + persistentVolumeClaim: + claimName: {{ .Release.Name }}-decoder-ebs-volume-clain {{- if .Values.decoder.nodeSelector.enabled }} nodeSelector: {{ .Values.decoder.nodeSelector.label_key }} : {{ .Values.decoder.nodeSelector.label_value }} diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index c2af06c..d0d1b96 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -15,7 +15,7 @@ encoder: env: GitSha: 47553b0190b63932d832454c044f5362be2967ab persistenceDisk: - enabled: true + enabled: false AWS_ElasticBlockStore_volumeID: vol-0888a8922504d118a AWS_ElasticBlockStore_size: 50Gi MODEL_DIR: /home/model-server/volume @@ -39,6 +39,9 @@ decoder: GitSha: 47553b0190b63932d832454c044f5362be2967ab persistenceDisk: enabled: false + AWS_ElasticBlockStore_volumeID: vol-084afed32a29fdc9a + AWS_ElasticBlockStore_size: 50Gi + MODEL_DIR: /home/model-server/volume nodeSelector: enabled: true label_key: nodegroup_type From a0b451f30d90547015eb397cc19a9a154ed750b4 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 27 Sep 2023 14:20:49 -0500 Subject: [PATCH 06/17] Update charts --- .../templates/decoder/deployment.yml | 4 +++ sam-helm-charts/templates/decoder/pvc.yml | 14 ----------- .../templates/encoder/deployment.yml | 4 +++ sam-helm-charts/templates/encoder/pvc.yml | 14 ----------- sam-helm-charts/values.yaml | 25 +++++++++++++++++-- 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/sam-helm-charts/templates/decoder/deployment.yml b/sam-helm-charts/templates/decoder/deployment.yml index d0febe3..d7983b2 100644 --- a/sam-helm-charts/templates/decoder/deployment.yml +++ b/sam-helm-charts/templates/decoder/deployment.yml @@ -31,13 +31,17 @@ spec: env: - name: GitSha value: {{ .Values.decoder.env.GitSha }} + {{- if .Values.decoder.persistenceDisk.enabled -}} volumeMounts: - name: sam-ebs-volume mountPath: {{ .Values.decoder.persistenceDisk.MODEL_DIR }} + {{- end }} + {{- if .Values.decoder.persistenceDisk.enabled -}} volumes: - name: sam-ebs-volume persistentVolumeClaim: claimName: {{ .Release.Name }}-decoder-ebs-volume-clain + {{- end }} {{- if .Values.decoder.nodeSelector.enabled }} nodeSelector: {{ .Values.decoder.nodeSelector.label_key }} : {{ .Values.decoder.nodeSelector.label_value }} diff --git a/sam-helm-charts/templates/decoder/pvc.yml b/sam-helm-charts/templates/decoder/pvc.yml index 3c4e4fd..e45de07 100644 --- a/sam-helm-charts/templates/decoder/pvc.yml +++ b/sam-helm-charts/templates/decoder/pvc.yml @@ -29,19 +29,5 @@ spec: resources: requests: storage: {{ .Values.decoder.persistenceDisk.AWS_ElasticBlockStore_size }} -# {{- else }} -# --- -# apiVersion: v1 -# kind: PersistentVolumeClaim -# metadata: -# name: {{ .Release.Name }}-decoder-ebs-volume-clain -# namespace: {{ .Values.namespace }} -# spec: -# accessModes: -# - ReadWriteOnce -# storageClassName: "gp2" -# resources: -# requests: -# storage: 20Gi {{- end }} {{- end }} diff --git a/sam-helm-charts/templates/encoder/deployment.yml b/sam-helm-charts/templates/encoder/deployment.yml index 9bcd0db..f077083 100644 --- a/sam-helm-charts/templates/encoder/deployment.yml +++ b/sam-helm-charts/templates/encoder/deployment.yml @@ -31,13 +31,17 @@ spec: env: - name: GitSha value: {{ .Values.encoder.env.GitSha }} + {{- if .Values.encoder.persistenceDisk.enabled -}} volumeMounts: - name: sam-ebs-volume mountPath: {{ .Values.encoder.persistenceDisk.MODEL_DIR }} + {{- end }} + {{- if .Values.encoder.persistenceDisk.enabled -}} volumes: - name: sam-ebs-volume persistentVolumeClaim: claimName: {{ .Release.Name }}-encoder-ebs-volume-clain + {{- end }} {{- if .Values.encoder.nodeSelector.enabled }} nodeSelector: {{ .Values.encoder.nodeSelector.label_key }} : {{ .Values.encoder.nodeSelector.label_value }} diff --git a/sam-helm-charts/templates/encoder/pvc.yml b/sam-helm-charts/templates/encoder/pvc.yml index 17edafa..5a809c1 100644 --- a/sam-helm-charts/templates/encoder/pvc.yml +++ b/sam-helm-charts/templates/encoder/pvc.yml @@ -29,19 +29,5 @@ spec: resources: requests: storage: {{ .Values.encoder.persistenceDisk.AWS_ElasticBlockStore_size }} -# {{- else }} -# --- -# apiVersion: v1 -# kind: PersistentVolumeClaim -# metadata: -# name: {{ .Release.Name }}-encoder-ebs-volume-clain -# namespace: {{ .Values.namespace }} -# spec: -# accessModes: -# - ReadWriteOnce -# storageClassName: "gp2" -# resources: -# requests: -# storage: 20Gi {{- end }} {{- end }} diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index d0d1b96..c87133b 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -9,6 +9,9 @@ domain: segmentanythingservice.com encoder: enabled: true replicas: 1 + serviceType: LoadBalancer + serviceAnnotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service tag: gpu-a198226c2ddb2271896f1e950e3b43a3d3bbac48 @@ -22,12 +25,12 @@ encoder: nodeSelector: enabled: true label_key: nodegroup_type - label_value: g4dn-xlarge-spot + label_value: g4dn-xlarge-ondemand # ==================================================== # decoder # ==================================================== decoder: - enabled: true + enabled: false replicas: 1 serviceType: LoadBalancer serviceAnnotations: @@ -46,3 +49,21 @@ decoder: enabled: true label_key: nodegroup_type label_value: cpu-worker-spot +# ==================================================== +# Development +# ==================================================== +dev: + enabled: false + replicas: 1 + serviceType: LoadBalancer + serviceAnnotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" + image: + name: 685980093638.dkr.ecr.us-west-1.amazonaws.com/sam-builder + tag: v1 + env: + GitSha: 47553b0190b63932d832454c044f5362be2967ab + nodeSelector: + enabled: true + label_key: nodegroup_type + label_value: g4dn-xlarge-ondemand From 1f83c51a5a56e5ff719d9dd4f9314666fea7aeb9 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 27 Sep 2023 16:57:50 -0500 Subject: [PATCH 07/17] Fix port for encoder --- sam-helm-charts/templates/encoder/deployment.yml | 2 +- sam-helm-charts/values.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sam-helm-charts/templates/encoder/deployment.yml b/sam-helm-charts/templates/encoder/deployment.yml index f077083..0a82115 100644 --- a/sam-helm-charts/templates/encoder/deployment.yml +++ b/sam-helm-charts/templates/encoder/deployment.yml @@ -26,7 +26,7 @@ spec: - -c - "./start" ports: - - containerPort: 7080 + - containerPort: 8080 name: backend-http env: - name: GitSha diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index c87133b..9eb0a9c 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -25,12 +25,12 @@ encoder: nodeSelector: enabled: true label_key: nodegroup_type - label_value: g4dn-xlarge-ondemand + label_value: g4dn-xlarge-spot # ==================================================== # decoder # ==================================================== decoder: - enabled: false + enabled: true replicas: 1 serviceType: LoadBalancer serviceAnnotations: @@ -53,14 +53,14 @@ decoder: # Development # ==================================================== dev: - enabled: false + enabled: true replicas: 1 serviceType: LoadBalancer serviceAnnotations: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-west-1.amazonaws.com/sam-builder - tag: v1 + tag: develop.v1 env: GitSha: 47553b0190b63932d832454c044f5362be2967ab nodeSelector: From 4122947c70937d074525578c4155797f97e32d8b Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 27 Sep 2023 17:10:09 -0500 Subject: [PATCH 08/17] Development env container --- sam-helm-charts/templates/dev/deployment.yml | 34 ++++++++++++++++++++ sam-helm-charts/templates/dev/service.yml | 26 +++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 sam-helm-charts/templates/dev/deployment.yml create mode 100644 sam-helm-charts/templates/dev/service.yml diff --git a/sam-helm-charts/templates/dev/deployment.yml b/sam-helm-charts/templates/dev/deployment.yml new file mode 100644 index 0000000..e334242 --- /dev/null +++ b/sam-helm-charts/templates/dev/deployment.yml @@ -0,0 +1,34 @@ +{{- if .Values.dev.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-dev + namespace: {{ .Values.namespace }} + labels: + component: dev + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.dev.replicas }} + selector: + matchLabels: + app: {{ .Release.Name }}-dev-app + template: + metadata: + labels: + app: {{ .Release.Name }}-dev-app + spec: + containers: + - name: dev + image: "{{ .Values.dev.image.name }}:{{ .Values.dev.image.tag }}" + ports: + - containerPort: 8888 + name: backend-http + env: + - name: GitSha + value: {{ .Values.dev.env.GitSha }} + {{- if .Values.dev.nodeSelector.enabled }} + nodeSelector: + {{ .Values.dev.nodeSelector.label_key }} : {{ .Values.dev.nodeSelector.label_value }} + {{- end }} +{{- end }} diff --git a/sam-helm-charts/templates/dev/service.yml b/sam-helm-charts/templates/dev/service.yml new file mode 100644 index 0000000..5709012 --- /dev/null +++ b/sam-helm-charts/templates/dev/service.yml @@ -0,0 +1,26 @@ +{{- if .Values.dev.enabled -}} +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" + name: {{ .Release.Name }}-dev-service + namespace: {{ .Values.namespace }} + labels: + component: dev-service + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + type: {{.Values.decoder.serviceType}} + ports: + - port: 80 + targetPort: backend-http + protocol: TCP + name: http + - port: 443 + targetPort: backend-http + protocol: TCP + name: https + selector: + app: {{ .Release.Name }}-decoder-app +{{- end }} From 26decae3e90882d4b59d0404faa5195d6689a1fe Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 9 Nov 2023 09:24:44 +0700 Subject: [PATCH 09/17] Remove develop config for sam --- sam-helm-charts/templates/dev/deployment.yml | 34 -------------------- sam-helm-charts/templates/dev/service.yml | 26 --------------- sam-helm-charts/values.yaml | 20 +----------- 3 files changed, 1 insertion(+), 79 deletions(-) delete mode 100644 sam-helm-charts/templates/dev/deployment.yml delete mode 100644 sam-helm-charts/templates/dev/service.yml diff --git a/sam-helm-charts/templates/dev/deployment.yml b/sam-helm-charts/templates/dev/deployment.yml deleted file mode 100644 index e334242..0000000 --- a/sam-helm-charts/templates/dev/deployment.yml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.dev.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Release.Name }}-dev - namespace: {{ .Values.namespace }} - labels: - component: dev - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.dev.replicas }} - selector: - matchLabels: - app: {{ .Release.Name }}-dev-app - template: - metadata: - labels: - app: {{ .Release.Name }}-dev-app - spec: - containers: - - name: dev - image: "{{ .Values.dev.image.name }}:{{ .Values.dev.image.tag }}" - ports: - - containerPort: 8888 - name: backend-http - env: - - name: GitSha - value: {{ .Values.dev.env.GitSha }} - {{- if .Values.dev.nodeSelector.enabled }} - nodeSelector: - {{ .Values.dev.nodeSelector.label_key }} : {{ .Values.dev.nodeSelector.label_value }} - {{- end }} -{{- end }} diff --git a/sam-helm-charts/templates/dev/service.yml b/sam-helm-charts/templates/dev/service.yml deleted file mode 100644 index 5709012..0000000 --- a/sam-helm-charts/templates/dev/service.yml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.dev.enabled -}} -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" - name: {{ .Release.Name }}-dev-service - namespace: {{ .Values.namespace }} - labels: - component: dev-service - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - type: {{.Values.decoder.serviceType}} - ports: - - port: 80 - targetPort: backend-http - protocol: TCP - name: http - - port: 443 - targetPort: backend-http - protocol: TCP - name: https - selector: - app: {{ .Release.Name }}-decoder-app -{{- end }} diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index 9eb0a9c..b777eb4 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -48,22 +48,4 @@ decoder: nodeSelector: enabled: true label_key: nodegroup_type - label_value: cpu-worker-spot -# ==================================================== -# Development -# ==================================================== -dev: - enabled: true - replicas: 1 - serviceType: LoadBalancer - serviceAnnotations: - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" - image: - name: 685980093638.dkr.ecr.us-west-1.amazonaws.com/sam-builder - tag: develop.v1 - env: - GitSha: 47553b0190b63932d832454c044f5362be2967ab - nodeSelector: - enabled: true - label_key: nodegroup_type - label_value: g4dn-xlarge-ondemand + label_value: cpu-worker-spot-2x From 106e2b1f16eebdb21ca70a17220b0900f6be48fc Mon Sep 17 00:00:00 2001 From: Rub21 Date: Sat, 25 Nov 2023 10:37:31 +0700 Subject: [PATCH 10/17] Update gitsha for containers --- sam-helm-charts/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index b777eb4..5b0241c 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -14,9 +14,9 @@ encoder: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service - tag: gpu-a198226c2ddb2271896f1e950e3b43a3d3bbac48 + tag: gpu-6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 env: - GitSha: 47553b0190b63932d832454c044f5362be2967ab + GitSha: 6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 persistenceDisk: enabled: false AWS_ElasticBlockStore_volumeID: vol-0888a8922504d118a @@ -37,9 +37,9 @@ decoder: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service - tag: cpu-e9b3d1c112f3f26838318e03f52d9f777e548054 + tag: cpu-6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 env: - GitSha: 47553b0190b63932d832454c044f5362be2967ab + GitSha: 6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 persistenceDisk: enabled: false AWS_ElasticBlockStore_volumeID: vol-084afed32a29fdc9a From 5667a4181361dda90fed19e78799270c9189228f Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 10:27:36 -0500 Subject: [PATCH 11/17] Version that sam is woking fine --- sam-helm-charts/values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index 5b0241c..1c999a3 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -14,9 +14,9 @@ encoder: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service - tag: gpu-6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 + tag: gpu-9f21a4918e0ead256221e0524a9ebefe7ddcc55b env: - GitSha: 6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 + GitSha: 9f21a4918e0ead256221e0524a9ebefe7ddcc55b persistenceDisk: enabled: false AWS_ElasticBlockStore_volumeID: vol-0888a8922504d118a @@ -25,7 +25,7 @@ encoder: nodeSelector: enabled: true label_key: nodegroup_type - label_value: g4dn-xlarge-spot + label_value: g4dn-xlarge-ondemand # ==================================================== # decoder # ==================================================== @@ -37,9 +37,9 @@ decoder: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service - tag: cpu-6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 + tag: cpu-9f21a4918e0ead256221e0524a9ebefe7ddcc55b env: - GitSha: 6a4183a9fd8b5ae33d9a8b30a474ed7f27d6a312 + GitSha: 9f21a4918e0ead256221e0524a9ebefe7ddcc55b persistenceDisk: enabled: false AWS_ElasticBlockStore_volumeID: vol-084afed32a29fdc9a From 5b89caf6186cc06b440d06a809944ba327dcab77 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 12:04:04 -0500 Subject: [PATCH 12/17] Add github action to deploy stack in k8s --- .github/workflows/k8s.yml | 55 +++++++++++++++++++++++++++++++++++++ sam-helm-charts/values.yaml | 8 +++--- 2 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/k8s.yml diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml new file mode 100644 index 0000000..bd3354a --- /dev/null +++ b/.github/workflows/k8s.yml @@ -0,0 +1,55 @@ +name: Deploy stack with Helm +on: + workflow_run: + workflows: ["model", "ecr"] + types: + - completed + branches: [main, '*'] + +jobs: + k8s: + runs-on: ubuntu-22.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v1 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Get Git SHA + run: echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Update values.yaml + run: sed -i 's/gitsha-placeholder/${{ env.GIT_SHA }}/g' sam-helm-charts/values.yaml + + - name: AWS Credentials + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-1 + + - name: Setup Kubectl and Helm Dependencies + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" + + - name: Update kube-config + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: aws eks --region us-west-1 update-kubeconfig --name devseed-k8s-staging + + - name: Update kube-config + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: kubectl get pods + + - name: Update values.yaml + run: cat sam-helm-charts/values.yaml + + - name: Staging - helm deploy + if: github.ref == 'refs/heads/staging' + run: cd sam-helm-charts/ && helm upgrade staging . -i -f values.yaml --set environment=staging + + - name: Production - helm deploy + if: github.ref == 'refs/heads/main' + run: cd sam-helm-charts/ && helm upgrade production . -i -f values.yaml --set environment=production diff --git a/sam-helm-charts/values.yaml b/sam-helm-charts/values.yaml index 1c999a3..9f095f7 100644 --- a/sam-helm-charts/values.yaml +++ b/sam-helm-charts/values.yaml @@ -14,9 +14,9 @@ encoder: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service - tag: gpu-9f21a4918e0ead256221e0524a9ebefe7ddcc55b + tag: gpu-gitsha-placeholder env: - GitSha: 9f21a4918e0ead256221e0524a9ebefe7ddcc55b + GitSha: gitsha-placeholder persistenceDisk: enabled: false AWS_ElasticBlockStore_volumeID: vol-0888a8922504d118a @@ -37,9 +37,9 @@ decoder: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" image: name: 685980093638.dkr.ecr.us-east-1.amazonaws.com/sam-service - tag: cpu-9f21a4918e0ead256221e0524a9ebefe7ddcc55b + tag: cpu-gitsha-placeholder env: - GitSha: 9f21a4918e0ead256221e0524a9ebefe7ddcc55b + GitSha: gitsha-placeholder persistenceDisk: enabled: false AWS_ElasticBlockStore_volumeID: vol-084afed32a29fdc9a From f192da80f0edeaa9d5b1605c088dea90d0a9a508 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 12:16:13 -0500 Subject: [PATCH 13/17] Update k8s action Update k8s action Update action to run after other jobs Fix process Remove push and pull_request Run jobs after other using name Check job on push Update gitignore --- .github/workflows/k8s.yml | 94 +++++++++++++++++++-------------------- .gitignore | 2 +- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index bd3354a..ad2dd0d 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -1,55 +1,53 @@ -name: Deploy stack with Helm +name: Deploy stack in k8s on: + push: workflow_run: - workflows: ["model", "ecr"] - types: - - completed - branches: [main, '*'] - + workflows: ["AWS ECR Builder", "AWS Model Builder"] + types: [completed] jobs: k8s: runs-on: ubuntu-22.04 timeout-minutes: 120 steps: - - uses: actions/checkout@v1 - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Get Git SHA - run: echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV - - - name: Update values.yaml - run: sed -i 's/gitsha-placeholder/${{ env.GIT_SHA }}/g' sam-helm-charts/values.yaml - - - name: AWS Credentials - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-1 - - - name: Setup Kubectl and Helm Dependencies - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" - - - name: Update kube-config - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: aws eks --region us-west-1 update-kubeconfig --name devseed-k8s-staging - - - name: Update kube-config - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: kubectl get pods - - - name: Update values.yaml - run: cat sam-helm-charts/values.yaml - - - name: Staging - helm deploy - if: github.ref == 'refs/heads/staging' - run: cd sam-helm-charts/ && helm upgrade staging . -i -f values.yaml --set environment=staging - - - name: Production - helm deploy - if: github.ref == 'refs/heads/main' - run: cd sam-helm-charts/ && helm upgrade production . -i -f values.yaml --set environment=production + - uses: actions/checkout@v1 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + + - name: Get Git SHA + run: echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Update values.yaml + run: sed -i 's/gitsha-placeholder/${{ env.GIT_SHA }}/g' sam-helm-charts/values.yaml + + - name: AWS Credentials + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-1 + + - name: Setup Kubectl and Helm Dependencies + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" + + - name: Update kube-config + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: aws eks --region us-west-1 update-kubeconfig --name devseed-k8s-staging + + - name: Update kube-config + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: kubectl get pods + + - name: Update values.yaml + run: cat sam-helm-charts/values.yaml + + - name: Staging - helm deploy + if: github.ref == 'refs/heads/staging' + run: cd sam-helm-charts/ && helm upgrade staging . -i -f values.yaml --set environment=staging + + - name: Production - helm deploy + if: github.ref == 'refs/heads/main' + run: cd sam-helm-charts/ && helm upgrade production . -i -f values.yaml --set environment=production diff --git a/.gitignore b/.gitignore index 4f70967..c41005c 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ sam_decode_mar/ scripts/save_npz_embeds.py src/sam_serve/__pycache__/ tests/__pycache__/ - +.secrets From c645c35a940487f0bec8dc5b7f6ec3d0bc5e49ea Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 15:19:52 -0500 Subject: [PATCH 14/17] Run jobs in sequence --- .github/workflows/k8s.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index ad2dd0d..a2b6928 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -1,9 +1,9 @@ name: Deploy stack in k8s on: - push: workflow_run: - workflows: ["AWS ECR Builder", "AWS Model Builder"] - types: [completed] + workflows: ["AWS ECR Builder"] + types: + - completed jobs: k8s: runs-on: ubuntu-22.04 From c4d4abdabc0beb2aa7bb28169e07e37f5507b4fe Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 15:31:58 -0500 Subject: [PATCH 15/17] Run jobs in sequence - fixed --- .github/workflows/ecr.yml | 1 + .github/workflows/k8s.yml | 6 +----- .github/workflows/model.yml | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ecr.yml b/.github/workflows/ecr.yml index 9596f3c..85f3613 100644 --- a/.github/workflows/ecr.yml +++ b/.github/workflows/ecr.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - staging pull_request: types: - opened diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index a2b6928..95c309e 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -1,7 +1,7 @@ name: Deploy stack in k8s on: workflow_run: - workflows: ["AWS ECR Builder"] + workflows: ["AWS ECR Builder", "AWS Model Builder"] types: - completed jobs: @@ -37,10 +37,6 @@ jobs: if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' run: aws eks --region us-west-1 update-kubeconfig --name devseed-k8s-staging - - name: Update kube-config - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: kubectl get pods - - name: Update values.yaml run: cat sam-helm-charts/values.yaml diff --git a/.github/workflows/model.yml b/.github/workflows/model.yml index 17229cc..f082a5f 100644 --- a/.github/workflows/model.yml +++ b/.github/workflows/model.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - staging pull_request: types: - opened From 948c96d3b41281f69f23f0b39c1dd63475e22d0c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 15:55:41 -0500 Subject: [PATCH 16/17] Enable in push and pull_request --- .github/workflows/k8s.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index 95c309e..b1330c9 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -1,5 +1,15 @@ name: Deploy stack in k8s on: + push: + branches: + - main + - staging + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review workflow_run: workflows: ["AWS ECR Builder", "AWS Model Builder"] types: @@ -9,7 +19,7 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 120 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Setup python uses: actions/setup-python@v2 with: From b1f3a8cdc8e2cb65a52ac0606e4133dd660356c2 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Dec 2023 15:57:34 -0500 Subject: [PATCH 17/17] Remove staging --- .github/workflows/ecr.yml | 1 - .github/workflows/k8s.yml | 1 - .github/workflows/model.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/ecr.yml b/.github/workflows/ecr.yml index 85f3613..9596f3c 100644 --- a/.github/workflows/ecr.yml +++ b/.github/workflows/ecr.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - staging pull_request: types: - opened diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index b1330c9..a4876d5 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -3,7 +3,6 @@ on: push: branches: - main - - staging pull_request: types: - opened diff --git a/.github/workflows/model.yml b/.github/workflows/model.yml index f082a5f..17229cc 100644 --- a/.github/workflows/model.yml +++ b/.github/workflows/model.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - staging pull_request: types: - opened