Merge branch 'main' into fix-version

developmentseed · Dec 14, 2023 · 7c2bcd7 · 7c2bcd7
2 parents 213335e + 76259a0
commit 7c2bcd7
Show file tree

Hide file tree

Showing 13 changed files with 402 additions and 1 deletion.
diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml
@@ -0,0 +1,58 @@
+name: Deploy stack in k8s
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+  workflow_run:
+    workflows: ["AWS ECR Builder", "AWS Model Builder"]
+    types:
+      - completed
+jobs:
+  k8s:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
+
+      - name: Get Git SHA
+        run: echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+
+      - name: Update values.yaml
+        run: sed -i 's/gitsha-placeholder/${{ env.GIT_SHA }}/g' sam-helm-charts/values.yaml
+
+      - name: AWS Credentials
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-1
+
+      - name: Setup Kubectl and Helm Dependencies
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
+        run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n"
+
+      - name: Update kube-config
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
+        run: aws eks --region us-west-1 update-kubeconfig --name devseed-k8s-staging
+
+      - name: Update values.yaml
+        run: cat sam-helm-charts/values.yaml
+
+      - name: Staging - helm deploy
+        if: github.ref == 'refs/heads/staging'
+        run: cd sam-helm-charts/ && helm upgrade staging . -i -f values.yaml --set environment=staging
+
+      - name: Production - helm deploy
+        if: github.ref == 'refs/heads/main'
+        run: cd sam-helm-charts/ && helm upgrade production . -i -f values.yaml --set environment=production
diff --git a/.gitignore b/.gitignore
@@ -41,4 +41,4 @@ sam_decode_mar/
 scripts/save_npz_embeds.py
 src/sam_serve/__pycache__/
 tests/__pycache__/
-
+.secrets
diff --git a/sam-helm-charts/.DS_Store b/sam-helm-charts/.DS_Store
diff --git a/sam-helm-charts/.helmignore b/sam-helm-charts/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/sam-helm-charts/Chart.yaml b/sam-helm-charts/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: sam
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.27.0"
diff --git a/sam-helm-charts/templates/.DS_Store b/sam-helm-charts/templates/.DS_Store
diff --git a/sam-helm-charts/templates/decoder/deployment.yml b/sam-helm-charts/templates/decoder/deployment.yml
@@ -0,0 +1,49 @@
+{{- if .Values.decoder.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-decoder
+  namespace: {{ .Values.namespace }}
+  labels:
+    component: decoder
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  replicas: {{ .Values.decoder.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-decoder-app
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-decoder-app
+    spec:
+      containers:
+        - name: decoder
+          image: "{{ .Values.decoder.image.name }}:{{ .Values.decoder.image.tag }}"
+          command:
+            - sh
+            - -c
+            - "./start"
+          ports:
+            - containerPort: 7080
+              name: backend-http
+          env:
+            - name: GitSha
+              value: {{ .Values.decoder.env.GitSha }}
+          {{- if .Values.decoder.persistenceDisk.enabled -}}
+          volumeMounts:
+            - name: sam-ebs-volume
+              mountPath: {{ .Values.decoder.persistenceDisk.MODEL_DIR }}
+          {{- end }}
+      {{- if .Values.decoder.persistenceDisk.enabled -}}
+      volumes:
+        - name: sam-ebs-volume
+          persistentVolumeClaim:
+            claimName:  {{ .Release.Name }}-decoder-ebs-volume-clain
+      {{- end }}
+      {{- if .Values.decoder.nodeSelector.enabled }}
+      nodeSelector:
+        {{ .Values.decoder.nodeSelector.label_key }} : {{ .Values.decoder.nodeSelector.label_value }}
+      {{- end }}
+{{- end }}
diff --git a/sam-helm-charts/templates/decoder/pvc.yml b/sam-helm-charts/templates/decoder/pvc.yml
@@ -0,0 +1,33 @@
+{{- if .Values.decoder.enabled -}}
+{{- if .Values.decoder.persistenceDisk.enabled -}}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Release.Name }}-decoder-ebs-volume
+  namespace: {{ .Values.namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: "gp2"
+  capacity:
+    storage: {{ .Values.decoder.persistenceDisk.AWS_ElasticBlockStore_size }}
+  awsElasticBlockStore:
+    volumeID: {{ .Values.decoder.persistenceDisk.AWS_ElasticBlockStore_volumeID }}
+    fsType: ext4
+  persistentVolumeReclaimPolicy: Retain
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Release.Name }}-decoder-ebs-volume-clain
+  namespace: {{ .Values.namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: "gp2"
+  volumeName: {{ .Release.Name }}-decoder-ebs-volume
+  resources:
+    requests:
+      storage: {{ .Values.decoder.persistenceDisk.AWS_ElasticBlockStore_size }}
+{{- end }}
+{{- end }}
diff --git a/sam-helm-charts/templates/decoder/service.yml b/sam-helm-charts/templates/decoder/service.yml
@@ -0,0 +1,41 @@
+{{- if .Values.decoder.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    {{- if eq .Values.decoder.serviceType "LoadBalancer" }}
+    service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300"
+    {{- end }}
+    {{- if and (eq .Values.decoder.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }}
+    service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }}
+    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
+    service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https
+    {{- end }}
+    {{- if eq .Values.decoder.serviceType "ClusterIP" }}
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: letsencrypt-prod-issuer
+    {{- end }}
+    {{- with .Values.decoder.serviceAnnotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  name: {{ .Release.Name }}-decoder-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    component: decoder-service
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  type: {{.Values.decoder.serviceType}}
+  ports:
+    - port: 80
+      targetPort: backend-http
+      protocol: TCP
+      name: http
+    - port: 443
+      targetPort: backend-http
+      protocol: TCP
+      name: https
+
+  selector:
+    app:  {{ .Release.Name }}-decoder-app
+{{- end }}
diff --git a/sam-helm-charts/templates/encoder/deployment.yml b/sam-helm-charts/templates/encoder/deployment.yml
@@ -0,0 +1,49 @@
+{{- if .Values.encoder.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-encoder
+  namespace: {{ .Values.namespace }}
+  labels:
+    component: encoder
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  replicas: {{ .Values.encoder.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-encoder-app
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-encoder-app
+    spec:
+      containers:
+        - name: encoder
+          image: "{{ .Values.encoder.image.name }}:{{ .Values.encoder.image.tag }}"
+          command:
+            - sh
+            - -c
+            - "./start"
+          ports:
+            - containerPort: 8080
+              name: backend-http
+          env:
+            - name: GitSha
+              value: {{ .Values.encoder.env.GitSha }}
+          {{- if .Values.encoder.persistenceDisk.enabled -}}
+          volumeMounts:
+            - name: sam-ebs-volume
+              mountPath: {{ .Values.encoder.persistenceDisk.MODEL_DIR }}
+          {{- end }}
+      {{- if .Values.encoder.persistenceDisk.enabled -}}
+      volumes:
+        - name: sam-ebs-volume
+          persistentVolumeClaim:
+            claimName:  {{ .Release.Name }}-encoder-ebs-volume-clain
+      {{- end }}
+      {{- if .Values.encoder.nodeSelector.enabled }}
+      nodeSelector:
+        {{ .Values.encoder.nodeSelector.label_key }} : {{ .Values.encoder.nodeSelector.label_value }}
+      {{- end }}
+{{- end }}
diff --git a/sam-helm-charts/templates/encoder/pvc.yml b/sam-helm-charts/templates/encoder/pvc.yml
@@ -0,0 +1,33 @@
+{{- if .Values.encoder.enabled -}}
+{{- if .Values.encoder.persistenceDisk.enabled -}}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Release.Name }}-encoder-ebs-volume
+  namespace: {{ .Values.namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: "gp2"
+  capacity:
+    storage: {{ .Values.encoder.persistenceDisk.AWS_ElasticBlockStore_size }}
+  awsElasticBlockStore:
+    volumeID: {{ .Values.encoder.persistenceDisk.AWS_ElasticBlockStore_volumeID }}
+    fsType: ext4
+  persistentVolumeReclaimPolicy: Retain
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Release.Name }}-encoder-ebs-volume-clain
+  namespace: {{ .Values.namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: "gp2"
+  volumeName: {{ .Release.Name }}-encoder-ebs-volume
+  resources:
+    requests:
+      storage: {{ .Values.encoder.persistenceDisk.AWS_ElasticBlockStore_size }}
+{{- end }}
+{{- end }}
diff --git a/sam-helm-charts/templates/encoder/service.yml b/sam-helm-charts/templates/encoder/service.yml
@@ -0,0 +1,40 @@
+{{- if .Values.encoder.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    {{- if eq .Values.encoder.serviceType "LoadBalancer" }}
+    service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300"
+    {{- end }}
+    {{- if and (eq .Values.encoder.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }}
+    service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }}
+    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
+    service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https
+    {{- end }}
+    {{- if eq .Values.encoder.serviceType "ClusterIP" }}
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: letsencrypt-prod-issuer
+    {{- end }}
+    {{- with .Values.encoder.serviceAnnotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  name: {{ .Release.Name }}-encoder-service
+  namespace: {{ .Values.namespace }}
+  labels:
+    component: encoder-service
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  type: {{.Values.encoder.serviceType}}
+  ports:
+    - port: 80
+      targetPort: backend-http
+      protocol: TCP
+      name: http
+    - port: 443
+      targetPort: backend-http
+      protocol: TCP
+      name: https
+  selector:
+    app:  {{ .Release.Name }}-encoder-app
+{{- end }}