Merge remote-tracking branch 'origin/main' into add_cosmos_to_kube

.gitignore

@@ -38,3 +38,4 @@ nb-configuration.xml
 secrets.env
 /docker/dev/docker/
 mi6/export
+kubernetes/.idea

README.md

@@ -1,7 +1,7 @@
 [![Lint & Format](https://github.com/DARPA-ASKEM/orchestration/actions/workflows/lint.yaml/badge.svg?branch=main)](https://github.com/DARPA-ASKEM/orchestration/actions/workflows/lint_format.yaml)
 [![Build Docker Images](https://github.com/DARPA-ASKEM/orchestration/actions/workflows/publish.yaml/badge.svg?branch=main)](https://github.com/DARPA-ASKEM/orchestration/actions/workflows/publish.yaml)
 # Orchestration
-Scripts and deployment information needed to setup and run TERArium
+Scripts and deployment information needed to setup and run Terarium
 
 ## Common Tasks
 

kubernetes/base/keycloak/keycloak-deployment.yaml

@@ -23,6 +23,9 @@ spec:
         - name: keycloak
           image: keycloak-image
           imagePullPolicy: Always
+          env:
+            - name: KC_DB
+              value: postgres
           args:
             - start
           ports:
@@ -48,6 +51,9 @@ spec:
             - import
             - --dir
             - /data
+          env:
+            - name: KC_DB
+              value: postgres
           ports:
             - containerPort: 8080
               protocol: TCP

kubernetes/orctl

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ -f ../orctl/target/debug/orctl ]]; then
+  ../orctl/target/debug/orctl "$@"
+else
+  echo "Please build orctl, consult the README.md in ../orctl"
+fi

kubernetes/overlays/prod/overlays/askem-dev/check-latest/secrets.enc.yaml

@@ -1,15 +1,14 @@
----
 apiVersion: v1
 kind: Secret
 metadata:
     name: check-latest-creds
     namespace: terarium
 type: Opaque
 data:
-    username: ENC[AES256_GCM,data:99XHG9E2Jl976O+QoesgCupLogzksXwN,iv:9+9/AYEe29a93BR0WorkfFlIGHN/CqtmwPGP1PMyzeU=,tag:VX82coM4UGL9w5+XjR3eoA==,type:str]
-    pat: ENC[AES256_GCM,data:9fIbq6xUUlOCOLEbOjD+Wv+qfAxHC6n4JhsZKaM0ju9b2jRAECQ6Zw17sfALWKKw3m8trkeqPro=,iv:x1wQ3pX/aeo+xVLIFyIpezgYaIeD6kW/rJq82CWLNM0=,tag:Jmumr+kiM2/xd5eG29DRpg==,type:str]
-    ssh_kubectl: ENC[AES256_GCM,data:xXXIYfgh52OKtt6GQd7iQxIgE2x4CKNja2Xd9QXAOorA2V6514a6OyYuOSAZdYKMrHL+Gah0lbEC2REhDbiiiIEK31dAyriDtFF7V7lrVKUwxrverVd3Z5Qg4Xe4o4fcUnfp4gNElYZ0MYuln/cb3kh9Yjok2fm2ffJLoXdHJpHrZaVxsDFCuLpIOWg=,iv:/5U5/nRzux5kf81kfJMHxZ7kkrFVm/LfUjntpDGigH8=,tag:iuh6dt2AHGUwhGBB5Wfqrw==,type:str]
-    images_file: ENC[AES256_GCM,data:mcGBUoAQ1gY2GXWhllab/cxiVWvljxpapZIUZHAN2s0z6eIZtWD88w==,iv:c2f/fKNLL7wxBmhviGzr4mAnk6JKQDJJ9t81HDop/Yw=,tag:HyOYAHY9vPk9Dg6FcvnYuQ==,type:str]
+    username: ENC[AES256_GCM,data:pFu4ZztCIZBXRZ3IxKFD459/fvYvs7tj,iv:CnbfSrtdYHVTs8bDczQ5xD4dNlESeVxqVeZRvGoVPX8=,tag:16eBA3Ca97GkbSKYZXEjDg==,type:str]
+    pat: ENC[AES256_GCM,data:cO1kdwnZE3hJzXz4938zGmSWy5JHJfJ5MBk1twdLWp7YnSXRD9gIEUIrlFqN2a7vGHRXpgx6c2o=,iv:dU5mEDdJz6mqnYL7sb+xbYjbqAVyo9sVDLWRIM/6ukM=,tag:PwddeXVKK4soHHoqh1XNPg==,type:str]
+    ssh_kubectl: ENC[AES256_GCM,data:R6fuyjXuBdP/GyuZ6xqp3b63DshTZAgZ1dQ1n87KZklKYDMQNXgaIbNkGwBjpjcJB46O4UsckWYdSfsc2ThhVCq02H5uk/N47e803E/2rOfGbMvTwG5XbrHbmnfwqXEiPHLHC4eUNpeX1nCvOIQi8EJobqcIwgWhpzMz+mIUKdMyS6prHxGaMw==,iv:eY9zZYBB4z+qChWpEhBgNr8JABP5uPmjUAx5P5+mDp4=,tag:UK58A7TqGf95iqto9DRcww==,type:str]
+    images_file: ENC[AES256_GCM,data:96CQzyS1CztYNUAIr8L4Yr7+6nkbizRQ4/MOYTPSedglLARA9y0fZQ==,iv:hhvAtaS/9sotiAejd9cMNybZJutT/CdKx1d6/UEvY9o=,tag:IdHP596UlabPkC37iQgKdg==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -19,14 +18,14 @@ sops:
         - recipient: age1q5q7lt6twttye7h5dx4zu0eek04pu6szqfx28w00vq3z6nmlc9dq7md4xd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2VXNWaWFVUkVKbHVFR3lL
-            U3o5eFUzT0ZVMkp5cWs0a3Q4bHdKazRBN2k4CjE1VnZPWHZSRjBYNFdhOTBoemt5
-            OWJLbG9KSVNXaGQ3cURXa0lDcFAySVUKLS0tICsyQTdCVEduTTFEQVFLZS9yWUlG
-            dDAvQm5URk5jVW9qOXY0ZlF5NUo1MjgKx5hUzYDYH3z4MkiWm0bJbWXegl4fuLI0
-            U0clQ4rzl73JkQpTtgdE50FVWa2juUI+3fCWhkLiP02wRh8GRi1OxA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqUkNwaEFiZWx1aWRjdE1i
+            RkxKOTlSVVVWL3E4dlJoNU1zeE9HMDFhZ3lRCk53TWZHNUJsbCtFOWtxbDViRmp5
+            SkZIOHJtZUJDRzVrMGtUQnFoRU5PVjgKLS0tIHVkaTRoaGxndzZnYjUvdzlUbGE0
+            bjlkTzdET2V1Mi94Ri9pUGdNejJIaEUKmQLqu99KkdkDnhLqfsPZjNLYOBJ4VPAc
+            bgu6dex9dJgiukK7QbiU29TtsnXT+7ZVm5VGkMs2KIZ/bAuH8mVTwA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-02-21T14:31:30Z"
-    mac: ENC[AES256_GCM,data:dr0Nh5hcyBdMWhCMMLYpAIf0GpaYjmubIgcflWhc8G8EfEt3pyD+AWUE9yBQscC+9QiNDcztVKeTEJLfObVSQkm7ki4FWr4UbqSekNdBGv2L2sUlbyv7yDomQfS2nyDDb6ZzxuxNwgaio7/6B9okTUgZ8GpW6NcGFeIuosUBr4c=,iv:L0/HgAX3zvqrI+m9C1ymTVde+GdS2RT2qmiFxPQra74=,tag:ymYeKDb85MSJm1Qrn7J8rA==,type:str]
+    lastmodified: "2024-06-18T21:08:20Z"
+    mac: ENC[AES256_GCM,data:ia/dFkcz/+jVS07vTf1zwOexDlxBpAp4AcaaPkRy9HDq/Jth50Yw3LwGVTOG/TbEqZM/Srd9K4n6739gDzED4Z4WR63T8UExRmpf0iolbJCKQkZ8Cfl4VnTy+uEfv5wODuK2QLFlrXSzwbZB8Q9gg2lJ0sxFnOUl6bRGa0cT7Hs=,iv:qoYlmdQNylouxqpQpBS4iwW4hAh8W/P1emaGtfbVS3A=,tag:SaLi2D5xG+kzqXHEF/6fsw==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.8.1

kubernetes/overlays/prod/overlays/askem-dev/ingress/private-web-ingress.yaml

@@ -3,6 +3,12 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: private-web-ingress
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "on"
+    nginx.ingress.kubernetes.io/proxy-buffers-number: "16"
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "32k"
 spec:
   ingressClassName: nginx
   rules:

kubernetes/overlays/prod/overlays/askem-dev/ingress/private-web-ssl-ingress.yaml

@@ -5,6 +5,11 @@ metadata:
   name: private-web-ssl-ingress
   annotations:
     nginx.ingress.kubernetes.io/backend-protocol: 'HTTPS'
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "on"
+    nginx.ingress.kubernetes.io/proxy-buffers-number: "16"
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "32k"
 spec:
   ingressClassName: nginx
   rules:

kubernetes/overlays/prod/overlays/askem-dev/keycloak/keycloak-deployment.yaml

@@ -8,13 +8,23 @@ spec:
     spec:
       containers:
         - name: keycloak
+          args:
+            - start
           env:
             - name: KC_HOSTNAME_URL
               value: https://keycloak.dev.terarium.ai
             - name: KC_HOSTNAME_ADMIN_URL
               value: https://keycloak.dev.terarium.ai
             - name: KC_DB_URL
               value: 'jdbc:postgresql://10.64.22.49:5432/keycloak'
+            - name: PROXY_ADDRESS_FORWARDING
+              value: 'true'
+            - name: KC_HTTP_ENABLED
+              value: 'true'
+            - name: KC_HOSTNAME_STRICT_HTTPS
+              value: 'false'
+            - name: KC_HOSTNAME_STRICT
+              value: 'false'
       initContainers:
         - name: init-keycloak
           env:

kubernetes/overlays/prod/overlays/askem-dev/kustomization.yaml

@@ -52,8 +52,8 @@ configMapGenerator:
 images:
   # Auth
   - name: keycloak-image
-    newName: ghcr.io/unchartedsoftware/keycloak
-    newTag: '0.2.2'
+    newName: quay.io/keycloak/keycloak
+    newTag: '25.0.0'
   - name: terarium-login-theme-image
     newName: ghcr.io/darpa-askem/terarium-login-theme
     newTag: 'latest'

kubernetes/overlays/prod/overlays/askem-production/kustomization.yaml

@@ -53,13 +53,13 @@ images:
   # Terarium HMI
   - name: hmi-client-image
     newName: ghcr.io/darpa-askem/hmi-client
-    newTag: '2.12.2'
+    newTag: '2.13.0'
   - name: terarium-docs-image
     newName: ghcr.io/darpa-askem/terarium-docs
     newTag: 'latest'
   - name: hmi-server-image
     newName: ghcr.io/darpa-askem/hmi-server
-    newTag: '2.12.2'
+    newTag: '2.13.0'
   - name: spicedb-image
     newName: ghcr.io/authzed/spicedb
     newTag: 'v1.24.0'
@@ -72,15 +72,15 @@ images:
   # Jupyter LLM service
   - name: beaker-image
     newName: ghcr.io/darpa-askem/beaker-kernel
-    newTag: '1.5.0'
+    newTag: '1.6.0'
 
   # Skema
   - name: skema-py-image
     newName: ghcr.io/darpa-askem/skema-py
-    newTag: '1.0.0'
+    newTag: '1.1.0'
   - name: skema-rs-image
     newName: ghcr.io/darpa-askem/skema-rs
-    newTag: '1.0.0'
+    newTag: '1.1.0'
   - name: skema-memgraph-image
     newName: ghcr.io/darpa-askem/memgraph-platform
     newTag: '2.6.5-memgraph2.5.2-lab2.4.0-mage1.6'
@@ -96,20 +96,20 @@ images:
   # Simulation Service
   - name: sciml-service-image
     newName: ghcr.io/darpa-askem/sciml-service
-    newTag: '1.0.0'
+    newTag: '1.1.0'
 
   # PyCIEMSS service
   - name: pyciemss-api-image
     newName: ghcr.io/darpa-askem/pyciemss-api
-    newTag: '1.10.0'
+    newTag: '1.11.0'
   - name: pyciemss-worker-image
     newName: ghcr.io/darpa-askem/pyciemss-worker
-    newTag: '1.10.0'
+    newTag: '1.11.0'
 
   # GoLLM service
   - name: gollm-taskrunner-image
     newName: ghcr.io/darpa-askem/gollm-taskrunner
-    newTag: '2.12.0'
+    newTag: '2.13.0'
 
   # Integration Dashboard
   - name: integration-dashboard-image
@@ -124,7 +124,7 @@ images:
   # mira taskrunner
   - name: mira-taskrunner-image
     newName: ghcr.io/darpa-askem/mira-taskrunner
-    newTag: '2.12.0'
+    newTag: '2.13.0'
 
   # cosmos (xdd)
   - name: cosmos-image
@@ -134,7 +134,7 @@ images:
   # funman taskrunner
   - name: funman-taskrunner-image
     newName: ghcr.io/darpa-askem/funman-taskrunner
-    newTag: '2.12.0'
+    newTag: '2.13.0'
 
   # Redis
   - name: redis-image