From 0cb12fad5373b31ef003eb71cfd88cc6eb8e0c8f Mon Sep 17 00:00:00 2001 From: Joe M Date: Mon, 27 May 2024 21:44:28 -0700 Subject: [PATCH] Updates for 0.4.0 k8s (#35) * Updates to 0.4.0 along with an attempt at keeping l7 istio changes separate. * Fixed issue with load balancer never being set to true * Service monitor updates needed to detect proximus nodes from promtheus --- kubernetes/full-create-and-install.sh | 67 +++++++++++++------ ...us-gke-values.yaml => avs-gke-values.yaml} | 29 +++++--- kubernetes/manifests/gateway.yaml | 23 +++++++ .../istio-ingressgateway-values.yaml | 17 +++++ ...cemonitor.yaml => avs-servicemonitor.yaml} | 6 +- .../manifests/quote-search-gke-values.yaml | 5 ++ .../manifests/sematic-search-values.yaml | 5 -- kubernetes/manifests/servicemonitor.yaml | 14 ++++ .../manifests/ssd_storage_cluster_cr.yaml | 8 +-- .../virtual-service-vector-search.yaml | 29 ++++++++ quote-semantic-search/quote-search/config.py | 6 +- 11 files changed, 163 insertions(+), 46 deletions(-) rename kubernetes/manifests/{proximus-gke-values.yaml => avs-gke-values.yaml} (60%) create mode 100644 kubernetes/manifests/gateway.yaml create mode 100644 kubernetes/manifests/istio-ingressgateway-values.yaml rename kubernetes/manifests/monitoring/{proximus-servicemonitor.yaml => avs-servicemonitor.yaml} (80%) create mode 100644 kubernetes/manifests/quote-search-gke-values.yaml delete mode 100644 kubernetes/manifests/sematic-search-values.yaml create mode 100644 kubernetes/manifests/servicemonitor.yaml create mode 100644 kubernetes/manifests/virtual-service-vector-search.yaml diff --git a/kubernetes/full-create-and-install.sh b/kubernetes/full-create-and-install.sh index cdbec18..04a4eb1 100755 --- a/kubernetes/full-create-and-install.sh +++ b/kubernetes/full-create-and-install.sh @@ -1,27 +1,33 @@ #!/bin/bash -# This script sets up a GKE cluster with specific configurations for Aerospike and Proximus node pools. +# This script sets up a GKE cluster with specific configurations for Aerospike and AVS node pools. # It handles the creation of the cluster, node pools, labeling, tainting of nodes, and deployment of necessary operators and configurations. -# Additionally, it sets up monitoring using Prometheus and deploys a specific Helm chart for Proximus. +# Additionally, it sets up monitoring using Prometheus and deploys a specific Helm chart for AVS. # Function to print environment variables for verification +set -eo pipefail +if [ -n "$DEBUG" ]; then set -x; fi +trap 'echo "Error: $? at line $LINENO" >&2' ERR + print_env() { echo "Environment Variables:" echo "export PROJECT_ID=$PROJECT_ID" echo "export CLUSTER_NAME=$CLUSTER_NAME" echo "export NODE_POOL_NAME_AEROSPIKE=$NODE_POOL_NAME_AEROSPIKE" - echo "export NODE_POOL_NAME_PROXIMUS=$NODE_POOL_NAME_PROXIMUS" + echo "export NODE_POOL_NAME_AVS=$NODE_POOL_NAME_AVS" echo "export ZONE=$ZONE" echo "export FEATURES_CONF=$FEATURES_CONF" echo "export AEROSPIKE_CR=$AEROSPIKE_CR" } # Set environment variables for the GKE cluster setup -export PROJECT_ID="performance-eco" -export CLUSTER_NAME="my-world-eco" +export PROJECT_ID="$(gcloud config get-value project)" +export CLUSTER_NAME="${PROJECT_ID}-modern-world" export NODE_POOL_NAME_AEROSPIKE="aerospike-pool" -export NODE_POOL_NAME_PROXIMUS="proximus-pool" +export NODE_POOL_NAME_AVS="avs-pool" export ZONE="us-central1-c" +#export HELM_CHART="aerospike/aerospike-avs" +export HELM_CHART="/home/joem/src/helm-charts/aerospike-vector-search" export FEATURES_CONF="./features.conf" export AEROSPIKE_CR="./manifests/ssd_storage_cluster_cr.yaml" @@ -97,12 +103,13 @@ kubectl apply -f https://raw.githubusercontent.com/aerospike/aerospike-kubernete echo "Deploying Aerospike cluster..." kubectl apply -f "$AEROSPIKE_CR" + ############################################## -# Proximus name space +# AVS name space ############################################## -echo "Adding Proximus node pool..." -if ! gcloud container node-pools create "$NODE_POOL_NAME_PROXIMUS" \ +echo "Adding avs node pool..." +if ! gcloud container node-pools create "$NODE_POOL_NAME_AVS" \ --cluster "$CLUSTER_NAME" \ --project "$PROJECT_ID" \ --zone "$ZONE" \ @@ -110,27 +117,45 @@ if ! gcloud container node-pools create "$NODE_POOL_NAME_PROXIMUS" \ --disk-type "pd-standard" \ --disk-size "100" \ --machine-type "e2-highmem-4"; then - echo "Failed to create Proximus node pool" + echo "Failed to create avs node pool" exit 1 else - echo "Proximus node pool added successfully." + echo "avs node pool added successfully." fi -echo "Labeling Proximus nodes..." -kubectl get nodes -l cloud.google.com/gke-nodepool="$NODE_POOL_NAME_PROXIMUS" -o name | \ - xargs -I {} kubectl label {} aerospike.com/node-pool=proximus --overwrite +echo "Labeling avs nodes..." +kubectl get nodes -l cloud.google.com/gke-nodepool="$NODE_POOL_NAME_AVS" -o name | \ + xargs -I {} kubectl label {} aerospike.com/node-pool=avs --overwrite echo "Setup complete. Cluster and node pools are configured." -kubectl create namespace proximus +kubectl create namespace avs + +echo "Setting secrets for avs cluster..." +kubectl --namespace avs create secret generic aerospike-secret --from-file=features.conf="$FEATURES_CONF" +kubectl --namespace avs create secret generic auth-secret --from-literal=password='admin123' + + +# echo "Deploying Istio" +# helm repo add istio https://istio-release.storage.googleapis.com/charts +# helm repo update + +# helm install istio-base istio/base --namespace istio-system --set defaultRevision=default --create-namespace --wait +# helm install istiod istio/istiod --namespace istio-system --create-namespace --wait +# helm install istio-ingress istio/gateway \ +# --values "manifests/istio-ingressgateway-values.yaml" \ +# --namespace istio-ingress \ +# --create-namespace \ +# --wait + +# kubectl apply -f "manifests/gateway.yaml" +# kubectl apply -f "manifests/virtual-service-vector-search.yaml" + -echo "Setting secrets for proximus cluster..." -kubectl --namespace proximus create secret generic aerospike-secret --from-file=features.conf="$FEATURES_CONF" -kubectl --namespace proximus create secret generic auth-secret --from-literal=password='admin123' -helm install proximus-gke --values "manifests/proximus-gke-values.yaml" --namespace proximus aerospike/aerospike-proximus --wait +helm install avs-gke --values "manifests/avs-gke-values.yaml" --namespace avs $HELM_CHART --wait ############################################## @@ -152,6 +177,6 @@ echo "To expose grafana ports publically 'kubectl apply -f helpers/EXPOSE-GRAFAN echo "To find the exposed port with 'kubectl get svc -n monitoring' " -#To run the quote search sample app on your new cluster you can use -# helm install sematic-search-app aerospike/quote-semantic-search --namespace proximus --values manifests/sematic-search-values.yaml --wait +echo To run the quote search sample app on your new cluster you can use +echo helm install sematic-search-app aerospike/quote-semantic-search --namespace avs --values manifests/sematic-search-values.yaml --wait diff --git a/kubernetes/manifests/proximus-gke-values.yaml b/kubernetes/manifests/avs-gke-values.yaml similarity index 60% rename from kubernetes/manifests/proximus-gke-values.yaml rename to kubernetes/manifests/avs-gke-values.yaml index 533ec77..5e4fae5 100644 --- a/kubernetes/manifests/proximus-gke-values.yaml +++ b/kubernetes/manifests/avs-gke-values.yaml @@ -1,23 +1,23 @@ replicaCount: 3 -proximusConfig: +aerospikeVectorSearchConfig: heartbeat: seeds: - - address: proximus-gke-aerospike-proximus-0.proximus-gke-aerospike-proximus.proximus.svc.cluster.local + - address: avs-gke-aerospike-vector-search-0.avs-gke-aerospike-vector-search.avs.svc.cluster.local port: 5001 - - address: proximus-gke-aerospike-proximus-1.proximus-gke-aerospike-proximus.proximus.svc.cluster.local + - address: avs-gke-aerospike-vector-search-1.avs-gke-aerospike-vector-search.avs.svc.cluster.local port: 5001 - - address: proximus-gke-aerospike-proximus-2.proximus-gke-aerospike-proximus.proximus.svc.cluster.local + - address: avs-gke-aerospike-vector-search-2.avs-gke-aerospike-vector-search.avs.svc.cluster.local port: 5001 - + interconnect: ports: 5001: addresses: 0.0.0.0 aerospike: - metadata-namespace: "proximus-meta" + metadata-namespace: "avs-meta" seeds: - aerocluster-0-0.aerocluster.aerospike.svc.cluster.local: port: 3000 @@ -30,16 +30,27 @@ proximusConfig: metrics-ticker: info root: info com.aerospike.vector.embedded.client: debug + client: debug ticker-interval: 10 service: enabled: true annotations: - networking.gke.io/load-balancer-type: "Internal" + networking.gke.io/load-balancer-type: "External" ports: - name: "svc-port" port: 5000 targetPort: 5000 +# service: +# enabled: false +# type: LoadBalancer +# annotations: +# cloud.google.com/l4-rbs: "enabled" +# # networking.gke.io/load-balancer-type: "Internal" +# ports: +# - name: "svc-port" +# port: 5000 +# targetPort: 5000 # schedule proximus nodes affinity: @@ -50,7 +61,7 @@ service: - key: aerospike.com/node-pool operator: In values: - - "proximus" + - "aerospike-vector-search" podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: "kubernetes.io/hostname" @@ -59,4 +70,4 @@ service: - key: "app.kubernetes.io/name" operator: In values: - - "aerospike-proximus" + - "aerospike-vector-search" diff --git a/kubernetes/manifests/gateway.yaml b/kubernetes/manifests/gateway.yaml new file mode 100644 index 0000000..d8903eb --- /dev/null +++ b/kubernetes/manifests/gateway.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: avs-gw + namespace: aerospike +spec: + selector: + istio: ingress + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" + - port: + number: 5000 + name: grpc + protocol: GRPC + hosts: + - "*" + tls: + mode: PASSTHROUGH diff --git a/kubernetes/manifests/istio-ingressgateway-values.yaml b/kubernetes/manifests/istio-ingressgateway-values.yaml new file mode 100644 index 0000000..e14f490 --- /dev/null +++ b/kubernetes/manifests/istio-ingressgateway-values.yaml @@ -0,0 +1,17 @@ +service: + type: LoadBalancer + annotations: + cloud.google.com/l4-rbs: "enabled" + ports: + - name: http + port: 80 + targetPort: 80 + - name: https + port: 443 + targetPort: 443 + - name: status-port + port: 15021 + targetPort: 15021 + - name: grpc + port: 5000 + targetPort: 5000 diff --git a/kubernetes/manifests/monitoring/proximus-servicemonitor.yaml b/kubernetes/manifests/monitoring/avs-servicemonitor.yaml similarity index 80% rename from kubernetes/manifests/monitoring/proximus-servicemonitor.yaml rename to kubernetes/manifests/monitoring/avs-servicemonitor.yaml index d2230d6..967790d 100644 --- a/kubernetes/manifests/monitoring/proximus-servicemonitor.yaml +++ b/kubernetes/manifests/monitoring/avs-servicemonitor.yaml @@ -1,17 +1,17 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: proximus-monitor + name: avs-monitor namespace: monitoring labels: release: monitoring-stack # Ensure this matches the Helm release name spec: selector: matchLabels: - app: proximus-gke-aerospike-proximus + app: avs-gke-aerospike-vector-search namespaceSelector: matchNames: - - aerospike + - avs endpoints: - port: manage-5040 path: /manage/rest/v1/prometheus diff --git a/kubernetes/manifests/quote-search-gke-values.yaml b/kubernetes/manifests/quote-search-gke-values.yaml new file mode 100644 index 0000000..594ca71 --- /dev/null +++ b/kubernetes/manifests/quote-search-gke-values.yaml @@ -0,0 +1,5 @@ +quoteSearchConfig: + avsHost: "avs-gke-aerospike-vector-search-lb.svc.cluster.local" + avsIsLoadbalancer: "True" + avsNamespace: "avs-meta" + diff --git a/kubernetes/manifests/sematic-search-values.yaml b/kubernetes/manifests/sematic-search-values.yaml deleted file mode 100644 index 7463c0f..0000000 --- a/kubernetes/manifests/sematic-search-values.yaml +++ /dev/null @@ -1,5 +0,0 @@ -quoteSearchConfig: - proximusHost: proximus-gke-aerospike-proximus-lb.proximus.svc.cluster.local - proximusIsLoadbalancer: True - proximusIndexName: "quotesindex" - diff --git a/kubernetes/manifests/servicemonitor.yaml b/kubernetes/manifests/servicemonitor.yaml new file mode 100644 index 0000000..3b98e4f --- /dev/null +++ b/kubernetes/manifests/servicemonitor.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: avs-service-monitor + namespace: aerospike +spec: + selector: + matchLabels: + app.kubernetes.io/name: aerospike-vector-search + endpoints: + - port: "manage-5040" + interval: 10s + path: "/manage/rest/v1/prometheus" diff --git a/kubernetes/manifests/ssd_storage_cluster_cr.yaml b/kubernetes/manifests/ssd_storage_cluster_cr.yaml index 5aca5e2..5ad7c84 100644 --- a/kubernetes/manifests/ssd_storage_cluster_cr.yaml +++ b/kubernetes/manifests/ssd_storage_cluster_cr.yaml @@ -22,9 +22,9 @@ spec: storageClass: ssd volumeMode: Filesystem size: 1Gi - - name: proximus-meta + - name: avs-meta aerospike: - path: /proximus/dev/xvdf + path: /avs/dev/xvdf source: persistentVolume: storageClass: ssd @@ -91,7 +91,7 @@ spec: devices: - /test/dev/xvdf - - name: proximus-meta + - name: avs-meta nsup-period: 600 nsup-threads: 2 evict-tenths-pct: 5 @@ -99,4 +99,4 @@ spec: storage-engine: type: device devices: - - /proximus/dev/xvdf \ No newline at end of file + - /avs/dev/xvdf \ No newline at end of file diff --git a/kubernetes/manifests/virtual-service-vector-search.yaml b/kubernetes/manifests/virtual-service-vector-search.yaml new file mode 100644 index 0000000..40fd335 --- /dev/null +++ b/kubernetes/manifests/virtual-service-vector-search.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: avs-vs + namespace: aerospike +spec: + hosts: + - "*" + gateways: + - avs-gw + http: + - match: + - uri: + prefix: / + port: 80 + route: + - destination: + port: + number: 8080 + host: quote-search-quote-semantic-search.aerospike.svc.cluster.local + - match: + - uri: + prefix: / + port: 5000 + route: + - destination: + port: + number: 5000 + host: avs-gke-aerospike-vector-search.aerospike.svc.cluster.local diff --git a/quote-semantic-search/quote-search/config.py b/quote-semantic-search/quote-search/config.py index fbb3992..7b4f378 100644 --- a/quote-semantic-search/quote-search/config.py +++ b/quote-semantic-search/quote-search/config.py @@ -3,13 +3,11 @@ def get_bool_env(name, default): env = os.environ.get(name) - if env is None: return default + env = env.lower() - env = env.lower() - - if os.environ.get(name) in ["true", "1"]: + if env in ["true", "1"]: return True else: return False