Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/zenko 4641 #2183

Open
wants to merge 8 commits into
base: development/2.10
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/scripts/end2end/configs/zenko.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ metadata:
name: ${ZENKO_NAME}
${ZENKO_ANNOTATIONS}
zenko.io/x-backbeat-oneshard-replicaset: data-db-mongodb-sharded-shard-0
zenko.io/x-backbeat-oneshard-replicaset-hosts: data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.default.svc.cluster.local:27017
zenko.io/x-backbeat-oneshard-replicaset-hosts: ${ZENKO_BACKBEAT_SHARD_HOSTS}
spec:
version: ${ZENKO_VERSION_NAME}
replicas: 1
Expand Down
11 changes: 11 additions & 0 deletions .github/scripts/end2end/deploy-zenko.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export ZENKO_STS_INGRESS=${ZENKO_STS_INGRESS:-'sts.zenko.local'}
export ZENKO_MANAGEMENT_INGRESS=${ZENKO_MANAGEMENT_INGRESS:-'management.zenko.local'}
export ZENKO_S3_INGRESS=${ZENKO_S3_INGRESS:-'s3.zenko.local'}
export ZENKO_UI_INGRESS=${ZENKO_UI_INGRESS:-'ui.zenko.local'}
export MONGODB_SHARD_COUNT=${MONGODB_SHARD_COUNT:-1}

export BACKBEAT_LCC_CRON_RULE=${BACKBEAT_LCC_CRON_RULE:-'*/5 * * * * *'}

Expand Down Expand Up @@ -128,7 +129,17 @@ create_encryption_secret()
export AZURE_SECRET_KEY_ENCRYPTED
}

generate_shard_hosts() {
local hosts=""
for ((i=0; i<MONGODB_SHARD_COUNT; i++)); do
if [ $i -gt 0 ]; then hosts+=","; fi
hosts+="data-db-mongodb-sharded-shard${i}-data-0.data-db-mongodb-sharded-headless.default.svc.cluster.local:27017"
done
export ZENKO_BACKBEAT_SHARD_HOSTS="$hosts"
}

create_encryption_secret
generate_shard_hosts

env $(dependencies_env) envsubst < ${ZENKOVERSION_PATH} | kubectl -n ${NAMESPACE} apply -f -
env $(dependencies_env) envsubst < ${ZENKO_CR_PATH} | kubectl -n ${NAMESPACE} apply -f -
Expand Down
79 changes: 79 additions & 0 deletions .github/scripts/end2end/generate-kustomization.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#!/bin/bash

set -euo pipefail

# Constants for valid topologies for CI tests
readonly VALID_TOPOLOGIES=(
"1:1" "1:2" "3:1" "3:3" "6:1" "6:3" "6:2" "6:6" "9:1" "9:3" "9:6" "9:9"
)

generate_kustomization() {
local node_count=$1
local shard_count=$2
local kustomization_file="${DIR}/kustomization.yaml"
local base_yaml="mongodb-sharded-${node_count}-node"

touch "$kustomization_file"

# Adjust file name if there are multiple shards
[[ "$shard_count" -gt 1 ]] && base_yaml="${base_yaml}-${shard_count}-shards"
base_yaml="${base_yaml}.yaml"

# Validate topology
local topology_key="${node_count}:${shard_count}"
[[ ! " ${VALID_TOPOLOGIES[*]} " =~ ${topology_key} ]] && {
echo "Error: Invalid topology - ${node_count} nodes, ${shard_count} shards"
exit 1
}

# Generate base kustomization file with the right base resource
cat > "$kustomization_file" << EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ${DIR}/_build/root/deploy/${base_yaml}
patchesStrategicMerge:
EOF

# Add configsvr patch with correct path to add volumeClaimTemplates
cat >> "$kustomization_file" << EOF
- |-
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: data-db-mongodb-sharded-configsvr
spec:
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "8Gi"
storageClassName: standard
EOF

# Add shard patches for N shards with correct path to add volumeClaimTemplates
for ((i=0; i<shard_count; i++)); do
cat >> "$kustomization_file" << EOF
- |-
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: data-db-mongodb-sharded-shard${i}-data
spec:
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "8Gi"
storageClassName: standard
EOF
done
}
17 changes: 12 additions & 5 deletions .github/scripts/end2end/install-kind-dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ MONGODB_APP_PASSWORD=datapass
MONGODB_APP_DATABASE=${ZENKO_MONGODB_DATABASE:-datadb}
MONGODB_RS_KEY=0123456789abcdef

MONGODB_SHARD_COUNT=${MONGODB_SHARD_COUNT:-1}

source "${DIR}/generate-kustomization.sh" && generate_kustomization "${NODE_COUNT:-1}" "${MONGODB_SHARD_COUNT}"

ENABLE_KEYCLOAK_HTTPS=${ENABLE_KEYCLOAK_HTTPS:-'false'}

KAFKA_CHART=banzaicloud-stable/kafka-operator
Expand Down Expand Up @@ -186,7 +190,7 @@ mongodb_wait_for_shards() {
--eval "db.runCommand({ listshards: 1 }).shards.length"
)

[ $count == "1" ]
[ $count == "$MONGODB_SHARD_COUNT" ]
}

mongodb_sharded() {
Expand All @@ -197,11 +201,14 @@ mongodb_sharded() {
$SOLUTION_REGISTRY/os-shell=$(get_image_from_deps mongodb-shell) \
$SOLUTION_REGISTRY/mongodb-exporter=$(get_image_from_deps mongodb-sharded-exporter)

kubectl apply -k .
kubectl apply -k "${DIR}"

kubectl rollout status statefulset data-db-mongodb-sharded-mongos
kubectl rollout status statefulset data-db-mongodb-sharded-configsvr
kubectl rollout status statefulset data-db-mongodb-sharded-shard0-data
kubectl rollout status statefulset data-db-mongodb-sharded-mongos --timeout=5m
kubectl rollout status statefulset data-db-mongodb-sharded-configsvr --timeout=5m

for ((i=0; i<MONGODB_SHARD_COUNT; i++)); do
kubectl rollout status statefulset "data-db-mongodb-sharded-shard${i}-data" --timeout=5m
done

retry mongodb_wait_for_shards "no shards found"

Expand Down
39 changes: 0 additions & 39 deletions .github/scripts/end2end/kustomization.yaml

This file was deleted.

43 changes: 43 additions & 0 deletions .github/workflows/end2end.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,48 @@ jobs:
- name: Clean Up
run: kind delete cluster

end2end-sharded-2-shards:
needs: [build-kafka]
runs-on:
- ubuntu-22.04-8core
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dependencies
uses: ./.github/actions/install-end2end-dependencies
- name: Wait for Docker daemon to be ready
uses: ./.github/actions/wait-docker-ready
- name: Kubectl tool installer
uses: Azure/setup-kubectl@v4
- name: Login to Registry
uses: docker/login-action@v3
with:
username: "${{ github.repository_owner }}"
password: "${{ github.token }}"
registry: ghcr.io
- name: Deploy Zenko
uses: ./.github/actions/deploy
env:
GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
ZENKO_ENABLE_SOSAPI: true
TIME_PROGRESSION_FACTOR: 86400
TRANSITION_ONE_DAY_EARLIER: false
EXPIRE_ONE_DAY_EARLIER: false
MONGODB_SHARD_COUNT: 2
- name: Debug wait
uses: ./.github/actions/debug-wait
timeout-minutes: 60
if: failure() && runner.debug == '1'
- name: Archive and publish artifacts
uses: ./.github/actions/archive-artifacts
with:
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
trunk_token: ${{ secrets.TRUNK_TOKEN }}
if: always()
- name: Clean Up
run: kind delete cluster

write-final-status:
runs-on: ubuntu-latest
needs:
Expand All @@ -652,6 +694,7 @@ jobs:
- end2end-http
- end2end-https
- end2end-sharded
- end2end-sharded-2-shards
- end2end-pra
- ctst-end2end-sharded
steps:
Expand Down
32 changes: 27 additions & 5 deletions solution-base/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,17 @@ SKOPEO_OPTS="--override-os linux --insecure-policy"
SOLUTION_REGISTRY=metalk8s-registry-from-config.invalid/${PRODUCT_LOWERNAME}-${VERSION_FULL}

MONGODB_SHARDED_SINGLE_NODE_PATH=${ISO_ROOT}/deploy/mongodb-sharded-1-node.yaml
MONGODB_SHARDED_SINGLE_NODE_TWO_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-1-node-2-shards.yaml
MONGODB_SHARDED_THREE_NODE_PATH=${ISO_ROOT}/deploy/mongodb-sharded-3-nodes.yaml
MONGODB_SHARDED_THREE_NODE_THREE_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-3-nodes-3-shards.yaml
MONGODB_SHARDED_SIX_NODE_PATH=${ISO_ROOT}/deploy/mongodb-sharded-6-nodes.yaml
MONGODB_SHARDED_SIX_NODE_TWO_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-6-nodes-2-shards.yaml
MONGODB_SHARDED_SIX_NODE_THREE_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-6-nodes-3-shards.yaml
MONGODB_SHARDED_SIX_NODE_SIX_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-6-nodes-6-shards.yaml
MONGODB_SHARDED_NINE_NODE_PATH=${ISO_ROOT}/deploy/mongodb-sharded-9-nodes.yaml
MONGODB_SHARDED_NINE_NODE_THREE_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-9-nodes-3-shards.yaml
MONGODB_SHARDED_NINE_NODE_SIX_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-9-nodes-6-shards.yaml
MONGODB_SHARDED_NINE_NODE_NINE_SHARDS_PATH=${ISO_ROOT}/deploy/mongodb-sharded-9-nodes-9-shards.yaml

SOLUTION_ENV='SOLUTION_ENV'

Expand Down Expand Up @@ -74,7 +84,7 @@ function render_mongodb_sharded_yamls()
local OUTPUT_PATH=${1:-${OPERATOR_PATH}}
local SHARD_COUNT=${2:-1}
local NODE_COUNT=${3:-1}
local ADD_OPTIONS=${4:-""}
local REPLICA_COUNT=${3:-${NODE_COUNT}}

echo creating mongodb-sharded ${NODE_COUNT}-node yamls
CHART_PATH="$SOLUTION_BASE_DIR/mongodb/charts/mongodb-sharded"
Expand All @@ -87,10 +97,10 @@ function render_mongodb_sharded_yamls()
--set shards=${SHARD_COUNT} \
--set mongos.replicaCount=${NODE_COUNT} \
--set mongos.useStatefulSet=true \
--set shardsvr.dataNode.replicaCount=${NODE_COUNT} \
--set shardsvr.dataNode.replicaCount=${REPLICA_COUNT} \
--set shardsvr.persistence.enabled=true \
--set shardsvr.persistence.storageClass=${MONGODB_STORAGE_CLASS} \
--set configsvr.replicaCount=${NODE_COUNT} \
--set configsvr.replicaCount=${REPLICA_COUNT} \
--set configsvr.persistence.enabled=true \
--set configsvr.persistence.storageClass=${MONGODB_STORAGE_CLASS} \
--set metrics.enabled=true \
Expand Down Expand Up @@ -120,8 +130,20 @@ function render_mongodb_sharded_yamls()

function mongodb_sharded_yamls()
{
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SINGLE_NODE_PATH}" 1 1
render_mongodb_sharded_yamls "${MONGODB_SHARDED_THREE_NODE_PATH}" 1 3
# For now we maximize the number of replicas to 3, so each shard is a P-S-S
# Do we want to support more combinations that is, N replicas?
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SINGLE_NODE_PATH}" 1 1 1
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SINGLE_NODE_TWO_SHARDS_PATH}" 2 1 1
render_mongodb_sharded_yamls "${MONGODB_SHARDED_THREE_NODE_PATH}" 1 3 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_THREE_NODE_THREE_SHARDS_PATH}" 3 3 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SIX_NODE_PATH}" 1 6 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SIX_NODE_TWO_SHARDS_PATH}" 2 6 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SIX_NODE_THREE_SHARDS_PATH}" 3 6 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_SIX_NODE_SIX_SHARDS_PATH}" 6 6 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_NINE_NODE_PATH}" 1 9 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_NINE_NODE_THREE_SHARDS_PATH}" 3 9 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_NINE_NODE_SIX_SHARDS_PATH}" 6 9 3
render_mongodb_sharded_yamls "${MONGODB_SHARDED_NINE_NODE_NINE_SHARDS_PATH}" 9 9 3
}

function gen_manifest_yaml()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ SPDX-License-Identifier: APACHE-2.0
*/}}

{{- if and .Values.shards .Values.metrics.enabled .Values.metrics.podMonitor.enabled }}
{{- $i := 0 }}
{{- $replicas := .Values.shards | int }}
{{- range $i, $e := until $replicas }}
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
Expand Down Expand Up @@ -35,4 +36,9 @@ spec:
selector:
matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }}
app.kubernetes.io/component: shardsvr
shard: {{ $i | quote }}
{{- if lt $i (sub $replicas 1) }}
---
{{- end }}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,11 @@ spec:
{{- end }}
{{- if $.Values.shardsvr.persistence.selector }}
selector:
{{ toYaml $.Values.shardsvr.persistence.selector | indent 10 }}
matchLabels:
{{- if gt $.Values.shards 1 }}
shard: {{ $i | quote }}
{{- end }}
{{ toYaml $.Values.shardsvr.persistence.selector.matchLabels | indent 12 }}
{{- end }}
resources:
requests:
Expand Down
2 changes: 1 addition & 1 deletion solution-base/mongodb/charts/mongodb-sharded/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -680,7 +680,7 @@ mongos:
## @param mongos.podAntiAffinityPreset Mongos Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard`
## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity
##
podAntiAffinityPreset: soft
podAntiAffinityPreset: hard
## Node affinity preset
## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity
##
Expand Down
15 changes: 15 additions & 0 deletions solution-base/mongodb/patches/config-server-statefulset.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
diff --git a/solution-base/mongodb/charts/mongodb-sharded/templates/config-server/config-server-statefulset.yaml b/solution-base/mongodb/charts/mongodb-sharded/templates/config-server/config-server-statefulset.yaml
index 2e4f2b25..aba11021 100644
--- a/solution-base/mongodb/charts/mongodb-sharded/templates/config-server/config-server-statefulset.yaml
+++ b/solution-base/mongodb/charts/mongodb-sharded/templates/config-server/config-server-statefulset.yaml
@@ -401,6 +401,10 @@ spec:
{{- range .Values.configsvr.persistence.accessModes }}
- {{ . | quote }}
{{- end }}
+ {{- if .Values.configsvr.persistence.selector }}
+ selector:
+{{ toYaml .Values.configsvr.persistence.selector | indent 10 }}
+ {{- end }}
resources:
requests:
storage: {{ .Values.configsvr.persistence.size | quote }}
Loading
Loading