Merge branch 'master' into montoring-stack

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM --platform=$BUILDPLATFORM golang:1.21 as builder
+FROM --platform=$BUILDPLATFORM golang:1.21 AS builder
 
 # OS and Arch args
 ARG TARGETOS
@@ -27,7 +27,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} GO111MODULE=on go
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
 
 # Version of Operator (build arg)
-ARG VERSION="3.3.0"
+ARG VERSION="3.3.1"
 
 # User to run container as
 ARG USER="root"

Jenkinsfile

@@ -80,8 +80,8 @@ pipeline {
                                sh "./snyk-linux test  --severity-threshold=high --fail-on=all"
 
                                // Scan the operator images
-                               sh "./snyk-linux container test  ${OPERATOR_CONTAINER_IMAGE_CANDIDATE_NAME} --severity-threshold=high --file=Dockerfile --policy-path=.snyk --fail-on=all"
-                               sh "./snyk-linux container test  ${OPERATOR_BUNDLE_IMAGE_CANDIDATE_NAME} --severity-threshold=high --file=Dockerfile --policy-path=.snyk --fail-on=all"
+                               sh "./snyk-linux container test ${OPERATOR_CONTAINER_IMAGE_CANDIDATE_NAME} --severity-threshold=high --file=Dockerfile --policy-path=.snyk --fail-on=all"
+                               sh "./snyk-linux container test ${OPERATOR_BUNDLE_IMAGE_CANDIDATE_NAME} --severity-threshold=high --file=Dockerfile --policy-path=.snyk --fail-on=all"
                            }
                         }
                     }
@@ -119,7 +119,7 @@ boolean isNightly() {
 }
 
 String getVersion() {
-    def prefix = "3.3.0"
+    def prefix = "3.3.1"
     def candidateName = ""
     if(isNightly()) {
         def timestamp = new Date().format("yyyy-MM-dd")

Makefile

@@ -11,7 +11,7 @@ OPENSHIFT_VERSION="v4.9"
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 # TODO: Version must be pulled from git tags
-VERSION ?= 3.3.0
+VERSION ?= 3.3.1
 
 # Platforms supported
 PLATFORMS ?= linux/amd64,linux/arm64
@@ -140,7 +140,7 @@ go-lint: golanci-lint ## Run golangci-lint against code.
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
 	# KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" cd $(shell pwd)/test; go run github.com/onsi/ginkgo/v2/ginkgo -coverprofile cover.out -progress -v -timeout=12h0m0s -focus=${FOCUS} --junit-report="junit.xml"  -- ${ARGS}
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" cd $(shell pwd)/test; go run github.com/onsi/ginkgo/v2/ginkgo -coverprofile cover.out -show-node-events -v -timeout=12h0m0s -focus=${FOCUS} --junit-report="junit.xml"  -- ${ARGS}
 
 ##@ Build
 
@@ -254,7 +254,7 @@ submodules: ## Pull and update git submodules recursively
 
 # Generate bundle manifests and metadata, then validate generated files.
 # For OpenShift bundles run
-# CHANNELS=stable DEFAULT_CHANNEL=stable OPENSHIFT_VERSION=v4.6 IMG=docker.io/aerospike/aerospike-kubernetes-operator-nightly:3.3.0 make bundle
+# CHANNELS=stable DEFAULT_CHANNEL=stable OPENSHIFT_VERSION=v4.6 IMG=docker.io/aerospike/aerospike-kubernetes-operator-nightly:3.3.1 make bundle
 .PHONY: bundle
 bundle: manifests kustomize
 	rm -rf $(ROOT_DIR)/bundle.Dockerfile $(BUNDLE_DIR)

README.md

@@ -42,7 +42,7 @@ Run the following command with the appropriate name and version for the operator
 
 ```sh
 IMAGE_TAG_BASE=aerospike/aerospike-kubernetes-operator-nightly
-VERSION=3.3.0
+VERSION=3.3.1
 make docker-buildx IMG=${IMAGE_TAG_BASE}:${VERSION} PLATFORMS=linux/amd64
 ```
 **Note**: Change `PLATFORMS` var as per host machine or remove it to build multi-arch image
@@ -94,7 +94,7 @@ Set up the environment with image names.
 ```shell
 export ACCOUNT=aerospike
 export IMAGE_TAG_BASE=${ACCOUNT}/aerospike-kubernetes-operator
-export VERSION=3.3.0
+export VERSION=3.3.1
 export IMG=docker.io/${IMAGE_TAG_BASE}-nightly:${VERSION}
 export BUNDLE_IMG=docker.io/${IMAGE_TAG_BASE}-bundle-nightly:${VERSION}
 export CATALOG_IMG=docker.io/${IMAGE_TAG_BASE}-catalog-nightly:${VERSION}

api/v1/aerospikecluster_types.go

@@ -124,6 +124,31 @@ type AerospikeClusterSpec struct { //nolint:govet // for readability
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Kubernetes Node BlockList"
 	// +kubebuilder:validation:MinItems:=1
 	K8sNodeBlockList []string `json:"k8sNodeBlockList,omitempty"`
+	// Operations is a list of on-demand operations to be performed on the Aerospike cluster.
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Operations"
+	// +kubebuilder:validation:MaxItems:=1
+	Operations []OperationSpec `json:"operations,omitempty"`
+}
+
+type OperationKind string
+
+const (
+	// OperationWarmRestart is the on-demand operation that leads to the warm restart of the aerospike pods
+	// (Restarting ASD in the pods). https://aerospike.com/docs/cloud/kubernetes/operator/Warm-restart
+	OperationWarmRestart OperationKind = "WarmRestart"
+
+	// OperationPodRestart is the on-demand operation that leads to the restart of aerospike pods.
+	OperationPodRestart OperationKind = "PodRestart"
+)
+
+type OperationSpec struct {
+	// Kind is the type of operation to be performed on the Aerospike cluster.
+	// +kubebuilder:validation:Enum=WarmRestart;PodRestart
+	Kind OperationKind `json:"kind"`
+	// +kubebuilder:validation:MaxLength=20
+	// +kubebuilder:validation:MinLength=1
+	ID      string   `json:"id"`
+	PodList []string `json:"podList,omitempty"`
 }
 
 type SeedsFinderServices struct {
@@ -699,6 +724,8 @@ type AerospikeClusterStatusSpec struct { //nolint:govet // for readability
 	RosterNodeBlockList []string `json:"rosterNodeBlockList,omitempty"`
 	// K8sNodeBlockList is a list of Kubernetes nodes which are not used for Aerospike pods.
 	K8sNodeBlockList []string `json:"k8sNodeBlockList,omitempty"`
+	// Operations is a list of on-demand operation to be performed on the Aerospike cluster.
+	Operations []OperationSpec `json:"operations,omitempty"`
 }
 
 // AerospikeClusterStatus defines the observed state of AerospikeCluster
@@ -924,7 +951,7 @@ type AerospikePodStatus struct { //nolint:govet // for readability
 
 // AerospikeCluster is the schema for the AerospikeCluster API
 // +operator-sdk:csv:customresourcedefinitions:displayName="Aerospike Cluster",resources={{Service, v1},{Pod,v1},{StatefulSet,v1}}
-// +kubebuilder:metadata:annotations="aerospike-kubernetes-operator/version=3.3.0"
+// +kubebuilder:metadata:annotations="aerospike-kubernetes-operator/version=3.3.1"
 //
 //nolint:lll // for readability
 type AerospikeCluster struct { //nolint:govet // for readability
@@ -1044,6 +1071,12 @@ func CopySpecToStatus(spec *AerospikeClusterSpec) (*AerospikeClusterStatusSpec,
 		status.K8sNodeBlockList = *k8sNodeBlockList
 	}
 
+	if len(spec.Operations) != 0 {
+		operations := lib.DeepCopy(&spec.Operations).(*[]OperationSpec)
+
+		status.Operations = *operations
+	}
+
 	return &status, nil
 }
 
@@ -1143,5 +1176,10 @@ func CopyStatusToSpec(status *AerospikeClusterStatusSpec) (*AerospikeClusterSpec
 		spec.K8sNodeBlockList = *k8sNodeBlockList
 	}
 
+	if len(status.Operations) != 0 {
+		operations := lib.DeepCopy(&status.Operations).(*[]OperationSpec)
+		spec.Operations = *operations
+	}
+
 	return &spec, nil
 }

api/v1/aerospikecluster_validating_webhook.go

@@ -117,6 +117,12 @@ func (c *AerospikeCluster) ValidateUpdate(oldObj runtime.Object) (admission.Warn
 		return nil, err
 	}
 
+	if err := validateOperationUpdate(
+		&old.Spec, &c.Spec, &c.Status,
+	); err != nil {
+		return nil, err
+	}
+
 	// Validate AerospikeConfig update
 	if err := validateAerospikeConfigUpdate(
 		aslog, incomingVersion, outgoingVersion,
@@ -191,6 +197,10 @@ func (c *AerospikeCluster) validate(aslog logr.Logger) error {
 		return err
 	}
 
+	if err := c.validateOperation(); err != nil {
+		return err
+	}
+
 	// Storage should be validated before validating aerospikeConfig and fileStorage
 	if err := validateStorage(&c.Spec.Storage, &c.Spec.PodSpec); err != nil {
 		return err
@@ -263,6 +273,19 @@ func (c *AerospikeCluster) validate(aslog logr.Logger) error {
 	return c.validateSCNamespaces()
 }
 
+func (c *AerospikeCluster) validateOperation() error {
+	// Nothing to validate if no operation
+	if len(c.Spec.Operations) == 0 {
+		return nil
+	}
+
+	if c.Status.AerospikeConfig == nil {
+		return fmt.Errorf("operation cannot be added during aerospike cluster creation")
+	}
+
+	return nil
+}
+
 func (c *AerospikeCluster) validateSCNamespaces() error {
 	scNamespaceSet := sets.NewString()
 
@@ -1292,20 +1315,22 @@ func validateSecurityConfigUpdate(
 func validateEnableSecurityConfig(newConfSpec, oldConfSpec *AerospikeConfigSpec) error {
 	newConf := newConfSpec.Value
 	oldConf := oldConfSpec.Value
+
 	oldSec, oldSecConfFound := oldConf["security"]
-	newSec, newSecConfFound := newConf["security"]
+	if !oldSecConfFound {
+		return nil
+	}
 
-	if oldSecConfFound && !newSecConfFound {
+	newSec, newSecConfFound := newConf["security"]
+	if !newSecConfFound {
 		return fmt.Errorf("cannot remove cluster security config")
 	}
 
-	if oldSecConfFound && newSecConfFound {
-		oldSecFlag, oldEnableSecurityFlagFound := oldSec.(map[string]interface{})["enable-security"]
-		newSecFlag, newEnableSecurityFlagFound := newSec.(map[string]interface{})["enable-security"]
+	oldSecFlag, oldEnableSecurityFlagFound := oldSec.(map[string]interface{})["enable-security"]
+	newSecFlag, newEnableSecurityFlagFound := newSec.(map[string]interface{})["enable-security"]
 
-		if oldEnableSecurityFlagFound && oldSecFlag.(bool) && (!newEnableSecurityFlagFound || !newSecFlag.(bool)) {
-			return fmt.Errorf("cannot disable cluster security in running cluster")
-		}
+	if oldEnableSecurityFlagFound && oldSecFlag.(bool) && (!newEnableSecurityFlagFound || !newSecFlag.(bool)) {
+		return fmt.Errorf("cannot disable cluster security in running cluster")
 	}
 
 	return nil
@@ -2360,33 +2385,46 @@ func (c *AerospikeCluster) validateEnableDynamicConfigUpdate() error {
 	return nil
 }
 
-func getMinRunningInitVersion(pods map[string]AerospikePodStatus) (string, error) {
-	minVersion := ""
+func validateOperationUpdate(oldSpec, newSpec *AerospikeClusterSpec, status *AerospikeClusterStatus) error {
+	if len(newSpec.Operations) == 0 {
+		return nil
+	}
 
-	for idx := range pods {
-		if pods[idx].InitImage != "" {
-			version, err := GetImageVersion(pods[idx].InitImage)
-			if err != nil {
-				return "", err
-			}
+	newOp := &newSpec.Operations[0]
 
-			if minVersion == "" {
-				minVersion = version
-				continue
-			}
+	var oldOp *OperationSpec
 
-			val, err := lib.CompareVersions(version, minVersion)
-			if err != nil {
-				return "", fmt.Errorf("failed to check image version: %v", err)
-			}
+	if len(oldSpec.Operations) != 0 {
+		oldOp = &oldSpec.Operations[0]
+	}
 
-			if val < 0 {
-				minVersion = version
-			}
-		} else {
-			return baseInitVersion, nil
+	if oldOp != nil && oldOp.ID == newOp.ID && !reflect.DeepEqual(oldOp, newOp) {
+		return fmt.Errorf("operation %s cannot be updated", newOp.ID)
+	}
+
+	allPodNames := GetAllPodNames(status.Pods)
+
+	podSet := sets.New(newSpec.Operations[0].PodList...)
+	if !allPodNames.IsSuperset(podSet) {
+		return fmt.Errorf("invalid pod names in operation %v", podSet.Difference(allPodNames).UnsortedList())
+	}
+
+	// Don't allow any on-demand operation along with these cluster change:
+	// 1- scale up
+	// 2- racks added or removed
+	// 3- image update
+	// New pods won't be available for operation
+	if !reflect.DeepEqual(newSpec.Operations, status.Operations) {
+		switch {
+		case newSpec.Size > status.Size:
+			return fmt.Errorf("cannot change Spec.Operations along with cluster scale-up")
+		case len(newSpec.RackConfig.Racks) != len(status.RackConfig.Racks) ||
+			len(newSpec.RackConfig.Racks) != len(oldSpec.RackConfig.Racks):
+			return fmt.Errorf("cannot change Spec.Operations along with rack addition/removal")
+		case newSpec.Image != status.Image || newSpec.Image != oldSpec.Image:
+			return fmt.Errorf("cannot change Spec.Operations along with image update")
 		}
 	}
 
-	return minVersion, nil
+	return nil
 }

api/v1/utils.go

@@ -7,6 +7,8 @@ import (
 	"regexp"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
 
@@ -71,7 +73,7 @@ const (
 	AerospikeInitContainerRegistryEnvVar           = "AEROSPIKE_KUBERNETES_INIT_REGISTRY"
 	AerospikeInitContainerDefaultRegistry          = "docker.io"
 	AerospikeInitContainerDefaultRegistryNamespace = "aerospike"
-	AerospikeInitContainerDefaultRepoAndTag        = "aerospike-kubernetes-init:2.2.0"
+	AerospikeInitContainerDefaultRepoAndTag        = "aerospike-kubernetes-init:2.2.1"
 	AerospikeAppLabel                              = "app"
 	AerospikeAppLabelValue                         = "aerospike-cluster"
 	AerospikeCustomResourceLabel                   = "aerospike.com/cr"
@@ -536,3 +538,62 @@ func GetDefaultPasswordFilePath(aerospikeConfigSpec *AerospikeConfigSpec) *strin
 
 	return &passFile
 }
+
+func getMinRunningInitVersion(pods map[string]AerospikePodStatus) (string, error) {
+	minVersion := ""
+
+	for idx := range pods {
+		if pods[idx].InitImage != "" {
+			version, err := GetImageVersion(pods[idx].InitImage)
+			if err != nil {
+				return "", err
+			}
+
+			if minVersion == "" {
+				minVersion = version
+				continue
+			}
+
+			val, err := lib.CompareVersions(version, minVersion)
+			if err != nil {
+				return "", fmt.Errorf("failed to check image version: %v", err)
+			}
+
+			if val < 0 {
+				minVersion = version
+			}
+		} else {
+			return baseInitVersion, nil
+		}
+	}
+
+	return minVersion, nil
+}
+
+func DistributeItems(totalItems, totalGroups int) []int {
+	itemsPerGroup, extraItems := totalItems/totalGroups, totalItems%totalGroups
+
+	// Distributing nodes in given racks
+	var topology []int
+
+	for groupIdx := 0; groupIdx < totalGroups; groupIdx++ {
+		itemsForThisGroup := itemsPerGroup
+		if groupIdx < extraItems {
+			itemsForThisGroup++
+		}
+
+		topology = append(topology, itemsForThisGroup)
+	}
+
+	return topology
+}
+
+func GetAllPodNames(pods map[string]AerospikePodStatus) sets.Set[string] {
+	podNames := make(sets.Set[string])
+
+	for podName := range pods {
+		podNames.Insert(podName)
+	}
+
+	return podNames
+}