Adding XSITE to an existing Infinispan CR breaks the cluster. Fixes i…

…nfinispan#2037

pkg/kubernetes/pods.go

@@ -2,10 +2,7 @@ package kubernetes
 
 import (
 	"context"
-	"fmt"
 	"reflect"
-	"strconv"
-	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -162,47 +159,20 @@ func GetInitContainer(name string, spec *corev1.PodSpec) *corev1.Container {
 	return nil
 }
 
-func GetPodMemoryLimitBytes(container, podName, namespace string, kube *Kubernetes) (uint64, error) {
-	execOut, err := kube.ExecWithOptions(ExecOptions{
-		Container: container,
-		Command:   []string{"/bin/bash", "-c", "cat /sys/fs/cgroup/memory/memory.limit_in_bytes || cat /sys/fs/cgroup/memory.max"},
-		PodName:   podName,
-		Namespace: namespace,
-	})
-
-	if err != nil {
-		return 0, fmt.Errorf("unexpected error getting memory limit bytes, err: %w", err)
-	}
-
-	result := strings.TrimSuffix(execOut.String(), "\n")
-	limitBytes, err := strconv.ParseUint(result, 10, 64)
-	if err != nil {
-		return 0, err
+func VolumeExists(name string, spec *corev1.PodSpec) bool {
+	for _, volume := range spec.Volumes {
+		if volume.Name == name {
+			return true
+		}
 	}
-	return limitBytes, nil
+	return false
 }
 
-func GetPodMaxMemoryUnboundedBytes(container, podName, namespace string, kube *Kubernetes) (uint64, error) {
-	execOut, err := kube.ExecWithOptions(ExecOptions{
-		Container: container,
-		Command:   []string{"cat", "/proc/meminfo"},
-		PodName:   podName,
-		Namespace: namespace,
-	})
-
-	if err != nil {
-		return 0, fmt.Errorf("unexpected error getting max unbounded memory, err: %w", err)
-	}
-
-	for _, line := range strings.Split(execOut.String(), "\n") {
-		if strings.Contains(line, "MemTotal:") {
-			tokens := strings.Fields(line)
-			maxUnboundKb, err := strconv.ParseUint(tokens[1], 10, 64)
-			if err != nil {
-				return 0, err
-			}
-			return maxUnboundKb * 1024, nil
+func VolumeMountExists(name string, container *corev1.Container) bool {
+	for _, mount := range container.VolumeMounts {
+		if mount.Name == name {
+			return true
 		}
 	}
-	return 0, fmt.Errorf("meminfo lacking MemTotal information")
+	return false
 }

pkg/reconcile/pipeline/infinispan/handler/manage/statefulset_updates.go

@@ -194,6 +194,8 @@ func StatefulSetRollingUpgrade(i *ispnv1.Infinispan, ctx pipeline.Context) {
 		}
 	}
 
+	updateNeeded = provision.AddXSiteTLSVolumes(ctx, i, statefulSet) || updateNeeded
+
 	if updateNeeded {
 		log.Info("updateNeeded")
 		// If updating the parameters results in a rolling upgrade, we can update the labels here too

pkg/reconcile/pipeline/infinispan/handler/provision/statefulsets.go

@@ -171,7 +171,7 @@ func ClusterStatefulSetSpec(statefulSetName string, i *ispnv1.Infinispan, ctx pi
 	addUserIdentities(ctx, i, statefulSet)
 	addUserConfigVolumes(ctx, i, statefulSet)
 	addTLS(ctx, i, statefulSet)
-	addXSiteTLS(ctx, i, statefulSet)
+	AddXSiteTLSVolumes(ctx, i, statefulSet)
 	return statefulSet, nil
 }
 
@@ -348,12 +348,13 @@ func addTLS(ctx pipeline.Context, i *ispnv1.Infinispan, statefulSet *appsv1.Stat
 	}
 }
 
-func addXSiteTLS(ctx pipeline.Context, i *ispnv1.Infinispan, statefulset *appsv1.StatefulSet) {
+func AddXSiteTLSVolumes(ctx pipeline.Context, i *ispnv1.Infinispan, statefulset *appsv1.StatefulSet) (updated bool) {
 	if i.IsSiteTLSEnabled() {
 		spec := &statefulset.Spec.Template.Spec
-		AddSecretVolume(i.GetSiteTransportSecretName(), SiteTransportKeystoreVolumeName, consts.SiteTransportKeyStoreRoot, spec, InfinispanContainer)
+		updated = AddSecretVolume(i.GetSiteTransportSecretName(), SiteTransportKeystoreVolumeName, consts.SiteTransportKeyStoreRoot, spec, InfinispanContainer)
 		if ctx.ConfigFiles().Transport.Truststore != nil {
-			AddSecretVolume(i.GetSiteTrustoreSecretName(), SiteTruststoreVolumeName, consts.SiteTrustStoreRoot, spec, InfinispanContainer)
+			updated = AddSecretVolume(i.GetSiteTrustoreSecretName(), SiteTruststoreVolumeName, consts.SiteTrustStoreRoot, spec, InfinispanContainer) || updated
 		}
 	}
+	return
 }

test/e2e/infinispan/spec_update_test.go

@@ -6,9 +6,15 @@ import (
 	"testing"
 
 	ispnv1 "github.com/infinispan/infinispan-operator/api/v1"
+	consts "github.com/infinispan/infinispan-operator/controllers/constants"
+	"github.com/infinispan/infinispan-operator/pkg/kubernetes"
+	"github.com/infinispan/infinispan-operator/pkg/reconcile/pipeline/infinispan/handler/provision"
 	tutils "github.com/infinispan/infinispan-operator/test/e2e/utils"
+	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/pointer"
@@ -124,6 +130,83 @@ func TestProbeUpdates(t *testing.T) {
 	genericTestForContainerUpdated(*spec, modifier, verifier)
 }
 
+func xsiteTlsSecret(name, namespace, filename string, file []byte) *corev1.Secret {
+	secret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Type: corev1.SecretTypeOpaque,
+		StringData: map[string]string{
+			"password": tutils.KeystorePassword,
+		},
+		Data: map[string][]byte{
+			filename: file,
+		},
+	}
+	testKube.CreateSecret(secret)
+	return secret
+}
+
+func TestXSiteUpdates(t *testing.T) {
+	t.Parallel()
+	defer testKube.CleanNamespaceAndLogOnPanic(t, tutils.Namespace)
+
+	spec := tutils.DefaultSpec(t, testKube, nil)
+	transport, router, trust := tutils.CreateDefaultCrossSiteKeyAndTrustStore()
+	keystoreSecret := xsiteTlsSecret(spec.Name+"-transport", spec.Namespace, consts.DefaultSiteKeyStoreFileName, transport)
+	routerSecret := xsiteTlsSecret(spec.Name+"-router", spec.Namespace, consts.DefaultSiteKeyStoreFileName, router)
+	trustSecret := xsiteTlsSecret(spec.Name+"-trust", spec.Namespace, consts.DefaultSiteTrustStoreFileName, trust)
+
+	var modifier = func(ispn *ispnv1.Infinispan) {
+		ispn.Spec.Service.Sites = &ispnv1.InfinispanSitesSpec{
+			Local: ispnv1.InfinispanSitesLocalSpec{
+				Name: "local",
+				Expose: ispnv1.CrossSiteExposeSpec{
+					Type: ispnv1.CrossSiteExposeTypeClusterIP,
+				},
+				MaxRelayNodes: 1,
+				Discovery: &ispnv1.DiscoverySiteSpec{
+					Memory: "500Mi",
+					CPU:    "500m",
+				},
+				Encryption: &ispnv1.EncryptionSiteSpec{
+					TransportKeyStore: ispnv1.CrossSiteKeyStore{
+						SecretName: keystoreSecret.Name,
+					},
+					TrustStore: &ispnv1.CrossSiteTrustStore{
+						SecretName: trustSecret.Name,
+					},
+					RouterKeyStore: ispnv1.CrossSiteKeyStore{
+						SecretName: routerSecret.Name,
+					},
+				},
+			},
+			Locations: []ispnv1.InfinispanSiteLocationSpec{
+				{
+					Name: "remote-site-1",
+					URL:  "infinispan+xsite://fake-site-1.svc.local:7900",
+				},
+			},
+		}
+	}
+	var verifier = func(ispn *ispnv1.Infinispan, ss *appsv1.StatefulSet) {
+		podSpec := &ss.Spec.Template.Spec
+		container := kubernetes.GetContainer(provision.InfinispanContainer, podSpec)
+		_assert := assert.New(t)
+		_assert.True(kubernetes.VolumeExists(provision.SiteTransportKeystoreVolumeName, podSpec))
+		_assert.True(kubernetes.VolumeMountExists(provision.SiteTruststoreVolumeName, container))
+		_assert.True(kubernetes.VolumeExists(provision.SiteTruststoreVolumeName, podSpec))
+		_assert.True(kubernetes.VolumeMountExists(provision.SiteTruststoreVolumeName, container))
+
+	}
+	genericTestForContainerUpdated(*spec, modifier, verifier)
+}
+
 func genericTestForContainerUpdated(ispn ispnv1.Infinispan, modifier func(*ispnv1.Infinispan), verifier func(*ispnv1.Infinispan, *appsv1.StatefulSet)) {
 	testKube.CreateInfinispan(&ispn, tutils.Namespace)
 	testKube.WaitForInfinispanPods(int(ispn.Spec.Replicas), tutils.SinglePodTimeout, ispn.Name, tutils.Namespace)