[#821] Support broker-N in -bp suffix secrets

rh-messaging · Mar 27, 2024 · 16447e4 · 16447e4
1 parent 1381995
commit 16447e4
Show file tree

Hide file tree

Showing 3 changed files with 191 additions and 12 deletions.
diff --git a/controllers/activemqartemis_controller_test.go b/controllers/activemqartemis_controller_test.go
@@ -9322,6 +9322,111 @@ var _ = Describe("artemis controller", func() {
 			Expect(k8sClient.Delete(ctx, &crd)).To(Succeed())
 		})
 
+		It("-bp suffix secret broker-n support", Label("broker-n-bp-secret"), func() {
+
+			ctx := context.Background()
+
+			_, bpSecret1 := DeploySecret(defaultNamespace, func(candidate *corev1.Secret) {
+				candidate.Name = "config-1-bp"
+				candidate.StringData = map[string]string{
+					"journal1.properties":           `journalFileSize=12345`,
+					"broker-0.globalMem.properties": `globalMaxSize=512M`,
+				}
+			})
+
+			_, bpSecret2 := DeploySecret(defaultNamespace, func(candidate *corev1.Secret) {
+				candidate.Name = "config-2-bp"
+				candidate.StringData = map[string]string{
+					"journal2.properties":           `journalMinFiles=3`,
+					"broker-1.globalMem.properties": `globalMaxSize=12M`,
+				}
+			})
+
+			By("Deploying the CRD")
+			_, crd := DeployCustomBroker(defaultNamespace, func(candidate *brokerv1beta1.ActiveMQArtemis) {
+				candidate.Spec.DeploymentPlan.Size = common.Int32ToPtr(2)
+				candidate.Spec.DeploymentPlan.ExtraMounts.Secrets = []string{bpSecret1.Name, bpSecret2.Name}
+			})
+
+			if os.Getenv("USE_EXISTING_CLUSTER") == "true" {
+				ssKey := types.NamespacedName{
+					Name:      namer.CrToSS(crd.Name),
+					Namespace: defaultNamespace,
+				}
+
+				By("checking statefulset is ready")
+				currentSS := &appsv1.StatefulSet{}
+				Eventually(func(g Gomega) {
+					g.Expect(k8sClient.Get(ctx, ssKey, currentSS)).Should(Succeed())
+					g.Expect(currentSS.Status.ReadyReplicas).To(Equal(int32(2)))
+				}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
+
+				By("checking pod 0 status that has properties applied")
+				podWithOrdinal0 := namer.CrToSS(crd.Name) + "-0"
+
+				curlUrl := "http://" + podWithOrdinal0 + ":8161/console/jolokia/read/org.apache.activemq.artemis:broker=\"amq-broker\"/Status"
+				curlCmd := []string{"curl", "-s", "-H", "Origin: http://localhost:8161", "-u", "user:password", curlUrl}
+				Eventually(func(g Gomega) {
+					result, err := RunCommandInPod(podWithOrdinal0, crd.Name+"-container", curlCmd)
+					g.Expect(err).To(BeNil())
+					g.Expect(*result).To(ContainSubstring("journal1.properties"))
+					g.Expect(*result).To(ContainSubstring("broker-0.globalMem.properties"))
+					g.Expect(*result).To(ContainSubstring("journal2.properties"))
+					g.Expect(*result).NotTo(ContainSubstring("broker-1.globalMem.properties"))
+				}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
+
+				curlUrl = "http://" + podWithOrdinal0 + ":8161/console/jolokia/read/org.apache.activemq.artemis:broker=\"amq-broker\"/GlobalMaxSize"
+				curlCmd = []string{"curl", "-s", "-H", "Origin: http://localhost:8161", "-u", "user:password", curlUrl}
+				Eventually(func(g Gomega) {
+					result, err := RunCommandInPod(podWithOrdinal0, crd.Name+"-container", curlCmd)
+					g.Expect(err).To(BeNil())
+					// 512M = 512 * 1024 * 1024
+					g.Expect(*result).To(ContainSubstring("\"value\":536870912"))
+				}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
+
+				By("checking pod 1 status that has properties applied")
+				podWithOrdinal1 := namer.CrToSS(crd.Name) + "-1"
+
+				curlUrl = "http://" + podWithOrdinal1 + ":8161/console/jolokia/read/org.apache.activemq.artemis:broker=\"amq-broker\"/Status"
+				curlCmd = []string{"curl", "-s", "-H", "Origin: http://localhost:8161", "-u", "user:password", curlUrl}
+				Eventually(func(g Gomega) {
+					result, err := RunCommandInPod(podWithOrdinal1, crd.Name+"-container", curlCmd)
+					g.Expect(err).To(BeNil())
+					g.Expect(*result).To(ContainSubstring("journal1.properties"))
+					g.Expect(*result).To(ContainSubstring("broker-1.globalMem.properties"))
+					g.Expect(*result).To(ContainSubstring("journal2.properties"))
+					g.Expect(*result).NotTo(ContainSubstring("broker-0.globalMem.properties"))
+				}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
+
+				curlUrl = "http://" + podWithOrdinal1 + ":8161/console/jolokia/read/org.apache.activemq.artemis:broker=\"amq-broker\"/GlobalMaxSize"
+				curlCmd = []string{"curl", "-s", "-H", "Origin: http://localhost:8161", "-u", "user:password", curlUrl}
+				Eventually(func(g Gomega) {
+					result, err := RunCommandInPod(podWithOrdinal1, crd.Name+"-container", curlCmd)
+					g.Expect(err).To(BeNil())
+					// 12M = 12 * 1024 * 1024
+					g.Expect(*result).To(ContainSubstring("\"value\":12582912"))
+				}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
+
+				createdCrd := &brokerv1beta1.ActiveMQArtemis{}
+				brokerKey := types.NamespacedName{Name: crd.Name, Namespace: crd.Namespace}
+
+				By("verify status ok")
+				Eventually(func(g Gomega) {
+					g.Expect(k8sClient.Get(ctx, brokerKey, createdCrd)).Should(Succeed())
+
+					condition := meta.FindStatusCondition(createdCrd.Status.Conditions, brokerv1beta1.ConfigAppliedConditionType)
+					g.Expect(condition).NotTo(BeNil())
+					g.Expect(condition.Status).To(Equal(metav1.ConditionTrue))
+
+				}, existingClusterTimeout, existingClusterInterval).Should(Succeed())
+
+			}
+
+			Expect(k8sClient.Delete(ctx, crd)).To(Succeed())
+			Expect(k8sClient.Delete(ctx, bpSecret1)).To(Succeed())
+			Expect(k8sClient.Delete(ctx, bpSecret2)).To(Succeed())
+		})
+
 		It("extraMount.configMap logging config manually", func() {
 
 			ctx := context.Background()

diff --git a/controllers/activemqartemis_reconciler.go b/controllers/activemqartemis_reconciler.go
@@ -1769,17 +1769,23 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) NewPodTemplateSpecForCR(customR
 
 	configMapsToCreate := customResource.Spec.DeploymentPlan.ExtraMounts.ConfigMaps
 	secretsToCreate := customResource.Spec.DeploymentPlan.ExtraMounts.Secrets
-	brokerPropertiesResourceName, isSecret, brokerPropertiesMapData := reconciler.addResourceForBrokerProperties(customResource, namer)
+	brokerPropertiesResourceName, isSecret, brokerPropertiesMapData, serr := reconciler.addResourceForBrokerProperties(customResource, namer)
+	if serr != nil {
+		return nil, serr
+	}
 	if isSecret {
 		secretsToCreate = append(secretsToCreate, brokerPropertiesResourceName)
 	} else {
 		configMapsToCreate = append(configMapsToCreate, brokerPropertiesResourceName)
 	}
-	extraVolumes, extraVolumeMounts := reconciler.createExtraConfigmapsAndSecretsVolumeMounts(configMapsToCreate, secretsToCreate, brokerPropertiesResourceName, brokerPropertiesMapData)
+	extraVolumes, extraVolumeMounts, err := reconciler.createExtraConfigmapsAndSecretsVolumeMounts(configMapsToCreate, secretsToCreate, brokerPropertiesResourceName, brokerPropertiesMapData, client)
+	if err != nil {
+		return nil, err
+	}
 
 	reqLogger.V(2).Info("Extra volumes", "volumes", extraVolumes)
 	reqLogger.V(2).Info("Extra mounts", "mounts", extraVolumeMounts)
-	var err error
+
 	container.VolumeMounts, err = reconciler.MakeVolumeMounts(customResource, namer)
 	if err != nil {
 		return nil, err
@@ -2070,8 +2076,8 @@ func brokerPropertiesConfigSystemPropValue(customResource *brokerv1beta1.ActiveM
 
 	for _, extraSecretName := range customResource.Spec.DeploymentPlan.ExtraMounts.Secrets {
 		if strings.HasSuffix(extraSecretName, brokerPropsSuffix) {
-			// formated append to result with comma
-			result = fmt.Sprintf("%s,%s%s/", result, secretPathBase, extraSecretName)
+			// append to ordinal path
+			result = fmt.Sprintf("%s,%s%s/,%s%s/%s${STATEFUL_SET_ORDINAL}/", result, secretPathBase, extraSecretName, secretPathBase, extraSecretName, OrdinalPrefix)
 		}
 	}
 	return result
@@ -2258,7 +2264,7 @@ func getConfigAppliedConfigMapName(artemis *brokerv1beta1.ActiveMQArtemis) types
 	}
 }
 
-func (reconciler *ActiveMQArtemisReconcilerImpl) addResourceForBrokerProperties(customResource *brokerv1beta1.ActiveMQArtemis, namer common.Namers) (string, bool, map[string]string) {
+func (reconciler *ActiveMQArtemisReconcilerImpl) addResourceForBrokerProperties(customResource *brokerv1beta1.ActiveMQArtemis, namer common.Namers) (string, bool, map[string]string, error) {
 
 	// fetch and do idempotent transform based on CR
 
@@ -2277,7 +2283,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) addResourceForBrokerProperties(
 		reconciler.log.V(1).Info("Requesting configMap for broker properties", "name", resourceName.Name)
 		reconciler.trackDesired(existing)
 
-		return resourceName.Name, false, existing.Data
+		return resourceName.Name, false, existing.Data, nil
 	}
 
 	var desired *corev1.Secret
@@ -2300,7 +2306,7 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) addResourceForBrokerProperties(
 	reconciler.trackDesired(desired)
 
 	reconciler.log.V(1).Info("Requesting mount for broker properties secret")
-	return resourceName.Name, true, data
+	return resourceName.Name, true, data, nil
 }
 
 func alder32StringValue(alder32Bytes []byte) string {
@@ -2453,7 +2459,7 @@ func (r *ActiveMQArtemisReconcilerImpl) configPodSecurity(podSpec *corev1.PodSpe
 	}
 }
 
-func (r *ActiveMQArtemisReconcilerImpl) createExtraConfigmapsAndSecretsVolumeMounts(configMaps []string, secrets []string, brokePropertiesResourceName string, brokerPropsData map[string]string) ([]corev1.Volume, []corev1.VolumeMount) {
+func (r *ActiveMQArtemisReconcilerImpl) createExtraConfigmapsAndSecretsVolumeMounts(configMaps []string, secrets []string, brokePropertiesResourceName string, brokerPropsData map[string]string, client rtclient.Client) ([]corev1.Volume, []corev1.VolumeMount, error) {
 
 	var extraVolumes []corev1.Volume
 	var extraVolumeMounts []corev1.VolumeMount
@@ -2495,13 +2501,36 @@ func (r *ActiveMQArtemisReconcilerImpl) createExtraConfigmapsAndSecretsVolumeMou
 					}
 				}
 			}
+
+			if strings.HasSuffix(secret, brokerPropsSuffix) {
+				bpSecret := &corev1.Secret{}
+				bpSecretKey := types.NamespacedName{
+					Name:      secret,
+					Namespace: r.customResource.Namespace,
+				}
+				if err := resources.Retrieve(bpSecretKey, client, bpSecret); err != nil {
+					return nil, nil, err
+				}
+
+				if len(bpSecret.Data) > 0 {
+					for _, key := range sortedKeysStringKeyByteValue(bpSecret.Data) {
+						if hasOrdinal, separatorIndex := extractOrdinalPrefixSeperatorIndex(key); hasOrdinal {
+							subPath := key[:separatorIndex]
+							secretVol.VolumeSource.Secret.Items = append(secretVol.VolumeSource.Secret.Items, corev1.KeyToPath{Key: key, Path: fmt.Sprintf("%s/%s", subPath, key)})
+						} else {
+							secretVol.VolumeSource.Secret.Items = append(secretVol.VolumeSource.Secret.Items, corev1.KeyToPath{Key: key, Path: key})
+						}
+					}
+				}
+
+			}
 			secretVolumeMount := volumes.MakeVolumeMountForCfg(secretVol.Name, secretPath, true)
 			extraVolumes = append(extraVolumes, secretVol)
 			extraVolumeMounts = append(extraVolumeMounts, secretVolumeMount)
 		}
 	}
 
-	return extraVolumes, extraVolumeMounts
+	return extraVolumes, extraVolumeMounts, nil
 }
 
 func extractOrdinalPrefixSeperatorIndex(key string) (bool, int) {
@@ -2815,7 +2844,6 @@ func AssertBrokerPropertiesStatus(cr *brokerv1beta1.ActiveMQArtemis, client rtcl
 	if errorStatus == nil {
 		for _, extraSecretName := range cr.Spec.DeploymentPlan.ExtraMounts.Secrets {
 			if strings.HasSuffix(extraSecretName, brokerPropsSuffix) {
-
 				secretProjection, err = getSecretProjection(types.NamespacedName{Name: extraSecretName, Namespace: cr.Namespace}, client)
 				if err != nil {
 					reqLogger.V(2).Info("error retrieving -bp extra mount resource. requeing")
@@ -3051,7 +3079,7 @@ func getConfigMappedJaasProperties(cr *brokerv1beta1.ActiveMQArtemis, client rtc
 func newProjectionFromByteValues(resourceMeta metav1.ObjectMeta, configKeyValue map[string][]byte) *projection {
 	projection := projection{Name: resourceMeta.Name, ResourceVersion: resourceMeta.ResourceVersion, Generation: resourceMeta.Generation, Files: map[string]propertyFile{}}
 	for prop_file_name, data := range configKeyValue {
-		projection.Files[prop_file_name] = propertyFile{Alder32: alder32FromData([]byte(data))}
+		projection.Files[prop_file_name] = propertyFile{Alder32: alder32FromData(data)}
 	}
 	return &projection
 }

diff --git a/docs/help/operator.md b/docs/help/operator.md
@@ -844,9 +844,55 @@ spec:
 ## Providing additional brokerProperties configuration from a secret
 It is possible to replace the use of the activemqartemisaddresses CRD and much of the activemqartemissecurities CRD with configuration via broker properties. This can necessitate a large amount of configuration in the CR.brokerProperties field.
 In order to provide a way to split or orgainse these properties by file or by secret, an extra mount can be used to provide a secret that will be treated as an additional source of broker properties configuration.
+
 Using an **extraMounts** secret with a suffix "-bp" will cause the operator to auto mount the secret and make the broker aware of it's location. In addition the CR.Status.Condition[BrokerPropertiesApplied] will reflect the content of this secret.
+
 Broker properties are applied in order, starting with the CR.brokerProperties and then with the "-bp" auto mounts in turn. Keys (or property files) from secrets are applied in alphabetical order.
 
+To configure a specific broker instance in a "-bp" secret, use `broker-N` as the prefix for a key in the secret data. For example:
+
+Create two -bp secrets:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: config-1-bp
+stringData:
+  journal1.properties: |
+    journalFileSize=12345
+  broker-0.globalMem.properties: |
+    globalMaxSize=512M
+```
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: config-2-bp
+stringData:
+  journal2.properties: |
+    journalMinFiles=3
+  broker-1.globalMem.properties: |
+    globalMaxSize=12M
+```
+
+and add the above secrets to extraMounts in the CR:
+
+```yaml
+apiVersion: broker.amq.io/v1beta1
+kind: ActiveMQArtemis
+metadata:
+  name: ex-aao
+spec:
+  deploymentPlan:
+    size: 2
+    extraMounts:
+      secrets:
+      - "config-1-bp"
+      - "config-2-bp"
+```
+When the CR is deployed the broker in pod 0 broker will get `globalMaxSize=512M` and pod 1 broker will get `globalMaxSize=12M`. While both will get properties from `journal1.properties` of secret **config-1-bp** and `journal2.properties` from secret **config-2-bp**.
 
 ## Configuring Logging for Brokers