feat: customize maxBytes and maxMsgs for EventBus (#1272)

* feat: customize maxBytes and maxMsgs for EventBus Signed-off-by: Derek Wang <[email protected]>
argoproj · Jul 2, 2021 · b38bed1 · b38bed1
1 parent 4163d44
commit b38bed1
Show file tree

Hide file tree

Showing 11 changed files with 270 additions and 89 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -8,9 +8,9 @@ RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-ENV ARGO_VERSION=v3.0.2
+ENV ARGO_VERSION=v3.1.1
 
-RUN wget -q https://github.com/argoproj/argo/releases/download/${ARGO_VERSION}/argo-linux-${ARCH}.gz
+RUN wget -q https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-${ARCH}.gz
 RUN gunzip -f argo-linux-${ARCH}.gz
 RUN chmod +x argo-linux-${ARCH}
 RUN mv ./argo-linux-${ARCH} /usr/local/bin/argo

diff --git a/api/event-bus.html b/api/event-bus.html
diff --git a/api/event-bus.md b/api/event-bus.md
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
diff --git a/controllers/eventbus/installer/nats.go b/controllers/eventbus/installer/nats.go
@@ -445,6 +445,14 @@ func (i *natsInstaller) buildConfigMap() (*corev1.ConfigMap, error) {
 	if err != nil {
 		return nil, err
 	}
+	maxMsgs := uint64(1000000)
+	if i.eventBus.Spec.NATS.Native.MaxMsgs != nil {
+		maxMsgs = *i.eventBus.Spec.NATS.Native.MaxMsgs
+	}
+	maxBytes := "1GB"
+	if i.eventBus.Spec.NATS.Native.MaxBytes != nil {
+		maxBytes = *i.eventBus.Spec.NATS.Native.MaxBytes
+	}
 	peers := []string{}
 	routes := []string{}
 	for j := 0; j < replicas; j++ {
@@ -470,8 +478,10 @@ streaming {
   }
   store_limits {
     max_age: %s
+	max_msgs: %v
+	max_bytes: %s
   }
-}`, strconv.Itoa(int(monitorPort)), strconv.Itoa(int(clusterPort)), strings.Join(routes, ","), clusterID, strings.Join(peers, ","), maxAge)
+}`, strconv.Itoa(int(monitorPort)), strconv.Itoa(int(clusterPort)), strings.Join(routes, ","), clusterID, strings.Join(peers, ","), maxAge, maxMsgs, maxBytes)
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: i.eventBus.Namespace,

diff --git a/docs/eventbus.md b/docs/eventbus.md
@@ -155,7 +155,11 @@ affinity:
 
 #### More About Native NATS EventBus
 
-- Messages limit is 1,000,000.
+- Messages limit per channel defaults to 1,000,000. It could be customized by
+  setting `spec.nats.native.maxMsgs`, `0` means unlimited.
+
+- Message bytes per channel defaults to `1GB`, setting
+  `spec.nats.native.maxBytes` to customize it, `"0"` means unlimited.
 
 - Max age of messages is 72 hours, which means messages over 72 hours will be
   deleted automatically. It can be cutomized by setting

diff --git a/pkg/apis/eventbus/v1alpha1/generated.pb.go b/pkg/apis/eventbus/v1alpha1/generated.pb.go
diff --git a/pkg/apis/eventbus/v1alpha1/generated.proto b/pkg/apis/eventbus/v1alpha1/generated.proto
diff --git a/pkg/apis/eventbus/v1alpha1/openapi_generated.go b/pkg/apis/eventbus/v1alpha1/openapi_generated.go
diff --git a/pkg/apis/eventbus/v1alpha1/types.go b/pkg/apis/eventbus/v1alpha1/types.go
@@ -125,6 +125,10 @@ type NativeStrategy struct {
 	// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
 	// +optional
 	Affinity *corev1.Affinity `json:"affinity,omitempty" protobuf:"bytes,16,opt,name=affinity"`
+	// Maximum number of messages per channel, 0 means unlimited. Defaults to 1000000
+	MaxMsgs *uint64 `json:"maxMsgs,omitempty" protobuf:"bytes,17,opt,name=maxMsgs"`
+	// Total size of messages per channel, 0 means unlimited. Defaults to 1GB
+	MaxBytes *string `json:"maxBytes,omitempty" protobuf:"bytes,18,opt,name=maxBytes"`
 }
 
 // ContainerTemplate defines customized spec for a container

diff --git a/pkg/apis/eventbus/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventbus/v1alpha1/zz_generated.deepcopy.go