Skip to content

Commit

Permalink
reconcile requeue limitter was implemented
Browse files Browse the repository at this point in the history
  • Loading branch information
Bohdan Siryk authored and Bohdan Siryk committed Sep 21, 2023
1 parent 88ad6c2 commit 4a50ffc
Show file tree
Hide file tree
Showing 4 changed files with 68 additions and 2 deletions.
2 changes: 1 addition & 1 deletion config/samples/clusters_v1beta1_cassandra.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ kind: Cassandra
metadata:
name: cassandra-cluster
spec:
name: "username-Cassandra"
name: "bohdan-Cassandra"
version: "4.0.10"
privateNetworkCluster: false
dataCentres:
Expand Down
3 changes: 2 additions & 1 deletion controllers/clusters/cassandra_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package clusters
import (
"context"
"errors"
"github.com/instaclustr/operator/pkg/reconcileutil"
"strconv"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -1307,5 +1308,5 @@ func (r *CassandraReconciler) SetupWithManager(mgr ctrl.Manager) error {
event.Object.GetAnnotations()[models.ResourceStateAnnotation] = models.GenericEvent
return true
},
})).Complete(r)
})).Complete(reconcileutil.ReconcilerWithLimit(r))
}
4 changes: 4 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package main

import (
"context"
"flag"
"os"
"time"
Expand Down Expand Up @@ -89,6 +90,9 @@ func main() {
HealthProbeBindAddress: probeAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: "680bba91.instaclustr.com",
BaseContext: func() context.Context {
return context.WithValue(context.Background(), "reconcile", "true")
},
// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
// when the Manager ends. This requires the binary to immediately end when the
// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
Expand Down
61 changes: 61 additions & 0 deletions pkg/reconcileutil/reconcile-limitation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package reconcileutil

import (
"context"
"fmt"
"github.com/instaclustr/operator/pkg/models"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

const maxReconcileRequeueRetries = 1

func ReconcilerWithLimit(r reconcile.Reconciler) reconcile.Func {
limiter := &ReconcileRequeueLimiter{
maxRetries: maxReconcileRequeueRetries,
m: make(map[types.NamespacedName]int),
}

return func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
fmt.Println("Reconciler with limit")
if !limiter.RequeueAllowed(req.NamespacedName) {
fmt.Println("reconcile is not allowed")
l := log.FromContext(ctx)
l.Info("Amount of reconcile requeue retries reached its maximum for the resource",
"req", req.NamespacedName,
)

limiter.Reset(req.NamespacedName)

return models.ExitReconcile, nil
}

result, err := r.Reconcile(ctx, req)
if err != nil || result == models.ReconcileRequeue {
limiter.Requeue(req.NamespacedName)
return result, err
}

limiter.Reset(req.NamespacedName)

return result, nil
}
}

type ReconcileRequeueLimiter struct {
maxRetries int
m map[types.NamespacedName]int
}

func (r *ReconcileRequeueLimiter) Requeue(key types.NamespacedName) {
r.m[key]++
}

func (r *ReconcileRequeueLimiter) RequeueAllowed(key types.NamespacedName) bool {
return r.m[key] <= r.maxRetries
}

func (r *ReconcileRequeueLimiter) Reset(key types.NamespacedName) {
delete(r.m, key)
}

0 comments on commit 4a50ffc

Please sign in to comment.