cmd/tetra: add retries with exponential backoff

[upstream commit 4f4feb8]

Allow the user to retry failed gRPC connections with exponential backoff.

Signed-off-by: William Findlay <[email protected]>

cmd/tetra/common/client.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/cilium/tetragon/api/v1/tetragon"
 	"github.com/cilium/tetragon/pkg/logger"
@@ -15,18 +16,47 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
+func connect(ctx context.Context) (*grpc.ClientConn, string, error) {
+	connCtx, connCancel := context.WithTimeout(ctx, viper.GetDuration(KeyTimeout))
+	defer connCancel()
+
+	var conn *grpc.ClientConn
+	var serverAddr string
+	var err error
+
+	conn, err = grpc.DialContext(connCtx, viper.GetString(KeyServerAddress), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
+
+	return conn, serverAddr, err
+}
+
 func CliRunErr(fn func(ctx context.Context, cli tetragon.FineGuidanceSensorsClient), fnErr func(err error)) {
 	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer cancel()
 
-	connCtx, connCancel := context.WithTimeout(ctx, viper.GetDuration(KeyTimeout))
-	defer connCancel()
-	conn, err := grpc.DialContext(connCtx, viper.GetString(KeyServerAddress), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
-	if err != nil {
-		fnErr(err)
-		logger.GetLogger().WithError(err).Fatal("Failed to connect")
+	var conn *grpc.ClientConn
+	var serverAddr string
+	var err error
+
+	backoff := time.Second
+	attempts := 0
+	for {
+		conn, serverAddr, err = connect(ctx)
+		if err != nil {
+			if attempts < viper.GetInt(KeyRetries) {
+				// Exponential backoff
+				attempts++
+				logger.GetLogger().WithField("server-address", serverAddr).WithField("attempts", attempts).WithError(err).Error("Connection attempt failed, retrying...")
+				time.Sleep(backoff)
+				backoff *= 2
+				continue
+			}
+			logger.GetLogger().WithField("server-address", serverAddr).WithField("attempts", attempts).WithError(err).Fatal("Failed to connect to server")
+			fnErr(err)
+		}
+		break
 	}
 	defer conn.Close()
+
 	client := tetragon.NewFineGuidanceSensorsClient(conn)
 	fn(ctx, client)
 }

cmd/tetra/common/flags.go

@@ -9,4 +9,5 @@ const (
 	KeyOutput        = "output"         // string
 	KeyServerAddress = "server-address" // string
 	KeyTimeout       = "timeout"        // duration
+	KeyRetries       = "retries"        // int
 )

cmd/tetra/main.go

@@ -44,6 +44,7 @@ func new() *cobra.Command {
 	flags.BoolP(common.KeyDebug, "d", false, "Enable debug messages")
 	flags.String(common.KeyServerAddress, "localhost:54321", "gRPC server address")
 	flags.Duration(common.KeyTimeout, 10*time.Second, "Connection timeout")
+	flags.Int(common.KeyRetries, 0, "Connection retries with exponential backoff")
 	viper.BindPFlags(flags)
 	return rootCmd
 }