feat: add Redis Sentinel support (#6)

Co-authored-by: Baptiste Mayelle <[email protected]>
Co-authored-by: Fernandez Ludovic <[email protected]>

.github/workflows/build.yml

@@ -3,7 +3,7 @@ name: Build and test
 on: [push, pull_request]
 
 env:
-  GOLANGCI_LINT_VERSION: v1.54.1
+  GOLANGCI_LINT_VERSION: v1.54.2
 
 jobs:
   build:

.gitignore

@@ -1 +1,4 @@
 .idea/
+sentinel1.conf
+sentinel2.conf
+sentinel3.conf

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all
+m.PHONY: all
 all: validate test clean
 
 ## Run validates
@@ -17,8 +17,14 @@ test:
 test-start-stack:
 	docker-compose -f script/docker-compose.yml up --wait
 
+	PORT=26379 envsubst < ./script/conf/sentinel_template.conf > ./script/conf/sentinel1.conf
+	PORT=36379 envsubst < ./script/conf/sentinel_template.conf > ./script/conf/sentinel2.conf
+	PORT=46379 envsubst < ./script/conf/sentinel_template.conf > ./script/conf/sentinel3.conf
+	docker-compose -f script/docker-compose-sentinel.yml up --wait
+
 ## Clean local data
 .PHONY: clean
 clean:
 	docker-compose -f script/docker-compose.yml down
+	docker-compose -f script/docker-compose-sentinel.yml down
 	$(RM) goverage.report $(shell find . -type f -name *.out)

go.mod

@@ -3,8 +3,8 @@ module github.com/kvtools/redis
 go 1.19
 
 require (
-	github.com/go-redis/redis/v8 v8.11.5
 	github.com/kvtools/valkeyrie v1.0.0
+	github.com/redis/go-redis/v9 v9.2.1
 	github.com/stretchr/testify v1.8.4
 )
 

go.sum

@@ -1,31 +1,24 @@
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
-github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kvtools/valkeyrie v1.0.0 h1:LAITop2wPoYCMitR24GZZsW0b57hmI+ePD18VRTtOf0=
 github.com/kvtools/valkeyrie v1.0.0/go.mod h1:bDi/OdhJCSbGPMsCgUQl881yuEweKCSItAtTBI+ZjpU=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
+github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

redis.go

@@ -10,9 +10,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-redis/redis/v8"
 	"github.com/kvtools/valkeyrie"
 	"github.com/kvtools/valkeyrie/store"
+	"github.com/redis/go-redis/v9"
 )
 
 // StoreName the name of the store.
@@ -32,6 +32,14 @@ var (
 	// by sending a signal to the stop chan,
 	// this is used to verify if the operation succeeded.
 	ErrAbortTryLock = errors.New("redis: lock operation aborted")
+
+	// ErrMasterSetMustBeProvided is thrown when Redis Sentinel is enabled
+	// and the MasterName option is undefined.
+	ErrMasterSetMustBeProvided = errors.New("master set name must be provided")
+
+	// ErrInvalidRoutesOptions is thrown when Redis Sentinel is enabled
+	// with RouteByLatency & RouteRandomly options without the ClusterClient.
+	ErrInvalidRoutesOptions = errors.New("RouteByLatency and RouteRandomly options are only allowed with the ClusterClient")
 )
 
 // registers Redis to Valkeyrie.
@@ -45,6 +53,32 @@ type Config struct {
 	Username string
 	Password string
 	DB       int
+	Sentinel *Sentinel
+}
+
+// Sentinel holds the Redis Sentinel configuration.
+type Sentinel struct {
+	MasterName string
+	Username   string
+	Password   string
+
+	// ClusterClient indicates whether to use the NewFailoverClusterClient to build the client.
+	ClusterClient bool
+
+	// Allows routing read-only commands to the closest master or replica node.
+	// This option only works with NewFailoverClusterClient.
+	RouteByLatency bool
+
+	// Allows routing read-only commands to the random master or replica node.
+	// This option only works with NewFailoverClusterClient.
+	RouteRandomly bool
+
+	// Route all commands to replica read-only nodes.
+	ReplicaOnly bool
+
+	// Use replicas disconnected with master when cannot get connected replicas
+	// Now, this option only works in RandomReplicaAddr function.
+	UseDisconnectedReplicas bool
 }
 
 func newStore(ctx context.Context, endpoints []string, options valkeyrie.Config) (store.Store, error) {
@@ -58,7 +92,7 @@ func newStore(ctx context.Context, endpoints []string, options valkeyrie.Config)
 
 // Store implements the store.Store interface.
 type Store struct {
-	client *redis.Client
+	client redis.UniversalClient
 	script *redis.Script
 	codec  Codec
 }
@@ -70,14 +104,54 @@ func New(ctx context.Context, endpoints []string, options *Config) (*Store, erro
 
 // NewWithCodec creates a new Redis client with codec config.
 func NewWithCodec(ctx context.Context, endpoints []string, options *Config, codec Codec) (*Store, error) {
-	if len(endpoints) > 1 {
-		return nil, ErrMultipleEndpointsUnsupported
+	client, err := newClient(endpoints, options)
+	if err != nil {
+		return nil, err
 	}
 
-	return newRedis(ctx, endpoints, options, codec), nil
+	return makeStore(ctx, client, codec), nil
 }
 
-func newRedis(ctx context.Context, endpoints []string, options *Config, codec Codec) *Store {
+func newClient(endpoints []string, options *Config) (redis.UniversalClient, error) {
+	if options != nil && options.Sentinel != nil {
+		if options.Sentinel.MasterName == "" {
+			return nil, ErrMasterSetMustBeProvided
+		}
+
+		if !options.Sentinel.ClusterClient && (options.Sentinel.RouteByLatency || options.Sentinel.RouteRandomly) {
+			return nil, ErrInvalidRoutesOptions
+		}
+
+		cfg := &redis.FailoverOptions{
+			SentinelAddrs:           endpoints,
+			SentinelUsername:        options.Sentinel.Username,
+			SentinelPassword:        options.Sentinel.Password,
+			MasterName:              options.Sentinel.MasterName,
+			RouteByLatency:          options.Sentinel.RouteByLatency,
+			RouteRandomly:           options.Sentinel.RouteRandomly,
+			ReplicaOnly:             options.Sentinel.ReplicaOnly,
+			UseDisconnectedReplicas: options.Sentinel.UseDisconnectedReplicas,
+			Username:                options.Username,
+			Password:                options.Password,
+			DB:                      options.DB,
+			DialTimeout:             5 * time.Second,
+			ReadTimeout:             30 * time.Second,
+			WriteTimeout:            30 * time.Second,
+			ContextTimeoutEnabled:   true,
+			TLSConfig:               options.TLS,
+		}
+
+		if options.Sentinel.ClusterClient {
+			return redis.NewFailoverClusterClient(cfg), nil
+		}
+
+		return redis.NewFailoverClient(cfg), nil
+	}
+
+	if len(endpoints) > 1 {
+		return nil, ErrMultipleEndpointsUnsupported
+	}
+
 	opt := &redis.Options{
 		Addr:         endpoints[0],
 		DialTimeout:  5 * time.Second,
@@ -93,8 +167,10 @@ func newRedis(ctx context.Context, endpoints []string, options *Config, codec Co
 	}
 
 	// TODO: use *redis.ClusterClient if we support multiple endpoints.
-	client := redis.NewClient(opt)
+	return redis.NewClient(opt), nil
+}
 
+func makeStore(ctx context.Context, client redis.UniversalClient, codec Codec) *Store {
 	// Listen to Keyspace events.
 	client.ConfigSet(ctx, "notify-keyspace-events", "KEA")
 
@@ -513,7 +589,7 @@ type subscribe struct {
 	closeCh chan struct{}
 }
 
-func newSubscribe(ctx context.Context, client *redis.Client, regex string) *subscribe {
+func newSubscribe(ctx context.Context, client redis.UniversalClient, regex string) *subscribe {
 	return &subscribe{
 		pubsub:  client.PSubscribe(ctx, regex),
 		closeCh: make(chan struct{}),

redis_test.go

@@ -14,15 +14,16 @@ import (
 
 const testTimeout = 60 * time.Second
 
-const client = "localhost:6379"
+const testAddress = "localhost:6379"
 
-func makeRedisClient(t *testing.T) store.Store {
+func makeRedisClient(t *testing.T, endpoints []string, config *Config) store.Store {
 	t.Helper()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	kv := newRedis(ctx, []string{client}, nil, nil)
+	kv, err := NewWithCodec(ctx, endpoints, config, nil)
+	require.NoError(t, err)
 
 	// NOTE: please turn on redis's notification
 	// before you using watch/watchtree/lock related features.
@@ -35,17 +36,57 @@ func TestRegister(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
 	defer cancel()
 
-	kv, err := valkeyrie.NewStore(ctx, StoreName, []string{client}, nil)
+	kv, err := valkeyrie.NewStore(ctx, StoreName, []string{testAddress}, nil)
 	require.NoError(t, err)
 	assert.NotNil(t, kv)
 
 	assert.IsTypef(t, kv, new(Store), "Error registering and initializing Redis")
 }
 
 func TestRedisStore(t *testing.T) {
-	kv := makeRedisClient(t)
-	lockTTL := makeRedisClient(t)
-	kvTTL := makeRedisClient(t)
+	kv := makeRedisClient(t, []string{testAddress}, nil)
+	lockTTL := makeRedisClient(t, []string{testAddress}, nil)
+	kvTTL := makeRedisClient(t, []string{testAddress}, nil)
+
+	t.Cleanup(func() {
+		testsuite.RunCleanup(t, kv)
+	})
+
+	testsuite.RunTestCommon(t, kv)
+	testsuite.RunTestAtomic(t, kv)
+	testsuite.RunTestWatch(t, kv)
+	testsuite.RunTestLock(t, kv)
+	testsuite.RunTestLockTTL(t, kv, lockTTL)
+	testsuite.RunTestTTL(t, kv, kvTTL)
+}
+
+func TestRedisSentinelStore(t *testing.T) {
+	endpoints := []string{"localhost:26379", "localhost:36379", "localhost:46379"}
+	config := &Config{Sentinel: &Sentinel{MasterName: "mymaster"}}
+
+	kv := makeRedisClient(t, endpoints, config)
+	lockTTL := makeRedisClient(t, endpoints, config)
+	kvTTL := makeRedisClient(t, endpoints, config)
+
+	t.Cleanup(func() {
+		testsuite.RunCleanup(t, kv)
+	})
+
+	testsuite.RunTestCommon(t, kv)
+	testsuite.RunTestAtomic(t, kv)
+	testsuite.RunTestWatch(t, kv)
+	testsuite.RunTestLock(t, kv)
+	testsuite.RunTestLockTTL(t, kv, lockTTL)
+	testsuite.RunTestTTL(t, kv, kvTTL)
+}
+
+func TestRedisSentinelStore_withClientCluster(t *testing.T) {
+	endpoints := []string{"localhost:26379", "localhost:36379", "localhost:46379"}
+	config := &Config{Sentinel: &Sentinel{MasterName: "mymaster", ClusterClient: true}}
+
+	kv := makeRedisClient(t, endpoints, config)
+	lockTTL := makeRedisClient(t, endpoints, config)
+	kvTTL := makeRedisClient(t, endpoints, config)
 
 	t.Cleanup(func() {
 		testsuite.RunCleanup(t, kv)

script/conf/sentinel_template.conf

@@ -0,0 +1,5 @@
+port ${PORT}
+dir "/tmp"
+sentinel resolve-hostnames yes
+sentinel monitor mymaster master 6380 2
+sentinel deny-scripts-reconfig yes

script/docker-compose-sentinel.yml

@@ -0,0 +1,60 @@
+version: "3.9"
+
+# when running test local, you can specify the image version using the env var REDIS_VERSION.
+# Example: REDIS_VERSION=7.0.5 make
+
+services:
+  master:
+    image: redis:${REDIS_VERSION:-7.2.1}
+    container_name: redis-master
+    command: redis-server --port 6380
+    ports:
+      - 6380:6380
+    healthcheck:
+      test: redis-cli -p 6380 ping
+  node1:
+    image: redis:${REDIS_VERSION:-7.2.1}
+    container_name: redis-node-1
+    ports:
+      - 6381:6381
+    command:  redis-server --port 6381 --slaveof redis-master 6380
+    healthcheck:
+      test: redis-cli -p 6381 ping
+  node2:
+    image: redis:${REDIS_VERSION:-7.2.1}
+    container_name: redis-node-2
+    ports:
+      - 6382:6382
+    command: redis-server --port 6382 --slaveof redis-master 6380
+    healthcheck:
+      test: redis-cli -p 6382 ping
+  sentinel1:
+    image: redis:${REDIS_VERSION:-7.2.1}
+    container_name: redis-sentinel-1
+    ports:
+      - 26379:26379
+    command: redis-sentinel /usr/local/etc/redis/conf/sentinel1.conf
+    healthcheck:
+      test: redis-cli -p 26379 ping
+    volumes:
+      - ./conf:/usr/local/etc/redis/conf
+  sentinel2:
+    image: redis:${REDIS_VERSION:-7.2.1}
+    container_name: redis-sentinel-2
+    ports:
+      - 36379:26379
+    command: redis-sentinel /usr/local/etc/redis/conf/sentinel2.conf
+    healthcheck:
+      test: redis-cli -p 36379 ping
+    volumes:
+      - ./conf:/usr/local/etc/redis/conf
+  sentinel3:
+    image: redis:${REDIS_VERSION:-7.2.1}
+    container_name: redis-sentinel-3
+    ports:
+      - 46379:26379
+    command: redis-sentinel /usr/local/etc/redis/conf/sentinel3.conf
+    healthcheck:
+      test: redis-cli -p 46379 ping
+    volumes:
+      - ./conf:/usr/local/etc/redis/conf

script/docker-compose.yml

@@ -5,7 +5,7 @@ version: "3.9"
 
 services:
   redis:
-    image: redis:${REDIS_VERSION:-4.0.10}
+    image: redis:${REDIS_VERSION:-7.2.1}
     container_name: redis
     healthcheck:
       test: redis-cli ping