From e90db5e4b75a3ddde0e3931378fc325e09b92738 Mon Sep 17 00:00:00 2001 From: fishu Date: Thu, 12 Mar 2020 18:49:54 +0800 Subject: [PATCH 1/9] supprot backup and restore from pvc --- go.mod | 2 +- go.sum | 2 + hack/docker/redis-tools/redis-tools.sh | 12 +- pkg/apis/redis/v1alpha1/default.go | 6 +- pkg/controller/manager/ensurer.go | 2 +- .../redisclusterbackup/sync_handler.go | 22 +-- pkg/osm/ceph.go | 57 -------- pkg/osm/osm.go | 138 ++++-------------- pkg/osm/rclone.go | 92 ++++++++++++ pkg/resources/statefulsets/statefulset.go | 49 +++++-- version/version.go | 2 +- 11 files changed, 177 insertions(+), 207 deletions(-) delete mode 100644 pkg/osm/ceph.go create mode 100644 pkg/osm/rclone.go diff --git a/go.mod b/go.mod index db9825f1a..f6725a479 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ require ( github.com/appscode/go v0.0.0-20191006073906-e3d193d493fc github.com/appscode/osm v0.12.0 github.com/aws/aws-sdk-go v1.20.20 - github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-logr/logr v0.1.0 github.com/go-openapi/spec v0.19.2 github.com/mediocregopher/radix.v2 v0.0.0-20181115013041-b67df6e626f9 @@ -19,6 +18,7 @@ require ( k8s.io/client-go v12.0.0+incompatible k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d k8s.io/kubernetes v1.16.2 + kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6 kmodules.xyz/objectstore-api v0.0.0-20191014210450-ac380fa650a3 sigs.k8s.io/controller-runtime v0.4.0 ) diff --git a/go.sum b/go.sum index 546569928..b013f8a32 100644 --- a/go.sum +++ b/go.sum @@ -1008,6 +1008,8 @@ k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl k8s.io/utils v0.0.0-20190801114015-581e00157fb1 h1:+ySTxfHnfzZb9ys375PXNlLhkJPLKgHajBU0N62BDvE= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= kmodules.xyz/client-go v0.0.0-20190715080709-7162a6c90b04/go.mod h1:3h1/ZglH6fV2uv3p9AB1JnHUknhWniShJ7fBJfSf7Js= +kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6 h1:hFv3DzanQJ/bjgahqosmthGLkVgMB2KuQIsltOA02t0= +kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY= kmodules.xyz/objectstore-api v0.0.0-20191014210450-ac380fa650a3 h1:64QSexLk/Dio4+L8Ge1tb4c44aBiwmUwTNP2kCu6YQU= kmodules.xyz/objectstore-api v0.0.0-20191014210450-ac380fa650a3/go.mod h1:mT7lEi2IehAi64DomCPMPtlsWXOD5Fr3/mPqLIzU7T8= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= diff --git a/hack/docker/redis-tools/redis-tools.sh b/hack/docker/redis-tools/redis-tools.sh index 77729f055..b25b9f637 100644 --- a/hack/docker/redis-tools/redis-tools.sh +++ b/hack/docker/redis-tools/redis-tools.sh @@ -14,6 +14,7 @@ show_help() { echo " --host=HOST database host" echo " --user=USERNAME database username" echo " --bucket=BUCKET name of bucket" + echo " --location=LOCATION location of backend (:)" echo " --folder=FOLDER name of folder in bucket" echo " --snapshot=SNAPSHOT name of snapshot" } @@ -25,6 +26,7 @@ REDIS_PORT=${REDIS_PORT:-6379} REDIS_USER=${REDIS_USER:-} REDIS_PASSWORD=${REDIS_PASSWORD:-} REDIS_BUCKET=${REDIS_BUCKET:-} +REDIS_LOCATION=${REDIS_LOCATION:-} REDIS_FOLDER=${REDIS_FOLDER:-} REDIS_SNAPSHOT=${REDIS_SNAPSHOT:-} REDIS_DATA_DIR=${REDIS_DATA_DIR:-/data} @@ -56,6 +58,10 @@ while test $# -gt 0; do export REDIS_BUCKET=$(echo $1 | sed -e 's/^[^=]*=//g') shift ;; + --location*) + export REDIS_LOCATION=$(echo $1 | sed -e 's/^[^=]*=//g') + shift + ;; --folder*) export REDIS_FOLDER=$(echo $1 | sed -e 's/^[^=]*=//g') shift @@ -108,7 +114,7 @@ case "$op" in ls -lh "$SOURCE_DIR" echo "Uploading dump file to the backend......." echo "From $SOURCE_DIR" - osm --config "$OSM_CONFIG_FILE" copy "$SOURCE_DIR" ceph:"$REDIS_BUCKET"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" -v + osm --config "$OSM_CONFIG_FILE" copy "$SOURCE_DIR" "$REDIS_LOCATION"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" -v echo "Backup successful" ;; @@ -120,9 +126,9 @@ case "$op" in fi index=$(echo "${POD_NAME}" | awk -F- '{print $(NF-1)}') REDIS_SNAPSHOT=${REDIS_SNAPSHOT}-${index} - SOURCE_SNAPSHOT="$REDIS_BUCKET"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" + SOURCE_SNAPSHOT="$REDIS_LOCATION"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" echo "From $SOURCE_SNAPSHOT" - osm --config "$OSM_CONFIG_FILE" sync ceph:"$SOURCE_SNAPSHOT" "$REDIS_DATA_DIR" -v + osm --config "$OSM_CONFIG_FILE" sync "$SOURCE_SNAPSHOT" "$REDIS_DATA_DIR" -v echo "Recovery successful" ;; diff --git a/pkg/apis/redis/v1alpha1/default.go b/pkg/apis/redis/v1alpha1/default.go index 97337bda9..c3c32228f 100644 --- a/pkg/apis/redis/v1alpha1/default.go +++ b/pkg/apis/redis/v1alpha1/default.go @@ -114,7 +114,7 @@ func (in *RedisClusterBackup) Validate() error { return nil } -func (in *RedisClusterBackup) Location() (string, error) { +func (in *RedisClusterBackup) RemotePath() (string, error) { spec := in.Spec.Backend timePrefix := in.Status.StartTime.Format("20060102150405") if spec.S3 != nil { @@ -138,3 +138,7 @@ func (in *RedisClusterBackup) OSMSecretName() string { func (in *RedisClusterBackup) JobName() string { return fmt.Sprintf("redisbackup-%v", in.Name) } + +func (in *RedisClusterBackup) IsRefLocalPVC() bool { + return in.Spec.Local != nil && in.Spec.Local.PersistentVolumeClaim != nil +} diff --git a/pkg/controller/manager/ensurer.go b/pkg/controller/manager/ensurer.go index d2aa94560..72e422a1e 100644 --- a/pkg/controller/manager/ensurer.go +++ b/pkg/controller/manager/ensurer.go @@ -236,7 +236,7 @@ func (r *realEnsureResource) EnsureRedisOSMSecret(cluster *redisv1alpha1.Distrib return nil } backup := cluster.Status.Restore.Backup - secret, err := osm.NewCephSecret(r.client, backup.OSMSecretName(), cluster.Namespace, backup.Spec.Backend) + secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), cluster.Namespace, backup.Spec.Backend) if err != nil { return err } diff --git a/pkg/controller/redisclusterbackup/sync_handler.go b/pkg/controller/redisclusterbackup/sync_handler.go index 3add486c3..1a6bc159c 100644 --- a/pkg/controller/redisclusterbackup/sync_handler.go +++ b/pkg/controller/redisclusterbackup/sync_handler.go @@ -50,20 +50,6 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi return nil } - //if backup.Labels == nil { - // backup.Labels = make(map[string]string) - //} - //backup.Labels[redisv1alpha1.LabelClusterName] = backup.Spec.RedisClusterName - //if err := r.crController.UpdateCR(backup); err != nil { - // r.recorder.Event( - // backup, - // corev1.EventTypeWarning, - // event.BackupError, - // err.Error(), - // ) - // return err - //} - if err := r.ValidateBackup(backup); err != nil { if k8sutil.IsRequestRetryable(err) { return err @@ -103,7 +89,7 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi return err } - secret, err := osm.NewCephSecret(r.client, backup.OSMSecretName(), backup.Namespace, backup.Spec.Backend) + secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), backup.Namespace, backup.Spec.Backend) if err != nil { msg := fmt.Sprintf("Failed to generate osm secret. Reason: %v", err) r.markAsFailedBackup(backup, msg) @@ -301,7 +287,7 @@ func (r *ReconcileRedisClusterBackup) getBackupJob(reqLogger logr.Logger, backup func (r *ReconcileRedisClusterBackup) backupContainers(backup *redisv1alpha1.RedisClusterBackup, cluster *redisv1alpha1.DistributedRedisCluster, reqLogger logr.Logger) ([]corev1.Container, error) { backupSpec := backup.Spec.Backend - bucket, err := backupSpec.Container() + location, err := backupSpec.Location() if err != nil { return nil, err } @@ -313,7 +299,7 @@ func (r *ReconcileRedisClusterBackup) backupContainers(backup *redisv1alpha1.Red if i == masterNum { break } - folderName, err := backup.Location() + folderName, err := backup.RemotePath() if err != nil { r.recorder.Event( backup, @@ -331,7 +317,7 @@ func (r *ReconcileRedisClusterBackup) backupContainers(backup *redisv1alpha1.Red Args: []string{ redisv1alpha1.JobTypeBackup, fmt.Sprintf(`--data-dir=%s`, redisv1alpha1.BackupDumpDir), - fmt.Sprintf(`--bucket=%s`, bucket), + fmt.Sprintf(`--location=%s`, location), fmt.Sprintf(`--host=%s`, node.IP), fmt.Sprintf(`--folder=%s`, folderName), fmt.Sprintf(`--snapshot=%s-%d`, backup.Name, i), diff --git a/pkg/osm/ceph.go b/pkg/osm/ceph.go deleted file mode 100644 index d2e332529..000000000 --- a/pkg/osm/ceph.go +++ /dev/null @@ -1,57 +0,0 @@ -package osm - -import ( - "context" - "fmt" - - core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ktypes "k8s.io/apimachinery/pkg/types" - api "kmodules.xyz/objectstore-api/api/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func NewCephSecret(kc client.Client, name, namespace string, spec api.Backend) (*core.Secret, error) { - if spec.S3 == nil { - return nil, fmt.Errorf("only suport ceph s3") - } - - config := make(map[string][]byte) - if spec.StorageSecretName != "" { - secret := &core.Secret{} - err := kc.Get(context.TODO(), ktypes.NamespacedName{ - Name: spec.StorageSecretName, - Namespace: namespace, - }, secret) - if err != nil { - return nil, err - } - config = secret.Data - } - - keyID := config[api.AWS_ACCESS_KEY_ID] - key := config[api.AWS_SECRET_ACCESS_KEY] - - osmBytes := fmt.Sprintf(`[ceph] -type = s3 -provider = Ceph -env_auth = false -access_key_id = %s -secret_access_key = %s -region = -endpoint = %s -location_constraint = -acl = -server_side_encryption = -storage_class =`, keyID, key, spec.S3.Endpoint) - out := &core.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: map[string][]byte{ - "config": []byte(osmBytes), - }, - } - return out, nil -} diff --git a/pkg/osm/osm.go b/pkg/osm/osm.go index 920e5c576..f4a2b9b73 100644 --- a/pkg/osm/osm.go +++ b/pkg/osm/osm.go @@ -2,13 +2,15 @@ package osm import ( "context" - "io/ioutil" "net/url" - "os" - "path/filepath" "strconv" "strings" + awsconst "kmodules.xyz/constants/aws" + azconst "kmodules.xyz/constants/azure" + googconst "kmodules.xyz/constants/google" + osconst "kmodules.xyz/constants/openstack" + stringz "github.com/appscode/go/strings" "github.com/appscode/go/types" otx "github.com/appscode/osm/context" @@ -18,7 +20,6 @@ import ( "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" _s3 "github.com/aws/aws-sdk-go/service/s3" - "github.com/ghodss/yaml" "github.com/pkg/errors" "gomodules.xyz/stow" "gomodules.xyz/stow/azure" @@ -27,7 +28,6 @@ import ( "gomodules.xyz/stow/s3" "gomodules.xyz/stow/swift" core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ktypes "k8s.io/apimachinery/pkg/types" api "kmodules.xyz/objectstore-api/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,92 +38,6 @@ const ( CaCertFileName = "ca.crt" ) -// NewOSMSecret creates a secret that contains the config file of OSM. -// So, generally, if this secret is mounted in `etc/osm`, -// the tree of `/etc/osm` directory will be similar to, -// -// /etc/osm -// └── config -// -// However, if the EndPoint is `S3 Minio Server`, then the secret will contain two file, -// `config` and `ca.crt`. So, the tree of the file path will look as, -// -// /etc/osm -// ├── ca.crt -// └── config - -func NewOSMSecret(kc client.Client, name, namespace string, spec api.Backend) (*core.Secret, error) { - osmCtx, err := NewOSMContext(kc, spec, namespace) - if err != nil { - return nil, err - } - cacertData, certDataFound := osmCtx.Config[s3.ConfigCACertData] - if certDataFound { - // assume that CA cert file is mounted at SecretMountPath directory - osmCtx.Config[s3.ConfigCACertFile] = filepath.Join(SecretMountPath, CaCertFileName) - delete(osmCtx.Config, s3.ConfigCACertData) - } - - osmCfg := &otx.OSMConfig{ - CurrentContext: osmCtx.Name, - Contexts: []*otx.Context{osmCtx}, - } - osmBytes, err := yaml.Marshal(osmCfg) - if err != nil { - return nil, err - } - out := &core.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: map[string][]byte{ - "config": osmBytes, - }, - } - if certDataFound { - // inject ca cert data as a file into the osm secret so that CaCertFileName exists. - out.Data[CaCertFileName] = []byte(cacertData) - } - return out, nil -} - -func WriteOSMConfig(kc client.Client, namespace string, spec api.Backend, filename string) error { - osmCtx, err := NewOSMContext(kc, spec, namespace) - if err != nil { - return err - } - - dir := filepath.Dir(filename) - - err = os.MkdirAll(dir, 0755) - if err != nil { - return err - } - - cacertData, certDataFound := osmCtx.Config[s3.ConfigCACertData] - if certDataFound { - // assume that CA cert file is mounted at SecretMountPath directory - f2 := filepath.Join(dir, CaCertFileName) - err = ioutil.WriteFile(f2, []byte(cacertData), 0644) - if err != nil { - return err - } - osmCtx.Config[s3.ConfigCACertFile] = f2 - delete(osmCtx.Config, s3.ConfigCACertData) - } - - osmCfg := &otx.OSMConfig{ - CurrentContext: osmCtx.Name, - Contexts: []*otx.Context{osmCtx}, - } - osmBytes, err := yaml.Marshal(osmCfg) - if err != nil { - return err - } - return ioutil.WriteFile(filename, osmBytes, 0644) -} - func CheckBucketAccess(client client.Client, spec api.Backend, namespace string) error { cfg, err := NewOSMContext(client, spec, namespace) if err != nil { @@ -167,8 +81,8 @@ func NewOSMContext(client client.Client, spec api.Backend, namespace string) (*o if spec.S3 != nil { nc.Provider = s3.Kind - keyID, foundKeyID := config[api.AWS_ACCESS_KEY_ID] - key, foundKey := config[api.AWS_SECRET_ACCESS_KEY] + keyID, foundKeyID := config[awsconst.AWS_ACCESS_KEY_ID] + key, foundKey := config[awsconst.AWS_SECRET_ACCESS_KEY] if foundKey && foundKeyID { nc.Config[s3.ConfigAccessKeyID] = string(keyID) nc.Config[s3.ConfigSecretKey] = string(key) @@ -226,7 +140,7 @@ func NewOSMContext(client client.Client, spec api.Backend, namespace string) (*o } nc.Config[s3.ConfigDisableSSL] = strconv.FormatBool(u.Scheme == "http") - cacertData, ok := config[api.CA_CERT_DATA] + cacertData, ok := config[awsconst.CA_CERT_DATA] if ok && u.Scheme == "https" { nc.Config[s3.ConfigCACertData] = string(cacertData) } @@ -234,13 +148,13 @@ func NewOSMContext(client client.Client, spec api.Backend, namespace string) (*o return nc, nil } else if spec.GCS != nil { nc.Provider = gcs.Kind - nc.Config[gcs.ConfigProjectId] = string(config[api.GOOGLE_PROJECT_ID]) - nc.Config[gcs.ConfigJSON] = string(config[api.GOOGLE_SERVICE_ACCOUNT_JSON_KEY]) + nc.Config[gcs.ConfigProjectId] = string(config[googconst.GOOGLE_PROJECT_ID]) + nc.Config[gcs.ConfigJSON] = string(config[googconst.GOOGLE_SERVICE_ACCOUNT_JSON_KEY]) return nc, nil } else if spec.Azure != nil { nc.Provider = azure.Kind - nc.Config[azure.ConfigAccount] = string(config[api.AZURE_ACCOUNT_NAME]) - nc.Config[azure.ConfigKey] = string(config[api.AZURE_ACCOUNT_KEY]) + nc.Config[azure.ConfigAccount] = string(config[azconst.AZURE_ACCOUNT_NAME]) + nc.Config[azure.ConfigKey] = string(config[azconst.AZURE_ACCOUNT_KEY]) return nc, nil } else if spec.Local != nil { nc.Provider = local.Kind @@ -254,28 +168,28 @@ func NewOSMContext(client client.Client, spec api.Backend, namespace string) (*o secretKey string }{ // v2/v3 specific - {swift.ConfigUsername, api.OS_USERNAME}, - {swift.ConfigKey, api.OS_PASSWORD}, - {swift.ConfigRegion, api.OS_REGION_NAME}, - {swift.ConfigTenantAuthURL, api.OS_AUTH_URL}, + {swift.ConfigUsername, osconst.OS_USERNAME}, + {swift.ConfigKey, osconst.OS_PASSWORD}, + {swift.ConfigRegion, osconst.OS_REGION_NAME}, + {swift.ConfigTenantAuthURL, osconst.OS_AUTH_URL}, // v3 specific - {swift.ConfigDomain, api.OS_USER_DOMAIN_NAME}, - {swift.ConfigTenantName, api.OS_PROJECT_NAME}, - {swift.ConfigTenantDomain, api.OS_PROJECT_DOMAIN_NAME}, + {swift.ConfigDomain, osconst.OS_USER_DOMAIN_NAME}, + {swift.ConfigTenantName, osconst.OS_PROJECT_NAME}, + {swift.ConfigTenantDomain, osconst.OS_PROJECT_DOMAIN_NAME}, // v2 specific - {swift.ConfigTenantId, api.OS_TENANT_ID}, - {swift.ConfigTenantName, api.OS_TENANT_NAME}, + {swift.ConfigTenantId, osconst.OS_TENANT_ID}, + {swift.ConfigTenantName, osconst.OS_TENANT_NAME}, // v1 specific - {swift.ConfigTenantAuthURL, api.ST_AUTH}, - {swift.ConfigUsername, api.ST_USER}, - {swift.ConfigKey, api.ST_KEY}, + {swift.ConfigTenantAuthURL, osconst.ST_AUTH}, + {swift.ConfigUsername, osconst.ST_USER}, + {swift.ConfigKey, osconst.ST_KEY}, // Manual authentication - {swift.ConfigStorageURL, api.OS_STORAGE_URL}, - {swift.ConfigAuthToken, api.OS_AUTH_TOKEN}, + {swift.ConfigStorageURL, osconst.OS_STORAGE_URL}, + {swift.ConfigAuthToken, osconst.OS_AUTH_TOKEN}, } { if _, exists := nc.Config.Config(val.stowKey); !exists { nc.Config[val.stowKey] = string(config[val.secretKey]) diff --git a/pkg/osm/rclone.go b/pkg/osm/rclone.go new file mode 100644 index 000000000..2d1106502 --- /dev/null +++ b/pkg/osm/rclone.go @@ -0,0 +1,92 @@ +package osm + +import ( + "context" + "fmt" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ktypes "k8s.io/apimachinery/pkg/types" + awsconst "kmodules.xyz/constants/aws" + api "kmodules.xyz/objectstore-api/api/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// NewRcloneSecret creates a secret that contains the config file of Rclone. +// So, generally, if this secret is mounted in `etc/osm`, +// the tree of `/etc/osm` directory will be similar to, +// +// /etc/osm +// └── config +func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend) (*core.Secret, error) { + osmCtx, err := newContext(kc, spec, namespace) + if err != nil { + return nil, err + } + + osmBytes := []byte(osmCtx) + + out := &core.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + "config": osmBytes, + }, + } + return out, nil +} + +func newContext(kc client.Client, spec api.Backend, namespace string) (string, error) { + config := make(map[string][]byte) + if spec.StorageSecretName != "" { + secret := &core.Secret{} + err := kc.Get(context.TODO(), ktypes.NamespacedName{ + Name: spec.StorageSecretName, + Namespace: namespace, + }, secret) + if err != nil { + return "", err + } + config = secret.Data + } + provider, err := spec.Provider() + if err != nil { + return "", err + } + + if spec.S3 != nil { + return cephContext(config, provider, spec), nil + } + if spec.Local != nil { + return localContext(provider), nil + } + + return "", fmt.Errorf("no storage provider is configured") +} + +func cephContext(config map[string][]byte, provider string, spec api.Backend) string { + keyID := config[awsconst.AWS_ACCESS_KEY_ID] + key := config[awsconst.AWS_SECRET_ACCESS_KEY] + + return fmt.Sprintf(`[%s] +type = s3 +provider = Ceph +env_auth = false +access_key_id = %s +secret_access_key = %s +region = +endpoint = %s +location_constraint = +acl = +server_side_encryption = +storage_class = +`, provider, keyID, key, spec.S3.Endpoint) +} + +func localContext(provider string) string { + return fmt.Sprintf(`[%s] +type = local +`, provider) +} diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 938ef9d67..d90ec2e13 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -20,9 +20,10 @@ import ( var log = logf.Log.WithName("resource_statefulset") const ( - redisStorageVolumeName = "redis-data" - redisServerName = "redis" - hostnameTopologyKey = "kubernetes.io/hostname" + redisStorageVolumeName = "redis-data" + redisRestoreLocalVolumeName = "redis-local" + redisServerName = "redis" + hostnameTopologyKey = "kubernetes.io/hostname" graceTime = 30 @@ -299,11 +300,11 @@ func redisExporterContainer(cluster *redisv1alpha1.DistributedRedisCluster, pass func redisInitContainer(cluster *redisv1alpha1.DistributedRedisCluster, password *corev1.EnvVar) (corev1.Container, error) { backup := cluster.Status.Restore.Backup backupSpec := backup.Spec.Backend - bucket, err := backupSpec.Container() + location, err := backupSpec.Location() if err != nil { return corev1.Container{}, err } - folderName, err := backup.Location() + folderName, err := backup.RemotePath() if err != nil { return corev1.Container{}, err } @@ -315,7 +316,7 @@ func redisInitContainer(cluster *redisv1alpha1.DistributedRedisCluster, password Args: []string{ redisv1alpha1.JobTypeRestore, fmt.Sprintf(`--data-dir=%s`, redisv1alpha1.BackupDumpDir), - fmt.Sprintf(`--bucket=%s`, bucket), + fmt.Sprintf(`--location=%s`, location), fmt.Sprintf(`--folder=%s`, folderName), fmt.Sprintf(`--snapshot=%s`, backup.Name), "--", @@ -353,15 +354,27 @@ func redisInitContainer(cluster *redisv1alpha1.DistributedRedisCluster, password }, }, } + + if backup.IsRefLocalPVC() { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: redisRestoreLocalVolumeName, + MountPath: backup.Spec.Backend.Local.MountPath, + SubPath: backup.Spec.Backend.Local.SubPath, + ReadOnly: true, + }) + } + if password != nil { container.Env = append(container.Env, *password) } + if backup.Spec.PodSpec != nil { container.Resources = backup.Spec.PodSpec.Resources container.LivenessProbe = backup.Spec.PodSpec.LivenessProbe container.ReadinessProbe = backup.Spec.PodSpec.ReadinessProbe container.Lifecycle = backup.Spec.PodSpec.Lifecycle } + return container, nil } @@ -418,14 +431,24 @@ func redisVolumes(cluster *redisv1alpha1.DistributedRedisCluster) []corev1.Volum if dataVolume != nil { volumes = append(volumes, *dataVolume) } - if cluster.IsRestoreFromBackup() && cluster.Status.Restore.Backup != nil { - volumes = append(volumes, corev1.Volume{ - Name: "osmconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Status.Restore.Backup.OSMSecretName(), - }, + + if !cluster.IsRestoreFromBackup() || cluster.Status.Restore.Backup == nil { + return volumes + } + + volumes = append(volumes, corev1.Volume{ + Name: "osmconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Status.Restore.Backup.OSMSecretName(), }, + }, + }) + + if cluster.Status.Restore.Backup.IsRefLocalPVC() { + volumes = append(volumes, corev1.Volume{ + Name: redisRestoreLocalVolumeName, + VolumeSource: cluster.Status.Restore.Backup.Spec.Local.VolumeSource, }) } diff --git a/version/version.go b/version/version.go index 5c6104329..b50bcbf7e 100644 --- a/version/version.go +++ b/version/version.go @@ -2,7 +2,7 @@ package version var ( // Version represents the software version of the Redis Cluster Operator - Version = "0.2.0" + Version = "0.2.2" // GitSHA represents the Git commit hash in short format GitSHA = "" ) From 22725b3528f4ce9a3b1443ee1ecd7054df00548c Mon Sep 17 00:00:00 2001 From: fishu Date: Mon, 16 Mar 2020 16:03:48 +0800 Subject: [PATCH 2/9] remove restore initContainer and volume when restore has been successfully --- pkg/resources/statefulsets/statefulset.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index d90ec2e13..720ddea87 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -85,7 +85,7 @@ func NewStatefulSetForCR(cluster *redisv1alpha1.DistributedRedisCluster, ssName, if spec.Monitor != nil { ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, redisExporterContainer(cluster, password)) } - if cluster.IsRestoreFromBackup() && cluster.Status.Restore.Backup != nil { + if cluster.IsRestoreFromBackup() && !cluster.IsRestored() && cluster.Status.Restore.Backup != nil { initContainer, err := redisInitContainer(cluster, password) if err != nil { return nil, err @@ -432,7 +432,9 @@ func redisVolumes(cluster *redisv1alpha1.DistributedRedisCluster) []corev1.Volum volumes = append(volumes, *dataVolume) } - if !cluster.IsRestoreFromBackup() || cluster.Status.Restore.Backup == nil { + if !cluster.IsRestoreFromBackup() || + cluster.Status.Restore.Backup == nil || + cluster.IsRestored() { return volumes } From d4c46580de24385f90359b3c9bc1835b3e613ef3 Mon Sep 17 00:00:00 2001 From: fishu Date: Mon, 16 Mar 2020 17:57:27 +0800 Subject: [PATCH 3/9] determine finished Jobs by conditions --- go.sum | 25 +++++++++++++++++++ pkg/controller/redisclusterbackup/helper.go | 10 ++++++++ .../redisclusterbackup_controller.go | 15 +++-------- .../redisclusterbackup/sync_handler.go | 11 +++++--- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/go.sum b/go.sum index b013f8a32..a365d0d01 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= @@ -140,18 +141,23 @@ github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kw github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coredns/corefile-migration v1.0.2/go.mod h1:OFwBp/Wc9dJt5cAZzHWMNhK1r5L0p0jDwIBc6j8NC8E= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/prometheus-operator v0.34.0 h1:TF9qaydNeUamLKs0hniaapa4FBz8U8TIlRRtJX987A4= github.com/coreos/prometheus-operator v0.34.0/go.mod h1:Li6rMllG/hYIyXfMuvUwhyC+hqwJVHdsDdP21hypT1M= @@ -185,6 +191,7 @@ github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20180612054059-a9fbbdc8dd87/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -340,6 +347,7 @@ github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAO github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/cadvisor v0.34.0/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= @@ -380,6 +388,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -387,12 +396,15 @@ github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17/go.mod h1:Fecb github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.8.6/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0= github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-health-probe v0.2.1-0.20181220223928-2bf0a5b182db/go.mod h1:uBKkC2RbarFsvS5jMJHpVhTLvGlGQj9JJwkaePE3FWI= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= @@ -431,6 +443,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/go-ogle-analytics v0.0.0-20161213085824-14b04e0594ef/go.mod h1:PlwhC7q1VSK73InDzdDatVetQrTsQHIbOvcJAZzitY0= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -453,6 +466,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -550,6 +564,7 @@ github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v1.0.0-rc2.0.20190611121236-6cc515888830/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -571,6 +586,7 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776/go.mod h1:3HNVkVOU7vZeFXocWuvtcS0XSFLcf2XUSDHkq9t1jU4= github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw= github.com/otiai10/mint v1.2.4/go.mod h1:d+b7n/0R3tdyUYYylALXpWQ/kTN+QobSq/4SRGBkR3M= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -639,14 +655,17 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -676,6 +695,7 @@ github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLY github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -690,6 +710,7 @@ github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVT github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -698,6 +719,7 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDf github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -932,6 +954,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3/go.mod h1:l0 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -959,6 +982,7 @@ k8s.io/apiextensions-apiserver v0.0.0-20191016113550-5357c4baaf65 h1:kThoiqgMsSw k8s.io/apiextensions-apiserver v0.0.0-20191016113550-5357c4baaf65/go.mod h1:5BINdGqggRXXKnDgpwoJ7PyQH8f+Ypp02fvVNcIFy9s= k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8 h1:Iieh/ZEgT3BWwbLD5qEKcY06jKuPEl6zC7gPSehoLw4= k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= +k8s.io/apiserver v0.0.0-20191016112112-5190913f932d h1:leksCBKKBrPJmW1jV4dZUvwqmVtXpKdzpHsqXfFS094= k8s.io/apiserver v0.0.0-20191016112112-5190913f932d/go.mod h1:7OqfAolfWxUM/jJ/HBLyE+cdaWFBUoo5Q5pHgJVj2ws= k8s.io/autoscaler v0.0.0-20190607113959-1b4f1855cb8e/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= k8s.io/cli-runtime v0.0.0-20191016114015-74ad18325ed5/go.mod h1:sDl6WKSQkDM6zS1u9F49a0VooQ3ycYFBFLqd2jf2Xfo= @@ -967,6 +991,7 @@ k8s.io/client-go v0.0.0-20191016111102-bec269661e48/go.mod h1:hrwktSwYGI4JK+TJA3 k8s.io/cloud-provider v0.0.0-20191016115326-20453efc2458/go.mod h1:O5SO5xcgxrjJV9EC9R/47RuBpbk5YX9URDBlg++FA5o= k8s.io/cluster-bootstrap v0.0.0-20191016115129-c07a134afb42/go.mod h1:MzCL6kLExQuHruGaqibd8cugC8nw8QRxm3+lzR5l8SI= k8s.io/code-generator v0.0.0-20191004115455-8e001e5d1894/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY= +k8s.io/component-base v0.0.0-20191016111319-039242c015a9 h1:2D+G/CCNVdYc0h9D+tX+0SmtcyQmby6uzNityrps1s0= k8s.io/component-base v0.0.0-20191016111319-039242c015a9/go.mod h1:SuWowIgd/dtU/m/iv8OD9eOxp3QZBBhTIiWMsBQvKjI= k8s.io/cri-api v0.0.0-20190828162817-608eb1dad4ac/go.mod h1:BvtUaNBr0fEpzb11OfrQiJLsLPtqbmulpo1fPwcpP6Q= k8s.io/csi-translation-lib v0.0.0-20191016115521-756ffa5af0bd/go.mod h1:lf1VBseeLanBpSXD0N9tuPx1ylI8sA0j6f+rckCKiIk= diff --git a/pkg/controller/redisclusterbackup/helper.go b/pkg/controller/redisclusterbackup/helper.go index dd87802b4..2333a5ab9 100644 --- a/pkg/controller/redisclusterbackup/helper.go +++ b/pkg/controller/redisclusterbackup/helper.go @@ -3,6 +3,7 @@ package redisclusterbackup import ( "context" + batch "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" @@ -100,3 +101,12 @@ func newDirectClient(config *rest.Config) client.Client { } return c } + +func isJobFinished(j *batch.Job) bool { + for _, c := range j.Status.Conditions { + if (c.Type == batch.JobComplete || c.Type == batch.JobFailed) && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} diff --git a/pkg/controller/redisclusterbackup/redisclusterbackup_controller.go b/pkg/controller/redisclusterbackup/redisclusterbackup_controller.go index 483919688..2f80eea29 100644 --- a/pkg/controller/redisclusterbackup/redisclusterbackup_controller.go +++ b/pkg/controller/redisclusterbackup/redisclusterbackup_controller.go @@ -119,9 +119,8 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { log.WithValues("namespace", e.MetaNew.GetNamespace(), "name", e.MetaNew.GetName()).V(4).Info("Job UpdateFunc Not Manage") return false } - oldObj := e.ObjectOld.(*batch.Job) newObj := e.ObjectNew.(*batch.Job) - if isJobCompleted(oldObj, newObj) { + if isJobCompleted(newObj) { return true } return false @@ -284,15 +283,9 @@ func remove(list []string, s string) []string { return list } -func isJobCompleted(old, new *batch.Job) bool { - log.WithValues("Request.Namespace", new.Namespace).V(4).Info("isJobCompleted", "old.Succeeded", old.Status.Succeeded, - "new.Succeeded", new.Status.Succeeded, "old.Failed", old.Status.Failed, "new.Failed", new.Status.Failed) - if old.Status.Succeeded == 0 && new.Status.Succeeded > 0 { - log.WithValues("Request.Namespace", new.Namespace).Info("JobCompleted Succeeded", "job", new.Name) - return true - } - if old.Status.Failed < utils.Int32(old.Spec.BackoffLimit) && new.Status.Failed >= utils.Int32(new.Spec.BackoffLimit) { - log.WithValues("Request.Namespace", new.Namespace).Info("JobCompleted Failed", "job", new.Name) +func isJobCompleted(newJob *batch.Job) bool { + if isJobFinished(newJob) { + log.WithValues("Request.Namespace", newJob.Namespace).Info("JobFinished", "type", newJob.Status.Conditions[0].Type, "job", newJob.Name) return true } return false diff --git a/pkg/controller/redisclusterbackup/sync_handler.go b/pkg/controller/redisclusterbackup/sync_handler.go index 1a6bc159c..0fb373c7b 100644 --- a/pkg/controller/redisclusterbackup/sync_handler.go +++ b/pkg/controller/redisclusterbackup/sync_handler.go @@ -435,7 +435,7 @@ func (r *ReconcileRedisClusterBackup) createPVCForBackup(backup *redisv1alpha1.R func (r *ReconcileRedisClusterBackup) handleBackupJob(reqLogger logr.Logger, backup *redisv1alpha1.RedisClusterBackup) error { reqLogger.Info("Handle Backup Job") - job, err := r.jobController.GetJob(backup.Namespace, backup.JobName()) + jobObj, err := r.jobController.GetJob(backup.Namespace, backup.JobName()) if err != nil { // TODO: Sometimes the job is created successfully, but it cannot be obtained immediately. if errors.IsNotFound(err) { @@ -462,7 +462,7 @@ func (r *ReconcileRedisClusterBackup) handleBackupJob(reqLogger logr.Logger, bac } return err } - if job.Status.Succeeded == 0 && job.Status.Failed < utils.Int32(job.Spec.BackoffLimit) { + if !isJobFinished(jobObj) { return fmt.Errorf("wait for job Succeeded or Failed") } @@ -476,10 +476,13 @@ func (r *ReconcileRedisClusterBackup) handleBackupJob(reqLogger logr.Logger, bac ) return err } - for _, o := range job.OwnerReferences { + + jobType := jobObj.Status.Conditions[0].Type + + for _, o := range jobObj.OwnerReferences { if o.Kind == redisv1alpha1.RedisClusterBackupKind { if o.Name == backup.Name { - jobSucceeded := job.Status.Succeeded > 0 + jobSucceeded := jobType == batchv1.JobComplete if jobSucceeded { backup.Status.Phase = redisv1alpha1.BackupPhaseSucceeded } else { From 5ba3193fd463d8fbf2e40c47be64b15c9a2104bd Mon Sep 17 00:00:00 2001 From: fishu Date: Tue, 17 Mar 2020 16:10:49 +0800 Subject: [PATCH 4/9] add OwnerReference for secret --- pkg/controller/manager/ensurer.go | 2 +- pkg/controller/redisclusterbackup/sync_handler.go | 9 ++++++++- pkg/osm/rclone.go | 7 ++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/controller/manager/ensurer.go b/pkg/controller/manager/ensurer.go index 72e422a1e..ce5770724 100644 --- a/pkg/controller/manager/ensurer.go +++ b/pkg/controller/manager/ensurer.go @@ -236,7 +236,7 @@ func (r *realEnsureResource) EnsureRedisOSMSecret(cluster *redisv1alpha1.Distrib return nil } backup := cluster.Status.Restore.Backup - secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), cluster.Namespace, backup.Spec.Backend) + secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), cluster.Namespace, backup.Spec.Backend, redisv1alpha1.DefaultOwnerReferences(cluster)) if err != nil { return err } diff --git a/pkg/controller/redisclusterbackup/sync_handler.go b/pkg/controller/redisclusterbackup/sync_handler.go index 0fb373c7b..16ed1ee96 100644 --- a/pkg/controller/redisclusterbackup/sync_handler.go +++ b/pkg/controller/redisclusterbackup/sync_handler.go @@ -89,7 +89,14 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi return err } - secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), backup.Namespace, backup.Spec.Backend) + secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), backup.Namespace, backup.Spec.Backend, []metav1.OwnerReference{ + { + APIVersion: redisv1alpha1.SchemeGroupVersion.String(), + Kind: redisv1alpha1.RedisClusterBackupKind, + Name: backup.Name, + UID: backup.UID, + }, + }) if err != nil { msg := fmt.Sprintf("Failed to generate osm secret. Reason: %v", err) r.markAsFailedBackup(backup, msg) diff --git a/pkg/osm/rclone.go b/pkg/osm/rclone.go index 2d1106502..226e69392 100644 --- a/pkg/osm/rclone.go +++ b/pkg/osm/rclone.go @@ -18,7 +18,7 @@ import ( // // /etc/osm // └── config -func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend) (*core.Secret, error) { +func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend, ownerReference []metav1.OwnerReference) (*core.Secret, error) { osmCtx, err := newContext(kc, spec, namespace) if err != nil { return nil, err @@ -28,8 +28,9 @@ func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend) out := &core.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, + Name: name, + Namespace: namespace, + OwnerReferences: ownerReference, }, Data: map[string][]byte{ "config": osmBytes, From f30ca5d3a1cc0c4999b8a161f0267be4cc929670 Mon Sep 17 00:00:00 2001 From: fishu Date: Sun, 22 Mar 2020 21:43:08 +0800 Subject: [PATCH 5/9] remove unused osm code --- go.mod | 3 +++ go.sum | 3 +++ hack/docker/redis-tools/Dockerfile | 2 +- hack/docker/redis-tools/make.sh | 16 ++++++++-------- hack/docker/redis-tools/redis-tools.sh | 6 +++--- hack/e2e.sh | 5 +++++ pkg/apis/redis/v1alpha1/default.go | 4 ++-- .../distributedrediscluster/sync_handler.go | 4 ++-- pkg/controller/manager/ensurer.go | 6 +++--- .../redisclusterbackup/sync_handler.go | 10 +++++----- pkg/osm/osm.go | 2 +- pkg/osm/rclone.go | 12 ++++++------ pkg/resources/statefulsets/statefulset.go | 6 +++--- 13 files changed, 45 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index f6725a479..f61ca38f1 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,15 @@ require ( github.com/aws/aws-sdk-go v1.20.20 github.com/go-logr/logr v0.1.0 github.com/go-openapi/spec v0.19.2 + github.com/go-redis/redis v6.15.7+incompatible github.com/mediocregopher/radix.v2 v0.0.0-20181115013041-b67df6e626f9 github.com/onsi/ginkgo v1.8.0 github.com/onsi/gomega v1.5.0 github.com/operator-framework/operator-sdk v0.13.0 github.com/pkg/errors v0.8.1 + github.com/satori/go.uuid v1.2.0 github.com/spf13/pflag v1.0.5 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e gomodules.xyz/stow v0.2.3 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 diff --git a/go.sum b/go.sum index a365d0d01..7bdd898aa 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,8 @@ github.com/go-openapi/swag v0.19.4/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= +github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -815,6 +817,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/hack/docker/redis-tools/Dockerfile b/hack/docker/redis-tools/Dockerfile index fd950bbf9..be3135b34 100644 --- a/hack/docker/redis-tools/Dockerfile +++ b/hack/docker/redis-tools/Dockerfile @@ -8,7 +8,7 @@ RUN set -x \ zip \ && rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man /tmp/* -COPY osm /usr/local/bin/osm +COPY rclone /usr/local/bin/rclone COPY redis-tools.sh /usr/local/bin/redis-tools.sh RUN chmod +x /usr/local/bin/redis-tools.sh diff --git a/hack/docker/redis-tools/make.sh b/hack/docker/redis-tools/make.sh index 680890f59..fd821f0c1 100644 --- a/hack/docker/redis-tools/make.sh +++ b/hack/docker/redis-tools/make.sh @@ -17,7 +17,7 @@ IMG=redis-tools DB_VERSION=5.0.4 TAG="$DB_VERSION" -OSM_VER=${OSM_VER:-v1.50.2} +RCLONE_VER=${RCLONE_VER:-v1.50.2} DIST=$REPO_ROOT/dist mkdir -p $DIST @@ -25,19 +25,19 @@ mkdir -p $DIST build() { pushd "$REPO_ROOT/hack/docker/redis-tools" - if [ ! -f "osm" ]; then + if [ ! -f "rclone" ]; then # Download rclone - wget https://downloads.rclone.org/"${OSM_VER}"/rclone-"${OSM_VER}"-linux-amd64.zip - unzip rclone-"${OSM_VER}"-linux-amd64.zip - chmod +x rclone-"${OSM_VER}"-linux-amd64/rclone - mv rclone-"${OSM_VER}"-linux-amd64/rclone osm + wget https://downloads.rclone.org/"${RCLONE_VER}"/rclone-"${RCLONE_VER}"-linux-amd64.zip + unzip rclone-"${RCLONE_VER}"-linux-amd64.zip + chmod +x rclone-"${RCLONE_VER}"-linux-amd64/rclone + mv rclone-"${RCLONE_VER}"-linux-amd64/rclone rclone fi local cmd="docker build --pull -t $DOCKER_REGISTRY/$IMG:$TAG ." echo $cmd; $cmd - rm -rf rclone-"${OSM_VER}"-linux-amd64* - rm osm + rm -rf rclone-"${RCLONE_VER}"-linux-amd64* + rm rclone popd } diff --git a/hack/docker/redis-tools/redis-tools.sh b/hack/docker/redis-tools/redis-tools.sh index b25b9f637..0c157023b 100644 --- a/hack/docker/redis-tools/redis-tools.sh +++ b/hack/docker/redis-tools/redis-tools.sh @@ -31,7 +31,7 @@ REDIS_FOLDER=${REDIS_FOLDER:-} REDIS_SNAPSHOT=${REDIS_SNAPSHOT:-} REDIS_DATA_DIR=${REDIS_DATA_DIR:-/data} REDIS_RESTORE_SUCCEEDED=${REDIS_RESTORE_SUCCEEDED:-0} -OSM_CONFIG_FILE=/etc/osm/config +RCLONE_CONFIG_FILE=/etc/rclone/config op=$1 shift @@ -114,7 +114,7 @@ case "$op" in ls -lh "$SOURCE_DIR" echo "Uploading dump file to the backend......." echo "From $SOURCE_DIR" - osm --config "$OSM_CONFIG_FILE" copy "$SOURCE_DIR" "$REDIS_LOCATION"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" -v + rclone --config "$RCLONE_CONFIG_FILE" copy "$SOURCE_DIR" "$REDIS_LOCATION"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" -v echo "Backup successful" ;; @@ -128,7 +128,7 @@ case "$op" in REDIS_SNAPSHOT=${REDIS_SNAPSHOT}-${index} SOURCE_SNAPSHOT="$REDIS_LOCATION"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" echo "From $SOURCE_SNAPSHOT" - osm --config "$OSM_CONFIG_FILE" sync "$SOURCE_SNAPSHOT" "$REDIS_DATA_DIR" -v + rclone --config "$RCLONE_CONFIG_FILE" sync "$SOURCE_SNAPSHOT" "$REDIS_DATA_DIR" -v echo "Recovery successful" ;; diff --git a/hack/e2e.sh b/hack/e2e.sh index 2cccf7193..4dda36096 100644 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -9,6 +9,11 @@ if [[ -z ${STORAGECLASSNAME} ]]; then exit 1 fi +if [[ -z ${CLUSTER_DOMAIN} ]]; then + echo "env CLUSTER_DOMAIN not set" + exit 1 +fi + if [[ -z ${GINKGO_SKIP} ]]; then export GINKGO_SKIP="" fi diff --git a/pkg/apis/redis/v1alpha1/default.go b/pkg/apis/redis/v1alpha1/default.go index c3c32228f..b71092ebb 100644 --- a/pkg/apis/redis/v1alpha1/default.go +++ b/pkg/apis/redis/v1alpha1/default.go @@ -131,8 +131,8 @@ func (in *RedisClusterBackup) RemotePath() (string, error) { return "", fmt.Errorf("no storage provider is configured") } -func (in *RedisClusterBackup) OSMSecretName() string { - return fmt.Sprintf("osmconfig-%v", in.Name) +func (in *RedisClusterBackup) RCloneSecretName() string { + return fmt.Sprintf("rcloneconfig-%v", in.Name) } func (in *RedisClusterBackup) JobName() string { diff --git a/pkg/controller/distributedrediscluster/sync_handler.go b/pkg/controller/distributedrediscluster/sync_handler.go index 471a33c85..33e89dc33 100644 --- a/pkg/controller/distributedrediscluster/sync_handler.go +++ b/pkg/controller/distributedrediscluster/sync_handler.go @@ -69,9 +69,9 @@ func (r *ReconcileDistributedRedisCluster) ensureCluster(ctx *syncContext) error if err := r.ensurer.EnsureRedisSvc(cluster, labels); err != nil { return Kubernetes.Wrap(err, "EnsureRedisSvc") } - if err := r.ensurer.EnsureRedisOSMSecret(cluster, labels); err != nil { + if err := r.ensurer.EnsureRedisRCloneSecret(cluster, labels); err != nil { if k8sutil.IsRequestRetryable(err) { - return Kubernetes.Wrap(err, "EnsureRedisOSMSecret") + return Kubernetes.Wrap(err, "EnsureRedisRCloneSecret") } return StopRetry.Wrap(err, "stop retry") } diff --git a/pkg/controller/manager/ensurer.go b/pkg/controller/manager/ensurer.go index ce5770724..50784f338 100644 --- a/pkg/controller/manager/ensurer.go +++ b/pkg/controller/manager/ensurer.go @@ -24,7 +24,7 @@ type IEnsureResource interface { EnsureRedisHeadLessSvcs(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error EnsureRedisSvc(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error EnsureRedisConfigMap(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error - EnsureRedisOSMSecret(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error + EnsureRedisRCloneSecret(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error } type realEnsureResource struct { @@ -231,12 +231,12 @@ func (r *realEnsureResource) EnsureRedisConfigMap(cluster *redisv1alpha1.Distrib return nil } -func (r *realEnsureResource) EnsureRedisOSMSecret(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error { +func (r *realEnsureResource) EnsureRedisRCloneSecret(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error { if !cluster.IsRestoreFromBackup() || cluster.IsRestored() { return nil } backup := cluster.Status.Restore.Backup - secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), cluster.Namespace, backup.Spec.Backend, redisv1alpha1.DefaultOwnerReferences(cluster)) + secret, err := osm.NewRcloneSecret(r.client, backup.RCloneSecretName(), cluster.Namespace, backup.Spec.Backend, redisv1alpha1.DefaultOwnerReferences(cluster)) if err != nil { return err } diff --git a/pkg/controller/redisclusterbackup/sync_handler.go b/pkg/controller/redisclusterbackup/sync_handler.go index 16ed1ee96..864e9c490 100644 --- a/pkg/controller/redisclusterbackup/sync_handler.go +++ b/pkg/controller/redisclusterbackup/sync_handler.go @@ -89,7 +89,7 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi return err } - secret, err := osm.NewRcloneSecret(r.client, backup.OSMSecretName(), backup.Namespace, backup.Spec.Backend, []metav1.OwnerReference{ + secret, err := osm.NewRcloneSecret(r.client, backup.RCloneSecretName(), backup.Namespace, backup.Spec.Backend, []metav1.OwnerReference{ { APIVersion: redisv1alpha1.SchemeGroupVersion.String(), Kind: redisv1alpha1.RedisClusterBackupKind, @@ -98,7 +98,7 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi }, }) if err != nil { - msg := fmt.Sprintf("Failed to generate osm secret. Reason: %v", err) + msg := fmt.Sprintf("Failed to generate rclone secret. Reason: %v", err) r.markAsFailedBackup(backup, msg) r.recorder.Event( backup, @@ -252,10 +252,10 @@ func (r *ReconcileRedisClusterBackup) getBackupJob(reqLogger logr.Logger, backup VolumeSource: persistentVolume.VolumeSource, }, { - Name: "osmconfig", + Name: "rcloneconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: backup.OSMSecretName(), + SecretName: backup.RCloneSecretName(), }, }, }, @@ -336,7 +336,7 @@ func (r *ReconcileRedisClusterBackup) backupContainers(backup *redisv1alpha1.Red MountPath: redisv1alpha1.BackupDumpDir, }, { - Name: "osmconfig", + Name: "rcloneconfig", ReadOnly: true, MountPath: osm.SecretMountPath, }, diff --git a/pkg/osm/osm.go b/pkg/osm/osm.go index f4a2b9b73..988185627 100644 --- a/pkg/osm/osm.go +++ b/pkg/osm/osm.go @@ -34,7 +34,7 @@ import ( ) const ( - SecretMountPath = "/etc/osm" + SecretMountPath = "/etc/rclone" CaCertFileName = "ca.crt" ) diff --git a/pkg/osm/rclone.go b/pkg/osm/rclone.go index 226e69392..7e0a05108 100644 --- a/pkg/osm/rclone.go +++ b/pkg/osm/rclone.go @@ -13,18 +13,18 @@ import ( ) // NewRcloneSecret creates a secret that contains the config file of Rclone. -// So, generally, if this secret is mounted in `etc/osm`, -// the tree of `/etc/osm` directory will be similar to, +// So, generally, if this secret is mounted in `etc/rclone`, +// the tree of `/etc/rclone` directory will be similar to, // -// /etc/osm +// /etc/rclone // └── config func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend, ownerReference []metav1.OwnerReference) (*core.Secret, error) { - osmCtx, err := newContext(kc, spec, namespace) + rcloneCtx, err := newContext(kc, spec, namespace) if err != nil { return nil, err } - osmBytes := []byte(osmCtx) + rcloneBytes := []byte(rcloneCtx) out := &core.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -33,7 +33,7 @@ func NewRcloneSecret(kc client.Client, name, namespace string, spec api.Backend, OwnerReferences: ownerReference, }, Data: map[string][]byte{ - "config": osmBytes, + "config": rcloneBytes, }, } return out, nil diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 720ddea87..b69384814 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -348,7 +348,7 @@ func redisInitContainer(cluster *redisv1alpha1.DistributedRedisCluster, password MountPath: redisv1alpha1.BackupDumpDir, }, { - Name: "osmconfig", + Name: "rcloneconfig", ReadOnly: true, MountPath: osm.SecretMountPath, }, @@ -439,10 +439,10 @@ func redisVolumes(cluster *redisv1alpha1.DistributedRedisCluster) []corev1.Volum } volumes = append(volumes, corev1.Volume{ - Name: "osmconfig", + Name: "rcloneconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cluster.Status.Restore.Backup.OSMSecretName(), + SecretName: cluster.Status.Restore.Backup.RCloneSecretName(), }, }, }) From 58aaa21ab825f3e391dab46e061720e4f31d3f65 Mon Sep 17 00:00:00 2001 From: fishu Date: Sun, 22 Mar 2020 21:44:04 +0800 Subject: [PATCH 6/9] stuffing data to redis and check dbsize for e2e --- deploy/e2e.yml | 2 ++ test/e2e/drc/drc_test.go | 16 +++++++++++++ test/e2e/drcb/drcb_test.go | 18 +++++++++++++++ test/e2e/framework.go | 4 ++-- test/e2e/goredis_util.go | 47 ++++++++++++++++++++++++++++++++++++++ test/e2e/operator_util.go | 18 ++++++++++++++- 6 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 test/e2e/goredis_util.go diff --git a/deploy/e2e.yml b/deploy/e2e.yml index 3447fee14..aec959694 100644 --- a/deploy/e2e.yml +++ b/deploy/e2e.yml @@ -30,6 +30,8 @@ spec: value: "" - name: S3_BUCKET value: "" + - name: CLUSTER_DOMAIN + value: "" --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/test/e2e/drc/drc_test.go b/test/e2e/drc/drc_test.go index ee7cc9549..8796a7c7b 100644 --- a/test/e2e/drc/drc_test.go +++ b/test/e2e/drc/drc_test.go @@ -7,6 +7,12 @@ import ( "github.com/ucloud/redis-cluster-operator/test/e2e" ) +var ( + goredis *e2e.GoRedis + dbsize int64 + err error +) + var _ = Describe("DistributedRedisCluster CRUD", func() { It("should create a DistributedRedisCluster", func() { name := e2e.RandString(8) @@ -15,6 +21,11 @@ var _ = Describe("DistributedRedisCluster CRUD", func() { Ω(f.CreateRedisClusterPassword(password)).Should(Succeed()) Ω(f.CreateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(name, f.Namespace(), password) + Expect(goredis.StuffingData(10, 300000)).NotTo(HaveOccurred()) + dbsize, err = goredis.DBSize() + Expect(err).NotTo(HaveOccurred()) + f.Logf("%s DBSIZE: %d", name, dbsize) }) Context("when the DistributedRedisCluster is created", func() { @@ -22,27 +33,32 @@ var _ = Describe("DistributedRedisCluster CRUD", func() { e2e.ChangeDRCRedisConfig(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) It("should recover from accidentally deleting master pods", func() { e2e.DeleteMasterPodForDRC(drc, f.Client) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) It("should scale up a DistributedRedisCluster", func() { e2e.ScaleUPDRC(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when the scale up succeeded", func() { It("should scale down a DistributedRedisCluster", func() { e2e.ScaleUPDown(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) It("should update the DistributedRedisCluster minor version", func() { e2e.RollingUpdateDRC(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) }) diff --git a/test/e2e/drcb/drcb_test.go b/test/e2e/drcb/drcb_test.go index 569964503..6db9b0341 100644 --- a/test/e2e/drcb/drcb_test.go +++ b/test/e2e/drcb/drcb_test.go @@ -8,6 +8,12 @@ import ( "github.com/ucloud/redis-cluster-operator/test/e2e" ) +var ( + goredis *e2e.GoRedis + dbsize int64 + err error +) + var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func() { It("should create a DistributedRedisCluster", func() { name := e2e.RandString(8) @@ -16,6 +22,11 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func Ω(f.CreateRedisClusterPassword(password)).Should(Succeed()) Ω(f.CreateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(name, f.Namespace(), password) + Expect(goredis.StuffingData(10, 300000)).NotTo(HaveOccurred()) + dbsize, err = goredis.DBSize() + Expect(err).NotTo(HaveOccurred()) + f.Logf("%s DBSIZE: %d", name, dbsize) }) Context("when the DistributedRedisCluster is created", func() { @@ -29,6 +40,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func Ω(f.CreateS3Secret(s3ID, s3Key)).Should(Succeed()) Ω(f.CreateRedisClusterBackup(drcb)).Should(Succeed()) Eventually(e2e.IsRedisClusterBackupProperly(f, drcb), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when the RedisClusterBackup is created", func() { It("should restore from backup", func() { @@ -36,33 +48,39 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func rdrc = e2e.RestoreDRC(drc, drcb) Ω(f.CreateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when restore is succeeded", func() { It("should change redis config for a DistributedRedisCluster", func() { e2e.ChangeDRCRedisConfig(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) It("should recover from accidentally deleting master pods", func() { e2e.DeleteMasterPodForDRC(rdrc, f.Client) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) It("should scale up a DistributedRedisCluster", func() { e2e.ScaleUPDRC(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when the scale up succeeded", func() { It("should scale down a DistributedRedisCluster", func() { e2e.ScaleUPDown(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) It("should update the DistributedRedisCluster minor version", func() { e2e.RollingUpdateDRC(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) }) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 12cf975f5..034175edf 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -71,13 +71,13 @@ func (f *Framework) AfterEach() error { // Logf write log to ginkgo output func (f *Framework) Logf(format string, a ...interface{}) { l := fmt.Sprintf(format, a...) - Logf("namespace:%s %s", f.Namespace(), l) + Logf("namespace: %s %s", f.Namespace(), l) } // Logf write log to ginkgo output func (f *Framework) Failf(format string, a ...interface{}) { l := fmt.Sprintf(format, a...) - Failf("namespace:%s %s", f.Namespace(), l) + Failf("namespace: %s %s", f.Namespace(), l) } // Namespace return the test namespace name diff --git a/test/e2e/goredis_util.go b/test/e2e/goredis_util.go new file mode 100644 index 000000000..a02eeb3d1 --- /dev/null +++ b/test/e2e/goredis_util.go @@ -0,0 +1,47 @@ +package e2e + +import ( + "github.com/go-redis/redis" + uuid "github.com/satori/go.uuid" + "golang.org/x/sync/errgroup" +) + +// GoRedis contains ClusterClient. +type GoRedis struct { + client *redis.ClusterClient +} + +// NewGoRedis return a new ClusterClient. +func NewGoRedis(addr, password string) *GoRedis { + return &GoRedis{ + client: redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: []string{addr}, + Password: password, + }), + } +} + +// StuffingData filled with (round * n)'s key. +func (g *GoRedis) StuffingData(round, n int) error { + var group errgroup.Group + for i := 0; i < round; i++ { + group.Go(func() error { + for j := 0; j < n; j++ { + key := uuid.NewV4().String() + if err := g.client.Set(key, key, 0).Err(); err != nil { + return err + } + } + return nil + }) + } + if err := group.Wait(); err != nil { + return err + } + return nil +} + +// DBSize return DBsize of all master nodes. +func (g *GoRedis) DBSize() (int64, error) { + return g.client.DBSize().Result() +} diff --git a/test/e2e/operator_util.go b/test/e2e/operator_util.go index 45f84ab27..92b628967 100644 --- a/test/e2e/operator_util.go +++ b/test/e2e/operator_util.go @@ -90,7 +90,7 @@ func NewDistributedRedisCluster(name, namespace, image, passwordName string, mas }, Storage: &redisv1alpha1.RedisStorage{ Type: "persistent-claim", - Size: resource.MustParse("1Gi"), + Size: resource.MustParse("10Gi"), Class: storageClassName, DeleteClaim: true, }, @@ -339,3 +339,19 @@ func IsRedisClusterBackupProperly(f *Framework, drcb *redisv1alpha1.RedisCluster return nil } } + +func NewGoRedisClient(svc, namespaces, password string) *GoRedis { + addr := fmt.Sprintf("%s.%s.svc.%s:6379", svc, namespaces, os.Getenv("CLUSTER_DOMAIN")) + return NewGoRedis(addr, password) +} + +func IsDBSizeConsistent(originalDBSize int64, goredis *GoRedis) error { + curDBSize, err := goredis.DBSize() + if err != nil { + return err + } + if curDBSize != originalDBSize { + return LogAndReturnErrorf("DBSize do not Equal current: %d, original: %d", curDBSize, originalDBSize) + } + return nil +} From 31885edb65ea6660b38b0facc11e0accee9ba7d2 Mon Sep 17 00:00:00 2001 From: fishu Date: Mon, 23 Mar 2020 18:28:15 +0800 Subject: [PATCH 7/9] fixed e2e: wait pod deleted --- test/e2e/drc/drc_test.go | 1 + test/e2e/drcb/drcb_test.go | 2 ++ test/e2e/goredis_util.go | 22 +++++++++++++++++++--- test/e2e/operator_util.go | 19 +++++++++++++++++++ test/e2e/util.go | 2 +- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/test/e2e/drc/drc_test.go b/test/e2e/drc/drc_test.go index 8796a7c7b..09aa3fca1 100644 --- a/test/e2e/drc/drc_test.go +++ b/test/e2e/drc/drc_test.go @@ -37,6 +37,7 @@ var _ = Describe("DistributedRedisCluster CRUD", func() { }) It("should recover from accidentally deleting master pods", func() { e2e.DeleteMasterPodForDRC(drc, f.Client) + Eventually(e2e.IsDRCPodBeDeleted(f, drc), "5m", "10s").ShouldNot(HaveOccurred()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) diff --git a/test/e2e/drcb/drcb_test.go b/test/e2e/drcb/drcb_test.go index 6db9b0341..42c2e2d57 100644 --- a/test/e2e/drcb/drcb_test.go +++ b/test/e2e/drcb/drcb_test.go @@ -48,6 +48,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func rdrc = e2e.RestoreDRC(drc, drcb) Ω(f.CreateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(rdrc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when restore is succeeded", func() { @@ -59,6 +60,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func }) It("should recover from accidentally deleting master pods", func() { e2e.DeleteMasterPodForDRC(rdrc, f.Client) + Eventually(e2e.IsDRCPodBeDeleted(f, rdrc), "5m", "10s").ShouldNot(HaveOccurred()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) diff --git a/test/e2e/goredis_util.go b/test/e2e/goredis_util.go index a02eeb3d1..e756f58c1 100644 --- a/test/e2e/goredis_util.go +++ b/test/e2e/goredis_util.go @@ -4,20 +4,31 @@ import ( "github.com/go-redis/redis" uuid "github.com/satori/go.uuid" "golang.org/x/sync/errgroup" + "time" ) +const defaultTimeOut = time.Second * 2 + // GoRedis contains ClusterClient. type GoRedis struct { - client *redis.ClusterClient + client *redis.ClusterClient + password string } // NewGoRedis return a new ClusterClient. func NewGoRedis(addr, password string) *GoRedis { return &GoRedis{ client: redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{addr}, - Password: password, + Addrs: []string{addr}, + Password: password, + MaxRetries: 5, + + PoolSize: 3, + MinIdleConns: 1, + PoolTimeout: defaultTimeOut, + IdleTimeout: defaultTimeOut, }), + password: password, } } @@ -45,3 +56,8 @@ func (g *GoRedis) StuffingData(round, n int) error { func (g *GoRedis) DBSize() (int64, error) { return g.client.DBSize().Result() } + +// Password return redis password. +func (g *GoRedis) Password() string { + return g.password +} diff --git a/test/e2e/operator_util.go b/test/e2e/operator_util.go index 92b628967..f86a7ba66 100644 --- a/test/e2e/operator_util.go +++ b/test/e2e/operator_util.go @@ -122,6 +122,9 @@ func IsDistributedRedisClusterProperly(f *Framework, drc *redisv1alpha1.Distribu return err } if result.Status.Status != redisv1alpha1.ClusterStatusOK { + if result.Status.Status == redisv1alpha1.ClusterStatusKO { + f.Logf("DistributedRedisCluster %s is %s, reason: %s", drc.Name, result.Status.Status, result.Status.Reason) + } return LogAndReturnErrorf("DistributedRedisCluster %s status not healthy, current: %s", drc.Name, result.Status.Status) } stsList, err := f.GetDRCStatefulSetByLabels(getLabels(drc)) @@ -177,6 +180,7 @@ func IsDistributedRedisClusterProperly(f *Framework, drc *redisv1alpha1.Distribu } } + drc.Spec = result.Spec return nil } } @@ -299,6 +303,21 @@ func DeleteMasterPodForDRC(drc *redisv1alpha1.DistributedRedisCluster, client cl } } +func IsDRCPodBeDeleted(f *Framework, drc *redisv1alpha1.DistributedRedisCluster) func() error { + return func() error { + stsList, err := f.GetDRCStatefulSetByLabels(getLabels(drc)) + if err != nil { + return LogAndReturnErrorf("GetDRCStatefulSetByLabels err: %s", err) + } + for _, sts := range stsList.Items { + if sts.Status.ReadyReplicas != (drc.Spec.ClusterReplicas + 1) { + return nil + } + } + return LogAndReturnErrorf("StatefulSet's Pod still running") + } +} + func NewRedisClusterBackup(name, namespace, image, drcName, storageSecretName, s3Endpoint, s3Bucket string) *redisv1alpha1.RedisClusterBackup { return &redisv1alpha1.RedisClusterBackup{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/util.go b/test/e2e/util.go index 533b1b2ec..fda98badc 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -24,7 +24,7 @@ func Logf(format string, args ...interface{}) { // Failf reports a failure in the current e2e func Failf(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) - log("INFO", msg) + log("ERROR", msg) ginkgo.Fail(nowStamp()+": "+msg, 1) } From 5a3b1912d31fc83226ccf627a8c6259f1ec94bd7 Mon Sep 17 00:00:00 2001 From: fishu Date: Mon, 23 Mar 2020 23:21:23 +0800 Subject: [PATCH 8/9] add phase for restore --- pkg/apis/redis/v1alpha1/constants.go | 11 +++++++ pkg/apis/redis/v1alpha1/default.go | 14 ++++++++- .../v1alpha1/distributedrediscluster_types.go | 6 ++-- .../distributedrediscluster_controller.go | 31 +++++++++++++++++-- .../distributedrediscluster/status.go | 2 +- .../distributedrediscluster/sync_handler.go | 23 +++++++++----- pkg/controller/manager/ensurer.go | 28 +++++++++++++++-- pkg/resources/configmaps/configmap.go | 3 +- pkg/resources/statefulsets/statefulset.go | 4 +-- 9 files changed, 101 insertions(+), 21 deletions(-) diff --git a/pkg/apis/redis/v1alpha1/constants.go b/pkg/apis/redis/v1alpha1/constants.go index 50e49f9ca..410ab52e8 100644 --- a/pkg/apis/redis/v1alpha1/constants.go +++ b/pkg/apis/redis/v1alpha1/constants.go @@ -59,6 +59,17 @@ const ( NodesPlacementInfoOptimal NodesPlacementInfo = "Optimal" ) +type RestorePhase string + +const ( + // RestorePhaseRunning used for Restore that are currently running. + RestorePhaseRunning RestorePhase = "Running" + // RestorePhaseRestart used for Restore that are restart master nodes. + RestorePhaseRestart RestorePhase = "Restart" + // RestorePhaseSucceeded used for Restore that are Succeeded. + RestorePhaseSucceeded RestorePhase = "Succeeded" +) + const ( DatabaseNamePrefix = "redis" diff --git a/pkg/apis/redis/v1alpha1/default.go b/pkg/apis/redis/v1alpha1/default.go index b71092ebb..49e96095d 100644 --- a/pkg/apis/redis/v1alpha1/default.go +++ b/pkg/apis/redis/v1alpha1/default.go @@ -70,7 +70,19 @@ func (in *DistributedRedisCluster) IsRestoreFromBackup() bool { } func (in *DistributedRedisCluster) IsRestored() bool { - return in.Status.Restore.RestoreSucceeded > 0 + return in.Status.Restore.Phase == RestorePhaseSucceeded +} + +func (in *DistributedRedisCluster) ShouldInitRestorePhase() bool { + return in.Status.Restore.Phase == "" +} + +func (in *DistributedRedisCluster) IsRestoreRunning() bool { + return in.Status.Restore.Phase == RestorePhaseRunning +} + +func (in *DistributedRedisCluster) IsRestoreRestarting() bool { + return in.Status.Restore.Phase == RestorePhaseRestart } func defaultResource() *v1.ResourceRequirements { diff --git a/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go b/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go index 644d06763..7403846f2 100644 --- a/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go +++ b/pkg/apis/redis/v1alpha1/distributedrediscluster_types.go @@ -116,10 +116,8 @@ type DistributedRedisClusterStatus struct { } type Restore struct { - // The number of restore which reached phase Succeeded. - RestoreSucceeded int32 `json:"restoreSucceeded,omitempty"` - Backup *RedisClusterBackup `json:"backup, omitempty"` - //BackupSourceSpec `json:",inline"` + Phase RestorePhase `json:"phase,omitempty"` + Backup *RedisClusterBackup `json:"backup, omitempty"` } // RedisClusterNode represent a RedisCluster Node diff --git a/pkg/controller/distributedrediscluster/distributedrediscluster_controller.go b/pkg/controller/distributedrediscluster/distributedrediscluster_controller.go index ad949a5bc..6a41e518d 100644 --- a/pkg/controller/distributedrediscluster/distributedrediscluster_controller.go +++ b/pkg/controller/distributedrediscluster/distributedrediscluster_controller.go @@ -249,10 +249,35 @@ func (r *ReconcileDistributedRedisCluster) Reconcile(request reconcile.Request) return reconcile.Result{}, err } - // update cr and wait for the next Reconcile loop - if instance.IsRestoreFromBackup() && !instance.IsRestored() { + // mark .Status.Restore.Phase = RestorePhaseRestart, will + // remove init container and restore volume that referenced in stateulset for + // dump RDB file from backup, then the redis master node will be restart. + if instance.IsRestoreFromBackup() && instance.IsRestoreRunning() { reqLogger.Info("update restore redis cluster cr") - instance.Status.Restore.RestoreSucceeded = 1 + instance.Status.Restore.Phase = redisv1alpha1.RestorePhaseRestart + if err := r.crController.UpdateCRStatus(instance); err != nil { + return reconcile.Result{}, err + } + if err := r.ensurer.UpdateRedisStatefulsets(instance, getLabels(instance)); err != nil { + return reconcile.Result{}, err + } + waiter := &waitStatefulSetUpdating{ + name: "waitMasterNodeRestarting", + timeout: 60 * time.Second, + tick: 5 * time.Second, + statefulSetController: r.statefulSetController, + cluster: instance, + } + if err := waiting(waiter, ctx.reqLogger); err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{Requeue: true}, nil + } + + // restore succeeded, then update cr and wait for the next Reconcile loop + if instance.IsRestoreFromBackup() && instance.IsRestoreRestarting() { + reqLogger.Info("update restore redis cluster cr") + instance.Status.Restore.Phase = redisv1alpha1.RestorePhaseSucceeded if err := r.crController.UpdateCRStatus(instance); err != nil { return reconcile.Result{}, err } diff --git a/pkg/controller/distributedrediscluster/status.go b/pkg/controller/distributedrediscluster/status.go index cc1d6819a..44df692c8 100644 --- a/pkg/controller/distributedrediscluster/status.go +++ b/pkg/controller/distributedrediscluster/status.go @@ -144,7 +144,7 @@ func compareStatus(old, new *redisv1alpha1.DistributedRedisClusterStatus, reqLog return true } - if utils.CompareInt32("restoreSucceeded", old.Restore.RestoreSucceeded, new.Restore.RestoreSucceeded, reqLogger) { + if utils.CompareStringValue("restoreSucceeded", string(old.Restore.Phase), string(new.Restore.Phase), reqLogger) { return true } diff --git a/pkg/controller/distributedrediscluster/sync_handler.go b/pkg/controller/distributedrediscluster/sync_handler.go index 33e89dc33..8ae48cdb2 100644 --- a/pkg/controller/distributedrediscluster/sync_handler.go +++ b/pkg/controller/distributedrediscluster/sync_handler.go @@ -30,7 +30,7 @@ type syncContext struct { func (r *ReconcileDistributedRedisCluster) ensureCluster(ctx *syncContext) error { cluster := ctx.cluster - if err := r.validate(cluster, ctx.reqLogger); err != nil { + if err := r.validateAndSetDefault(cluster, ctx.reqLogger); err != nil { if k8sutil.IsRequestRetryable(err) { return Kubernetes.Wrap(err, "Validate") } @@ -89,16 +89,22 @@ func (r *ReconcileDistributedRedisCluster) waitPodReady(ctx *syncContext) error return nil } -func (r *ReconcileDistributedRedisCluster) validate(cluster *redisv1alpha1.DistributedRedisCluster, reqLogger logr.Logger) error { +func (r *ReconcileDistributedRedisCluster) validateAndSetDefault(cluster *redisv1alpha1.DistributedRedisCluster, reqLogger logr.Logger) error { var update bool var err error - if cluster.IsRestoreFromBackup() && !cluster.IsRestored() { - update, err = r.validateRestore(cluster, reqLogger) + if cluster.IsRestoreFromBackup() && cluster.ShouldInitRestorePhase() { + update, err = r.initRestore(cluster, reqLogger) if err != nil { return err } } + + if cluster.IsRestoreFromBackup() && (cluster.IsRestoreRunning() || cluster.IsRestoreRestarting()) { + // Set ClusterReplicas = 0, only start master node in first reconcile loop when do restore + cluster.Spec.ClusterReplicas = 0 + } + updateDefault := cluster.DefaultSpec(reqLogger) if update || updateDefault { return r.crController.UpdateCR(cluster) @@ -116,7 +122,7 @@ func dbLoadedFromDiskWhenRestore(cluster *redisv1alpha1.DistributedRedisCluster, } } -func (r *ReconcileDistributedRedisCluster) validateRestore(cluster *redisv1alpha1.DistributedRedisCluster, reqLogger logr.Logger) (bool, error) { +func (r *ReconcileDistributedRedisCluster) initRestore(cluster *redisv1alpha1.DistributedRedisCluster, reqLogger logr.Logger) (bool, error) { update := false if cluster.Status.Restore.Backup == nil { initSpec := cluster.Spec.Init @@ -130,6 +136,10 @@ func (r *ReconcileDistributedRedisCluster) validateRestore(cluster *redisv1alpha return update, fmt.Errorf("backup is still running") } cluster.Status.Restore.Backup = backup + cluster.Status.Restore.Phase = redisv1alpha1.RestorePhaseRunning + if err := r.crController.UpdateCRStatus(cluster); err != nil { + return update, err + } } backup := cluster.Status.Restore.Backup if cluster.Spec.Image == "" { @@ -140,8 +150,7 @@ func (r *ReconcileDistributedRedisCluster) validateRestore(cluster *redisv1alpha cluster.Spec.MasterSize = backup.Status.MasterSize update = true } - // Set ClusterReplicas = 0, only start master node in first reconcile loop when do restore - cluster.Spec.ClusterReplicas = 0 + return update, nil } diff --git a/pkg/controller/manager/ensurer.go b/pkg/controller/manager/ensurer.go index 50784f338..390e20f3c 100644 --- a/pkg/controller/manager/ensurer.go +++ b/pkg/controller/manager/ensurer.go @@ -1,7 +1,6 @@ package manager import ( - "strconv" "strings" "github.com/go-logr/logr" @@ -25,6 +24,7 @@ type IEnsureResource interface { EnsureRedisSvc(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error EnsureRedisConfigMap(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error EnsureRedisRCloneSecret(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error + UpdateRedisStatefulsets(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error } type realEnsureResource struct { @@ -223,7 +223,7 @@ func (r *realEnsureResource) EnsureRedisConfigMap(cluster *redisv1alpha1.Distrib } return err } - if restoreCm.Data[configmaps.RestoreSucceeded] != strconv.Itoa(int(cluster.Status.Restore.RestoreSucceeded)) { + if cluster.Status.Restore.Phase == redisv1alpha1.RestorePhaseRestart && restoreCm.Data[configmaps.RestoreSucceeded] == "0" { cm := configmaps.NewConfigMapForRestore(cluster, labels) return r.configMapClient.UpdateConfigMap(cm) } @@ -268,3 +268,27 @@ func isRedisConfChanged(confInCm string, currentConf map[string]string, log logr } return false } + +func (r *realEnsureResource) UpdateRedisStatefulsets(cluster *redisv1alpha1.DistributedRedisCluster, labels map[string]string) error { + for i := 0; i < int(cluster.Spec.MasterSize); i++ { + name := statefulsets.ClusterStatefulSetName(cluster.Name, i) + svcName := statefulsets.ClusterHeadlessSvcName(cluster.Spec.ServiceName, i) + // assign label + labels[redisv1alpha1.StatefulSetLabel] = name + if err := r.updateRedisStatefulset(cluster, name, svcName, labels); err != nil { + return err + } + } + return nil +} + +func (r *realEnsureResource) updateRedisStatefulset(cluster *redisv1alpha1.DistributedRedisCluster, ssName, svcName string, + labels map[string]string) error { + r.logger.WithValues("StatefulSet.Namespace", cluster.Namespace, "StatefulSet.Name", ssName). + Info("updating statefulSet immediately") + newSS, err := statefulsets.NewStatefulSetForCR(cluster, ssName, svcName, labels) + if err != nil { + return err + } + return r.statefulSetClient.UpdateStatefulSet(newSS) +} diff --git a/pkg/resources/configmaps/configmap.go b/pkg/resources/configmaps/configmap.go index 42fd2376d..c02165fa5 100644 --- a/pkg/resources/configmaps/configmap.go +++ b/pkg/resources/configmaps/configmap.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "sort" + "strconv" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -106,7 +107,7 @@ func NewConfigMapForRestore(cluster *redisv1alpha1.DistributedRedisCluster, labe OwnerReferences: redisv1alpha1.DefaultOwnerReferences(cluster), }, Data: map[string]string{ - RestoreSucceeded: fmt.Sprintf("%d", cluster.Status.Restore.RestoreSucceeded), + RestoreSucceeded: strconv.Itoa(0), }, } } diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index b69384814..9718d50bd 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -85,7 +85,7 @@ func NewStatefulSetForCR(cluster *redisv1alpha1.DistributedRedisCluster, ssName, if spec.Monitor != nil { ss.Spec.Template.Spec.Containers = append(ss.Spec.Template.Spec.Containers, redisExporterContainer(cluster, password)) } - if cluster.IsRestoreFromBackup() && !cluster.IsRestored() && cluster.Status.Restore.Backup != nil { + if cluster.IsRestoreFromBackup() && cluster.IsRestoreRunning() && cluster.Status.Restore.Backup != nil { initContainer, err := redisInitContainer(cluster, password) if err != nil { return nil, err @@ -434,7 +434,7 @@ func redisVolumes(cluster *redisv1alpha1.DistributedRedisCluster) []corev1.Volum if !cluster.IsRestoreFromBackup() || cluster.Status.Restore.Backup == nil || - cluster.IsRestored() { + !cluster.IsRestoreRunning() { return volumes } From 9bc4ace500784de359a82e79065e2ea09b5f19e3 Mon Sep 17 00:00:00 2001 From: fishu Date: Mon, 23 Mar 2020 23:22:01 +0800 Subject: [PATCH 9/9] reset redis client when redis node updated --- test/e2e/drc/drc_test.go | 4 ++++ test/e2e/drcb/drcb_test.go | 4 ++++ test/e2e/goredis_util.go | 21 +++++++++++++-------- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/test/e2e/drc/drc_test.go b/test/e2e/drc/drc_test.go index 09aa3fca1..5c57f4dae 100644 --- a/test/e2e/drc/drc_test.go +++ b/test/e2e/drc/drc_test.go @@ -39,12 +39,14 @@ var _ = Describe("DistributedRedisCluster CRUD", func() { e2e.DeleteMasterPodForDRC(drc, f.Client) Eventually(e2e.IsDRCPodBeDeleted(f, drc), "5m", "10s").ShouldNot(HaveOccurred()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(drc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) It("should scale up a DistributedRedisCluster", func() { e2e.ScaleUPDRC(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(drc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when the scale up succeeded", func() { @@ -52,6 +54,7 @@ var _ = Describe("DistributedRedisCluster CRUD", func() { e2e.ScaleUPDown(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(drc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) @@ -59,6 +62,7 @@ var _ = Describe("DistributedRedisCluster CRUD", func() { e2e.RollingUpdateDRC(drc) Ω(f.UpdateRedisCluster(drc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, drc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(drc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/drcb/drcb_test.go b/test/e2e/drcb/drcb_test.go index 42c2e2d57..bb5a38e18 100644 --- a/test/e2e/drcb/drcb_test.go +++ b/test/e2e/drcb/drcb_test.go @@ -62,12 +62,14 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func e2e.DeleteMasterPodForDRC(rdrc, f.Client) Eventually(e2e.IsDRCPodBeDeleted(f, rdrc), "5m", "10s").ShouldNot(HaveOccurred()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(rdrc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) It("should scale up a DistributedRedisCluster", func() { e2e.ScaleUPDRC(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(rdrc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) Context("when the scale up succeeded", func() { @@ -75,6 +77,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func e2e.ScaleUPDown(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(rdrc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) @@ -82,6 +85,7 @@ var _ = Describe("Restore DistributedRedisCluster From RedisClusterBackup", func e2e.RollingUpdateDRC(rdrc) Ω(f.UpdateRedisCluster(rdrc)).Should(Succeed()) Eventually(e2e.IsDistributedRedisClusterProperly(f, rdrc), "10m", "10s").ShouldNot(HaveOccurred()) + goredis = e2e.NewGoRedisClient(rdrc.Name, f.Namespace(), goredis.Password()) Expect(e2e.IsDBSizeConsistent(dbsize, goredis)).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/goredis_util.go b/test/e2e/goredis_util.go index e756f58c1..ab5f96aac 100644 --- a/test/e2e/goredis_util.go +++ b/test/e2e/goredis_util.go @@ -19,14 +19,14 @@ type GoRedis struct { func NewGoRedis(addr, password string) *GoRedis { return &GoRedis{ client: redis.NewClusterClient(&redis.ClusterOptions{ - Addrs: []string{addr}, - Password: password, - MaxRetries: 5, - - PoolSize: 3, - MinIdleConns: 1, - PoolTimeout: defaultTimeOut, - IdleTimeout: defaultTimeOut, + Addrs: []string{addr}, + Password: password, + //MaxRetries: 5, + // + //PoolSize: 3, + //MinIdleConns: 1, + //PoolTimeout: defaultTimeOut, + //IdleTimeout: defaultTimeOut, }), password: password, } @@ -61,3 +61,8 @@ func (g *GoRedis) DBSize() (int64, error) { func (g *GoRedis) Password() string { return g.password } + +// Close closes the cluster client. +func (g *GoRedis) Close() error { + return g.client.Close() +}