From 1d2bffc64ef61b73618b5015d1ae1dca78acb35d Mon Sep 17 00:00:00 2001 From: fishu Date: Fri, 29 Nov 2019 14:45:44 +0800 Subject: [PATCH] replace of osm with rclone to do backup and restore --- hack/docker/redis-tools/Dockerfile | 1 + hack/docker/redis-tools/make.sh | 12 ++-- hack/docker/redis-tools/redis-tools.sh | 4 +- pkg/apis/redis/v1alpha1/default.go | 2 +- pkg/controller/manager/ensurer.go | 4 +- .../redisclusterbackup/sync_handler.go | 4 +- pkg/k8sutil/util.go | 10 ++-- pkg/osm/ceph.go | 57 +++++++++++++++++++ pkg/resources/statefulsets/statefulset.go | 4 +- 9 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 pkg/osm/ceph.go diff --git a/hack/docker/redis-tools/Dockerfile b/hack/docker/redis-tools/Dockerfile index 86bfedc09..fd950bbf9 100644 --- a/hack/docker/redis-tools/Dockerfile +++ b/hack/docker/redis-tools/Dockerfile @@ -5,6 +5,7 @@ RUN set -x \ && apt-get install -y --no-install-recommends \ ca-certificates \ netcat \ + zip \ && rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man /tmp/* COPY osm /usr/local/bin/osm diff --git a/hack/docker/redis-tools/make.sh b/hack/docker/redis-tools/make.sh index c8189e73e..680890f59 100644 --- a/hack/docker/redis-tools/make.sh +++ b/hack/docker/redis-tools/make.sh @@ -17,7 +17,7 @@ IMG=redis-tools DB_VERSION=5.0.4 TAG="$DB_VERSION" -OSM_VER=${OSM_VER:-0.9.1} +OSM_VER=${OSM_VER:-v1.50.2} DIST=$REPO_ROOT/dist mkdir -p $DIST @@ -26,15 +26,17 @@ build() { pushd "$REPO_ROOT/hack/docker/redis-tools" if [ ! -f "osm" ]; then - # Download osm - wget https://cdn.appscode.com/binaries/osm/${OSM_VER}/osm-alpine-amd64 - chmod +x osm-alpine-amd64 - mv osm-alpine-amd64 osm + # Download rclone + wget https://downloads.rclone.org/"${OSM_VER}"/rclone-"${OSM_VER}"-linux-amd64.zip + unzip rclone-"${OSM_VER}"-linux-amd64.zip + chmod +x rclone-"${OSM_VER}"-linux-amd64/rclone + mv rclone-"${OSM_VER}"-linux-amd64/rclone osm fi local cmd="docker build --pull -t $DOCKER_REGISTRY/$IMG:$TAG ." echo $cmd; $cmd + rm -rf rclone-"${OSM_VER}"-linux-amd64* rm osm popd } diff --git a/hack/docker/redis-tools/redis-tools.sh b/hack/docker/redis-tools/redis-tools.sh index 1e7b9dfe3..aaae597d2 100644 --- a/hack/docker/redis-tools/redis-tools.sh +++ b/hack/docker/redis-tools/redis-tools.sh @@ -104,7 +104,7 @@ case "$op" in redis-cli --rdb dump.rdb -h "${REDIS_HOST}" -a "${REDIS_PASSWORD}" redis-cli -h "${REDIS_HOST}" -a "${REDIS_PASSWORD}" CLUSTER NODES | grep myself > nodes.conf echo "Uploading dump file to the backend......." - osm push --enable-analytics="$ENABLE_ANALYTICS" --osmconfig="$OSM_CONFIG_FILE" -c "$REDIS_BUCKET" "$REDIS_DATA_DIR" "$REDIS_FOLDER/$REDIS_SNAPSHOT" + osm --config "$OSM_CONFIG_FILE" sync "$REDIS_DATA_DIR" ceph:"$REDIS_BUCKET"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" -v echo "Backup successful" ;; @@ -116,7 +116,7 @@ case "$op" in fi index=$(echo "${POD_NAME}" | awk -F- '{print $NF}') REDIS_SNAPSHOT=${REDIS_SNAPSHOT}-${index} - osm pull --enable-analytics="$ENABLE_ANALYTICS" --osmconfig="$OSM_CONFIG_FILE" -c "$REDIS_BUCKET" "$REDIS_FOLDER/$REDIS_SNAPSHOT" "$REDIS_DATA_DIR" + osm --config "$OSM_CONFIG_FILE" sync ceph:"$REDIS_BUCKET"/"$REDIS_FOLDER/$REDIS_SNAPSHOT" "$REDIS_DATA_DIR" -v echo "Recovery successful" ;; diff --git a/pkg/apis/redis/v1alpha1/default.go b/pkg/apis/redis/v1alpha1/default.go index 2f3d577c2..9fa55e467 100644 --- a/pkg/apis/redis/v1alpha1/default.go +++ b/pkg/apis/redis/v1alpha1/default.go @@ -114,7 +114,7 @@ func (in *RedisClusterBackup) Location() (string, error) { } func (in *RedisClusterBackup) OSMSecretName() string { - return fmt.Sprintf("osm-%v", in.Name) + return fmt.Sprintf("osmconfig-%v", in.Name) } func (in *RedisClusterBackup) JobName() string { diff --git a/pkg/controller/manager/ensurer.go b/pkg/controller/manager/ensurer.go index 358b7d640..b160bdf1d 100644 --- a/pkg/controller/manager/ensurer.go +++ b/pkg/controller/manager/ensurer.go @@ -138,11 +138,11 @@ func (r *realEnsureResource) EnsureRedisOSMSecret(cluster *redisv1alpha1.Distrib if cluster.Spec.Init == nil || cluster.Status.RestoreSucceeded > 0 { return nil } - secret, err := osm.NewOSMSecret(r.client, k8sutil.OSMSecretName(backup.Name), backup.Namespace, backup.Spec.Backend) + secret, err := osm.NewCephSecret(r.client, backup.OSMSecretName(), cluster.Namespace, backup.Spec.Backend) if err != nil { return err } - if err := k8sutil.CreateSecret(r.client, secret); err != nil { + if err := k8sutil.CreateSecret(r.client, secret, r.logger); err != nil { return err } return nil diff --git a/pkg/controller/redisclusterbackup/sync_handler.go b/pkg/controller/redisclusterbackup/sync_handler.go index cd2ccf658..b5e29d1f7 100644 --- a/pkg/controller/redisclusterbackup/sync_handler.go +++ b/pkg/controller/redisclusterbackup/sync_handler.go @@ -103,7 +103,7 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi return err } - secret, err := osm.NewOSMSecret(r.client, k8sutil.OSMSecretName(backup.Name), backup.Namespace, backup.Spec.Backend) + secret, err := osm.NewCephSecret(r.client, backup.OSMSecretName(), backup.Namespace, backup.Spec.Backend) if err != nil { msg := fmt.Sprintf("Failed to generate osm secret. Reason: %v", err) r.markAsFailedBackup(backup, msg) @@ -116,7 +116,7 @@ func (r *ReconcileRedisClusterBackup) create(reqLogger logr.Logger, backup *redi return nil // don't retry } - if err := k8sutil.CreateSecret(r.client, secret); err != nil { + if err := k8sutil.CreateSecret(r.client, secret, reqLogger); err != nil { r.recorder.Event( backup, corev1.EventTypeWarning, diff --git a/pkg/k8sutil/util.go b/pkg/k8sutil/util.go index 6ba2be74f..6f038f145 100644 --- a/pkg/k8sutil/util.go +++ b/pkg/k8sutil/util.go @@ -2,8 +2,8 @@ package k8sutil import ( "context" - "fmt" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" kerr "k8s.io/apimachinery/pkg/api/errors" @@ -18,11 +18,7 @@ func IsRequestRetryable(err error) bool { kerr.IsTooManyRequests(err) } -func OSMSecretName(name string) string { - return fmt.Sprintf("osm-%v", name) -} - -func CreateSecret(client client.Client, secret *corev1.Secret) error { +func CreateSecret(client client.Client, secret *corev1.Secret, logger logr.Logger) error { ctx := context.TODO() s := &corev1.Secret{} err := client.Get(ctx, types.NamespacedName{ @@ -31,6 +27,8 @@ func CreateSecret(client client.Client, secret *corev1.Secret) error { }, s) if err != nil { if errors.IsNotFound(err) { + logger.WithValues("Secret.Namespace", secret.Namespace, "Secret.Name", secret.Name). + Info("creating a new secret") return client.Create(ctx, secret) } } diff --git a/pkg/osm/ceph.go b/pkg/osm/ceph.go new file mode 100644 index 000000000..d2e332529 --- /dev/null +++ b/pkg/osm/ceph.go @@ -0,0 +1,57 @@ +package osm + +import ( + "context" + "fmt" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ktypes "k8s.io/apimachinery/pkg/types" + api "kmodules.xyz/objectstore-api/api/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func NewCephSecret(kc client.Client, name, namespace string, spec api.Backend) (*core.Secret, error) { + if spec.S3 == nil { + return nil, fmt.Errorf("only suport ceph s3") + } + + config := make(map[string][]byte) + if spec.StorageSecretName != "" { + secret := &core.Secret{} + err := kc.Get(context.TODO(), ktypes.NamespacedName{ + Name: spec.StorageSecretName, + Namespace: namespace, + }, secret) + if err != nil { + return nil, err + } + config = secret.Data + } + + keyID := config[api.AWS_ACCESS_KEY_ID] + key := config[api.AWS_SECRET_ACCESS_KEY] + + osmBytes := fmt.Sprintf(`[ceph] +type = s3 +provider = Ceph +env_auth = false +access_key_id = %s +secret_access_key = %s +region = +endpoint = %s +location_constraint = +acl = +server_side_encryption = +storage_class =`, keyID, key, spec.S3.Endpoint) + out := &core.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + "config": []byte(osmBytes), + }, + } + return out, nil +} diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 258370e89..020a47fdc 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -235,7 +235,7 @@ func redisExporterContainer(cluster *redisv1alpha1.DistributedRedisCluster, pass fmt.Sprintf("--web.telemetry-path=%v", redisv1alpha1.PrometheusExporterTelemetryPath), }, cluster.Spec.Monitor.Args...), Image: cluster.Spec.Monitor.Image, - ImagePullPolicy: corev1.PullIfNotPresent, + ImagePullPolicy: corev1.PullAlways, Ports: []corev1.ContainerPort{ { Name: "prom-http", @@ -266,7 +266,7 @@ func redisInitContainer(cluster *redisv1alpha1.DistributedRedisCluster, backup * container := corev1.Container{ Name: redisv1alpha1.JobTypeRestore, Image: backup.Spec.Image, - ImagePullPolicy: "Always", + ImagePullPolicy: corev1.PullAlways, Args: []string{ redisv1alpha1.JobTypeRestore, fmt.Sprintf(`--data-dir=%s`, redisv1alpha1.BackupDumpDir),