Skip to content

Commit

Permalink
*: support juicefs #745
Browse files Browse the repository at this point in the history
  • Loading branch information
acekingke committed Jan 12, 2023
1 parent 41b6f3c commit 4f726a9
Show file tree
Hide file tree
Showing 26 changed files with 1,123 additions and 10 deletions.
13 changes: 10 additions & 3 deletions Dockerfile.sidecar
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,20 @@ RUN set -ex; \
ARG XTRABACKUP_PKG=percona-xtrabackup-24
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \
wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
apt-get update; \
apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs
# COPY juicefs/juicefs /usr/local/bin/juicefs
RUN wget --no-check-certificate "https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz" && tar -zxf "juicefs-1.0.2-linux-amd64.tar.gz" ;\
mv juicefs /usr/local/bin/juicefs; \
chmod +x /usr/local/bin/juicefs ; mkdir -p /run/sshd; \
mkdir -p /root/.ssh; \
chmod 700 /root/.ssh
WORKDIR /
COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
ENTRYPOINT ["sidecar"]
COPY script/*.sh /
CMD [ "sidecar" ]
11 changes: 11 additions & 0 deletions api/v1alpha1/backup_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type JuiceOpt struct {
// sqlite or redis
JuiceMeta string `json:"juiceMeta"`
// backupSecrete name for S3
BackupSecretName string `json:"backupSecretName"`
JuiceName string `json:"juiceName"`
}

// This is the backup Job CRD.
// BackupSpec defines the desired state of Backup
type BackupSpec struct {
Expand All @@ -40,6 +48,9 @@ type BackupSpec struct {
// +optional
NFSServerAddress string `json:"nfsServerAddress,omitempty"`

// Represents the juicefs parameters which need.
// +optional
JuiceOpt *JuiceOpt `json:"juiceOpt,omitempty"`
// ClusterName represents the cluster name to backup
ClusterName string `json:"clusterName"`

Expand Down
20 changes: 20 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

90 changes: 90 additions & 0 deletions backup/syncer/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,16 @@ limitations under the License.
package syncer

import (
"context"
"fmt"
"strings"

"github.com/presslabs/controller-util/pkg/syncer"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

v1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
Expand All @@ -33,6 +36,7 @@ import (
)

type jobSyncer struct {
client client.Client
job *batchv1.Job
backup *backup.Backup
}
Expand All @@ -50,6 +54,7 @@ func NewJobSyncer(c client.Client, backup *backup.Backup) syncer.Interface {
}

sync := &jobSyncer{
client: c,
job: obj,
backup: backup,
}
Expand Down Expand Up @@ -174,6 +179,10 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
MountPath: utils.XtrabckupLocal,
},
}
} else if s.backup.Spec.JuiceOpt != nil {
// Deal it for juiceOpt
s.buildJuicefsBackPod(&in)

} else {
// in.Containers[0].ImagePullPolicy = s.opt.ImagePullPolicy
in.Containers[0].Args = []string{
Expand Down Expand Up @@ -238,3 +247,84 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
}
return in
}

func (s *jobSyncer) buildJuicefsBackPod(in *corev1.PodSpec) error {
// add volumn about pvc
var defMode int32 = 0600
var err error
var cmdstr string
in.Volumes = []corev1.Volume{
{
Name: utils.SShVolumnName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: fmt.Sprintf("%s-ssh-key", s.backup.Spec.ClusterName),
DefaultMode: &defMode,
},
},
},
}

in.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: utils.SShVolumnName,
MountPath: utils.SshVolumnPath,
},
}

// PodName.clusterName-mysql.Namespace
// sample-mysql-0.sample-mysql.default
hostname := fmt.Sprintf("%s.%s-mysql.%s", s.backup.Spec.HostName, s.backup.Spec.ClusterName, s.backup.Namespace)
if cmdstr, err = s.buildJuicefsCmd(s.backup.Spec.JuiceOpt.BackupSecretName); err != nil {
return err
}

in.Containers[0].Command = []string{"bash", "-c", "--", `cp /etc/secret-ssh/* /root/.ssh
chmod 600 /root/.ssh/authorized_keys ;` +
strings.Join([]string{
"ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", hostname, cmdstr,
}, " ")}

return nil
}

func (s *jobSyncer) buildJuicefsCmd(secName string) (string, error) {
juiceopt := s.backup.Spec.JuiceOpt
secret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: metav1.ObjectMeta{
Name: secName,
Namespace: s.backup.Namespace,
},
}
err := s.client.Get(context.TODO(),
types.NamespacedName{Namespace: s.backup.Namespace,
Name: secName}, secret)

if err != nil {
return "", err
}
url, bucket := secret.Data["s3-endpoint"], secret.Data["s3-bucket"]
accesskey, secretkey := secret.Data["s3-access-key"], secret.Data["s3-secret-key"]
juicebucket := utils.InstallBucket(string(url), string(bucket))
cmdstr := fmt.Sprintf(`<<EOF
export CLUSTER_NAME=%s
juicefs format --storage s3 \
--bucket %s \
--access-key %s \
--secret-key %s \
%s \
%s`, s.backup.Spec.ClusterName, juicebucket, accesskey, secretkey, juiceopt.JuiceMeta, juiceopt.JuiceName)
cmdstr += fmt.Sprintf(`
juicefs mount -d %s /%s/
`, juiceopt.JuiceMeta, juiceopt.JuiceName)
cmdstr += fmt.Sprintf(`
source /backup.sh
backup
juicefs umount /%s/
EOF`, juiceopt.JuiceName)
return cmdstr, nil
}
16 changes: 16 additions & 0 deletions charts/mysql-operator/crds/mysql.radondb.com_backups.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,22 @@ spec:
default: radondb/mysql57-sidecar:v2.3.0
description: To specify the image that will be used for sidecar container.
type: string
juiceOpt:
description: Represents the juicefs parameters which need.
properties:
backupSecretName:
description: backupSecrete name for S3
type: string
juiceMeta:
description: sqlite or redis
type: string
juiceName:
type: string
required:
- backupSecretName
- juiceMeta
- juiceName
type: object
nfsServerAddress:
description: Represents the ip address of the nfs server.
type: string
Expand Down
16 changes: 16 additions & 0 deletions config/crd/bases/mysql.radondb.com_backups.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,22 @@ spec:
default: radondb/mysql57-sidecar:v2.3.0
description: To specify the image that will be used for sidecar container.
type: string
juiceOpt:
description: Represents the juicefs parameters which need.
properties:
backupSecretName:
description: backupSecrete name for S3
type: string
juiceMeta:
description: sqlite or redis
type: string
juiceName:
type: string
required:
- backupSecretName
- juiceMeta
- juiceName
type: object
nfsServerAddress:
description: Represents the ip address of the nfs server.
type: string
Expand Down
9 changes: 8 additions & 1 deletion controllers/mysqlcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ import (
"github.com/presslabs/controller-util/pkg/syncer"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
policyv1beta1 "k8s.io/api/policy/v1beta1"

// policyv1beta1 "k8s.io/api/policy/v1beta1"
policyv1beta1 "k8s.io/api/policy/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, err
}

secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance)
if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil {
return ctrl.Result{}, err
}

// Todo: modify mysql cm will trigger rolling update but it will not be applied.
cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion
Expand Down
98 changes: 98 additions & 0 deletions docs/en-us/juicefs_backup_and_restore.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
English | [简体中文](../zh-cn/juicefs_backup_and_restore.md) |

目录
=============
* [juiceopt backup]()
* [prerequest](#startup-juiceopt-backup)
* [configuration](#fill-backup-crds-yaml-configs)
* [running backup](#running-backup)

* [restore](#restore)
* [prepare job](#prerequest-for-restore)
* [add configmap](#and-config-map)
* [config mysql cluster](#config-mysql-clusters-yaml)
* [app the cluster restore from backup](#use-kubectl-apply-the-yaml)

# startup juiceopt backup
## prerequire
1. prepare S3 strorage (if you want other types of storage, reference the juicfs document),Obtain the access-key and secret-key . In the example of this article, it use minio ,and the instance is minio-1668754867, and the bucket named `test` so the url is http://test.minio-1668754867.minio:9000/ , you can modify it according your situations, and how to do you can refer to juicefs documents.

2. Install the redis , although juicefs also support sqlite as meta data storage, buf if you do so, you should sync the meta file from s3 at first,and I do not recommend it. redis url is the form as follow:
```
redis://<redis-server-name/IP>:<port>/<NO of database>
```
in the example of this article, redis-server-name is redis-leader, the number of database is 1, So the redis url is `redis://redis-leader:6379/1`

3. Verfiy whether it works: suppose the backup directory is juicefs , you can login in Pod's backup container , execute commanas as follow:

```
juicefs format --storage s3 \
--bucket http://test.minio-1668754867.minio:9000/ \
--access-key <your access key> \
--secret-key <your secrete key> > \
redis://redis-leader:6379/1 \
juicefs
```
then execute :
`juicefs mount -d redis://redis-leader:6379/1 /juicefs`

check whether juicefs is exist, write files, and check S3 storage whether has changed.

## fill backup crd's yaml configs
In backup crd's yaml file, such as in samples/mysql_v1alpha_backup.yaml, add fields information under spec:

```
juiceOpt:
juiceMeta: <fill your redis url>
backupSecretName: <S3's secret name, for example:sample-backup-secret>
juiceName: <juicefs backup directory>
```
for example:
```
juiceOpt:
juiceMeta: "redis://redis-leader:6379/1"
backupSecretName: sample-backup-secret
juiceName: juicefs
```
Others refer to [backup and restore config](./backup_and_restoration_s3.md)

## Running backup.

use command `kubectl apply -f <your backup crd's yaml>` , for examples:

```
kubectl apply -f config/samples/mysql_v1alpha1_backup.yaml
```

# Restore
## prerequest for restore
I suppose that the cluster you want restore is `sample2`
### and `config map`
1. At first give the `config map` a name,name's form is <name of restore cluster>-restore, this article suppose that cluster name is sample2, so `config map`'s name is `sample2-restore`
2. Create config map
* prepare for juiceopt parameters:
build a yaml file, named `juiceopt.yaml`, fill it with:
```
juiceMeta: <redis url>
backupSecretName: <S3's secret name>
juiceName: <backup directory under S3 bucket>
```
for example, in the example of this article, juiceopt.yaml is:
```
juiceMeta: "redis://redis-leader:6379/1"
backupSecretName: sample-backup-secret
juiceName: juicefs
```
* use `kubectl create configmap` create a configmap
configmap has two keys , `from` and `juice.opt` that respectively indicate the cluster has been backuped which we should restore from, and the juice parameter.
but `date` key is optional, it indicates the time where restore to (format is:"2006-01-02 09:07:41"), if it does not have got this key , it will restore to now, use the commands as follows:
`kubectl create configmap sample2-restore --from-literal=from=sample --from-file="juice.opt"=./juiceopt.yaml `


### config mysql cluster's yaml
in the example of this article, we suppose the cluster need to restore is sample2, the config method can refer to [radondb cluster configuration](./deploy_radondb-mysql_operator_on_k8s.md)
### use kubectl apply the yaml
use `kubectl apply ` apply the yaml file, for the example, use the commands as follow:

`kubectl apply -f config/samples/mysql_v1alpha1_mysqlcluster.yaml `

Loading

0 comments on commit 4f726a9

Please sign in to comment.