From b42224518443d4134e5b7509cf7c6659e1441adf Mon Sep 17 00:00:00 2001 From: Derek Su Date: Sun, 13 Aug 2023 17:07:59 +0800 Subject: [PATCH] backup: move common functions and types to backupstore lib Longhorn 6138 Signed-off-by: Derek Su --- app/cmd/backup.go | 5 +- pkg/backup/main.go | 4 +- pkg/replica/backup.go | 10 +-- pkg/sync/rpc/server.go | 11 +-- pkg/util/credential.go | 159 ----------------------------------------- 5 files changed, 16 insertions(+), 173 deletions(-) delete mode 100644 pkg/util/credential.go diff --git a/app/cmd/backup.go b/app/cmd/backup.go index 676c3c41b..92e254f4e 100644 --- a/app/cmd/backup.go +++ b/app/cmd/backup.go @@ -10,6 +10,7 @@ import ( "github.com/urfave/cli" "github.com/longhorn/backupstore/cmd" + butil "github.com/longhorn/backupstore/util" replicaClient "github.com/longhorn/longhorn-engine/pkg/replica/client" "github.com/longhorn/longhorn-engine/pkg/sync" @@ -262,7 +263,7 @@ func createBackup(c *cli.Context) error { } } - credential, err := util.GetBackupCredential(dest) + credential, err := butil.GetBackupCredential(dest) if err != nil { return err } @@ -308,7 +309,7 @@ func restoreBackup(c *cli.Context) error { } backupURL := util.UnescapeURL(backup) - credential, err := util.GetBackupCredential(backup) + credential, err := butil.GetBackupCredential(backup) if err != nil { return err } diff --git a/pkg/backup/main.go b/pkg/backup/main.go index 8d3dcce9a..ef97447cf 100644 --- a/pkg/backup/main.go +++ b/pkg/backup/main.go @@ -31,7 +31,7 @@ type CreateBackupParameters struct { BackingImageChecksum string CompressionMethod string ConcurrentLimit int32 - StorageClassname string + StorageClassName string Labels []string } @@ -110,7 +110,7 @@ func DoBackupInit(params *CreateBackupParameters) (*replica.BackupStatus, *backu BackingImageName: params.BackingImageName, BackingImageChecksum: params.BackingImageChecksum, CompressionMethod: params.CompressionMethod, - StorageClassName: params.StorageClassname, + StorageClassName: params.StorageClassName, CreatedTime: util.Now(), } diff --git a/pkg/replica/backup.go b/pkg/replica/backup.go index 27749c34c..d08a9c29b 100644 --- a/pkg/replica/backup.go +++ b/pkg/replica/backup.go @@ -8,7 +8,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/longhorn/backupstore" + btypes "github.com/longhorn/backupstore/types" butil "github.com/longhorn/backupstore/util" "github.com/longhorn/longhorn-engine/pkg/backingfile" @@ -288,7 +288,7 @@ func (rb *BackupStatus) CloseSnapshot(snapID, volumeID string) error { return err } -func (rb *BackupStatus) CompareSnapshot(snapID, compareSnapID, volumeID string) (*backupstore.Mappings, error) { +func (rb *BackupStatus) CompareSnapshot(snapID, compareSnapID, volumeID string) (*btypes.Mappings, error) { id := diskutil.GenerateSnapshotDiskName(snapID) compareID := "" if compareSnapID != "" { @@ -314,10 +314,10 @@ func (rb *BackupStatus) CompareSnapshot(snapID, compareSnapID, volumeID string) return nil, fmt.Errorf("failed to find snapshot %s in chain", compareID) } - mappings := &backupstore.Mappings{ + mappings := &btypes.Mappings{ BlockSize: snapBlockSize, } - mapping := backupstore.Mapping{ + mapping := btypes.Mapping{ Offset: -1, } @@ -331,7 +331,7 @@ func (rb *BackupStatus) CompareSnapshot(snapID, compareSnapID, volumeID string) // align offset -= (offset % snapBlockSize) if mapping.Offset != offset { - mapping = backupstore.Mapping{ + mapping = btypes.Mapping{ Offset: offset, Size: snapBlockSize, } diff --git a/pkg/sync/rpc/server.go b/pkg/sync/rpc/server.go index b13bba886..75500ca9b 100644 --- a/pkg/sync/rpc/server.go +++ b/pkg/sync/rpc/server.go @@ -22,6 +22,7 @@ import ( "google.golang.org/grpc/reflection" "github.com/longhorn/backupstore" + butil "github.com/longhorn/backupstore/util" "github.com/longhorn/sparse-tools/sparse" sparserest "github.com/longhorn/sparse-tools/sparse/rest" @@ -681,12 +682,12 @@ func (s *SyncAgentServer) SnapshotCloneStatus(ctx context.Context, req *empty.Em } func (s *SyncAgentServer) BackupCreate(ctx context.Context, req *ptypes.BackupCreateRequest) (*ptypes.BackupCreateResponse, error) { - backupType, err := util.CheckBackupType(req.BackupTarget) + backupType, err := butil.CheckBackupType(req.BackupTarget) if err != nil { return nil, err } - if err := util.SetupCredential(backupType, req.Credential); err != nil { + if err := butil.SetupCredential(backupType, req.Credential); err != nil { return nil, err } @@ -706,7 +707,7 @@ func (s *SyncAgentServer) BackupCreate(ctx context.Context, req *ptypes.BackupCr BackingImageChecksum: req.BackingImageChecksum, CompressionMethod: req.CompressionMethod, ConcurrentLimit: req.ConcurrentLimit, - StorageClassname: req.StorageClassName, + StorageClassName: req.StorageClassName, Labels: req.Labels, }) if err != nil { @@ -815,12 +816,12 @@ func (s *SyncAgentServer) BackupRestore(ctx context.Context, req *ptypes.BackupR return nil, fmt.Errorf("empty backup URL for the restore") } - backupType, err := util.CheckBackupType(req.Backup) + backupType, err := butil.CheckBackupType(req.Backup) if err != nil { return nil, errors.Wrapf(err, "failed to check the type for backup %v", req.Backup) } - if err := util.SetupCredential(backupType, req.Credential); err != nil { + if err := butil.SetupCredential(backupType, req.Credential); err != nil { return nil, errors.Wrapf(err, "failed to setup credential for backup %v", req.Backup) } diff --git a/pkg/util/credential.go b/pkg/util/credential.go deleted file mode 100644 index c04f4c17d..000000000 --- a/pkg/util/credential.go +++ /dev/null @@ -1,159 +0,0 @@ -package util - -import ( - "errors" - "os" - - "github.com/longhorn/longhorn-engine/pkg/types" -) - -func SetupCredential(backupType string, credential map[string]string) error { - switch backupType { - case "s3": - return setupS3Credential(credential) - case "cifs": - return setupCIFSCredential(credential) - case "azblob": - return setupAZBlobCredential(credential) - default: - return nil - } -} - -func setupS3Credential(credential map[string]string) error { - if credential == nil { - return nil - } - - if credential[types.AWSAccessKey] == "" && credential[types.AWSSecretKey] != "" { - return errors.New("s3 credential access key not found") - } - if credential[types.AWSAccessKey] != "" && credential[types.AWSSecretKey] == "" { - return errors.New("s3 credential secret access key not found") - } - if credential[types.AWSAccessKey] != "" && credential[types.AWSSecretKey] != "" { - os.Setenv(types.AWSAccessKey, credential[types.AWSAccessKey]) - os.Setenv(types.AWSSecretKey, credential[types.AWSSecretKey]) - } - - os.Setenv(types.AWSEndPoint, credential[types.AWSEndPoint]) - os.Setenv(types.HTTPSProxy, credential[types.HTTPSProxy]) - os.Setenv(types.HTTPProxy, credential[types.HTTPProxy]) - os.Setenv(types.NOProxy, credential[types.NOProxy]) - os.Setenv(types.VirtualHostedStyle, credential[types.VirtualHostedStyle]) - - // set a custom ca cert if available - if credential[types.AWSCert] != "" { - os.Setenv(types.AWSCert, credential[types.AWSCert]) - } - - return nil -} - -func setupCIFSCredential(credential map[string]string) error { - if credential == nil { - return nil - } - - os.Setenv(types.CIFSUsername, credential[types.CIFSUsername]) - os.Setenv(types.CIFSPassword, credential[types.CIFSPassword]) - - return nil -} - -func setupAZBlobCredential(credential map[string]string) error { - if credential == nil { - return nil - } - - if credential[types.AZBlobAccountName] == "" && credential[types.AZBlobAccountKey] != "" { - return errors.New("Azure Blob Storage credential account name not found") - } - if credential[types.AZBlobAccountName] != "" && credential[types.AZBlobAccountKey] == "" { - return errors.New("Azure Blob Storage credential account key not found") - } - - os.Setenv(types.AZBlobAccountName, credential[types.AZBlobAccountName]) - os.Setenv(types.AZBlobAccountKey, credential[types.AZBlobAccountKey]) - os.Setenv(types.AZBlobEndpoint, credential[types.AZBlobEndpoint]) - os.Setenv(types.HTTPSProxy, credential[types.HTTPSProxy]) - os.Setenv(types.HTTPProxy, credential[types.HTTPProxy]) - os.Setenv(types.NOProxy, credential[types.NOProxy]) - - if credential[types.AZBlobCert] != "" { - os.Setenv(types.AZBlobCert, credential[types.AZBlobCert]) - } - - return nil -} - -func getCredentialFromEnvVars(backupType string) (map[string]string, error) { - switch backupType { - case "s3": - return getS3CredentialFromEnvVars() - case "cifs": - return getCIFSCredentialFromEnvVars() - case "azblob": - return getAZBlobCredentialFromEnvVars() - default: - return nil, nil - } -} - -func getAZBlobCredentialFromEnvVars() (map[string]string, error) { - credential := map[string]string{} - - credential[types.AZBlobAccountName] = os.Getenv(types.AZBlobAccountName) - credential[types.AZBlobAccountKey] = os.Getenv(types.AZBlobAccountKey) - credential[types.AZBlobEndpoint] = os.Getenv(types.AZBlobEndpoint) - credential[types.AZBlobCert] = os.Getenv(types.AZBlobCert) - credential[types.HTTPSProxy] = os.Getenv(types.HTTPSProxy) - credential[types.HTTPProxy] = os.Getenv(types.HTTPProxy) - credential[types.NOProxy] = os.Getenv(types.NOProxy) - - return credential, nil -} - -func getCIFSCredentialFromEnvVars() (map[string]string, error) { - credential := map[string]string{} - - credential[types.CIFSUsername] = os.Getenv(types.CIFSUsername) - credential[types.CIFSPassword] = os.Getenv(types.CIFSPassword) - - return credential, nil -} - -func getS3CredentialFromEnvVars() (map[string]string, error) { - credential := map[string]string{} - - accessKey := os.Getenv(types.AWSAccessKey) - secretKey := os.Getenv(types.AWSSecretKey) - if accessKey == "" && secretKey != "" { - return nil, errors.New("s3 credential access key not found") - } - if accessKey != "" && secretKey == "" { - return nil, errors.New("s3 credential secret access key not found") - } - if accessKey != "" && secretKey != "" { - credential[types.AWSAccessKey] = accessKey - credential[types.AWSSecretKey] = secretKey - } - - credential[types.AWSEndPoint] = os.Getenv(types.AWSEndPoint) - credential[types.AWSCert] = os.Getenv(types.AWSCert) - credential[types.HTTPSProxy] = os.Getenv(types.HTTPSProxy) - credential[types.HTTPProxy] = os.Getenv(types.HTTPProxy) - credential[types.NOProxy] = os.Getenv(types.NOProxy) - credential[types.VirtualHostedStyle] = os.Getenv(types.VirtualHostedStyle) - - return credential, nil -} - -func GetBackupCredential(backupURL string) (map[string]string, error) { - backupType, err := CheckBackupType(backupURL) - if err != nil { - return nil, err - } - - return getCredentialFromEnvVars(backupType) -}