Migrated blob and file code to Track 2 (#2242)

ChangeLog.md

@@ -1,6 +1,19 @@
 
 # Change Log
 
+## Version 10.21.0-Preview
+
+### New Features
+
+1. Migrated to the latest [azblob SDK](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob).
+2. Migrated to the latest [azfile SDK](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azfile).
+3. Migrated from deprecated ADAL to MSAL through the latest [azidentity SDK](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity).
+4. Deprecated support for object IDs in MSI. Client ID or Resource ID can be used as an alternative.
+
+### Special notes
+
+1. Due to the migration from ADAL to MSAL, tenant ID must now be set when authorizing with single tenant applications created after 10/15/2018.
+
 ## Version 10.20.1
 
 ### Bug Fixes
@@ -28,6 +41,7 @@
 7. Fixed an issue where `--skip-version-check` would not be honored for `login`,` logout`, `help` commands. [#2299](https://github.com/Azure/azure-storage-azcopy/issues/2299)
 
 ### Documentation
+
 1. Add a log for LMTs when a mismatch is encountered.
 2. Added documentation indicating the `login` and `logout` commands will be deprecated in the future.
 

cmd/benchmark.go

@@ -23,15 +23,15 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	sharefile "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
 
 	"github.com/Azure/azure-storage-azcopy/v10/azbfs"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
-	"github.com/Azure/azure-storage-blob-go/azblob"
-	"github.com/Azure/azure-storage-file-go/azfile"
 	"github.com/spf13/cobra"
 )
 
@@ -176,43 +176,47 @@ func (raw rawBenchmarkCmdArgs) cook() (CookedCopyCmdArgs, error) {
 }
 
 func (raw rawBenchmarkCmdArgs) appendVirtualDir(target, virtualDir string) (string, error) {
-
-	u, err := url.Parse(target)
-	if err != nil {
-		return "", fmt.Errorf("error parsing the url %s. Failed with error %s", target, err.Error())
-	}
-
-	var result url.URL
-
 	switch InferArgumentLocation(target) {
 	case common.ELocation.Blob():
-		p := azblob.NewBlobURLParts(*u)
+		p, err := blob.ParseURL(target)
+		if err != nil {
+			return "", fmt.Errorf("error parsing the url %s. Failed with error %s", target, err.Error())
+		}
 		if p.ContainerName == "" || p.BlobName != "" {
 			return "", errors.New("the blob target must be a container")
 		}
 		p.BlobName = virtualDir
-		result = p.URL()
+		return p.String(), err
 
 	case common.ELocation.File():
-		p := azfile.NewFileURLParts(*u)
+		p, err := sharefile.ParseURL(target)
+		if err != nil {
+			return "", fmt.Errorf("error parsing the url %s. Failed with error %s", target, err.Error())
+		}
 		if p.ShareName == "" || p.DirectoryOrFilePath != "" {
-			return "", errors.New("the Azure Files target must be a file share root")
+			return "", errors.New("the file share target must be a file share root")
 		}
 		p.DirectoryOrFilePath = virtualDir
-		result = p.URL()
+		return p.String(), err
 
 	case common.ELocation.BlobFS():
+		u, err := url.Parse(target)
+		if err != nil {
+			return "", fmt.Errorf("error parsing the url %s. Failed with error %s", target, err.Error())
+		}
+
+		var result url.URL
 		p := azbfs.NewBfsURLParts(*u)
 		if p.FileSystemName == "" || p.DirectoryOrFilePath != "" {
 			return "", errors.New("the blobFS target must be a file system")
 		}
 		p.DirectoryOrFilePath = virtualDir
 		result = p.URL()
+		return result.String(), nil
 	default:
 		return "", errors.New("benchmarking only supports https connections to Blob, Azure Files, and ADLS Gen2")
 	}
 
-	return result.String(), nil
 }
 
 // define a cleanup job

cmd/copy.go

@@ -26,6 +26,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"io"
 	"math"
 	"net/url"
@@ -39,7 +41,6 @@ import (
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/spf13/cobra"
 
 	"github.com/Azure/azure-storage-azcopy/v10/common"
@@ -837,7 +838,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 			if err != nil {
 				return cooked, fmt.Errorf("error parsing the exclude-blob-type %s provided with exclude-blob-type flag ", blobType)
 			}
-			cooked.excludeBlobType = append(cooked.excludeBlobType, eBlobType.ToAzBlobType())
+			cooked.excludeBlobType = append(cooked.excludeBlobType, eBlobType.ToBlobType())
 		}
 	}
 
@@ -947,7 +948,7 @@ func validatePreserveSMBPropertyOption(toPreserve bool, fromTo common.FromTo, ov
 	} else if toPreserve && !(fromTo == common.EFromTo.LocalFile() ||
 		fromTo == common.EFromTo.FileLocal() ||
 		fromTo == common.EFromTo.FileFile()) {
-		return fmt.Errorf("%s is set but the job is not between %s-aware resources", flagName, common.IffString(flagName == PreservePermissionsFlag, "permission", "SMB"))
+		return fmt.Errorf("%s is set but the job is not between %s-aware resources", flagName, common.Iff(flagName == PreservePermissionsFlag, "permission", "SMB"))
 	}
 
 	if toPreserve && (fromTo.IsUpload() || fromTo.IsDownload()) &&
@@ -1116,7 +1117,7 @@ type CookedCopyCmdArgs struct {
 	// options from flags
 	blockSize int64
 	// list of blobTypes to exclude while enumerating the transfer
-	excludeBlobType []azblob.BlobType
+	excludeBlobType []blob.BlobType
 	blobType        common.BlobType
 	// Blob index tags categorize data in your storage account utilizing key-value tag attributes.
 	// These tags are automatically indexed and exposed as a queryable multi-dimensional index to easily find data.
@@ -1281,11 +1282,8 @@ func (cca *CookedCopyCmdArgs) processRedirectionDownload(blobResource common.Res
 		return fmt.Errorf("fatal: cannot find auth on source blob URL: %s", err.Error())
 	}
 
-	// step 1: initialize pipeline
-	p, err := createBlobPipeline(ctx, credInfo, pipeline.LogNone)
-	if err != nil {
-		return err
-	}
+	// step 1: create client options
+	options := createClientOptions(pipeline.LogNone, nil, nil)
 
 	// step 2: parse source url
 	u, err := blobResource.FullURL()
@@ -1294,17 +1292,17 @@ func (cca *CookedCopyCmdArgs) processRedirectionDownload(blobResource common.Res
 	}
 
 	// step 3: start download
-	blobURL := azblob.NewBlobURL(*u, p)
-	clientProvidedKey := azblob.ClientProvidedKeyOptions{}
-	if cca.CpkOptions.IsSourceEncrypted {
-		clientProvidedKey = common.GetClientProvidedKey(cca.CpkOptions)
-	}
-	blobStream, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, clientProvidedKey)
+	blobClient := common.CreateBlobClient(u.String(), credInfo, nil, options)
+
+	blobStream, err := blobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{
+		CPKInfo:      cca.CpkOptions.GetCPKInfo(),
+		CPKScopeInfo: cca.CpkOptions.GetCPKScopeInfo(),
+	})
 	if err != nil {
 		return fmt.Errorf("fatal: cannot download blob due to error: %s", err.Error())
 	}
 
-	blobBody := blobStream.Body(azblob.RetryReaderOptions{MaxRetryRequests: ste.MaxRetryPerDownloadBody, ClientProvidedKeyOptions: clientProvidedKey})
+	blobBody := blobStream.NewRetryReader(ctx, &blob.RetryReaderOptions{MaxRetries: ste.MaxRetryPerDownloadBody})
 	defer blobBody.Close()
 
 	// step 4: pipe everything into Stdout
@@ -1328,14 +1326,11 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
 	credInfo, _, err := GetCredentialInfoForLocation(ctx, common.ELocation.Blob(), blobResource.Value, blobResource.SAS, false, cca.CpkOptions)
 
 	if err != nil {
-		return fmt.Errorf("fatal: cannot find auth on source blob URL: %s", err.Error())
+		return fmt.Errorf("fatal: cannot find auth on destination blob URL: %s", err.Error())
 	}
 
 	// step 0: initialize pipeline
-	p, err := createBlobPipeline(ctx, credInfo, pipeline.LogNone)
-	if err != nil {
-		return err
-	}
+	options := createClientOptions(pipeline.LogNone, nil, nil)
 
 	// step 1: parse destination url
 	u, err := blobResource.FullURL()
@@ -1344,34 +1339,36 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
 	}
 
 	// step 2: leverage high-level call in Blob SDK to upload stdin in parallel
-	blockBlobUrl := azblob.NewBlockBlobURL(*u, p)
+	blockBlobClient := common.CreateBlockBlobClient(u.String(), credInfo, nil, options)
+
 	metadataString := cca.metadata
 	metadataMap := common.Metadata{}
 	if len(metadataString) > 0 {
 		for _, keyAndValue := range strings.Split(metadataString, ";") { // key/value pairs are separated by ';'
 			kv := strings.Split(keyAndValue, "=") // key/value are separated by '='
-			metadataMap[kv[0]] = kv[1]
+			metadataMap[kv[0]] = &kv[1]
 		}
 	}
 	blobTags := cca.blobTags
-	bbAccessTier := azblob.DefaultAccessTier
+	bbAccessTier := blob.AccessTier("")
 	if cca.blockBlobTier != common.EBlockBlobTier.None() {
-		bbAccessTier = azblob.AccessTierType(cca.blockBlobTier.String())
-	}
-	_, err = azblob.UploadStreamToBlockBlob(ctx, os.Stdin, blockBlobUrl, azblob.UploadStreamToBlockBlobOptions{
-		BufferSize:  int(blockSize),
-		MaxBuffers:  pipingUploadParallelism,
-		Metadata:    metadataMap.ToAzBlobMetadata(),
-		BlobTagsMap: blobTags.ToAzBlobTagsMap(),
-		BlobHTTPHeaders: azblob.BlobHTTPHeaders{
-			ContentType:        cca.contentType,
-			ContentLanguage:    cca.contentLanguage,
-			ContentEncoding:    cca.contentEncoding,
-			ContentDisposition: cca.contentDisposition,
-			CacheControl:       cca.cacheControl,
+		bbAccessTier = blob.AccessTier(cca.blockBlobTier.String())
+	}
+	_, err = blockBlobClient.UploadStream(ctx, os.Stdin, &blockblob.UploadStreamOptions{
+		BlockSize:   blockSize,
+		Concurrency: pipingUploadParallelism,
+		Metadata:    metadataMap,
+		Tags:        blobTags,
+		HTTPHeaders: &blob.HTTPHeaders{
+			BlobContentType:        &cca.contentType,
+			BlobContentLanguage:    &cca.contentLanguage,
+			BlobContentEncoding:    &cca.contentEncoding,
+			BlobContentDisposition: &cca.contentDisposition,
+			BlobCacheControl:       &cca.cacheControl,
 		},
-		BlobAccessTier:           bbAccessTier,
-		ClientProvidedKeyOptions: common.GetClientProvidedKey(cca.CpkOptions),
+		AccessTier:   &bbAccessTier,
+		CPKInfo:      cca.CpkOptions.GetCPKInfo(),
+		CPKScopeInfo: cca.CpkOptions.GetCPKScopeInfo(),
 	})
 
 	return err
@@ -1407,13 +1404,12 @@ func (cca *CookedCopyCmdArgs) getSrcCredential(ctx context.Context, jpo *common.
 				cca.credentialInfo.OAuthTokenInfo = *tokenInfo
 				jpo.CredentialInfo.OAuthTokenInfo = *tokenInfo
 			}
+			jpo.CredentialInfo.S2SSourceTokenCredential, err = common.GetSourceBlobCredential(srcCredInfo, common.CredentialOpOptions{LogError: glcm.Info})
+			if err != nil {
+				return srcCredInfo, err
+			}
 			// if the source is not local then store the credential token if it was OAuth to avoid constant refreshing
-			jpo.CredentialInfo.SourceBlobToken = common.CreateBlobCredential(ctx, srcCredInfo, common.CredentialOpOptions{
-				// LogInfo:  glcm.Info, //Comment out for debugging
-				LogError: glcm.Info,
-			})
-			cca.credentialInfo.SourceBlobToken = jpo.CredentialInfo.SourceBlobToken
-			srcCredInfo.SourceBlobToken = jpo.CredentialInfo.SourceBlobToken
+			cca.credentialInfo.S2SSourceTokenCredential = jpo.CredentialInfo.S2SSourceTokenCredential
 		}
 	}
 	return srcCredInfo, nil
@@ -1685,7 +1681,7 @@ func (cca *CookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) (tot
 		cca.intervalStartTime = time.Now()
 		cca.intervalBytesTransferred = summary.BytesOverWire
 
-		return common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8
+		return common.Iff(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8
 	}
 	glcm.Progress(func(format common.OutputFormat) string {
 		if format == common.EOutputFormat.Json() {
@@ -2054,4 +2050,4 @@ func init() {
 	// Deprecate the old persist-smb-permissions flag
 	_ = cpCmd.PersistentFlags().MarkHidden("preserve-smb-permissions")
 	cpCmd.PersistentFlags().BoolVar(&raw.preservePermissions, PreservePermissionsFlag, false, "False by default. Preserves ACLs between aware resources (Windows and Azure Files, or ADLS Gen 2 to ADLS Gen 2). For Hierarchical Namespace accounts, you will need a container SAS or OAuth token with Modify Ownership and Modify Permissions permissions. For downloads, you will also need the --backup flag to restore permissions where the new Owner will not be the user running AzCopy. This flag applies to both files and folders, unless a file-only filter is specified (e.g. include-pattern).")
-}
+}