Skip to content

Commit

Permalink
Merge pull request #106 from flownative/feature/cluster-flag
Browse files Browse the repository at this point in the history 
Allow to specify cluster for resource download/upload
  • Loading branch information
@kdambekalns
kdambekalns authored Jan 9, 2025
2 parents 6087373 + 5fb20b1 commit 91db934
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 25 deletions.
12 changes: 10 additions & 2 deletions cmd/beach/cmd/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ import (
asset "github.com/flownative/localbeach/assets"
)

var instanceIdentifier, projectNamespace, clusterIdentifier string

func copyFileFromAssets(src, dst string) (int64, error) {
source, err := asset.Assets.Open(src)
if err != nil {
Expand Down Expand Up @@ -90,12 +92,18 @@ func getRelativePersistentResourcePathByHash(hash string) string {
}
}

func retrieveCloudStorageCredentials(instanceIdentifier string, projectNamespace string) (err error, bucketName string, privateKey []byte) {
func retrieveCloudStorageCredentials(instanceIdentifier string, projectNamespace string, clusterIdentifier string) (err error, bucketName string, privateKey []byte) {
log.Info("Retrieving cloud storage access data from instance")

internalHost := "beach@" + instanceIdentifier + "." + projectNamespace
jumpHost := ""
if clusterIdentifier != "" {
jumpHost = "beach@ssh." + clusterIdentifier + ".flownative.cloud"
} else {
jumpHost = "[email protected]"
}
output, err := exec.RunCommand("ssh", []string{
"-J", "[email protected]", internalHost,
"-J", jumpHost, internalHost,
"/bin/bash", "-c", "env | grep BEACH_GOOGLE_CLOUD_STORAGE_",
})
if err != nil {
Expand Down
27 changes: 15 additions & 12 deletions cmd/beach/cmd/resource-download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import (
"path/filepath"
)

var sourceBucketName, targetResourcesPath string

// resourceDownloadCmd represents the resource-download command
var resourceDownloadCmd = &cobra.Command{
Use: "resource-download",
Expand Down Expand Up @@ -59,8 +61,9 @@ Notes:
func init() {
resourceDownloadCmd.Flags().StringVar(&instanceIdentifier, "instance", "", "instance identifier of the Beach instance to download from, eg. 'instance-123abc45-def6-7890-abcd-1234567890ab'")
resourceDownloadCmd.Flags().StringVar(&projectNamespace, "namespace", "", "The project namespace of the Beach instance to download from, eg. 'beach-project-123abc45-def6-7890-abcd-1234567890ab'")
resourceDownloadCmd.Flags().StringVar(&bucketName, "bucket", "", "name of the bucket to download resources from")
resourceDownloadCmd.Flags().StringVar(&resourcesPath, "resources-path", "", "custom path where to store the downloaded resources, e.g. 'Data/Persistent/Protected'")
resourceDownloadCmd.Flags().StringVar(&clusterIdentifier, "cluster", "", "The cluster identifier of the Beach instance to download from, eg. 'h9acc4'")
resourceDownloadCmd.Flags().StringVar(&sourceBucketName, "bucket", "", "name of the bucket to download resources from")
resourceDownloadCmd.Flags().StringVar(&targetResourcesPath, "resources-path", "", "custom path where to store the downloaded resources, e.g. 'Data/Persistent/Protected'")
_ = resourceDownloadCmd.MarkFlagRequired("instance")
_ = resourceDownloadCmd.MarkFlagRequired("namespace")
rootCmd.AddCommand(resourceDownloadCmd)
Expand All @@ -73,24 +76,24 @@ func handleResourceDownloadRun(cmd *cobra.Command, args []string) {
return
}

if resourcesPath == "" {
resourcesPath = sandbox.ProjectDataPersistentResourcesPath
if targetResourcesPath == "" {
targetResourcesPath = sandbox.ProjectDataPersistentResourcesPath
}

_, err = os.Stat(resourcesPath)
_, err = os.Stat(targetResourcesPath)
if err != nil {
log.Fatal(fmt.Sprintf("The path %v does not exist", resourcesPath))
log.Fatal(fmt.Sprintf("The path %v does not exist", targetResourcesPath))
return
}

err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace)
err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace, clusterIdentifier)
if err != nil {
log.Fatal(err)
return
}

if bucketName == "" {
bucketName = bucketNameFromCredentials
if sourceBucketName == "" {
sourceBucketName = bucketNameFromCredentials
}

ctx := context.Background()
Expand All @@ -100,9 +103,9 @@ func handleResourceDownloadRun(cmd *cobra.Command, args []string) {
return
}

log.Info(fmt.Sprintf("Downloading resources from bucket %v to local directory %v ...", bucketName, resourcesPath))
log.Info(fmt.Sprintf("Downloading resources from bucket %v to local directory %v ...", sourceBucketName, targetResourcesPath))

bucket := client.Bucket(bucketName)
bucket := client.Bucket(sourceBucketName)
it := bucket.Objects(ctx, nil)
for {
attributes, err := it.Next()
Expand All @@ -113,7 +116,7 @@ func handleResourceDownloadRun(cmd *cobra.Command, args []string) {
log.Error(err)
} else {
source := bucket.Object(attributes.Name)
targetPathAndFilename := filepath.Join(resourcesPath, getRelativePersistentResourcePathByHash(attributes.Name), filepath.Base(attributes.Name))
targetPathAndFilename := filepath.Join(targetResourcesPath, getRelativePersistentResourcePathByHash(attributes.Name), filepath.Base(attributes.Name))

err = os.MkdirAll(filepath.Dir(targetPathAndFilename), 0755)
if err != nil {
Expand Down
23 changes: 12 additions & 11 deletions cmd/beach/cmd/resource-upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
"google.golang.org/api/option"
)

var instanceIdentifier, projectNamespace, bucketName, resourcesPath, resumeWithFile string
var targetBucketName, sourceResourcesPath, resumeWithFile string
var force bool

// resourceUploadCmd represents the resource-upload command
Expand Down Expand Up @@ -62,6 +62,7 @@ Notes:
func init() {
resourceUploadCmd.Flags().StringVar(&instanceIdentifier, "instance", "", "instance identifier of the Beach instance to upload to, eg. 'instance-123abc45-def6-7890-abcd-1234567890ab'")
resourceUploadCmd.Flags().StringVar(&projectNamespace, "namespace", "", "The project namespace of the Beach instance to upload to, eg. 'beach-project-123abc45-def6-7890-abcd-1234567890ab'")
resourceUploadCmd.Flags().StringVar(&clusterIdentifier, "cluster", "", "The cluster identifier of the Beach instance to upload to, eg. 'h9acc4'")
resourceUploadCmd.Flags().BoolVar(&force, "force", false, "Force uploading resources which already exist in the target bucket")
resourceUploadCmd.Flags().StringVar(&resumeWithFile, "resume-with-file", "", "If specified, resume uploading resources starting with the given filename, eg. '12dcde4c13142942288c5a973caf0fa720ed2794'")
_ = resourceUploadCmd.MarkFlagRequired("instance")
Expand All @@ -75,23 +76,23 @@ func handleResourceUploadRun(cmd *cobra.Command, args []string) {
log.Fatal("Could not activate sandbox: ", err)
return
}
if resourcesPath == "" {
resourcesPath = sandbox.ProjectDataPersistentResourcesPath
if sourceResourcesPath == "" {
sourceResourcesPath = sandbox.ProjectDataPersistentResourcesPath
}
_, err = os.Stat(resourcesPath)
_, err = os.Stat(sourceResourcesPath)
if err != nil {
log.Fatal("The path %v does not exist", resourcesPath)
log.Fatal("The path %v does not exist", sourceResourcesPath)
return
}

err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace)
err, bucketNameFromCredentials, privateKeyDecoded := retrieveCloudStorageCredentials(instanceIdentifier, projectNamespace, clusterIdentifier)
if err != nil {
log.Fatal(err)
return
}

if bucketName == "" {
bucketName = bucketNameFromCredentials
if targetBucketName == "" {
targetBucketName = bucketNameFromCredentials
}

ctx := context.Background()
Expand All @@ -101,10 +102,10 @@ func handleResourceUploadRun(cmd *cobra.Command, args []string) {
return
}

log.Info(fmt.Sprintf("Uploading resources from local directory %v to bucket %v...", resourcesPath, bucketName))
log.Info(fmt.Sprintf("Uploading resources from local directory %v to bucket %v...", sourceResourcesPath, targetBucketName))

var fileList []string
err = filepath.Walk(resourcesPath, func(path string, f os.FileInfo, err error) error {
err = filepath.Walk(sourceResourcesPath, func(path string, f os.FileInfo, err error) error {
if !f.IsDir() {
fileList = append(fileList, path)
}
Expand All @@ -115,7 +116,7 @@ func handleResourceUploadRun(cmd *cobra.Command, args []string) {
return
}

bucket := client.Bucket(bucketName)
bucket := client.Bucket(targetBucketName)
for _, pathAndFilename := range fileList {
filename := filepath.Base(pathAndFilename)

Expand Down

0 comments on commit 91db934

Please sign in to comment.