Support for sub-folders when mounting S3 buckets (#263)

* initial support for s3 folders Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * adding support for building local component containers with podman Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * bug fixes in build bash scripts Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * bug fixes in build bash scripts Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * bug fixes in build bash scripts Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * bug fixes in build bash scripts Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * bug fixes in build bash scripts Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * bug fixes in build bash scripts Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * docker buildx load after build Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> * docker buildx load after build Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]> --------- Signed-off-by: SRIKUMAR VENUGOPAL <[email protected]>
datashim-io · Jan 22, 2024 · 24062e9 · 24062e9
1 parent b4c16e4
commit 24062e9
Show file tree

Hide file tree

Showing 14 changed files with 301 additions and 67 deletions.
diff --git a/build-tools/build_components.sh b/build-tools/build_components.sh
@@ -30,16 +30,47 @@ do
     esac
 done
 
+shift $((OPTIND-1))
+
+REGISTRY_URL="${1:-quay.io/datashim-io}"
+VERSION="${2:-latest}"
+
+DOCKERCMD="docker"
+ALTDOCKERCMD="podman"
+if !(command -v ${DOCKERCMD} &> /dev/null)
+then
+    echo "Docker command not found"
+    if !(command -v ${ALTDOCKERCMD} &> /dev/null)
+    then
+        echo "Neither ${DOCKERCMD} nor ${ALTDOCKERCMD} commands found.. cannot build "
+        exit 1
+    else
+        DOCKERCMD=${ALTDOCKERCMD}  
+    fi
+else
+    echo "Docker command found"
+    cmd_type=$(type -t ${DOCKERCMD})
+    if [ $cmd_type == "alias" ]
+    then
+        echo "${DOCKERCMD} is an alias, switching to ${ALTDOCKERCMD}"
+        DOCKERCMD=${ALTDOCKERCMD}  
+    fi
+fi 
+
+if [ ${DOCKERCMD} == "docker" ]
+then 
+    if [ $CREATE_NEW_BUILDX_CONTEXT = "yes" ]; then
+        docker buildx create --use
+    fi
+fi
+
 if [ $BUILD_AND_PUSH = "yes" ]; then
       if [ $SKIP_LOGIN = "no" ]; then
             echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USERNAME --password-stdin $REGISTRY_URL
       fi
-      if [ $CREATE_NEW_BUILDX_CONTEXT = "yes" ]; then
-            docker buildx create --use
-      fi
-      (cd ../src/dataset-operator && ./build_and_push_multiarch_dataset_operator.sh $REGISTRY_URL)
-      (cd ../src/generate-keys && ./build_and_push_multiarch_generate_keys.sh $REGISTRY_URL)
+      (cd ../src/dataset-operator && ./build_multiarch_dataset_operator.sh -p $REGISTRY_URL $VERSION)
+      (cd ../src/generate-keys && ./build_multiarch_generate_keys.sh -p $REGISTRY_URL $VERSION)
 else
-      (cd ../src/dataset-operator && ./build_dataset_operator.sh $REGISTRY_URL)
-      (cd ../src/generate-keys && ./build_generate_keys.sh $REGISTRY_URL)
+      (cd ../src/dataset-operator && ./build_multiarch_dataset_operator.sh $REGISTRY_URL $VERSION)
+      (cd ../src/generate-keys && ./build_multiarch_generate_keys.sh $REGISTRY_URL $VERSION)
 fi
diff --git a/src/csi-s3/pkg/s3/config.go b/src/csi-s3/pkg/s3/config.go
@@ -8,6 +8,7 @@ type Config struct {
 	Endpoint        string
 	Mounter         string
 	Bucket          string
+	Folder          string
 	RemoveOnDelete  bool
 	Readonly        bool
 	Provision       bool

diff --git a/src/csi-s3/pkg/s3/mounter_goofys.go b/src/csi-s3/pkg/s3/mounter_goofys.go
@@ -2,13 +2,14 @@ package s3
 
 import (
 	"fmt"
-	"github.com/golang/glog"
 	"log"
 	"os"
+
+	"github.com/golang/glog"
 )
 
 const (
-	goofysCmd     = "goofys"
+	goofysCmd = "goofys"
 )
 
 // Implements Mounter
@@ -18,12 +19,12 @@ type goofysMounter struct {
 	region          string
 	accessKeyID     string
 	secretAccessKey string
-	volumeID		string
-	readonly 		bool
+	volumeID        string
+	readonly        bool
 }
 
 func newGoofysMounter(b *bucket, cfg *Config, volume string) (Mounter, error) {
-	glog.V(3).Infof("Mounting using goofys volume %s",volume)
+	glog.V(3).Infof("Mounting using goofys volume %s", volume)
 	//TODO we need to handle regions as well
 	//region := cfg.Region
 	//// if endpoint is set we need a default region
@@ -36,8 +37,8 @@ func newGoofysMounter(b *bucket, cfg *Config, volume string) (Mounter, error) {
 		region:          cfg.Region,
 		accessKeyID:     cfg.AccessKeyID,
 		secretAccessKey: cfg.SecretAccessKey,
-		readonly: 		 cfg.Readonly,
-		volumeID:		 volume,
+		readonly:        cfg.Readonly,
+		volumeID:        volume,
 	}, nil
 }
 
@@ -57,11 +58,11 @@ func (goofys *goofysMounter) Mount(source string, target string) error {
 	}
 	args := []string{
 		fmt.Sprintf("--endpoint=%s", goofys.endpoint),
-		"--type-cache-ttl","1s",
-		"--stat-cache-ttl","1s",
-		"--dir-mode","0777",
-		"--file-mode","0777",
-		"--http-timeout","5m",
+		"--type-cache-ttl", "1s",
+		"--stat-cache-ttl", "1s",
+		"--dir-mode", "0777",
+		"--file-mode", "0777",
+		"--http-timeout", "5m",
 		//fmt.Sprintf("--cheap=%s", os.Getenv("cheap")),
 		"-o", "allow_other",
 	}
@@ -71,21 +72,28 @@ func (goofys *goofysMounter) Mount(source string, target string) error {
 	if goofys.region != "" {
 		args = append(args, "--region", goofys.region)
 	}
-	if(goofys.readonly) {
-		args = append(args, "-o","ro")
+	if goofys.readonly {
+		args = append(args, "-o", "ro")
+	}
+	if goofys.bucket.Folder == "" {
+		glog.V(4).Infof("This bucket %s contains no folder prefixes", goofys.bucket.Name)
+		args = append(args,
+			fmt.Sprintf("%s", goofys.bucket.Name),
+			fmt.Sprintf("%s", target))
+	} else {
+		glog.V(4).Infof("This bucket %s contains folder prefix %s", goofys.bucket.Name, goofys.bucket.Folder)
+		args = append(args,
+			fmt.Sprintf("%s:%s", goofys.bucket.Name, goofys.bucket.Folder),
+			fmt.Sprintf("%s", target))
 	}
-	args = append(args,
-		fmt.Sprintf("%s", goofys.bucket.Name),
-		fmt.Sprintf("%s", target))
 	return fuseMount(target, goofysCmd, args)
 
 }
 
-
 func writes3fsPassGoofy(goofys *goofysMounter) error {
 	awsPath := fmt.Sprintf("%s/.aws", os.Getenv("HOME"))
 	if _, err := os.Stat(awsPath); os.IsNotExist(err) {
-		mkdir_err := os.Mkdir(awsPath,0700)
+		mkdir_err := os.Mkdir(awsPath, 0700)
 		if mkdir_err != nil {
 			return mkdir_err
 		}
@@ -98,10 +106,9 @@ func writes3fsPassGoofy(goofys *goofysMounter) error {
 		log.Println(err)
 	}
 	defer f.Close()
-	textToAdd := "["+goofys.volumeID+"]\naws_access_key_id = "+goofys.accessKeyID+"\naws_secret_access_key ="+goofys.secretAccessKey+"\n"
+	textToAdd := "[" + goofys.volumeID + "]\naws_access_key_id = " + goofys.accessKeyID + "\naws_secret_access_key =" + goofys.secretAccessKey + "\n"
 	if _, err := f.WriteString(textToAdd); err != nil {
 		log.Println(err)
 	}
 	return nil
 }
-
diff --git a/src/csi-s3/pkg/s3/nodeserver.go b/src/csi-s3/pkg/s3/nodeserver.go
@@ -122,6 +122,15 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 		return nil, status.Error(codes.InvalidArgument, "Bucket name not provided for mounting")
 	}
 
+	// srikumarv - this is a hack to support folders for the current csi-s3 implementation used in
+	// Datashim
+	folder := ""
+	if len(s3.cfg.Folder) != 0 {
+		folder = s3.cfg.Folder
+	} else {
+		glog.V(2).Infof("s3: no fspath found for bucket %s", bucketName)
+	}
+
 	//b, err := s3.getBucket(bucketName)
 	//if err != nil {
 	//	return nil, err
@@ -130,6 +139,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 
 	b := &bucket{
 		Name:    bucketName,
+		Folder:  folder,
 		Mounter: volContext[mounterTypeKey],
 	}
 

diff --git a/src/csi-s3/pkg/s3/s3-client.go b/src/csi-s3/pkg/s3/s3-client.go
@@ -20,6 +20,7 @@ type s3Client struct {
 
 type bucket struct {
 	Name    string
+	Folder  string
 	Mounter string
 }
 
@@ -54,6 +55,7 @@ func newS3ClientFromSecrets(secrets map[string]string) (*s3Client, error) {
 		Region:          secrets["region"],
 		Endpoint:        secrets["endpoint"],
 		Bucket:          secrets["bucket"],
+		Folder:          secrets["folder"],
 		Readonly:        readonly,
 		Provision:       provision,
 		RemoveOnDelete:  removeOnDelete,
@@ -127,7 +129,7 @@ func (client *s3Client) emptyBucket(bucketName string) error {
 //	return err
 //}
 
-//TODO is it not used at all?
+// TODO is it not used at all?
 func (client *s3Client) metadataExist(bucketName string) bool {
 	opts := minio.GetObjectOptions{}
 	_, err := client.minio.GetObject(bucketName, metadataName, opts)

diff --git a/src/dataset-operator/build_and_push_multiarch_dataset_operator.sh b/src/dataset-operator/build_and_push_multiarch_dataset_operator.sh
diff --git a/src/dataset-operator/build_dataset_operator.sh b/src/dataset-operator/build_dataset_operator.sh
diff --git a/src/dataset-operator/build_multiarch_dataset_operator.sh b/src/dataset-operator/build_multiarch_dataset_operator.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+set -e
+
+print_usage() {
+    echo "usage: $0 [-p] <REGISTRY_URL> <VERSION>"
+    echo "Use -p to build and push multiarch images"
+}
+
+PUSH="false"
+while getopts 'p' flag; do
+case "$flag" in
+    p) 
+        PUSH="true"
+        ;;
+    ?) 
+        print_usage >&2
+        exit 1
+        ;;
+esac
+done
+
+shift $((OPTIND-1))
+
+REGISTRY_URL="${1:-quay.io/datashim-io}"
+VERSION="${2:-latest}"
+
+docker_build () {
+    docker buildx build --platform linux/amd64 -t ${REGISTRY_URL}/dataset-operator:${VERSION} .
+    docker buildx build --load -t ${REGISTRY_URL}/dataset-operator:${VERSION} .
+}
+
+docker_build_and_push () {
+    docker buildx build --platform linux/amd64,linux/arm64,linux/ppc64le --push -t ${REGISTRY_URL}/dataset-operator:${VERSION} .
+}
+
+podman_build () {
+    podman manifest create ${REGISTRY_URL}/dataset-operator:${VERSION}
+    podman buildx build --platform linux/amd64,linux/arm64  --manifest ${REGISTRY_URL}/dataset-operator:${VERSION} .
+}
+
+podman_push () {
+    podman manifest push ${REGISTRY_URL}/dataset-operator:${VERSION} 
+
+}
+
+DOCKERCMD="docker"
+ALTDOCKERCMD="podman"
+if !(command -v ${DOCKERCMD} &> /dev/null)
+then
+    echo "Docker command not found"
+    if !(command -v ${ALTDOCKERCMD} &> /dev/null)
+    then
+        echo "Neither ${DOCKERCMD} nor ${ALTDOCKERCMD} commands found.. cannot build "
+        exit 1
+    else
+        DOCKERCMD=${ALTDOCKERCMD}  
+    fi
+else
+    echo "Docker command found"
+    cmd_type=$(type -t ${DOCKERCMD})
+    if [ $cmd_type == "alias" ]
+    then
+        echo "${DOCKERCMD} is an alias, switching to ${ALTDOCKERCMD}"
+        DOCKERCMD=${ALTDOCKERCMD}  
+    fi
+fi 
+
+if [ $PUSH == "true" ]
+then
+    echo "pushing images to the registry"
+    if [ ${DOCKERCMD} == "docker" ]
+    then
+        docker_build_and_push
+    else 
+        podman_build
+        podman_push
+    fi
+else
+    echo "building image locally"
+    if [ ${DOCKERCMD} == "docker" ]
+    then
+        docker_build 
+    else
+        podman_build
+    fi
+fi