From e5e3f3d48b6d31c09c16443e8b37ceafe00f8c36 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Mon, 27 May 2024 11:20:13 +0200 Subject: [PATCH] feat(fs): Export Path method for s3ng and ocis blobstores Signed-off-by: jkoberg --- .../add-path-method-to-blobstore.md | 5 ++ pkg/storage/fs/ocis/blobstore/blobstore.go | 53 +++++++++---------- pkg/storage/fs/s3ng/blobstore/blobstore.go | 27 +++++----- 3 files changed, 45 insertions(+), 40 deletions(-) create mode 100644 changelog/unreleased/add-path-method-to-blobstore.md diff --git a/changelog/unreleased/add-path-method-to-blobstore.md b/changelog/unreleased/add-path-method-to-blobstore.md new file mode 100644 index 0000000000..5651a9982a --- /dev/null +++ b/changelog/unreleased/add-path-method-to-blobstore.md @@ -0,0 +1,5 @@ +Enhancement: Add a Path method to blobstore + +Add a method to get the path of a blob to the ocis and s3ng blobstores. + +https://github.com/cs3org/reva/pull/4699 diff --git a/pkg/storage/fs/ocis/blobstore/blobstore.go b/pkg/storage/fs/ocis/blobstore/blobstore.go index 43dadd4ceb..b4293fa645 100644 --- a/pkg/storage/fs/ocis/blobstore/blobstore.go +++ b/pkg/storage/fs/ocis/blobstore/blobstore.go @@ -32,6 +32,9 @@ import ( "github.com/pkg/errors" ) +// ErrBlobIDEmpty is returned when the BlobID is empty +var ErrBlobIDEmpty = fmt.Errorf("blobstore: BlobID is empty") + // Blobstore provides an interface to an filesystem based blobstore type Blobstore struct { root string @@ -51,10 +54,12 @@ func New(root string) (*Blobstore, error) { // Upload stores some data in the blobstore under the given key func (bs *Blobstore) Upload(node *node.Node, source string) error { - dest, err := bs.path(node) - if err != nil { - return err + if node.BlobID == "" { + return ErrBlobIDEmpty } + + dest := bs.Path(node) + // ensure parent path exists if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating parent folders for blob") @@ -87,10 +92,11 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { // Download retrieves a blob from the blobstore for reading func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { - dest, err := bs.path(node) - if err != nil { - return nil, err + if node.BlobID == "" { + return nil, ErrBlobIDEmpty } + + dest := bs.Path(node) file, err := os.Open(dest) if err != nil { return nil, errors.Wrapf(err, "could not read blob '%s'", dest) @@ -100,10 +106,10 @@ func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { // Delete deletes a blob from the blobstore func (bs *Blobstore) Delete(node *node.Node) error { - dest, err := bs.path(node) - if err != nil { - return err + if node.BlobID == "" { + return ErrBlobIDEmpty } + dest := bs.Path(node) if err := utils.RemoveItem(dest); err != nil { return errors.Wrapf(err, "could not delete blob '%s'", dest) } @@ -111,37 +117,28 @@ func (bs *Blobstore) Delete(node *node.Node) error { } // List lists all blobs in the Blobstore -func (bs *Blobstore) List() ([]string, error) { +func (bs *Blobstore) List() ([]*node.Node, error) { dirs, err := filepath.Glob(filepath.Join(bs.root, "spaces", "*", "*", "blobs", "*", "*", "*", "*", "*")) if err != nil { return nil, err } - blobids := make([]string, 0, len(dirs)) + blobids := make([]*node.Node, 0, len(dirs)) for _, d := range dirs { - seps := strings.Split(d, "/") - var b string - var now bool - for _, s := range seps { - if now { - b += s - } - if s == "blobs" { - now = true - } - } - blobids = append(blobids, b) + _, s, _ := strings.Cut(d, "spaces") + spaceraw, blobraw, _ := strings.Cut(s, "blobs") + blobids = append(blobids, &node.Node{ + SpaceID: strings.ReplaceAll(spaceraw, "/", ""), + BlobID: strings.ReplaceAll(blobraw, "/", ""), + }) } return blobids, nil } -func (bs *Blobstore) path(node *node.Node) (string, error) { - if node.BlobID == "" { - return "", fmt.Errorf("blobstore: BlobID is empty") - } +func (bs *Blobstore) Path(node *node.Node) string { return filepath.Join( bs.root, filepath.Clean(filepath.Join( "/", "spaces", lookup.Pathify(node.SpaceID, 1, 2), "blobs", lookup.Pathify(node.BlobID, 4, 2)), ), - ), nil + ) } diff --git a/pkg/storage/fs/s3ng/blobstore/blobstore.go b/pkg/storage/fs/s3ng/blobstore/blobstore.go index 2105ddef09..aaeecf65f8 100644 --- a/pkg/storage/fs/s3ng/blobstore/blobstore.go +++ b/pkg/storage/fs/s3ng/blobstore/blobstore.go @@ -84,7 +84,7 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { } defer reader.Close() - _, err = bs.client.PutObject(context.Background(), bs.bucket, bs.path(node), reader, node.Blobsize, minio.PutObjectOptions{ + _, err = bs.client.PutObject(context.Background(), bs.bucket, bs.Path(node), reader, node.Blobsize, minio.PutObjectOptions{ ContentType: "application/octet-stream", SendContentMd5: bs.defaultPutOptions.SendContentMd5, ConcurrentStreamParts: bs.defaultPutOptions.ConcurrentStreamParts, @@ -95,21 +95,21 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { }) if err != nil { - return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.path(node), bs.bucket) + return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.Path(node), bs.bucket) } return nil } // Download retrieves a blob from the blobstore for reading func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { - reader, err := bs.client.GetObject(context.Background(), bs.bucket, bs.path(node), minio.GetObjectOptions{}) + reader, err := bs.client.GetObject(context.Background(), bs.bucket, bs.Path(node), minio.GetObjectOptions{}) if err != nil { - return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", bs.path(node), bs.bucket) + return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", bs.Path(node), bs.bucket) } stat, err := reader.Stat() if err != nil { - return nil, errors.Wrapf(err, "blob path: %s", bs.path(node)) + return nil, errors.Wrapf(err, "blob path: %s", bs.Path(node)) } if node.Blobsize != stat.Size { @@ -121,31 +121,34 @@ func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { // Delete deletes a blob from the blobstore func (bs *Blobstore) Delete(node *node.Node) error { - err := bs.client.RemoveObject(context.Background(), bs.bucket, bs.path(node), minio.RemoveObjectOptions{}) + err := bs.client.RemoveObject(context.Background(), bs.bucket, bs.Path(node), minio.RemoveObjectOptions{}) if err != nil { - return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", bs.path(node), bs.bucket) + return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", bs.Path(node), bs.bucket) } return nil } // List lists all blobs in the Blobstore -func (bs *Blobstore) List() ([]string, error) { +func (bs *Blobstore) List() ([]*node.Node, error) { ch := bs.client.ListObjects(context.Background(), bs.bucket, minio.ListObjectsOptions{Recursive: true}) var err error - ids := make([]string, 0) + ids := make([]*node.Node, 0) for oi := range ch { if oi.Err != nil { err = oi.Err continue } - _, blobid, _ := strings.Cut(oi.Key, "/") - ids = append(ids, strings.ReplaceAll(blobid, "/", "")) + spaceid, blobid, _ := strings.Cut(oi.Key, "/") + ids = append(ids, &node.Node{ + SpaceID: strings.ReplaceAll(spaceid, "/", ""), + BlobID: strings.ReplaceAll(blobid, "/", ""), + }) } return ids, err } -func (bs *Blobstore) path(node *node.Node) string { +func (bs *Blobstore) Path(node *node.Node) string { // https://aws.amazon.com/de/premiumsupport/knowledge-center/s3-prefix-nested-folders-difference/ // Prefixes are used to partion a bucket. A prefix is everything except the filename. // For a file `BucketName/foo/bar/lorem.ipsum`, `BucketName/foo/bar/` is the prefix.