Pass lock metadata on uploads + use upstream EOS GRPC bindings

changelog/unreleased/fix-ocm-open-driver.md

@@ -0,0 +1,8 @@
+Bugfix: ocm: fixed domain not having a protocol scheme
+
+This PR fixes a bug in the OCM open driver that causes it to be unable to probe
+OCM services at the remote server due to the domain having an unsupported
+protocol scheme. in this case domain doesn't have a scheme and the changes in
+this PR add a scheme to the domain before doing the probe.
+
+https://github.com/cs3org/reva/pull/4790

changelog/unreleased/locks-uploads.md

@@ -0,0 +1,6 @@
+Enhancement: Pass lock holder metadata on uploads
+
+We now pass relevant metadata (lock id and lock holder) downstream
+on uploads, and handle the case of conflicts due to lock mismatch.
+
+https://github.com/cs3org/reva/pull/4514

docker/Dockerfile.revad-ceph

@@ -18,13 +18,7 @@
 
 FROM quay.io/ceph/ceph:v18
 
-RUN mkdir -p /etc/selinux/config
-
-# this is a workaround as the Ceph docker image is still based on CentOS 8 Stream, which is EOL
-RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-
-RUN dnf update --exclude=ceph-iscsi,chrony -y && dnf install -y \
+RUN dnf update --exclude=ceph-iscsi -y  && dnf install -y \
   git \
   gcc \
   make \

examples/cernbox/cernbox.toml

@@ -280,8 +280,10 @@ sender_mail = "sciencemesh@{{ vars.provider_domain }}"
 smtp_server = "smtp.{{ vars.provider_domain }}"
 smtp_port = 25
 
-[http.services.wellknown.ocmprovider]
+[http.services.wellknown]
 address = ":443"
+
+[http.services.wellknown.ocmprovider]
 ocm_prefix = "ocm"
 provider = "Reva for CERNBox"
 endpoint = "{{ vars.external_reva_endpoint }}"

examples/sciencemesh/sciencemesh.toml

@@ -253,8 +253,10 @@ sender_mail = "sciencemesh@{{ vars.provider_domain }}"
 smtp_server = "smtp.{{ vars.provider_domain }}"
 smtp_port = 25
 
-[http.services.wellknown.ocmprovider]
+[http.services.wellknown]
 address = ":443"
+
+[http.services.wellknown.ocmprovider]
 ocm_prefix = "ocm"
 provider = "Reva for ownCloud/Nextcloud"
 endpoint = "{{ vars.external_reva_endpoint }}"

go.mod

@@ -72,6 +72,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect
+	github.com/cern-eos/go-eosgrpc v0.0.0-20240812132646-f105d2304f38 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

go.sum

@@ -848,6 +848,8 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
 github.com/ceph/go-ceph v0.26.0 h1:LZoATo25ZH5aeL5t85BwIbrNLKCDfcDM+e0qV0cmwHY=
 github.com/ceph/go-ceph v0.26.0/go.mod h1:ISxb295GszZwtLPkeWi+L2uLYBVsqbsh0M104jZMOX4=
+github.com/cern-eos/go-eosgrpc v0.0.0-20240812132646-f105d2304f38 h1:+81ss4Vut1khzEhl7ximWF/V+EadspY47V4JrQkwlI4=
+github.com/cern-eos/go-eosgrpc v0.0.0-20240812132646-f105d2304f38/go.mod h1:ZiIzbg4sDO2MwYlspcnauUR2dfwZHUzxker+HP9k+20=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

internal/grpc/services/storageprovider/storageprovider.go

@@ -284,10 +284,10 @@ func (s *service) SetLock(ctx context.Context, req *provider.SetLockRequest) (*p
 		var st *rpc.Status
 		switch err.(type) {
 		case errtypes.IsNotFound:
-			st = status.NewNotFound(ctx, "path not found when setting lock")
+			st = status.NewNotFound(ctx, "resource not found when setting lock")
 		case errtypes.PermissionDenied:
 			st = status.NewPermissionDenied(ctx, err, "permission denied")
-		case errtypes.BadRequest:
+		case errtypes.Conflict:
 			st = status.NewFailedPrecondition(ctx, err, "reference already locked")
 		default:
 			st = status.NewInternal(ctx, err, "error setting lock: "+req.Ref.String())

internal/http/services/owncloud/ocdav/put.go

@@ -271,6 +271,12 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		return
 	}
 	httpReq.Header.Set(datagateway.TokenTransportHeader, token)
+	if lockid := r.Header.Get(HeaderLockID); lockid != "" {
+		httpReq.Header.Set(HeaderLockID, lockid)
+	}
+	if lockholder := r.Header.Get(HeaderLockHolder); lockholder != "" {
+		httpReq.Header.Set(HeaderLockHolder, lockholder)
+	}
 
 	httpRes, err := s.client.Do(httpReq)
 	if err != nil {

internal/http/services/owncloud/ocdav/webdav.go

@@ -76,6 +76,8 @@ const (
 	HeaderOCMtime              = "X-OC-Mtime"
 	HeaderExpectedEntityLength = "X-Expected-Entity-Length"
 	HeaderTransferAuth         = "TransferHeaderAuthorization"
+	HeaderLockID               = "X-Lock-Id"
+	HeaderLockHolder           = "X-Lock-Holder"
 )
 
 // WebDavHandler implements a dav endpoint.

pkg/eosclient/eosbinary/eosbinary.go

@@ -203,6 +203,11 @@ func (c *Client) executeXRDCopy(ctx context.Context, cmdArgs []string) (string,
 		err = errtypes.InvalidCredentials("eosclient: no sufficient permissions for the operation")
 	}
 
+	// check for lock mismatch error
+	if strings.Contains(errBuf.String(), "file has a valid extended attribute lock") {
+		err = errtypes.Conflict("eosclient: lock mismatch")
+	}
+
 	args := fmt.Sprintf("%s", cmd.Args)
 	env := fmt.Sprintf("%s", cmd.Env)
 	log.Info().Str("args", args).Str("env", env).Int("exit", exitStatus).Msg("eos cmd")
@@ -455,7 +460,7 @@ func (c *Client) mergeACLsAndAttrsForFiles(ctx context.Context, auth eosclient.A
 }
 
 // SetAttr sets an extended attributes on a path.
-func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path string) error {
+func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path, app string) error {
 	if !isValidAttribute(attr) {
 		return errors.New("eos: attr is invalid: " + serializeAttribute(attr))
 	}
@@ -468,11 +473,15 @@ func (c *Client) SetAttr(ctx context.Context, auth eosclient.Authorization, attr
 		}
 		return c.handleFavAttr(ctx, auth, attr, recursive, path, info, true)
 	}
-	return c.setEOSAttr(ctx, auth, attr, errorIfExists, recursive, path)
+	return c.setEOSAttr(ctx, auth, attr, errorIfExists, recursive, path, app)
 }
 
-func (c *Client) setEOSAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path string) error {
-	args := []string{"attr"}
+func (c *Client) setEOSAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, errorIfExists, recursive bool, path, app string) error {
+	args := []string{}
+	if app != "" {
+		args = append(args, "-a", app)
+	}
+	args = append(args, "attr")
 	if recursive {
 		args = append(args, "-r")
 	}
@@ -485,9 +494,12 @@ func (c *Client) setEOSAttr(ctx context.Context, auth eosclient.Authorization, a
 	_, _, err := c.executeEOS(ctx, args, auth)
 	if err != nil {
 		var exErr *exec.ExitError
-		if errors.As(err, &exErr) && exErr.ExitCode() == 17 {
+		if errors.As(err, &exErr) && exErr.ExitCode() == 17 { // EEXIST
 			return eosclient.AttrAlreadyExistsError
 		}
+		if errors.As(err, &exErr) && exErr.ExitCode() == 16 { // EBUSY -> Locked
+			return eosclient.FileIsLockedError
+		}
 		return err
 	}
 	return nil
@@ -516,11 +528,11 @@ func (c *Client) handleFavAttr(ctx context.Context, auth eosclient.Authorization
 		favs.DeleteEntry(acl.TypeUser, u.Id.OpaqueId)
 	}
 	attr.Val = favs.Serialize()
-	return c.setEOSAttr(ctx, auth, attr, false, recursive, path)
+	return c.setEOSAttr(ctx, auth, attr, false, recursive, path, "")
 }
 
 // UnsetAttr unsets an extended attribute on a path.
-func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path string) error {
+func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, attr *eosclient.Attribute, recursive bool, path, app string) error {
 	if !isValidAttribute(attr) {
 		return errors.New("eos: attr is invalid: " + serializeAttribute(attr))
 	}
@@ -536,11 +548,15 @@ func (c *Client) UnsetAttr(ctx context.Context, auth eosclient.Authorization, at
 	}
 
 	var args []string
+	if app != "" {
+		args = append(args, "-a", app)
+	}
+	args = append(args, "attr")
 	if recursive {
-		args = []string{"attr", "-r", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path}
-	} else {
-		args = []string{"attr", "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path}
+		args = append(args, "-r")
 	}
+	args = append(args, "rm", fmt.Sprintf("%s.%s", attrTypeToString(attr.Type), attr.Key), path)
+
 	_, _, err = c.executeEOS(ctx, args, auth)
 	if err != nil {
 		var exErr *exec.ExitError
@@ -707,7 +723,7 @@ func (c *Client) Read(ctx context.Context, auth eosclient.Authorization, path st
 }
 
 // Write writes a stream to the mgm.
-func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser) error {
+func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser, app string) error {
 	fd, err := os.CreateTemp(c.opt.CacheDirectory, "eoswrite-")
 	if err != nil {
 		return err
@@ -720,19 +736,18 @@ func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path s
 	if err != nil {
 		return err
 	}
-
-	return c.WriteFile(ctx, auth, path, fd.Name())
+	return c.writeFile(ctx, auth, path, fd.Name(), app)
 }
 
 // WriteFile writes an existing file to the mgm.
-func (c *Client) WriteFile(ctx context.Context, auth eosclient.Authorization, path, source string) error {
+func (c *Client) writeFile(ctx context.Context, auth eosclient.Authorization, path, source, app string) error {
 	xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path)
 	args := []string{"--nopbar", "--silent", "-f", source, xrdPath}
 
 	if auth.Token != "" {
 		args[4] += "?authz=" + auth.Token
 	} else if auth.Role.UID != "" && auth.Role.GID != "" {
-		args = append(args, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s&eos.app=reva_eosclient::write", auth.Role.UID, auth.Role.GID))
+		args = append(args, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s&eos.app=%s", auth.Role.UID, auth.Role.GID, app))
 	}
 
 	_, _, err := c.executeXRDCopy(ctx, args)

pkg/eosclient/eosclient.go

@@ -37,8 +37,8 @@ type EOSClient interface {
 	GetFileInfoByInode(ctx context.Context, auth Authorization, inode uint64) (*FileInfo, error)
 	GetFileInfoByFXID(ctx context.Context, auth Authorization, fxid string) (*FileInfo, error)
 	GetFileInfoByPath(ctx context.Context, auth Authorization, path string) (*FileInfo, error)
-	SetAttr(ctx context.Context, auth Authorization, attr *Attribute, errorIfExists, recursive bool, path string) error
-	UnsetAttr(ctx context.Context, auth Authorization, attr *Attribute, recursive bool, path string) error
+	SetAttr(ctx context.Context, auth Authorization, attr *Attribute, errorIfExists, recursive bool, path, app string) error
+	UnsetAttr(ctx context.Context, auth Authorization, attr *Attribute, recursive bool, path, app string) error
 	GetAttr(ctx context.Context, auth Authorization, key, path string) (*Attribute, error)
 	GetAttrs(ctx context.Context, auth Authorization, path string) ([]*Attribute, error)
 	GetQuota(ctx context.Context, username string, rootAuth Authorization, path string) (*QuotaInfo, error)
@@ -51,8 +51,7 @@ type EOSClient interface {
 	Rename(ctx context.Context, auth Authorization, oldPath, newPath string) error
 	List(ctx context.Context, auth Authorization, path string) ([]*FileInfo, error)
 	Read(ctx context.Context, auth Authorization, path string) (io.ReadCloser, error)
-	Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser) error
-	WriteFile(ctx context.Context, auth Authorization, path, source string) error
+	Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser, app string) error
 	ListDeletedEntries(ctx context.Context, auth Authorization, maxentries int, from, to time.Time) ([]*DeletedEntry, error)
 	RestoreDeletedEntry(ctx context.Context, auth Authorization, key string) error
 	PurgeDeletedEntries(ctx context.Context, auth Authorization) error
@@ -154,3 +153,7 @@ const AttrAlreadyExistsError = errtypes.BadRequest("attr already exists")
 // AttrNotExistsError is the error raised when removing
 // an attribute that does not exist.
 const AttrNotExistsError = errtypes.BadRequest("attr not exists")
+
+// FileIsLockedError is the error raised when attempting to set a lock
+// attribute to an already locked file with a mismatched lock.
+const FileIsLockedError = errtypes.BadRequest("file is locked")