From d7adeacc37b3e6fe6175af9991072bcf80eb1b04 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Mon, 29 Jul 2024 16:44:23 +0200 Subject: [PATCH 1/3] chore: bump reva to v2.22.0 --- changelog/unreleased/bump-reva.md | 18 +- go.mod | 4 +- go.sum | 8 +- .../publicshareprovider.go | 53 ++-- .../pkg/storage/fs/posix/tree/assimilation.go | 300 ++++++++++++------ .../posix/tree/gpfsfilauditloggingwatcher.go | 6 +- .../fs/posix/tree/gpfswatchfolderwatcher.go | 8 +- .../storage/fs/posix/tree/inotifywatcher.go | 9 +- .../reva/v2/pkg/storage/fs/posix/tree/tree.go | 11 +- .../inotifywaitgo/inotifywaitgo/models.go | 4 + .../inotifywaitgo/inotifywaitgo/watcher.go | 7 + vendor/modules.txt | 4 +- 12 files changed, 292 insertions(+), 140 deletions(-) diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md index c72b812b38b..84288fd522e 100644 --- a/changelog/unreleased/bump-reva.md +++ b/changelog/unreleased/bump-reva.md @@ -1,4 +1,20 @@ -Enhancement: Bump reva +Enhancement: Bump reva to v.2.22.0 + +* Bugfix [cs3org/reva#4741](https://github.com/cs3org/reva/pull/4741): Always find unique providers +* Bugfix [cs3org/reva#4762](https://github.com/cs3org/reva/pull/4762): Blanks in dav Content-Disposition header +* Bugfix [cs3org/reva#4775](https://github.com/cs3org/reva/pull/4775): Fixed the response code when copying the shared from to personal +* Bugfix [cs3org/reva#4633](https://github.com/cs3org/reva/pull/4633): Allow all users to create internal links +* Bugfix [cs3org/reva#4771](https://github.com/cs3org/reva/pull/4771): Deleting resources via their id +* Bugfix [cs3org/reva#4768](https://github.com/cs3org/reva/pull/4768): Fixed the file name validation if nodeid is used +* Bugfix [cs3org/reva#4758](https://github.com/cs3org/reva/pull/4758): Fix moving locked files, enable handling locked files via ocdav +* Bugfix [cs3org/reva#4774](https://github.com/cs3org/reva/pull/4774): Fix micro ocdav service init and registration +* Bugfix [cs3org/reva#4776](https://github.com/cs3org/reva/pull/4776): Fix response code for DEL file that in postprocessing +* Bugfix [cs3org/reva#4746](https://github.com/cs3org/reva/pull/4746): Uploading the same file multiple times leads to orphaned blobs +* Bugfix [cs3org/reva#4778](https://github.com/cs3org/reva/pull/4778): Zero byte uploads +* Change [cs3org/reva#4759](https://github.com/cs3org/reva/pull/4759): Updated to the latest version of the go-cs3apis +* Change [cs3org/reva#4773](https://github.com/cs3org/reva/pull/4773): Ocis bumped +* Enhancement [cs3org/reva#4766](https://github.com/cs3org/reva/pull/4766): Set archiver output format via query parameter +* Enhancement [cs3org/reva#4763](https://github.com/cs3org/reva/pull/4763): Improve posixfs storage driver https://github.com/owncloud/ocis/pull/9690 https://github.com/owncloud/ocis/pull/9662 diff --git a/go.mod b/go.mod index b60acd7b850..f7ad78d0dac 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.10.0 github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb - github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6 + github.com/cs3org/reva/v2 v2.22.0 github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 @@ -288,7 +288,7 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect - github.com/pablodz/inotifywaitgo v0.0.6 // indirect + github.com/pablodz/inotifywaitgo v0.0.7 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect diff --git a/go.sum b/go.sum index 2309f36abdf..869b08fab49 100644 --- a/go.sum +++ b/go.sum @@ -1025,8 +1025,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb h1:KmYZDReplv/yfwc1LNYpDcVhVujC3Pasv6WjXx1haSU= github.com/cs3org/go-cs3apis v0.0.0-20240724121416-062c4e3046cb/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE= -github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6 h1:kGRQf90rhI4L2vU+RxPIP8LHmCkGhESV3VpML3Owkh0= -github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6/go.mod h1:vz5FHfuRaLvSO6bscoWWx+29RtyJm0wIWLZciCLN3L0= +github.com/cs3org/reva/v2 v2.22.0 h1:dMZGJDdrTCYYyMlVSzchbEUyFrHgvFI2Md/aLLiez54= +github.com/cs3org/reva/v2 v2.22.0/go.mod h1:y9ujkcxepugOsv50baQSCtudfY5VY5S5eigNTvGfSZI= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= @@ -1808,8 +1808,8 @@ github.com/owncloud/libre-graph-api-go v1.0.5-0.20240618162722-2298241331d1 h1:w github.com/owncloud/libre-graph-api-go v1.0.5-0.20240618162722-2298241331d1/go.mod h1:yXI+rmE8yYx+ZsGVrnCpprw/gZMcxjwntnX2y2+VKxY= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= -github.com/pablodz/inotifywaitgo v0.0.6 h1:BTjQfnixXwG7oYmlIiyhWA6iyO9BtxatB3YgiibOTFc= -github.com/pablodz/inotifywaitgo v0.0.6/go.mod h1:OtzRCsYTJlIr+vAzlOtauTkfQ1c25ebFuXq8tbbf8cw= +github.com/pablodz/inotifywaitgo v0.0.7 h1:1ii49dGBnRn0t1Sz7RGZS6/NberPEDQprwKHN49Bv6U= +github.com/pablodz/inotifywaitgo v0.0.7/go.mod h1:OtzRCsYTJlIr+vAzlOtauTkfQ1c25ebFuXq8tbbf8cw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go index 3202b8c7559..7876750b670 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -53,7 +53,7 @@ import ( const getUserCtxErrMsg = "error getting user from context" func init() { - rgrpc.Register("publicshareprovider", New) + rgrpc.Register("publicshareprovider", NewDefault) } type config struct { @@ -127,9 +127,8 @@ func parsePasswordPolicy(m map[string]interface{}) (*passwordPolicy, error) { return p, nil } -// New creates a new user share provider svc -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { - +// New creates a new public share provider svc initialized from defaults +func NewDefault(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -146,6 +145,15 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, err } + gatewaySelector, err := pool.GatewaySelector(sharedconf.GetGatewaySVC(c.GatewayAddr)) + if err != nil { + return nil, err + } + return New(gatewaySelector, sm, c, p) +} + +// New creates a new user share provider svc +func New(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], sm publicshare.Manager, c *config, p *passwordPolicy) (rgrpc.Service, error) { allowedPathsForShares := make([]*regexp.Regexp, 0, len(c.AllowedPathsForShares)) for _, s := range c.AllowedPathsForShares { regex, err := regexp.Compile(s) @@ -155,11 +163,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { allowedPathsForShares = append(allowedPathsForShares, regex) } - gatewaySelector, err := pool.GatewaySelector(sharedconf.GetGatewaySVC(c.GatewayAddr)) - if err != nil { - return nil, err - } - service := &service{ conf: c, sm: sm, @@ -233,7 +236,7 @@ func (s *service) CreatePublicShare(ctx context.Context, req *link.CreatePublicS } // check that user has share permissions - if !sRes.GetInfo().GetPermissionSet().AddGrant { + if !isInternalLink && !sRes.GetInfo().GetPermissionSet().AddGrant { return &link.CreatePublicShareResponse{ Status: status.NewInvalidArg(ctx, "no share permission"), }, nil @@ -477,14 +480,12 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS isInternalLink := isInternalLink(req, ps) - // users should always be able to downgrade links to internal links - // when they are the creator of the link - // all other users should have the WritePublicLink permission - if !isInternalLink && !publicshare.IsCreatedByUser(ps, user) { + // check if the user has the permission in the user role + if !publicshare.IsCreatedByUser(ps, user) { canWriteLink, err := utils.CheckPermission(ctx, permission.WritePublicLink, gatewayClient) if err != nil { return &link.UpdatePublicShareResponse{ - Status: status.NewInternal(ctx, "error loading public share"), + Status: status.NewInternal(ctx, "error checking permission to write public share"), }, err } if !canWriteLink { @@ -501,8 +502,14 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS Status: status.NewInternal(ctx, "failed to stat shared resource"), }, err } + if sRes.Status.Code != rpc.Code_CODE_OK { + return &link.UpdatePublicShareResponse{ + Status: sRes.GetStatus(), + }, nil - if !publicshare.IsCreatedByUser(ps, user) { + } + + if !isInternalLink && !publicshare.IsCreatedByUser(ps, user) { if !sRes.GetInfo().GetPermissionSet().UpdateGrant { return &link.UpdatePublicShareResponse{ Status: status.NewPermissionDenied(ctx, nil, "no permission to update public share"), @@ -547,12 +554,16 @@ func (s *service) UpdatePublicShare(ctx context.Context, req *link.UpdatePublicS } // enforce password if needed - canOptOut, err := utils.CheckPermission(ctx, permission.DeleteReadOnlyPassword, gatewayClient) - if err != nil { - return &link.UpdatePublicShareResponse{ - Status: status.NewInternal(ctx, err.Error()), - }, nil + var canOptOut bool + if !isInternalLink { + canOptOut, err = utils.CheckPermission(ctx, permission.DeleteReadOnlyPassword, gatewayClient) + if err != nil { + return &link.UpdatePublicShareResponse{ + Status: status.NewInternal(ctx, err.Error()), + }, nil + } } + updatePassword := req.GetUpdate().GetType() == link.UpdatePublicShareRequest_Update_TYPE_PASSWORD setPassword := grant.GetPassword() diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go index 966b2a61cf1..a350175807a 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/assimilation.go @@ -46,52 +46,89 @@ import ( type ScanDebouncer struct { after time.Duration f func(item scanItem) - pending map[string]*time.Timer + pending sync.Map inProgress sync.Map mutex sync.Mutex } +type EventAction int + +const ( + ActionCreate EventAction = iota + ActionUpdate + ActionMove + ActionDelete +) + +type queueItem struct { + item scanItem + timer *time.Timer +} + // NewScanDebouncer returns a new SpaceDebouncer instance func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer { return &ScanDebouncer{ after: d, f: f, - pending: map[string]*time.Timer{}, + pending: sync.Map{}, inProgress: sync.Map{}, } } -// Debounce restars the debounce timer for the given space +// Debounce restarts the debounce timer for the given space func (d *ScanDebouncer) Debounce(item scanItem) { d.mutex.Lock() defer d.mutex.Unlock() path := item.Path force := item.ForceRescan - if t := d.pending[item.Path]; t != nil { - force = force || item.ForceRescan - t.Stop() + recurse := item.Recurse + if i, ok := d.pending.Load(item.Path); ok { + queueItem := i.(*queueItem) + force = force || queueItem.item.ForceRescan + recurse = recurse || queueItem.item.Recurse + queueItem.timer.Stop() } - d.pending[item.Path] = time.AfterFunc(d.after, func() { - if _, ok := d.inProgress.Load(path); ok { - // Reschedule this run for when the previous run has finished - d.mutex.Lock() - d.pending[path].Reset(d.after) - d.mutex.Unlock() - return - } + d.pending.Store(item.Path, &queueItem{ + item: item, + timer: time.AfterFunc(d.after, func() { + if _, ok := d.inProgress.Load(path); ok { + // Reschedule this run for when the previous run has finished + d.mutex.Lock() + if i, ok := d.pending.Load(path); ok { + i.(*queueItem).timer.Reset(d.after) + } - d.inProgress.Store(path, true) - defer d.inProgress.Delete(path) - d.f(scanItem{ - Path: path, - ForceRescan: force, - }) + d.mutex.Unlock() + return + } + + d.pending.Delete(path) + d.inProgress.Store(path, true) + defer d.inProgress.Delete(path) + d.f(scanItem{ + Path: path, + ForceRescan: force, + Recurse: recurse, + }) + }), }) } +// InProgress returns true if the given path is currently being processed +func (d *ScanDebouncer) InProgress(path string) bool { + d.mutex.Lock() + defer d.mutex.Unlock() + if _, ok := d.pending.Load(path); ok { + return true + } + + _, ok := d.inProgress.Load(path) + return ok +} + func (t *Tree) workScanQueue() { for i := 0; i < t.options.MaxConcurrency; i++ { go func() { @@ -103,17 +140,73 @@ func (t *Tree) workScanQueue() { log.Error().Err(err).Str("path", item.Path).Msg("failed to assimilate item") continue } + + if item.Recurse { + err = t.WarmupIDCache(item.Path, true) + if err != nil { + log.Error().Err(err).Str("path", item.Path).Msg("failed to warmup id cache") + } + } } }() } } // Scan scans the given path and updates the id chache -func (t *Tree) Scan(path string, forceRescan bool) error { - t.scanDebouncer.Debounce(scanItem{ - Path: path, - ForceRescan: forceRescan, - }) +func (t *Tree) Scan(path string, action EventAction, isDir bool, recurse bool) error { + // cases: + switch action { + case ActionCreate: + if !isDir { + // 1. New file (could be emitted as part of a new directory) + // -> assimilate file + // -> scan parent directory recursively + if !t.scanDebouncer.InProgress(filepath.Dir(path)) { + t.scanDebouncer.Debounce(scanItem{ + Path: path, + ForceRescan: false, + }) + } + t.scanDebouncer.Debounce(scanItem{ + Path: filepath.Dir(path), + ForceRescan: true, + Recurse: true, + }) + } else { + // 2. New directory + // -> scan directory + t.scanDebouncer.Debounce(scanItem{ + Path: path, + ForceRescan: true, + Recurse: true, + }) + } + + case ActionUpdate: + // 3. Updated file + // -> update file unless parent directory is being rescanned + if !t.scanDebouncer.InProgress(filepath.Dir(path)) { + t.scanDebouncer.Debounce(scanItem{ + Path: path, + ForceRescan: true, + }) + } + + case ActionMove: + // 4. Moved file + // -> update file + // 5. Moved directory + // -> update directory and all children + t.scanDebouncer.Debounce(scanItem{ + Path: path, + ForceRescan: isDir, + Recurse: isDir, + }) + + case ActionDelete: + _ = t.HandleFileDelete(path) + } + return nil } @@ -183,44 +276,43 @@ func (t *Tree) getOwnerAndIDs(path string) (*userv1beta1.UserId, string, string, return owner, nodeID, spaceID, parentID, nil } -func (t *Tree) assimilate(item scanItem) error { - var err error +func (t *Tree) findSpaceId(path string) (string, node.Attributes, error) { // find the space id, scope by the according user - spaceID := []byte("") - spaceCandidate := item.Path + spaceCandidate := path spaceAttrs := node.Attributes{} for strings.HasPrefix(spaceCandidate, t.options.Root) { - spaceAttrs, err = t.lookup.MetadataBackend().All(context.Background(), spaceCandidate) - if err == nil && len(spaceAttrs[prefixes.SpaceIDAttr]) > 0 { - spaceID = spaceAttrs[prefixes.SpaceIDAttr] + spaceAttrs, err := t.lookup.MetadataBackend().All(context.Background(), spaceCandidate) + spaceID := spaceAttrs[prefixes.SpaceIDAttr] + if err == nil && len(spaceID) > 0 { if t.options.UseSpaceGroups { // set the uid and gid for the space fi, err := os.Stat(spaceCandidate) if err != nil { - return err + return "", spaceAttrs, err } sys := fi.Sys().(*syscall.Stat_t) gid := int(sys.Gid) _, err = t.userMapper.ScopeUserByIds(-1, gid) if err != nil { - return err + return "", spaceAttrs, err } } - break + + return string(spaceID), spaceAttrs, nil } spaceCandidate = filepath.Dir(spaceCandidate) } - if len(spaceID) == 0 { - return fmt.Errorf("did not find space id for path") - } + return "", spaceAttrs, fmt.Errorf("could not find space for path %s", path) +} +func (t *Tree) assimilate(item scanItem) error { var id []byte - if !item.ForceRescan { - // already assimilated? - id, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr) - if err == nil { - return t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path) - } + var err error + + // First find the space id + spaceID, spaceAttrs, err := t.findSpaceId(item.Path) + if err != nil { + return err } // lock the file for assimilation @@ -240,64 +332,62 @@ func (t *Tree) assimilate(item scanItem) error { // check for the id attribute again after grabbing the lock, maybe the file was assimilated/created by us in the meantime id, err = t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr) if err == nil { - previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), string(spaceID), string(id)) + previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), spaceID, string(id)) - _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), item.Path) - if item.ForceRescan { - previousParentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) - if err != nil { - return err - } + // This item had already been assimilated in the past. Update the path + _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), spaceID, string(id), item.Path) - fi, err := t.updateFile(item.Path, string(id), string(spaceID)) - if err != nil { - return err - } + previousParentID, _ := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) - // was it moved? - if ok && previousPath != item.Path { - // purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair. - _ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath) - _ = t.lookup.MetadataBackend().Purge(previousPath) + fi, err := t.updateFile(item.Path, string(id), spaceID) + if err != nil { + return err + } - if fi.IsDir() { - // if it was moved and it is a directory we need to propagate the move - go func() { _ = t.WarmupIDCache(item.Path, false) }() - } + // was it moved? + if ok && len(previousParentID) > 0 && previousPath != item.Path { + // purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair. + _ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath) + _ = t.lookup.MetadataBackend().Purge(previousPath) - parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) - if err == nil && len(parentID) > 0 { - ref := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: t.options.MountID, - SpaceId: string(spaceID), - OpaqueId: string(parentID), - }, - Path: filepath.Base(item.Path), - } - oldRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: t.options.MountID, - SpaceId: string(spaceID), - OpaqueId: string(previousParentID), - }, - Path: filepath.Base(previousPath), - } - t.PublishEvent(events.ItemMoved{ - SpaceOwner: user, - Executant: user, - Owner: user, - Ref: ref, - OldReference: oldRef, - Timestamp: utils.TSNow(), - }) + if fi.IsDir() { + // if it was moved and it is a directory we need to propagate the move + go func() { _ = t.WarmupIDCache(item.Path, false) }() + } + + parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) + if err == nil && len(parentID) > 0 { + ref := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: spaceID, + OpaqueId: string(parentID), + }, + Path: filepath.Base(item.Path), } + oldRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: spaceID, + OpaqueId: string(previousParentID), + }, + Path: filepath.Base(previousPath), + } + t.PublishEvent(events.ItemMoved{ + SpaceOwner: user, + Executant: user, + Owner: user, + Ref: ref, + OldReference: oldRef, + Timestamp: utils.TSNow(), + }) } + // } } } else { // assimilate new file newId := uuid.New().String() - fi, err := t.updateFile(item.Path, newId, string(spaceID)) + fi, err := t.updateFile(item.Path, newId, spaceID) if err != nil { return err } @@ -305,7 +395,7 @@ func (t *Tree) assimilate(item scanItem) error { ref := &provider.Reference{ ResourceId: &provider.ResourceId{ StorageId: t.options.MountID, - SpaceId: string(spaceID), + SpaceId: spaceID, OpaqueId: newId, }, } @@ -455,17 +545,25 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error { return err } - return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + sizes := make(map[string]int64) + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock") { + // skip lock files + if isLockFile(path) { return nil } + // calculate tree sizes + if !info.IsDir() { + dir := filepath.Dir(path) + sizes[dir] += info.Size() + } + attribs, err := t.lookup.MetadataBackend().All(context.Background(), path) - if err == nil { + if err == nil && len(attribs[prefixes.IDAttr]) > 0 { nodeSpaceID := attribs[prefixes.SpaceIDAttr] if len(nodeSpaceID) > 0 { spaceID = nodeSpaceID @@ -496,10 +594,18 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error { id, ok := attribs[prefixes.IDAttr] if ok { _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), path) - } else if assimilate { - _ = t.Scan(path, false) } + } else if assimilate { + _ = t.assimilate(scanItem{Path: path, ForceRescan: true}) } return nil }) + if err != nil { + return err + } + + for dir, size := range sizes { + _ = t.lookup.MetadataBackend().Set(context.Background(), dir, prefixes.TreesizeAttr, []byte(fmt.Sprintf("%d", size))) + } + return nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go index 4f9f173e555..a61c1728026 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go @@ -61,15 +61,15 @@ start: } switch ev.Event { case "CREATE": - go func() { _ = w.tree.Scan(ev.Path, false) }() + go func() { _ = w.tree.Scan(ev.Path, ActionCreate, false, false) }() case "CLOSE": bytesWritten, err := strconv.Atoi(ev.BytesWritten) if err == nil && bytesWritten > 0 { - go func() { _ = w.tree.Scan(ev.Path, true) }() + go func() { _ = w.tree.Scan(ev.Path, ActionUpdate, false, true) }() } case "RENAME": go func() { - _ = w.tree.Scan(ev.Path, true) + _ = w.tree.Scan(ev.Path, ActionMove, false, true) _ = w.tree.WarmupIDCache(ev.Path, false) }() } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go index 67b4d5828ad..6d1e295269b 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go @@ -41,21 +41,21 @@ func (w *GpfsWatchFolderWatcher) Watch(topic string) { continue } - if strings.HasSuffix(lwev.Path, ".flock") || strings.HasSuffix(lwev.Path, ".mlock") { + if isLockFile(lwev.Path) { continue } switch { case strings.Contains(lwev.Event, "IN_CREATE"): - go func() { _ = w.tree.Scan(lwev.Path, false) }() + go func() { _ = w.tree.Scan(lwev.Path, ActionCreate, false, false) }() case strings.Contains(lwev.Event, "IN_CLOSE_WRITE"): bytesWritten, err := strconv.Atoi(lwev.BytesWritten) if err == nil && bytesWritten > 0 { - go func() { _ = w.tree.Scan(lwev.Path, true) }() + go func() { _ = w.tree.Scan(lwev.Path, ActionUpdate, false, true) }() } case strings.Contains(lwev.Event, "IN_MOVED_TO"): go func() { - _ = w.tree.Scan(lwev.Path, true) + _ = w.tree.Scan(lwev.Path, ActionMove, false, true) _ = w.tree.WarmupIDCache(lwev.Path, false) }() } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go index 9a6fa22a39f..c50ddc1d464 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/inotifywatcher.go @@ -2,7 +2,6 @@ package tree import ( "fmt" - "strings" "github.com/pablodz/inotifywaitgo/inotifywaitgo" ) @@ -43,20 +42,20 @@ func (iw *InotifyWatcher) Watch(path string) { select { case event := <-events: for _, e := range event.Events { - if strings.HasSuffix(event.Filename, ".flock") || strings.HasSuffix(event.Filename, ".mlock") { + if isLockFile(event.Filename) { continue } switch e { case inotifywaitgo.DELETE: go func() { _ = iw.tree.HandleFileDelete(event.Filename) }() case inotifywaitgo.CREATE: - go func() { _ = iw.tree.Scan(event.Filename, false) }() + go func() { _ = iw.tree.Scan(event.Filename, ActionCreate, event.IsDir, false) }() case inotifywaitgo.MOVED_TO: go func() { - _ = iw.tree.Scan(event.Filename, true) + _ = iw.tree.Scan(event.Filename, ActionMove, event.IsDir, true) }() case inotifywaitgo.CLOSE_WRITE: - go func() { _ = iw.tree.Scan(event.Filename, true) }() + go func() { _ = iw.tree.Scan(event.Filename, ActionUpdate, event.IsDir, true) }() } } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go index 68498f20704..bacceefe1ce 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree/tree.go @@ -75,6 +75,7 @@ type Watcher interface { type scanItem struct { Path string ForceRescan bool + Recurse bool } // Tree manages a hierarchical tree @@ -110,7 +111,7 @@ func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Opti idCache: cache, propagator: propagator.New(lu, &o.Options), scanQueue: scanQueue, - scanDebouncer: NewScanDebouncer(500*time.Millisecond, func(item scanItem) { + scanDebouncer: NewScanDebouncer(1000*time.Millisecond, func(item scanItem) { scanQueue <- item }), es: es, @@ -393,6 +394,10 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro g.Go(func() error { defer close(work) for _, name := range names { + if isLockFile(name) { + continue + } + select { case work <- name: case <-ctx.Done(): @@ -877,3 +882,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( return } + +func isLockFile(path string) bool { + return strings.HasSuffix(path, ".lock") || strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock") +} diff --git a/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/models.go b/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/models.go index 9eaf89641d0..48701fcb403 100644 --- a/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/models.go +++ b/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/models.go @@ -59,6 +59,9 @@ const ( EventDeleteSelf = "delete_self" // The filesystem on which a watched file or directory resides was unmounted. After this event the file or directory is no longer being watched. Note that this event can occur even if it is not explicitly being listened to. EventUnmount = "unmount" + + // The subject of this event is a directory + FlagIsdir = "ISDIR" ) type EVENT int @@ -84,6 +87,7 @@ const ( type FileEvent struct { Filename string Events []EVENT + IsDir bool } var EVENT_MAP = map[int]string{ diff --git a/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/watcher.go b/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/watcher.go index b953744d009..6529331967c 100644 --- a/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/watcher.go +++ b/vendor/github.com/pablodz/inotifywaitgo/inotifywaitgo/watcher.go @@ -75,8 +75,14 @@ func WatchPath(s *Settings) { } } var eventsEvents []EVENT + isDir := false for _, eventStr := range eventsStr { + if eventStr == FlagIsdir { + isDir = true + continue + } + eventStr = strings.ToLower(eventStr) event, ok := EVENT_MAP_REVERSE[eventStr] if !ok { @@ -89,6 +95,7 @@ func WatchPath(s *Settings) { event := FileEvent{ Filename: prefix + file, Events: eventsEvents, + IsDir: isDir, } // Send the file name to the channel diff --git a/vendor/modules.txt b/vendor/modules.txt index 047f7b61b2e..6db680f92ab 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -366,7 +366,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.21.1-0.20240725094850-569a027d5cc6 +# github.com/cs3org/reva/v2 v2.22.0 ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime @@ -1626,7 +1626,7 @@ github.com/owncloud/libre-graph-api-go # github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c ## explicit; go 1.12 github.com/oxtoacart/bpool -# github.com/pablodz/inotifywaitgo v0.0.6 +# github.com/pablodz/inotifywaitgo v0.0.7 ## explicit; go 1.21 github.com/pablodz/inotifywaitgo/inotifywaitgo # github.com/patrickmn/go-cache v2.1.0+incompatible From b83f2b4d2d336bc161e350220b01886ea5b816e3 Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Mon, 29 Jul 2024 17:45:22 +0200 Subject: [PATCH 2/3] tests: adapt acceptance tests --- .../apiSharingNgLinkSharePermission/createLinkShare.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature b/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature index f1d8cb3fccc..9e24619e7ce 100644 --- a/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature +++ b/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature @@ -1414,7 +1414,6 @@ Feature: Create a link share for a resource Examples: | drive | message | | Personal | cannot create link on personal space root | - | Shares | no share permission | Scenario Outline: try to create an internal link share with password of a Personal and Shares drives using permissions endpoint From c82c331dadb6feea04aec47e412af8506322ec6b Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Mon, 29 Jul 2024 18:04:46 +0200 Subject: [PATCH 3/3] tests: adapt expected failures line numbers --- ...expected-failures-localAPI-on-OCIS-storage.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md index 623342739be..758a28a5737 100644 --- a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md @@ -229,16 +229,16 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiSharingNgLinkSharePermission/createLinkShare.feature:1221](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1221) - [apiSharingNgLinkSharePermission/createLinkShare.feature:1222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1222) - [apiSharingNgLinkSharePermission/createLinkShare.feature:1374](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1374) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:1633](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1633) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:1632](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1632) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:1686](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1686) - [apiSharingNgLinkSharePermission/createLinkShare.feature:1687](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1687) - [apiSharingNgLinkSharePermission/createLinkShare.feature:1688](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1688) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:1689](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1689) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:1757](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1757) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:1823](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1823) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:1945](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1945) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:2072](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2072) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:2199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2199) -- [apiSharingNgLinkSharePermission/createLinkShare.feature:2327](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2327) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:1756](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1756) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:1822](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1822) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:1944](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L1944) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:2071](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2071) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:2198](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2198) +- [apiSharingNgLinkSharePermission/createLinkShare.feature:2326](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/createLinkShare.feature#L2326) - [apiSharingNgLinkSharePermission/updateLinkShare.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/updateLinkShare.feature#L73) - [apiSharingNgLinkSharePermission/updateLinkShare.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/updateLinkShare.feature#L75) - [apiSharingNgLinkSharePermission/updateLinkShare.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNgLinkSharePermission/updateLinkShare.feature#L76)