Skip to content

Commit

Permalink
Fix slicer upload in some cases (#486)
Browse files Browse the repository at this point in the history
  • Loading branch information
cthulhu-rider authored Aug 10, 2023
2 parents 38848dc + d61cba5 commit 4c702aa
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 12 deletions.
26 changes: 14 additions & 12 deletions object/slicer/slicer.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,24 +173,26 @@ func slice(ctx context.Context, ow ObjectWriter, header object.Object, data io.R

for {
n, err = data.Read(bChunk)
if err != nil {
if !errors.Is(err, io.EOF) {
return rootID, fmt.Errorf("read payload chunk: %w", err)
if n > 0 {
if _, err = writer.Write(bChunk[:n]); err != nil {
return oid.ID{}, err
}
}

// no more data to read

if err = writer.Close(); err != nil {
return rootID, fmt.Errorf("writer close: %w", err)
}
if err == nil {
continue
}

rootID = writer.ID()
break
if !errors.Is(err, io.EOF) {
return rootID, fmt.Errorf("read payload chunk: %w", err)
}

if _, err = writer.Write(bChunk[:n]); err != nil {
return oid.ID{}, err
if err = writer.Close(); err != nil {
return rootID, fmt.Errorf("writer close: %w", err)
}

rootID = writer.ID()
break
}

return rootID, nil
Expand Down
75 changes: 75 additions & 0 deletions object/slicer/slicer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,39 @@ func testSlicer(t *testing.T, size, sizeLimit uint64) {
}
}

// eofOnLastChunkReader is a special reader for tests. It returns io.EOF with the last data portion.
type eofOnLastChunkReader struct {
// this option enable returning 0, nil before fina; result with io.EOF error.
// Only in case when we return 0, EOF result.
isZeroNilShowed bool
isZeroOnEOF bool
payload []byte
i int
}

func (l *eofOnLastChunkReader) Read(p []byte) (int, error) {
n := copy(p, l.payload[l.i:])
l.i += n

if l.i == len(l.payload) {
if n == 0 {
// nothing happened case from io.Reader docs.
if !l.isZeroNilShowed {
l.isZeroNilShowed = true
return 0, nil
}

return 0, io.EOF
}

if !l.isZeroOnEOF {
return n, io.EOF
}
}

return n, nil
}

func testSlicerByHeaderType(t *testing.T, checker *slicedObjectChecker, in input, opts slicer.Options) {
ctx := context.Background()

Expand Down Expand Up @@ -364,6 +397,48 @@ func testSlicerByHeaderType(t *testing.T, checker *slicedObjectChecker, in input

checker.chainCollector.verify(checker.input, w.ID())
})

t.Run("slicer.Put, io.EOF in last chunk", func(t *testing.T) {
checker.chainCollector = newChainCollector(t)

var hdr object.Object
hdr.SetSessionToken(opts.Session())
hdr.SetContainerID(in.container)
hdr.SetOwnerID(&in.owner)
hdr.SetAttributes(in.attributes...)

rootID, err := slicer.Put(ctx, checker, hdr, checker.input.signer, &eofOnLastChunkReader{payload: in.payload, isZeroNilShowed: true}, opts)
require.NoError(t, err)
checker.chainCollector.verify(checker.input, rootID)
})

t.Run("slicer.Put, zeroNil before io.EOF in last chunk", func(t *testing.T) {
checker.chainCollector = newChainCollector(t)

var hdr object.Object
hdr.SetSessionToken(opts.Session())
hdr.SetContainerID(in.container)
hdr.SetOwnerID(&in.owner)
hdr.SetAttributes(in.attributes...)

rootID, err := slicer.Put(ctx, checker, hdr, checker.input.signer, &eofOnLastChunkReader{payload: in.payload}, opts)
require.NoError(t, err)
checker.chainCollector.verify(checker.input, rootID)
})

t.Run("slicer.Put, io.EOF after last chunk", func(t *testing.T) {
checker.chainCollector = newChainCollector(t)

var hdr object.Object
hdr.SetSessionToken(opts.Session())
hdr.SetContainerID(in.container)
hdr.SetOwnerID(&in.owner)
hdr.SetAttributes(in.attributes...)

rootID, err := slicer.Put(ctx, checker, hdr, checker.input.signer, &eofOnLastChunkReader{payload: in.payload, isZeroOnEOF: true, isZeroNilShowed: true}, opts)
require.NoError(t, err)
checker.chainCollector.verify(checker.input, rootID)
})
}

type slicedObjectChecker struct {
Expand Down

0 comments on commit 4c702aa

Please sign in to comment.