From 98ad62570ff848605c95180c003e8fd4598d0822 Mon Sep 17 00:00:00 2001 From: r-scheele Date: Thu, 29 Jun 2023 11:12:13 +0100 Subject: [PATCH 01/14] Enable stat operations for specific ranges --- api-get-object.go | 3 --- api-put-object_test.go | 3 --- functional_tests.go | 12 ++++++++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index e31e4cf92..1d426db2c 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -127,9 +127,6 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. - - // Remove range header if already set, for stat Operations to get original file size. - delete(opts.headers, "Range") objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ diff --git a/api-put-object_test.go b/api-put-object_test.go index ca35b12ed..dde4777ee 100644 --- a/api-put-object_test.go +++ b/api-put-object_test.go @@ -82,7 +82,6 @@ func Test_SSEHeaders(t *testing.T) { c, err := New("s3.amazonaws.com", &Options{ Transport: rt, }) - if err != nil { t.Error(err) } @@ -147,7 +146,6 @@ func Test_SSEHeaders(t *testing.T) { uploadID: "upId", sse: opts.ServerSideEncryption, }) - if err != nil { t.Error(err) } @@ -167,5 +165,4 @@ func Test_SSEHeaders(t *testing.T) { } }) } - } diff --git a/functional_tests.go b/functional_tests.go index 2bc6b8645..3c83d6f64 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2471,7 +2471,8 @@ func testTrailingChecksums() { PO minio.PutObjectOptions }{ // Currently there is no way to override the checksum type. - {header: "x-amz-checksum-crc32c", + { + header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ @@ -2481,7 +2482,8 @@ func testTrailingChecksums() { PartSize: 5 << 20, }, }, - {header: "x-amz-checksum-crc32c", + { + header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ @@ -2491,7 +2493,8 @@ func testTrailingChecksums() { PartSize: 6_645_654, // Rather arbitrary size }, }, - {header: "x-amz-checksum-crc32c", + { + header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ @@ -2501,7 +2504,8 @@ func testTrailingChecksums() { PartSize: 5 << 20, }, }, - {header: "x-amz-checksum-crc32c", + { + header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ From 336e059030a34867e37c808e3d73f9918e67b654 Mon Sep 17 00:00:00 2001 From: r-scheele Date: Wed, 5 Jul 2023 11:29:31 +0100 Subject: [PATCH 02/14] [updates] --- api-get-object.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api-get-object.go b/api-get-object.go index 1d426db2c..3b4e118ca 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -127,6 +127,8 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. + // Remove range header if already set, for stat Operations to get original file size. + delete(opts.headers, "Range") objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ @@ -412,7 +414,8 @@ func (o *Object) Stat() (ObjectInfo, error) { if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { return ObjectInfo{}, o.prevErr } - + o.isStarted = false + o.objectInfoSet = false // This is the first request. if !o.isStarted || !o.objectInfoSet { // Send the request and get the response. From c190491ae3ba3f518984cb1b049c9ae6395a2063 Mon Sep 17 00:00:00 2001 From: r-scheele Date: Wed, 5 Jul 2023 11:47:37 +0100 Subject: [PATCH 03/14] [updates] --- functional_tests.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/functional_tests.go b/functional_tests.go index 3c83d6f64..dac30046c 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2021,7 +2021,7 @@ func testObjectTaggingWithVersioning() { successLogger(testName, function, args, startTime).Info() } -// Test PutObject with custom checksums. +// testPutObjectWithChecksums tests PutObject with custom checksums. func testPutObjectWithChecksums() { // initialize logging params startTime := time.Now() @@ -2120,7 +2120,7 @@ func testPutObjectWithChecksums() { }) if err == nil { if i == 0 && resp.ChecksumCRC32 == "" { - ignoredLog(testName, function, args, startTime, "Checksums does not appear to be supported by backend").Info() + ignoredLog(testName, function, args, startTime, "Checksums do not appear to be supported by backend").Info() return } logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -2128,6 +2128,7 @@ func testPutObjectWithChecksums() { } // Set correct CRC. + h.Reset() h.Write(b) meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) reader.Close() From 5037bcf489daa4113c410596632ad1a7e9cb0e50 Mon Sep 17 00:00:00 2001 From: r-scheele Date: Wed, 5 Jul 2023 11:50:56 +0100 Subject: [PATCH 04/14] [updates] --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8ae0384eb..75b094402 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ validator golangci-lint functional_tests -.idea \ No newline at end of file +.idea +.DS_Store \ No newline at end of file From d3f1199d72c246705d5baf9904733344fe3c521b Mon Sep 17 00:00:00 2001 From: r-scheele Date: Wed, 5 Jul 2023 12:08:50 +0100 Subject: [PATCH 05/14] [updates] --- functional_tests.go | 169 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 150 insertions(+), 19 deletions(-) diff --git a/functional_tests.go b/functional_tests.go index dac30046c..be338cd1d 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2021,7 +2021,7 @@ func testObjectTaggingWithVersioning() { successLogger(testName, function, args, startTime).Info() } -// testPutObjectWithChecksums tests PutObject with custom checksums. +// Test put object with checksums. func testPutObjectWithChecksums() { // initialize logging params startTime := time.Now() @@ -2171,43 +2171,174 @@ func testPutObjectWithChecksums() { return } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Object Close failed", err) + // Discard the object + _, err = r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Discard object failed", err) return } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) - return + + delete(args, "range") + delete(args, "metadata") + } + + successLogger(testName, function, args, startTime).Info() +} +func testPutObjectWithChecksums() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", + } + + if !isFullMode() { + ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + defer cleanupBucket(bucketName, c) + tests := []struct { + header string + hasher hash.Hash + + // Checksum values + ChecksumCRC32 string + ChecksumCRC32C string + ChecksumSHA1 string + ChecksumSHA256 string + }{ + {header: "x-amz-checksum-crc32", hasher: crc32.NewIEEE()}, + {header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))}, + {header: "x-amz-checksum-sha1", hasher: sha1.New()}, + {header: "x-amz-checksum-sha256", hasher: sha256.New()}, + } + + for i, test := range tests { + bufSize := dataFileMap["datafile-10-kB"] + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + cmpChecksum := func(got, want string) { + if want != got { + logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) + return + } } - args["range"] = "true" - err = gopts.SetRange(100, 1000) + meta := map[string]string{} + reader := getDataReader("datafile-10-kB") + b, err := io.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "SetRange failed", err) + logError(testName, function, args, startTime, "", "Read failed", err) return } - r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) + h := test.hasher + h.Reset() + // Wrong CRC. + meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) + args["metadata"] = meta + args["range"] = "false" + + resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ + DisableMultipart: true, + UserMetadata: meta, + }) + if err == nil { + if i == 0 && resp.ChecksumCRC32 == "" { + ignoredLog(testName, function, args, startTime, "Checksums do not appear to be supported by backend").Info() + return + } + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Set correct CRC. + h.Reset() + h.Write(b) + meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) + reader.Close() + + resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ + DisableMultipart: true, + DisableContentSha256: true, + UserMetadata: meta, + }) if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } + cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + + // Read the data back + gopts := minio.GetObjectOptions{Checksum: true} - b, err = io.ReadAll(r) + r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { - logError(testName, function, args, startTime, "", "Read failed", err) + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - st, err = r.Stat() + + st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } + cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) + return + } - // Range requests should return empty checksums... - cmpChecksum(st.ChecksumSHA256, "") - cmpChecksum(st.ChecksumSHA1, "") - cmpChecksum(st.ChecksumCRC32, "") - cmpChecksum(st.ChecksumCRC32C, "") + // Discard the object + _, err = r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Discard object failed", err) + return + } delete(args, "range") delete(args, "metadata") From 474d3ace6359eeb91bbdffd5f7703ca9f4962dbd Mon Sep 17 00:00:00 2001 From: r-scheele Date: Wed, 5 Jul 2023 12:14:37 +0100 Subject: [PATCH 06/14] [updates] --- functional_tests.go | 162 -------------------------------------------- 1 file changed, 162 deletions(-) diff --git a/functional_tests.go b/functional_tests.go index be338cd1d..667912ae0 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2184,168 +2184,6 @@ func testPutObjectWithChecksums() { successLogger(testName, function, args, startTime).Info() } -func testPutObjectWithChecksums() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "PutObject(bucketName, objectName, reader,size, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", - } - - if !isFullMode() { - ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() - return - } - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.New(os.Getenv(serverEndpoint), - &minio.Options{ - Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), - Secure: mustParseBool(os.Getenv(enableHTTPS)), - }) - if err != nil { - logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket. - err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) - if err != nil { - logError(testName, function, args, startTime, "", "Make bucket failed", err) - return - } - - defer cleanupBucket(bucketName, c) - tests := []struct { - header string - hasher hash.Hash - - // Checksum values - ChecksumCRC32 string - ChecksumCRC32C string - ChecksumSHA1 string - ChecksumSHA256 string - }{ - {header: "x-amz-checksum-crc32", hasher: crc32.NewIEEE()}, - {header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))}, - {header: "x-amz-checksum-sha1", hasher: sha1.New()}, - {header: "x-amz-checksum-sha256", hasher: sha256.New()}, - } - - for i, test := range tests { - bufSize := dataFileMap["datafile-10-kB"] - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - cmpChecksum := func(got, want string) { - if want != got { - logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) - return - } - } - - meta := map[string]string{} - reader := getDataReader("datafile-10-kB") - b, err := io.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "Read failed", err) - return - } - h := test.hasher - h.Reset() - // Wrong CRC. - meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) - args["metadata"] = meta - args["range"] = "false" - - resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ - DisableMultipart: true, - UserMetadata: meta, - }) - if err == nil { - if i == 0 && resp.ChecksumCRC32 == "" { - ignoredLog(testName, function, args, startTime, "Checksums do not appear to be supported by backend").Info() - return - } - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - - // Set correct CRC. - h.Reset() - h.Write(b) - meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) - reader.Close() - - resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ - DisableMultipart: true, - DisableContentSha256: true, - UserMetadata: meta, - }) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } - cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) - cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) - cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) - cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) - - // Read the data back - gopts := minio.GetObjectOptions{Checksum: true} - - r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - - st, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) - cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) - cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) - cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) - - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) - return - } - - // Discard the object - _, err = r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Discard object failed", err) - return - } - - delete(args, "range") - delete(args, "metadata") - } - - successLogger(testName, function, args, startTime).Info() -} // Test PutObject with custom checksums. func testPutMultipartObjectWithChecksums() { From 87c173ec1c293392015ac2caf5f6ae82823959e3 Mon Sep 17 00:00:00 2001 From: r-scheele Date: Wed, 5 Jul 2023 17:48:12 +0100 Subject: [PATCH 07/14] [updates] --- api-get-object.go | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index 3b4e118ca..6e73f96c1 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -127,20 +127,31 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. - // Remove range header if already set, for stat Operations to get original file size. - delete(opts.headers, "Range") - objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) - if err != nil { + // Check if the range header is present. + _, rangeHeaderPresent := opts.headers["Range"] + if rangeHeaderPresent { + delete(opts.headers, "Range") + objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) + + if err != nil { + resCh <- getResponse{ + Error: err, + } + // Exit the go-routine. + return + } + etag = objectInfo.ETag + // Send back the first response. resCh <- getResponse{ - Error: err, + objectInfo: objectInfo, + } + } else { + // The range header is not present, continue with the existing objectInfo. + etag = objectInfo.ETag + // Send back the first response. + resCh <- getResponse{ + objectInfo: objectInfo, } - // Exit the go-routine. - return - } - etag = objectInfo.ETag - // Send back the first response. - resCh <- getResponse{ - objectInfo: objectInfo, } } } else if req.settingObjectInfo { // Request is just to get objectInfo. @@ -416,6 +427,7 @@ func (o *Object) Stat() (ObjectInfo, error) { } o.isStarted = false o.objectInfoSet = false + // This is the first request. if !o.isStarted || !o.objectInfoSet { // Send the request and get the response. From f30da45f9a4327c32e4f5d390d8b71f340482a5d Mon Sep 17 00:00:00 2001 From: r-scheele Date: Thu, 6 Jul 2023 11:03:37 +0100 Subject: [PATCH 08/14] Fix failing tests --- pkg/tags/tags_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tags/tags_test.go b/pkg/tags/tags_test.go index 89b68cc17..de43d3755 100644 --- a/pkg/tags/tags_test.go +++ b/pkg/tags/tags_test.go @@ -36,7 +36,7 @@ func TestParseTags(t *testing.T) { false, }, { - " store forever =false&factory=true", + "_store_forever_=false&factory=true", false, }, { From 2de26fd4d14e515b49ab675846b2383a857870ad Mon Sep 17 00:00:00 2001 From: r-scheele Date: Thu, 6 Jul 2023 12:08:00 +0100 Subject: [PATCH 09/14] [updates] --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 75b094402..2205aebe8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ validator golangci-lint functional_tests .idea -.DS_Store \ No newline at end of file +.DS_Store + +# Go workspace file +go.work \ No newline at end of file From e9c902a8ae91030d3dc313ff11d15b3b82ff7ee0 Mon Sep 17 00:00:00 2001 From: r-sc Date: Wed, 2 Aug 2023 11:40:42 +0100 Subject: [PATCH 10/14] [updates] - fixed failing tests --- api-get-object.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index 6e73f96c1..fdcbd6726 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -127,10 +127,10 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. - // Check if the range header is present. + + // Remove range header if already set, for stat Operations to get original file size. _, rangeHeaderPresent := opts.headers["Range"] if rangeHeaderPresent { - delete(opts.headers, "Range") objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { @@ -147,6 +147,16 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o } } else { // The range header is not present, continue with the existing objectInfo. + delete(opts.headers, "Range") + objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) + + if err != nil { + resCh <- getResponse{ + Error: err, + } + // Exit the go-routine. + return + } etag = objectInfo.ETag // Send back the first response. resCh <- getResponse{ From abe296eb5582dd5762354025ed1b7743f9a55029 Mon Sep 17 00:00:00 2001 From: r-sc Date: Wed, 2 Aug 2023 12:00:12 +0100 Subject: [PATCH 11/14] [updates] --- api-get-object.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api-get-object.go b/api-get-object.go index fdcbd6726..aea856f81 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -84,7 +84,9 @@ func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, o // Range is set with respect to the offset and length of the buffer requested. // Do not set objectInfo from the first readAt request because it will not get // the whole object. - opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + rangeStart := req.Offset + rangeEnd := req.Offset + int64(len(req.Buffer)) - 1 + opts.SetRange(rangeStart, rangeEnd) } else if req.Offset > 0 { opts.SetRange(req.Offset, 0) } From b13b28ebc4391f016d6a7a8b9140a737467ed44d Mon Sep 17 00:00:00 2001 From: r-sc Date: Mon, 7 Aug 2023 13:52:36 +0100 Subject: [PATCH 12/14] fixed failing tests --- api-get-object.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/api-get-object.go b/api-get-object.go index aea856f81..559900fea 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -437,9 +437,6 @@ func (o *Object) Stat() (ObjectInfo, error) { if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { return ObjectInfo{}, o.prevErr } - o.isStarted = false - o.objectInfoSet = false - // This is the first request. if !o.isStarted || !o.objectInfoSet { // Send the request and get the response. From fe34fff9f090b4777e32112c114b81bcfe46469b Mon Sep 17 00:00:00 2001 From: r-sc Date: Mon, 7 Aug 2023 14:13:28 +0100 Subject: [PATCH 13/14] [updates] --- .github/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vulncheck.yml b/.github/workflows/vulncheck.yml index 050445821..fd6b59963 100644 --- a/.github/workflows/vulncheck.yml +++ b/.github/workflows/vulncheck.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [ 1.20.6 ] + go-version: [ 1.20.7 ] steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 From 80aea4a2c4c1251bc0bf689e6c7dcceab4bb222f Mon Sep 17 00:00:00 2001 From: r-sc Date: Mon, 7 Aug 2023 16:09:13 +0100 Subject: [PATCH 14/14] [updates] --- functional_tests.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/functional_tests.go b/functional_tests.go index 73c4debef..79d499221 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -2170,7 +2170,28 @@ func testPutObjectWithChecksums() { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Object Close failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) + return + } + + args["range"] = "true" + err = gopts.SetRange(100, 1000) + if err != nil { + logError(testName, function, args, startTime, "", "SetRange failed", err) + return + } + r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + b, err = io.ReadAll(r) // Discard the object _, err = r.Stat() if err != nil {