diff --git a/mutate/compress.go b/mutate/compress.go index d8a2427e..9360e1c1 100644 --- a/mutate/compress.go +++ b/mutate/compress.go @@ -25,6 +25,10 @@ type Compressor interface { // WithOpt applies an option and can be chained. WithOpt(CompressorOpt) Compressor + + // BytesRead returns the number of bytes read from the uncompressed input + // stream at the current time, no guarantee of completion. + BytesRead() int64 } // CompressorOpt is a compressor option which can be used to configure a @@ -41,6 +45,10 @@ func (nc noopCompressor) MediaTypeSuffix() string { return "" } +func (nc noopCompressor) BytesRead() int64 { + return -1 +} + // NoopCompressor provides no compression. var NoopCompressor Compressor = noopCompressor{} @@ -55,9 +63,10 @@ type GzipBlockSize int type gzipCompressor struct { blockSize int + bytesRead int64 } -func (gz gzipCompressor) Compress(reader io.Reader) (io.ReadCloser, error) { +func (gz *gzipCompressor) Compress(reader io.Reader) (io.ReadCloser, error) { pipeReader, pipeWriter := io.Pipe() gzw := gzip.NewWriter(pipeWriter) @@ -65,12 +74,14 @@ func (gz gzipCompressor) Compress(reader io.Reader) (io.ReadCloser, error) { return nil, errors.Wrapf(err, "set concurrency level to %v blocks", 2*runtime.NumCPU()) } go func() { - if _, err := system.Copy(gzw, reader); err != nil { + bytesRead, err := system.Copy(gzw, reader) + if err != nil { log.Warnf("gzip compress: could not compress layer: %v", err) // #nosec G104 _ = pipeWriter.CloseWithError(errors.Wrap(err, "compressing layer")) return } + gz.bytesRead = bytesRead if err := gzw.Close(); err != nil { log.Warnf("gzip compress: could not close gzip writer: %v", err) // #nosec G104 @@ -100,12 +111,18 @@ func (gz gzipCompressor) WithOpt(opt CompressorOpt) Compressor { return gz } +func (gz gzipCompressor) BytesRead() int64 { + return gz.bytesRead +} + // ZstdCompressor provides zstd compression. -var ZstdCompressor Compressor = zstdCompressor{} +var ZstdCompressor Compressor = &zstdCompressor{} -type zstdCompressor struct{} +type zstdCompressor struct { + bytesRead int64 +} -func (zs zstdCompressor) Compress(reader io.Reader) (io.ReadCloser, error) { +func (zs *zstdCompressor) Compress(reader io.Reader) (io.ReadCloser, error) { pipeReader, pipeWriter := io.Pipe() zw, err := zstd.NewWriter(pipeWriter) @@ -113,12 +130,14 @@ func (zs zstdCompressor) Compress(reader io.Reader) (io.ReadCloser, error) { return nil, err } go func() { - if _, err := system.Copy(zw, reader); err != nil { + bytesRead, err := system.Copy(zw, reader) + if err != nil { log.Warnf("zstd compress: could not compress layer: %v", err) // #nosec G104 _ = pipeWriter.CloseWithError(errors.Wrap(err, "compressing layer")) return } + zs.bytesRead = bytesRead if err := zw.Close(); err != nil { log.Warnf("zstd compress: could not close gzip writer: %v", err) // #nosec G104 @@ -142,3 +161,7 @@ func (zs zstdCompressor) MediaTypeSuffix() string { func (zs zstdCompressor) WithOpt(CompressorOpt) Compressor { return zs } + +func (zs zstdCompressor) BytesRead() int64 { + return zs.bytesRead +} diff --git a/mutate/mutate.go b/mutate/mutate.go index 7f93b292..f0c8f5f0 100644 --- a/mutate/mutate.go +++ b/mutate/mutate.go @@ -24,6 +24,7 @@ package mutate import ( "context" + "fmt" "io" "reflect" "time" @@ -36,6 +37,8 @@ import ( "github.com/pkg/errors" ) +const UmociUncompressedBlobSizeAnnotation = "ci.umo.uncompressed_blob_size" + func configPtr(c ispec.Image) *ispec.Image { return &c } func manifestPtr(m ispec.Manifest) *ispec.Manifest { return &m } func timePtr(t time.Time) *time.Time { return &t } @@ -295,6 +298,13 @@ func (m *Mutator) Add(ctx context.Context, mediaType string, r io.Reader, histor compressedMediaType = compressedMediaType + "+" + compressor.MediaTypeSuffix() } + if annotations == nil { + annotations = make(map[string]string) + } + if compressor.BytesRead() >= 0 { + annotations[UmociUncompressedBlobSizeAnnotation] = fmt.Sprintf("%d", compressor.BytesRead()) + } + // Append to layers. desc = ispec.Descriptor{ MediaType: compressedMediaType, diff --git a/mutate/mutate_test.go b/mutate/mutate_test.go index 4dd859d5..3fc48843 100644 --- a/mutate/mutate_test.go +++ b/mutate/mutate_test.go @@ -214,6 +214,7 @@ func TestMutateAdd(t *testing.T) { // This isn't a valid image, but whatever. buffer := bytes.NewBufferString("contents") + bufferSize := buffer.Len() // Add a new layer. annotations := map[string]string{"hello": "world"} @@ -253,10 +254,18 @@ func TestMutateAdd(t *testing.T) { if mutator.manifest.Layers[1].Digest == expectedLayerDigest { t.Errorf("manifest.Layers[1].Digest is not the same!") } - if len(mutator.manifest.Layers[1].Annotations) != 1 || mutator.manifest.Layers[1].Annotations["hello"] != "world" { - t.Errorf("manifest.Layers[1].Annotations was not set correctly!") + if len(mutator.manifest.Layers[1].Annotations) != 2 { + t.Errorf("manifest.Layers[1].Annotations was not set correctly!: %+v", mutator.manifest.Layers[1].Annotations) + } + if mutator.manifest.Layers[1].Annotations["hello"] != "world" { + t.Errorf("manifest.Layers[1].Annotations['hello'] was not set correctly!: %+v", mutator.manifest.Layers[1].Annotations) + } + if mutator.manifest.Layers[1].Annotations[UmociUncompressedBlobSizeAnnotation] != fmt.Sprintf("%d", bufferSize) { + t.Errorf("manifest.Layers[1].Annotations['%s'] was not set correctly!: %q, expected %d", + UmociUncompressedBlobSizeAnnotation, + mutator.manifest.Layers[1].Annotations[UmociUncompressedBlobSizeAnnotation], + bufferSize) } - if mutator.manifest.Layers[1].Digest != newLayerDesc.Digest { t.Fatalf("unexpected digest for new layer: %v %v", mutator.manifest.Layers[1].Digest, newLayerDesc.Digest) } diff --git a/test/create.bats b/test/create.bats index ae7f783d..42ac0de5 100644 --- a/test/create.bats +++ b/test/create.bats @@ -277,8 +277,8 @@ function teardown() { # Verify that the hashes of the blobs and index match (blobs are # content-addressable so using hashes is a bit silly, but whatever). known_hashes=( - "6f3716b8ae2bb4b3fd0a851f5a553946ea351ed5fdda0bd708d325363c0487f7 $IMAGE/index.json" - "73ecb862d0ebee78acdf8553d34d1ae5c13df509030ac262e561d096bb480dfc $IMAGE/blobs/sha256/73ecb862d0ebee78acdf8553d34d1ae5c13df509030ac262e561d096bb480dfc" + "b780d08bfed4850ab807b7c308682fae6868ed9c09c0c842063c418ebe2a19fb $IMAGE/index.json" + "27bcd3b4f31740cc087346382aaba3fe1c01872d75ead1bd2b9f7053d2bb3231 $IMAGE/blobs/sha256/27bcd3b4f31740cc087346382aaba3fe1c01872d75ead1bd2b9f7053d2bb3231" "e7013826daf8b5d68f82c5b790ca5e9de222a834f2cb3fe3532030161bd72083 $IMAGE/blobs/sha256/e7013826daf8b5d68f82c5b790ca5e9de222a834f2cb3fe3532030161bd72083" "f4a39a97d97aa834da7ad2d92940f9636a57e3d9b3cc7c53242451b02a6cea89 $IMAGE/blobs/sha256/f4a39a97d97aa834da7ad2d92940f9636a57e3d9b3cc7c53242451b02a6cea89" )