From c127a0ef1effd06a5670f28ad6a41df72e4c1e02 Mon Sep 17 00:00:00 2001 From: Glyn Normington Date: Mon, 12 Aug 2019 16:25:29 +0100 Subject: [PATCH] Load images from docker daemon in export and relocate This is implemented by bumping pivotal/image-relocation to pick up the docker daemon support. Fixes https://github.com/deislabs/duffle/issues/828 --- Gopkg.lock | 15 +- Gopkg.toml | 4 +- .../cmd/ko/test/kodata/kenobi | 1 - .../pkg/authn/keychain.go | 8 +- .../pkg/internal/retry/retry.go | 68 ++++ .../go-containerregistry/pkg/logs/logs.go | 29 ++ .../go-containerregistry/pkg/name/digest.go | 1 - .../go-containerregistry/pkg/name/doc.go | 42 +++ .../go-containerregistry/pkg/v1/daemon/doc.go | 17 + .../pkg/v1/daemon/image.go | 133 +++++++ .../pkg/v1/daemon/options.go | 34 ++ .../pkg/v1/daemon/write.go | 80 ++++ .../go-containerregistry/pkg/v1/image.go | 1 + .../go-containerregistry/pkg/v1/index.go | 1 + .../go-containerregistry/pkg/v1/layer.go | 5 + .../pkg/v1/layout/image.go | 4 + .../pkg/v1/layout/options.go | 5 +- .../pkg/v1/partial/compressed.go | 4 + .../pkg/v1/partial/uncompressed.go | 3 + .../pkg/v1/partial/with.go | 7 + .../go-containerregistry/pkg/v1/platform.go | 1 + .../pkg/v1/random/image.go | 8 + .../pkg/v1/remote/check.go | 5 +- .../pkg/v1/remote/delete.go | 9 +- .../pkg/v1/remote/descriptor.go | 42 +-- .../pkg/v1/remote/image.go | 22 +- .../pkg/v1/remote/index.go | 55 ++- .../pkg/v1/remote/list.go | 12 +- .../pkg/v1/remote/options.go | 90 +++-- .../pkg/v1/remote/transport/basic.go | 2 +- .../pkg/v1/remote/transport/bearer.go | 45 ++- .../pkg/v1/remote/transport/error.go | 14 + .../pkg/v1/remote/transport/retry.go | 89 +++++ .../pkg/v1/remote/write.go | 135 ++++--- .../pkg/v1/stream/layer.go | 194 ---------- .../pkg/v1/tarball/doc.go | 17 + .../pkg/v1/tarball/image.go | 352 ++++++++++++++++++ .../pkg/v1/tarball/layer.go | 162 ++++++++ .../pkg/v1/tarball/write.go | 194 ++++++++++ .../image-relocation/pkg/registry/remote.go | 28 +- 40 files changed, 1588 insertions(+), 350 deletions(-) delete mode 120000 vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi create mode 100644 vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/logs/logs.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/name/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go delete mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go diff --git a/Gopkg.lock b/Gopkg.lock index bb951427..d091da41 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -471,24 +471,27 @@ version = "v1.0.0" [[projects]] - digest = "1:2e588e08805f44a6b53d08a47a48b1b7fdb6903076fc8c245f1ec769b28ceb74" + digest = "1:56a9f68e8f260c3be2b1d997b7b7cf8b2ea019d52fea350219158571a61e0236" name = "github.com/google/go-containerregistry" packages = [ "pkg/authn", + "pkg/internal/retry", + "pkg/logs", "pkg/name", "pkg/v1", + "pkg/v1/daemon", "pkg/v1/empty", "pkg/v1/layout", "pkg/v1/partial", "pkg/v1/random", "pkg/v1/remote", "pkg/v1/remote/transport", - "pkg/v1/stream", + "pkg/v1/tarball", "pkg/v1/types", "pkg/v1/v1util", ] pruneopts = "NUT" - revision = "5296537b6d5d12241dd0a8b1ef70a59cba62f48c" + revision = "ef12d49c8daf6a6d72978966d5945e39bb898b4a" [[projects]] digest = "1:52c5834e2bebac9030c97cc0798ac11c3aa8a39f098aeb419f142533da6cd3cc" @@ -733,7 +736,7 @@ version = "v3.0.0" [[projects]] - digest = "1:87bef79892c094091d2ce4e1d9244e79660baf2f839ab9f075f82ed19a006263" + digest = "1:f822c559b4c70e3493d03c6f3861a208507ef9bf148c57b0531c5ac85ab7fe30" name = "github.com/pivotal/image-relocation" packages = [ "pkg/image", @@ -741,8 +744,8 @@ "pkg/registry", ] pruneopts = "NUT" - revision = "3c9c32bb3d97fc213476f1c9846a99ca35ecba5d" - version = "v0.1" + revision = "ea07ec0b5a8a8e6fb53e33d479fa557bd5faa023" + version = "v0.2" [[projects]] digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24" diff --git a/Gopkg.toml b/Gopkg.toml index 3e077884..d8015d77 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -53,7 +53,7 @@ [[constraint]] name = "github.com/pivotal/image-relocation" - version = "v0.1" + version = "v0.2" [[constraint]] name = "github.com/deislabs/cnab-go" @@ -61,7 +61,7 @@ [[override]] name = "github.com/google/go-containerregistry" - revision = "5296537b6d5d12241dd0a8b1ef70a59cba62f48c" + revision = "ef12d49c8daf6a6d72978966d5945e39bb898b4a" [[override]] name = "k8s.io/apimachinery" diff --git a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi b/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi deleted file mode 120000 index 5d7eddc7..00000000 --- a/vendor/github.com/google/go-containerregistry/cmd/ko/test/kodata/kenobi +++ /dev/null @@ -1 +0,0 @@ -../kenobi \ No newline at end of file diff --git a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go index aee1fedb..8b2ead5d 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go +++ b/vendor/github.com/google/go-containerregistry/pkg/authn/keychain.go @@ -19,11 +19,11 @@ import ( "errors" "fmt" "io/ioutil" - "log" "os" "path/filepath" "runtime" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" ) @@ -100,19 +100,19 @@ var ( func (dk *defaultKeychain) Resolve(reg name.Registry) (Authenticator, error) { dir, err := configDir() if err != nil { - log.Printf("Unable to determine config dir: %v", err) + logs.Warn.Printf("Unable to determine config dir: %v", err) return Anonymous, nil } file := filepath.Join(dir, "config.json") content, err := ioutil.ReadFile(file) if err != nil { - log.Printf("Unable to read %q: %v", file, err) + logs.Warn.Printf("Unable to read %q: %v", file, err) return Anonymous, nil } var cf cfg if err := json.Unmarshal(content, &cf); err != nil { - log.Printf("Unable to parse %q: %v", file, err) + logs.Warn.Printf("Unable to parse %q: %v", file, err) return Anonymous, nil } diff --git a/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go new file mode 100644 index 00000000..87f73095 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/internal/retry/retry.go @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package retry provides methods for retrying operations. It is a thin wrapper +// around k8s.io/apimachinery/pkg/util/wait to make certain operations easier. +package retry + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// This is implemented by several errors in the net package as well as our +// transport.Error. +type temporary interface { + Temporary() bool +} + +// IsTemporary returns true if err implements Temporary() and it returns true. +func IsTemporary(err error) bool { + if te, ok := err.(temporary); ok && te.Temporary() { + return true + } + return false +} + +// IsNotNil returns true if err is not nil. +func IsNotNil(err error) bool { + return err != nil +} + +// Predicate determines whether an error should be retried. +type Predicate func(error) (retry bool) + +// Retry retries a given function, f, until a predicate is satisfied, using +// exponential backoff. If the predicate is never satisfied, it will return the +// last error returned by f. +func Retry(f func() error, p Predicate, backoff wait.Backoff) (err error) { + if f == nil { + return fmt.Errorf("nil f passed to retry") + } + if p == nil { + return fmt.Errorf("nil p passed to retry") + } + + condition := func() (bool, error) { + err = f() + if p(err) { + return false, nil + } + return true, err + } + + wait.ExponentialBackoff(backoff, condition) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go new file mode 100644 index 00000000..af3c1a3b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/logs/logs.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package logs exposes the loggers used by this library. +package logs + +import ( + "io/ioutil" + "log" +) + +var ( + // Warn is used to log non-fatal errors. + Warn = log.New(ioutil.Discard, "", log.LstdFlags) + + // Progress is used to log notable, successful events. + Progress = log.New(ioutil.Discard, "", log.LstdFlags) +) diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go index d10856de..2dc0f7f3 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/digest.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/digest.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package name defines structured types for representing image references. package name import ( diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/doc.go b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go new file mode 100644 index 00000000..b294794d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/name/doc.go @@ -0,0 +1,42 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package name defines structured types for representing image references. +// +// What's in a name? For image references, not nearly enough! +// +// Image references look a lot like URLs, but they differ in that they don't +// contain the scheme (http or https), they can end with a :tag or a @digest +// (the latter being validated), and they perform defaulting for missing +// components. +// +// Since image references don't contain the scheme, we do our best to infer +// if we use http or https from the given hostname. We allow http fallback for +// any host that looks like localhost (localhost, 127.0.0.1, ::1), ends in +// ".local", or is in the "private" address space per RFC 1918. For everything +// else, we assume https only. To override this heuristic, use the Insecure +// option. +// +// Image references with a digest signal to us that we should verify the content +// of the image matches the digest. E.g. when pulling a Digest reference, we'll +// calculate the sha256 of the manifest returned by the registry and error out +// if it doesn't match what we asked for. +// +// For defaulting, we interpret "ubuntu" as +// "index.docker.io/library/ubuntu:latest" because we add the missing repo +// "library", the missing registry "index.docker.io", and the missing tag +// "latest". To disable this defaulting, use the StrictValidation option. This +// is useful e.g. to only allow image references that explicitly set a tag or +// digest, so that you don't accidentally pull "latest". +package name diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go new file mode 100644 index 00000000..ac05d961 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package daemon provides facilities for reading/writing v1.Image from/to +// a running daemon. +package daemon diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go new file mode 100644 index 00000000..b7f5f3ef --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go @@ -0,0 +1,133 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "bytes" + "context" + "io" + "io/ioutil" + + "github.com/docker/docker/client" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// image accesses an image from a docker daemon +type image struct { + v1.Image + + opener tarball.Opener + ref name.Reference +} + +var _ v1.Image = (*image)(nil) + +type imageOpener struct { + ref name.Reference + buffered bool +} + +// ImageOption is a functional option for Image. +type ImageOption func(*imageOpener) error + +func (i *imageOpener) Open() (v1.Image, error) { + var opener tarball.Opener + var err error + if i.buffered { + opener, err = bufferedOpener(i.ref) + } else { + opener, err = unbufferedOpener(i.ref) + } + if err != nil { + return nil, err + } + + tb, err := tarball.Image(opener, nil) + if err != nil { + return nil, err + } + img := &image{ + Image: tb, + } + return img, nil +} + +// ImageSaver is an interface for testing. +type ImageSaver interface { + ImageSave(context.Context, []string) (io.ReadCloser, error) +} + +// This is a variable so we can override in tests. +var getImageSaver = func() (ImageSaver, error) { + cli, err := client.NewEnvClient() + if err != nil { + return nil, err + } + cli.NegotiateAPIVersion(context.Background()) + return cli, nil +} + +func saveImage(ref name.Reference) (io.ReadCloser, error) { + cli, err := getImageSaver() + if err != nil { + return nil, err + } + + return cli.ImageSave(context.Background(), []string{ref.Name()}) +} + +func bufferedOpener(ref name.Reference) (tarball.Opener, error) { + // Store the tarball in memory and return a new reader into the bytes each time we need to access something. + rc, err := saveImage(ref) + if err != nil { + return nil, err + } + defer rc.Close() + + imageBytes, err := ioutil.ReadAll(rc) + if err != nil { + return nil, err + } + // The tarball interface takes a function that it can call to return an opened reader-like object. + // Daemon comes from a set of bytes, so wrap them in a ReadCloser so it looks like an opened file. + return func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(imageBytes)), nil + }, nil +} + +func unbufferedOpener(ref name.Reference) (tarball.Opener, error) { + // To avoid storing the tarball in memory, do a save every time we need to access something. + return func() (io.ReadCloser, error) { + return saveImage(ref) + }, nil +} + +// Image provides access to an image reference from the Docker daemon, +// applying functional options to the underlying imageOpener before +// resolving the reference into a v1.Image. +func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { + i := &imageOpener{ + ref: ref, + buffered: true, // buffer by default + } + for _, option := range options { + if err := option(i); err != nil { + return nil, err + } + } + return i.Open() +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go new file mode 100644 index 00000000..4e03952e --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go @@ -0,0 +1,34 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +// WithBufferedOpener buffers the image. +func WithBufferedOpener() ImageOption { + return func(i *imageOpener) error { + return i.setBuffered(true) + } +} + +// WithUnbufferedOpener streams the image to avoid buffering. +func WithUnbufferedOpener() ImageOption { + return func(i *imageOpener) error { + return i.setBuffered(false) + } +} + +func (i *imageOpener) setBuffered(buffer bool) error { + i.buffered = buffer + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go new file mode 100644 index 00000000..09e9bbdb --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go @@ -0,0 +1,80 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "context" + "io" + "io/ioutil" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/pkg/errors" +) + +// ImageLoader is an interface for testing. +type ImageLoader interface { + ImageLoad(context.Context, io.Reader, bool) (types.ImageLoadResponse, error) + ImageTag(context.Context, string, string) error +} + +// GetImageLoader is a variable so we can override in tests. +var GetImageLoader = func() (ImageLoader, error) { + cli, err := client.NewEnvClient() + if err != nil { + return nil, err + } + cli.NegotiateAPIVersion(context.Background()) + return cli, nil +} + +// Tag adds a tag to an already existent image. +func Tag(src, dest name.Tag) error { + cli, err := GetImageLoader() + if err != nil { + return err + } + + return cli.ImageTag(context.Background(), src.String(), dest.String()) +} + +// Write saves the image into the daemon as the given tag. +func Write(tag name.Tag, img v1.Image) (string, error) { + cli, err := GetImageLoader() + if err != nil { + return "", err + } + + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(tarball.Write(tag, img, pw)) + }() + + // write the image in docker save format first, then load it + resp, err := cli.ImageLoad(context.Background(), pr, false) + if err != nil { + return "", errors.Wrapf(err, "error loading image") + } + defer resp.Body.Close() + b, readErr := ioutil.ReadAll(resp.Body) + response := string(b) + if readErr != nil { + return response, errors.Wrapf(err, "error reading load response body") + } + return response, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go index 17b9839a..9ef02679 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/image.go @@ -19,6 +19,7 @@ import ( ) // Image defines the interface for interacting with an OCI v1 image. +//go:generate counterfeiter -o fake/image.go . Image type Image interface { // Layers returns the ordered collection of filesystem layers that comprise this image. // The order of the list is oldest/base layer first, and most-recent/top layer last. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go index 604e6de3..b4e84e02 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/index.go @@ -19,6 +19,7 @@ import ( ) // ImageIndex defines the interface for interacting with an OCI image index. +//go:generate counterfeiter -o fake/index.go . ImageIndex type ImageIndex interface { // MediaType of this image's manifest. MediaType() (types.MediaType, error) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go index 8b5091e4..57447d26 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layer.go @@ -16,6 +16,8 @@ package v1 import ( "io" + + "github.com/google/go-containerregistry/pkg/v1/types" ) // Layer is an interface for accessing the properties of a particular layer of a v1.Image @@ -34,4 +36,7 @@ type Layer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // MediaType returns the media type of the Layer. + MediaType() (types.MediaType, error) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go index ea036771..7c76a10c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go @@ -125,3 +125,7 @@ func (b *compressedBlob) Compressed() (io.ReadCloser, error) { func (b *compressedBlob) Size() (int64, error) { return b.desc.Size, nil } + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go index b27633d7..5569e51d 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -28,10 +28,7 @@ func WithURLs(urls []string) Option { if desc.URLs == nil { desc.URLs = []string{} } - for _, url := range urls { - desc.URLs = append(desc.URLs, url) - } - + desc.URLs = append(desc.URLs, urls...) return nil } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go index 497d1af0..1c631b7a 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/compressed.go @@ -18,6 +18,7 @@ import ( "io" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -32,6 +33,9 @@ type CompressedLayer interface { // Size returns the compressed size of the Layer. Size() (int64, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // compressedLayerExtender implements v1.Image using the compressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go index 9f75723e..07658be4 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/uncompressed.go @@ -32,6 +32,9 @@ type UncompressedLayer interface { // Uncompressed returns an io.ReadCloser for the uncompressed layer contents. Uncompressed() (io.ReadCloser, error) + + // Returns the mediaType for the compressed Layer + MediaType() (types.MediaType, error) } // uncompressedLayerExtender implements v1.Image using the uncompressed base properties. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index f724ec8a..99267204 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -22,6 +22,7 @@ import ( "io/ioutil" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/go-containerregistry/pkg/v1/v1util" ) @@ -80,6 +81,12 @@ func (cl *configLayer) Size() (int64, error) { return int64(len(cl.content)), nil } +func (cl *configLayer) MediaType() (types.MediaType, error) { + // Defaulting this to OCIConfigJSON as it should remain + // backwards compatible with DockerConfigJSON + return types.OCIConfigJSON, nil +} + var _ v1.Layer = (*configLayer)(nil) // ConfigLayer implements v1.Layer from the raw config bytes. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go index df9b2959..bb988643 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/platform.go @@ -21,4 +21,5 @@ type Platform struct { OSVersion string `json:"os.version,omitempty"` OSFeatures []string `json:"os.features,omitempty"` Variant string `json:"variant,omitempty"` + Features []string `json:"features,omitempty"` } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go index cc269d6b..e09984cb 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/random/image.go @@ -45,6 +45,14 @@ func (ul *uncompressedLayer) Uncompressed() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer(ul.content)), nil } +// MediaType returns the media type of the layer +func (ul *uncompressedLayer) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + var _ partial.UncompressedLayer = (*uncompressedLayer)(nil) // Image returns a pseudo-randomly generated Image. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index aa574eb8..da0fa24c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -1,6 +1,7 @@ package remote import ( + "fmt" "net/http" "github.com/google/go-containerregistry/pkg/authn" @@ -18,13 +19,13 @@ import ( func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTripper) error { auth, err := kc.Resolve(ref.Context().Registry) if err != nil { - return err + return fmt.Errorf("resolving authorization for %v failed: %v", ref.Context().Registry, err) } scopes := []string{ref.Scope(transport.PushScope)} tr, err := transport.New(ref.Context().Registry, auth, t, scopes) if err != nil { - return err + return fmt.Errorf("creating push check transport for %v failed: %v", ref.Context().Registry, err) } // TODO(jasonhall): Against GCR, just doing the token handshake is // enough, but this doesn't extend to Dockerhub diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go index 2032e276..c034435f 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/delete.go @@ -20,15 +20,18 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) // Delete removes the specified image reference from the remote registry. -func Delete(ref name.Reference, auth authn.Authenticator, t http.RoundTripper) error { +func Delete(ref name.Reference, options ...Option) error { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.DeleteScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go index 9c570b7f..144b99ec 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/descriptor.go @@ -23,7 +23,6 @@ import ( "net/url" "strings" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" @@ -52,18 +51,10 @@ type Descriptor struct { platform v1.Platform } -type imageOpener struct { - auth authn.Authenticator - transport http.RoundTripper - ref name.Reference - client *http.Client - platform v1.Platform -} - // Get returns a remote.Descriptor for the given reference. The response from // the registry is left un-interpreted, for the most part. This is useful for // querying what kind of artifact a reference represents. -func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) { +func Get(ref name.Reference, options ...Option) (*Descriptor, error) { acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, @@ -78,26 +69,19 @@ func Get(ref name.Reference, options ...ImageOption) (*Descriptor, error) { // Handle options and fetch the manifest with the acceptable MediaTypes in the // Accept header. -func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOption) (*Descriptor, error) { - i := &imageOpener{ - auth: authn.Anonymous, - transport: http.DefaultTransport, - ref: ref, - platform: defaultPlatform, +func get(ref name.Reference, acceptable []types.MediaType, options ...Option) (*Descriptor, error) { + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return nil, err } - for _, option := range options { - if err := option(i); err != nil { - return nil, err - } - } - tr, err := transport.New(i.ref.Context().Registry, i.auth, i.transport, []string{i.ref.Scope(transport.PullScope)}) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, []string{ref.Scope(transport.PullScope)}) if err != nil { return nil, err } f := fetcher{ - Ref: i.ref, + Ref: ref, Client: &http.Client{Transport: tr}, } @@ -110,7 +94,7 @@ func get(ref name.Reference, acceptable []types.MediaType, options ...ImageOptio fetcher: f, Manifest: b, Descriptor: *desc, - platform: i.platform, + platform: o.platform, }, nil } @@ -241,12 +225,16 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType } mediaType := types.MediaType(resp.Header.Get("Content-Type")) + contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest")) + if err == nil && mediaType == types.DockerManifestSchema1Signed { + // If we can parse the digest from the header, and it's a signed schema 1 + // manifest, let's use that for the digest to appease older registries. + digest = contentDigest + } // Validate the digest matches what we asked for, if pulling by digest. if dgst, ok := ref.(name.Digest); ok { - if mediaType == types.DockerManifestSchema1Signed { - // Digests for this are stupid to calculate, ignore it. - } else if digest.String() != dgst.DigestStr() { + if digest.String() != dgst.DigestStr() { return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), f.Ref) } } else { diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go index 9f2d51e8..752c7160 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/image.go @@ -15,6 +15,7 @@ package remote import ( + "fmt" "io" "io/ioutil" "net/http" @@ -40,9 +41,8 @@ type remoteImage struct { var _ partial.CompressedImageCore = (*remoteImage)(nil) -// Image provides access to a remote image reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.Image. -func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) { +// Image provides access to a remote image reference. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { acceptable := []types.MediaType{ types.DockerManifestSchema2, types.OCIManifestSchema1, @@ -151,6 +151,22 @@ func (rl *remoteLayer) Manifest() (*v1.Manifest, error) { return partial.Manifest(rl.ri) } +// MediaType implements v1.Layer +func (rl *remoteLayer) MediaType() (types.MediaType, error) { + m, err := rl.Manifest() + if err != nil { + return "", err + } + + for _, layer := range m.Layers { + if layer.Digest == rl.digest { + return layer.MediaType, nil + } + } + + return "", fmt.Errorf("unable to find layer with digest: %v", rl.digest) +} + // Size implements partial.CompressedLayer func (rl *remoteLayer) Size() (int64, error) { // Look up the size of this digest in the manifest to avoid a request. diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go index 2cf9922c..043dc83b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/index.go @@ -33,9 +33,8 @@ type remoteIndex struct { mediaType types.MediaType } -// Index provides access to a remote index reference, applying functional options -// to the underlying imageOpener before resolving the reference into a v1.ImageIndex. -func Index(ref name.Reference, options ...ImageOption) (v1.ImageIndex, error) { +// Index provides access to a remote index reference. +func Index(ref name.Reference, options ...Option) (v1.ImageIndex, error) { acceptable := []types.MediaType{ types.DockerManifestList, types.OCIImageIndex, @@ -120,7 +119,7 @@ func (r *remoteIndex) imageByPlatform(platform v1.Platform) (v1.Image, error) { return desc.Image() } -// This naively matches the first manifest with matching Architecture and OS. +// This naively matches the first manifest with matching platform attributes. // // We should probably use this instead: // github.com/containerd/containerd/platforms @@ -139,7 +138,7 @@ func (r *remoteIndex) childByPlatform(platform v1.Platform) (*Descriptor, error) p = *childDesc.Platform } - if platform.Architecture == p.Architecture && platform.OS == p.OS { + if matchesPlatform(p, platform) { return r.childDescriptor(childDesc, platform) } } @@ -183,3 +182,49 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) platform: platform, }, nil } + +// matchesPlatform checks if the given platform matches the required platforms. +// The given platform matches the required platform if +// - architecture and OS are identical. +// - OS version and variant are identical if provided. +// - features and OS features of the required platform are subsets of those of the given platform. +func matchesPlatform(given, required v1.Platform) bool { + // Required fields that must be identical. + if given.Architecture != required.Architecture || given.OS != required.OS { + return false + } + + // Optional fields that may be empty, but must be identical if provided. + if required.OSVersion != "" && given.OSVersion != required.OSVersion { + return false + } + if required.Variant != "" && given.Variant != required.Variant { + return false + } + + // Verify required platform's features are a subset of given platform's features. + if !isSubset(given.OSFeatures, required.OSFeatures) { + return false + } + if !isSubset(given.Features, required.Features) { + return false + } + + return true +} + +// isSubset checks if the required array of strings is a subset of the given lst. +func isSubset(lst, required []string) bool { + set := make(map[string]bool) + for _, value := range lst { + set[value] = true + } + + for _, value := range required { + if _, ok := set[value]; !ok { + return false + } + } + + return true +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go index 1a36d0a4..4cbdc4f1 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/list.go @@ -20,7 +20,6 @@ import ( "net/http" "net/url" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) @@ -30,10 +29,15 @@ type tags struct { Tags []string `json:"tags"` } -// List calls /tags/list for the given repository. -func List(repo name.Repository, auth authn.Authenticator, t http.RoundTripper) ([]string, error) { +// List calls /tags/list for the given repository, returning the list of tags +// in the "tags" property. +func List(repo name.Repository, options ...Option) ([]string, error) { + o, err := makeOptions(repo.Registry, options...) + if err != nil { + return nil, err + } scopes := []string{repo.Scope(transport.PullScope)} - tr, err := transport.New(repo.Registry, auth, t, scopes) + tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes) if err != nil { return nil, err } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go index 1af76061..c6034484 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/options.go @@ -15,56 +15,96 @@ package remote import ( - "log" "net/http" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" ) -// ImageOption is a functional option for Image, index, and Get. -type ImageOption func(*imageOpener) error +// Option is a functional option for remote operations. +type Option func(*options) error + +type options struct { + auth authn.Authenticator + keychain authn.Keychain + transport http.RoundTripper + platform v1.Platform +} + +func makeOptions(reg name.Registry, opts ...Option) (*options, error) { + o := &options{ + auth: authn.Anonymous, + transport: http.DefaultTransport, + platform: defaultPlatform, + } + + for _, option := range opts { + if err := option(o); err != nil { + return nil, err + } + } + + if o.keychain != nil { + auth, err := o.keychain.Resolve(reg) + if err != nil { + return nil, err + } + if auth == authn.Anonymous { + logs.Warn.Println("No matching credentials were found, falling back on anonymous") + } + o.auth = auth + } + + // Wrap the transport in something that can retry network flakes. + o.transport = transport.NewRetry(o.transport) + + return o, nil +} // WithTransport is a functional option for overriding the default transport -// on a remote image -func WithTransport(t http.RoundTripper) ImageOption { - return func(i *imageOpener) error { - i.transport = t +// for remote operations. +// +// The default transport its http.DefaultTransport. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) error { + o.transport = t return nil } } // WithAuth is a functional option for overriding the default authenticator -// on a remote image -func WithAuth(auth authn.Authenticator) ImageOption { - return func(i *imageOpener) error { - i.auth = auth +// for remote operations. +// +// The default authenticator is authn.Anonymous. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) error { + o.auth = auth return nil } } // WithAuthFromKeychain is a functional option for overriding the default -// authenticator on a remote image using an authn.Keychain -func WithAuthFromKeychain(keys authn.Keychain) ImageOption { - return func(i *imageOpener) error { - auth, err := keys.Resolve(i.ref.Context().Registry) - if err != nil { - return err - } - if auth == authn.Anonymous { - log.Println("No matching credentials were found, falling back on anonymous") - } - i.auth = auth +// authenticator for remote operations, using an authn.Keychain to find +// credentials. +// +// The default authenticator is authn.Anonymous. +func WithAuthFromKeychain(keys authn.Keychain) Option { + return func(o *options) error { + o.keychain = keys return nil } } // WithPlatform is a functional option for overriding the default platform // that Image and Descriptor.Image use for resolving an index to an image. +// // The default platform is amd64/linux. -func WithPlatform(p v1.Platform) ImageOption { - return func(i *imageOpener) error { - i.platform = p +func WithPlatform(p v1.Platform) Option { + return func(o *options) error { + o.platform = p return nil } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go index e77f47f6..b98b4a62 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/basic.go @@ -40,7 +40,7 @@ func (bt *basicTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the host with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.target || in.URL.Host == bt.target { + if hdr != "" && (in.Host == bt.target || in.URL.Host == bt.target) { in.Header.Set("Authorization", hdr) } in.Header.Set("User-Agent", transportName) diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go index f72ab276..326708b9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/bearer.go @@ -18,8 +18,10 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/url" + "strings" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" @@ -45,6 +47,11 @@ type bearerTransport struct { var _ http.RoundTripper = (*bearerTransport)(nil) +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + // RoundTrip implements http.RoundTripper func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { sendRequest := func() (*http.Response, error) { @@ -58,12 +65,19 @@ func (bt *bearerTransport) RoundTrip(in *http.Request) (*http.Response, error) { // we are redirected, only set it when the authorization header matches // the registry with which we are interacting. // In case of redirect http.Client can use an empty Host, check URL too. - if in.Host == bt.registry.RegistryStr() || in.URL.Host == bt.registry.RegistryStr() { + canonicalHeaderHost := bt.canonicalAddress(in.Host) + canonicalURLHost := bt.canonicalAddress(in.URL.Host) + canonicalRegistryHost := bt.canonicalAddress(bt.registry.RegistryStr()) + if canonicalHeaderHost == canonicalRegistryHost || canonicalURLHost == canonicalRegistryHost { in.Header.Set("Authorization", hdr) + + // When we ping() the registry, we determine whether to use http or https + // based on which scheme was successful. That is only valid for the + // registry server and not e.g. a separate token server or blob storage, + // so we should only override the scheme if the host is the registry. + in.URL.Scheme = bt.scheme } in.Header.Set("User-Agent", transportName) - - in.URL.Scheme = bt.scheme return bt.inner.RoundTrip(in) } @@ -140,3 +154,28 @@ func (bt *bearerTransport) refresh() error { bt.bearer = &bearer return nil } + +func (bt *bearerTransport) canonicalAddress(host string) (address string) { + // The host may be any one of: + // - hostname + // - hostname:port + // - ipv4 + // - ipv4:port + // - ipv6 + // - [ipv6]:port + // As net.SplitHostPort returns an error if the host does not contain a port, we should only attempt + // to call it when we know that the address contains a port + if strings.Count(host, ":") == 1 || (strings.Count(host, ":") >= 2 && strings.Contains(host, "]:")) { + hostname, port, err := net.SplitHostPort(host) + if err != nil { + return host + } + if port == "" { + port = portMap[bt.scheme] + } + + return net.JoinHostPort(hostname, port) + } + + return net.JoinHostPort(host, portMap[bt.scheme]) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 44885eff..3673a341 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -48,6 +48,20 @@ func (e *Error) Error() string { } } +// Temporary returns whether the request that preceded the error is temporary. +func (e *Error) Temporary() bool { + if len(e.Errors) == 0 { + return false + } + for _, d := range e.Errors { + // TODO: Include other error types. + if d.Code != BlobUploadInvalidErrorCode { + return false + } + } + return true +} + // Diagnostic represents a single error returned by a Docker registry interaction. type Diagnostic struct { Code ErrorCode `json:"code"` diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go new file mode 100644 index 00000000..b6b2181d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/retry.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transport + +import ( + "net/http" + "time" + + "github.com/google/go-containerregistry/pkg/internal/retry" + "k8s.io/apimachinery/pkg/util/wait" +) + +// Sleep for 0.1, 0.3, 0.9, 2.7 seconds. This should cover networking blips. +var defaultBackoff = wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 3.0, + Jitter: 0.1, + Steps: 5, +} + +var _ http.RoundTripper = (*retryTransport)(nil) + +// retryTransport wraps a RoundTripper and retries temporary network errors. +type retryTransport struct { + inner http.RoundTripper + backoff wait.Backoff + predicate retry.Predicate +} + +// Option is a functional option for retryTransport. +type Option func(*options) + +type options struct { + backoff wait.Backoff + predicate retry.Predicate +} + +// WithRetryBackoff sets the backoff for retry operations. +func WithRetryBackoff(backoff wait.Backoff) Option { + return func(o *options) { + o.backoff = backoff + } +} + +// WithRetryPredicate sets the predicate for retry operations. +func WithRetryPredicate(predicate func(error) bool) Option { + return func(o *options) { + o.predicate = predicate + } +} + +// NewRetry returns a transport that retries errors. +func NewRetry(inner http.RoundTripper, opts ...Option) http.RoundTripper { + o := &options{ + backoff: defaultBackoff, + predicate: retry.IsTemporary, + } + + for _, opt := range opts { + opt(o) + } + + return &retryTransport{ + inner: inner, + backoff: o.backoff, + predicate: o.predicate, + } +} + +func (t *retryTransport) RoundTrip(in *http.Request) (out *http.Response, err error) { + roundtrip := func() error { + out, err = t.inner.RoundTrip(in) + return err + } + retry.Retry(roundtrip, t.predicate, t.backoff) + return +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 66f14815..8806aa18 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -19,18 +19,19 @@ import ( "errors" "fmt" "io" - "log" "net/http" "net/url" + "time" - "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/internal/retry" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote/transport" - "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/util/wait" ) type manifest interface { @@ -40,14 +41,19 @@ type manifest interface { } // Write pushes the provided img to the specified image reference. -func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.RoundTripper) error { +func Write(ref name.Reference, img v1.Image, options ...Option) error { ls, err := img.Layers() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } + scopes := scopesForUploadingImage(ref, ls) - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -57,17 +63,17 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro } // Upload individual layers in goroutines and collect any errors. - // If we can dedupe by the layer digest, try to do so. If the layer is - // a stream.Layer, we can't dedupe and might re-upload. + // If we can dedupe by the layer digest, try to do so. If we can't determine + // the digest for whatever reason, we can't dedupe and might re-upload. var g errgroup.Group uploaded := map[v1.Hash]bool{} for _, l := range ls { l := l - if _, ok := l.(*stream.Layer); !ok { - h, err := l.Digest() - if err != nil { - return err - } + + // Streaming layers calculate their digests while uploading them. Assume + // an error here indicates we need to upload the layer. + h, err := l.Digest() + if err == nil { // If we can determine the layer's digest ahead of // time, use it to dedupe uploads. if uploaded[h] { @@ -81,14 +87,15 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro }) } - if l, err := partial.ConfigLayer(img); err == stream.ErrNotComputed { - // We can't read the ConfigLayer, because of streaming layers, since the - // config hasn't been calculated yet. + if l, err := partial.ConfigLayer(img); err != nil { + // We can't read the ConfigLayer, possibly because of streaming layers, + // since the layer DiffIDs haven't been calculated yet. Attempt to wait + // for the other layers to be uploaded, then try the config again. if err := g.Wait(); err != nil { return err } - // Now that all the layers are uploaded, upload the config file blob. + // Now that all the layers are uploaded, try to upload the config file blob. l, err := partial.ConfigLayer(img) if err != nil { return err @@ -96,9 +103,6 @@ func Write(ref name.Reference, img v1.Image, auth authn.Authenticator, t http.Ro if err := w.uploadOne(l); err != nil { return err } - } else if err != nil { - // This is an actual error, not a streaming error, just return it. - return err } else { // We *can* read the ConfigLayer, so upload it concurrently with the layers. g.Go(func() error { @@ -285,25 +289,16 @@ func (w *writer) commitBlob(location, digest string) error { // uploadOne performs a complete upload of a single layer. func (w *writer) uploadOne(l v1.Layer) error { - var from, mount, digest string - if _, ok := l.(*stream.Layer); !ok { - // Layer isn't streamable, we should take advantage of that to - // skip uploading if possible. - // By sending ?digest= in the request, we'll also check that - // our computed digest matches the one computed by the - // registry. - h, err := l.Digest() - if err != nil { - return err - } - digest = h.String() - + var from, mount string + if h, err := l.Digest(); err == nil { + // If we know the digest, this isn't a streaming layer. Do an existence + // check so we can skip uploading the layer if possible. existing, err := w.checkExistingBlob(h) if err != nil { return err } if existing { - log.Printf("existing blob: %v", h) + logs.Progress.Printf("existing blob: %v", h) return nil } @@ -315,38 +310,50 @@ func (w *writer) uploadOne(l v1.Layer) error { } } - location, mounted, err := w.initiateUpload(from, mount) - if err != nil { - return err - } else if mounted { + tryUpload := func() error { + location, mounted, err := w.initiateUpload(from, mount) + if err != nil { + return err + } else if mounted { + h, err := l.Digest() + if err != nil { + return err + } + logs.Progress.Printf("mounted blob: %s", h.String()) + return nil + } + + blob, err := l.Compressed() + if err != nil { + return err + } + location, err = w.streamBlob(blob, location) + if err != nil { + return err + } + h, err := l.Digest() if err != nil { return err } - log.Printf("mounted blob: %s", h.String()) - return nil - } + digest := h.String() - blob, err := l.Compressed() - if err != nil { - return err - } - location, err = w.streamBlob(blob, location) - if err != nil { - return err + if err := w.commitBlob(location, digest); err != nil { + return err + } + logs.Progress.Printf("pushed blob: %s", digest) + return nil } - h, err := l.Digest() - if err != nil { - return err + // Try this three times, waiting 1s after first failure, 3s after second. + backoff := wait.Backoff{ + Duration: 1.0 * time.Second, + Factor: 3.0, + Jitter: 0.1, + Steps: 3, } - digest = h.String() - if err := w.commitBlob(location, digest); err != nil { - return err - } - log.Printf("pushed blob: %s", digest) - return nil + return retry.Retry(tryUpload, retry.IsTemporary, backoff) } // commitImage does a PUT of the image's manifest. @@ -385,7 +392,7 @@ func (w *writer) commitImage(man manifest) error { } // The image was successfully pushed! - log.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) + logs.Progress.Printf("%v: digest: %v size: %d", w.ref, digest, len(raw)) return nil } @@ -416,14 +423,18 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { // WriteIndex pushes the provided ImageIndex to the specified image reference. // WriteIndex will attempt to push all of the referenced manifests before // attempting to push the ImageIndex, to retain referential integrity. -func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, t http.RoundTripper) error { +func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { index, err := ii.IndexManifest() if err != nil { return err } + o, err := makeOptions(ref.Context().Registry, options...) + if err != nil { + return err + } scopes := []string{ref.Scope(transport.PushScope)} - tr, err := transport.New(ref.Context().Registry, auth, t, scopes) + tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } @@ -442,7 +453,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, return err } if exists { - log.Printf("existing manifest: %v", desc.Digest) + logs.Progress.Printf("existing manifest: %v", desc.Digest) continue } @@ -453,7 +464,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, return err } - if err := WriteIndex(ref, ii, auth, t); err != nil { + if err := WriteIndex(ref, ii, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } case types.OCIManifestSchema1, types.DockerManifestSchema2: @@ -461,7 +472,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, auth authn.Authenticator, if err != nil { return err } - if err := Write(ref, img, auth, t); err != nil { + if err := Write(ref, img, WithAuth(o.auth), WithTransport(o.transport)); err != nil { return err } } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go deleted file mode 100644 index f8895a22..00000000 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/stream/layer.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package stream - -import ( - "compress/gzip" - "crypto/sha256" - "encoding/hex" - "errors" - "hash" - "io" - "sync" - - v1 "github.com/google/go-containerregistry/pkg/v1" -) - -var ( - // ErrNotComputed is returned when the requested value is not yet - // computed because the stream has not been consumed yet. - ErrNotComputed = errors.New("value not computed until stream is consumed") - - // ErrConsumed is returned by Compressed when the underlying stream has - // already been consumed and closed. - ErrConsumed = errors.New("stream was already consumed") -) - -// Layer is a streaming implementation of v1.Layer. -type Layer struct { - blob io.ReadCloser - consumed bool - - mu sync.Mutex - digest, diffID *v1.Hash - size int64 -} - -var _ v1.Layer = (*Layer)(nil) - -// NewLayer creates a Layer from an io.ReadCloser. -func NewLayer(rc io.ReadCloser) *Layer { return &Layer{blob: rc} } - -// Digest implements v1.Layer. -func (l *Layer) Digest() (v1.Hash, error) { - l.mu.Lock() - defer l.mu.Unlock() - if l.digest == nil { - return v1.Hash{}, ErrNotComputed - } - return *l.digest, nil -} - -// DiffID implements v1.Layer. -func (l *Layer) DiffID() (v1.Hash, error) { - l.mu.Lock() - defer l.mu.Unlock() - if l.diffID == nil { - return v1.Hash{}, ErrNotComputed - } - return *l.diffID, nil -} - -// Size implements v1.Layer. -func (l *Layer) Size() (int64, error) { - l.mu.Lock() - defer l.mu.Unlock() - if l.size == 0 { - return 0, ErrNotComputed - } - return l.size, nil -} - -// Uncompressed implements v1.Layer. -func (l *Layer) Uncompressed() (io.ReadCloser, error) { - return nil, errors.New("NYI: stream.Layer.Uncompressed is not implemented") -} - -// Compressed implements v1.Layer. -func (l *Layer) Compressed() (io.ReadCloser, error) { - if l.consumed { - return nil, ErrConsumed - } - return newCompressedReader(l) -} - -type compressedReader struct { - closer io.Closer // original blob's Closer. - - h, zh hash.Hash // collects digests of compressed and uncompressed stream. - pr io.Reader - count *countWriter - - l *Layer // stream.Layer to update upon Close. -} - -func newCompressedReader(l *Layer) (*compressedReader, error) { - h := sha256.New() - zh := sha256.New() - count := &countWriter{} - - // gzip.Writer writes to the output stream via pipe, a hasher to - // capture compressed digest, and a countWriter to capture compressed - // size. - pr, pw := io.Pipe() - zw, err := gzip.NewWriterLevel(io.MultiWriter(pw, zh, count), gzip.BestSpeed) - if err != nil { - return nil, err - } - - cr := &compressedReader{ - closer: newMultiCloser(zw, l.blob), - pr: pr, - h: h, - zh: zh, - count: count, - l: l, - } - go func() { - if _, err := io.Copy(io.MultiWriter(h, zw), l.blob); err != nil { - pw.CloseWithError(err) - return - } - // Now close the compressed reader, to flush the gzip stream - // and calculate digest/diffID/size. This will cause pr to - // return EOF which will cause readers of the Compressed stream - // to finish reading. - pw.CloseWithError(cr.Close()) - }() - - return cr, nil -} - -func (cr *compressedReader) Read(b []byte) (int, error) { return cr.pr.Read(b) } - -func (cr *compressedReader) Close() error { - cr.l.mu.Lock() - defer cr.l.mu.Unlock() - - // Close the inner ReadCloser. - if err := cr.closer.Close(); err != nil { - return err - } - - diffID, err := v1.NewHash("sha256:" + hex.EncodeToString(cr.h.Sum(nil))) - if err != nil { - return err - } - cr.l.diffID = &diffID - - digest, err := v1.NewHash("sha256:" + hex.EncodeToString(cr.zh.Sum(nil))) - if err != nil { - return err - } - cr.l.digest = &digest - - cr.l.size = cr.count.n - cr.l.consumed = true - return nil -} - -// countWriter counts bytes written to it. -type countWriter struct{ n int64 } - -func (c *countWriter) Write(p []byte) (int, error) { - c.n += int64(len(p)) - return len(p), nil -} - -// multiCloser is a Closer that collects multiple Closers and Closes them in order. -type multiCloser []io.Closer - -var _ io.Closer = (multiCloser)(nil) - -func newMultiCloser(c ...io.Closer) multiCloser { return multiCloser(c) } - -func (m multiCloser) Close() error { - for _, c := range m { - if err := c.Close(); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go new file mode 100644 index 00000000..4eb79bb4 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarball provides facilities for reading/writing v1.Images from/to +// a tarball on-disk. +package tarball diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go new file mode 100644 index 00000000..06ecd9a7 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -0,0 +1,352 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "sync" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/v1util" +) + +type image struct { + opener Opener + td *tarDescriptor + config []byte + imgDescriptor *singleImageTarDescriptor + + tag *name.Tag +} + +type uncompressedImage struct { + *image +} + +type compressedImage struct { + *image + manifestLock sync.Mutex // Protects manifest + manifest *v1.Manifest +} + +var _ partial.UncompressedImageCore = (*uncompressedImage)(nil) +var _ partial.CompressedImageCore = (*compressedImage)(nil) + +// Opener is a thunk for opening a tar file. +type Opener func() (io.ReadCloser, error) + +func pathOpener(path string) Opener { + return func() (io.ReadCloser, error) { + return os.Open(path) + } +} + +// ImageFromPath returns a v1.Image from a tarball located on path. +func ImageFromPath(path string, tag *name.Tag) (v1.Image, error) { + return Image(pathOpener(path), tag) +} + +// Image exposes an image from the tarball at the provided path. +func Image(opener Opener, tag *name.Tag) (v1.Image, error) { + img := &image{ + opener: opener, + tag: tag, + } + if err := img.loadTarDescriptorAndConfig(); err != nil { + return nil, err + } + + // Peek at the first layer and see if it's compressed. + compressed, err := img.areLayersCompressed() + if err != nil { + return nil, err + } + if compressed { + c := compressedImage{ + image: img, + } + return partial.CompressedToImage(&c) + } + + uc := uncompressedImage{ + image: img, + } + return partial.UncompressedToImage(&uc) +} + +func (i *image) MediaType() (types.MediaType, error) { + return types.DockerManifestSchema2, nil +} + +// singleImageTarDescriptor is the struct used to represent a single image inside a `docker save` tarball. +type singleImageTarDescriptor struct { + Config string + RepoTags []string + Layers []string +} + +// tarDescriptor is the struct used inside the `manifest.json` file of a `docker save` tarball. +type tarDescriptor []singleImageTarDescriptor + +func (td tarDescriptor) findSpecifiedImageDescriptor(tag *name.Tag) (*singleImageTarDescriptor, error) { + if tag == nil { + if len(td) != 1 { + return nil, errors.New("tarball must contain only a single image to be used with tarball.Image") + } + return &(td)[0], nil + } + for _, img := range td { + for _, tagStr := range img.RepoTags { + repoTag, err := name.NewTag(tagStr) + if err != nil { + return nil, err + } + + // Compare the resolved names, since there are several ways to specify the same tag. + if repoTag.Name() == tag.Name() { + return &img, nil + } + } + } + return nil, fmt.Errorf("tag %s not found in tarball", tag) +} + +func (i *image) areLayersCompressed() (bool, error) { + if len(i.imgDescriptor.Layers) == 0 { + return false, errors.New("0 layers found in image") + } + layer := i.imgDescriptor.Layers[0] + blob, err := extractFileFromTar(i.opener, layer) + if err != nil { + return false, err + } + defer blob.Close() + return v1util.IsGzipped(blob) +} + +func (i *image) loadTarDescriptorAndConfig() error { + td, err := extractFileFromTar(i.opener, "manifest.json") + if err != nil { + return err + } + defer td.Close() + + if err := json.NewDecoder(td).Decode(&i.td); err != nil { + return err + } + + i.imgDescriptor, err = i.td.findSpecifiedImageDescriptor(i.tag) + if err != nil { + return err + } + + cfg, err := extractFileFromTar(i.opener, i.imgDescriptor.Config) + if err != nil { + return err + } + defer cfg.Close() + + i.config, err = ioutil.ReadAll(cfg) + if err != nil { + return err + } + return nil +} + +func (i *image) RawConfigFile() ([]byte, error) { + return i.config, nil +} + +// tarFile represents a single file inside a tar. Closing it closes the tar itself. +type tarFile struct { + io.Reader + io.Closer +} + +func extractFileFromTar(opener Opener, filePath string) (io.ReadCloser, error) { + f, err := opener() + if err != nil { + return nil, err + } + tf := tar.NewReader(f) + for { + hdr, err := tf.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if hdr.Name == filePath { + return tarFile{ + Reader: tf, + Closer: f, + }, nil + } + } + return nil, fmt.Errorf("file %s not found in tar", filePath) +} + +// uncompressedLayerFromTarball implements partial.UncompressedLayer +type uncompressedLayerFromTarball struct { + diffID v1.Hash + opener Opener + filePath string +} + +// DiffID implements partial.UncompressedLayer +func (ulft *uncompressedLayerFromTarball) DiffID() (v1.Hash, error) { + return ulft.diffID, nil +} + +// Uncompressed implements partial.UncompressedLayer +func (ulft *uncompressedLayerFromTarball) Uncompressed() (io.ReadCloser, error) { + return extractFileFromTar(ulft.opener, ulft.filePath) +} + +func (ulft *uncompressedLayerFromTarball) MediaType() (types.MediaType, error) { + // Technically the media type should be 'application/tar' but given that our + // v1.Layer doesn't force consumers to care about whether the layer is compressed + // we should be fine returning the DockerLayer media type + return types.DockerLayer, nil +} + +func (i *uncompressedImage) LayerByDiffID(h v1.Hash) (partial.UncompressedLayer, error) { + cfg, err := partial.ConfigFile(i) + if err != nil { + return nil, err + } + for idx, diffID := range cfg.RootFS.DiffIDs { + if diffID == h { + return &uncompressedLayerFromTarball{ + diffID: diffID, + opener: i.opener, + filePath: i.imgDescriptor.Layers[idx], + }, nil + } + } + return nil, fmt.Errorf("diff id %q not found", h) +} + +func (c *compressedImage) Manifest() (*v1.Manifest, error) { + c.manifestLock.Lock() + defer c.manifestLock.Unlock() + if c.manifest != nil { + return c.manifest, nil + } + + b, err := c.RawConfigFile() + if err != nil { + return nil, err + } + + cfgHash, cfgSize, err := v1.SHA256(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + c.manifest = &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.DockerManifestSchema2, + Config: v1.Descriptor{ + MediaType: types.DockerConfigJSON, + Size: cfgSize, + Digest: cfgHash, + }, + } + + for _, p := range c.imgDescriptor.Layers { + l, err := extractFileFromTar(c.opener, p) + if err != nil { + return nil, err + } + defer l.Close() + sha, size, err := v1.SHA256(l) + if err != nil { + return nil, err + } + c.manifest.Layers = append(c.manifest.Layers, v1.Descriptor{ + MediaType: types.DockerLayer, + Size: size, + Digest: sha, + }) + } + return c.manifest, nil +} + +func (c *compressedImage) RawManifest() ([]byte, error) { + return partial.RawManifest(c) +} + +// compressedLayerFromTarball implements partial.CompressedLayer +type compressedLayerFromTarball struct { + digest v1.Hash + opener Opener + filePath string +} + +// Digest implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Digest() (v1.Hash, error) { + return clft.digest, nil +} + +// Compressed implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Compressed() (io.ReadCloser, error) { + return extractFileFromTar(clft.opener, clft.filePath) +} + +// MediaType implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + +// Size implements partial.CompressedLayer +func (clft *compressedLayerFromTarball) Size() (int64, error) { + r, err := clft.Compressed() + if err != nil { + return -1, err + } + defer r.Close() + _, i, err := v1.SHA256(r) + return i, err +} + +func (c *compressedImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + m, err := c.Manifest() + if err != nil { + return nil, err + } + for i, l := range m.Layers { + if l.Digest == h { + fp := c.imgDescriptor.Layers[i] + return &compressedLayerFromTarball{ + digest: h, + opener: c.opener, + filePath: fp, + }, nil + } + } + return nil, fmt.Errorf("blob %v not found", h) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go new file mode 100644 index 00000000..85c1b783 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -0,0 +1,162 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "bytes" + "compress/gzip" + "io" + "io/ioutil" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/v1util" +) + +type layer struct { + digest v1.Hash + diffID v1.Hash + size int64 + opener Opener + compressed bool +} + +func (l *layer) Digest() (v1.Hash, error) { + return l.digest, nil +} + +func (l *layer) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +func (l *layer) Compressed() (io.ReadCloser, error) { + rc, err := l.opener() + if err == nil && !l.compressed { + return v1util.GzipReadCloser(rc) + } + + return rc, err +} + +func (l *layer) Uncompressed() (io.ReadCloser, error) { + rc, err := l.opener() + if err == nil && l.compressed { + return v1util.GunzipReadCloser(rc) + } + + return rc, err +} + +func (l *layer) Size() (int64, error) { + return l.size, nil +} + +func (l *layer) MediaType() (types.MediaType, error) { + return types.DockerLayer, nil +} + +// LayerFromFile returns a v1.Layer given a tarball +func LayerFromFile(path string) (v1.Layer, error) { + opener := func() (io.ReadCloser, error) { + return os.Open(path) + } + return LayerFromOpener(opener) +} + +// LayerFromOpener returns a v1.Layer given an Opener function +func LayerFromOpener(opener Opener) (v1.Layer, error) { + rc, err := opener() + if err != nil { + return nil, err + } + defer rc.Close() + + compressed, err := v1util.IsGzipped(rc) + if err != nil { + return nil, err + } + + var digest v1.Hash + var size int64 + if digest, size, err = computeDigest(opener, compressed); err != nil { + return nil, err + } + + diffID, err := computeDiffID(opener, compressed) + if err != nil { + return nil, err + } + + return &layer{ + digest: digest, + diffID: diffID, + size: size, + compressed: compressed, + opener: opener, + }, nil +} + +// LayerFromReader returns a v1.Layer given a io.Reader. +func LayerFromReader(reader io.Reader) (v1.Layer, error) { + // Buffering due to Opener requiring multiple calls. + a, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + return LayerFromOpener(func() (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewReader(a)), nil + }) +} + +func computeDigest(opener Opener, compressed bool) (v1.Hash, int64, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, 0, err + } + defer rc.Close() + + if compressed { + return v1.SHA256(rc) + } + + reader, err := v1util.GzipReadCloser(ioutil.NopCloser(rc)) + if err != nil { + return v1.Hash{}, 0, err + } + + return v1.SHA256(reader) +} + +func computeDiffID(opener Opener, compressed bool) (v1.Hash, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, err + } + defer rc.Close() + + if !compressed { + digest, _, err := v1.SHA256(rc) + return digest, err + } + + reader, err := gzip.NewReader(rc) + if err != nil { + return v1.Hash{}, err + } + + diffID, _, err := v1.SHA256(reader) + return diffID, err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go new file mode 100644 index 00000000..2ee81f0b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/write.go @@ -0,0 +1,194 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// WriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.Write with a new file. +func WriteToFile(p string, ref name.Reference, img v1.Image) error { + w, err := os.Create(p) + if err != nil { + return err + } + defer w.Close() + + return Write(ref, img, w) +} + +// MultiWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiWrite with a new file. +func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWriteToFile(p, refToImage) +} + +// MultiRefWriteToFile writes in the compressed format to a tarball, on disk. +// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file. +func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image) error { + w, err := os.Create(p) + if err != nil { + return err + } + defer w.Close() + + return MultiRefWrite(refToImage, w) +} + +// Write is a wrapper to write a single image and tag to a tarball. +func Write(ref name.Reference, img v1.Image, w io.Writer) error { + return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w) +} + +// MultiWrite writes the contents of each image to the provided reader, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error { + var refToImage map[name.Reference]v1.Image = make(map[name.Reference]v1.Image, len(tagToImage)) + for i, d := range tagToImage { + refToImage[i] = d + } + return MultiRefWrite(refToImage, w) +} + +// MultiRefWrite writes the contents of each image to the provided reader, in the compressed format. +// The contents are written in the following format: +// One manifest.json file at the top level containing information about several images. +// One file for each layer, named after the layer's SHA. +// One file for the config blob, named after its SHA. +func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error { + tf := tar.NewWriter(w) + defer tf.Close() + + imageToTags := dedupRefToImage(refToImage) + var td tarDescriptor + + for img, tags := range imageToTags { + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := writeTarEntry(tf, cfgName.String(), bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil { + return err + } + + // Write the layers. + layers, err := img.Layers() + if err != nil { + return err + } + layerFiles := make([]string, len(layers)) + for i, l := range layers { + d, err := l.Digest() + if err != nil { + return err + } + + // Munge the file name to appease ancient technology. + // + // tar assumes anything with a colon is a remote tape drive: + // https://www.gnu.org/software/tar/manual/html_section/tar_45.html + // Drop the algorithm prefix, e.g. "sha256:" + hex := d.Hex + + // gunzip expects certain file extensions: + // https://www.gnu.org/software/gzip/manual/html_node/Overview.html + layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex) + + r, err := l.Compressed() + if err != nil { + return err + } + blobSize, err := l.Size() + if err != nil { + return err + } + + if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil { + return err + } + } + + // Generate the tar descriptor and write it. + sitd := singleImageTarDescriptor{ + Config: cfgName.String(), + RepoTags: tags, + Layers: layerFiles, + } + + td = append(td, sitd) + } + + tdBytes, err := json.Marshal(td) + if err != nil { + return err + } + return writeTarEntry(tf, "manifest.json", bytes.NewReader(tdBytes), int64(len(tdBytes))) +} + +func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string { + imageToTags := make(map[v1.Image][]string) + + for ref, img := range refToImage { + if tag, ok := ref.(name.Tag); ok { + if tags, ok := imageToTags[img]; ok && tags != nil { + imageToTags[img] = append(tags, tag.String()) + } else { + imageToTags[img] = []string{tag.String()} + } + } else { + if _, ok := imageToTags[img]; !ok { + imageToTags[img] = nil + } + } + } + + return imageToTags +} + +// write a file to the provided writer with a corresponding tar header +func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error { + hdr := &tar.Header{ + Mode: 0644, + Typeflag: tar.TypeReg, + Size: size, + Name: path, + } + if err := tf.WriteHeader(hdr); err != nil { + return err + } + _, err := io.Copy(tf, r) + return err +} diff --git a/vendor/github.com/pivotal/image-relocation/pkg/registry/remote.go b/vendor/github.com/pivotal/image-relocation/pkg/registry/remote.go index 75b4308d..182a0f32 100644 --- a/vendor/github.com/pivotal/image-relocation/pkg/registry/remote.go +++ b/vendor/github.com/pivotal/image-relocation/pkg/registry/remote.go @@ -17,15 +17,25 @@ package registry import ( + "errors" + "fmt" "net/http" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/pivotal/image-relocation/pkg/image" ) +var ( + daemonImageFunc = daemon.Image + repoImageFunc = remote.Image + resolveFunc = authn.DefaultKeychain.Resolve + repoWriteFunc = remote.Write +) + func readRemoteImage(n image.Name) (v1.Image, error) { auth, err := resolve(n) if err != nil { @@ -44,7 +54,16 @@ func readRemoteImage(n image.Name) (v1.Image, error) { return nil, err } - return remote.Image(ref, remote.WithAuth(auth)) + img, err := daemonImageFunc(ref) + if err != nil { + var remoteErr error + img, remoteErr = repoImageFunc(ref, remote.WithAuth(auth)) + if remoteErr != nil { + return nil, fmt.Errorf("reading remote image %s failed: %v; attempting to read from daemon also failed: %v", n.String(), remoteErr, err) + } + } + + return img, nil } func writeRemoteImage(i v1.Image, n image.Name) error { @@ -58,14 +77,17 @@ func writeRemoteImage(i v1.Image, n image.Name) error { return err } - return remote.Write(ref, i, auth, http.DefaultTransport) + return repoWriteFunc(ref, i, remote.WithAuth(auth), remote.WithTransport(http.DefaultTransport)) } func resolve(n image.Name) (authn.Authenticator, error) { + if n == image.EmptyName { + return nil, errors.New("empty image name invalid") + } repo, err := name.NewRepository(n.WithoutTagOrDigest().String(), name.WeakValidation) if err != nil { return nil, err } - return authn.DefaultKeychain.Resolve(repo.Registry) + return resolveFunc(repo.Registry) }